diff --git a/cmd/kops/validate_cluster.go b/cmd/kops/validate_cluster.go index 7a1e66bfd02a7..a93023b5e90a3 100644 --- a/cmd/kops/validate_cluster.go +++ b/cmd/kops/validate_cluster.go @@ -33,7 +33,6 @@ import ( "k8s.io/client-go/tools/clientcmd" "k8s.io/kops/cmd/kops/util" api "k8s.io/kops/pkg/apis/kops" - apiutil "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/pkg/validation" "k8s.io/kops/util/pkg/tables" ) @@ -69,7 +68,7 @@ func NewCmdValidateCluster(f *util.Factory, out io.Writer) *cobra.Command { } // We want the validate command to exit non-zero if validation found a problem, // even if we didn't really hit an error during validation. - if len(result.PodFailures) != 0 { + if len(result.Failures) != 0 { os.Exit(2) } }, @@ -129,43 +128,43 @@ func RunValidateCluster(f *util.Factory, cmd *cobra.Command, args []string, out return nil, fmt.Errorf("Cannot build kubernetes api client for %q: %v", contextName, err) } - validationCluster, validationFailed := validation.ValidateCluster(cluster, list, k8sClient) - - if validationCluster == nil || validationCluster.NodeList == nil || validationCluster.NodeList.Items == nil { - return validationCluster, validationFailed + result, err := validation.ValidateCluster(cluster, list, k8sClient) + if err != nil { + return nil, fmt.Errorf("unexpected error during validation: %v", err) } switch options.output { case OutputTable: - if err := validateClusterOutputTable(validationCluster, validationFailed, instanceGroups, out); err != nil { + if err := validateClusterOutputTable(result, cluster, instanceGroups, out); err != nil { return nil, err } + case OutputYaml: - y, err := yaml.Marshal(validationCluster) + y, err := yaml.Marshal(result) if err != nil { return nil, fmt.Errorf("unable to marshal YAML: %v", err) } if _, err := out.Write(y); err != nil { - return nil, fmt.Errorf("unable to print data: %v", err) + return nil, fmt.Errorf("error writing to output: %v", err) } case OutputJSON: - j, err := json.Marshal(validationCluster) + j, err := json.Marshal(result) if err != nil { return nil, fmt.Errorf("unable to marshal JSON: %v", err) } if _, err := out.Write(j); err != nil { - return nil, fmt.Errorf("unable to print data: %v", err) + return nil, fmt.Errorf("error writing to output: %v", err) } default: return nil, fmt.Errorf("Unknown output format: %q", options.output) } - return validationCluster, validationFailed + return result, nil } -func validateClusterOutputTable(validationCluster *validation.ValidationCluster, validationFailed error, instanceGroups []api.InstanceGroup, out io.Writer) error { +func validateClusterOutputTable(result *validation.ValidationCluster, cluster *api.Cluster, instanceGroups []api.InstanceGroup, out io.Writer) error { t := &tables.Table{} t.AddColumn("NAME", func(c api.InstanceGroup) string { return c.ObjectMeta.Name @@ -190,73 +189,52 @@ func validateClusterOutputTable(validationCluster *validation.ValidationCluster, err := t.Render(instanceGroups, out, "NAME", "ROLE", "MACHINETYPE", "MIN", "MAX", "SUBNETS") if err != nil { - return fmt.Errorf("cannot render nodes for %q: %v", validationCluster.ClusterName, err) + return fmt.Errorf("cannot render nodes for %q: %v", cluster.Name, err) } - nodeTable := &tables.Table{} - - nodeTable.AddColumn("NAME", func(n v1.Node) string { - return n.Name - }) - - nodeTable.AddColumn("READY", func(n v1.Node) v1.ConditionStatus { - return validation.GetNodeReadyStatus(&n) - }) - - nodeTable.AddColumn("ROLE", func(n v1.Node) string { - // TODO: Maybe print the instance group role instead? - // TODO: Maybe include the instance group name? - role := apiutil.GetNodeRole(&n) - if role == "" { - role = "node" - } - return role - }) - - fmt.Fprintln(out, "\nNODE STATUS") - err = nodeTable.Render(validationCluster.NodeList.Items, out, "NAME", "ROLE", "READY") - - if err != nil { - return fmt.Errorf("cannot render nodes for %q: %v", validationCluster.ClusterName, err) - } + { + nodeTable := &tables.Table{} + nodeTable.AddColumn("NAME", func(n *validation.ValidationNode) string { + return n.Name + }) - if len(validationCluster.ComponentFailures) != 0 { - componentFailuresTable := &tables.Table{} - componentFailuresTable.AddColumn("NAME", func(s string) string { - return s + nodeTable.AddColumn("READY", func(n *validation.ValidationNode) v1.ConditionStatus { + return n.Status }) - fmt.Fprintln(out, "\nComponent Failures") - err = componentFailuresTable.Render(validationCluster.ComponentFailures, out, "NAME") + nodeTable.AddColumn("ROLE", func(n *validation.ValidationNode) string { + return n.Role + }) - if err != nil { - return fmt.Errorf("cannot render components for %q: %v", validationCluster.ClusterName, err) + fmt.Fprintln(out, "\nNODE STATUS") + if err := nodeTable.Render(result.Nodes, out, "NAME", "ROLE", "READY"); err != nil { + return fmt.Errorf("cannot render nodes for %q: %v", cluster.Name, err) } } - if len(validationCluster.PodFailures) != 0 { - podFailuresTable := &tables.Table{} - podFailuresTable.AddColumn("NAME", func(s string) string { - return s + if len(result.Failures) != 0 { + failuresTable := &tables.Table{} + failuresTable.AddColumn("KIND", func(e *validation.ValidationError) string { + return e.Kind + }) + failuresTable.AddColumn("NAME", func(e *validation.ValidationError) string { + return e.Name + }) + failuresTable.AddColumn("MESSAGE", func(e *validation.ValidationError) string { + return e.Message }) - fmt.Fprintln(out, "\nPod Failures in kube-system") - err = podFailuresTable.Render(validationCluster.PodFailures, out, "NAME") - - if err != nil { - return fmt.Errorf("cannot render pods for %q: %v", validationCluster.ClusterName, err) + fmt.Fprintln(out, "\nVALIDATION ERRORS") + if err := failuresTable.Render(result.Failures, out, "KIND", "NAME", "MESSAGE"); err != nil { + return fmt.Errorf("error rendering failures table: %v", err) } } - if validationFailed == nil { - fmt.Fprintf(out, "\nYour cluster %s is ready\n", validationCluster.ClusterName) - return nil + if len(result.Failures) == 0 { + fmt.Fprintf(out, "\nYour cluster %s is ready\n", cluster.Name) } else { - // do we need to print which instance group is not ready? - // nodes are going to be a pain fmt.Fprint(out, "\nValidation Failed\n") - fmt.Fprintf(out, "Ready Master(s) %d out of %d.\n", len(validationCluster.MastersReadyArray), validationCluster.MastersCount) - fmt.Fprintf(out, "Ready Node(s) %d out of %d.\n", len(validationCluster.NodesReadyArray), validationCluster.NodesCount) - return validationFailed } + + return nil } diff --git a/pkg/validation/BUILD.bazel b/pkg/validation/BUILD.bazel index 66fdba083a22d..12798e942b762 100644 --- a/pkg/validation/BUILD.bazel +++ b/pkg/validation/BUILD.bazel @@ -11,7 +11,9 @@ go_library( deps = [ "//pkg/apis/kops:go_default_library", "//pkg/apis/kops/util:go_default_library", + "//pkg/cloudinstances:go_default_library", "//pkg/dns:go_default_library", + "//upup/pkg/fi/cloudup:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", @@ -27,8 +29,9 @@ go_test( ], embed = [":go_default_library"], deps = [ + "//pkg/apis/kops:go_default_library", + "//pkg/cloudinstances:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", - "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library", diff --git a/pkg/validation/node_conditions.go b/pkg/validation/node_conditions.go index 3516a3c36d854..e2efd6eaa153e 100644 --- a/pkg/validation/node_conditions.go +++ b/pkg/validation/node_conditions.go @@ -21,7 +21,7 @@ import ( "k8s.io/api/core/v1" ) -func GetNodeReadyStatus(node *v1.Node) v1.ConditionStatus { +func getNodeReadyStatus(node *v1.Node) v1.ConditionStatus { cond := findNodeCondition(node, v1.NodeReady) if cond != nil { return cond.Status diff --git a/pkg/validation/validate_cluster.go b/pkg/validation/validate_cluster.go index d4d86fa3ae080..044184deade2a 100644 --- a/pkg/validation/validate_cluster.go +++ b/pkg/validation/validate_cluster.go @@ -21,52 +21,47 @@ import ( "net" "net/url" + "github.com/golang/glog" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/kops/util" + "k8s.io/kops/pkg/cloudinstances" "k8s.io/kops/pkg/dns" + "k8s.io/kops/upup/pkg/fi/cloudup" ) // ValidationCluster a cluster to validate. type ValidationCluster struct { - MastersReady bool `json:"mastersReady,omitempty"` - MastersReadyArray []*ValidationNode `json:"mastersReadyArray,omitempty"` - MastersNotReadyArray []*ValidationNode `json:"mastersNotReadyArray,omitempty"` - MastersCount int `json:"mastersCount,omitempty"` - - NodesReady bool `json:"nodesReady,omitempty"` - NodesReadyArray []*ValidationNode `json:"nodesReadyArray,omitempty"` - NodesNotReadyArray []*ValidationNode `json:"nodesNotReadyArray,omitempty"` - NodesCount int `json:"nodesCount,omitempty"` - - NodeList *v1.NodeList `json:"nodeList,omitempty"` - - ComponentFailures []string `json:"componentFailures,omitempty"` - PodFailures []string `json:"podFailures,omitempty"` - ErrorMessage string `json:"errorMessage,omitempty"` - Status string `json:"status"` - ClusterName string `json:"clusterName"` - InstanceGroups []*kops.InstanceGroup `json:"instanceGroups,omitempty"` + Failures []*ValidationError `json:"failures,omitempty"` + + Nodes []*ValidationNode `json:"nodes,omitempty"` +} + +// ValidationError holds a validation failure +type ValidationError struct { + Kind string `json:"type,omitempty"` + Name string `json:"name,omitempty"` + Message string `json:"message,omitempty"` } -// ValidationNode is A K8s node to be validated. +func (v *ValidationCluster) addError(failure *ValidationError) { + v.Failures = append(v.Failures, failure) +} + +// ValidationNode represents the validation status for a node type ValidationNode struct { + Name string `json:"name,omitempty"` Zone string `json:"zone,omitempty"` Role string `json:"role,omitempty"` Hostname string `json:"hostname,omitempty"` Status v1.ConditionStatus `json:"status,omitempty"` } -const ( - ClusterValidationFailed = "FAILED" - ClusterValidationPassed = "PASSED" -) - -// HasPlaceHolderIP checks if the API DNS has been updated. -func HasPlaceHolderIP(clusterName string) (bool, error) { +// hasPlaceHolderIP checks if the API DNS has been updated. +func hasPlaceHolderIP(clusterName string) (bool, error) { config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( clientcmd.NewDefaultClientConfigLoadingRules(), @@ -92,14 +87,16 @@ func HasPlaceHolderIP(clusterName string) (bool, error) { } // ValidateCluster validate a k8s cluster with a provided instance group list -func ValidateCluster(cluster *kops.Cluster, instanceGroupList *kops.InstanceGroupList, clusterKubernetesClient kubernetes.Interface) (*ValidationCluster, error) { +func ValidateCluster(cluster *kops.Cluster, instanceGroupList *kops.InstanceGroupList, k8sClient kubernetes.Interface) (*ValidationCluster, error) { clusterName := cluster.Name + v := &ValidationCluster{} + // Do not use if we are running gossip if !dns.IsGossipHostname(clusterName) { contextName := clusterName - hasPlaceHolderIPAddress, err := HasPlaceHolderIP(contextName) + hasPlaceHolderIPAddress, err := hasPlaceHolderIP(contextName) if err != nil { return nil, err } @@ -111,13 +108,12 @@ func ValidateCluster(cluster *kops.Cluster, instanceGroupList *kops.InstanceGrou " Please wait about 5-10 minutes for a master to start, dns-controller to launch, and DNS to propagate." + " The protokube container and dns-controller deployment logs may contain more diagnostic information." + " Etcd and the API DNS entries must be updated for a kops Kubernetes cluster to start." - validationCluster := &ValidationCluster{ - ClusterName: clusterName, - ErrorMessage: message, - Status: ClusterValidationFailed, - } - validationFailed := fmt.Errorf("\nCannot reach cluster's API server: unable to Validate Cluster: %s", clusterName) - return validationCluster, validationFailed + v.addError(&ValidationError{ + Kind: "dns", + Name: "apiserver", + Message: message, + }) + return v, nil } } @@ -132,146 +128,144 @@ func ValidateCluster(cluster *kops.Cluster, instanceGroupList *kops.InstanceGrou return nil, fmt.Errorf("no InstanceGroup objects found") } - validationCluster := &ValidationCluster{ - ClusterName: clusterName, - ErrorMessage: ClusterValidationPassed, - InstanceGroups: instanceGroups, + cloud, err := cloudup.BuildCloud(cluster) + if err != nil { + return nil, err } - nodes, err := clusterKubernetesClient.CoreV1().Nodes().List(metav1.ListOptions{}) + nodeList, err := k8sClient.CoreV1().Nodes().List(metav1.ListOptions{}) if err != nil { - return nil, fmt.Errorf("error querying nodes: %v", err) + return nil, fmt.Errorf("error listing nodes: %v", err) } - validationCluster.NodeList = nodes - - validationCluster.ComponentFailures, err = collectComponentFailures(clusterKubernetesClient) + warnUnmatched := false + cloudGroups, err := cloud.GetCloudGroups(cluster, instanceGroups, warnUnmatched, nodeList.Items) if err != nil { + return nil, err + } + v.validateNodes(cloudGroups) + + if err := v.collectComponentFailures(k8sClient); err != nil { return nil, fmt.Errorf("cannot get component status for %q: %v", clusterName, err) } - validationCluster.PodFailures, err = collectPodFailures(clusterKubernetesClient) - if err != nil { + if err = v.collectPodFailures(k8sClient); err != nil { return nil, fmt.Errorf("cannot get pod health for %q: %v", clusterName, err) } - return validateTheNodes(clusterName, validationCluster) - + return v, nil } -func collectComponentFailures(client kubernetes.Interface) (failures []string, err error) { +func (v *ValidationCluster) collectComponentFailures(client kubernetes.Interface) error { componentList, err := client.CoreV1().ComponentStatuses().List(metav1.ListOptions{}) - if err == nil { - for _, component := range componentList.Items { - for _, condition := range component.Conditions { - if condition.Status != v1.ConditionTrue { - failures = append(failures, component.Name) - } + if err != nil { + return fmt.Errorf("error listing ComponentStatuses: %v", err) + } + + for _, component := range componentList.Items { + for _, condition := range component.Conditions { + if condition.Status != v1.ConditionTrue { + v.addError(&ValidationError{ + Kind: "ComponentStatus", + Name: component.Name, + Message: "component is unhealthy", + }) } } } - return + return nil } -func collectPodFailures(client kubernetes.Interface) (failures []string, err error) { +func (v *ValidationCluster) collectPodFailures(client kubernetes.Interface) error { pods, err := client.CoreV1().Pods("kube-system").List(metav1.ListOptions{}) - if err == nil { - for _, pod := range pods.Items { - if pod.Status.Phase == v1.PodSucceeded { - continue - } - for _, status := range pod.Status.ContainerStatuses { - if !status.Ready { - failures = append(failures, pod.Name) - } + if err != nil { + return fmt.Errorf("error listing Pods: %v", err) + } + + for _, pod := range pods.Items { + if pod.Status.Phase == v1.PodSucceeded { + continue + } + for _, status := range pod.Status.ContainerStatuses { + if !status.Ready { + v.addError(&ValidationError{ + Kind: "Pod", + Name: "kube-system/" + pod.Name, + Message: fmt.Sprintf("kube-system pod %q is not healthy", pod.Name), + }) } } } - return + return nil } -func validateTheNodes(clusterName string, validationCluster *ValidationCluster) (*ValidationCluster, error) { - nodes := validationCluster.NodeList - - if nodes == nil || len(nodes.Items) == 0 { - return nil, fmt.Errorf("No nodes found in validationCluster") - } - // Needed for when NodesCount and MastersCounts are predefined, i.e tests - presetNodeCount := validationCluster.NodesCount == 0 - presetMasterCount := validationCluster.MastersCount == 0 - for i := range nodes.Items { - node := &nodes.Items[i] - - role := util.GetNodeRole(node) - if role == "" { - role = "node" - } - - n := &ValidationNode{ - Zone: node.ObjectMeta.Labels["failure-domain.beta.kubernetes.io/zone"], - Hostname: node.ObjectMeta.Labels["kubernetes.io/hostname"], - Role: role, - Status: GetNodeReadyStatus(node), +func (v *ValidationCluster) validateNodes(cloudGroups map[string]*cloudinstances.CloudInstanceGroup) { + for _, cloudGroup := range cloudGroups { + var allMembers []*cloudinstances.CloudInstanceGroupMember + allMembers = append(allMembers, cloudGroup.Ready...) + allMembers = append(allMembers, cloudGroup.NeedUpdate...) + if len(allMembers) < cloudGroup.MinSize { + v.addError(&ValidationError{ + Kind: "InstanceGroup", + Name: cloudGroup.InstanceGroup.Name, + Message: fmt.Sprintf("InstanceGroup %q did not have enough nodes %d vs %d", + cloudGroup.InstanceGroup.Name, + len(allMembers), + cloudGroup.MinSize), + }) } - ready := isNodeReady(node) + for _, member := range allMembers { + node := member.Node - // TODO: Use instance group role instead... - if n.Role == "master" { - if presetMasterCount { - validationCluster.MastersCount++ - } - if ready { - validationCluster.MastersReadyArray = append(validationCluster.MastersReadyArray, n) - } else { - validationCluster.MastersNotReadyArray = append(validationCluster.MastersNotReadyArray, n) - } - } else if n.Role == "node" { - if presetNodeCount { - validationCluster.NodesCount++ - } - if ready { - validationCluster.NodesReadyArray = append(validationCluster.NodesReadyArray, n) - } else { - validationCluster.NodesNotReadyArray = append(validationCluster.NodesNotReadyArray, n) + if node == nil { + v.addError(&ValidationError{ + Kind: "Machine", + Name: member.ID, + Message: fmt.Sprintf("machine %q has not yet joined cluster", member.ID), + }) + continue } - } - } - - validationCluster.MastersReady = true - if len(validationCluster.MastersNotReadyArray) != 0 || validationCluster.MastersCount != len(validationCluster.MastersReadyArray) { - validationCluster.MastersReady = false - } + role := util.GetNodeRole(node) + if role == "" { + role = "node" + } - validationCluster.NodesReady = true - if len(validationCluster.NodesNotReadyArray) != 0 || validationCluster.NodesCount > len(validationCluster.NodesReadyArray) { - validationCluster.NodesReady = false - } + n := &ValidationNode{ + Name: node.Name, + Zone: node.ObjectMeta.Labels["failure-domain.beta.kubernetes.io/zone"], + Hostname: node.ObjectMeta.Labels["kubernetes.io/hostname"], + Role: role, + Status: getNodeReadyStatus(node), + } - if !validationCluster.MastersReady { - validationCluster.Status = ClusterValidationFailed - validationCluster.ErrorMessage = fmt.Sprintf("your masters are NOT ready %s", clusterName) - return validationCluster, fmt.Errorf(validationCluster.ErrorMessage) - } + ready := isNodeReady(node) - if !validationCluster.NodesReady { - validationCluster.Status = ClusterValidationFailed - validationCluster.ErrorMessage = fmt.Sprintf("your nodes are NOT ready %s", clusterName) - return validationCluster, fmt.Errorf(validationCluster.ErrorMessage) - } + // TODO: Use instance group role instead... + if n.Role == "master" { + if !ready { + v.addError(&ValidationError{ + Kind: "Node", + Name: node.Name, + Message: fmt.Sprintf("master %q is not ready", node.Name), + }) + } - if len(validationCluster.ComponentFailures) != 0 { - validationCluster.Status = ClusterValidationFailed - validationCluster.ErrorMessage = fmt.Sprintf("your components are NOT healthy %s", clusterName) - return validationCluster, fmt.Errorf(validationCluster.ErrorMessage) - } + v.Nodes = append(v.Nodes, n) + } else if n.Role == "node" { + if !ready { + v.addError(&ValidationError{ + Kind: "Node", + Name: node.Name, + Message: fmt.Sprintf("node %q is not ready", node.Name), + }) + } - if len(validationCluster.PodFailures) != 0 { - validationCluster.Status = ClusterValidationFailed - validationCluster.ErrorMessage = fmt.Sprintf("your kube-system pods are NOT healthy %s", clusterName) - return validationCluster, fmt.Errorf(validationCluster.ErrorMessage) + v.Nodes = append(v.Nodes, n) + } else { + glog.Warningf("ignoring node with role %q", n.Role) + } + } } - - return validationCluster, nil } diff --git a/pkg/validation/validate_cluster_test.go b/pkg/validation/validate_cluster_test.go index 22c540b030432..e893201f47c96 100644 --- a/pkg/validation/validate_cluster_test.go +++ b/pkg/validation/validate_cluster_test.go @@ -21,128 +21,86 @@ import ( "testing" "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" + kopsapi "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/cloudinstances" ) -func Test_ValidateClusterPositive(t *testing.T) { - nodeList, err := dummyClient("true", "true").CoreV1().Nodes().List(metav1.ListOptions{}) - - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - validationCluster := &ValidationCluster{NodeList: nodeList, NodesCount: 1, MastersCount: 1} - validationCluster, err = validateTheNodes("foo", validationCluster) - - if err != nil { - printDebug(validationCluster) - t.Fatalf("unexpected error: %v", err) - } -} - -func Test_ValidateClusterMasterAndNodeNotReady(t *testing.T) { - nodeList, err := dummyClient("false", "false").CoreV1().Nodes().List(metav1.ListOptions{}) - - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - validationCluster := &ValidationCluster{NodeList: nodeList, NodesCount: 1, MastersCount: 1} - validationCluster, err = validateTheNodes("foo", validationCluster) - - if err == nil { - printDebug(validationCluster) - t.Fatalf("unexpected error: %v", err) - } -} - -func Test_ValidateClusterComponents(t *testing.T) { - nodeList, err := dummyClient("true", "true").CoreV1().Nodes().List(metav1.ListOptions{}) - - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - var component = make([]string, 1) - validationCluster := &ValidationCluster{NodeList: nodeList, NodesCount: 1, MastersCount: 1, ComponentFailures: component} - validationCluster, err = validateTheNodes("foo", validationCluster) - - if err == nil { - printDebug(validationCluster) - t.Fatalf("unexpected error: %v", err) - } -} - -func Test_ValidateClusterPods(t *testing.T) { - nodeList, err := dummyClient("true", "true").CoreV1().Nodes().List(metav1.ListOptions{}) - - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - var pod = make([]string, 1) - validationCluster := &ValidationCluster{NodeList: nodeList, NodesCount: 1, MastersCount: 1, PodFailures: pod} - validationCluster, err = validateTheNodes("foo", validationCluster) - - if err == nil { - printDebug(validationCluster) - t.Fatalf("unexpected error: %v", err) - } -} - -func Test_ValidateClusterNodeNotReady(t *testing.T) { - nodeList, err := dummyClient("true", "false").CoreV1().Nodes().List(metav1.ListOptions{}) - - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - validationCluster := &ValidationCluster{NodeList: nodeList, NodesCount: 1, MastersCount: 1} - validationCluster, err = validateTheNodes("foo", validationCluster) - - if err == nil { - printDebug(validationCluster) - t.Fatalf("unexpected error: %v", err) - } -} - -func Test_ValidateClusterMastersNotEnough(t *testing.T) { - nodeList, err := dummyClient("true", "true").CoreV1().Nodes().List(metav1.ListOptions{}) - - if err != nil { - t.Fatalf("unexpected error: %v", err) +func Test_ValidateNodesNotEnough(t *testing.T) { + groups := make(map[string]*cloudinstances.CloudInstanceGroup) + groups["node-1"] = &cloudinstances.CloudInstanceGroup{ + InstanceGroup: &kopsapi.InstanceGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + }, + Spec: kopsapi.InstanceGroupSpec{ + Role: kopsapi.InstanceGroupRoleNode, + }, + }, + Ready: []*cloudinstances.CloudInstanceGroupMember{ + { + ID: "i-00001", + Node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node-1a"}, + Status: v1.NodeStatus{ + Conditions: []v1.NodeCondition{ + {Type: "Ready", Status: v1.ConditionTrue}, + }, + }, + }, + }, + }, + NeedUpdate: []*cloudinstances.CloudInstanceGroupMember{ + { + ID: "i-00002", + Node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node-1b"}, + Status: v1.NodeStatus{ + Conditions: []v1.NodeCondition{ + {Type: "Ready", Status: v1.ConditionFalse}, + }, + }, + }, + }, + }, } - validationCluster := &ValidationCluster{NodeList: nodeList, NodesCount: 1, MastersCount: 3} - validationCluster, err = validateTheNodes("foo", validationCluster) - - if err == nil { - printDebug(validationCluster) - t.Fatalf("unexpected error: %v", err) + { + v := &ValidationCluster{} + groups["node-1"].MinSize = 3 + v.validateNodes(groups) + if len(v.Failures) != 2 { + printDebug(t, v) + t.Fatal("Too few nodes not caught") + } } -} - -func Test_ValidateNodesNotEnough(t *testing.T) { - nodeList, err := dummyClient("true", "true").CoreV1().Nodes().List(metav1.ListOptions{}) - if err != nil { - t.Fatalf("unexpected error: %v", err) + { + groups["node-1"].MinSize = 2 + v := &ValidationCluster{} + v.validateNodes(groups) + if len(v.Failures) != 1 { + printDebug(t, v) + t.Fatal("Not ready node not caught") + } } - validationCluster := &ValidationCluster{NodeList: nodeList, NodesCount: 3, MastersCount: 1} - validationCluster, err = validateTheNodes("foo", validationCluster) - - if err == nil { - printDebug(validationCluster) - t.Fatal("Too few nodes not caught") + { + groups["node-1"].NeedUpdate[0].Node.Status.Conditions[0].Status = v1.ConditionTrue + v := &ValidationCluster{} + v.validateNodes(groups) + if len(v.Failures) != 0 { + printDebug(t, v) + t.Fatal("unexpected errors") + } } } func Test_ValidateNoPodFailures(t *testing.T) { - failures, err := collectPodFailures(dummyPodClient( + v := &ValidationCluster{} + err := v.collectPodFailures(dummyPodClient( []map[string]string{ { "name": "pod1", @@ -161,14 +119,15 @@ func Test_ValidateNoPodFailures(t *testing.T) { t.Fatalf("unexpected error: %v", err) } - if len(failures) != 0 { - fmt.Printf("failures: %+v\n", failures) + if len(v.Failures) != 0 { + fmt.Printf("failures: %+v\n", v.Failures) t.Fatal("no failures expected") } } func Test_ValidatePodFailure(t *testing.T) { - failures, err := collectPodFailures(dummyPodClient( + v := &ValidationCluster{} + err := v.collectPodFailures(dummyPodClient( []map[string]string{ { "name": "pod1", @@ -182,43 +141,17 @@ func Test_ValidatePodFailure(t *testing.T) { t.Fatalf("unexpected error: %v", err) } - if len(failures) != 1 || failures[0] != "pod1" { - fmt.Printf("failures: %+v\n", failures) + if len(v.Failures) != 1 || v.Failures[0].Name != "kube-system/pod1" { + printDebug(t, v) t.Fatal("pod1 failure expected") } } -func printDebug(validationCluster *ValidationCluster) { - fmt.Printf("cluster - masters ready: %v, nodes ready: %v\n", validationCluster.MastersReady, validationCluster.NodesReady) - fmt.Printf("mastersNotReady %v\n", len(validationCluster.MastersNotReadyArray)) - fmt.Printf("mastersCount %v, mastersReady %v\n", validationCluster.MastersCount, len(validationCluster.MastersReadyArray)) - fmt.Printf("nodesNotReady %v\n", len(validationCluster.NodesNotReadyArray)) - fmt.Printf("nodesCount %v, nodesReady %v\n", validationCluster.NodesCount, len(validationCluster.NodesReadyArray)) - -} - -const NODE_READY = "nodeReady" - -func dummyClient(masterReady string, nodeReady string) kubernetes.Interface { - return fake.NewSimpleClientset(makeNodeList( - []map[string]string{ - { - "name": "master1", - "kubernetes.io/role": "master", - NODE_READY: masterReady, - }, - { - "name": "node1", - "kubernetes.io/role": "node", - NODE_READY: nodeReady, - }, - { - "name": "node2", - "kubernetes.io/role": "node", - NODE_READY: "true", - }, - }, - )) +func printDebug(t *testing.T, v *ValidationCluster) { + t.Logf("cluster - %d failures", len(v.Failures)) + for _, fail := range v.Failures { + t.Logf(" failure: %+v", fail) + } } func dummyPodClient(pods []map[string]string) kubernetes.Interface { @@ -251,98 +184,3 @@ func makePodList(pods []map[string]string) *v1.PodList { } return &list } - -func dummyNode(nodeMap map[string]string) v1.Node { - - nodeReady := v1.ConditionFalse - if nodeMap[NODE_READY] == "true" { - nodeReady = v1.ConditionTrue - } - expectedNode := v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: nodeMap["name"], - Labels: map[string]string{ - "kubernetes.io/role": nodeMap["kubernetes.io/role"], - }, - }, - Spec: v1.NodeSpec{}, - Status: v1.NodeStatus{ - Conditions: []v1.NodeCondition{ - { - Type: v1.NodeOutOfDisk, - Status: v1.ConditionTrue, - Reason: "KubeletOutOfDisk", - Message: "out of disk space", - }, - { - Type: v1.NodeMemoryPressure, - Status: v1.ConditionFalse, - Reason: "KubeletHasSufficientMemory", - Message: "kubelet has sufficient memory available", - }, - { - Type: v1.NodeDiskPressure, - Status: v1.ConditionFalse, - Reason: "KubeletHasSufficientDisk", - Message: "kubelet has sufficient disk space available", - }, - { - Type: v1.NodeReady, - Status: nodeReady, - Reason: "KubeletReady", - Message: "kubelet is posting ready status", - }, - }, - NodeInfo: v1.NodeSystemInfo{ - MachineID: "123", - SystemUUID: "abc", - BootID: "1b3", - KernelVersion: "3.16.0-0.bpo.4-amd64", - OSImage: "Debian GNU/Linux 7 (wheezy)", - //OperatingSystem: goruntime.GOOS, - //Architecture: goruntime.GOARCH, - ContainerRuntimeVersion: "test://1.5.0", - //KubeletVersion: version.Get().String(), - //KubeProxyVersion: version.Get().String(), - }, - Capacity: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(20E9, resource.BinarySI), - v1.ResourcePods: *resource.NewQuantity(0, resource.DecimalSI), - v1.ResourceNvidiaGPU: *resource.NewQuantity(0, resource.DecimalSI), - }, - Allocatable: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(1800, resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(19900E6, resource.BinarySI), - v1.ResourcePods: *resource.NewQuantity(0, resource.DecimalSI), - v1.ResourceNvidiaGPU: *resource.NewQuantity(0, resource.DecimalSI), - }, - Addresses: []v1.NodeAddress{ - {Type: v1.NodeAddressType("LegacyHostIP"), Address: "127.0.0.1"}, - {Type: v1.NodeInternalIP, Address: "127.0.0.1"}, - {Type: v1.NodeHostName, Address: nodeMap["name"]}, - }, - // images will be sorted from max to min in node status. - Images: []v1.ContainerImage{ - { - Names: []string{"k8s.gcr.io:v3", "k8s.gcr.io:v4"}, - SizeBytes: 456, - }, - { - Names: []string{"k8s.gcr.io:v1", "k8s.gcr.io:v2"}, - SizeBytes: 123, - }, - }, - }, - } - return expectedNode -} - -// MakeNodeList constructs api.NodeList from list of node names and a NodeResource. -func makeNodeList(nodes []map[string]string) *v1.NodeList { - var list v1.NodeList - for _, node := range nodes { - list.Items = append(list.Items, dummyNode(node)) - } - return &list -}