Add ability to store cluster spec in userdata, so component config

changes are detected and marks nodes with 'NEEDUPDATE'

pkg/apis/kops/cluster.go

@@ -137,6 +137,12 @@ type ClusterSpec struct {
 	// Additional policies to add for roles
 	AdditionalPolicies *map[string]string `json:"additionalPolicies,omitempty"`
 
+	// Include cluster spec in user data to detect component config changes
+	EnableClusterSpecInUserData bool `json:"enableClusterSpecInUserData,omitempty"`
+
+	// Hash cluster spec yaml in user data to reduce file size
+	EnableClusterSpecHash bool `json:"enableClusterSpecHash,omitempty"`
+
 	//HairpinMode                   string `json:",omitempty"`
 	//
 	//OpencontrailTag               string `json:",omitempty"`

pkg/apis/kops/v1alpha1/cluster.go

@@ -134,6 +134,12 @@ type ClusterSpec struct {
 	// Additional policies to add for roles
 	AdditionalPolicies *map[string]string `json:"additionalPolicies,omitempty"`
 
+	// Include cluster spec in user data to detect component config changes
+	EnableClusterSpecInUserData bool `json:"enableClusterSpecInUserData,omitempty"`
+
+	// Hash cluster spec yaml in user data to reduce file size
+	EnableClusterSpecHash bool `json:"enableClusterSpecHash,omitempty"`
+
 	//HairpinMode                   string `json:",omitempty"`
 	//
 	//OpencontrailTag               string `json:",omitempty"`

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -524,6 +524,8 @@ func autoConvert_v1alpha1_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *
 	out.IsolateMasters = in.IsolateMasters
 	out.UpdatePolicy = in.UpdatePolicy
 	out.AdditionalPolicies = in.AdditionalPolicies
+	out.EnableClusterSpecInUserData = in.EnableClusterSpecInUserData
+	out.EnableClusterSpecHash = in.EnableClusterSpecHash
 	if in.EtcdClusters != nil {
 		in, out := &in.EtcdClusters, &out.EtcdClusters
 		*out = make([]*kops.EtcdClusterSpec, len(*in))
@@ -709,6 +711,8 @@ func autoConvert_kops_ClusterSpec_To_v1alpha1_ClusterSpec(in *kops.ClusterSpec,
 	out.IsolateMasters = in.IsolateMasters
 	out.UpdatePolicy = in.UpdatePolicy
 	out.AdditionalPolicies = in.AdditionalPolicies
+	out.EnableClusterSpecInUserData = in.EnableClusterSpecInUserData
+	out.EnableClusterSpecHash = in.EnableClusterSpecHash
 	if in.EtcdClusters != nil {
 		in, out := &in.EtcdClusters, &out.EtcdClusters
 		*out = make([]*EtcdClusterSpec, len(*in))

pkg/apis/kops/v1alpha2/cluster.go

@@ -138,6 +138,12 @@ type ClusterSpec struct {
 	// Additional policies to add for roles
 	AdditionalPolicies *map[string]string `json:"additionalPolicies,omitempty"`
 
+	// Include cluster spec in user data to detect component config changes
+	EnableClusterSpecInUserData bool `json:"enableClusterSpecInUserData,omitempty"`
+
+	// Hash cluster spec yaml in user data to reduce file size
+	EnableClusterSpecHash bool `json:"enableClusterSpecHash,omitempty"`
+
 	// EtcdClusters stores the configuration for each cluster
 	EtcdClusters []*EtcdClusterSpec `json:"etcdClusters,omitempty"`
 

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -562,6 +562,8 @@ func autoConvert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out *
 	out.IsolateMasters = in.IsolateMasters
 	out.UpdatePolicy = in.UpdatePolicy
 	out.AdditionalPolicies = in.AdditionalPolicies
+	out.EnableClusterSpecInUserData = in.EnableClusterSpecInUserData
+	out.EnableClusterSpecHash = in.EnableClusterSpecHash
 	if in.EtcdClusters != nil {
 		in, out := &in.EtcdClusters, &out.EtcdClusters
 		*out = make([]*kops.EtcdClusterSpec, len(*in))
@@ -762,6 +764,8 @@ func autoConvert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec,
 	out.IsolateMasters = in.IsolateMasters
 	out.UpdatePolicy = in.UpdatePolicy
 	out.AdditionalPolicies = in.AdditionalPolicies
+	out.EnableClusterSpecInUserData = in.EnableClusterSpecInUserData
+	out.EnableClusterSpecHash = in.EnableClusterSpecHash
 	if in.EtcdClusters != nil {
 		in, out := &in.EtcdClusters, &out.EtcdClusters
 		*out = make([]*EtcdClusterSpec, len(*in))

pkg/model/awsmodel/autoscalinggroup.go

@@ -18,6 +18,7 @@ package awsmodel
 
 import (
 	"fmt"
+
 	"github.com/golang/glog"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/model"
@@ -117,7 +118,7 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 				return err
 			}
 
-			if t.UserData, err = b.BootstrapScript.ResourceNodeUp(ig); err != nil {
+			if t.UserData, err = b.BootstrapScript.ResourceNodeUp(ig, &b.Cluster.Spec); err != nil {
 				return err
 			}
 

pkg/model/bootstrapscript.go

@@ -17,13 +17,18 @@ limitations under the License.
 package model
 
 import (
+	"encoding/base64"
+	"encoding/json"
 	"fmt"
+	"os"
+	"text/template"
+
+	"github.com/ghodss/yaml"
+
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/nodeup"
 	"k8s.io/kops/pkg/model/resources"
 	"k8s.io/kops/upup/pkg/fi"
-	"os"
-	"text/template"
 )
 
 // BootstrapScript creates the bootstrap script
@@ -33,12 +38,49 @@ type BootstrapScript struct {
 	NodeUpConfigBuilder func(ig *kops.InstanceGroup) (*nodeup.NodeUpConfig, error)
 }
 
-func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup) (*fi.ResourceHolder, error) {
+func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup, cs *kops.ClusterSpec) (*fi.ResourceHolder, error) {
 	if ig.Spec.Role == kops.InstanceGroupRoleBastion {
 		// Bastions are just bare machines (currently), used as SSH jump-hosts
 		return nil, nil
 	}
 
+	var igSpec string
+	if cs.EnableClusterSpecInUserData {
+		spec := make(map[string]interface{})
+
+		spec["docker"] = cs.Docker
+		spec["kubeProxy"] = cs.KubeProxy
+		spec["kubelet"] = cs.Kubelet
+		spec["cloudConfig"] = cs.CloudConfig
+
+		if ig.IsMaster() {
+			spec["kubeAPIServer"] = cs.KubeAPIServer
+			spec["kubeControllerManager"] = cs.KubeControllerManager
+			spec["kubeScheduler"] = cs.KubeScheduler
+			spec["masterKubelet"] = cs.MasterKubelet
+		}
+
+		j, err := json.Marshal(spec)
+		if err != nil {
+			return nil, err
+		}
+		content, err := yaml.JSONToYAML(j)
+		if err != nil {
+			return nil, err
+		}
+
+		if cs.EnableClusterSpecHash {
+			igSpec = base64.StdEncoding.EncodeToString(content)
+		} else {
+			igSpec = string(content)
+		}
+	}
+
+	context := map[string]interface{}{
+		"IncludeClusterSpec": cs.EnableClusterSpecInUserData,
+		"ClusterSpecContent": igSpec,
+	}
+
 	functions := template.FuncMap{
 		"NodeUpSource": func() string {
 			return b.NodeUpSource
@@ -81,7 +123,7 @@ func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup) (*fi.ResourceHo
 		},
 	}
 
-	templateResource, err := NewTemplateResource("nodeup", resources.AWSNodeUpTemplate, functions, nil)
+	templateResource, err := NewTemplateResource("nodeup", resources.AWSNodeUpTemplate, functions, context)
 	if err != nil {
 		return nil, err
 	}

pkg/model/bootstrapscript_test.go

@@ -0,0 +1,130 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package model
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/apis/nodeup"
+)
+
+func TestBootstrapUserData(t *testing.T) {
+	cs := []struct {
+		EnableClusterSpec bool
+		HashClusterSpec   bool
+		Role              kops.InstanceGroupRole
+		ExpectedFilePath  string
+	}{
+		{
+			EnableClusterSpec: false,
+			HashClusterSpec:   false,
+			Role:              "Master",
+			ExpectedFilePath:  "tests/data/bootstrapscript_0.txt",
+		},
+		{
+			EnableClusterSpec: true,
+			HashClusterSpec:   false,
+			Role:              "Master",
+			ExpectedFilePath:  "tests/data/bootstrapscript_1.txt",
+		},
+		{
+			EnableClusterSpec: true,
+			HashClusterSpec:   false,
+			Role:              "Node",
+			ExpectedFilePath:  "tests/data/bootstrapscript_2.txt",
+		},
+		{
+			EnableClusterSpec: true,
+			HashClusterSpec:   true,
+			Role:              "Master",
+			ExpectedFilePath:  "tests/data/bootstrapscript_3.txt",
+		},
+	}
+
+	for i, x := range cs {
+		spec := makeTestCluster(x.EnableClusterSpec, x.HashClusterSpec).Spec
+		group := makeTestInstanceGroup(x.Role)
+
+		renderNodeUpConfig := func(ig *kops.InstanceGroup) (*nodeup.NodeUpConfig, error) {
+			return &nodeup.NodeUpConfig{}, nil
+		}
+
+		bs := &BootstrapScript{
+			NodeUpSource:        "NUSource",
+			NodeUpSourceHash:    "NUSHash",
+			NodeUpConfigBuilder: renderNodeUpConfig,
+		}
+
+		res, err := bs.ResourceNodeUp(group, &spec)
+		if err != nil {
+			t.Errorf("case %d failed to create nodeup resource. error: %s", i, err)
+			continue
+		}
+
+		actual, err := res.AsString()
+		if err != nil {
+			t.Errorf("case %d failed to render nodeup resource. error: %s", i, err)
+			continue
+		}
+
+		expectedBytes, err := ioutil.ReadFile(x.ExpectedFilePath)
+		if err != nil {
+			t.Fatalf("unexpected error reading ExpectedFilePath %q: %v", x.ExpectedFilePath, err)
+		}
+
+		if actual != string(expectedBytes) {
+			t.Errorf("case %d, expected: %s. got: %s", i, string(expectedBytes), actual)
+		}
+	}
+}
+
+func makeTestCluster(enableClusterSpec bool, hashClusterSpec bool) *kops.Cluster {
+	return &kops.Cluster{
+		Spec: kops.ClusterSpec{
+			EnableClusterSpecInUserData: enableClusterSpec,
+			EnableClusterSpecHash:       hashClusterSpec,
+			CloudProvider:               "aws",
+			KubernetesVersion:           "1.7.0",
+			Subnets: []kops.ClusterSubnetSpec{
+				{Name: "test", Zone: "eu-west-1a"},
+			},
+			NonMasqueradeCIDR: "10.100.0.0/16",
+			EtcdClusters: []*kops.EtcdClusterSpec{
+				{
+					Name: "main",
+					Members: []*kops.EtcdMemberSpec{
+						{
+							Name:          "test",
+							InstanceGroup: s("master-1"),
+						},
+					},
+				},
+			},
+			NetworkCIDR: "10.79.0.0/24",
+		},
+	}
+}
+
+func makeTestInstanceGroup(role kops.InstanceGroupRole) *kops.InstanceGroup {
+	return &kops.InstanceGroup{
+		Spec: kops.InstanceGroupSpec{
+			Role: role,
+		},
+	}
+}

pkg/model/gcemodel/autoscalinggroup.go

@@ -18,6 +18,7 @@ package gcemodel
 
 import (
 	"fmt"
+
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/model"
@@ -44,7 +45,7 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 	for _, ig := range b.InstanceGroups {
 		name := b.SafeObjectName(ig.ObjectMeta.Name)
 
-		startupScript, err := b.BootstrapScript.ResourceNodeUp(ig)
+		startupScript, err := b.BootstrapScript.ResourceNodeUp(ig, &b.Cluster.Spec)
 		if err != nil {
 			return err
 		}

pkg/model/resources/nodeup.go

@@ -138,7 +138,11 @@ function download-release() {
 
 echo "== nodeup node config starting =="
 ensure-install-dir
-
+{{ if .IncludeClusterSpec }}
+cat > cluster_spec.yaml << __EOF_CLUSTER_SPEC
+{{ .ClusterSpecContent }}
+__EOF_CLUSTER_SPEC
+{{ end }}
 cat > kube_env.yaml << __EOF_KUBE_ENV
 {{ KubeEnv }}
 __EOF_KUBE_ENV