From 5a223581ae9b5b8be67f2415b6a4a25feb7bb90a Mon Sep 17 00:00:00 2001 From: Kashif Saadat Date: Tue, 1 Aug 2017 22:50:38 +0100 Subject: [PATCH] Add ability to store cluster spec in userdata, so component config changes are detected and marks nodes with 'NEEDUPDATE' --- pkg/apis/kops/cluster.go | 6 + pkg/apis/kops/v1alpha1/cluster.go | 6 + .../kops/v1alpha1/zz_generated.conversion.go | 4 + pkg/apis/kops/v1alpha2/cluster.go | 6 + .../kops/v1alpha2/zz_generated.conversion.go | 4 + pkg/model/awsmodel/autoscalinggroup.go | 3 +- pkg/model/bootstrapscript.go | 50 +++++- pkg/model/bootstrapscript_test.go | 130 ++++++++++++++++ pkg/model/gcemodel/autoscalinggroup.go | 3 +- pkg/model/resources/nodeup.go | 6 +- pkg/model/tests/data/bootstrapscript_0.txt | 130 ++++++++++++++++ pkg/model/tests/data/bootstrapscript_1.txt | 142 ++++++++++++++++++ pkg/model/tests/data/bootstrapscript_2.txt | 138 +++++++++++++++++ pkg/model/tests/data/bootstrapscript_3.txt | 134 +++++++++++++++++ pkg/model/vspheremodel/autoscalinggroup.go | 5 +- upup/pkg/fi/cloudup/vspheretasks/attachiso.go | 19 +-- 16 files changed, 768 insertions(+), 18 deletions(-) create mode 100644 pkg/model/bootstrapscript_test.go create mode 100644 pkg/model/tests/data/bootstrapscript_0.txt create mode 100644 pkg/model/tests/data/bootstrapscript_1.txt create mode 100644 pkg/model/tests/data/bootstrapscript_2.txt create mode 100644 pkg/model/tests/data/bootstrapscript_3.txt diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index 6925122c17f02..6c81b0db20a34 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -137,6 +137,12 @@ type ClusterSpec struct { // Additional policies to add for roles AdditionalPolicies *map[string]string `json:"additionalPolicies,omitempty"` + // Include cluster spec in user data to detect component config changes + EnableClusterSpecInUserData bool `json:"enableClusterSpecInUserData,omitempty"` + + // Hash cluster spec yaml in user data to reduce file size + EnableClusterSpecHash bool `json:"enableClusterSpecHash,omitempty"` + //HairpinMode string `json:",omitempty"` // //OpencontrailTag string `json:",omitempty"` diff --git a/pkg/apis/kops/v1alpha1/cluster.go b/pkg/apis/kops/v1alpha1/cluster.go index d0697dc3cd643..22b1094c881a8 100644 --- a/pkg/apis/kops/v1alpha1/cluster.go +++ b/pkg/apis/kops/v1alpha1/cluster.go @@ -134,6 +134,12 @@ type ClusterSpec struct { // Additional policies to add for roles AdditionalPolicies *map[string]string `json:"additionalPolicies,omitempty"` + // Include cluster spec in user data to detect component config changes + EnableClusterSpecInUserData bool `json:"enableClusterSpecInUserData,omitempty"` + + // Hash cluster spec yaml in user data to reduce file size + EnableClusterSpecHash bool `json:"enableClusterSpecHash,omitempty"` + //HairpinMode string `json:",omitempty"` // //OpencontrailTag string `json:",omitempty"` diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index c2553ef1a3e06..7f977bffcaceb 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -524,6 +524,8 @@ func autoConvert_v1alpha1_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out * out.IsolateMasters = in.IsolateMasters out.UpdatePolicy = in.UpdatePolicy out.AdditionalPolicies = in.AdditionalPolicies + out.EnableClusterSpecInUserData = in.EnableClusterSpecInUserData + out.EnableClusterSpecHash = in.EnableClusterSpecHash if in.EtcdClusters != nil { in, out := &in.EtcdClusters, &out.EtcdClusters *out = make([]*kops.EtcdClusterSpec, len(*in)) @@ -709,6 +711,8 @@ func autoConvert_kops_ClusterSpec_To_v1alpha1_ClusterSpec(in *kops.ClusterSpec, out.IsolateMasters = in.IsolateMasters out.UpdatePolicy = in.UpdatePolicy out.AdditionalPolicies = in.AdditionalPolicies + out.EnableClusterSpecInUserData = in.EnableClusterSpecInUserData + out.EnableClusterSpecHash = in.EnableClusterSpecHash if in.EtcdClusters != nil { in, out := &in.EtcdClusters, &out.EtcdClusters *out = make([]*EtcdClusterSpec, len(*in)) diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 99ad3b766d5f3..119319f9b4fe1 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -138,6 +138,12 @@ type ClusterSpec struct { // Additional policies to add for roles AdditionalPolicies *map[string]string `json:"additionalPolicies,omitempty"` + // Include cluster spec in user data to detect component config changes + EnableClusterSpecInUserData bool `json:"enableClusterSpecInUserData,omitempty"` + + // Hash cluster spec yaml in user data to reduce file size + EnableClusterSpecHash bool `json:"enableClusterSpecHash,omitempty"` + // EtcdClusters stores the configuration for each cluster EtcdClusters []*EtcdClusterSpec `json:"etcdClusters,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 91f929fc3db09..a7affd2a9038d 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -562,6 +562,8 @@ func autoConvert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out * out.IsolateMasters = in.IsolateMasters out.UpdatePolicy = in.UpdatePolicy out.AdditionalPolicies = in.AdditionalPolicies + out.EnableClusterSpecInUserData = in.EnableClusterSpecInUserData + out.EnableClusterSpecHash = in.EnableClusterSpecHash if in.EtcdClusters != nil { in, out := &in.EtcdClusters, &out.EtcdClusters *out = make([]*kops.EtcdClusterSpec, len(*in)) @@ -762,6 +764,8 @@ func autoConvert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec, out.IsolateMasters = in.IsolateMasters out.UpdatePolicy = in.UpdatePolicy out.AdditionalPolicies = in.AdditionalPolicies + out.EnableClusterSpecInUserData = in.EnableClusterSpecInUserData + out.EnableClusterSpecHash = in.EnableClusterSpecHash if in.EtcdClusters != nil { in, out := &in.EtcdClusters, &out.EtcdClusters *out = make([]*EtcdClusterSpec, len(*in)) diff --git a/pkg/model/awsmodel/autoscalinggroup.go b/pkg/model/awsmodel/autoscalinggroup.go index b41a12f192cc6..9af52103f64c8 100644 --- a/pkg/model/awsmodel/autoscalinggroup.go +++ b/pkg/model/awsmodel/autoscalinggroup.go @@ -18,6 +18,7 @@ package awsmodel import ( "fmt" + "github.com/golang/glog" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/model" @@ -117,7 +118,7 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { return err } - if t.UserData, err = b.BootstrapScript.ResourceNodeUp(ig); err != nil { + if t.UserData, err = b.BootstrapScript.ResourceNodeUp(ig, &b.Cluster.Spec); err != nil { return err } diff --git a/pkg/model/bootstrapscript.go b/pkg/model/bootstrapscript.go index 6b93ad2813c6a..d49fd45193814 100644 --- a/pkg/model/bootstrapscript.go +++ b/pkg/model/bootstrapscript.go @@ -17,13 +17,18 @@ limitations under the License. package model import ( + "encoding/base64" + "encoding/json" "fmt" + "os" + "text/template" + + "github.com/ghodss/yaml" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/apis/nodeup" "k8s.io/kops/pkg/model/resources" "k8s.io/kops/upup/pkg/fi" - "os" - "text/template" ) // BootstrapScript creates the bootstrap script @@ -33,12 +38,49 @@ type BootstrapScript struct { NodeUpConfigBuilder func(ig *kops.InstanceGroup) (*nodeup.NodeUpConfig, error) } -func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup) (*fi.ResourceHolder, error) { +func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup, cs *kops.ClusterSpec) (*fi.ResourceHolder, error) { if ig.Spec.Role == kops.InstanceGroupRoleBastion { // Bastions are just bare machines (currently), used as SSH jump-hosts return nil, nil } + var igSpec string + if cs.EnableClusterSpecInUserData { + spec := make(map[string]interface{}) + + spec["docker"] = cs.Docker + spec["kubeProxy"] = cs.KubeProxy + spec["kubelet"] = cs.Kubelet + spec["cloudConfig"] = cs.CloudConfig + + if ig.IsMaster() { + spec["kubeAPIServer"] = cs.KubeAPIServer + spec["kubeControllerManager"] = cs.KubeControllerManager + spec["kubeScheduler"] = cs.KubeScheduler + spec["masterKubelet"] = cs.MasterKubelet + } + + j, err := json.Marshal(spec) + if err != nil { + return nil, err + } + content, err := yaml.JSONToYAML(j) + if err != nil { + return nil, err + } + + if cs.EnableClusterSpecHash { + igSpec = base64.StdEncoding.EncodeToString(content) + } else { + igSpec = string(content) + } + } + + context := map[string]interface{}{ + "IncludeClusterSpec": cs.EnableClusterSpecInUserData, + "ClusterSpecContent": igSpec, + } + functions := template.FuncMap{ "NodeUpSource": func() string { return b.NodeUpSource @@ -81,7 +123,7 @@ func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup) (*fi.ResourceHo }, } - templateResource, err := NewTemplateResource("nodeup", resources.AWSNodeUpTemplate, functions, nil) + templateResource, err := NewTemplateResource("nodeup", resources.AWSNodeUpTemplate, functions, context) if err != nil { return nil, err } diff --git a/pkg/model/bootstrapscript_test.go b/pkg/model/bootstrapscript_test.go new file mode 100644 index 0000000000000..5f85f32aa8131 --- /dev/null +++ b/pkg/model/bootstrapscript_test.go @@ -0,0 +1,130 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "io/ioutil" + "testing" + + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/apis/nodeup" +) + +func TestBootstrapUserData(t *testing.T) { + cs := []struct { + EnableClusterSpec bool + HashClusterSpec bool + Role kops.InstanceGroupRole + ExpectedFilePath string + }{ + { + EnableClusterSpec: false, + HashClusterSpec: false, + Role: "Master", + ExpectedFilePath: "tests/data/bootstrapscript_0.txt", + }, + { + EnableClusterSpec: true, + HashClusterSpec: false, + Role: "Master", + ExpectedFilePath: "tests/data/bootstrapscript_1.txt", + }, + { + EnableClusterSpec: true, + HashClusterSpec: false, + Role: "Node", + ExpectedFilePath: "tests/data/bootstrapscript_2.txt", + }, + { + EnableClusterSpec: true, + HashClusterSpec: true, + Role: "Master", + ExpectedFilePath: "tests/data/bootstrapscript_3.txt", + }, + } + + for i, x := range cs { + spec := makeTestCluster(x.EnableClusterSpec, x.HashClusterSpec).Spec + group := makeTestInstanceGroup(x.Role) + + renderNodeUpConfig := func(ig *kops.InstanceGroup) (*nodeup.NodeUpConfig, error) { + return &nodeup.NodeUpConfig{}, nil + } + + bs := &BootstrapScript{ + NodeUpSource: "NUSource", + NodeUpSourceHash: "NUSHash", + NodeUpConfigBuilder: renderNodeUpConfig, + } + + res, err := bs.ResourceNodeUp(group, &spec) + if err != nil { + t.Errorf("case %d failed to create nodeup resource. error: %s", i, err) + continue + } + + actual, err := res.AsString() + if err != nil { + t.Errorf("case %d failed to render nodeup resource. error: %s", i, err) + continue + } + + expectedBytes, err := ioutil.ReadFile(x.ExpectedFilePath) + if err != nil { + t.Fatalf("unexpected error reading ExpectedFilePath %q: %v", x.ExpectedFilePath, err) + } + + if actual != string(expectedBytes) { + t.Errorf("case %d, expected: %s. got: %s", i, string(expectedBytes), actual) + } + } +} + +func makeTestCluster(enableClusterSpec bool, hashClusterSpec bool) *kops.Cluster { + return &kops.Cluster{ + Spec: kops.ClusterSpec{ + EnableClusterSpecInUserData: enableClusterSpec, + EnableClusterSpecHash: hashClusterSpec, + CloudProvider: "aws", + KubernetesVersion: "1.7.0", + Subnets: []kops.ClusterSubnetSpec{ + {Name: "test", Zone: "eu-west-1a"}, + }, + NonMasqueradeCIDR: "10.100.0.0/16", + EtcdClusters: []*kops.EtcdClusterSpec{ + { + Name: "main", + Members: []*kops.EtcdMemberSpec{ + { + Name: "test", + InstanceGroup: s("master-1"), + }, + }, + }, + }, + NetworkCIDR: "10.79.0.0/24", + }, + } +} + +func makeTestInstanceGroup(role kops.InstanceGroupRole) *kops.InstanceGroup { + return &kops.InstanceGroup{ + Spec: kops.InstanceGroupSpec{ + Role: role, + }, + } +} diff --git a/pkg/model/gcemodel/autoscalinggroup.go b/pkg/model/gcemodel/autoscalinggroup.go index d0e46c7d7444a..8ae8ba177aba2 100644 --- a/pkg/model/gcemodel/autoscalinggroup.go +++ b/pkg/model/gcemodel/autoscalinggroup.go @@ -18,6 +18,7 @@ package gcemodel import ( "fmt" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/model" @@ -44,7 +45,7 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { for _, ig := range b.InstanceGroups { name := b.SafeObjectName(ig.ObjectMeta.Name) - startupScript, err := b.BootstrapScript.ResourceNodeUp(ig) + startupScript, err := b.BootstrapScript.ResourceNodeUp(ig, &b.Cluster.Spec) if err != nil { return err } diff --git a/pkg/model/resources/nodeup.go b/pkg/model/resources/nodeup.go index 0188e14e504a5..bb771319f499f 100644 --- a/pkg/model/resources/nodeup.go +++ b/pkg/model/resources/nodeup.go @@ -138,7 +138,11 @@ function download-release() { echo "== nodeup node config starting ==" ensure-install-dir - +{{ if .IncludeClusterSpec }} +cat > cluster_spec.yaml << __EOF_CLUSTER_SPEC +{{ .ClusterSpecContent }} +__EOF_CLUSTER_SPEC +{{ end }} cat > kube_env.yaml << __EOF_KUBE_ENV {{ KubeEnv }} __EOF_KUBE_ENV diff --git a/pkg/model/tests/data/bootstrapscript_0.txt b/pkg/model/tests/data/bootstrapscript_0.txt new file mode 100644 index 0000000000000..548526ebabadd --- /dev/null +++ b/pkg/model/tests/data/bootstrapscript_0.txt @@ -0,0 +1,130 @@ +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=NUSource +NODEUP_HASH=NUSHash + + + + +function ensure-install-dir() { + INSTALL_DIR="/var/cache/kubernetes-install" + # On ContainerOS, we install to /var/lib/toolbox install (because of noexec) + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kubernetes-install" + fi + mkdir -p ${INSTALL_DIR} + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. Takes a hash and a set of URLs. +# +# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown. +# $2+ are the URLs to download. +download-or-bust() { + local -r hash="$1" + shift 1 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + local file="${url##*/}" + rm -f "${file}" + if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then + echo "== Failed to download ${url}. Retrying. ==" + elif [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha1sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + # TODO(zmerlynn): Now we REALLY have no excuse not to do the reboot + # optimization. + + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + local -r nodeup_filename="${nodeup_urls[0]##*/}" + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha1 (not found in env)" + download-or-bust "" "${nodeup_urls[@]/%/.sha1}" + local -r nodeup_hash=$(cat "${nodeup_filename}.sha1") + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + # In case of failure checking integrity of release, retry. + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > kube_env.yaml << __EOF_KUBE_ENV +{} + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/pkg/model/tests/data/bootstrapscript_1.txt b/pkg/model/tests/data/bootstrapscript_1.txt new file mode 100644 index 0000000000000..7c926e11e7552 --- /dev/null +++ b/pkg/model/tests/data/bootstrapscript_1.txt @@ -0,0 +1,142 @@ +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=NUSource +NODEUP_HASH=NUSHash + + + + +function ensure-install-dir() { + INSTALL_DIR="/var/cache/kubernetes-install" + # On ContainerOS, we install to /var/lib/toolbox install (because of noexec) + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kubernetes-install" + fi + mkdir -p ${INSTALL_DIR} + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. Takes a hash and a set of URLs. +# +# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown. +# $2+ are the URLs to download. +download-or-bust() { + local -r hash="$1" + shift 1 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + local file="${url##*/}" + rm -f "${file}" + if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then + echo "== Failed to download ${url}. Retrying. ==" + elif [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha1sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + # TODO(zmerlynn): Now we REALLY have no excuse not to do the reboot + # optimization. + + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + local -r nodeup_filename="${nodeup_urls[0]##*/}" + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha1 (not found in env)" + download-or-bust "" "${nodeup_urls[@]/%/.sha1}" + local -r nodeup_hash=$(cat "${nodeup_filename}.sha1") + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + # In case of failure checking integrity of release, retry. + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > cluster_spec.yaml << __EOF_CLUSTER_SPEC +cloudConfig: null +docker: null +kubeAPIServer: null +kubeControllerManager: null +kubeProxy: null +kubeScheduler: null +kubelet: null +masterKubelet: null + +__EOF_CLUSTER_SPEC + +cat > kube_env.yaml << __EOF_KUBE_ENV +{} + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/pkg/model/tests/data/bootstrapscript_2.txt b/pkg/model/tests/data/bootstrapscript_2.txt new file mode 100644 index 0000000000000..4e3a8c61d99b8 --- /dev/null +++ b/pkg/model/tests/data/bootstrapscript_2.txt @@ -0,0 +1,138 @@ +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=NUSource +NODEUP_HASH=NUSHash + + + + +function ensure-install-dir() { + INSTALL_DIR="/var/cache/kubernetes-install" + # On ContainerOS, we install to /var/lib/toolbox install (because of noexec) + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kubernetes-install" + fi + mkdir -p ${INSTALL_DIR} + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. Takes a hash and a set of URLs. +# +# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown. +# $2+ are the URLs to download. +download-or-bust() { + local -r hash="$1" + shift 1 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + local file="${url##*/}" + rm -f "${file}" + if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then + echo "== Failed to download ${url}. Retrying. ==" + elif [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha1sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + # TODO(zmerlynn): Now we REALLY have no excuse not to do the reboot + # optimization. + + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + local -r nodeup_filename="${nodeup_urls[0]##*/}" + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha1 (not found in env)" + download-or-bust "" "${nodeup_urls[@]/%/.sha1}" + local -r nodeup_hash=$(cat "${nodeup_filename}.sha1") + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + # In case of failure checking integrity of release, retry. + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > cluster_spec.yaml << __EOF_CLUSTER_SPEC +cloudConfig: null +docker: null +kubeProxy: null +kubelet: null + +__EOF_CLUSTER_SPEC + +cat > kube_env.yaml << __EOF_KUBE_ENV +{} + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/pkg/model/tests/data/bootstrapscript_3.txt b/pkg/model/tests/data/bootstrapscript_3.txt new file mode 100644 index 0000000000000..97c1f1440bc68 --- /dev/null +++ b/pkg/model/tests/data/bootstrapscript_3.txt @@ -0,0 +1,134 @@ +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=NUSource +NODEUP_HASH=NUSHash + + + + +function ensure-install-dir() { + INSTALL_DIR="/var/cache/kubernetes-install" + # On ContainerOS, we install to /var/lib/toolbox install (because of noexec) + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kubernetes-install" + fi + mkdir -p ${INSTALL_DIR} + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. Takes a hash and a set of URLs. +# +# $1 is the sha1 of the URL. Can be "" if the sha1 is unknown. +# $2+ are the URLs to download. +download-or-bust() { + local -r hash="$1" + shift 1 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + local file="${url##*/}" + rm -f "${file}" + if ! curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10 "${url}"; then + echo "== Failed to download ${url}. Retrying. ==" + elif [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha1sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, sha1 ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + # TODO(zmerlynn): Now we REALLY have no excuse not to do the reboot + # optimization. + + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + local -r nodeup_filename="${nodeup_urls[0]##*/}" + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha1 (not found in env)" + download-or-bust "" "${nodeup_urls[@]/%/.sha1}" + local -r nodeup_hash=$(cat "${nodeup_filename}.sha1") + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + # In case of failure checking integrity of release, retry. + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > cluster_spec.yaml << __EOF_CLUSTER_SPEC +Y2xvdWRDb25maWc6IG51bGwKZG9ja2VyOiBudWxsCmt1YmVBUElTZXJ2ZXI6IG51bGwKa3ViZUNvbnRyb2xsZXJNYW5hZ2VyOiBudWxsCmt1YmVQcm94eTogbnVsbAprdWJlU2NoZWR1bGVyOiBudWxsCmt1YmVsZXQ6IG51bGwKbWFzdGVyS3ViZWxldDogbnVsbAo= +__EOF_CLUSTER_SPEC + +cat > kube_env.yaml << __EOF_KUBE_ENV +{} + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/pkg/model/vspheremodel/autoscalinggroup.go b/pkg/model/vspheremodel/autoscalinggroup.go index a97753d49466b..13e161a1a96e7 100644 --- a/pkg/model/vspheremodel/autoscalinggroup.go +++ b/pkg/model/vspheremodel/autoscalinggroup.go @@ -19,11 +19,12 @@ package vspheremodel // autoscalinggroup is a model for vSphere cloud. It's responsible for building tasks, necessary for kubernetes cluster deployment. import ( + "strconv" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/model" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/vspheretasks" - "strconv" ) // AutoscalingGroupModelBuilder configures AutoscalingGroup objects @@ -61,7 +62,7 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { VM: createVmTask, IG: ig, BootstrapScript: b.BootstrapScript, - EtcdClusters: b.Cluster.Spec.EtcdClusters, + Spec: &b.Cluster.Spec, } c.AddTask(attachISOTask) diff --git a/upup/pkg/fi/cloudup/vspheretasks/attachiso.go b/upup/pkg/fi/cloudup/vspheretasks/attachiso.go index eee4a8c2570f8..f5aefce44ae31 100644 --- a/upup/pkg/fi/cloudup/vspheretasks/attachiso.go +++ b/upup/pkg/fi/cloudup/vspheretasks/attachiso.go @@ -21,13 +21,7 @@ package vspheretasks import ( "bytes" "fmt" - "github.com/golang/glog" - "github.com/pborman/uuid" "io/ioutil" - "k8s.io/kops/pkg/apis/kops" - "k8s.io/kops/pkg/model" - "k8s.io/kops/upup/pkg/fi" - "k8s.io/kops/upup/pkg/fi/cloudup/vsphere" "net" "net/url" "os" @@ -35,6 +29,13 @@ import ( "path/filepath" "runtime" "strings" + + "github.com/golang/glog" + "github.com/pborman/uuid" + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/model" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/vsphere" ) // AttachISO represents the cloud-init ISO file attached to a VM on vSphere cloud. @@ -44,7 +45,7 @@ type AttachISO struct { VM *VirtualMachine IG *kops.InstanceGroup BootstrapScript *model.BootstrapScript - EtcdClusters []*kops.EtcdClusterSpec + Spec *kops.ClusterSpec } var _ fi.HasName = &AttachISO{} @@ -91,7 +92,7 @@ func (_ *AttachISO) CheckChanges(a, e, changes *AttachISO) error { // RenderVSphere executes the actual task logic, for vSphere cloud. func (_ *AttachISO) RenderVSphere(t *vsphere.VSphereAPITarget, a, e, changes *AttachISO) error { - startupScript, err := changes.BootstrapScript.ResourceNodeUp(changes.IG) + startupScript, err := changes.BootstrapScript.ResourceNodeUp(changes.IG, changes.Spec) if err != nil { return fmt.Errorf("error on resource nodeup: %v", err) } @@ -195,7 +196,7 @@ func getVolMetadata(changes *AttachISO) (string, error) { var volsMetadata []vsphere.VolumeMetadata // Creating vsphere.VolumeMetadata using clusters EtcdClusterSpec - for i, etcd := range changes.EtcdClusters { + for i, etcd := range changes.Spec.EtcdClusters { volMetadata := vsphere.VolumeMetadata{} volMetadata.EtcdClusterName = etcd.Name volMetadata.VolumeId = vsphere.GetVolumeId(i + 1)