diff --git a/hack/verify-terraform.sh b/hack/verify-terraform.sh index d53f5c2a15b5d..cf981f82afeeb 100755 --- a/hack/verify-terraform.sh +++ b/hack/verify-terraform.sh @@ -36,7 +36,7 @@ while IFS= read -r -d '' -u 3 test_dir; do cluster=$(basename "${test_dir}") kube::util::array_contains "${cluster}" "${CLUSTERS_0_11[@]}" && tag=$TAG_0_11 || tag=$TAG_0_12 - docker run --rm -it -v "${test_dir}":"${test_dir}" -w "${test_dir}" --entrypoint=sh hashicorp/terraform:$tag -c '/bin/terraform init >/dev/null && /bin/terraform validate' || RC=$? + docker run --rm -v "${test_dir}":"${test_dir}" -w "${test_dir}" --entrypoint=sh hashicorp/terraform:$tag -c '/bin/terraform init >/dev/null && /bin/terraform validate' || RC=$? done 3< <(find "${KOPS_ROOT}/tests/integration/update_cluster" -type d -maxdepth 1 -print0) if [ $RC != 0 ]; then diff --git a/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1a.masters.additionalcidr.example.com_user_data b/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1a.masters.additionalcidr.example.com_user_data index 002ba98635c3f..3216f0988cde5 100644 --- a/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1a.masters.additionalcidr.example.com_user_data +++ b/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1a.masters.additionalcidr.example.com_user_data @@ -1 +1,302 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMwogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBhZGRpdGlvbmFsY2lkci5leGFtcGxlLmNvbQogIGNvbmZpZ3VyZUNsb3VkUm91dGVzOiB0cnVlCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1jb250cm9sbGVyLW1hbmFnZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgogIHVzZVNlcnZpY2VBY2NvdW50Q3JlZGVudGlhbHM6IHRydWUKa3ViZVByb3h5OgogIGNsdXN0ZXJDSURSOiAxMDAuOTYuMC4wLzExCiAgY3B1UmVxdWVzdDogMTAwbQogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtcHJveHk6djEuMTQuMAogIGxvZ0xldmVsOiAyCmt1YmVTY2hlZHVsZXI6CiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1zY2hlZHVsZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgprdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5NVFU6IDkwMDEKICBuZXR3b3JrUGx1Z2luTmFtZToga3ViZW5ldAogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKbWFzdGVyS3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCiAgcmVnaXN0ZXJTY2hlZHVsYWJsZTogZmFsc2UKCl9fRU9GX0NMVVNURVJfU1BFQwoKY2F0ID4gY29uZi9pZ19zcGVjLnlhbWwgPDwgJ19fRU9GX0lHX1NQRUMnCmt1YmVsZXQ6IG51bGwKbm9kZUxhYmVsczogbnVsbAp0YWludHM6IG51bGwKCl9fRU9GX0lHX1NQRUMKCmNhdCA+IGNvbmYva3ViZV9lbnYueWFtbCA8PCAnX19FT0ZfS1VCRV9FTlYnCkFzc2V0czoKLSBjM2I3MzZmZDBmMDAzNzY1YzEyZDk5ZjJjOTk1YTgzNjllNjI0MWY0QGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlbGV0Ci0gN2UzYTNlYTY2MzE1M2Y5MDBjYmQ1MjkwMGEzOWM5MWZhOWYzMzRiZUBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDUyZTlkMmRlOGE1ZjkyNzMwN2Q5Mzk3MzA4NzM1NjU4ZWU0NGFiOGRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLXBsdWdpbnMtYW1kNjQtdjAuNy41LnRnegpDbHVzdGVyTmFtZTogYWRkaXRpb25hbGNpZHIuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9hZGRpdGlvbmFsY2lkci5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2FkZGl0aW9uYWxjaWRyLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCmV0Y2RNYW5pZmVzdHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9hZGRpdGlvbmFsY2lkci5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9tYWluLnlhbWwKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2FkZGl0aW9uYWxjaWRyLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: additionalcidr.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: additionalcidr.example.com +ConfigBase: memfs://clusters.example.com/additionalcidr.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/additionalcidr.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/additionalcidr.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/additionalcidr.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1b.masters.additionalcidr.example.com_user_data b/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1b.masters.additionalcidr.example.com_user_data index 18f849e0ff23a..0499a28f3ba3a 100644 --- a/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1b.masters.additionalcidr.example.com_user_data +++ b/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1b.masters.additionalcidr.example.com_user_data @@ -1 +1,302 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMwogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBhZGRpdGlvbmFsY2lkci5leGFtcGxlLmNvbQogIGNvbmZpZ3VyZUNsb3VkUm91dGVzOiB0cnVlCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1jb250cm9sbGVyLW1hbmFnZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgogIHVzZVNlcnZpY2VBY2NvdW50Q3JlZGVudGlhbHM6IHRydWUKa3ViZVByb3h5OgogIGNsdXN0ZXJDSURSOiAxMDAuOTYuMC4wLzExCiAgY3B1UmVxdWVzdDogMTAwbQogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtcHJveHk6djEuMTQuMAogIGxvZ0xldmVsOiAyCmt1YmVTY2hlZHVsZXI6CiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1zY2hlZHVsZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgprdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5NVFU6IDkwMDEKICBuZXR3b3JrUGx1Z2luTmFtZToga3ViZW5ldAogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKbWFzdGVyS3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCiAgcmVnaXN0ZXJTY2hlZHVsYWJsZTogZmFsc2UKCl9fRU9GX0NMVVNURVJfU1BFQwoKY2F0ID4gY29uZi9pZ19zcGVjLnlhbWwgPDwgJ19fRU9GX0lHX1NQRUMnCmt1YmVsZXQ6IG51bGwKbm9kZUxhYmVsczogbnVsbAp0YWludHM6IG51bGwKCl9fRU9GX0lHX1NQRUMKCmNhdCA+IGNvbmYva3ViZV9lbnYueWFtbCA8PCAnX19FT0ZfS1VCRV9FTlYnCkFzc2V0czoKLSBjM2I3MzZmZDBmMDAzNzY1YzEyZDk5ZjJjOTk1YTgzNjllNjI0MWY0QGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlbGV0Ci0gN2UzYTNlYTY2MzE1M2Y5MDBjYmQ1MjkwMGEzOWM5MWZhOWYzMzRiZUBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDUyZTlkMmRlOGE1ZjkyNzMwN2Q5Mzk3MzA4NzM1NjU4ZWU0NGFiOGRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLXBsdWdpbnMtYW1kNjQtdjAuNy41LnRnegpDbHVzdGVyTmFtZTogYWRkaXRpb25hbGNpZHIuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9hZGRpdGlvbmFsY2lkci5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWIKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2FkZGl0aW9uYWxjaWRyLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCmV0Y2RNYW5pZmVzdHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9hZGRpdGlvbmFsY2lkci5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9tYWluLnlhbWwKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2FkZGl0aW9uYWxjaWRyLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: additionalcidr.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: additionalcidr.example.com +ConfigBase: memfs://clusters.example.com/additionalcidr.example.com +InstanceGroupName: master-us-test-1b +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/additionalcidr.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/additionalcidr.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/additionalcidr.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1c.masters.additionalcidr.example.com_user_data b/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1c.masters.additionalcidr.example.com_user_data index b820af155a00d..dc1e58c1aeaf3 100644 --- a/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1c.masters.additionalcidr.example.com_user_data +++ b/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_master-us-test-1c.masters.additionalcidr.example.com_user_data @@ -1 +1,302 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMwogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBhZGRpdGlvbmFsY2lkci5leGFtcGxlLmNvbQogIGNvbmZpZ3VyZUNsb3VkUm91dGVzOiB0cnVlCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1jb250cm9sbGVyLW1hbmFnZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgogIHVzZVNlcnZpY2VBY2NvdW50Q3JlZGVudGlhbHM6IHRydWUKa3ViZVByb3h5OgogIGNsdXN0ZXJDSURSOiAxMDAuOTYuMC4wLzExCiAgY3B1UmVxdWVzdDogMTAwbQogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtcHJveHk6djEuMTQuMAogIGxvZ0xldmVsOiAyCmt1YmVTY2hlZHVsZXI6CiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1zY2hlZHVsZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgprdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5NVFU6IDkwMDEKICBuZXR3b3JrUGx1Z2luTmFtZToga3ViZW5ldAogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKbWFzdGVyS3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCiAgcmVnaXN0ZXJTY2hlZHVsYWJsZTogZmFsc2UKCl9fRU9GX0NMVVNURVJfU1BFQwoKY2F0ID4gY29uZi9pZ19zcGVjLnlhbWwgPDwgJ19fRU9GX0lHX1NQRUMnCmt1YmVsZXQ6IG51bGwKbm9kZUxhYmVsczogbnVsbAp0YWludHM6IG51bGwKCl9fRU9GX0lHX1NQRUMKCmNhdCA+IGNvbmYva3ViZV9lbnYueWFtbCA8PCAnX19FT0ZfS1VCRV9FTlYnCkFzc2V0czoKLSBjM2I3MzZmZDBmMDAzNzY1YzEyZDk5ZjJjOTk1YTgzNjllNjI0MWY0QGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlbGV0Ci0gN2UzYTNlYTY2MzE1M2Y5MDBjYmQ1MjkwMGEzOWM5MWZhOWYzMzRiZUBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDUyZTlkMmRlOGE1ZjkyNzMwN2Q5Mzk3MzA4NzM1NjU4ZWU0NGFiOGRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLXBsdWdpbnMtYW1kNjQtdjAuNy41LnRnegpDbHVzdGVyTmFtZTogYWRkaXRpb25hbGNpZHIuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9hZGRpdGlvbmFsY2lkci5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWMKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2FkZGl0aW9uYWxjaWRyLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCmV0Y2RNYW5pZmVzdHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9hZGRpdGlvbmFsY2lkci5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9tYWluLnlhbWwKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2FkZGl0aW9uYWxjaWRyLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: additionalcidr.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: additionalcidr.example.com +ConfigBase: memfs://clusters.example.com/additionalcidr.example.com +InstanceGroupName: master-us-test-1c +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/additionalcidr.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/additionalcidr.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/additionalcidr.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_nodes.additionalcidr.example.com_user_data b/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_nodes.additionalcidr.example.com_user_data index 423a546a0fe2a..1d60426e5b899 100644 --- a/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_nodes.additionalcidr.example.com_user_data +++ b/tests/integration/update_cluster/additional_cidr/data/aws_launch_template_nodes.additionalcidr.example.com_user_data @@ -1 +1,209 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IGFkZGl0aW9uYWxjaWRyLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vYWRkaXRpb25hbGNpZHIuZXhhbXBsZS5jb20KSW5zdGFuY2VHcm91cE5hbWU6IG5vZGVzClRhZ3M6Ci0gX2F1dG9tYXRpY191cGdyYWRlcwotIF9hd3MKY2hhbm5lbHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9hZGRpdGlvbmFsY2lkci5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbAoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg== +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: additionalcidr.example.com +ConfigBase: memfs://clusters.example.com/additionalcidr.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/additionalcidr.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/api_elb_cross_zone/data/aws_launch_template_master-us-test-1a.masters.crosszone.example.com_user_data b/tests/integration/update_cluster/api_elb_cross_zone/data/aws_launch_template_master-us-test-1a.masters.crosszone.example.com_user_data index ade6bea45420f..d9b4ffb84c39b 100644 --- a/tests/integration/update_cluster/api_elb_cross_zone/data/aws_launch_template_master-us-test-1a.masters.crosszone.example.com_user_data +++ b/tests/integration/update_cluster/api_elb_cross_zone/data/aws_launch_template_master-us-test-1a.masters.crosszone.example.com_user_data @@ -1 +1,303 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc2VydmljZU5vZGVQb3J0UmFuZ2U6IDI4MDAwLTMyNzY3CiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBjcm9zc3pvbmUuZXhhbXBsZS5jb20KICBjb25maWd1cmVDbG91ZFJvdXRlczogdHJ1ZQogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKICB1c2VTZXJ2aWNlQWNjb3VudENyZWRlbnRpYWxzOiB0cnVlCmt1YmVQcm94eToKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNwdVJlcXVlc3Q6IDEwMG0KICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXByb3h5OnYxLjE0LjAKICBsb2dMZXZlbDogMgprdWJlU2NoZWR1bGVyOgogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCm1hc3Rlckt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk1UVTogOTAwMQogIG5ldHdvcmtQbHVnaW5OYW1lOiBrdWJlbmV0CiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwogIHJlZ2lzdGVyU2NoZWR1bGFibGU6IGZhbHNlCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IGNyb3Nzem9uZS5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2Nyb3Nzem9uZS5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2Nyb3Nzem9uZS5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbApldGNkTWFuaWZlc3RzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vY3Jvc3N6b25lLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL21haW4ueWFtbAotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vY3Jvc3N6b25lLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + serviceNodePortRange: 28000-32767 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: crosszone.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: crosszone.example.com +ConfigBase: memfs://clusters.example.com/crosszone.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/crosszone.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/crosszone.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/crosszone.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/api_elb_cross_zone/data/aws_launch_template_nodes.crosszone.example.com_user_data b/tests/integration/update_cluster/api_elb_cross_zone/data/aws_launch_template_nodes.crosszone.example.com_user_data index 010cb4a75b457..ce65bc64fb875 100644 --- a/tests/integration/update_cluster/api_elb_cross_zone/data/aws_launch_template_nodes.crosszone.example.com_user_data +++ b/tests/integration/update_cluster/api_elb_cross_zone/data/aws_launch_template_nodes.crosszone.example.com_user_data @@ -1 +1,209 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IGNyb3Nzem9uZS5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2Nyb3Nzem9uZS5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbm9kZXMKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2Nyb3Nzem9uZS5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbAoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg== +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: crosszone.example.com +ConfigBase: memfs://clusters.example.com/crosszone.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/crosszone.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_bastion.bastionuserdata.example.com_user_data b/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_bastion.bastionuserdata.example.com_user_data index 77b9558bbf598..b79aaa959e65e 100644 --- a/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_bastion.bastionuserdata.example.com_user_data +++ b/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_bastion.bastionuserdata.example.com_user_data @@ -1 +1,13 @@ -Q29udGVudC1UeXBlOiBtdWx0aXBhcnQvbWl4ZWQ7IGJvdW5kYXJ5PSJNSU1FQk9VTkRBUlkiDQpNSU1FLVZlcnNpb246IDEuMA0KDQotLU1JTUVCT1VOREFSWQ0KQ29udGVudC1EaXNwb3NpdGlvbjogYXR0YWNobWVudDsgZmlsZW5hbWU9Im15c2NyaXB0LnNoIg0KQ29udGVudC1UcmFuc2Zlci1FbmNvZGluZzogN2JpdA0KQ29udGVudC1UeXBlOiB0ZXh0L3gtc2hlbGxzY3JpcHQNCk1pbWUtVmVyc2lvbjogMS4wDQoNCiMhL2Jpbi9zaAplY2hvICJIZWxsbyBXb3JsZCwgZnJvbSB0aGUgYmFzdGlvbiEgIFRoZSB0aW1lIGlzIG5vdyAkKGRhdGUgLVIpISIgfCB0ZWUgL3Jvb3Qvb3V0cHV0LnR4dAoNCi0tTUlNRUJPVU5EQVJZLS0NCg== +Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Disposition: attachment; filename="myscript.sh" +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript +Mime-Version: 1.0 + +#!/bin/sh +echo "Hello World, from the bastion! The time is now $(date -R)!" | tee /root/output.txt + +--MIMEBOUNDARY-- diff --git a/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_master-us-test-1a.masters.bastionuserdata.example.com_user_data b/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_master-us-test-1a.masters.bastionuserdata.example.com_user_data index b8d724fc20c3f..0864fbaebbdad 100644 --- a/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_master-us-test-1a.masters.bastionuserdata.example.com_user_data +++ b/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_master-us-test-1a.masters.bastionuserdata.example.com_user_data @@ -1 +1,300 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBiYXN0aW9udXNlcmRhdGEuZXhhbXBsZS5jb20KICBjb25maWd1cmVDbG91ZFJvdXRlczogZmFsc2UKICBpbWFnZTogazhzLmdjci5pby9rdWJlLWNvbnRyb2xsZXItbWFuYWdlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCiAgdXNlU2VydmljZUFjY291bnRDcmVkZW50aWFsczogdHJ1ZQprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZVNjaGVkdWxlcjoKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXNjaGVkdWxlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCmt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk5hbWU6IGNuaQogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKbWFzdGVyS3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwogIHJlZ2lzdGVyU2NoZWR1bGFibGU6IGZhbHNlCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IGJhc3Rpb251c2VyZGF0YS5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2Jhc3Rpb251c2VyZGF0YS5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2Jhc3Rpb251c2VyZGF0YS5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbApldGNkTWFuaWZlc3RzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vYmFzdGlvbnVzZXJkYXRhLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL21haW4ueWFtbAotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vYmFzdGlvbnVzZXJkYXRhLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: bastionuserdata.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: bastionuserdata.example.com +ConfigBase: memfs://clusters.example.com/bastionuserdata.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/bastionuserdata.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/bastionuserdata.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/bastionuserdata.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_nodes.bastionuserdata.example.com_user_data b/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_nodes.bastionuserdata.example.com_user_data index 6f1ec4bb01355..e1fa5b670e0a3 100644 --- a/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_nodes.bastionuserdata.example.com_user_data +++ b/tests/integration/update_cluster/bastionadditional_user-data/data/aws_launch_template_nodes.bastionuserdata.example.com_user_data @@ -1 +1,228 @@ -Q29udGVudC1UeXBlOiBtdWx0aXBhcnQvbWl4ZWQ7IGJvdW5kYXJ5PSJNSU1FQk9VTkRBUlkiDQpNSU1FLVZlcnNpb246IDEuMA0KDQotLU1JTUVCT1VOREFSWQ0KQ29udGVudC1EaXNwb3NpdGlvbjogYXR0YWNobWVudDsgZmlsZW5hbWU9Im5vZGV1cC5zaCINCkNvbnRlbnQtVHJhbnNmZXItRW5jb2Rpbmc6IDdiaXQNCkNvbnRlbnQtVHlwZTogdGV4dC94LXNoZWxsc2NyaXB0DQpNaW1lLVZlcnNpb246IDEuMA0KDQojIS9iaW4vYmFzaAojIENvcHlyaWdodCAyMDE2IFRoZSBLdWJlcm5ldGVzIEF1dGhvcnMgQWxsIHJpZ2h0cyByZXNlcnZlZC4KIwojIExpY2Vuc2VkIHVuZGVyIHRoZSBBcGFjaGUgTGljZW5zZSwgVmVyc2lvbiAyLjAgKHRoZSAiTGljZW5zZSIpOwojIHlvdSBtYXkgbm90IHVzZSB0aGlzIGZpbGUgZXhjZXB0IGluIGNvbXBsaWFuY2Ugd2l0aCB0aGUgTGljZW5zZS4KIyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKIwojICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKIwojIFVubGVzcyByZXF1aXJlZCBieSBhcHBsaWNhYmxlIGxhdyBvciBhZ3JlZWQgdG8gaW4gd3JpdGluZywgc29mdHdhcmUKIyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAojIFdJVEhPVVQgV0FSUkFOVElFUyBPUiBDT05ESVRJT05TIE9GIEFOWSBLSU5ELCBlaXRoZXIgZXhwcmVzcyBvciBpbXBsaWVkLgojIFNlZSB0aGUgTGljZW5zZSBmb3IgdGhlIHNwZWNpZmljIGxhbmd1YWdlIGdvdmVybmluZyBwZXJtaXNzaW9ucyBhbmQKIyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCnNldCAtbyBlcnJleGl0CnNldCAtbyBub3Vuc2V0CnNldCAtbyBwaXBlZmFpbAoKTk9ERVVQX1VSTD1odHRwczovL2FydGlmYWN0cy5rOHMuaW8vYmluYXJpZXMva29wcy8xLjE1LjAvbGludXgvYW1kNjQvbm9kZXVwLGh0dHBzOi8vZ2l0aHViLmNvbS9rdWJlcm5ldGVzL2tvcHMvcmVsZWFzZXMvZG93bmxvYWQvdjEuMTUuMC9saW51eC1hbWQ2NC1ub2RldXAsaHR0cHM6Ly9rdWJldXB2Mi5zMy5hbWF6b25hd3MuY29tL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cApOT0RFVVBfSEFTSD05NjA0ZWYxODI2N2FkN2I1Y2Y0Y2ViYmY3YWI2NDQyM2NmNWJiMDM0MmQxNjljNjA4YWM2Mzc2ZTZhZjI2ZDgxCgpleHBvcnQgQVdTX1JFR0lPTj11cy10ZXN0LTEKCgoKCmZ1bmN0aW9uIGVuc3VyZS1pbnN0YWxsLWRpcigpIHsKICBJTlNUQUxMX0RJUj0iL29wdC9rb3BzIgogICMgT24gQ29udGFpbmVyT1MsIHdlIGluc3RhbGwgdW5kZXIgL3Zhci9saWIvdG9vbGJveDsgL29wdCBpcyBybyBhbmQgbm9leGVjCiAgaWYgW1sgLWQgL3Zhci9saWIvdG9vbGJveCBdXTsgdGhlbgogICAgSU5TVEFMTF9ESVI9Ii92YXIvbGliL3Rvb2xib3gva29wcyIKICBmaQogIG1rZGlyIC1wICR7SU5TVEFMTF9ESVJ9L2JpbgogIG1rZGlyIC1wICR7SU5TVEFMTF9ESVJ9L2NvbmYKICBjZCAke0lOU1RBTExfRElSfQp9CgojIFJldHJ5IGEgZG93bmxvYWQgdW50aWwgd2UgZ2V0IGl0LiBhcmdzOiBuYW1lLCBzaGEsIHVybDEsIHVybDIuLi4KZG93bmxvYWQtb3ItYnVzdCgpIHsKICBsb2NhbCAtciBmaWxlPSIkMSIKICBsb2NhbCAtciBoYXNoPSIkMiIKICBzaGlmdCAyCgogIHVybHM9KCAkKiApCiAgd2hpbGUgdHJ1ZTsgZG8KICAgIGZvciB1cmwgaW4gIiR7dXJsc1tAXX0iOyBkbwogICAgICBjb21tYW5kcz0oCiAgICAgICAgImN1cmwgLWYgLS1pcHY0IC0tY29tcHJlc3NlZCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtLWNvbXByZXNzaW9uPWF1dG8gLU8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0PTIwIC0tdHJpZXM9NiAtLXdhaXQ9MTAiCiAgICAgICAgImN1cmwgLWYgLS1pcHY0IC1MbyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQgMjAgLS1yZXRyeSA2IC0tcmV0cnktZGVsYXkgMTAiCiAgICAgICAgIndnZXQgLS1pbmV0NC1vbmx5IC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICApCiAgICAgIGZvciBjbWQgaW4gIiR7Y29tbWFuZHNbQF19IjsgZG8KICAgICAgICBlY2hvICJBdHRlbXB0aW5nIGRvd25sb2FkIHdpdGg6ICR7Y21kfSB7dXJsfSIKICAgICAgICBpZiAhICgke2NtZH0gIiR7dXJsfSIpOyB0aGVuCiAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZCBmYWlsZWQgd2l0aCAke2NtZH0gPT0iCiAgICAgICAgICBjb250aW51ZQogICAgICAgIGZpCiAgICAgICAgaWYgW1sgLW4gIiR7aGFzaH0iIF1dICYmICEgdmFsaWRhdGUtaGFzaCAiJHtmaWxlfSIgIiR7aGFzaH0iOyB0aGVuCiAgICAgICAgICBlY2hvICI9PSBIYXNoIHZhbGlkYXRpb24gb2YgJHt1cmx9IGZhaWxlZC4gUmV0cnlpbmcuID09IgogICAgICAgICAgcm0gLWYgIiR7ZmlsZX0iCiAgICAgICAgZWxzZQogICAgICAgICAgaWYgW1sgLW4gIiR7aGFzaH0iIF1dOyB0aGVuCiAgICAgICAgICAgIGVjaG8gIj09IERvd25sb2FkZWQgJHt1cmx9IChTSEExID0gJHtoYXNofSkgPT0iCiAgICAgICAgICBlbHNlCiAgICAgICAgICAgIGVjaG8gIj09IERvd25sb2FkZWQgJHt1cmx9ID09IgogICAgICAgICAgZmkKICAgICAgICAgIHJldHVybgogICAgICAgIGZpCiAgICAgIGRvbmUKICAgIGRvbmUKCiAgICBlY2hvICJBbGwgZG93bmxvYWRzIGZhaWxlZDsgc2xlZXBpbmcgYmVmb3JlIHJldHJ5aW5nIgogICAgc2xlZXAgNjAKICBkb25lCn0KCnZhbGlkYXRlLWhhc2goKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgZXhwZWN0ZWQ9IiQyIgogIGxvY2FsIGFjdHVhbAoKICBhY3R1YWw9JChzaGEyNTZzdW0gJHtmaWxlfSB8IGF3ayAneyBwcmludCAkMSB9JykgfHwgdHJ1ZQogIGlmIFtbICIke2FjdHVhbH0iICE9ICIke2V4cGVjdGVkfSIgXV07IHRoZW4KICAgIGVjaG8gIj09ICR7ZmlsZX0gY29ycnVwdGVkLCBoYXNoICR7YWN0dWFsfSBkb2Vzbid0IG1hdGNoIGV4cGVjdGVkICR7ZXhwZWN0ZWR9ID09IgogICAgcmV0dXJuIDEKICBmaQp9CgpmdW5jdGlvbiBzcGxpdC1jb21tYXMoKSB7CiAgZWNobyAkMSB8IHRyICIsIiAiXG4iCn0KCmZ1bmN0aW9uIHRyeS1kb3dubG9hZC1yZWxlYXNlKCkgewogIGxvY2FsIC1yIG5vZGV1cF91cmxzPSggJChzcGxpdC1jb21tYXMgIiR7Tk9ERVVQX1VSTH0iKSApCiAgaWYgW1sgLW4gIiR7Tk9ERVVQX0hBU0g6LX0iIF1dOyB0aGVuCiAgICBsb2NhbCAtciBub2RldXBfaGFzaD0iJHtOT0RFVVBfSEFTSH0iCiAgZWxzZQogICMgVE9ETzogUmVtb3ZlPwogICAgZWNobyAiRG93bmxvYWRpbmcgc2hhMjU2IChub3QgZm91bmQgaW4gZW52KSIKICAgIGRvd25sb2FkLW9yLWJ1c3Qgbm9kZXVwLnNoYTI1NiAiIiAiJHtub2RldXBfdXJsc1tAXS8lLy5zaGEyNTZ9IgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9JChjYXQgbm9kZXVwLnNoYTI1NikKICBmaQoKICBlY2hvICJEb3dubG9hZGluZyBub2RldXAgKCR7bm9kZXVwX3VybHNbQF19KSIKICBkb3dubG9hZC1vci1idXN0IG5vZGV1cCAiJHtub2RldXBfaGFzaH0iICIke25vZGV1cF91cmxzW0BdfSIKCiAgY2htb2QgK3ggbm9kZXVwCn0KCmZ1bmN0aW9uIGRvd25sb2FkLXJlbGVhc2UoKSB7CiAgY2FzZSAiJCh1bmFtZSAtbSkiIGluCiAgeDg2XzY0KnxpPzg2XzY0KnxhbWQ2NCopCiAgICBOT0RFVVBfVVJMPSIke05PREVVUF9VUkx9IgogICAgTk9ERVVQX0hBU0g9IiR7Tk9ERVVQX0hBU0h9IgogICAgOzsKICAqKQogICAgZWNobyAiVW5zdXBwb3J0ZWQgaG9zdCBhcmNoOiAkKHVuYW1lIC1tKSIgPiYyCiAgICBleGl0IDEKICAgIDs7CiAgZXNhYwoKICAjIEluIGNhc2Ugb2YgZmFpbHVyZSBjaGVja2luZyBpbnRlZ3JpdHkgb2YgcmVsZWFzZSwgcmV0cnkuCiAgY2QgJHtJTlNUQUxMX0RJUn0vYmluCiAgdW50aWwgdHJ5LWRvd25sb2FkLXJlbGVhc2U7IGRvCiAgICBzbGVlcCAxNQogICAgZWNobyAiQ291bGRuJ3QgZG93bmxvYWQgcmVsZWFzZS4gUmV0cnlpbmcuLi4iCiAgZG9uZQoKICBlY2hvICJSdW5uaW5nIG5vZGV1cCIKICAjIFdlIGNhbid0IHJ1biBpbiB0aGUgZm9yZWdyb3VuZCBiZWNhdXNlIG9mIGh0dHBzOi8vZ2l0aHViLmNvbS9kb2NrZXIvZG9ja2VyL2lzc3Vlcy8yMzc5MwogICggY2QgJHtJTlNUQUxMX0RJUn0vYmluOyAuL25vZGV1cCAtLWluc3RhbGwtc3lzdGVtZC11bml0IC0tY29uZj0ke0lOU1RBTExfRElSfS9jb25mL2t1YmVfZW52LnlhbWwgLS12PTggICkKfQoKIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjCgovYmluL3N5c3RlbWQtbWFjaGluZS1pZC1zZXR1cCB8fCBlY2hvICJmYWlsZWQgdG8gc2V0IHVwIGVuc3VyZSBtYWNoaW5lLWlkIGNvbmZpZ3VyZWQiCgplY2hvICI9PSBub2RldXAgbm9kZSBjb25maWcgc3RhcnRpbmcgPT0iCmVuc3VyZS1pbnN0YWxsLWRpcgoKY2F0ID4gY29uZi9jbHVzdGVyX3NwZWMueWFtbCA8PCAnX19FT0ZfQ0xVU1RFUl9TUEVDJwpjbG91ZENvbmZpZzogbnVsbApjb250YWluZXJSdW50aW1lOiBkb2NrZXIKY29udGFpbmVyZDoKICBza2lwSW5zdGFsbDogdHJ1ZQpkb2NrZXI6CiAgaXBNYXNxOiBmYWxzZQogIGlwVGFibGVzOiBmYWxzZQogIGxvZ0RyaXZlcjoganNvbi1maWxlCiAgbG9nTGV2ZWw6IGluZm8KICBsb2dPcHQ6CiAgLSBtYXgtc2l6ZT0xMG0KICAtIG1heC1maWxlPTUKICBzdG9yYWdlOiBvdmVybGF5MixvdmVybGF5LGF1ZnMKICB2ZXJzaW9uOiAxOC4wNi4zCmt1YmVQcm94eToKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNwdVJlcXVlc3Q6IDEwMG0KICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXByb3h5OnYxLjE0LjAKICBsb2dMZXZlbDogMgprdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5OYW1lOiBjbmkKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IGJhc3Rpb251c2VyZGF0YS5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2Jhc3Rpb251c2VyZGF0YS5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbm9kZXMKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2Jhc3Rpb251c2VyZGF0YS5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbAoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg0KLS1NSU1FQk9VTkRBUlkNCkNvbnRlbnQtRGlzcG9zaXRpb246IGF0dGFjaG1lbnQ7IGZpbGVuYW1lPSJteXNjcmlwdC5zaCINCkNvbnRlbnQtVHJhbnNmZXItRW5jb2Rpbmc6IDdiaXQNCkNvbnRlbnQtVHlwZTogdGV4dC94LXNoZWxsc2NyaXB0DQpNaW1lLVZlcnNpb246IDEuMA0KDQojIS9iaW4vc2gKZWNobyAiSGVsbG8gV29ybGQsIGZyb20gYSBub2RlISAgVGhlIHRpbWUgaXMgbm93ICQoZGF0ZSAtUikhIiB8IHRlZSAvcm9vdC9vdXRwdXQudHh0Cg0KLS1NSU1FQk9VTkRBUlktLQ0K +Content-Type: multipart/mixed; boundary="MIMEBOUNDARY" +MIME-Version: 1.0 + +--MIMEBOUNDARY +Content-Disposition: attachment; filename="nodeup.sh" +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript +Mime-Version: 1.0 + +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: bastionuserdata.example.com +ConfigBase: memfs://clusters.example.com/bastionuserdata.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/bastionuserdata.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" + +--MIMEBOUNDARY +Content-Disposition: attachment; filename="myscript.sh" +Content-Transfer-Encoding: 7bit +Content-Type: text/x-shellscript +Mime-Version: 1.0 + +#!/bin/sh +echo "Hello World, from a node! The time is now $(date -R)!" | tee /root/output.txt + +--MIMEBOUNDARY-- diff --git a/tests/integration/update_cluster/complex/data/aws_launch_template_master-us-test-1a.masters.complex.example.com_user_data b/tests/integration/update_cluster/complex/data/aws_launch_template_master-us-test-1a.masters.complex.example.com_user_data index 23e271de9f6e3..a725ffc53dc73 100644 --- a/tests/integration/update_cluster/complex/data/aws_launch_template_master-us-test-1a.masters.complex.example.com_user_data +++ b/tests/integration/update_cluster/complex/data/aws_launch_template_master-us-test-1a.masters.complex.example.com_user_data @@ -1 +1,304 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1ZGl0V2ViaG9va0JhdGNoVGhyb3R0bGVRcHM6IDMxNDBtCiAgYXV0aG9yaXphdGlvbk1vZGU6IEFsd2F5c0FsbG93CiAgYmluZEFkZHJlc3M6IDAuMC4wLjAKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBlbmFibGVBZG1pc3Npb25QbHVnaW5zOgogIC0gTmFtZXNwYWNlTGlmZWN5Y2xlCiAgLSBMaW1pdFJhbmdlcgogIC0gU2VydmljZUFjY291bnQKICAtIFBlcnNpc3RlbnRWb2x1bWVMYWJlbAogIC0gRGVmYXVsdFN0b3JhZ2VDbGFzcwogIC0gRGVmYXVsdFRvbGVyYXRpb25TZWNvbmRzCiAgLSBNdXRhdGluZ0FkbWlzc2lvbldlYmhvb2sKICAtIFZhbGlkYXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBOb2RlUmVzdHJpY3Rpb24KICAtIFJlc291cmNlUXVvdGEKICBldGNkU2VydmVyczoKICAtIGh0dHA6Ly8xMjcuMC4wLjE6NDAwMQogIGV0Y2RTZXJ2ZXJzT3ZlcnJpZGVzOgogIC0gL2V2ZW50cyNodHRwOi8vMTI3LjAuMC4xOjQwMDIKICBpbWFnZTogazhzLmdjci5pby9rdWJlLWFwaXNlcnZlcjp2MS4xNC4wCiAgaW5zZWN1cmVCaW5kQWRkcmVzczogMTI3LjAuMC4xCiAgaW5zZWN1cmVQb3J0OiA4MDgwCiAga3ViZWxldFByZWZlcnJlZEFkZHJlc3NUeXBlczoKICAtIEludGVybmFsSVAKICAtIEhvc3RuYW1lCiAgLSBFeHRlcm5hbElQCiAgbG9nTGV2ZWw6IDIKICByZXF1ZXN0aGVhZGVyQWxsb3dlZE5hbWVzOgogIC0gYWdncmVnYXRvcgogIHJlcXVlc3RoZWFkZXJFeHRyYUhlYWRlclByZWZpeGVzOgogIC0gWC1SZW1vdGUtRXh0cmEtCiAgcmVxdWVzdGhlYWRlckdyb3VwSGVhZGVyczoKICAtIFgtUmVtb3RlLUdyb3VwCiAgcmVxdWVzdGhlYWRlclVzZXJuYW1lSGVhZGVyczoKICAtIFgtUmVtb3RlLVVzZXIKICBzZWN1cmVQb3J0OiA0NDMKICBzZXJ2aWNlQ2x1c3RlcklQUmFuZ2U6IDEwMC42NC4wLjAvMTMKICBzZXJ2aWNlTm9kZVBvcnRSYW5nZTogMjgwMDAtMzI3NjcKICBzdG9yYWdlQmFja2VuZDogZXRjZDMKa3ViZUNvbnRyb2xsZXJNYW5hZ2VyOgogIGFsbG9jYXRlTm9kZUNJRFJzOiB0cnVlCiAgYXR0YWNoRGV0YWNoUmVjb25jaWxlU3luY1BlcmlvZDogMW0wcwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJDSURSOiAxMDAuOTYuMC4wLzExCiAgY2x1c3Rlck5hbWU6IGNvbXBsZXguZXhhbXBsZS5jb20KICBjb25maWd1cmVDbG91ZFJvdXRlczogdHJ1ZQogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKICB1c2VTZXJ2aWNlQWNjb3VudENyZWRlbnRpYWxzOiB0cnVlCmt1YmVQcm94eToKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNwdVJlcXVlc3Q6IDEwMG0KICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXByb3h5OnYxLjE0LjAKICBsb2dMZXZlbDogMgprdWJlU2NoZWR1bGVyOgogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCm1hc3Rlckt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk1UVTogOTAwMQogIG5ldHdvcmtQbHVnaW5OYW1lOiBrdWJlbmV0CiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwogIHJlZ2lzdGVyU2NoZWR1bGFibGU6IGZhbHNlCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IGNvbXBsZXguZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9jb21wbGV4LmV4YW1wbGUuY29tCkluc3RhbmNlR3JvdXBOYW1lOiBtYXN0ZXItdXMtdGVzdC0xYQpUYWdzOgotIF9hdXRvbWF0aWNfdXBncmFkZXMKLSBfYXdzCmNoYW5uZWxzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vY29tcGxleC5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbApldGNkTWFuaWZlc3RzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vY29tcGxleC5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9tYWluLnlhbWwKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2NvbXBsZXguZXhhbXBsZS5jb20vbWFuaWZlc3RzL2V0Y2QvZXZlbnRzLnlhbWwKcHJvdG9rdWJlSW1hZ2U6CiAgaGFzaDogNDJhOWM0MzI0ZmUyNmQ2M2NlMTFmM2RkNzgzNjM3MWJjOTNmYTA2Y2E4ZjQ3OTgwNzcyOGYzNzQ2ZTI3MDYxYgogIG5hbWU6IHByb3Rva3ViZToxLjE1LjAKICBzb3VyY2VzOgogIC0gaHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2ltYWdlcy9wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvaW1hZ2VzLXByb3Rva3ViZS50YXIuZ3oKICAtIGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg== +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + auditWebhookBatchThrottleQps: 3140m + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + serviceNodePortRange: 28000-32767 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: complex.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: complex.example.com +ConfigBase: memfs://clusters.example.com/complex.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/complex.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/complex.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/complex.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/complex/data/aws_launch_template_nodes.complex.example.com_user_data b/tests/integration/update_cluster/complex/data/aws_launch_template_nodes.complex.example.com_user_data index a08576319c737..b47fecfcb89a8 100644 --- a/tests/integration/update_cluster/complex/data/aws_launch_template_nodes.complex.example.com_user_data +++ b/tests/integration/update_cluster/complex/data/aws_launch_template_nodes.complex.example.com_user_data @@ -1 +1,209 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IGNvbXBsZXguZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9jb21wbGV4LmV4YW1wbGUuY29tCkluc3RhbmNlR3JvdXBOYW1lOiBub2RlcwpUYWdzOgotIF9hdXRvbWF0aWNfdXBncmFkZXMKLSBfYXdzCmNoYW5uZWxzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vY29tcGxleC5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbAoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg== +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: complex.example.com +ConfigBase: memfs://clusters.example.com/complex.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/complex.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1a.masters.existing-iam.example.com_user_data b/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1a.masters.existing-iam.example.com_user_data index c4c2bb41ee51d..353f360aeaadc 100644 --- a/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1a.masters.existing-iam.example.com_user_data +++ b/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1a.masters.existing-iam.example.com_user_data @@ -1 +1,302 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMwogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBleGlzdGluZy1pYW0uZXhhbXBsZS5jb20KICBjb25maWd1cmVDbG91ZFJvdXRlczogdHJ1ZQogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKICB1c2VTZXJ2aWNlQWNjb3VudENyZWRlbnRpYWxzOiB0cnVlCmt1YmVQcm94eToKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNwdVJlcXVlc3Q6IDEwMG0KICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXByb3h5OnYxLjE0LjAKICBsb2dMZXZlbDogMgprdWJlU2NoZWR1bGVyOgogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCm1hc3Rlckt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk1UVTogOTAwMQogIG5ldHdvcmtQbHVnaW5OYW1lOiBrdWJlbmV0CiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwogIHJlZ2lzdGVyU2NoZWR1bGFibGU6IGZhbHNlCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IGV4aXN0aW5nLWlhbS5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL3Rlc3RzL2V4aXN0aW5nLWlhbS5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL3Rlc3RzL2V4aXN0aW5nLWlhbS5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbApldGNkTWFuaWZlc3RzOgotIG1lbWZzOi8vdGVzdHMvZXhpc3RpbmctaWFtLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL21haW4ueWFtbAotIG1lbWZzOi8vdGVzdHMvZXhpc3RpbmctaWFtLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: existing-iam.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: existing-iam.example.com +ConfigBase: memfs://tests/existing-iam.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://tests/existing-iam.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://tests/existing-iam.example.com/manifests/etcd/main.yaml +- memfs://tests/existing-iam.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1b.masters.existing-iam.example.com_user_data b/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1b.masters.existing-iam.example.com_user_data index 04e65a5e0158b..7c0e551199aff 100644 --- a/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1b.masters.existing-iam.example.com_user_data +++ b/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1b.masters.existing-iam.example.com_user_data @@ -1 +1,302 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMwogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBleGlzdGluZy1pYW0uZXhhbXBsZS5jb20KICBjb25maWd1cmVDbG91ZFJvdXRlczogdHJ1ZQogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKICB1c2VTZXJ2aWNlQWNjb3VudENyZWRlbnRpYWxzOiB0cnVlCmt1YmVQcm94eToKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNwdVJlcXVlc3Q6IDEwMG0KICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXByb3h5OnYxLjE0LjAKICBsb2dMZXZlbDogMgprdWJlU2NoZWR1bGVyOgogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCm1hc3Rlckt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk1UVTogOTAwMQogIG5ldHdvcmtQbHVnaW5OYW1lOiBrdWJlbmV0CiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwogIHJlZ2lzdGVyU2NoZWR1bGFibGU6IGZhbHNlCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IGV4aXN0aW5nLWlhbS5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL3Rlc3RzL2V4aXN0aW5nLWlhbS5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWIKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL3Rlc3RzL2V4aXN0aW5nLWlhbS5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbApldGNkTWFuaWZlc3RzOgotIG1lbWZzOi8vdGVzdHMvZXhpc3RpbmctaWFtLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL21haW4ueWFtbAotIG1lbWZzOi8vdGVzdHMvZXhpc3RpbmctaWFtLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: existing-iam.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: existing-iam.example.com +ConfigBase: memfs://tests/existing-iam.example.com +InstanceGroupName: master-us-test-1b +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://tests/existing-iam.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://tests/existing-iam.example.com/manifests/etcd/main.yaml +- memfs://tests/existing-iam.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1c.masters.existing-iam.example.com_user_data b/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1c.masters.existing-iam.example.com_user_data index d0808be04e240..f1fe45a6f1e08 100644 --- a/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1c.masters.existing-iam.example.com_user_data +++ b/tests/integration/update_cluster/existing_iam/data/aws_launch_template_master-us-test-1c.masters.existing-iam.example.com_user_data @@ -1 +1,302 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMwogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBleGlzdGluZy1pYW0uZXhhbXBsZS5jb20KICBjb25maWd1cmVDbG91ZFJvdXRlczogdHJ1ZQogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKICB1c2VTZXJ2aWNlQWNjb3VudENyZWRlbnRpYWxzOiB0cnVlCmt1YmVQcm94eToKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNwdVJlcXVlc3Q6IDEwMG0KICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXByb3h5OnYxLjE0LjAKICBsb2dMZXZlbDogMgprdWJlU2NoZWR1bGVyOgogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCm1hc3Rlckt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk1UVTogOTAwMQogIG5ldHdvcmtQbHVnaW5OYW1lOiBrdWJlbmV0CiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwogIHJlZ2lzdGVyU2NoZWR1bGFibGU6IGZhbHNlCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IGV4aXN0aW5nLWlhbS5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL3Rlc3RzL2V4aXN0aW5nLWlhbS5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWMKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL3Rlc3RzL2V4aXN0aW5nLWlhbS5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbApldGNkTWFuaWZlc3RzOgotIG1lbWZzOi8vdGVzdHMvZXhpc3RpbmctaWFtLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL21haW4ueWFtbAotIG1lbWZzOi8vdGVzdHMvZXhpc3RpbmctaWFtLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: existing-iam.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: existing-iam.example.com +ConfigBase: memfs://tests/existing-iam.example.com +InstanceGroupName: master-us-test-1c +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://tests/existing-iam.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://tests/existing-iam.example.com/manifests/etcd/main.yaml +- memfs://tests/existing-iam.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/existing_iam/data/aws_launch_template_nodes.existing-iam.example.com_user_data b/tests/integration/update_cluster/existing_iam/data/aws_launch_template_nodes.existing-iam.example.com_user_data index aa6a4dbeaf30d..ce22513bc4536 100644 --- a/tests/integration/update_cluster/existing_iam/data/aws_launch_template_nodes.existing-iam.example.com_user_data +++ b/tests/integration/update_cluster/existing_iam/data/aws_launch_template_nodes.existing-iam.example.com_user_data @@ -1 +1,209 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IGV4aXN0aW5nLWlhbS5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL3Rlc3RzL2V4aXN0aW5nLWlhbS5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbm9kZXMKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL3Rlc3RzL2V4aXN0aW5nLWlhbS5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbAoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg== +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: existing-iam.example.com +ConfigBase: memfs://tests/existing-iam.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://tests/existing-iam.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1a.masters.existingsg.example.com_user_data b/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1a.masters.existingsg.example.com_user_data index bbbcb6eb7f2b0..1957ab1ea65b4 100644 --- a/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1a.masters.existingsg.example.com_user_data +++ b/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1a.masters.existingsg.example.com_user_data @@ -1 +1,302 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMwogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBleGlzdGluZ3NnLmV4YW1wbGUuY29tCiAgY29uZmlndXJlQ2xvdWRSb3V0ZXM6IHRydWUKICBpbWFnZTogazhzLmdjci5pby9rdWJlLWNvbnRyb2xsZXItbWFuYWdlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCiAgdXNlU2VydmljZUFjY291bnRDcmVkZW50aWFsczogdHJ1ZQprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZVNjaGVkdWxlcjoKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXNjaGVkdWxlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCmt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk1UVTogOTAwMQogIG5ldHdvcmtQbHVnaW5OYW1lOiBrdWJlbmV0CiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwptYXN0ZXJLdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5NVFU6IDkwMDEKICBuZXR3b3JrUGx1Z2luTmFtZToga3ViZW5ldAogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKICByZWdpc3RlclNjaGVkdWxhYmxlOiBmYWxzZQoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGMzYjczNmZkMGYwMDM3NjVjMTJkOTlmMmM5OTVhODM2OWU2MjQxZjRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA3ZTNhM2VhNjYzMTUzZjkwMGNiZDUyOTAwYTM5YzkxZmE5ZjMzNGJlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlY3RsCi0gNTJlOWQyZGU4YTVmOTI3MzA3ZDkzOTczMDg3MzU2NThlZTQ0YWI4ZEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL25ldHdvcmstcGx1Z2lucy9jbmktcGx1Z2lucy1hbWQ2NC12MC43LjUudGd6CkNsdXN0ZXJOYW1lOiBleGlzdGluZ3NnLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vZXhpc3RpbmdzZy5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2V4aXN0aW5nc2cuZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKZXRjZE1hbmlmZXN0czoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2V4aXN0aW5nc2cuZXhhbXBsZS5jb20vbWFuaWZlc3RzL2V0Y2QvbWFpbi55YW1sCi0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9leGlzdGluZ3NnLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: existingsg.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: existingsg.example.com +ConfigBase: memfs://clusters.example.com/existingsg.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/existingsg.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/existingsg.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/existingsg.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1b.masters.existingsg.example.com_user_data b/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1b.masters.existingsg.example.com_user_data index d1522d0c81eca..f7338f84198ef 100644 --- a/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1b.masters.existingsg.example.com_user_data +++ b/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1b.masters.existingsg.example.com_user_data @@ -1 +1,302 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMwogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBleGlzdGluZ3NnLmV4YW1wbGUuY29tCiAgY29uZmlndXJlQ2xvdWRSb3V0ZXM6IHRydWUKICBpbWFnZTogazhzLmdjci5pby9rdWJlLWNvbnRyb2xsZXItbWFuYWdlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCiAgdXNlU2VydmljZUFjY291bnRDcmVkZW50aWFsczogdHJ1ZQprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZVNjaGVkdWxlcjoKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXNjaGVkdWxlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCmt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk1UVTogOTAwMQogIG5ldHdvcmtQbHVnaW5OYW1lOiBrdWJlbmV0CiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwptYXN0ZXJLdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5NVFU6IDkwMDEKICBuZXR3b3JrUGx1Z2luTmFtZToga3ViZW5ldAogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKICByZWdpc3RlclNjaGVkdWxhYmxlOiBmYWxzZQoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGMzYjczNmZkMGYwMDM3NjVjMTJkOTlmMmM5OTVhODM2OWU2MjQxZjRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA3ZTNhM2VhNjYzMTUzZjkwMGNiZDUyOTAwYTM5YzkxZmE5ZjMzNGJlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlY3RsCi0gNTJlOWQyZGU4YTVmOTI3MzA3ZDkzOTczMDg3MzU2NThlZTQ0YWI4ZEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL25ldHdvcmstcGx1Z2lucy9jbmktcGx1Z2lucy1hbWQ2NC12MC43LjUudGd6CkNsdXN0ZXJOYW1lOiBleGlzdGluZ3NnLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vZXhpc3RpbmdzZy5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWIKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2V4aXN0aW5nc2cuZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKZXRjZE1hbmlmZXN0czoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2V4aXN0aW5nc2cuZXhhbXBsZS5jb20vbWFuaWZlc3RzL2V0Y2QvbWFpbi55YW1sCi0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9leGlzdGluZ3NnLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: existingsg.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: existingsg.example.com +ConfigBase: memfs://clusters.example.com/existingsg.example.com +InstanceGroupName: master-us-test-1b +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/existingsg.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/existingsg.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/existingsg.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1c.masters.existingsg.example.com_user_data b/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1c.masters.existingsg.example.com_user_data index cf5b77bc29ef1..2d3ec8e922342 100644 --- a/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1c.masters.existingsg.example.com_user_data +++ b/tests/integration/update_cluster/existing_sg/data/aws_launch_template_master-us-test-1c.masters.existingsg.example.com_user_data @@ -1 +1,302 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMwogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBleGlzdGluZ3NnLmV4YW1wbGUuY29tCiAgY29uZmlndXJlQ2xvdWRSb3V0ZXM6IHRydWUKICBpbWFnZTogazhzLmdjci5pby9rdWJlLWNvbnRyb2xsZXItbWFuYWdlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCiAgdXNlU2VydmljZUFjY291bnRDcmVkZW50aWFsczogdHJ1ZQprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZVNjaGVkdWxlcjoKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXNjaGVkdWxlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCmt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk1UVTogOTAwMQogIG5ldHdvcmtQbHVnaW5OYW1lOiBrdWJlbmV0CiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwptYXN0ZXJLdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5NVFU6IDkwMDEKICBuZXR3b3JrUGx1Z2luTmFtZToga3ViZW5ldAogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKICByZWdpc3RlclNjaGVkdWxhYmxlOiBmYWxzZQoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGMzYjczNmZkMGYwMDM3NjVjMTJkOTlmMmM5OTVhODM2OWU2MjQxZjRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA3ZTNhM2VhNjYzMTUzZjkwMGNiZDUyOTAwYTM5YzkxZmE5ZjMzNGJlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlY3RsCi0gNTJlOWQyZGU4YTVmOTI3MzA3ZDkzOTczMDg3MzU2NThlZTQ0YWI4ZEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL25ldHdvcmstcGx1Z2lucy9jbmktcGx1Z2lucy1hbWQ2NC12MC43LjUudGd6CkNsdXN0ZXJOYW1lOiBleGlzdGluZ3NnLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vZXhpc3RpbmdzZy5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWMKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2V4aXN0aW5nc2cuZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKZXRjZE1hbmlmZXN0czoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2V4aXN0aW5nc2cuZXhhbXBsZS5jb20vbWFuaWZlc3RzL2V0Y2QvbWFpbi55YW1sCi0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9leGlzdGluZ3NnLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: existingsg.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: existingsg.example.com +ConfigBase: memfs://clusters.example.com/existingsg.example.com +InstanceGroupName: master-us-test-1c +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/existingsg.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/existingsg.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/existingsg.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/existing_sg/data/aws_launch_template_nodes.existingsg.example.com_user_data b/tests/integration/update_cluster/existing_sg/data/aws_launch_template_nodes.existingsg.example.com_user_data index 26521cb7f3ff3..399acdff3e809 100644 --- a/tests/integration/update_cluster/existing_sg/data/aws_launch_template_nodes.existingsg.example.com_user_data +++ b/tests/integration/update_cluster/existing_sg/data/aws_launch_template_nodes.existingsg.example.com_user_data @@ -1 +1,209 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IGV4aXN0aW5nc2cuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9leGlzdGluZ3NnLmV4YW1wbGUuY29tCkluc3RhbmNlR3JvdXBOYW1lOiBub2RlcwpUYWdzOgotIF9hdXRvbWF0aWNfdXBncmFkZXMKLSBfYXdzCmNoYW5uZWxzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vZXhpc3RpbmdzZy5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbAoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg== +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: existingsg.example.com +ConfigBase: memfs://clusters.example.com/existingsg.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/existingsg.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/externallb/data/aws_launch_template_master-us-test-1a.masters.externallb.example.com_user_data b/tests/integration/update_cluster/externallb/data/aws_launch_template_master-us-test-1a.masters.externallb.example.com_user_data index d37c081f6fe1c..6a1b0b329795b 100644 --- a/tests/integration/update_cluster/externallb/data/aws_launch_template_master-us-test-1a.masters.externallb.example.com_user_data +++ b/tests/integration/update_cluster/externallb/data/aws_launch_template_master-us-test-1a.masters.externallb.example.com_user_data @@ -1 +1,302 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBleHRlcm5hbGxiLmV4YW1wbGUuY29tCiAgY29uZmlndXJlQ2xvdWRSb3V0ZXM6IHRydWUKICBpbWFnZTogazhzLmdjci5pby9rdWJlLWNvbnRyb2xsZXItbWFuYWdlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCiAgdXNlU2VydmljZUFjY291bnRDcmVkZW50aWFsczogdHJ1ZQprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZVNjaGVkdWxlcjoKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXNjaGVkdWxlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCmt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk1UVTogOTAwMQogIG5ldHdvcmtQbHVnaW5OYW1lOiBrdWJlbmV0CiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwptYXN0ZXJLdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5NVFU6IDkwMDEKICBuZXR3b3JrUGx1Z2luTmFtZToga3ViZW5ldAogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKICByZWdpc3RlclNjaGVkdWxhYmxlOiBmYWxzZQoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGMzYjczNmZkMGYwMDM3NjVjMTJkOTlmMmM5OTVhODM2OWU2MjQxZjRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA3ZTNhM2VhNjYzMTUzZjkwMGNiZDUyOTAwYTM5YzkxZmE5ZjMzNGJlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlY3RsCi0gNTJlOWQyZGU4YTVmOTI3MzA3ZDkzOTczMDg3MzU2NThlZTQ0YWI4ZEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL25ldHdvcmstcGx1Z2lucy9jbmktcGx1Z2lucy1hbWQ2NC12MC43LjUudGd6CkNsdXN0ZXJOYW1lOiBleHRlcm5hbGxiLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vZXh0ZXJuYWxsYi5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2V4dGVybmFsbGIuZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKZXRjZE1hbmlmZXN0czoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2V4dGVybmFsbGIuZXhhbXBsZS5jb20vbWFuaWZlc3RzL2V0Y2QvbWFpbi55YW1sCi0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9leHRlcm5hbGxiLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: externallb.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: externallb.example.com +ConfigBase: memfs://clusters.example.com/externallb.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/externallb.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/externallb.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/externallb.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/externallb/data/aws_launch_template_nodes.externallb.example.com_user_data b/tests/integration/update_cluster/externallb/data/aws_launch_template_nodes.externallb.example.com_user_data index 7e4a010c00fb6..166ab708c343d 100644 --- a/tests/integration/update_cluster/externallb/data/aws_launch_template_nodes.externallb.example.com_user_data +++ b/tests/integration/update_cluster/externallb/data/aws_launch_template_nodes.externallb.example.com_user_data @@ -1 +1,209 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IGV4dGVybmFsbGIuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9leHRlcm5hbGxiLmV4YW1wbGUuY29tCkluc3RhbmNlR3JvdXBOYW1lOiBub2RlcwpUYWdzOgotIF9hdXRvbWF0aWNfdXBncmFkZXMKLSBfYXdzCmNoYW5uZWxzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vZXh0ZXJuYWxsYi5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbAoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg== +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: externallb.example.com +ConfigBase: memfs://clusters.example.com/externallb.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/externallb.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/externalpolicies/data/aws_launch_template_master-us-test-1a.masters.externalpolicies.example.com_user_data b/tests/integration/update_cluster/externalpolicies/data/aws_launch_template_master-us-test-1a.masters.externalpolicies.example.com_user_data index f6055deb711c8..d575b93fc96e9 100644 --- a/tests/integration/update_cluster/externalpolicies/data/aws_launch_template_master-us-test-1a.masters.externalpolicies.example.com_user_data +++ b/tests/integration/update_cluster/externalpolicies/data/aws_launch_template_master-us-test-1a.masters.externalpolicies.example.com_user_data @@ -1 +1,304 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1ZGl0V2ViaG9va0JhdGNoVGhyb3R0bGVRcHM6IDMxNDBtCiAgYXV0aG9yaXphdGlvbk1vZGU6IEFsd2F5c0FsbG93CiAgYmluZEFkZHJlc3M6IDAuMC4wLjAKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBlbmFibGVBZG1pc3Npb25QbHVnaW5zOgogIC0gTmFtZXNwYWNlTGlmZWN5Y2xlCiAgLSBMaW1pdFJhbmdlcgogIC0gU2VydmljZUFjY291bnQKICAtIFBlcnNpc3RlbnRWb2x1bWVMYWJlbAogIC0gRGVmYXVsdFN0b3JhZ2VDbGFzcwogIC0gRGVmYXVsdFRvbGVyYXRpb25TZWNvbmRzCiAgLSBNdXRhdGluZ0FkbWlzc2lvbldlYmhvb2sKICAtIFZhbGlkYXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBOb2RlUmVzdHJpY3Rpb24KICAtIFJlc291cmNlUXVvdGEKICBldGNkU2VydmVyczoKICAtIGh0dHA6Ly8xMjcuMC4wLjE6NDAwMQogIGV0Y2RTZXJ2ZXJzT3ZlcnJpZGVzOgogIC0gL2V2ZW50cyNodHRwOi8vMTI3LjAuMC4xOjQwMDIKICBpbWFnZTogazhzLmdjci5pby9rdWJlLWFwaXNlcnZlcjp2MS4xNC4wCiAgaW5zZWN1cmVCaW5kQWRkcmVzczogMTI3LjAuMC4xCiAgaW5zZWN1cmVQb3J0OiA4MDgwCiAga3ViZWxldFByZWZlcnJlZEFkZHJlc3NUeXBlczoKICAtIEludGVybmFsSVAKICAtIEhvc3RuYW1lCiAgLSBFeHRlcm5hbElQCiAgbG9nTGV2ZWw6IDIKICByZXF1ZXN0aGVhZGVyQWxsb3dlZE5hbWVzOgogIC0gYWdncmVnYXRvcgogIHJlcXVlc3RoZWFkZXJFeHRyYUhlYWRlclByZWZpeGVzOgogIC0gWC1SZW1vdGUtRXh0cmEtCiAgcmVxdWVzdGhlYWRlckdyb3VwSGVhZGVyczoKICAtIFgtUmVtb3RlLUdyb3VwCiAgcmVxdWVzdGhlYWRlclVzZXJuYW1lSGVhZGVyczoKICAtIFgtUmVtb3RlLVVzZXIKICBzZWN1cmVQb3J0OiA0NDMKICBzZXJ2aWNlQ2x1c3RlcklQUmFuZ2U6IDEwMC42NC4wLjAvMTMKICBzZXJ2aWNlTm9kZVBvcnRSYW5nZTogMjgwMDAtMzI3NjcKICBzdG9yYWdlQmFja2VuZDogZXRjZDMKa3ViZUNvbnRyb2xsZXJNYW5hZ2VyOgogIGFsbG9jYXRlTm9kZUNJRFJzOiB0cnVlCiAgYXR0YWNoRGV0YWNoUmVjb25jaWxlU3luY1BlcmlvZDogMW0wcwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJDSURSOiAxMDAuOTYuMC4wLzExCiAgY2x1c3Rlck5hbWU6IGV4dGVybmFscG9saWNpZXMuZXhhbXBsZS5jb20KICBjb25maWd1cmVDbG91ZFJvdXRlczogdHJ1ZQogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKICB1c2VTZXJ2aWNlQWNjb3VudENyZWRlbnRpYWxzOiB0cnVlCmt1YmVQcm94eToKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNwdVJlcXVlc3Q6IDEwMG0KICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXByb3h5OnYxLjE0LjAKICBsb2dMZXZlbDogMgprdWJlU2NoZWR1bGVyOgogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCm1hc3Rlckt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk1UVTogOTAwMQogIG5ldHdvcmtQbHVnaW5OYW1lOiBrdWJlbmV0CiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwogIHJlZ2lzdGVyU2NoZWR1bGFibGU6IGZhbHNlCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IGV4dGVybmFscG9saWNpZXMuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9leHRlcm5hbHBvbGljaWVzLmV4YW1wbGUuY29tCkluc3RhbmNlR3JvdXBOYW1lOiBtYXN0ZXItdXMtdGVzdC0xYQpUYWdzOgotIF9hdXRvbWF0aWNfdXBncmFkZXMKLSBfYXdzCmNoYW5uZWxzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vZXh0ZXJuYWxwb2xpY2llcy5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbApldGNkTWFuaWZlc3RzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vZXh0ZXJuYWxwb2xpY2llcy5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9tYWluLnlhbWwKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL2V4dGVybmFscG9saWNpZXMuZXhhbXBsZS5jb20vbWFuaWZlc3RzL2V0Y2QvZXZlbnRzLnlhbWwKcHJvdG9rdWJlSW1hZ2U6CiAgaGFzaDogNDJhOWM0MzI0ZmUyNmQ2M2NlMTFmM2RkNzgzNjM3MWJjOTNmYTA2Y2E4ZjQ3OTgwNzcyOGYzNzQ2ZTI3MDYxYgogIG5hbWU6IHByb3Rva3ViZToxLjE1LjAKICBzb3VyY2VzOgogIC0gaHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2ltYWdlcy9wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvaW1hZ2VzLXByb3Rva3ViZS50YXIuZ3oKICAtIGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg== +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + auditWebhookBatchThrottleQps: 3140m + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + serviceNodePortRange: 28000-32767 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: externalpolicies.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: externalpolicies.example.com +ConfigBase: memfs://clusters.example.com/externalpolicies.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/externalpolicies.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/externalpolicies.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/externalpolicies.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/externalpolicies/data/aws_launch_template_nodes.externalpolicies.example.com_user_data b/tests/integration/update_cluster/externalpolicies/data/aws_launch_template_nodes.externalpolicies.example.com_user_data index 437ae55bfa065..8e97cd2f464c7 100644 --- a/tests/integration/update_cluster/externalpolicies/data/aws_launch_template_nodes.externalpolicies.example.com_user_data +++ b/tests/integration/update_cluster/externalpolicies/data/aws_launch_template_nodes.externalpolicies.example.com_user_data @@ -1 +1,209 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IGV4dGVybmFscG9saWNpZXMuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9leHRlcm5hbHBvbGljaWVzLmV4YW1wbGUuY29tCkluc3RhbmNlR3JvdXBOYW1lOiBub2RlcwpUYWdzOgotIF9hdXRvbWF0aWNfdXBncmFkZXMKLSBfYXdzCmNoYW5uZWxzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vZXh0ZXJuYWxwb2xpY2llcy5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbAoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg== +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: externalpolicies.example.com +ConfigBase: memfs://clusters.example.com/externalpolicies.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/externalpolicies.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1a.masters.ha.example.com_user_data b/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1a.masters.ha.example.com_user_data index dcfb011fc1be6..615616abe2f25 100644 --- a/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1a.masters.ha.example.com_user_data +++ b/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1a.masters.ha.example.com_user_data @@ -1 +1,302 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMwogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBoYS5leGFtcGxlLmNvbQogIGNvbmZpZ3VyZUNsb3VkUm91dGVzOiB0cnVlCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1jb250cm9sbGVyLW1hbmFnZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgogIHVzZVNlcnZpY2VBY2NvdW50Q3JlZGVudGlhbHM6IHRydWUKa3ViZVByb3h5OgogIGNsdXN0ZXJDSURSOiAxMDAuOTYuMC4wLzExCiAgY3B1UmVxdWVzdDogMTAwbQogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtcHJveHk6djEuMTQuMAogIGxvZ0xldmVsOiAyCmt1YmVTY2hlZHVsZXI6CiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1zY2hlZHVsZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgprdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5NVFU6IDkwMDEKICBuZXR3b3JrUGx1Z2luTmFtZToga3ViZW5ldAogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKbWFzdGVyS3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCiAgcmVnaXN0ZXJTY2hlZHVsYWJsZTogZmFsc2UKCl9fRU9GX0NMVVNURVJfU1BFQwoKY2F0ID4gY29uZi9pZ19zcGVjLnlhbWwgPDwgJ19fRU9GX0lHX1NQRUMnCmt1YmVsZXQ6IG51bGwKbm9kZUxhYmVsczogbnVsbAp0YWludHM6IG51bGwKCl9fRU9GX0lHX1NQRUMKCmNhdCA+IGNvbmYva3ViZV9lbnYueWFtbCA8PCAnX19FT0ZfS1VCRV9FTlYnCkFzc2V0czoKLSBjM2I3MzZmZDBmMDAzNzY1YzEyZDk5ZjJjOTk1YTgzNjllNjI0MWY0QGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlbGV0Ci0gN2UzYTNlYTY2MzE1M2Y5MDBjYmQ1MjkwMGEzOWM5MWZhOWYzMzRiZUBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDUyZTlkMmRlOGE1ZjkyNzMwN2Q5Mzk3MzA4NzM1NjU4ZWU0NGFiOGRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLXBsdWdpbnMtYW1kNjQtdjAuNy41LnRnegpDbHVzdGVyTmFtZTogaGEuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly90ZXN0cy9oYS5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL3Rlc3RzL2hhLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCmV0Y2RNYW5pZmVzdHM6Ci0gbWVtZnM6Ly90ZXN0cy9oYS5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9tYWluLnlhbWwKLSBtZW1mczovL3Rlc3RzL2hhLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: ha.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: ha.example.com +ConfigBase: memfs://tests/ha.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://tests/ha.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://tests/ha.example.com/manifests/etcd/main.yaml +- memfs://tests/ha.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1b.masters.ha.example.com_user_data b/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1b.masters.ha.example.com_user_data index cab89688d3f58..7a2f2fa5ab44f 100644 --- a/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1b.masters.ha.example.com_user_data +++ b/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1b.masters.ha.example.com_user_data @@ -1 +1,302 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMwogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBoYS5leGFtcGxlLmNvbQogIGNvbmZpZ3VyZUNsb3VkUm91dGVzOiB0cnVlCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1jb250cm9sbGVyLW1hbmFnZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgogIHVzZVNlcnZpY2VBY2NvdW50Q3JlZGVudGlhbHM6IHRydWUKa3ViZVByb3h5OgogIGNsdXN0ZXJDSURSOiAxMDAuOTYuMC4wLzExCiAgY3B1UmVxdWVzdDogMTAwbQogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtcHJveHk6djEuMTQuMAogIGxvZ0xldmVsOiAyCmt1YmVTY2hlZHVsZXI6CiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1zY2hlZHVsZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgprdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5NVFU6IDkwMDEKICBuZXR3b3JrUGx1Z2luTmFtZToga3ViZW5ldAogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKbWFzdGVyS3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCiAgcmVnaXN0ZXJTY2hlZHVsYWJsZTogZmFsc2UKCl9fRU9GX0NMVVNURVJfU1BFQwoKY2F0ID4gY29uZi9pZ19zcGVjLnlhbWwgPDwgJ19fRU9GX0lHX1NQRUMnCmt1YmVsZXQ6IG51bGwKbm9kZUxhYmVsczogbnVsbAp0YWludHM6IG51bGwKCl9fRU9GX0lHX1NQRUMKCmNhdCA+IGNvbmYva3ViZV9lbnYueWFtbCA8PCAnX19FT0ZfS1VCRV9FTlYnCkFzc2V0czoKLSBjM2I3MzZmZDBmMDAzNzY1YzEyZDk5ZjJjOTk1YTgzNjllNjI0MWY0QGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlbGV0Ci0gN2UzYTNlYTY2MzE1M2Y5MDBjYmQ1MjkwMGEzOWM5MWZhOWYzMzRiZUBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDUyZTlkMmRlOGE1ZjkyNzMwN2Q5Mzk3MzA4NzM1NjU4ZWU0NGFiOGRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLXBsdWdpbnMtYW1kNjQtdjAuNy41LnRnegpDbHVzdGVyTmFtZTogaGEuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly90ZXN0cy9oYS5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWIKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL3Rlc3RzL2hhLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCmV0Y2RNYW5pZmVzdHM6Ci0gbWVtZnM6Ly90ZXN0cy9oYS5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9tYWluLnlhbWwKLSBtZW1mczovL3Rlc3RzL2hhLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: ha.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: ha.example.com +ConfigBase: memfs://tests/ha.example.com +InstanceGroupName: master-us-test-1b +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://tests/ha.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://tests/ha.example.com/manifests/etcd/main.yaml +- memfs://tests/ha.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1c.masters.ha.example.com_user_data b/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1c.masters.ha.example.com_user_data index 5adff34583324..abd5e502ccfba 100644 --- a/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1c.masters.ha.example.com_user_data +++ b/tests/integration/update_cluster/ha/data/aws_launch_template_master-us-test-1c.masters.ha.example.com_user_data @@ -1 +1,302 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMwogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBoYS5leGFtcGxlLmNvbQogIGNvbmZpZ3VyZUNsb3VkUm91dGVzOiB0cnVlCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1jb250cm9sbGVyLW1hbmFnZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgogIHVzZVNlcnZpY2VBY2NvdW50Q3JlZGVudGlhbHM6IHRydWUKa3ViZVByb3h5OgogIGNsdXN0ZXJDSURSOiAxMDAuOTYuMC4wLzExCiAgY3B1UmVxdWVzdDogMTAwbQogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtcHJveHk6djEuMTQuMAogIGxvZ0xldmVsOiAyCmt1YmVTY2hlZHVsZXI6CiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1zY2hlZHVsZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgprdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5NVFU6IDkwMDEKICBuZXR3b3JrUGx1Z2luTmFtZToga3ViZW5ldAogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKbWFzdGVyS3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCiAgcmVnaXN0ZXJTY2hlZHVsYWJsZTogZmFsc2UKCl9fRU9GX0NMVVNURVJfU1BFQwoKY2F0ID4gY29uZi9pZ19zcGVjLnlhbWwgPDwgJ19fRU9GX0lHX1NQRUMnCmt1YmVsZXQ6IG51bGwKbm9kZUxhYmVsczogbnVsbAp0YWludHM6IG51bGwKCl9fRU9GX0lHX1NQRUMKCmNhdCA+IGNvbmYva3ViZV9lbnYueWFtbCA8PCAnX19FT0ZfS1VCRV9FTlYnCkFzc2V0czoKLSBjM2I3MzZmZDBmMDAzNzY1YzEyZDk5ZjJjOTk1YTgzNjllNjI0MWY0QGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlbGV0Ci0gN2UzYTNlYTY2MzE1M2Y5MDBjYmQ1MjkwMGEzOWM5MWZhOWYzMzRiZUBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDUyZTlkMmRlOGE1ZjkyNzMwN2Q5Mzk3MzA4NzM1NjU4ZWU0NGFiOGRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLXBsdWdpbnMtYW1kNjQtdjAuNy41LnRnegpDbHVzdGVyTmFtZTogaGEuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly90ZXN0cy9oYS5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWMKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL3Rlc3RzL2hhLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCmV0Y2RNYW5pZmVzdHM6Ci0gbWVtZnM6Ly90ZXN0cy9oYS5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9tYWluLnlhbWwKLSBtZW1mczovL3Rlc3RzL2hhLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: ha.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: ha.example.com +ConfigBase: memfs://tests/ha.example.com +InstanceGroupName: master-us-test-1c +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://tests/ha.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://tests/ha.example.com/manifests/etcd/main.yaml +- memfs://tests/ha.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/ha/data/aws_launch_template_nodes.ha.example.com_user_data b/tests/integration/update_cluster/ha/data/aws_launch_template_nodes.ha.example.com_user_data index 34fa631fddae1..e3e86a6a30e68 100644 --- a/tests/integration/update_cluster/ha/data/aws_launch_template_nodes.ha.example.com_user_data +++ b/tests/integration/update_cluster/ha/data/aws_launch_template_nodes.ha.example.com_user_data @@ -1 +1,209 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IGhhLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vdGVzdHMvaGEuZXhhbXBsZS5jb20KSW5zdGFuY2VHcm91cE5hbWU6IG5vZGVzClRhZ3M6Ci0gX2F1dG9tYXRpY191cGdyYWRlcwotIF9hd3MKY2hhbm5lbHM6Ci0gbWVtZnM6Ly90ZXN0cy9oYS5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbAoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg== +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: ha.example.com +ConfigBase: memfs://tests/ha.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://tests/ha.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/minimal/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/tests/integration/update_cluster/minimal/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data index e85adce189f7a..3d5f2b03087c1 100644 --- a/tests/integration/update_cluster/minimal/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data +++ b/tests/integration/update_cluster/minimal/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data @@ -1 +1,302 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBtaW5pbWFsLmV4YW1wbGUuY29tCiAgY29uZmlndXJlQ2xvdWRSb3V0ZXM6IHRydWUKICBpbWFnZTogazhzLmdjci5pby9rdWJlLWNvbnRyb2xsZXItbWFuYWdlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCiAgdXNlU2VydmljZUFjY291bnRDcmVkZW50aWFsczogdHJ1ZQprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZVNjaGVkdWxlcjoKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXNjaGVkdWxlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCmt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk1UVTogOTAwMQogIG5ldHdvcmtQbHVnaW5OYW1lOiBrdWJlbmV0CiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwptYXN0ZXJLdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5NVFU6IDkwMDEKICBuZXR3b3JrUGx1Z2luTmFtZToga3ViZW5ldAogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKICByZWdpc3RlclNjaGVkdWxhYmxlOiBmYWxzZQoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGMzYjczNmZkMGYwMDM3NjVjMTJkOTlmMmM5OTVhODM2OWU2MjQxZjRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA3ZTNhM2VhNjYzMTUzZjkwMGNiZDUyOTAwYTM5YzkxZmE5ZjMzNGJlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlY3RsCi0gNTJlOWQyZGU4YTVmOTI3MzA3ZDkzOTczMDg3MzU2NThlZTQ0YWI4ZEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL25ldHdvcmstcGx1Z2lucy9jbmktcGx1Z2lucy1hbWQ2NC12MC43LjUudGd6CkNsdXN0ZXJOYW1lOiBtaW5pbWFsLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vbWluaW1hbC5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL21pbmltYWwuZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKZXRjZE1hbmlmZXN0czoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL21pbmltYWwuZXhhbXBsZS5jb20vbWFuaWZlc3RzL2V0Y2QvbWFpbi55YW1sCi0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taW5pbWFsLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: minimal.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: minimal.example.com +ConfigBase: memfs://clusters.example.com/minimal.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/minimal/data/aws_launch_template_nodes.minimal.example.com_user_data b/tests/integration/update_cluster/minimal/data/aws_launch_template_nodes.minimal.example.com_user_data index 7d67047f50877..c7f2e0062fce5 100644 --- a/tests/integration/update_cluster/minimal/data/aws_launch_template_nodes.minimal.example.com_user_data +++ b/tests/integration/update_cluster/minimal/data/aws_launch_template_nodes.minimal.example.com_user_data @@ -1 +1,209 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IG1pbmltYWwuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taW5pbWFsLmV4YW1wbGUuY29tCkluc3RhbmNlR3JvdXBOYW1lOiBub2RlcwpUYWdzOgotIF9hdXRvbWF0aWNfdXBncmFkZXMKLSBfYXdzCmNoYW5uZWxzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vbWluaW1hbC5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbAoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg== +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: minimal.example.com +ConfigBase: memfs://clusters.example.com/minimal.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1a.masters.mixedinstances.example.com_user_data b/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1a.masters.mixedinstances.example.com_user_data index 885c850fb1c38..caaa7eb9443c5 100644 --- a/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1a.masters.mixedinstances.example.com_user_data +++ b/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1a.masters.mixedinstances.example.com_user_data @@ -1 +1,304 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjIuMjQKICBtYWluOgogICAgdmVyc2lvbjogMy4yLjI0Cmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMwogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTIuOQogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBtaXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbQogIGNvbmZpZ3VyZUNsb3VkUm91dGVzOiB0cnVlCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1jb250cm9sbGVyLW1hbmFnZXI6djEuMTIuOQogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgogIHVzZVNlcnZpY2VBY2NvdW50Q3JlZGVudGlhbHM6IHRydWUKa3ViZVByb3h5OgogIGNsdXN0ZXJDSURSOiAxMDAuOTYuMC4wLzExCiAgY3B1UmVxdWVzdDogMTAwbQogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtcHJveHk6djEuMTIuOQogIGxvZ0xldmVsOiAyCmt1YmVTY2hlZHVsZXI6CiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1zY2hlZHVsZXI6djEuMTIuOQogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgprdWJlbGV0OgogIGFsbG93UHJpdmlsZWdlZDogdHJ1ZQogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5NVFU6IDkwMDEKICBuZXR3b3JrUGx1Z2luTmFtZToga3ViZW5ldAogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKbWFzdGVyS3ViZWxldDoKICBhbGxvd1ByaXZpbGVnZWQ6IHRydWUKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCiAgcmVnaXN0ZXJTY2hlZHVsYWJsZTogZmFsc2UKCl9fRU9GX0NMVVNURVJfU1BFQwoKY2F0ID4gY29uZi9pZ19zcGVjLnlhbWwgPDwgJ19fRU9GX0lHX1NQRUMnCmt1YmVsZXQ6IG51bGwKbm9kZUxhYmVsczogbnVsbAp0YWludHM6IG51bGwKCl9fRU9GX0lHX1NQRUMKCmNhdCA+IGNvbmYva3ViZV9lbnYueWFtbCA8PCAnX19FT0ZfS1VCRV9FTlYnCkFzc2V0czoKLSBlOTE0YjE3NTMyYzQxMWNiN2MwY2M0NzIxMzFiNjE5MzVmYjY2YjMxQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xMi45L2Jpbi9saW51eC9hbWQ2NC9rdWJlbGV0Ci0gYWEzZTkzODk3YTY5OTlkNmM3ZGVkYmM0MTc5M2M5MGQ0MWVlYjAwMEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTIuOS9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDUyZTlkMmRlOGE1ZjkyNzMwN2Q5Mzk3MzA4NzM1NjU4ZWU0NGFiOGRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLXBsdWdpbnMtYW1kNjQtdjAuNy41LnRnegpDbHVzdGVyTmFtZTogbWl4ZWRpbnN0YW5jZXMuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL21peGVkaW5zdGFuY2VzLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCmV0Y2RNYW5pZmVzdHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9tYWluLnlhbWwKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL21peGVkaW5zdGFuY2VzLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.2.24 + main: + version: 3.2.24 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.12.9 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: mixedinstances.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.12.9 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.12.9 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.12.9 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + allowPrivileged: true + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + allowPrivileged: true + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- e914b17532c411cb7c0cc472131b61935fb66b31@https://storage.googleapis.com/kubernetes-release/release/v1.12.9/bin/linux/amd64/kubelet +- aa3e93897a6999d6c7dedbc41793c90d41eeb000@https://storage.googleapis.com/kubernetes-release/release/v1.12.9/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: mixedinstances.example.com +ConfigBase: memfs://clusters.example.com/mixedinstances.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1b.masters.mixedinstances.example.com_user_data b/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1b.masters.mixedinstances.example.com_user_data index 902fa42a16b9a..bc4f0084d91b5 100644 --- a/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1b.masters.mixedinstances.example.com_user_data +++ b/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1b.masters.mixedinstances.example.com_user_data @@ -1 +1,304 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjIuMjQKICBtYWluOgogICAgdmVyc2lvbjogMy4yLjI0Cmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMwogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTIuOQogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBtaXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbQogIGNvbmZpZ3VyZUNsb3VkUm91dGVzOiB0cnVlCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1jb250cm9sbGVyLW1hbmFnZXI6djEuMTIuOQogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgogIHVzZVNlcnZpY2VBY2NvdW50Q3JlZGVudGlhbHM6IHRydWUKa3ViZVByb3h5OgogIGNsdXN0ZXJDSURSOiAxMDAuOTYuMC4wLzExCiAgY3B1UmVxdWVzdDogMTAwbQogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtcHJveHk6djEuMTIuOQogIGxvZ0xldmVsOiAyCmt1YmVTY2hlZHVsZXI6CiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1zY2hlZHVsZXI6djEuMTIuOQogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgprdWJlbGV0OgogIGFsbG93UHJpdmlsZWdlZDogdHJ1ZQogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5NVFU6IDkwMDEKICBuZXR3b3JrUGx1Z2luTmFtZToga3ViZW5ldAogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKbWFzdGVyS3ViZWxldDoKICBhbGxvd1ByaXZpbGVnZWQ6IHRydWUKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCiAgcmVnaXN0ZXJTY2hlZHVsYWJsZTogZmFsc2UKCl9fRU9GX0NMVVNURVJfU1BFQwoKY2F0ID4gY29uZi9pZ19zcGVjLnlhbWwgPDwgJ19fRU9GX0lHX1NQRUMnCmt1YmVsZXQ6IG51bGwKbm9kZUxhYmVsczogbnVsbAp0YWludHM6IG51bGwKCl9fRU9GX0lHX1NQRUMKCmNhdCA+IGNvbmYva3ViZV9lbnYueWFtbCA8PCAnX19FT0ZfS1VCRV9FTlYnCkFzc2V0czoKLSBlOTE0YjE3NTMyYzQxMWNiN2MwY2M0NzIxMzFiNjE5MzVmYjY2YjMxQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xMi45L2Jpbi9saW51eC9hbWQ2NC9rdWJlbGV0Ci0gYWEzZTkzODk3YTY5OTlkNmM3ZGVkYmM0MTc5M2M5MGQ0MWVlYjAwMEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTIuOS9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDUyZTlkMmRlOGE1ZjkyNzMwN2Q5Mzk3MzA4NzM1NjU4ZWU0NGFiOGRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLXBsdWdpbnMtYW1kNjQtdjAuNy41LnRnegpDbHVzdGVyTmFtZTogbWl4ZWRpbnN0YW5jZXMuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWIKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL21peGVkaW5zdGFuY2VzLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCmV0Y2RNYW5pZmVzdHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9tYWluLnlhbWwKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL21peGVkaW5zdGFuY2VzLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.2.24 + main: + version: 3.2.24 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.12.9 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: mixedinstances.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.12.9 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.12.9 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.12.9 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + allowPrivileged: true + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + allowPrivileged: true + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- e914b17532c411cb7c0cc472131b61935fb66b31@https://storage.googleapis.com/kubernetes-release/release/v1.12.9/bin/linux/amd64/kubelet +- aa3e93897a6999d6c7dedbc41793c90d41eeb000@https://storage.googleapis.com/kubernetes-release/release/v1.12.9/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: mixedinstances.example.com +ConfigBase: memfs://clusters.example.com/mixedinstances.example.com +InstanceGroupName: master-us-test-1b +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1c.masters.mixedinstances.example.com_user_data b/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1c.masters.mixedinstances.example.com_user_data index 99d2235f70e33..f8ab7db3e8ab2 100644 --- a/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1c.masters.mixedinstances.example.com_user_data +++ b/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_master-us-test-1c.masters.mixedinstances.example.com_user_data @@ -1 +1,304 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjIuMjQKICBtYWluOgogICAgdmVyc2lvbjogMy4yLjI0Cmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMwogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTIuOQogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBtaXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbQogIGNvbmZpZ3VyZUNsb3VkUm91dGVzOiB0cnVlCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1jb250cm9sbGVyLW1hbmFnZXI6djEuMTIuOQogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgogIHVzZVNlcnZpY2VBY2NvdW50Q3JlZGVudGlhbHM6IHRydWUKa3ViZVByb3h5OgogIGNsdXN0ZXJDSURSOiAxMDAuOTYuMC4wLzExCiAgY3B1UmVxdWVzdDogMTAwbQogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtcHJveHk6djEuMTIuOQogIGxvZ0xldmVsOiAyCmt1YmVTY2hlZHVsZXI6CiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1zY2hlZHVsZXI6djEuMTIuOQogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgprdWJlbGV0OgogIGFsbG93UHJpdmlsZWdlZDogdHJ1ZQogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5NVFU6IDkwMDEKICBuZXR3b3JrUGx1Z2luTmFtZToga3ViZW5ldAogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKbWFzdGVyS3ViZWxldDoKICBhbGxvd1ByaXZpbGVnZWQ6IHRydWUKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCiAgcmVnaXN0ZXJTY2hlZHVsYWJsZTogZmFsc2UKCl9fRU9GX0NMVVNURVJfU1BFQwoKY2F0ID4gY29uZi9pZ19zcGVjLnlhbWwgPDwgJ19fRU9GX0lHX1NQRUMnCmt1YmVsZXQ6IG51bGwKbm9kZUxhYmVsczogbnVsbAp0YWludHM6IG51bGwKCl9fRU9GX0lHX1NQRUMKCmNhdCA+IGNvbmYva3ViZV9lbnYueWFtbCA8PCAnX19FT0ZfS1VCRV9FTlYnCkFzc2V0czoKLSBlOTE0YjE3NTMyYzQxMWNiN2MwY2M0NzIxMzFiNjE5MzVmYjY2YjMxQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xMi45L2Jpbi9saW51eC9hbWQ2NC9rdWJlbGV0Ci0gYWEzZTkzODk3YTY5OTlkNmM3ZGVkYmM0MTc5M2M5MGQ0MWVlYjAwMEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTIuOS9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDUyZTlkMmRlOGE1ZjkyNzMwN2Q5Mzk3MzA4NzM1NjU4ZWU0NGFiOGRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLXBsdWdpbnMtYW1kNjQtdjAuNy41LnRnegpDbHVzdGVyTmFtZTogbWl4ZWRpbnN0YW5jZXMuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWMKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL21peGVkaW5zdGFuY2VzLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCmV0Y2RNYW5pZmVzdHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9tYWluLnlhbWwKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL21peGVkaW5zdGFuY2VzLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.2.24 + main: + version: 3.2.24 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.12.9 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: mixedinstances.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.12.9 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.12.9 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.12.9 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + allowPrivileged: true + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + allowPrivileged: true + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- e914b17532c411cb7c0cc472131b61935fb66b31@https://storage.googleapis.com/kubernetes-release/release/v1.12.9/bin/linux/amd64/kubelet +- aa3e93897a6999d6c7dedbc41793c90d41eeb000@https://storage.googleapis.com/kubernetes-release/release/v1.12.9/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: mixedinstances.example.com +ConfigBase: memfs://clusters.example.com/mixedinstances.example.com +InstanceGroupName: master-us-test-1c +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_nodes.mixedinstances.example.com_user_data b/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_nodes.mixedinstances.example.com_user_data index 6fee2cc7e2fed..9aef29bbcc73c 100644 --- a/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_nodes.mixedinstances.example.com_user_data +++ b/tests/integration/update_cluster/mixed_instances/data/aws_launch_template_nodes.mixedinstances.example.com_user_data @@ -1 +1,210 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xMi45CiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbGxvd1ByaXZpbGVnZWQ6IHRydWUKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gZTkxNGIxNzUzMmM0MTFjYjdjMGNjNDcyMTMxYjYxOTM1ZmI2NmIzMUBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTIuOS9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIGFhM2U5Mzg5N2E2OTk5ZDZjN2RlZGJjNDE3OTNjOTBkNDFlZWIwMDBAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjEyLjkvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IG1peGVkaW5zdGFuY2VzLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vbWl4ZWRpbnN0YW5jZXMuZXhhbXBsZS5jb20KSW5zdGFuY2VHcm91cE5hbWU6IG5vZGVzClRhZ3M6Ci0gX2F1dG9tYXRpY191cGdyYWRlcwotIF9hd3MKY2hhbm5lbHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbAoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg== +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.12.9 + logLevel: 2 +kubelet: + allowPrivileged: true + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- e914b17532c411cb7c0cc472131b61935fb66b31@https://storage.googleapis.com/kubernetes-release/release/v1.12.9/bin/linux/amd64/kubelet +- aa3e93897a6999d6c7dedbc41793c90d41eeb000@https://storage.googleapis.com/kubernetes-release/release/v1.12.9/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: mixedinstances.example.com +ConfigBase: memfs://clusters.example.com/mixedinstances.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1a.masters.mixedinstances.example.com_user_data b/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1a.masters.mixedinstances.example.com_user_data index 885c850fb1c38..caaa7eb9443c5 100644 --- a/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1a.masters.mixedinstances.example.com_user_data +++ b/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1a.masters.mixedinstances.example.com_user_data @@ -1 +1,304 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjIuMjQKICBtYWluOgogICAgdmVyc2lvbjogMy4yLjI0Cmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMwogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTIuOQogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBtaXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbQogIGNvbmZpZ3VyZUNsb3VkUm91dGVzOiB0cnVlCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1jb250cm9sbGVyLW1hbmFnZXI6djEuMTIuOQogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgogIHVzZVNlcnZpY2VBY2NvdW50Q3JlZGVudGlhbHM6IHRydWUKa3ViZVByb3h5OgogIGNsdXN0ZXJDSURSOiAxMDAuOTYuMC4wLzExCiAgY3B1UmVxdWVzdDogMTAwbQogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtcHJveHk6djEuMTIuOQogIGxvZ0xldmVsOiAyCmt1YmVTY2hlZHVsZXI6CiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1zY2hlZHVsZXI6djEuMTIuOQogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgprdWJlbGV0OgogIGFsbG93UHJpdmlsZWdlZDogdHJ1ZQogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5NVFU6IDkwMDEKICBuZXR3b3JrUGx1Z2luTmFtZToga3ViZW5ldAogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKbWFzdGVyS3ViZWxldDoKICBhbGxvd1ByaXZpbGVnZWQ6IHRydWUKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCiAgcmVnaXN0ZXJTY2hlZHVsYWJsZTogZmFsc2UKCl9fRU9GX0NMVVNURVJfU1BFQwoKY2F0ID4gY29uZi9pZ19zcGVjLnlhbWwgPDwgJ19fRU9GX0lHX1NQRUMnCmt1YmVsZXQ6IG51bGwKbm9kZUxhYmVsczogbnVsbAp0YWludHM6IG51bGwKCl9fRU9GX0lHX1NQRUMKCmNhdCA+IGNvbmYva3ViZV9lbnYueWFtbCA8PCAnX19FT0ZfS1VCRV9FTlYnCkFzc2V0czoKLSBlOTE0YjE3NTMyYzQxMWNiN2MwY2M0NzIxMzFiNjE5MzVmYjY2YjMxQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xMi45L2Jpbi9saW51eC9hbWQ2NC9rdWJlbGV0Ci0gYWEzZTkzODk3YTY5OTlkNmM3ZGVkYmM0MTc5M2M5MGQ0MWVlYjAwMEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTIuOS9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDUyZTlkMmRlOGE1ZjkyNzMwN2Q5Mzk3MzA4NzM1NjU4ZWU0NGFiOGRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLXBsdWdpbnMtYW1kNjQtdjAuNy41LnRnegpDbHVzdGVyTmFtZTogbWl4ZWRpbnN0YW5jZXMuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL21peGVkaW5zdGFuY2VzLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCmV0Y2RNYW5pZmVzdHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9tYWluLnlhbWwKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL21peGVkaW5zdGFuY2VzLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.2.24 + main: + version: 3.2.24 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.12.9 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: mixedinstances.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.12.9 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.12.9 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.12.9 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + allowPrivileged: true + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + allowPrivileged: true + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- e914b17532c411cb7c0cc472131b61935fb66b31@https://storage.googleapis.com/kubernetes-release/release/v1.12.9/bin/linux/amd64/kubelet +- aa3e93897a6999d6c7dedbc41793c90d41eeb000@https://storage.googleapis.com/kubernetes-release/release/v1.12.9/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: mixedinstances.example.com +ConfigBase: memfs://clusters.example.com/mixedinstances.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1b.masters.mixedinstances.example.com_user_data b/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1b.masters.mixedinstances.example.com_user_data index 902fa42a16b9a..bc4f0084d91b5 100644 --- a/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1b.masters.mixedinstances.example.com_user_data +++ b/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1b.masters.mixedinstances.example.com_user_data @@ -1 +1,304 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjIuMjQKICBtYWluOgogICAgdmVyc2lvbjogMy4yLjI0Cmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMwogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTIuOQogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBtaXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbQogIGNvbmZpZ3VyZUNsb3VkUm91dGVzOiB0cnVlCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1jb250cm9sbGVyLW1hbmFnZXI6djEuMTIuOQogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgogIHVzZVNlcnZpY2VBY2NvdW50Q3JlZGVudGlhbHM6IHRydWUKa3ViZVByb3h5OgogIGNsdXN0ZXJDSURSOiAxMDAuOTYuMC4wLzExCiAgY3B1UmVxdWVzdDogMTAwbQogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtcHJveHk6djEuMTIuOQogIGxvZ0xldmVsOiAyCmt1YmVTY2hlZHVsZXI6CiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1zY2hlZHVsZXI6djEuMTIuOQogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgprdWJlbGV0OgogIGFsbG93UHJpdmlsZWdlZDogdHJ1ZQogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5NVFU6IDkwMDEKICBuZXR3b3JrUGx1Z2luTmFtZToga3ViZW5ldAogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKbWFzdGVyS3ViZWxldDoKICBhbGxvd1ByaXZpbGVnZWQ6IHRydWUKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCiAgcmVnaXN0ZXJTY2hlZHVsYWJsZTogZmFsc2UKCl9fRU9GX0NMVVNURVJfU1BFQwoKY2F0ID4gY29uZi9pZ19zcGVjLnlhbWwgPDwgJ19fRU9GX0lHX1NQRUMnCmt1YmVsZXQ6IG51bGwKbm9kZUxhYmVsczogbnVsbAp0YWludHM6IG51bGwKCl9fRU9GX0lHX1NQRUMKCmNhdCA+IGNvbmYva3ViZV9lbnYueWFtbCA8PCAnX19FT0ZfS1VCRV9FTlYnCkFzc2V0czoKLSBlOTE0YjE3NTMyYzQxMWNiN2MwY2M0NzIxMzFiNjE5MzVmYjY2YjMxQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xMi45L2Jpbi9saW51eC9hbWQ2NC9rdWJlbGV0Ci0gYWEzZTkzODk3YTY5OTlkNmM3ZGVkYmM0MTc5M2M5MGQ0MWVlYjAwMEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTIuOS9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDUyZTlkMmRlOGE1ZjkyNzMwN2Q5Mzk3MzA4NzM1NjU4ZWU0NGFiOGRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLXBsdWdpbnMtYW1kNjQtdjAuNy41LnRnegpDbHVzdGVyTmFtZTogbWl4ZWRpbnN0YW5jZXMuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWIKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL21peGVkaW5zdGFuY2VzLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCmV0Y2RNYW5pZmVzdHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9tYWluLnlhbWwKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL21peGVkaW5zdGFuY2VzLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.2.24 + main: + version: 3.2.24 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.12.9 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: mixedinstances.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.12.9 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.12.9 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.12.9 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + allowPrivileged: true + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + allowPrivileged: true + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- e914b17532c411cb7c0cc472131b61935fb66b31@https://storage.googleapis.com/kubernetes-release/release/v1.12.9/bin/linux/amd64/kubelet +- aa3e93897a6999d6c7dedbc41793c90d41eeb000@https://storage.googleapis.com/kubernetes-release/release/v1.12.9/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: mixedinstances.example.com +ConfigBase: memfs://clusters.example.com/mixedinstances.example.com +InstanceGroupName: master-us-test-1b +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1c.masters.mixedinstances.example.com_user_data b/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1c.masters.mixedinstances.example.com_user_data index 99d2235f70e33..f8ab7db3e8ab2 100644 --- a/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1c.masters.mixedinstances.example.com_user_data +++ b/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_master-us-test-1c.masters.mixedinstances.example.com_user_data @@ -1 +1,304 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjIuMjQKICBtYWluOgogICAgdmVyc2lvbjogMy4yLjI0Cmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMwogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTIuOQogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBtaXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbQogIGNvbmZpZ3VyZUNsb3VkUm91dGVzOiB0cnVlCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1jb250cm9sbGVyLW1hbmFnZXI6djEuMTIuOQogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgogIHVzZVNlcnZpY2VBY2NvdW50Q3JlZGVudGlhbHM6IHRydWUKa3ViZVByb3h5OgogIGNsdXN0ZXJDSURSOiAxMDAuOTYuMC4wLzExCiAgY3B1UmVxdWVzdDogMTAwbQogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtcHJveHk6djEuMTIuOQogIGxvZ0xldmVsOiAyCmt1YmVTY2hlZHVsZXI6CiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1zY2hlZHVsZXI6djEuMTIuOQogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgprdWJlbGV0OgogIGFsbG93UHJpdmlsZWdlZDogdHJ1ZQogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5NVFU6IDkwMDEKICBuZXR3b3JrUGx1Z2luTmFtZToga3ViZW5ldAogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKbWFzdGVyS3ViZWxldDoKICBhbGxvd1ByaXZpbGVnZWQ6IHRydWUKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCiAgcmVnaXN0ZXJTY2hlZHVsYWJsZTogZmFsc2UKCl9fRU9GX0NMVVNURVJfU1BFQwoKY2F0ID4gY29uZi9pZ19zcGVjLnlhbWwgPDwgJ19fRU9GX0lHX1NQRUMnCmt1YmVsZXQ6IG51bGwKbm9kZUxhYmVsczogbnVsbAp0YWludHM6IG51bGwKCl9fRU9GX0lHX1NQRUMKCmNhdCA+IGNvbmYva3ViZV9lbnYueWFtbCA8PCAnX19FT0ZfS1VCRV9FTlYnCkFzc2V0czoKLSBlOTE0YjE3NTMyYzQxMWNiN2MwY2M0NzIxMzFiNjE5MzVmYjY2YjMxQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xMi45L2Jpbi9saW51eC9hbWQ2NC9rdWJlbGV0Ci0gYWEzZTkzODk3YTY5OTlkNmM3ZGVkYmM0MTc5M2M5MGQ0MWVlYjAwMEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTIuOS9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDUyZTlkMmRlOGE1ZjkyNzMwN2Q5Mzk3MzA4NzM1NjU4ZWU0NGFiOGRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLXBsdWdpbnMtYW1kNjQtdjAuNy41LnRnegpDbHVzdGVyTmFtZTogbWl4ZWRpbnN0YW5jZXMuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWMKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL21peGVkaW5zdGFuY2VzLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCmV0Y2RNYW5pZmVzdHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9tYWluLnlhbWwKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL21peGVkaW5zdGFuY2VzLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.2.24 + main: + version: 3.2.24 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 3 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.12.9 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: mixedinstances.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.12.9 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.12.9 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.12.9 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + allowPrivileged: true + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + allowPrivileged: true + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- e914b17532c411cb7c0cc472131b61935fb66b31@https://storage.googleapis.com/kubernetes-release/release/v1.12.9/bin/linux/amd64/kubelet +- aa3e93897a6999d6c7dedbc41793c90d41eeb000@https://storage.googleapis.com/kubernetes-release/release/v1.12.9/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: mixedinstances.example.com +ConfigBase: memfs://clusters.example.com/mixedinstances.example.com +InstanceGroupName: master-us-test-1c +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/mixedinstances.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_nodes.mixedinstances.example.com_user_data b/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_nodes.mixedinstances.example.com_user_data index 6fee2cc7e2fed..9aef29bbcc73c 100644 --- a/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_nodes.mixedinstances.example.com_user_data +++ b/tests/integration/update_cluster/mixed_instances_spot/data/aws_launch_template_nodes.mixedinstances.example.com_user_data @@ -1 +1,210 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xMi45CiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbGxvd1ByaXZpbGVnZWQ6IHRydWUKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gZTkxNGIxNzUzMmM0MTFjYjdjMGNjNDcyMTMxYjYxOTM1ZmI2NmIzMUBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTIuOS9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIGFhM2U5Mzg5N2E2OTk5ZDZjN2RlZGJjNDE3OTNjOTBkNDFlZWIwMDBAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjEyLjkvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IG1peGVkaW5zdGFuY2VzLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vbWl4ZWRpbnN0YW5jZXMuZXhhbXBsZS5jb20KSW5zdGFuY2VHcm91cE5hbWU6IG5vZGVzClRhZ3M6Ci0gX2F1dG9tYXRpY191cGdyYWRlcwotIF9hd3MKY2hhbm5lbHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9taXhlZGluc3RhbmNlcy5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbAoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg== +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.12.9 + logLevel: 2 +kubelet: + allowPrivileged: true + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- e914b17532c411cb7c0cc472131b61935fb66b31@https://storage.googleapis.com/kubernetes-release/release/v1.12.9/bin/linux/amd64/kubelet +- aa3e93897a6999d6c7dedbc41793c90d41eeb000@https://storage.googleapis.com/kubernetes-release/release/v1.12.9/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: mixedinstances.example.com +ConfigBase: memfs://clusters.example.com/mixedinstances.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/mixedinstances.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/nosshkey/data/aws_launch_template_master-us-test-1a.masters.nosshkey.example.com_user_data b/tests/integration/update_cluster/nosshkey/data/aws_launch_template_master-us-test-1a.masters.nosshkey.example.com_user_data index 6b85fcab9e6e6..b7b61a98835fb 100644 --- a/tests/integration/update_cluster/nosshkey/data/aws_launch_template_master-us-test-1a.masters.nosshkey.example.com_user_data +++ b/tests/integration/update_cluster/nosshkey/data/aws_launch_template_master-us-test-1a.masters.nosshkey.example.com_user_data @@ -1 +1,306 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTcuMDMuMgplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICBpbWFnZTogazhzLmdjci5pby9ldGNkOjIuMi4xCiAgICB2ZXJzaW9uOiAyLjIuMQogIG1haW46CiAgICBpbWFnZTogazhzLmdjci5pby9ldGNkOjIuMi4xCiAgICB2ZXJzaW9uOiAyLjIuMQprdWJlQVBJU2VydmVyOgogIGFsbG93UHJpdmlsZWdlZDogdHJ1ZQogIGFub255bW91c0F1dGg6IGZhbHNlCiAgYXBpU2VydmVyQ291bnQ6IDEKICBhdXRob3JpemF0aW9uTW9kZTogQWx3YXlzQWxsb3cKICBiaW5kQWRkcmVzczogMC4wLjAuMAogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGVuYWJsZUFkbWlzc2lvblBsdWdpbnM6CiAgLSBJbml0aWFsaXplcnMKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFF1b3J1bVJlYWQ6IGZhbHNlCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTEuMTAKICBpbnNlY3VyZUJpbmRBZGRyZXNzOiAxMjcuMC4wLjEKICBpbnNlY3VyZVBvcnQ6IDgwODAKICBrdWJlbGV0UHJlZmVycmVkQWRkcmVzc1R5cGVzOgogIC0gSW50ZXJuYWxJUAogIC0gSG9zdG5hbWUKICAtIEV4dGVybmFsSVAKICBsb2dMZXZlbDogMgogIHJlcXVlc3RoZWFkZXJBbGxvd2VkTmFtZXM6CiAgLSBhZ2dyZWdhdG9yCiAgcmVxdWVzdGhlYWRlckV4dHJhSGVhZGVyUHJlZml4ZXM6CiAgLSBYLVJlbW90ZS1FeHRyYS0KICByZXF1ZXN0aGVhZGVyR3JvdXBIZWFkZXJzOgogIC0gWC1SZW1vdGUtR3JvdXAKICByZXF1ZXN0aGVhZGVyVXNlcm5hbWVIZWFkZXJzOgogIC0gWC1SZW1vdGUtVXNlcgogIHNlY3VyZVBvcnQ6IDQ0MwogIHNlcnZpY2VDbHVzdGVySVBSYW5nZTogMTAwLjY0LjAuMC8xMwogIHNlcnZpY2VOb2RlUG9ydFJhbmdlOiAyODAwMC0zMjc2NwogIHN0b3JhZ2VCYWNrZW5kOiBldGNkMgprdWJlQ29udHJvbGxlck1hbmFnZXI6CiAgYWxsb2NhdGVOb2RlQ0lEUnM6IHRydWUKICBhdHRhY2hEZXRhY2hSZWNvbmNpbGVTeW5jUGVyaW9kOiAxbTBzCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjbHVzdGVyTmFtZTogbm9zc2hrZXkuZXhhbXBsZS5jb20KICBjb25maWd1cmVDbG91ZFJvdXRlczogdHJ1ZQogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjExLjEwCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCiAgdXNlU2VydmljZUFjY291bnRDcmVkZW50aWFsczogdHJ1ZQprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xMS4xMAogIGxvZ0xldmVsOiAyCmt1YmVTY2hlZHVsZXI6CiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1zY2hlZHVsZXI6djEuMTEuMTAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbGxvd1ByaXZpbGVnZWQ6IHRydWUKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCm1hc3Rlckt1YmVsZXQ6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk1UVTogOTAwMQogIG5ldHdvcmtQbHVnaW5OYW1lOiBrdWJlbmV0CiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwogIHJlZ2lzdGVyU2NoZWR1bGFibGU6IGZhbHNlCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYTFlNWQyYTdkYTRjYWJjMjlhZjBkZGE2MzA1NjQ1MTFhOWI0MzdkOEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTEuMTAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSBjMTMzZjU1MTUyYzc2YzMzZDliNDE4OTRkY2QzMTEwNjQ5MDQ1MDNlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xMS4xMC9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDUyZTlkMmRlOGE1ZjkyNzMwN2Q5Mzk3MzA4NzM1NjU4ZWU0NGFiOGRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLXBsdWdpbnMtYW1kNjQtdjAuNy41LnRnegpDbHVzdGVyTmFtZTogbm9zc2hrZXkuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9ub3NzaGtleS5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL25vc3Noa2V5LmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 17.03.2 +encryptionConfig: null +etcdClusters: + events: + image: k8s.gcr.io/etcd:2.2.1 + version: 2.2.1 + main: + image: k8s.gcr.io/etcd:2.2.1 + version: 2.2.1 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - Initializers + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdQuorumRead: false + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.11.10 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + serviceNodePortRange: 28000-32767 + storageBackend: etcd2 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: nosshkey.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.11.10 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.11.10 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.11.10 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + allowPrivileged: true + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + allowPrivileged: true + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- a1e5d2a7da4cabc29af0dda630564511a9b437d8@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/amd64/kubelet +- c133f55152c76c33d9b41894dcd311064904503e@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: nosshkey.example.com +ConfigBase: memfs://clusters.example.com/nosshkey.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/nosshkey.example.com/addons/bootstrap-channel.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/nosshkey/data/aws_launch_template_nodes.nosshkey.example.com_user_data b/tests/integration/update_cluster/nosshkey/data/aws_launch_template_nodes.nosshkey.example.com_user_data index be907abe1c492..af21eeaab5b73 100644 --- a/tests/integration/update_cluster/nosshkey/data/aws_launch_template_nodes.nosshkey.example.com_user_data +++ b/tests/integration/update_cluster/nosshkey/data/aws_launch_template_nodes.nosshkey.example.com_user_data @@ -1 +1,210 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTcuMDMuMgprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xMS4xMAogIGxvZ0xldmVsOiAyCmt1YmVsZXQ6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk1UVTogOTAwMQogIG5ldHdvcmtQbHVnaW5OYW1lOiBrdWJlbmV0CiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGExZTVkMmE3ZGE0Y2FiYzI5YWYwZGRhNjMwNTY0NTExYTliNDM3ZDhAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjExLjEwL2Jpbi9saW51eC9hbWQ2NC9rdWJlbGV0Ci0gYzEzM2Y1NTE1MmM3NmMzM2Q5YjQxODk0ZGNkMzExMDY0OTA0NTAzZUBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTEuMTAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IG5vc3Noa2V5LmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vbm9zc2hrZXkuZXhhbXBsZS5jb20KSW5zdGFuY2VHcm91cE5hbWU6IG5vZGVzClRhZ3M6Ci0gX2F1dG9tYXRpY191cGdyYWRlcwotIF9hd3MKY2hhbm5lbHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9ub3NzaGtleS5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbAoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg== +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 17.03.2 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.11.10 + logLevel: 2 +kubelet: + allowPrivileged: true + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- a1e5d2a7da4cabc29af0dda630564511a9b437d8@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/amd64/kubelet +- c133f55152c76c33d9b41894dcd311064904503e@https://storage.googleapis.com/kubernetes-release/release/v1.11.10/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: nosshkey.example.com +ConfigBase: memfs://clusters.example.com/nosshkey.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/nosshkey.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/private-shared-subnet/data/aws_launch_template_master-us-test-1a.masters.private-shared-subnet.example.com_user_data b/tests/integration/update_cluster/private-shared-subnet/data/aws_launch_template_master-us-test-1a.masters.private-shared-subnet.example.com_user_data index 685dc8c03ad73..c864523e24e8b 100644 --- a/tests/integration/update_cluster/private-shared-subnet/data/aws_launch_template_master-us-test-1a.masters.private-shared-subnet.example.com_user_data +++ b/tests/integration/update_cluster/private-shared-subnet/data/aws_launch_template_master-us-test-1a.masters.private-shared-subnet.example.com_user_data @@ -1 +1,300 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBwcml2YXRlLXNoYXJlZC1zdWJuZXQuZXhhbXBsZS5jb20KICBjb25maWd1cmVDbG91ZFJvdXRlczogZmFsc2UKICBpbWFnZTogazhzLmdjci5pby9rdWJlLWNvbnRyb2xsZXItbWFuYWdlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCiAgdXNlU2VydmljZUFjY291bnRDcmVkZW50aWFsczogdHJ1ZQprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZVNjaGVkdWxlcjoKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXNjaGVkdWxlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCmt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk5hbWU6IGNuaQogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKbWFzdGVyS3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwogIHJlZ2lzdGVyU2NoZWR1bGFibGU6IGZhbHNlCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IHByaXZhdGUtc2hhcmVkLXN1Ym5ldC5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGUtc2hhcmVkLXN1Ym5ldC5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGUtc2hhcmVkLXN1Ym5ldC5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbApldGNkTWFuaWZlc3RzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vcHJpdmF0ZS1zaGFyZWQtc3VibmV0LmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL21haW4ueWFtbAotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vcHJpdmF0ZS1zaGFyZWQtc3VibmV0LmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: private-shared-subnet.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: private-shared-subnet.example.com +ConfigBase: memfs://clusters.example.com/private-shared-subnet.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/private-shared-subnet.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/private-shared-subnet.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/private-shared-subnet.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/private-shared-subnet/data/aws_launch_template_nodes.private-shared-subnet.example.com_user_data b/tests/integration/update_cluster/private-shared-subnet/data/aws_launch_template_nodes.private-shared-subnet.example.com_user_data index 7c3395d68f33d..3ba4d732c98aa 100644 --- a/tests/integration/update_cluster/private-shared-subnet/data/aws_launch_template_nodes.private-shared-subnet.example.com_user_data +++ b/tests/integration/update_cluster/private-shared-subnet/data/aws_launch_template_nodes.private-shared-subnet.example.com_user_data @@ -1 +1,208 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGMzYjczNmZkMGYwMDM3NjVjMTJkOTlmMmM5OTVhODM2OWU2MjQxZjRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA3ZTNhM2VhNjYzMTUzZjkwMGNiZDUyOTAwYTM5YzkxZmE5ZjMzNGJlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlY3RsCi0gNTJlOWQyZGU4YTVmOTI3MzA3ZDkzOTczMDg3MzU2NThlZTQ0YWI4ZEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL25ldHdvcmstcGx1Z2lucy9jbmktcGx1Z2lucy1hbWQ2NC12MC43LjUudGd6CkNsdXN0ZXJOYW1lOiBwcml2YXRlLXNoYXJlZC1zdWJuZXQuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9wcml2YXRlLXNoYXJlZC1zdWJuZXQuZXhhbXBsZS5jb20KSW5zdGFuY2VHcm91cE5hbWU6IG5vZGVzClRhZ3M6Ci0gX2F1dG9tYXRpY191cGdyYWRlcwotIF9hd3MKY2hhbm5lbHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9wcml2YXRlLXNoYXJlZC1zdWJuZXQuZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: private-shared-subnet.example.com +ConfigBase: memfs://clusters.example.com/private-shared-subnet.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/private-shared-subnet.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatecalico/data/aws_launch_template_master-us-test-1a.masters.privatecalico.example.com_user_data b/tests/integration/update_cluster/privatecalico/data/aws_launch_template_master-us-test-1a.masters.privatecalico.example.com_user_data index 2e59aaa83b1a7..7d19d82e3b805 100644 --- a/tests/integration/update_cluster/privatecalico/data/aws_launch_template_master-us-test-1a.masters.privatecalico.example.com_user_data +++ b/tests/integration/update_cluster/privatecalico/data/aws_launch_template_master-us-test-1a.masters.privatecalico.example.com_user_data @@ -1 +1,300 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBwcml2YXRlY2FsaWNvLmV4YW1wbGUuY29tCiAgY29uZmlndXJlQ2xvdWRSb3V0ZXM6IGZhbHNlCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1jb250cm9sbGVyLW1hbmFnZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgogIHVzZVNlcnZpY2VBY2NvdW50Q3JlZGVudGlhbHM6IHRydWUKa3ViZVByb3h5OgogIGNsdXN0ZXJDSURSOiAxMDAuOTYuMC4wLzExCiAgY3B1UmVxdWVzdDogMTAwbQogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtcHJveHk6djEuMTQuMAogIGxvZ0xldmVsOiAyCmt1YmVTY2hlZHVsZXI6CiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1zY2hlZHVsZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgprdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5OYW1lOiBjbmkKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCm1hc3Rlckt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk5hbWU6IGNuaQogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKICByZWdpc3RlclNjaGVkdWxhYmxlOiBmYWxzZQoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGMzYjczNmZkMGYwMDM3NjVjMTJkOTlmMmM5OTVhODM2OWU2MjQxZjRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA3ZTNhM2VhNjYzMTUzZjkwMGNiZDUyOTAwYTM5YzkxZmE5ZjMzNGJlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlY3RsCi0gNTJlOWQyZGU4YTVmOTI3MzA3ZDkzOTczMDg3MzU2NThlZTQ0YWI4ZEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL25ldHdvcmstcGx1Z2lucy9jbmktcGx1Z2lucy1hbWQ2NC12MC43LjUudGd6CkNsdXN0ZXJOYW1lOiBwcml2YXRlY2FsaWNvLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vcHJpdmF0ZWNhbGljby5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVjYWxpY28uZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKZXRjZE1hbmlmZXN0czoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVjYWxpY28uZXhhbXBsZS5jb20vbWFuaWZlc3RzL2V0Y2QvbWFpbi55YW1sCi0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9wcml2YXRlY2FsaWNvLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: privatecalico.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: privatecalico.example.com +ConfigBase: memfs://clusters.example.com/privatecalico.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/privatecalico.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/privatecalico.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/privatecalico.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatecalico/data/aws_launch_template_nodes.privatecalico.example.com_user_data b/tests/integration/update_cluster/privatecalico/data/aws_launch_template_nodes.privatecalico.example.com_user_data index 41ab38aa7b07f..40944928fbf54 100644 --- a/tests/integration/update_cluster/privatecalico/data/aws_launch_template_nodes.privatecalico.example.com_user_data +++ b/tests/integration/update_cluster/privatecalico/data/aws_launch_template_nodes.privatecalico.example.com_user_data @@ -1 +1,208 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGMzYjczNmZkMGYwMDM3NjVjMTJkOTlmMmM5OTVhODM2OWU2MjQxZjRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA3ZTNhM2VhNjYzMTUzZjkwMGNiZDUyOTAwYTM5YzkxZmE5ZjMzNGJlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlY3RsCi0gNTJlOWQyZGU4YTVmOTI3MzA3ZDkzOTczMDg3MzU2NThlZTQ0YWI4ZEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL25ldHdvcmstcGx1Z2lucy9jbmktcGx1Z2lucy1hbWQ2NC12MC43LjUudGd6CkNsdXN0ZXJOYW1lOiBwcml2YXRlY2FsaWNvLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vcHJpdmF0ZWNhbGljby5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbm9kZXMKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVjYWxpY28uZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: privatecalico.example.com +ConfigBase: memfs://clusters.example.com/privatecalico.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/privatecalico.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatecanal/data/aws_launch_template_master-us-test-1a.masters.privatecanal.example.com_user_data b/tests/integration/update_cluster/privatecanal/data/aws_launch_template_master-us-test-1a.masters.privatecanal.example.com_user_data index a6a8e2fd965fe..27d6cae15acca 100644 --- a/tests/integration/update_cluster/privatecanal/data/aws_launch_template_master-us-test-1a.masters.privatecanal.example.com_user_data +++ b/tests/integration/update_cluster/privatecanal/data/aws_launch_template_master-us-test-1a.masters.privatecanal.example.com_user_data @@ -1 +1,300 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBwcml2YXRlY2FuYWwuZXhhbXBsZS5jb20KICBjb25maWd1cmVDbG91ZFJvdXRlczogZmFsc2UKICBpbWFnZTogazhzLmdjci5pby9rdWJlLWNvbnRyb2xsZXItbWFuYWdlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCiAgdXNlU2VydmljZUFjY291bnRDcmVkZW50aWFsczogdHJ1ZQprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZVNjaGVkdWxlcjoKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXNjaGVkdWxlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCmt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk5hbWU6IGNuaQogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKbWFzdGVyS3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwogIHJlZ2lzdGVyU2NoZWR1bGFibGU6IGZhbHNlCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IHByaXZhdGVjYW5hbC5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVjYW5hbC5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVjYW5hbC5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbApldGNkTWFuaWZlc3RzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vcHJpdmF0ZWNhbmFsLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL21haW4ueWFtbAotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vcHJpdmF0ZWNhbmFsLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: privatecanal.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: privatecanal.example.com +ConfigBase: memfs://clusters.example.com/privatecanal.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/privatecanal.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/privatecanal.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/privatecanal.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatecanal/data/aws_launch_template_nodes.privatecanal.example.com_user_data b/tests/integration/update_cluster/privatecanal/data/aws_launch_template_nodes.privatecanal.example.com_user_data index 9bf869bd205b9..918955d585cab 100644 --- a/tests/integration/update_cluster/privatecanal/data/aws_launch_template_nodes.privatecanal.example.com_user_data +++ b/tests/integration/update_cluster/privatecanal/data/aws_launch_template_nodes.privatecanal.example.com_user_data @@ -1 +1,208 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGMzYjczNmZkMGYwMDM3NjVjMTJkOTlmMmM5OTVhODM2OWU2MjQxZjRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA3ZTNhM2VhNjYzMTUzZjkwMGNiZDUyOTAwYTM5YzkxZmE5ZjMzNGJlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlY3RsCi0gNTJlOWQyZGU4YTVmOTI3MzA3ZDkzOTczMDg3MzU2NThlZTQ0YWI4ZEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL25ldHdvcmstcGx1Z2lucy9jbmktcGx1Z2lucy1hbWQ2NC12MC43LjUudGd6CkNsdXN0ZXJOYW1lOiBwcml2YXRlY2FuYWwuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9wcml2YXRlY2FuYWwuZXhhbXBsZS5jb20KSW5zdGFuY2VHcm91cE5hbWU6IG5vZGVzClRhZ3M6Ci0gX2F1dG9tYXRpY191cGdyYWRlcwotIF9hd3MKY2hhbm5lbHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9wcml2YXRlY2FuYWwuZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: privatecanal.example.com +ConfigBase: memfs://clusters.example.com/privatecanal.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/privatecanal.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatecilium/data/aws_launch_template_master-us-test-1a.masters.privatecilium.example.com_user_data b/tests/integration/update_cluster/privatecilium/data/aws_launch_template_master-us-test-1a.masters.privatecilium.example.com_user_data index 1b91b46fa47f9..dcf7e6df35db3 100644 --- a/tests/integration/update_cluster/privatecilium/data/aws_launch_template_master-us-test-1a.masters.privatecilium.example.com_user_data +++ b/tests/integration/update_cluster/privatecilium/data/aws_launch_template_master-us-test-1a.masters.privatecilium.example.com_user_data @@ -1 +1,300 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBwcml2YXRlY2lsaXVtLmV4YW1wbGUuY29tCiAgY29uZmlndXJlQ2xvdWRSb3V0ZXM6IGZhbHNlCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1jb250cm9sbGVyLW1hbmFnZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgogIHVzZVNlcnZpY2VBY2NvdW50Q3JlZGVudGlhbHM6IHRydWUKa3ViZVByb3h5OgogIGNsdXN0ZXJDSURSOiAxMDAuOTYuMC4wLzExCiAgY3B1UmVxdWVzdDogMTAwbQogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtcHJveHk6djEuMTQuMAogIGxvZ0xldmVsOiAyCmt1YmVTY2hlZHVsZXI6CiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1zY2hlZHVsZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgprdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5OYW1lOiBjbmkKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCm1hc3Rlckt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk5hbWU6IGNuaQogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKICByZWdpc3RlclNjaGVkdWxhYmxlOiBmYWxzZQoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGMzYjczNmZkMGYwMDM3NjVjMTJkOTlmMmM5OTVhODM2OWU2MjQxZjRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA3ZTNhM2VhNjYzMTUzZjkwMGNiZDUyOTAwYTM5YzkxZmE5ZjMzNGJlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlY3RsCi0gNTJlOWQyZGU4YTVmOTI3MzA3ZDkzOTczMDg3MzU2NThlZTQ0YWI4ZEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL25ldHdvcmstcGx1Z2lucy9jbmktcGx1Z2lucy1hbWQ2NC12MC43LjUudGd6CkNsdXN0ZXJOYW1lOiBwcml2YXRlY2lsaXVtLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vcHJpdmF0ZWNpbGl1bS5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVjaWxpdW0uZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKZXRjZE1hbmlmZXN0czoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVjaWxpdW0uZXhhbXBsZS5jb20vbWFuaWZlc3RzL2V0Y2QvbWFpbi55YW1sCi0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9wcml2YXRlY2lsaXVtLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: privatecilium.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: privatecilium.example.com +ConfigBase: memfs://clusters.example.com/privatecilium.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/privatecilium.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/privatecilium.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/privatecilium.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatecilium/data/aws_launch_template_nodes.privatecilium.example.com_user_data b/tests/integration/update_cluster/privatecilium/data/aws_launch_template_nodes.privatecilium.example.com_user_data index 3f28a81c00e50..4f919f9694512 100644 --- a/tests/integration/update_cluster/privatecilium/data/aws_launch_template_nodes.privatecilium.example.com_user_data +++ b/tests/integration/update_cluster/privatecilium/data/aws_launch_template_nodes.privatecilium.example.com_user_data @@ -1 +1,208 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGMzYjczNmZkMGYwMDM3NjVjMTJkOTlmMmM5OTVhODM2OWU2MjQxZjRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA3ZTNhM2VhNjYzMTUzZjkwMGNiZDUyOTAwYTM5YzkxZmE5ZjMzNGJlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlY3RsCi0gNTJlOWQyZGU4YTVmOTI3MzA3ZDkzOTczMDg3MzU2NThlZTQ0YWI4ZEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL25ldHdvcmstcGx1Z2lucy9jbmktcGx1Z2lucy1hbWQ2NC12MC43LjUudGd6CkNsdXN0ZXJOYW1lOiBwcml2YXRlY2lsaXVtLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vcHJpdmF0ZWNpbGl1bS5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbm9kZXMKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVjaWxpdW0uZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: privatecilium.example.com +ConfigBase: memfs://clusters.example.com/privatecilium.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/privatecilium.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_master-us-test-1a.masters.privateciliumadvanced.example.com_user_data b/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_master-us-test-1a.masters.privateciliumadvanced.example.com_user_data index d6927de1d7586..64394ff9d03b4 100644 --- a/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_master-us-test-1a.masters.privateciliumadvanced.example.com_user_data +++ b/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_master-us-test-1a.masters.privateciliumadvanced.example.com_user_data @@ -1 +1,304 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBjaWxpdW06CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBwcml2YXRlY2lsaXVtYWR2YW5jZWQuZXhhbXBsZS5jb20KICBjb25maWd1cmVDbG91ZFJvdXRlczogZmFsc2UKICBpbWFnZTogazhzLmdjci5pby9rdWJlLWNvbnRyb2xsZXItbWFuYWdlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCiAgdXNlU2VydmljZUFjY291bnRDcmVkZW50aWFsczogdHJ1ZQprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgZW5hYmxlZDogZmFsc2UKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXByb3h5OnYxLjE0LjAKICBsb2dMZXZlbDogMgprdWJlU2NoZWR1bGVyOgogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwptYXN0ZXJLdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5OYW1lOiBjbmkKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCiAgcmVnaXN0ZXJTY2hlZHVsYWJsZTogZmFsc2UKCl9fRU9GX0NMVVNURVJfU1BFQwoKY2F0ID4gY29uZi9pZ19zcGVjLnlhbWwgPDwgJ19fRU9GX0lHX1NQRUMnCmt1YmVsZXQ6IG51bGwKbm9kZUxhYmVsczogbnVsbAp0YWludHM6IG51bGwKCl9fRU9GX0lHX1NQRUMKCmNhdCA+IGNvbmYva3ViZV9lbnYueWFtbCA8PCAnX19FT0ZfS1VCRV9FTlYnCkFzc2V0czoKLSBjM2I3MzZmZDBmMDAzNzY1YzEyZDk5ZjJjOTk1YTgzNjllNjI0MWY0QGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlbGV0Ci0gN2UzYTNlYTY2MzE1M2Y5MDBjYmQ1MjkwMGEzOWM5MWZhOWYzMzRiZUBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDUyZTlkMmRlOGE1ZjkyNzMwN2Q5Mzk3MzA4NzM1NjU4ZWU0NGFiOGRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLXBsdWdpbnMtYW1kNjQtdjAuNy41LnRnegpDbHVzdGVyTmFtZTogcHJpdmF0ZWNpbGl1bWFkdmFuY2VkLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vcHJpdmF0ZWNpbGl1bWFkdmFuY2VkLmV4YW1wbGUuY29tCkluc3RhbmNlR3JvdXBOYW1lOiBtYXN0ZXItdXMtdGVzdC0xYQpUYWdzOgotIF9hdXRvbWF0aWNfdXBncmFkZXMKLSBfYXdzCmNoYW5uZWxzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vcHJpdmF0ZWNpbGl1bWFkdmFuY2VkLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCmV0Y2RNYW5pZmVzdHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9wcml2YXRlY2lsaXVtYWR2YW5jZWQuZXhhbXBsZS5jb20vbWFuaWZlc3RzL2V0Y2QvbWFpbi55YW1sCi0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9wcml2YXRlY2lsaXVtYWR2YW5jZWQuZXhhbXBsZS5jb20vbWFuaWZlc3RzL2V0Y2QvZXZlbnRzLnlhbWwKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVjaWxpdW1hZHZhbmNlZC5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9jaWxpdW0ueWFtbApwcm90b2t1YmVJbWFnZToKICBoYXNoOiA0MmE5YzQzMjRmZTI2ZDYzY2UxMWYzZGQ3ODM2MzcxYmM5M2ZhMDZjYThmNDc5ODA3NzI4ZjM3NDZlMjcwNjFiCiAgbmFtZTogcHJvdG9rdWJlOjEuMTUuMAogIHNvdXJjZXM6CiAgLSBodHRwczovL2FydGlmYWN0cy5rOHMuaW8vYmluYXJpZXMva29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKICAtIGh0dHBzOi8vZ2l0aHViLmNvbS9rdWJlcm5ldGVzL2tvcHMvcmVsZWFzZXMvZG93bmxvYWQvdjEuMTUuMC9pbWFnZXMtcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9rdWJldXB2Mi5zMy5hbWF6b25hd3MuY29tL2tvcHMvMS4xNS4wL2ltYWdlcy9wcm90b2t1YmUudGFyLmd6CgpfX0VPRl9LVUJFX0VOVgoKZG93bmxvYWQtcmVsZWFzZQplY2hvICI9PSBub2RldXAgbm9kZSBjb25maWcgZG9uZSA9PSIK +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + cilium: + version: 3.3.10 + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: privateciliumadvanced.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + enabled: false + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: privateciliumadvanced.example.com +ConfigBase: memfs://clusters.example.com/privateciliumadvanced.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/privateciliumadvanced.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/privateciliumadvanced.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/privateciliumadvanced.example.com/manifests/etcd/events.yaml +- memfs://clusters.example.com/privateciliumadvanced.example.com/manifests/etcd/cilium.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_nodes.privateciliumadvanced.example.com_user_data b/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_nodes.privateciliumadvanced.example.com_user_data index b2ec7cb2a7318..48c84d385baa8 100644 --- a/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_nodes.privateciliumadvanced.example.com_user_data +++ b/tests/integration/update_cluster/privateciliumadvanced/data/aws_launch_template_nodes.privateciliumadvanced.example.com_user_data @@ -1 +1,209 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgZW5hYmxlZDogZmFsc2UKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXByb3h5OnYxLjE0LjAKICBsb2dMZXZlbDogMgprdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5OYW1lOiBjbmkKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IHByaXZhdGVjaWxpdW1hZHZhbmNlZC5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVjaWxpdW1hZHZhbmNlZC5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbm9kZXMKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVjaWxpdW1hZHZhbmNlZC5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbAoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg== +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + enabled: false + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: privateciliumadvanced.example.com +ConfigBase: memfs://clusters.example.com/privateciliumadvanced.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/privateciliumadvanced.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatedns1/data/aws_launch_template_master-us-test-1a.masters.privatedns1.example.com_user_data b/tests/integration/update_cluster/privatedns1/data/aws_launch_template_master-us-test-1a.masters.privatedns1.example.com_user_data index 503e31df00c3b..ba9796051a6b3 100644 --- a/tests/integration/update_cluster/privatedns1/data/aws_launch_template_master-us-test-1a.masters.privatedns1.example.com_user_data +++ b/tests/integration/update_cluster/privatedns1/data/aws_launch_template_master-us-test-1a.masters.privatedns1.example.com_user_data @@ -1 +1,300 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBwcml2YXRlZG5zMS5leGFtcGxlLmNvbQogIGNvbmZpZ3VyZUNsb3VkUm91dGVzOiBmYWxzZQogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKICB1c2VTZXJ2aWNlQWNjb3VudENyZWRlbnRpYWxzOiB0cnVlCmt1YmVQcm94eToKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNwdVJlcXVlc3Q6IDEwMG0KICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXByb3h5OnYxLjE0LjAKICBsb2dMZXZlbDogMgprdWJlU2NoZWR1bGVyOgogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwptYXN0ZXJLdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5OYW1lOiBjbmkKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCiAgcmVnaXN0ZXJTY2hlZHVsYWJsZTogZmFsc2UKCl9fRU9GX0NMVVNURVJfU1BFQwoKY2F0ID4gY29uZi9pZ19zcGVjLnlhbWwgPDwgJ19fRU9GX0lHX1NQRUMnCmt1YmVsZXQ6IG51bGwKbm9kZUxhYmVsczogbnVsbAp0YWludHM6IG51bGwKCl9fRU9GX0lHX1NQRUMKCmNhdCA+IGNvbmYva3ViZV9lbnYueWFtbCA8PCAnX19FT0ZfS1VCRV9FTlYnCkFzc2V0czoKLSBjM2I3MzZmZDBmMDAzNzY1YzEyZDk5ZjJjOTk1YTgzNjllNjI0MWY0QGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlbGV0Ci0gN2UzYTNlYTY2MzE1M2Y5MDBjYmQ1MjkwMGEzOWM5MWZhOWYzMzRiZUBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDUyZTlkMmRlOGE1ZjkyNzMwN2Q5Mzk3MzA4NzM1NjU4ZWU0NGFiOGRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLXBsdWdpbnMtYW1kNjQtdjAuNy41LnRnegpDbHVzdGVyTmFtZTogcHJpdmF0ZWRuczEuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9wcml2YXRlZG5zMS5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVkbnMxLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCmV0Y2RNYW5pZmVzdHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9wcml2YXRlZG5zMS5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9tYWluLnlhbWwKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVkbnMxLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: privatedns1.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: privatedns1.example.com +ConfigBase: memfs://clusters.example.com/privatedns1.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/privatedns1.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/privatedns1.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/privatedns1.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatedns1/data/aws_launch_template_nodes.privatedns1.example.com_user_data b/tests/integration/update_cluster/privatedns1/data/aws_launch_template_nodes.privatedns1.example.com_user_data index f98b09e424c58..2a9ca5e87ae29 100644 --- a/tests/integration/update_cluster/privatedns1/data/aws_launch_template_nodes.privatedns1.example.com_user_data +++ b/tests/integration/update_cluster/privatedns1/data/aws_launch_template_nodes.privatedns1.example.com_user_data @@ -1 +1,208 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGMzYjczNmZkMGYwMDM3NjVjMTJkOTlmMmM5OTVhODM2OWU2MjQxZjRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA3ZTNhM2VhNjYzMTUzZjkwMGNiZDUyOTAwYTM5YzkxZmE5ZjMzNGJlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlY3RsCi0gNTJlOWQyZGU4YTVmOTI3MzA3ZDkzOTczMDg3MzU2NThlZTQ0YWI4ZEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL25ldHdvcmstcGx1Z2lucy9jbmktcGx1Z2lucy1hbWQ2NC12MC43LjUudGd6CkNsdXN0ZXJOYW1lOiBwcml2YXRlZG5zMS5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVkbnMxLmV4YW1wbGUuY29tCkluc3RhbmNlR3JvdXBOYW1lOiBub2RlcwpUYWdzOgotIF9hdXRvbWF0aWNfdXBncmFkZXMKLSBfYXdzCmNoYW5uZWxzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vcHJpdmF0ZWRuczEuZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: privatedns1.example.com +ConfigBase: memfs://clusters.example.com/privatedns1.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/privatedns1.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatedns2/data/aws_launch_template_master-us-test-1a.masters.privatedns2.example.com_user_data b/tests/integration/update_cluster/privatedns2/data/aws_launch_template_master-us-test-1a.masters.privatedns2.example.com_user_data index 9795e0755dc09..9af9f745488bd 100644 --- a/tests/integration/update_cluster/privatedns2/data/aws_launch_template_master-us-test-1a.masters.privatedns2.example.com_user_data +++ b/tests/integration/update_cluster/privatedns2/data/aws_launch_template_master-us-test-1a.masters.privatedns2.example.com_user_data @@ -1 +1,300 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBwcml2YXRlZG5zMi5leGFtcGxlLmNvbQogIGNvbmZpZ3VyZUNsb3VkUm91dGVzOiBmYWxzZQogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKICB1c2VTZXJ2aWNlQWNjb3VudENyZWRlbnRpYWxzOiB0cnVlCmt1YmVQcm94eToKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNwdVJlcXVlc3Q6IDEwMG0KICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXByb3h5OnYxLjE0LjAKICBsb2dMZXZlbDogMgprdWJlU2NoZWR1bGVyOgogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwptYXN0ZXJLdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5OYW1lOiBjbmkKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCiAgcmVnaXN0ZXJTY2hlZHVsYWJsZTogZmFsc2UKCl9fRU9GX0NMVVNURVJfU1BFQwoKY2F0ID4gY29uZi9pZ19zcGVjLnlhbWwgPDwgJ19fRU9GX0lHX1NQRUMnCmt1YmVsZXQ6IG51bGwKbm9kZUxhYmVsczogbnVsbAp0YWludHM6IG51bGwKCl9fRU9GX0lHX1NQRUMKCmNhdCA+IGNvbmYva3ViZV9lbnYueWFtbCA8PCAnX19FT0ZfS1VCRV9FTlYnCkFzc2V0czoKLSBjM2I3MzZmZDBmMDAzNzY1YzEyZDk5ZjJjOTk1YTgzNjllNjI0MWY0QGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlbGV0Ci0gN2UzYTNlYTY2MzE1M2Y5MDBjYmQ1MjkwMGEzOWM5MWZhOWYzMzRiZUBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDUyZTlkMmRlOGE1ZjkyNzMwN2Q5Mzk3MzA4NzM1NjU4ZWU0NGFiOGRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLXBsdWdpbnMtYW1kNjQtdjAuNy41LnRnegpDbHVzdGVyTmFtZTogcHJpdmF0ZWRuczIuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9wcml2YXRlZG5zMi5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVkbnMyLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCmV0Y2RNYW5pZmVzdHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9wcml2YXRlZG5zMi5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9tYWluLnlhbWwKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVkbnMyLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: privatedns2.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: privatedns2.example.com +ConfigBase: memfs://clusters.example.com/privatedns2.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/privatedns2.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/privatedns2.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/privatedns2.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatedns2/data/aws_launch_template_nodes.privatedns2.example.com_user_data b/tests/integration/update_cluster/privatedns2/data/aws_launch_template_nodes.privatedns2.example.com_user_data index ce1dc0188c9da..669d216d9c6a1 100644 --- a/tests/integration/update_cluster/privatedns2/data/aws_launch_template_nodes.privatedns2.example.com_user_data +++ b/tests/integration/update_cluster/privatedns2/data/aws_launch_template_nodes.privatedns2.example.com_user_data @@ -1 +1,208 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGMzYjczNmZkMGYwMDM3NjVjMTJkOTlmMmM5OTVhODM2OWU2MjQxZjRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA3ZTNhM2VhNjYzMTUzZjkwMGNiZDUyOTAwYTM5YzkxZmE5ZjMzNGJlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlY3RsCi0gNTJlOWQyZGU4YTVmOTI3MzA3ZDkzOTczMDg3MzU2NThlZTQ0YWI4ZEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL25ldHdvcmstcGx1Z2lucy9jbmktcGx1Z2lucy1hbWQ2NC12MC43LjUudGd6CkNsdXN0ZXJOYW1lOiBwcml2YXRlZG5zMi5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVkbnMyLmV4YW1wbGUuY29tCkluc3RhbmNlR3JvdXBOYW1lOiBub2RlcwpUYWdzOgotIF9hdXRvbWF0aWNfdXBncmFkZXMKLSBfYXdzCmNoYW5uZWxzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vcHJpdmF0ZWRuczIuZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: privatedns2.example.com +ConfigBase: memfs://clusters.example.com/privatedns2.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/privatedns2.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privateflannel/data/aws_launch_template_master-us-test-1a.masters.privateflannel.example.com_user_data b/tests/integration/update_cluster/privateflannel/data/aws_launch_template_master-us-test-1a.masters.privateflannel.example.com_user_data index 173676db7d5bb..f487ad7cc949d 100644 --- a/tests/integration/update_cluster/privateflannel/data/aws_launch_template_master-us-test-1a.masters.privateflannel.example.com_user_data +++ b/tests/integration/update_cluster/privateflannel/data/aws_launch_template_master-us-test-1a.masters.privateflannel.example.com_user_data @@ -1 +1,300 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBwcml2YXRlZmxhbm5lbC5leGFtcGxlLmNvbQogIGNvbmZpZ3VyZUNsb3VkUm91dGVzOiBmYWxzZQogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKICB1c2VTZXJ2aWNlQWNjb3VudENyZWRlbnRpYWxzOiB0cnVlCmt1YmVQcm94eToKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNwdVJlcXVlc3Q6IDEwMG0KICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXByb3h5OnYxLjE0LjAKICBsb2dMZXZlbDogMgprdWJlU2NoZWR1bGVyOgogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwptYXN0ZXJLdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5OYW1lOiBjbmkKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCiAgcmVnaXN0ZXJTY2hlZHVsYWJsZTogZmFsc2UKCl9fRU9GX0NMVVNURVJfU1BFQwoKY2F0ID4gY29uZi9pZ19zcGVjLnlhbWwgPDwgJ19fRU9GX0lHX1NQRUMnCmt1YmVsZXQ6IG51bGwKbm9kZUxhYmVsczogbnVsbAp0YWludHM6IG51bGwKCl9fRU9GX0lHX1NQRUMKCmNhdCA+IGNvbmYva3ViZV9lbnYueWFtbCA8PCAnX19FT0ZfS1VCRV9FTlYnCkFzc2V0czoKLSBjM2I3MzZmZDBmMDAzNzY1YzEyZDk5ZjJjOTk1YTgzNjllNjI0MWY0QGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlbGV0Ci0gN2UzYTNlYTY2MzE1M2Y5MDBjYmQ1MjkwMGEzOWM5MWZhOWYzMzRiZUBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDUyZTlkMmRlOGE1ZjkyNzMwN2Q5Mzk3MzA4NzM1NjU4ZWU0NGFiOGRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLXBsdWdpbnMtYW1kNjQtdjAuNy41LnRnegpDbHVzdGVyTmFtZTogcHJpdmF0ZWZsYW5uZWwuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9wcml2YXRlZmxhbm5lbC5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVmbGFubmVsLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCmV0Y2RNYW5pZmVzdHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9wcml2YXRlZmxhbm5lbC5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9tYWluLnlhbWwKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVmbGFubmVsLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: privateflannel.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: privateflannel.example.com +ConfigBase: memfs://clusters.example.com/privateflannel.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/privateflannel.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/privateflannel.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/privateflannel.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privateflannel/data/aws_launch_template_nodes.privateflannel.example.com_user_data b/tests/integration/update_cluster/privateflannel/data/aws_launch_template_nodes.privateflannel.example.com_user_data index c60dac0ce7929..a88e73116f0ca 100644 --- a/tests/integration/update_cluster/privateflannel/data/aws_launch_template_nodes.privateflannel.example.com_user_data +++ b/tests/integration/update_cluster/privateflannel/data/aws_launch_template_nodes.privateflannel.example.com_user_data @@ -1 +1,208 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGMzYjczNmZkMGYwMDM3NjVjMTJkOTlmMmM5OTVhODM2OWU2MjQxZjRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA3ZTNhM2VhNjYzMTUzZjkwMGNiZDUyOTAwYTM5YzkxZmE5ZjMzNGJlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlY3RsCi0gNTJlOWQyZGU4YTVmOTI3MzA3ZDkzOTczMDg3MzU2NThlZTQ0YWI4ZEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL25ldHdvcmstcGx1Z2lucy9jbmktcGx1Z2lucy1hbWQ2NC12MC43LjUudGd6CkNsdXN0ZXJOYW1lOiBwcml2YXRlZmxhbm5lbC5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVmbGFubmVsLmV4YW1wbGUuY29tCkluc3RhbmNlR3JvdXBOYW1lOiBub2RlcwpUYWdzOgotIF9hdXRvbWF0aWNfdXBncmFkZXMKLSBfYXdzCmNoYW5uZWxzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vcHJpdmF0ZWZsYW5uZWwuZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: privateflannel.example.com +ConfigBase: memfs://clusters.example.com/privateflannel.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/privateflannel.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatekopeio/data/aws_launch_template_master-us-test-1a.masters.privatekopeio.example.com_user_data b/tests/integration/update_cluster/privatekopeio/data/aws_launch_template_master-us-test-1a.masters.privatekopeio.example.com_user_data index 0bb1f849e318b..667a36eb76242 100644 --- a/tests/integration/update_cluster/privatekopeio/data/aws_launch_template_master-us-test-1a.masters.privatekopeio.example.com_user_data +++ b/tests/integration/update_cluster/privatekopeio/data/aws_launch_template_master-us-test-1a.masters.privatekopeio.example.com_user_data @@ -1 +1,300 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBwcml2YXRla29wZWlvLmV4YW1wbGUuY29tCiAgY29uZmlndXJlQ2xvdWRSb3V0ZXM6IGZhbHNlCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1jb250cm9sbGVyLW1hbmFnZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgogIHVzZVNlcnZpY2VBY2NvdW50Q3JlZGVudGlhbHM6IHRydWUKa3ViZVByb3h5OgogIGNsdXN0ZXJDSURSOiAxMDAuOTYuMC4wLzExCiAgY3B1UmVxdWVzdDogMTAwbQogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtcHJveHk6djEuMTQuMAogIGxvZ0xldmVsOiAyCmt1YmVTY2hlZHVsZXI6CiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1zY2hlZHVsZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgprdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5OYW1lOiBjbmkKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCm1hc3Rlckt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk5hbWU6IGNuaQogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKICByZWdpc3RlclNjaGVkdWxhYmxlOiBmYWxzZQoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGMzYjczNmZkMGYwMDM3NjVjMTJkOTlmMmM5OTVhODM2OWU2MjQxZjRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA3ZTNhM2VhNjYzMTUzZjkwMGNiZDUyOTAwYTM5YzkxZmE5ZjMzNGJlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlY3RsCi0gNTJlOWQyZGU4YTVmOTI3MzA3ZDkzOTczMDg3MzU2NThlZTQ0YWI4ZEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL25ldHdvcmstcGx1Z2lucy9jbmktcGx1Z2lucy1hbWQ2NC12MC43LjUudGd6CkNsdXN0ZXJOYW1lOiBwcml2YXRla29wZWlvLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vcHJpdmF0ZWtvcGVpby5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVrb3BlaW8uZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKZXRjZE1hbmlmZXN0czoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVrb3BlaW8uZXhhbXBsZS5jb20vbWFuaWZlc3RzL2V0Y2QvbWFpbi55YW1sCi0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9wcml2YXRla29wZWlvLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: privatekopeio.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: privatekopeio.example.com +ConfigBase: memfs://clusters.example.com/privatekopeio.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/privatekopeio.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/privatekopeio.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/privatekopeio.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privatekopeio/data/aws_launch_template_nodes.privatekopeio.example.com_user_data b/tests/integration/update_cluster/privatekopeio/data/aws_launch_template_nodes.privatekopeio.example.com_user_data index 2ff59c3ffb819..6b527522046cf 100644 --- a/tests/integration/update_cluster/privatekopeio/data/aws_launch_template_nodes.privatekopeio.example.com_user_data +++ b/tests/integration/update_cluster/privatekopeio/data/aws_launch_template_nodes.privatekopeio.example.com_user_data @@ -1 +1,208 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGMzYjczNmZkMGYwMDM3NjVjMTJkOTlmMmM5OTVhODM2OWU2MjQxZjRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA3ZTNhM2VhNjYzMTUzZjkwMGNiZDUyOTAwYTM5YzkxZmE5ZjMzNGJlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlY3RsCi0gNTJlOWQyZGU4YTVmOTI3MzA3ZDkzOTczMDg3MzU2NThlZTQ0YWI4ZEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL25ldHdvcmstcGx1Z2lucy9jbmktcGx1Z2lucy1hbWQ2NC12MC43LjUudGd6CkNsdXN0ZXJOYW1lOiBwcml2YXRla29wZWlvLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vcHJpdmF0ZWtvcGVpby5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbm9kZXMKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGVrb3BlaW8uZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: privatekopeio.example.com +ConfigBase: memfs://clusters.example.com/privatekopeio.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/privatekopeio.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privateweave/data/aws_launch_template_master-us-test-1a.masters.privateweave.example.com_user_data b/tests/integration/update_cluster/privateweave/data/aws_launch_template_master-us-test-1a.masters.privateweave.example.com_user_data index 12171e26642ce..a222477ed2e73 100644 --- a/tests/integration/update_cluster/privateweave/data/aws_launch_template_master-us-test-1a.masters.privateweave.example.com_user_data +++ b/tests/integration/update_cluster/privateweave/data/aws_launch_template_master-us-test-1a.masters.privateweave.example.com_user_data @@ -1 +1,300 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBwcml2YXRld2VhdmUuZXhhbXBsZS5jb20KICBjb25maWd1cmVDbG91ZFJvdXRlczogZmFsc2UKICBpbWFnZTogazhzLmdjci5pby9rdWJlLWNvbnRyb2xsZXItbWFuYWdlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCiAgdXNlU2VydmljZUFjY291bnRDcmVkZW50aWFsczogdHJ1ZQprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZVNjaGVkdWxlcjoKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXNjaGVkdWxlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCmt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk5hbWU6IGNuaQogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKbWFzdGVyS3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwogIHJlZ2lzdGVyU2NoZWR1bGFibGU6IGZhbHNlCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IHByaXZhdGV3ZWF2ZS5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGV3ZWF2ZS5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3ByaXZhdGV3ZWF2ZS5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbApldGNkTWFuaWZlc3RzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vcHJpdmF0ZXdlYXZlLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL21haW4ueWFtbAotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vcHJpdmF0ZXdlYXZlLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: privateweave.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: privateweave.example.com +ConfigBase: memfs://clusters.example.com/privateweave.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/privateweave.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/privateweave.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/privateweave.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/privateweave/data/aws_launch_template_nodes.privateweave.example.com_user_data b/tests/integration/update_cluster/privateweave/data/aws_launch_template_nodes.privateweave.example.com_user_data index 8c253669436c2..103b396881f7d 100644 --- a/tests/integration/update_cluster/privateweave/data/aws_launch_template_nodes.privateweave.example.com_user_data +++ b/tests/integration/update_cluster/privateweave/data/aws_launch_template_nodes.privateweave.example.com_user_data @@ -1 +1,208 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGMzYjczNmZkMGYwMDM3NjVjMTJkOTlmMmM5OTVhODM2OWU2MjQxZjRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA3ZTNhM2VhNjYzMTUzZjkwMGNiZDUyOTAwYTM5YzkxZmE5ZjMzNGJlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlY3RsCi0gNTJlOWQyZGU4YTVmOTI3MzA3ZDkzOTczMDg3MzU2NThlZTQ0YWI4ZEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL25ldHdvcmstcGx1Z2lucy9jbmktcGx1Z2lucy1hbWQ2NC12MC43LjUudGd6CkNsdXN0ZXJOYW1lOiBwcml2YXRld2VhdmUuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9wcml2YXRld2VhdmUuZXhhbXBsZS5jb20KSW5zdGFuY2VHcm91cE5hbWU6IG5vZGVzClRhZ3M6Ci0gX2F1dG9tYXRpY191cGdyYWRlcwotIF9hd3MKY2hhbm5lbHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9wcml2YXRld2VhdmUuZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: privateweave.example.com +ConfigBase: memfs://clusters.example.com/privateweave.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/privateweave.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/restrict_access/data/aws_launch_template_master-us-test-1a.masters.restrictaccess.example.com_user_data b/tests/integration/update_cluster/restrict_access/data/aws_launch_template_master-us-test-1a.masters.restrictaccess.example.com_user_data index 35df2dc46f9bd..935ca6595b2e9 100644 --- a/tests/integration/update_cluster/restrict_access/data/aws_launch_template_master-us-test-1a.masters.restrictaccess.example.com_user_data +++ b/tests/integration/update_cluster/restrict_access/data/aws_launch_template_master-us-test-1a.masters.restrictaccess.example.com_user_data @@ -1 +1,302 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiByZXN0cmljdGFjY2Vzcy5leGFtcGxlLmNvbQogIGNvbmZpZ3VyZUNsb3VkUm91dGVzOiB0cnVlCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1jb250cm9sbGVyLW1hbmFnZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgogIHVzZVNlcnZpY2VBY2NvdW50Q3JlZGVudGlhbHM6IHRydWUKa3ViZVByb3h5OgogIGNsdXN0ZXJDSURSOiAxMDAuOTYuMC4wLzExCiAgY3B1UmVxdWVzdDogMTAwbQogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtcHJveHk6djEuMTQuMAogIGxvZ0xldmVsOiAyCmt1YmVTY2hlZHVsZXI6CiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1zY2hlZHVsZXI6djEuMTQuMAogIGxlYWRlckVsZWN0aW9uOgogICAgbGVhZGVyRWxlY3Q6IHRydWUKICBsb2dMZXZlbDogMgprdWJlbGV0OgogIGFub255bW91c0F1dGg6IGZhbHNlCiAgY2dyb3VwUm9vdDogLwogIGNsb3VkUHJvdmlkZXI6IGF3cwogIGNsdXN0ZXJETlM6IDEwMC42NC4wLjEwCiAgY2x1c3RlckRvbWFpbjogY2x1c3Rlci5sb2NhbAogIGVuYWJsZURlYnVnZ2luZ0hhbmRsZXJzOiB0cnVlCiAgZXZpY3Rpb25IYXJkOiBtZW1vcnkuYXZhaWxhYmxlPDEwME1pLG5vZGVmcy5hdmFpbGFibGU8MTAlLG5vZGVmcy5pbm9kZXNGcmVlPDUlLGltYWdlZnMuYXZhaWxhYmxlPDEwJSxpbWFnZWZzLmlub2Rlc0ZyZWU8NSUKICBmZWF0dXJlR2F0ZXM6CiAgICBFeHBlcmltZW50YWxDcml0aWNhbFBvZEFubm90YXRpb246ICJ0cnVlIgogIGhvc3RuYW1lT3ZlcnJpZGU6ICdAYXdzJwogIGt1YmVjb25maWdQYXRoOiAvdmFyL2xpYi9rdWJlbGV0L2t1YmVjb25maWcKICBsb2dMZXZlbDogMgogIG5ldHdvcmtQbHVnaW5NVFU6IDkwMDEKICBuZXR3b3JrUGx1Z2luTmFtZToga3ViZW5ldAogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKbWFzdGVyS3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCiAgcmVnaXN0ZXJTY2hlZHVsYWJsZTogZmFsc2UKCl9fRU9GX0NMVVNURVJfU1BFQwoKY2F0ID4gY29uZi9pZ19zcGVjLnlhbWwgPDwgJ19fRU9GX0lHX1NQRUMnCmt1YmVsZXQ6IG51bGwKbm9kZUxhYmVsczogbnVsbAp0YWludHM6IG51bGwKCl9fRU9GX0lHX1NQRUMKCmNhdCA+IGNvbmYva3ViZV9lbnYueWFtbCA8PCAnX19FT0ZfS1VCRV9FTlYnCkFzc2V0czoKLSBjM2I3MzZmZDBmMDAzNzY1YzEyZDk5ZjJjOTk1YTgzNjllNjI0MWY0QGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlbGV0Ci0gN2UzYTNlYTY2MzE1M2Y5MDBjYmQ1MjkwMGEzOWM5MWZhOWYzMzRiZUBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWN0bAotIDUyZTlkMmRlOGE1ZjkyNzMwN2Q5Mzk3MzA4NzM1NjU4ZWU0NGFiOGRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9uZXR3b3JrLXBsdWdpbnMvY25pLXBsdWdpbnMtYW1kNjQtdjAuNy41LnRnegpDbHVzdGVyTmFtZTogcmVzdHJpY3RhY2Nlc3MuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9yZXN0cmljdGFjY2Vzcy5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3Jlc3RyaWN0YWNjZXNzLmV4YW1wbGUuY29tL2FkZG9ucy9ib290c3RyYXAtY2hhbm5lbC55YW1sCmV0Y2RNYW5pZmVzdHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9yZXN0cmljdGFjY2Vzcy5leGFtcGxlLmNvbS9tYW5pZmVzdHMvZXRjZC9tYWluLnlhbWwKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3Jlc3RyaWN0YWNjZXNzLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: restrictaccess.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: restrictaccess.example.com +ConfigBase: memfs://clusters.example.com/restrictaccess.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/restrictaccess.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/restrictaccess.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/restrictaccess.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/restrict_access/data/aws_launch_template_nodes.restrictaccess.example.com_user_data b/tests/integration/update_cluster/restrict_access/data/aws_launch_template_nodes.restrictaccess.example.com_user_data index dc9d40c39ee1e..f28057795f384 100644 --- a/tests/integration/update_cluster/restrict_access/data/aws_launch_template_nodes.restrictaccess.example.com_user_data +++ b/tests/integration/update_cluster/restrict_access/data/aws_launch_template_nodes.restrictaccess.example.com_user_data @@ -1 +1,209 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IHJlc3RyaWN0YWNjZXNzLmV4YW1wbGUuY29tCkNvbmZpZ0Jhc2U6IG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vcmVzdHJpY3RhY2Nlc3MuZXhhbXBsZS5jb20KSW5zdGFuY2VHcm91cE5hbWU6IG5vZGVzClRhZ3M6Ci0gX2F1dG9tYXRpY191cGdyYWRlcwotIF9hd3MKY2hhbm5lbHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS9yZXN0cmljdGFjY2Vzcy5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbAoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg== +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: restrictaccess.example.com +ConfigBase: memfs://clusters.example.com/restrictaccess.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/restrictaccess.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/shared_subnet/data/aws_launch_template_master-us-test-1a.masters.sharedsubnet.example.com_user_data b/tests/integration/update_cluster/shared_subnet/data/aws_launch_template_master-us-test-1a.masters.sharedsubnet.example.com_user_data index 2edc8541e5de5..ed4a79340afd1 100644 --- a/tests/integration/update_cluster/shared_subnet/data/aws_launch_template_master-us-test-1a.masters.sharedsubnet.example.com_user_data +++ b/tests/integration/update_cluster/shared_subnet/data/aws_launch_template_master-us-test-1a.masters.sharedsubnet.example.com_user_data @@ -1 +1,302 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBzaGFyZWRzdWJuZXQuZXhhbXBsZS5jb20KICBjb25maWd1cmVDbG91ZFJvdXRlczogdHJ1ZQogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKICB1c2VTZXJ2aWNlQWNjb3VudENyZWRlbnRpYWxzOiB0cnVlCmt1YmVQcm94eToKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNwdVJlcXVlc3Q6IDEwMG0KICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXByb3h5OnYxLjE0LjAKICBsb2dMZXZlbDogMgprdWJlU2NoZWR1bGVyOgogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCm1hc3Rlckt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk1UVTogOTAwMQogIG5ldHdvcmtQbHVnaW5OYW1lOiBrdWJlbmV0CiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwogIHJlZ2lzdGVyU2NoZWR1bGFibGU6IGZhbHNlCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IHNoYXJlZHN1Ym5ldC5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3NoYXJlZHN1Ym5ldC5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3NoYXJlZHN1Ym5ldC5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbApldGNkTWFuaWZlc3RzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vc2hhcmVkc3VibmV0LmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL21haW4ueWFtbAotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vc2hhcmVkc3VibmV0LmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: sharedsubnet.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: sharedsubnet.example.com +ConfigBase: memfs://clusters.example.com/sharedsubnet.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/sharedsubnet.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/sharedsubnet.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/sharedsubnet.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/shared_subnet/data/aws_launch_template_nodes.sharedsubnet.example.com_user_data b/tests/integration/update_cluster/shared_subnet/data/aws_launch_template_nodes.sharedsubnet.example.com_user_data index 4028a0f50d76c..718f9f9702133 100644 --- a/tests/integration/update_cluster/shared_subnet/data/aws_launch_template_nodes.sharedsubnet.example.com_user_data +++ b/tests/integration/update_cluster/shared_subnet/data/aws_launch_template_nodes.sharedsubnet.example.com_user_data @@ -1 +1,209 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IHNoYXJlZHN1Ym5ldC5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3NoYXJlZHN1Ym5ldC5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbm9kZXMKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3NoYXJlZHN1Ym5ldC5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbAoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg== +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: sharedsubnet.example.com +ConfigBase: memfs://clusters.example.com/sharedsubnet.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/sharedsubnet.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/shared_vpc/data/aws_launch_template_master-us-test-1a.masters.sharedvpc.example.com_user_data b/tests/integration/update_cluster/shared_vpc/data/aws_launch_template_master-us-test-1a.masters.sharedvpc.example.com_user_data index 1e060cbc03e65..68de723ed9737 100644 --- a/tests/integration/update_cluster/shared_vpc/data/aws_launch_template_master-us-test-1a.masters.sharedvpc.example.com_user_data +++ b/tests/integration/update_cluster/shared_vpc/data/aws_launch_template_master-us-test-1a.masters.sharedvpc.example.com_user_data @@ -1 +1,302 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiBzaGFyZWR2cGMuZXhhbXBsZS5jb20KICBjb25maWd1cmVDbG91ZFJvdXRlczogdHJ1ZQogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKICB1c2VTZXJ2aWNlQWNjb3VudENyZWRlbnRpYWxzOiB0cnVlCmt1YmVQcm94eToKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNwdVJlcXVlc3Q6IDEwMG0KICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXByb3h5OnYxLjE0LjAKICBsb2dMZXZlbDogMgprdWJlU2NoZWR1bGVyOgogIGltYWdlOiBrOHMuZ2NyLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjE0LjAKICBsZWFkZXJFbGVjdGlvbjoKICAgIGxlYWRlckVsZWN0OiB0cnVlCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCm1hc3Rlckt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk1UVTogOTAwMQogIG5ldHdvcmtQbHVnaW5OYW1lOiBrdWJlbmV0CiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwogIHJlZ2lzdGVyU2NoZWR1bGFibGU6IGZhbHNlCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IHNoYXJlZHZwYy5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3NoYXJlZHZwYy5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3NoYXJlZHZwYy5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbApldGNkTWFuaWZlc3RzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vc2hhcmVkdnBjLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL21haW4ueWFtbAotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vc2hhcmVkdnBjLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: sharedvpc.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: sharedvpc.example.com +ConfigBase: memfs://clusters.example.com/sharedvpc.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/sharedvpc.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/sharedvpc.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/sharedvpc.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/shared_vpc/data/aws_launch_template_nodes.sharedvpc.example.com_user_data b/tests/integration/update_cluster/shared_vpc/data/aws_launch_template_nodes.sharedvpc.example.com_user_data index 7f74184745d0c..625c3be00c72e 100644 --- a/tests/integration/update_cluster/shared_vpc/data/aws_launch_template_nodes.sharedvpc.example.com_user_data +++ b/tests/integration/update_cluster/shared_vpc/data/aws_launch_template_nodes.sharedvpc.example.com_user_data @@ -1 +1,209 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTVRVOiA5MDAxCiAgbmV0d29ya1BsdWdpbk5hbWU6IGt1YmVuZXQKICBub25NYXNxdWVyYWRlQ0lEUjogMTAwLjY0LjAuMC8xMAogIHBvZEluZnJhQ29udGFpbmVySW1hZ2U6IGs4cy5nY3IuaW8vcGF1c2U6My4yCiAgcG9kTWFuaWZlc3RQYXRoOiAvZXRjL2t1YmVybmV0ZXMvbWFuaWZlc3RzCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IHNoYXJlZHZwYy5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3NoYXJlZHZwYy5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbm9kZXMKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3NoYXJlZHZwYy5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbAoKX19FT0ZfS1VCRV9FTlYKCmRvd25sb2FkLXJlbGVhc2UKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIGRvbmUgPT0iCg== +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: sharedvpc.example.com +ConfigBase: memfs://clusters.example.com/sharedvpc.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/sharedvpc.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/unmanaged/data/aws_launch_template_master-us-test-1a.masters.unmanaged.example.com_user_data b/tests/integration/update_cluster/unmanaged/data/aws_launch_template_master-us-test-1a.masters.unmanaged.example.com_user_data index d24fd517ed596..6ae8290798654 100644 --- a/tests/integration/update_cluster/unmanaged/data/aws_launch_template_master-us-test-1a.masters.unmanaged.example.com_user_data +++ b/tests/integration/update_cluster/unmanaged/data/aws_launch_template_master-us-test-1a.masters.unmanaged.example.com_user_data @@ -1 +1,300 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwplbmNyeXB0aW9uQ29uZmlnOiBudWxsCmV0Y2RDbHVzdGVyczoKICBldmVudHM6CiAgICB2ZXJzaW9uOiAzLjMuMTAKICBtYWluOgogICAgdmVyc2lvbjogMy4zLjEwCmt1YmVBUElTZXJ2ZXI6CiAgYWxsb3dQcml2aWxlZ2VkOiB0cnVlCiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBhcGlTZXJ2ZXJDb3VudDogMQogIGF1dGhvcml6YXRpb25Nb2RlOiBBbHdheXNBbGxvdwogIGJpbmRBZGRyZXNzOiAwLjAuMC4wCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgZW5hYmxlQWRtaXNzaW9uUGx1Z2luczoKICAtIE5hbWVzcGFjZUxpZmVjeWNsZQogIC0gTGltaXRSYW5nZXIKICAtIFNlcnZpY2VBY2NvdW50CiAgLSBQZXJzaXN0ZW50Vm9sdW1lTGFiZWwKICAtIERlZmF1bHRTdG9yYWdlQ2xhc3MKICAtIERlZmF1bHRUb2xlcmF0aW9uU2Vjb25kcwogIC0gTXV0YXRpbmdBZG1pc3Npb25XZWJob29rCiAgLSBWYWxpZGF0aW5nQWRtaXNzaW9uV2ViaG9vawogIC0gTm9kZVJlc3RyaWN0aW9uCiAgLSBSZXNvdXJjZVF1b3RhCiAgZXRjZFNlcnZlcnM6CiAgLSBodHRwOi8vMTI3LjAuMC4xOjQwMDEKICBldGNkU2VydmVyc092ZXJyaWRlczoKICAtIC9ldmVudHMjaHR0cDovLzEyNy4wLjAuMTo0MDAyCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1hcGlzZXJ2ZXI6djEuMTQuMAogIGluc2VjdXJlQmluZEFkZHJlc3M6IDEyNy4wLjAuMQogIGluc2VjdXJlUG9ydDogODA4MAogIGt1YmVsZXRQcmVmZXJyZWRBZGRyZXNzVHlwZXM6CiAgLSBJbnRlcm5hbElQCiAgLSBIb3N0bmFtZQogIC0gRXh0ZXJuYWxJUAogIGxvZ0xldmVsOiAyCiAgcmVxdWVzdGhlYWRlckFsbG93ZWROYW1lczoKICAtIGFnZ3JlZ2F0b3IKICByZXF1ZXN0aGVhZGVyRXh0cmFIZWFkZXJQcmVmaXhlczoKICAtIFgtUmVtb3RlLUV4dHJhLQogIHJlcXVlc3RoZWFkZXJHcm91cEhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Hcm91cAogIHJlcXVlc3RoZWFkZXJVc2VybmFtZUhlYWRlcnM6CiAgLSBYLVJlbW90ZS1Vc2VyCiAgc2VjdXJlUG9ydDogNDQzCiAgc2VydmljZUNsdXN0ZXJJUFJhbmdlOiAxMDAuNjQuMC4wLzEzCiAgc3RvcmFnZUJhY2tlbmQ6IGV0Y2QzCmt1YmVDb250cm9sbGVyTWFuYWdlcjoKICBhbGxvY2F0ZU5vZGVDSURSczogdHJ1ZQogIGF0dGFjaERldGFjaFJlY29uY2lsZVN5bmNQZXJpb2Q6IDFtMHMKICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyQ0lEUjogMTAwLjk2LjAuMC8xMQogIGNsdXN0ZXJOYW1lOiB1bm1hbmFnZWQuZXhhbXBsZS5jb20KICBjb25maWd1cmVDbG91ZFJvdXRlczogZmFsc2UKICBpbWFnZTogazhzLmdjci5pby9rdWJlLWNvbnRyb2xsZXItbWFuYWdlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCiAgdXNlU2VydmljZUFjY291bnRDcmVkZW50aWFsczogdHJ1ZQprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZVNjaGVkdWxlcjoKICBpbWFnZTogazhzLmdjci5pby9rdWJlLXNjaGVkdWxlcjp2MS4xNC4wCiAgbGVhZGVyRWxlY3Rpb246CiAgICBsZWFkZXJFbGVjdDogdHJ1ZQogIGxvZ0xldmVsOiAyCmt1YmVsZXQ6CiAgYW5vbnltb3VzQXV0aDogZmFsc2UKICBjZ3JvdXBSb290OiAvCiAgY2xvdWRQcm92aWRlcjogYXdzCiAgY2x1c3RlckROUzogMTAwLjY0LjAuMTAKICBjbHVzdGVyRG9tYWluOiBjbHVzdGVyLmxvY2FsCiAgZW5hYmxlRGVidWdnaW5nSGFuZGxlcnM6IHRydWUKICBldmljdGlvbkhhcmQ6IG1lbW9yeS5hdmFpbGFibGU8MTAwTWksbm9kZWZzLmF2YWlsYWJsZTwxMCUsbm9kZWZzLmlub2Rlc0ZyZWU8NSUsaW1hZ2Vmcy5hdmFpbGFibGU8MTAlLGltYWdlZnMuaW5vZGVzRnJlZTw1JQogIGZlYXR1cmVHYXRlczoKICAgIEV4cGVyaW1lbnRhbENyaXRpY2FsUG9kQW5ub3RhdGlvbjogInRydWUiCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAga3ViZWNvbmZpZ1BhdGg6IC92YXIvbGliL2t1YmVsZXQva3ViZWNvbmZpZwogIGxvZ0xldmVsOiAyCiAgbmV0d29ya1BsdWdpbk5hbWU6IGNuaQogIG5vbk1hc3F1ZXJhZGVDSURSOiAxMDAuNjQuMC4wLzEwCiAgcG9kSW5mcmFDb250YWluZXJJbWFnZTogazhzLmdjci5pby9wYXVzZTozLjIKICBwb2RNYW5pZmVzdFBhdGg6IC9ldGMva3ViZXJuZXRlcy9tYW5pZmVzdHMKbWFzdGVyS3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwogIHJlZ2lzdGVyU2NoZWR1bGFibGU6IGZhbHNlCgpfX0VPRl9DTFVTVEVSX1NQRUMKCmNhdCA+IGNvbmYvaWdfc3BlYy55YW1sIDw8ICdfX0VPRl9JR19TUEVDJwprdWJlbGV0OiBudWxsCm5vZGVMYWJlbHM6IG51bGwKdGFpbnRzOiBudWxsCgpfX0VPRl9JR19TUEVDCgpjYXQgPiBjb25mL2t1YmVfZW52LnlhbWwgPDwgJ19fRU9GX0tVQkVfRU5WJwpBc3NldHM6Ci0gYzNiNzM2ZmQwZjAwMzc2NWMxMmQ5OWYyYzk5NWE4MzY5ZTYyNDFmNEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL3JlbGVhc2UvdjEuMTQuMC9iaW4vbGludXgvYW1kNjQva3ViZWxldAotIDdlM2EzZWE2NjMxNTNmOTAwY2JkNTI5MDBhMzljOTFmYTlmMzM0YmVAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVjdGwKLSA1MmU5ZDJkZThhNWY5MjczMDdkOTM5NzMwODczNTY1OGVlNDRhYjhkQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvbmV0d29yay1wbHVnaW5zL2NuaS1wbHVnaW5zLWFtZDY0LXYwLjcuNS50Z3oKQ2x1c3Rlck5hbWU6IHVubWFuYWdlZC5leGFtcGxlLmNvbQpDb25maWdCYXNlOiBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3VubWFuYWdlZC5leGFtcGxlLmNvbQpJbnN0YW5jZUdyb3VwTmFtZTogbWFzdGVyLXVzLXRlc3QtMWEKVGFnczoKLSBfYXV0b21hdGljX3VwZ3JhZGVzCi0gX2F3cwpjaGFubmVsczoKLSBtZW1mczovL2NsdXN0ZXJzLmV4YW1wbGUuY29tL3VubWFuYWdlZC5leGFtcGxlLmNvbS9hZGRvbnMvYm9vdHN0cmFwLWNoYW5uZWwueWFtbApldGNkTWFuaWZlc3RzOgotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vdW5tYW5hZ2VkLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL21haW4ueWFtbAotIG1lbWZzOi8vY2x1c3RlcnMuZXhhbXBsZS5jb20vdW5tYW5hZ2VkLmV4YW1wbGUuY29tL21hbmlmZXN0cy9ldGNkL2V2ZW50cy55YW1sCnByb3Rva3ViZUltYWdlOgogIGhhc2g6IDQyYTljNDMyNGZlMjZkNjNjZTExZjNkZDc4MzYzNzFiYzkzZmEwNmNhOGY0Nzk4MDc3MjhmMzc0NmUyNzA2MWIKICBuYW1lOiBwcm90b2t1YmU6MS4xNS4wCiAgc291cmNlczoKICAtIGh0dHBzOi8vYXJ0aWZhY3RzLms4cy5pby9iaW5hcmllcy9rb3BzLzEuMTUuMC9pbWFnZXMvcHJvdG9rdWJlLnRhci5negogIC0gaHR0cHM6Ly9naXRodWIuY29tL2t1YmVybmV0ZXMva29wcy9yZWxlYXNlcy9kb3dubG9hZC92MS4xNS4wL2ltYWdlcy1wcm90b2t1YmUudGFyLmd6CiAgLSBodHRwczovL2t1YmV1cHYyLnMzLmFtYXpvbmF3cy5jb20va29wcy8xLjE1LjAvaW1hZ2VzL3Byb3Rva3ViZS50YXIuZ3oKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: false + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: unmanaged.example.com + configureCloudRoutes: false + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: unmanaged.example.com +ConfigBase: memfs://clusters.example.com/unmanaged.example.com +InstanceGroupName: master-us-test-1a +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/unmanaged.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/unmanaged.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/unmanaged.example.com/manifests/etcd/events.yaml +protokubeImage: + hash: 42a9c4324fe26d63ce11f3dd7836371bc93fa06ca8f479807728f3746e27061b + name: protokube:1.15.0 + sources: + - https://artifacts.k8s.io/binaries/kops/1.15.0/images/protokube.tar.gz + - https://github.com/kubernetes/kops/releases/download/v1.15.0/images-protokube.tar.gz + - https://kubeupv2.s3.amazonaws.com/kops/1.15.0/images/protokube.tar.gz + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/unmanaged/data/aws_launch_template_nodes.unmanaged.example.com_user_data b/tests/integration/update_cluster/unmanaged/data/aws_launch_template_nodes.unmanaged.example.com_user_data index e3c54e666faee..cad7fc2786494 100644 --- a/tests/integration/update_cluster/unmanaged/data/aws_launch_template_nodes.unmanaged.example.com_user_data +++ b/tests/integration/update_cluster/unmanaged/data/aws_launch_template_nodes.unmanaged.example.com_user_data @@ -1 +1,208 @@ -IyEvYmluL2Jhc2gKIyBDb3B5cmlnaHQgMjAxNiBUaGUgS3ViZXJuZXRlcyBBdXRob3JzIEFsbCByaWdodHMgcmVzZXJ2ZWQuCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpzZXQgLW8gZXJyZXhpdApzZXQgLW8gbm91bnNldApzZXQgLW8gcGlwZWZhaWwKCk5PREVVUF9VUkw9aHR0cHM6Ly9hcnRpZmFjdHMuazhzLmlvL2JpbmFyaWVzL2tvcHMvMS4xNS4wL2xpbnV4L2FtZDY0L25vZGV1cCxodHRwczovL2dpdGh1Yi5jb20va3ViZXJuZXRlcy9rb3BzL3JlbGVhc2VzL2Rvd25sb2FkL3YxLjE1LjAvbGludXgtYW1kNjQtbm9kZXVwLGh0dHBzOi8va3ViZXVwdjIuczMuYW1hem9uYXdzLmNvbS9rb3BzLzEuMTUuMC9saW51eC9hbWQ2NC9ub2RldXAKTk9ERVVQX0hBU0g9OTYwNGVmMTgyNjdhZDdiNWNmNGNlYmJmN2FiNjQ0MjNjZjViYjAzNDJkMTY5YzYwOGFjNjM3NmU2YWYyNmQ4MQoKZXhwb3J0IEFXU19SRUdJT049dXMtdGVzdC0xCgoKCgpmdW5jdGlvbiBlbnN1cmUtaW5zdGFsbC1kaXIoKSB7CiAgSU5TVEFMTF9ESVI9Ii9vcHQva29wcyIKICAjIE9uIENvbnRhaW5lck9TLCB3ZSBpbnN0YWxsIHVuZGVyIC92YXIvbGliL3Rvb2xib3g7IC9vcHQgaXMgcm8gYW5kIG5vZXhlYwogIGlmIFtbIC1kIC92YXIvbGliL3Rvb2xib3ggXV07IHRoZW4KICAgIElOU1RBTExfRElSPSIvdmFyL2xpYi90b29sYm94L2tvcHMiCiAgZmkKICBta2RpciAtcCAke0lOU1RBTExfRElSfS9iaW4KICBta2RpciAtcCAke0lOU1RBTExfRElSfS9jb25mCiAgY2QgJHtJTlNUQUxMX0RJUn0KfQoKIyBSZXRyeSBhIGRvd25sb2FkIHVudGlsIHdlIGdldCBpdC4gYXJnczogbmFtZSwgc2hhLCB1cmwxLCB1cmwyLi4uCmRvd25sb2FkLW9yLWJ1c3QoKSB7CiAgbG9jYWwgLXIgZmlsZT0iJDEiCiAgbG9jYWwgLXIgaGFzaD0iJDIiCiAgc2hpZnQgMgoKICB1cmxzPSggJCogKQogIHdoaWxlIHRydWU7IGRvCiAgICBmb3IgdXJsIGluICIke3VybHNbQF19IjsgZG8KICAgICAgY29tbWFuZHM9KAogICAgICAgICJjdXJsIC1mIC0taXB2NCAtLWNvbXByZXNzZWQgLUxvICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dCAyMCAtLXJldHJ5IDYgLS1yZXRyeS1kZWxheSAxMCIKICAgICAgICAid2dldCAtLWluZXQ0LW9ubHkgLS1jb21wcmVzc2lvbj1hdXRvIC1PICIke2ZpbGV9IiAtLWNvbm5lY3QtdGltZW91dD0yMCAtLXRyaWVzPTYgLS13YWl0PTEwIgogICAgICAgICJjdXJsIC1mIC0taXB2NCAtTG8gIiR7ZmlsZX0iIC0tY29ubmVjdC10aW1lb3V0IDIwIC0tcmV0cnkgNiAtLXJldHJ5LWRlbGF5IDEwIgogICAgICAgICJ3Z2V0IC0taW5ldDQtb25seSAtTyAiJHtmaWxlfSIgLS1jb25uZWN0LXRpbWVvdXQ9MjAgLS10cmllcz02IC0td2FpdD0xMCIKICAgICAgKQogICAgICBmb3IgY21kIGluICIke2NvbW1hbmRzW0BdfSI7IGRvCiAgICAgICAgZWNobyAiQXR0ZW1wdGluZyBkb3dubG9hZCB3aXRoOiAke2NtZH0ge3VybH0iCiAgICAgICAgaWYgISAoJHtjbWR9ICIke3VybH0iKTsgdGhlbgogICAgICAgICAgZWNobyAiPT0gRG93bmxvYWQgZmFpbGVkIHdpdGggJHtjbWR9ID09IgogICAgICAgICAgY29udGludWUKICAgICAgICBmaQogICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXSAmJiAhIHZhbGlkYXRlLWhhc2ggIiR7ZmlsZX0iICIke2hhc2h9IjsgdGhlbgogICAgICAgICAgZWNobyAiPT0gSGFzaCB2YWxpZGF0aW9uIG9mICR7dXJsfSBmYWlsZWQuIFJldHJ5aW5nLiA9PSIKICAgICAgICAgIHJtIC1mICIke2ZpbGV9IgogICAgICAgIGVsc2UKICAgICAgICAgIGlmIFtbIC1uICIke2hhc2h9IiBdXTsgdGhlbgogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSAoU0hBMSA9ICR7aGFzaH0pID09IgogICAgICAgICAgZWxzZQogICAgICAgICAgICBlY2hvICI9PSBEb3dubG9hZGVkICR7dXJsfSA9PSIKICAgICAgICAgIGZpCiAgICAgICAgICByZXR1cm4KICAgICAgICBmaQogICAgICBkb25lCiAgICBkb25lCgogICAgZWNobyAiQWxsIGRvd25sb2FkcyBmYWlsZWQ7IHNsZWVwaW5nIGJlZm9yZSByZXRyeWluZyIKICAgIHNsZWVwIDYwCiAgZG9uZQp9Cgp2YWxpZGF0ZS1oYXNoKCkgewogIGxvY2FsIC1yIGZpbGU9IiQxIgogIGxvY2FsIC1yIGV4cGVjdGVkPSIkMiIKICBsb2NhbCBhY3R1YWwKCiAgYWN0dWFsPSQoc2hhMjU2c3VtICR7ZmlsZX0gfCBhd2sgJ3sgcHJpbnQgJDEgfScpIHx8IHRydWUKICBpZiBbWyAiJHthY3R1YWx9IiAhPSAiJHtleHBlY3RlZH0iIF1dOyB0aGVuCiAgICBlY2hvICI9PSAke2ZpbGV9IGNvcnJ1cHRlZCwgaGFzaCAke2FjdHVhbH0gZG9lc24ndCBtYXRjaCBleHBlY3RlZCAke2V4cGVjdGVkfSA9PSIKICAgIHJldHVybiAxCiAgZmkKfQoKZnVuY3Rpb24gc3BsaXQtY29tbWFzKCkgewogIGVjaG8gJDEgfCB0ciAiLCIgIlxuIgp9CgpmdW5jdGlvbiB0cnktZG93bmxvYWQtcmVsZWFzZSgpIHsKICBsb2NhbCAtciBub2RldXBfdXJscz0oICQoc3BsaXQtY29tbWFzICIke05PREVVUF9VUkx9IikgKQogIGlmIFtbIC1uICIke05PREVVUF9IQVNIOi19IiBdXTsgdGhlbgogICAgbG9jYWwgLXIgbm9kZXVwX2hhc2g9IiR7Tk9ERVVQX0hBU0h9IgogIGVsc2UKICAjIFRPRE86IFJlbW92ZT8KICAgIGVjaG8gIkRvd25sb2FkaW5nIHNoYTI1NiAobm90IGZvdW5kIGluIGVudikiCiAgICBkb3dubG9hZC1vci1idXN0IG5vZGV1cC5zaGEyNTYgIiIgIiR7bm9kZXVwX3VybHNbQF0vJS8uc2hhMjU2fSIKICAgIGxvY2FsIC1yIG5vZGV1cF9oYXNoPSQoY2F0IG5vZGV1cC5zaGEyNTYpCiAgZmkKCiAgZWNobyAiRG93bmxvYWRpbmcgbm9kZXVwICgke25vZGV1cF91cmxzW0BdfSkiCiAgZG93bmxvYWQtb3ItYnVzdCBub2RldXAgIiR7bm9kZXVwX2hhc2h9IiAiJHtub2RldXBfdXJsc1tAXX0iCgogIGNobW9kICt4IG5vZGV1cAp9CgpmdW5jdGlvbiBkb3dubG9hZC1yZWxlYXNlKCkgewogIGNhc2UgIiQodW5hbWUgLW0pIiBpbgogIHg4Nl82NCp8aT84Nl82NCp8YW1kNjQqKQogICAgTk9ERVVQX1VSTD0iJHtOT0RFVVBfVVJMfSIKICAgIE5PREVVUF9IQVNIPSIke05PREVVUF9IQVNIfSIKICAgIDs7CiAgKikKICAgIGVjaG8gIlVuc3VwcG9ydGVkIGhvc3QgYXJjaDogJCh1bmFtZSAtbSkiID4mMgogICAgZXhpdCAxCiAgICA7OwogIGVzYWMKCiAgIyBJbiBjYXNlIG9mIGZhaWx1cmUgY2hlY2tpbmcgaW50ZWdyaXR5IG9mIHJlbGVhc2UsIHJldHJ5LgogIGNkICR7SU5TVEFMTF9ESVJ9L2JpbgogIHVudGlsIHRyeS1kb3dubG9hZC1yZWxlYXNlOyBkbwogICAgc2xlZXAgMTUKICAgIGVjaG8gIkNvdWxkbid0IGRvd25sb2FkIHJlbGVhc2UuIFJldHJ5aW5nLi4uIgogIGRvbmUKCiAgZWNobyAiUnVubmluZyBub2RldXAiCiAgIyBXZSBjYW4ndCBydW4gaW4gdGhlIGZvcmVncm91bmQgYmVjYXVzZSBvZiBodHRwczovL2dpdGh1Yi5jb20vZG9ja2VyL2RvY2tlci9pc3N1ZXMvMjM3OTMKICAoIGNkICR7SU5TVEFMTF9ESVJ9L2JpbjsgLi9ub2RldXAgLS1pbnN0YWxsLXN5c3RlbWQtdW5pdCAtLWNvbmY9JHtJTlNUQUxMX0RJUn0vY29uZi9rdWJlX2Vudi55YW1sIC0tdj04ICApCn0KCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwoKL2Jpbi9zeXN0ZW1kLW1hY2hpbmUtaWQtc2V0dXAgfHwgZWNobyAiZmFpbGVkIHRvIHNldCB1cCBlbnN1cmUgbWFjaGluZS1pZCBjb25maWd1cmVkIgoKZWNobyAiPT0gbm9kZXVwIG5vZGUgY29uZmlnIHN0YXJ0aW5nID09IgplbnN1cmUtaW5zdGFsbC1kaXIKCmNhdCA+IGNvbmYvY2x1c3Rlcl9zcGVjLnlhbWwgPDwgJ19fRU9GX0NMVVNURVJfU1BFQycKY2xvdWRDb25maWc6IG51bGwKY29udGFpbmVyUnVudGltZTogZG9ja2VyCmNvbnRhaW5lcmQ6CiAgc2tpcEluc3RhbGw6IHRydWUKZG9ja2VyOgogIGlwTWFzcTogZmFsc2UKICBpcFRhYmxlczogZmFsc2UKICBsb2dEcml2ZXI6IGpzb24tZmlsZQogIGxvZ0xldmVsOiBpbmZvCiAgbG9nT3B0OgogIC0gbWF4LXNpemU9MTBtCiAgLSBtYXgtZmlsZT01CiAgc3RvcmFnZTogb3ZlcmxheTIsb3ZlcmxheSxhdWZzCiAgdmVyc2lvbjogMTguMDYuMwprdWJlUHJveHk6CiAgY2x1c3RlckNJRFI6IDEwMC45Ni4wLjAvMTEKICBjcHVSZXF1ZXN0OiAxMDBtCiAgaG9zdG5hbWVPdmVycmlkZTogJ0Bhd3MnCiAgaW1hZ2U6IGs4cy5nY3IuaW8va3ViZS1wcm94eTp2MS4xNC4wCiAgbG9nTGV2ZWw6IDIKa3ViZWxldDoKICBhbm9ueW1vdXNBdXRoOiBmYWxzZQogIGNncm91cFJvb3Q6IC8KICBjbG91ZFByb3ZpZGVyOiBhd3MKICBjbHVzdGVyRE5TOiAxMDAuNjQuMC4xMAogIGNsdXN0ZXJEb21haW46IGNsdXN0ZXIubG9jYWwKICBlbmFibGVEZWJ1Z2dpbmdIYW5kbGVyczogdHJ1ZQogIGV2aWN0aW9uSGFyZDogbWVtb3J5LmF2YWlsYWJsZTwxMDBNaSxub2RlZnMuYXZhaWxhYmxlPDEwJSxub2RlZnMuaW5vZGVzRnJlZTw1JSxpbWFnZWZzLmF2YWlsYWJsZTwxMCUsaW1hZ2Vmcy5pbm9kZXNGcmVlPDUlCiAgZmVhdHVyZUdhdGVzOgogICAgRXhwZXJpbWVudGFsQ3JpdGljYWxQb2RBbm5vdGF0aW9uOiAidHJ1ZSIKICBob3N0bmFtZU92ZXJyaWRlOiAnQGF3cycKICBrdWJlY29uZmlnUGF0aDogL3Zhci9saWIva3ViZWxldC9rdWJlY29uZmlnCiAgbG9nTGV2ZWw6IDIKICBuZXR3b3JrUGx1Z2luTmFtZTogY25pCiAgbm9uTWFzcXVlcmFkZUNJRFI6IDEwMC42NC4wLjAvMTAKICBwb2RJbmZyYUNvbnRhaW5lckltYWdlOiBrOHMuZ2NyLmlvL3BhdXNlOjMuMgogIHBvZE1hbmlmZXN0UGF0aDogL2V0Yy9rdWJlcm5ldGVzL21hbmlmZXN0cwoKX19FT0ZfQ0xVU1RFUl9TUEVDCgpjYXQgPiBjb25mL2lnX3NwZWMueWFtbCA8PCAnX19FT0ZfSUdfU1BFQycKa3ViZWxldDogbnVsbApub2RlTGFiZWxzOiBudWxsCnRhaW50czogbnVsbAoKX19FT0ZfSUdfU1BFQwoKY2F0ID4gY29uZi9rdWJlX2Vudi55YW1sIDw8ICdfX0VPRl9LVUJFX0VOVicKQXNzZXRzOgotIGMzYjczNmZkMGYwMDM3NjVjMTJkOTlmMmM5OTVhODM2OWU2MjQxZjRAaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2t1YmVybmV0ZXMtcmVsZWFzZS9yZWxlYXNlL3YxLjE0LjAvYmluL2xpbnV4L2FtZDY0L2t1YmVsZXQKLSA3ZTNhM2VhNjYzMTUzZjkwMGNiZDUyOTAwYTM5YzkxZmE5ZjMzNGJlQGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9rdWJlcm5ldGVzLXJlbGVhc2UvcmVsZWFzZS92MS4xNC4wL2Jpbi9saW51eC9hbWQ2NC9rdWJlY3RsCi0gNTJlOWQyZGU4YTVmOTI3MzA3ZDkzOTczMDg3MzU2NThlZTQ0YWI4ZEBodHRwczovL3N0b3JhZ2UuZ29vZ2xlYXBpcy5jb20va3ViZXJuZXRlcy1yZWxlYXNlL25ldHdvcmstcGx1Z2lucy9jbmktcGx1Z2lucy1hbWQ2NC12MC43LjUudGd6CkNsdXN0ZXJOYW1lOiB1bm1hbmFnZWQuZXhhbXBsZS5jb20KQ29uZmlnQmFzZTogbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS91bm1hbmFnZWQuZXhhbXBsZS5jb20KSW5zdGFuY2VHcm91cE5hbWU6IG5vZGVzClRhZ3M6Ci0gX2F1dG9tYXRpY191cGdyYWRlcwotIF9hd3MKY2hhbm5lbHM6Ci0gbWVtZnM6Ly9jbHVzdGVycy5leGFtcGxlLmNvbS91bm1hbmFnZWQuZXhhbXBsZS5jb20vYWRkb25zL2Jvb3RzdHJhcC1jaGFubmVsLnlhbWwKCl9fRU9GX0tVQkVfRU5WCgpkb3dubG9hZC1yZWxlYXNlCmVjaG8gIj09IG5vZGV1cCBub2RlIGNvbmZpZyBkb25lID09Igo= +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL=https://artifacts.k8s.io/binaries/kops/1.15.0/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.15.0/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.15.0/linux/amd64/nodeup +NODEUP_HASH=9604ef18267ad7b5cf4cebbf7ab64423cf5bb0342d169c608ac6376e6af26d81 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL}" + NODEUP_HASH="${NODEUP_HASH}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginName: cni + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +kubelet: null +nodeLabels: null +taints: null + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: +- c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet +- 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl +- 52e9d2de8a5f927307d9397308735658ee44ab8d@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz +ClusterName: unmanaged.example.com +ConfigBase: memfs://clusters.example.com/unmanaged.example.com +InstanceGroupName: nodes +Tags: +- _automatic_upgrades +- _aws +channels: +- memfs://clusters.example.com/unmanaged.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/upup/pkg/fi/cloudup/awstasks/iamrole.go b/upup/pkg/fi/cloudup/awstasks/iamrole.go index 3b5087641f7c0..a2b36e29ddf14 100644 --- a/upup/pkg/fi/cloudup/awstasks/iamrole.go +++ b/upup/pkg/fi/cloudup/awstasks/iamrole.go @@ -195,7 +195,7 @@ type terraformIAMRole struct { } func (_ *IAMRole) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *IAMRole) error { - policy, err := t.AddFile("aws_iam_role", *e.Name, "policy", e.RolePolicyDocument) + policy, err := t.AddFile("aws_iam_role", *e.Name, "policy", e.RolePolicyDocument, false) if err != nil { return fmt.Errorf("error rendering RolePolicyDocument: %v", err) } diff --git a/upup/pkg/fi/cloudup/awstasks/iamrolepolicy.go b/upup/pkg/fi/cloudup/awstasks/iamrolepolicy.go index c045cd2ae9ec1..6d597cd8a0368 100644 --- a/upup/pkg/fi/cloudup/awstasks/iamrolepolicy.go +++ b/upup/pkg/fi/cloudup/awstasks/iamrolepolicy.go @@ -317,7 +317,7 @@ func (_ *IAMRolePolicy) RenderTerraform(t *terraform.TerraformTarget, a, e, chan return nil } - policy, err := t.AddFile("aws_iam_role_policy", *e.Name, "policy", e.PolicyDocument) + policy, err := t.AddFile("aws_iam_role_policy", *e.Name, "policy", e.PolicyDocument, false) if err != nil { return fmt.Errorf("error rendering PolicyDocument: %v", err) } diff --git a/upup/pkg/fi/cloudup/awstasks/launchconfiguration.go b/upup/pkg/fi/cloudup/awstasks/launchconfiguration.go index 7080869bad177..f47bf832b3aa5 100644 --- a/upup/pkg/fi/cloudup/awstasks/launchconfiguration.go +++ b/upup/pkg/fi/cloudup/awstasks/launchconfiguration.go @@ -543,7 +543,7 @@ func (_ *LaunchConfiguration) RenderTerraform(t *terraform.TerraformTarget, a, e return err } if userData != "" { - tf.UserData, err = t.AddFile("aws_launch_configuration", *e.Name, "user_data", e.UserData) + tf.UserData, err = t.AddFile("aws_launch_configuration", *e.Name, "user_data", e.UserData, false) if err != nil { return err } diff --git a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go index 276a8d27c7e2a..a6f931b33dce4 100644 --- a/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go +++ b/upup/pkg/fi/cloudup/awstasks/launchtemplate_target_terraform.go @@ -19,6 +19,7 @@ package awstasks import ( "encoding/base64" + "k8s.io/kops/pkg/featureflag" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/cloudup/terraform" @@ -212,13 +213,23 @@ func (t *LaunchTemplate) RenderTerraform(target *terraform.TerraformTarget, a, e if err != nil { return err } - b64d := base64.StdEncoding.EncodeToString(d) - if b64d != "" { - b64UserDataResource := fi.WrapResource(fi.NewStringResource(b64d)) + if d != nil { + if featureflag.Terraform012.Enabled() { + userDataResource := fi.WrapResource(fi.NewBytesResource(d)) - tf.UserData, err = target.AddFile("aws_launch_template", fi.StringValue(e.Name), "user_data", b64UserDataResource) - if err != nil { - return err + tf.UserData, err = target.AddFile("aws_launch_template", fi.StringValue(e.Name), "user_data", userDataResource, true) + if err != nil { + return err + } + } else { + b64d := base64.StdEncoding.EncodeToString(d) + if b64d != "" { + b64UserDataResource := fi.WrapResource(fi.NewStringResource(b64d)) + tf.UserData, err = target.AddFile("aws_launch_template", fi.StringValue(e.Name), "user_data", b64UserDataResource, false) + if err != nil { + return err + } + } } } } diff --git a/upup/pkg/fi/cloudup/awstasks/sshkey.go b/upup/pkg/fi/cloudup/awstasks/sshkey.go index f2c88cf092392..4bb78c697d4b9 100644 --- a/upup/pkg/fi/cloudup/awstasks/sshkey.go +++ b/upup/pkg/fi/cloudup/awstasks/sshkey.go @@ -176,7 +176,7 @@ func (_ *SSHKey) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *SS return nil } tfName := strings.Replace(*e.Name, ":", "", -1) - publicKey, err := t.AddFile("aws_key_pair", tfName, "public_key", e.PublicKey) + publicKey, err := t.AddFile("aws_key_pair", tfName, "public_key", e.PublicKey, false) if err != nil { return fmt.Errorf("error rendering PublicKey: %v", err) } diff --git a/upup/pkg/fi/cloudup/gcetasks/instancetemplate.go b/upup/pkg/fi/cloudup/gcetasks/instancetemplate.go index 967ca5d6a064d..d548bc9421940 100644 --- a/upup/pkg/fi/cloudup/gcetasks/instancetemplate.go +++ b/upup/pkg/fi/cloudup/gcetasks/instancetemplate.go @@ -500,7 +500,7 @@ func addMetadata(target *terraform.TerraformTarget, name string, metadata *compu val := fi.StringValue(g.Value) if strings.Contains(val, "\n") { v := fi.NewStringResource(val) - tfResource, err := target.AddFile("google_compute_instance_template", name, "metadata_"+g.Key, v) + tfResource, err := target.AddFile("google_compute_instance_template", name, "metadata_"+g.Key, v, false) if err != nil { return nil, err } diff --git a/upup/pkg/fi/cloudup/spotinsttasks/elastigroup.go b/upup/pkg/fi/cloudup/spotinsttasks/elastigroup.go index 82ba32b22b1ce..8529d034cc871 100644 --- a/upup/pkg/fi/cloudup/spotinsttasks/elastigroup.go +++ b/upup/pkg/fi/cloudup/spotinsttasks/elastigroup.go @@ -1430,7 +1430,7 @@ func (_ *Elastigroup) RenderTerraform(t *terraform.TerraformTarget, a, e, change // User data. if e.UserData != nil { var err error - tf.UserData, err = t.AddFile("spotinst_elastigroup_aws", *e.Name, "user_data", e.UserData) + tf.UserData, err = t.AddFile("spotinst_elastigroup_aws", *e.Name, "user_data", e.UserData, false) if err != nil { return err } diff --git a/upup/pkg/fi/cloudup/spotinsttasks/launch_spec.go b/upup/pkg/fi/cloudup/spotinsttasks/launch_spec.go index 86e90eec3d897..591895d29cfa1 100644 --- a/upup/pkg/fi/cloudup/spotinsttasks/launch_spec.go +++ b/upup/pkg/fi/cloudup/spotinsttasks/launch_spec.go @@ -667,7 +667,7 @@ func (_ *LaunchSpec) RenderTerraform(t *terraform.TerraformTarget, a, e, changes { if e.UserData != nil { var err error - tf.UserData, err = t.AddFile("spotinst_ocean_aws_launch_spec", *e.Name, "user_data", e.UserData) + tf.UserData, err = t.AddFile("spotinst_ocean_aws_launch_spec", *e.Name, "user_data", e.UserData, false) if err != nil { return err } diff --git a/upup/pkg/fi/cloudup/spotinsttasks/ocean.go b/upup/pkg/fi/cloudup/spotinsttasks/ocean.go index 41a773057c6ab..20c9e8b0740b8 100644 --- a/upup/pkg/fi/cloudup/spotinsttasks/ocean.go +++ b/upup/pkg/fi/cloudup/spotinsttasks/ocean.go @@ -1095,7 +1095,7 @@ func (_ *Ocean) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Oce // User data. if e.UserData != nil { var err error - tf.UserData, err = t.AddFile("spotinst_ocean_aws", *e.Name, "user_data", e.UserData) + tf.UserData, err = t.AddFile("spotinst_ocean_aws", *e.Name, "user_data", e.UserData, false) if err != nil { return err } diff --git a/upup/pkg/fi/cloudup/terraform/literal.go b/upup/pkg/fi/cloudup/terraform/literal.go index c7409788d3c0d..75dae890d1b1d 100644 --- a/upup/pkg/fi/cloudup/terraform/literal.go +++ b/upup/pkg/fi/cloudup/terraform/literal.go @@ -47,9 +47,13 @@ func (l *Literal) MarshalJSON() ([]byte, error) { return json.Marshal(&l.Value) } -func LiteralFileExpression(modulePath string) *Literal { +func LiteralFileExpression(modulePath string, base64 bool) *Literal { + fn := "file" + if base64 { + fn = "filebase64" + } return &Literal{ - Value: fmt.Sprintf("${file(%q)}", modulePath), + Value: fmt.Sprintf("${%v(%q)}", fn, modulePath), FilePath: modulePath, } } diff --git a/upup/pkg/fi/cloudup/terraform/target.go b/upup/pkg/fi/cloudup/terraform/target.go index 6f0b0ea389684..d0a31ed00b761 100644 --- a/upup/pkg/fi/cloudup/terraform/target.go +++ b/upup/pkg/fi/cloudup/terraform/target.go @@ -106,7 +106,7 @@ func tfSanitize(name string) string { return strings.NewReplacer(".", "-", "/", "--", ":", "_").Replace(name) } -func (t *TerraformTarget) AddFile(resourceType string, resourceName string, key string, r fi.Resource) (*Literal, error) { +func (t *TerraformTarget) AddFile(resourceType string, resourceName string, key string, r fi.Resource, base64 bool) (*Literal, error) { id := resourceType + "_" + resourceName + "_" + key d, err := fi.ResourceAsBytes(r) @@ -121,7 +121,7 @@ func (t *TerraformTarget) AddFile(resourceType string, resourceName string, key t.files[p] = d modulePath := path.Join("${path.module}", p) - l := LiteralFileExpression(modulePath) + l := LiteralFileExpression(modulePath, base64) return l, nil }