From be0da5a9f6fdb2ff9051c5c84a1e098ac1dd1f68 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 25 Aug 2020 11:02:07 -0400 Subject: [PATCH] Expose JWKS via a feature-flag When the PublicJWKS feature-flag is set, we expose the apiserver JWKS document publicly (including enabling anonymous access). This is a stepping stone to a more hardened configuration where we copy the JWKS document to S3/GCS/etc. --- cloudmock/aws/mockiam/oidcprovider.go | 26 +- cmd/kops/integration_test.go | 43 +- pkg/featureflag/featureflag.go | 2 + pkg/model/awsmodel/BUILD.bazel | 3 + pkg/model/awsmodel/oidc_provider.go | 79 +++ pkg/model/components/BUILD.bazel | 3 + pkg/model/components/apiserver.go | 7 + pkg/model/components/discovery.go | 83 +++ pkg/model/iam/BUILD.bazel | 2 + pkg/model/iam/pod_roles.go | 33 ++ .../integration/update_cluster/jwks/README.md | 9 + tests/integration/update_cluster/jwks/ca.crt | 11 + tests/integration/update_cluster/jwks/ca.key | 10 + ...am_role_masters.minimal.example.com_policy | 10 + ..._iam_role_nodes.minimal.example.com_policy | 10 + ..._policy_masters.minimal.example.com_policy | 170 ++++++ ...le_policy_nodes.minimal.example.com_policy | 15 + ...4a6ed9aa889b9e2c39cd663eb9c7157_public_key | 1 + ...t-1a.masters.minimal.example.com_user_data | 333 +++++++++++ ...mplate_nodes.minimal.example.com_user_data | 237 ++++++++ .../update_cluster/jwks/id_rsa.pub | 1 + .../update_cluster/jwks/in-v1alpha2.yaml | 80 +++ .../update_cluster/jwks/kubernetes.tf | 548 ++++++++++++++++++ upup/models/bindata.go | 36 ++ .../k8s-1.16.yaml.template | 15 + upup/pkg/fi/cloudup/BUILD.bazel | 1 + upup/pkg/fi/cloudup/apply_cluster.go | 1 + .../fi/cloudup/awstasks/iamoidcprovider.go | 87 ++- .../pkg/fi/cloudup/bootstrapchannelbuilder.go | 19 + .../cloudup/bootstrapchannelbuilder_test.go | 15 + upup/pkg/fi/cloudup/populate_cluster_spec.go | 1 + ...onymous-access.addons.k8s.io-k8s-1.16.yaml | 15 + .../bootstrapchannelbuilder/jwks/cluster.yaml | 42 ++ ...dns-controller.addons.k8s.io-k8s-1.12.yaml | 109 ++++ ...ops-controller.addons.k8s.io-k8s-1.16.yaml | 175 ++++++ .../jwks/manifest.yaml | 96 +++ upup/pkg/fi/fitasks/keypair.go | 49 ++ upup/pkg/fi/resources.go | 13 +- upup/pkg/fi/topological_sort.go | 2 +- 39 files changed, 2356 insertions(+), 36 deletions(-) create mode 100644 pkg/model/awsmodel/oidc_provider.go create mode 100644 pkg/model/components/discovery.go create mode 100644 pkg/model/iam/pod_roles.go create mode 100644 tests/integration/update_cluster/jwks/README.md create mode 100644 tests/integration/update_cluster/jwks/ca.crt create mode 100644 tests/integration/update_cluster/jwks/ca.key create mode 100644 tests/integration/update_cluster/jwks/data/aws_iam_role_masters.minimal.example.com_policy create mode 100644 tests/integration/update_cluster/jwks/data/aws_iam_role_nodes.minimal.example.com_policy create mode 100644 tests/integration/update_cluster/jwks/data/aws_iam_role_policy_masters.minimal.example.com_policy create mode 100644 tests/integration/update_cluster/jwks/data/aws_iam_role_policy_nodes.minimal.example.com_policy create mode 100644 tests/integration/update_cluster/jwks/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key create mode 100644 tests/integration/update_cluster/jwks/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data create mode 100644 tests/integration/update_cluster/jwks/data/aws_launch_template_nodes.minimal.example.com_user_data create mode 100755 tests/integration/update_cluster/jwks/id_rsa.pub create mode 100644 tests/integration/update_cluster/jwks/in-v1alpha2.yaml create mode 100644 tests/integration/update_cluster/jwks/kubernetes.tf create mode 100644 upup/models/cloudup/resources/addons/anonymous-access.addons.k8s.io/k8s-1.16.yaml.template create mode 100644 upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/anonymous-access.addons.k8s.io-k8s-1.16.yaml create mode 100644 upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/cluster.yaml create mode 100644 upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/dns-controller.addons.k8s.io-k8s-1.12.yaml create mode 100644 upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/kops-controller.addons.k8s.io-k8s-1.16.yaml create mode 100644 upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/manifest.yaml diff --git a/cloudmock/aws/mockiam/oidcprovider.go b/cloudmock/aws/mockiam/oidcprovider.go index e7b15b7e8f0e3..fb30a18137a28 100644 --- a/cloudmock/aws/mockiam/oidcprovider.go +++ b/cloudmock/aws/mockiam/oidcprovider.go @@ -17,6 +17,7 @@ limitations under the License. package mockiam import ( + "context" "fmt" "github.com/aws/aws-sdk-go/aws" @@ -48,11 +49,28 @@ func (m *MockIAM) ListOpenIDConnectProvidersRequest(*iam.ListOpenIDConnectProvid panic("Not implemented") } -func (m *MockIAM) GetOpenIDConnectProviderWithContext(aws.Context, *iam.GetOpenIDConnectProviderInput, ...request.Option) (*iam.GetOpenIDConnectProviderOutput, error) { - panic("Not implemented") +func (m *MockIAM) GetOpenIDConnectProviderWithContext(ctx aws.Context, request *iam.GetOpenIDConnectProviderInput, options ...request.Option) (*iam.GetOpenIDConnectProviderOutput, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + arn := aws.StringValue(request.OpenIDConnectProviderArn) + + provider := m.OIDCProviders[arn] + if provider == nil { + return nil, fmt.Errorf("OpenIDConnectProvider with arn=%q not found", arn) + } + + response := &iam.GetOpenIDConnectProviderOutput{ + ClientIDList: provider.ClientIDList, + CreateDate: provider.CreateDate, + ThumbprintList: provider.ThumbprintList, + Url: provider.Url, + } + return response, nil } -func (m *MockIAM) GetOpenIDConnectProvider(*iam.GetOpenIDConnectProviderInput) (*iam.GetOpenIDConnectProviderOutput, error) { - panic("Not implemented") + +func (m *MockIAM) GetOpenIDConnectProvider(request *iam.GetOpenIDConnectProviderInput) (*iam.GetOpenIDConnectProviderOutput, error) { + return m.GetOpenIDConnectProviderWithContext(context.Background(), request) } func (m *MockIAM) GetOpenIDConnectProviderRequest(*iam.GetOpenIDConnectProviderInput) (*request.Request, *iam.GetOpenIDConnectProviderOutput) { diff --git a/cmd/kops/integration_test.go b/cmd/kops/integration_test.go index 2901ac27338e8..0858ad328ef33 100644 --- a/cmd/kops/integration_test.go +++ b/cmd/kops/integration_test.go @@ -61,8 +61,10 @@ type integrationTest struct { launchConfiguration bool lifecycleOverrides []string sshKey bool - jsonOutput bool - bastionUserData bool + // caKey is true if we should use a provided ca.crt & ca.key as our CA + caKey bool + jsonOutput bool + bastionUserData bool } func newIntegrationTest(clusterName, srcDir string) *integrationTest { @@ -91,6 +93,13 @@ func (i *integrationTest) withoutSSHKey() *integrationTest { return i } +// withCAKey indicates that we should use a fixed ca.crt & ca.key from the source directory as our CA. +// This is needed when the CA is exposed, for example when using AWS WebIdentity federation. +func (i *integrationTest) withCAKey() *integrationTest { + i.caKey = true + return i +} + func (i *integrationTest) withoutPolicies() *integrationTest { i.expectPolicies = false return i @@ -314,15 +323,27 @@ func TestContainerdCloudformation(t *testing.T) { // TestLaunchConfigurationASG tests ASGs using launch configurations instead of launch templates func TestLaunchConfigurationASG(t *testing.T) { featureflag.ParseFlags("-EnableLaunchTemplates") - unsetFeaureFlag := func() { + unsetFeatureFlags := func() { featureflag.ParseFlags("+EnableLaunchTemplates") } - defer unsetFeaureFlag() + defer unsetFeatureFlags() newIntegrationTest("launchtemplates.example.com", "launch_templates").withZones(3).withLaunchConfiguration().runTestTerraformAWS(t) newIntegrationTest("launchtemplates.example.com", "launch_templates").withZones(3).withLaunchConfiguration().runTestCloudformation(t) } +// TestJWKS runs a simple configuration, but with PublicJWKS enabled +func TestJWKS(t *testing.T) { + featureflag.ParseFlags("+PublicJWKS") + unsetFeatureFlags := func() { + featureflag.ParseFlags("-PublicJWKS") + } + defer unsetFeatureFlags() + + // We have to use a fixed CA because the fingerprint is inserted into the AWS WebIdentity configuration. + newIntegrationTest("minimal.example.com", "jwks").withCAKey().runTestTerraformAWS(t) +} + func (i *integrationTest) runTest(t *testing.T, h *testutils.IntegrationTestHarness, expectedDataFilenames []string, tfFileName string, expectedTfFileName string, phase *cloudup.Phase) { ctx := context.Background() @@ -364,7 +385,19 @@ func (i *integrationTest) runTest(t *testing.T, h *testutils.IntegrationTestHarn err := RunCreateSecretPublicKey(ctx, factory, &stdout, options) if err != nil { - t.Fatalf("error running %q create: %v", inputYAML, err) + t.Fatalf("error running %q create public key: %v", inputYAML, err) + } + } + + if i.caKey { + options := &CreateSecretCaCertOptions{} + options.ClusterName = i.clusterName + options.CaPrivateKeyPath = path.Join(i.srcDir, "ca.key") + options.CaCertPath = path.Join(i.srcDir, "ca.crt") + + err := RunCreateSecretCaCert(ctx, factory, &stdout, options) + if err != nil { + t.Fatalf("error running %q create CA keypair: %v", inputYAML, err) } } diff --git a/pkg/featureflag/featureflag.go b/pkg/featureflag/featureflag.go index 3ae14150aba58..21689e38e289c 100644 --- a/pkg/featureflag/featureflag.go +++ b/pkg/featureflag/featureflag.go @@ -92,6 +92,8 @@ var ( Terraform012 = New("Terraform-0.12", Bool(true)) // LegacyIAM will permit use of legacy IAM permissions. LegacyIAM = New("LegacyIAM", Bool(false)) + // PublicJWKS enables public jwks access. This is generally not as secure as republishing. + PublicJWKS = New("PublicJWKS", Bool(false)) ) // FeatureFlag defines a feature flag diff --git a/pkg/model/awsmodel/BUILD.bazel b/pkg/model/awsmodel/BUILD.bazel index dcc36a5a1e95c..b7b2ccfa8280c 100644 --- a/pkg/model/awsmodel/BUILD.bazel +++ b/pkg/model/awsmodel/BUILD.bazel @@ -6,6 +6,7 @@ go_library( "api_loadbalancer.go", "autoscalinggroup.go", "context.go", + "oidc_provider.go", ], importpath = "k8s.io/kops/pkg/model/awsmodel", visibility = ["//visibility:public"], @@ -15,9 +16,11 @@ go_library( "//pkg/featureflag:go_default_library", "//pkg/model:go_default_library", "//pkg/model/defaults:go_default_library", + "//pkg/model/iam:go_default_library", "//pkg/model/spotinstmodel:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/awstasks:go_default_library", + "//upup/pkg/fi/fitasks:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/klog/v2:go_default_library", diff --git a/pkg/model/awsmodel/oidc_provider.go b/pkg/model/awsmodel/oidc_provider.go new file mode 100644 index 0000000000000..5cf129decc54b --- /dev/null +++ b/pkg/model/awsmodel/oidc_provider.go @@ -0,0 +1,79 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package awsmodel + +import ( + "fmt" + + "k8s.io/kops/pkg/featureflag" + "k8s.io/kops/pkg/model" + "k8s.io/kops/pkg/model/iam" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awstasks" + "k8s.io/kops/upup/pkg/fi/fitasks" +) + +// OIDCProviderBuilder configures IAM OIDC Provider +type OIDCProviderBuilder struct { + *model.KopsModelContext + + KeyStore fi.CAStore + Lifecycle *fi.Lifecycle +} + +var _ fi.ModelBuilder = &OIDCProviderBuilder{} + +const ( + defaultAudience = "amazonaws.com" +) + +func (b *OIDCProviderBuilder) Build(c *fi.ModelBuilderContext) error { + var thumbprints []fi.Resource + var issuerURL string + + if featureflag.PublicJWKS.Enabled() { + serviceAccountIssuer, err := iam.ServiceAccountIssuer(b.ClusterName(), &b.Cluster.Spec) + if err != nil { + return err + } + issuerURL = serviceAccountIssuer + + caTaskObject, found := c.Tasks["Keypair/ca"] + if !found { + return fmt.Errorf("keypair/ca task not found") + } + + caTask := caTaskObject.(*fitasks.Keypair) + fingerprint := caTask.CertificateSHA1Fingerprint() + + thumbprints = []fi.Resource{fingerprint} + } + + if issuerURL == "" { + return nil + } + + c.AddTask(&awstasks.IAMOIDCProvider{ + Name: fi.String(b.ClusterName()), + Lifecycle: b.Lifecycle, + URL: fi.String(issuerURL), + ClientIDs: []*string{fi.String(defaultAudience)}, + Thumbprints: thumbprints, + }) + + return nil +} diff --git a/pkg/model/components/BUILD.bazel b/pkg/model/components/BUILD.bazel index f800296e6f92d..b27123957d0a1 100644 --- a/pkg/model/components/BUILD.bazel +++ b/pkg/model/components/BUILD.bazel @@ -8,6 +8,7 @@ go_library( "containerd.go", "context.go", "defaults.go", + "discovery.go", "docker.go", "etcd.go", "kubecontrollermanager.go", @@ -24,7 +25,9 @@ go_library( "//pkg/apis/kops:go_default_library", "//pkg/apis/kops/util:go_default_library", "//pkg/assets:go_default_library", + "//pkg/featureflag:go_default_library", "//pkg/k8sversion:go_default_library", + "//pkg/model/iam:go_default_library", "//pkg/wellknownports:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/gce:go_default_library", diff --git a/pkg/model/components/apiserver.go b/pkg/model/components/apiserver.go index f290d25abc043..281d07bc234ac 100644 --- a/pkg/model/components/apiserver.go +++ b/pkg/model/components/apiserver.go @@ -22,6 +22,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/featureflag" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/loader" @@ -208,6 +209,12 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(o interface{}) error { // We make sure to disable AnonymousAuth c.AnonymousAuth = fi.Bool(false) + // For the development of JWKS functionality, re-enable anonymous auth for PublicJWKS + // (this likely isn't a production-suitable configuration, currently) + if featureflag.PublicJWKS.Enabled() { + c.AnonymousAuth = fi.Bool(true) + } + if b.IsKubernetesGTE("1.17") { // We query via the kube-apiserver-healthcheck proxy, which listens on port 3990 c.InsecurePort = 0 diff --git a/pkg/model/components/discovery.go b/pkg/model/components/discovery.go new file mode 100644 index 0000000000000..4e8979640b839 --- /dev/null +++ b/pkg/model/components/discovery.go @@ -0,0 +1,83 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package components + +import ( + "strings" + + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/featureflag" + "k8s.io/kops/pkg/model/iam" + "k8s.io/kops/upup/pkg/fi/loader" +) + +// DiscoveryOptionsBuilder adds options for identity discovery to the model (mostly kube-apiserver) +type DiscoveryOptionsBuilder struct { + *OptionsContext +} + +var _ loader.OptionsBuilder = &DiscoveryOptionsBuilder{} + +func (b *DiscoveryOptionsBuilder) BuildOptions(o interface{}) error { + clusterSpec := o.(*kops.ClusterSpec) + + useJWKS := featureflag.PublicJWKS.Enabled() + if !useJWKS { + return nil + } + + if clusterSpec.KubeAPIServer == nil { + clusterSpec.KubeAPIServer = &kops.KubeAPIServerConfig{} + } + + kubeAPIServer := clusterSpec.KubeAPIServer + + if kubeAPIServer.FeatureGates == nil { + kubeAPIServer.FeatureGates = make(map[string]string) + } + kubeAPIServer.FeatureGates["ServiceAccountIssuerDiscovery"] = "true" + + if len(kubeAPIServer.APIAudiences) == 0 { + kubeAPIServer.APIAudiences = []string{"kubernetes.svc.default"} + } + + if kubeAPIServer.ServiceAccountIssuer == nil { + serviceAccountIssuer, err := iam.ServiceAccountIssuer(b.ClusterName, clusterSpec) + if err != nil { + return err + } + kubeAPIServer.ServiceAccountIssuer = &serviceAccountIssuer + } + + if kubeAPIServer.ServiceAccountJWKSURI == nil { + jwksURL := *kubeAPIServer.ServiceAccountIssuer + jwksURL = strings.TrimSuffix(jwksURL, "/") + "/openid/v1/jwks" + + kubeAPIServer.ServiceAccountJWKSURI = &jwksURL + } + + if kubeAPIServer.ServiceAccountSigningKeyFile == nil { + s := "/srv/kubernetes/server.key" + kubeAPIServer.ServiceAccountSigningKeyFile = &s + } + + if len(kubeAPIServer.ServiceAccountKeyFile) == 0 { + kubeAPIServer.ServiceAccountKeyFile = []string{"/srv/kubernetes/server.key"} + } + + return nil +} diff --git a/pkg/model/iam/BUILD.bazel b/pkg/model/iam/BUILD.bazel index 7b715453fb54b..6a60ed2d754f5 100644 --- a/pkg/model/iam/BUILD.bazel +++ b/pkg/model/iam/BUILD.bazel @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "iam_builder.go", + "pod_roles.go", "types.go", ], importpath = "k8s.io/kops/pkg/model/iam", @@ -11,6 +12,7 @@ go_library( deps = [ "//pkg/apis/kops:go_default_library", "//pkg/apis/kops/model:go_default_library", + "//pkg/featureflag:go_default_library", "//pkg/util/stringorslice:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/awstasks:go_default_library", diff --git a/pkg/model/iam/pod_roles.go b/pkg/model/iam/pod_roles.go new file mode 100644 index 0000000000000..d8bf104829152 --- /dev/null +++ b/pkg/model/iam/pod_roles.go @@ -0,0 +1,33 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package iam + +import ( + "fmt" + + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/featureflag" +) + +// ServiceAccountIssuer determines the issuer in the ServiceAccount JWTs +func ServiceAccountIssuer(clusterName string, clusterSpec *kops.ClusterSpec) (string, error) { + if featureflag.PublicJWKS.Enabled() { + return "https://api." + clusterName, nil + } + + return "", fmt.Errorf("ServiceAcccountIssuer not (currently) supported without PublicJWKS") +} diff --git a/tests/integration/update_cluster/jwks/README.md b/tests/integration/update_cluster/jwks/README.md new file mode 100644 index 0000000000000..9f2479a8bb4ca --- /dev/null +++ b/tests/integration/update_cluster/jwks/README.md @@ -0,0 +1,9 @@ +Simple test of (experimental) JWKS functionality + +We have to use a fixed CA because the fingerprint is inserted into the AWS WebIdentity configuration. + +ca.crt & ca.key generated with: + +`openssl req -new -newkey rsa:512 -days 3650 -nodes -x509 -subj "/CN=kubernetes" -keyout ca.key -out ca.crt` + + diff --git a/tests/integration/update_cluster/jwks/ca.crt b/tests/integration/update_cluster/jwks/ca.crt new file mode 100644 index 0000000000000..f195442cd8302 --- /dev/null +++ b/tests/integration/update_cluster/jwks/ca.crt @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBgTCCASugAwIBAgIUZrxLCo6MlBXbjRWuIBXdlRkM2EcwDQYJKoZIhvcNAQEL +BQAwFTETMBEGA1UEAwwKa3ViZXJuZXRlczAeFw0yMDA4MTUyMTM3NDhaFw0zMDA4 +MTMyMTM3NDhaMBUxEzARBgNVBAMMCmt1YmVybmV0ZXMwXDANBgkqhkiG9w0BAQEF +AANLADBIAkEA5eJVxg/iR9zq2wQrk2VjdavGYiPu1Q0cmNb4LvItHBO0eiSVA7EV +D/7qAgnB13ASaQHLMuG50qK3wihMJC9/6QIDAQABo1MwUTAdBgNVHQ4EFgQU4/Jf +ZYu5ziuhZRnpcxvDOlYGA+4wHwYDVR0jBBgwFoAU4/JfZYu5ziuhZRnpcxvDOlYG +A+4wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAANBAEHceMm6tpH6Yc+H +5uu5wY8Q4pmYJt+HOkIpoXO1KD4/8h90y6XY8Z0Nu3dOZSwBSCWChrYAIndtzJfC +PtQHwNM= +-----END CERTIFICATE----- diff --git a/tests/integration/update_cluster/jwks/ca.key b/tests/integration/update_cluster/jwks/ca.key new file mode 100644 index 0000000000000..c8a4715ae7236 --- /dev/null +++ b/tests/integration/update_cluster/jwks/ca.key @@ -0,0 +1,10 @@ +-----BEGIN PRIVATE KEY----- +MIIBVQIBADANBgkqhkiG9w0BAQEFAASCAT8wggE7AgEAAkEA5eJVxg/iR9zq2wQr +k2VjdavGYiPu1Q0cmNb4LvItHBO0eiSVA7EVD/7qAgnB13ASaQHLMuG50qK3wihM +JC9/6QIDAQABAkEAug/7RJfOmkOggyxY6LADVFZ39y8GO8KlBr/XmIfDIxj20yIG +W2SmoSGPqoWDpr8G2LUSVrdaQ9ZyDqG0AqUN0QIhAPx5JQRoRDo2hiS+Ioaty/NA +7/iInYFkS5hMvud1QSKDAiEA6RhpLIFZbLAoof6/fdIUy7QWU1UHJ6PKq/3qpR7u +mCMCIQCVmHKGmgFTPNtfCgoLIw+louSNruUktfjU1SSIoMFnYQIgLxR8Ib4ahsZp +3pZqrQoioyZDoB87a7k8dVK68xD1VgsCIHFjAVxGmS2MgT80UjwPNs9XkT5WOpoR +BzhivO3D3oOn +-----END PRIVATE KEY----- diff --git a/tests/integration/update_cluster/jwks/data/aws_iam_role_masters.minimal.example.com_policy b/tests/integration/update_cluster/jwks/data/aws_iam_role_masters.minimal.example.com_policy new file mode 100644 index 0000000000000..66d5de1d5ae1e --- /dev/null +++ b/tests/integration/update_cluster/jwks/data/aws_iam_role_masters.minimal.example.com_policy @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/tests/integration/update_cluster/jwks/data/aws_iam_role_nodes.minimal.example.com_policy b/tests/integration/update_cluster/jwks/data/aws_iam_role_nodes.minimal.example.com_policy new file mode 100644 index 0000000000000..66d5de1d5ae1e --- /dev/null +++ b/tests/integration/update_cluster/jwks/data/aws_iam_role_nodes.minimal.example.com_policy @@ -0,0 +1,10 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/tests/integration/update_cluster/jwks/data/aws_iam_role_policy_masters.minimal.example.com_policy b/tests/integration/update_cluster/jwks/data/aws_iam_role_policy_masters.minimal.example.com_policy new file mode 100644 index 0000000000000..107ccaf8e6c24 --- /dev/null +++ b/tests/integration/update_cluster/jwks/data/aws_iam_role_policy_masters.minimal.example.com_policy @@ -0,0 +1,170 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeInstances", + "ec2:DescribeInternetGateways", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes" + ], + "Resource": [ + "*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:DescribeVolumesModifications", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume" + ], + "Resource": [ + "*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateRoute", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:RevokeSecurityGroupIngress" + ], + "Resource": [ + "*" + ], + "Condition": { + "StringEquals": { + "ec2:ResourceTag/KubernetesCluster": "minimal.example.com" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeLaunchTemplateVersions" + ], + "Resource": [ + "*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "autoscaling:UpdateAutoScalingGroup" + ], + "Resource": [ + "*" + ], + "Condition": { + "StringEquals": { + "autoscaling:ResourceTag/KubernetesCluster": "minimal.example.com" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer" + ], + "Resource": [ + "*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener" + ], + "Resource": [ + "*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "iam:ListServerCertificates", + "iam:GetServerCertificate" + ], + "Resource": [ + "*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "route53:ChangeResourceRecordSets", + "route53:ListResourceRecordSets", + "route53:GetHostedZone" + ], + "Resource": [ + "arn:aws:route53:::hostedzone/Z1AFAKE1ZON3YO" + ] + }, + { + "Effect": "Allow", + "Action": [ + "route53:GetChange" + ], + "Resource": [ + "arn:aws:route53:::change/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "route53:ListHostedZones" + ], + "Resource": [ + "*" + ] + } + ] +} diff --git a/tests/integration/update_cluster/jwks/data/aws_iam_role_policy_nodes.minimal.example.com_policy b/tests/integration/update_cluster/jwks/data/aws_iam_role_policy_nodes.minimal.example.com_policy new file mode 100644 index 0000000000000..ec7bf70d63350 --- /dev/null +++ b/tests/integration/update_cluster/jwks/data/aws_iam_role_policy_nodes.minimal.example.com_policy @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeRegions" + ], + "Resource": [ + "*" + ] + } + ] +} diff --git a/tests/integration/update_cluster/jwks/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key b/tests/integration/update_cluster/jwks/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key new file mode 100644 index 0000000000000..81cb0127830e7 --- /dev/null +++ b/tests/integration/update_cluster/jwks/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/jwks/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data b/tests/integration/update_cluster/jwks/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data new file mode 100644 index 0000000000000..6fa2ebdc5596f --- /dev/null +++ b/tests/integration/update_cluster/jwks/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data @@ -0,0 +1,333 @@ +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.1/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.1/linux/amd64/nodeup +NODEUP_HASH_AMD64=deff1179f80bd65232af30c252a2b5f390b160bbfe9990637c0ec88d76cdd647 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.1/linux-arm64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.1/linux/arm64/nodeup +NODEUP_HASH_ARM64=300bfade024ede4fe8996c9ee6393e97463bea4eaffd0876310710dcdebbf8d3 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +encryptionConfig: null +etcdClusters: + events: + version: 3.3.10 + main: + version: 3.3.10 +kubeAPIServer: + allowPrivileged: true + anonymousAuth: true + apiAudiences: + - kubernetes.svc.default + apiServerCount: 1 + authorizationMode: AlwaysAllow + bindAddress: 0.0.0.0 + cloudProvider: aws + enableAdmissionPlugins: + - NamespaceLifecycle + - LimitRanger + - ServiceAccount + - PersistentVolumeLabel + - DefaultStorageClass + - DefaultTolerationSeconds + - MutatingAdmissionWebhook + - ValidatingAdmissionWebhook + - NodeRestriction + - ResourceQuota + etcdServers: + - http://127.0.0.1:4001 + etcdServersOverrides: + - /events#http://127.0.0.1:4002 + featureGates: + ServiceAccountIssuerDiscovery: "true" + image: k8s.gcr.io/kube-apiserver:v1.14.0 + insecureBindAddress: 127.0.0.1 + insecurePort: 8080 + kubeletPreferredAddressTypes: + - InternalIP + - Hostname + - ExternalIP + logLevel: 2 + requestheaderAllowedNames: + - aggregator + requestheaderExtraHeaderPrefixes: + - X-Remote-Extra- + requestheaderGroupHeaders: + - X-Remote-Group + requestheaderUsernameHeaders: + - X-Remote-User + securePort: 443 + serviceAccountIssuer: https://api.minimal.example.com + serviceAccountJWKSURI: https://api.minimal.example.com/openid/v1/jwks + serviceAccountKeyFile: + - /srv/kubernetes/server.key + serviceAccountSigningKeyFile: /srv/kubernetes/server.key + serviceClusterIPRange: 100.64.0.0/13 + storageBackend: etcd3 +kubeControllerManager: + allocateNodeCIDRs: true + attachDetachReconcileSyncPeriod: 1m0s + cloudProvider: aws + clusterCIDR: 100.96.0.0/11 + clusterName: minimal.example.com + configureCloudRoutes: true + image: k8s.gcr.io/kube-controller-manager:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 + useServiceAccountCredentials: true +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubeScheduler: + image: k8s.gcr.io/kube-scheduler:v1.14.0 + leaderElection: + leaderElect: true + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +masterKubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +{} + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: + amd64: + - c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet + - 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl + - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz + arm64: + - df38e04576026393055ccc77c0dce73612996561@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubelet + - 01c2b6b43d36b6bfafc80a3737391c19ebfb8ad5@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubectl + - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz +ClusterName: minimal.example.com +ConfigBase: memfs://clusters.example.com/minimal.example.com +InstanceGroupName: master-us-test-1a +InstanceGroupRole: Master +KubeletConfig: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nodeLabels: + kubernetes.io/role: master + node-role.kubernetes.io/master: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + registerSchedulable: false +channels: +- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml +etcdManifests: +- memfs://clusters.example.com/minimal.example.com/manifests/etcd/main.yaml +- memfs://clusters.example.com/minimal.example.com/manifests/etcd/events.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/jwks/data/aws_launch_template_nodes.minimal.example.com_user_data b/tests/integration/update_cluster/jwks/data/aws_launch_template_nodes.minimal.example.com_user_data new file mode 100644 index 0000000000000..ea7f8190da879 --- /dev/null +++ b/tests/integration/update_cluster/jwks/data/aws_launch_template_nodes.minimal.example.com_user_data @@ -0,0 +1,237 @@ +#!/bin/bash +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.1/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.1/linux-amd64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.1/linux/amd64/nodeup +NODEUP_HASH_AMD64=deff1179f80bd65232af30c252a2b5f390b160bbfe9990637c0ec88d76cdd647 +NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.1/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.1/linux-arm64-nodeup,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.1/linux/arm64/nodeup +NODEUP_HASH_ARM64=300bfade024ede4fe8996c9ee6393e97463bea4eaffd0876310710dcdebbf8d3 + +export AWS_REGION=us-test-1 + + + + +function ensure-install-dir() { + INSTALL_DIR="/opt/kops" + # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec + if [[ -d /var/lib/toolbox ]]; then + INSTALL_DIR="/var/lib/toolbox/kops" + fi + mkdir -p ${INSTALL_DIR}/bin + mkdir -p ${INSTALL_DIR}/conf + cd ${INSTALL_DIR} +} + +# Retry a download until we get it. args: name, sha, url1, url2... +download-or-bust() { + local -r file="$1" + local -r hash="$2" + shift 2 + + urls=( $* ) + while true; do + for url in "${urls[@]}"; do + commands=( + "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10" + "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10" + ) + for cmd in "${commands[@]}"; do + echo "Attempting download with: ${cmd} {url}" + if ! (${cmd} "${url}"); then + echo "== Download failed with ${cmd} ==" + continue + fi + if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then + echo "== Hash validation of ${url} failed. Retrying. ==" + rm -f "${file}" + else + if [[ -n "${hash}" ]]; then + echo "== Downloaded ${url} (SHA1 = ${hash}) ==" + else + echo "== Downloaded ${url} ==" + fi + return + fi + done + done + + echo "All downloads failed; sleeping before retrying" + sleep 60 + done +} + +validate-hash() { + local -r file="$1" + local -r expected="$2" + local actual + + actual=$(sha256sum ${file} | awk '{ print $1 }') || true + if [[ "${actual}" != "${expected}" ]]; then + echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} ==" + return 1 + fi +} + +function split-commas() { + echo $1 | tr "," "\n" +} + +function try-download-release() { + local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") ) + if [[ -n "${NODEUP_HASH:-}" ]]; then + local -r nodeup_hash="${NODEUP_HASH}" + else + # TODO: Remove? + echo "Downloading sha256 (not found in env)" + download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}" + local -r nodeup_hash=$(cat nodeup.sha256) + fi + + echo "Downloading nodeup (${nodeup_urls[@]})" + download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}" + + chmod +x nodeup +} + +function download-release() { + case "$(uname -m)" in + x86_64*|i?86_64*|amd64*) + NODEUP_URL="${NODEUP_URL_AMD64}" + NODEUP_HASH="${NODEUP_HASH_AMD64}" + ;; + aarch64*|arm64*) + NODEUP_URL="${NODEUP_URL_ARM64}" + NODEUP_HASH="${NODEUP_HASH_ARM64}" + ;; + *) + echo "Unsupported host arch: $(uname -m)" >&2 + exit 1 + ;; + esac + + # In case of failure checking integrity of release, retry. + cd ${INSTALL_DIR}/bin + until try-download-release; do + sleep 15 + echo "Couldn't download release. Retrying..." + done + + echo "Running nodeup" + # We can't run in the foreground because of https://github.com/docker/docker/issues/23793 + ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8 ) +} + +#################################################################################### + +/bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured" + +echo "== nodeup node config starting ==" +ensure-install-dir + +cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC' +cloudConfig: null +containerRuntime: docker +containerd: + skipInstall: true +docker: + ipMasq: false + ipTables: false + logDriver: json-file + logLevel: info + logOpt: + - max-size=10m + - max-file=5 + storage: overlay2,overlay,aufs + version: 18.06.3 +kubeProxy: + clusterCIDR: 100.96.0.0/11 + cpuRequest: 100m + hostnameOverride: '@aws' + image: k8s.gcr.io/kube-proxy:v1.14.0 + logLevel: 2 +kubelet: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests + +__EOF_CLUSTER_SPEC + +cat > conf/ig_spec.yaml << '__EOF_IG_SPEC' +{} + +__EOF_IG_SPEC + +cat > conf/kube_env.yaml << '__EOF_KUBE_ENV' +Assets: + amd64: + - c3b736fd0f003765c12d99f2c995a8369e6241f4@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubelet + - 7e3a3ea663153f900cbd52900a39c91fa9f334be@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/amd64/kubectl + - 3ca15c0a18ee830520cf3a95408be826cbd255a1535a38e0be9608b25ad8bf64@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-amd64-v0.7.5.tgz + arm64: + - df38e04576026393055ccc77c0dce73612996561@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubelet + - 01c2b6b43d36b6bfafc80a3737391c19ebfb8ad5@https://storage.googleapis.com/kubernetes-release/release/v1.14.0/bin/linux/arm64/kubectl + - 7fec91af78e9548df306f0ec43bea527c8c10cc3a9682c33e971c8522a7fcded@https://storage.googleapis.com/kubernetes-release/network-plugins/cni-plugins-arm64-v0.7.5.tgz +ClusterName: minimal.example.com +ConfigBase: memfs://clusters.example.com/minimal.example.com +InstanceGroupName: nodes +InstanceGroupRole: Node +KubeletConfig: + anonymousAuth: false + cgroupRoot: / + cloudProvider: aws + clusterDNS: 100.64.0.10 + clusterDomain: cluster.local + enableDebuggingHandlers: true + evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5% + featureGates: + ExperimentalCriticalPodAnnotation: "true" + hostnameOverride: '@aws' + kubeconfigPath: /var/lib/kubelet/kubeconfig + logLevel: 2 + networkPluginMTU: 9001 + networkPluginName: kubenet + nodeLabels: + kubernetes.io/role: node + node-role.kubernetes.io/node: "" + nonMasqueradeCIDR: 100.64.0.0/10 + podInfraContainerImage: k8s.gcr.io/pause:3.2 + podManifestPath: /etc/kubernetes/manifests +channels: +- memfs://clusters.example.com/minimal.example.com/addons/bootstrap-channel.yaml + +__EOF_KUBE_ENV + +download-release +echo "== nodeup node config done ==" diff --git a/tests/integration/update_cluster/jwks/id_rsa.pub b/tests/integration/update_cluster/jwks/id_rsa.pub new file mode 100755 index 0000000000000..81cb0127830e7 --- /dev/null +++ b/tests/integration/update_cluster/jwks/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ== diff --git a/tests/integration/update_cluster/jwks/in-v1alpha2.yaml b/tests/integration/update_cluster/jwks/in-v1alpha2.yaml new file mode 100644 index 0000000000000..02fcc271521ce --- /dev/null +++ b/tests/integration/update_cluster/jwks/in-v1alpha2.yaml @@ -0,0 +1,80 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2016-12-10T22:42:27Z" + name: minimal.example.com +spec: + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/minimal.example.com + etcdClusters: + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: main + - etcdMembers: + - instanceGroup: master-us-test-1a + name: us-test-1a + name: events + iam: {} + kubelet: + anonymousAuth: false + kubernetesVersion: v1.14.0 + masterInternalName: api.internal.minimal.example.com + masterPublicName: api.minimal.example.com + networkCIDR: 172.20.0.0/16 + networking: + kubenet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + topology: + masters: public + nodes: public + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-10T22:42:28Z" + name: nodes + labels: + kops.k8s.io/cluster: minimal.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 + machineType: t2.medium + maxSize: 2 + minSize: 2 + role: Node + subnets: + - us-test-1a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2016-12-10T22:42:28Z" + name: master-us-test-1a + labels: + kops.k8s.io/cluster: minimal.example.com +spec: + associatePublicIp: true + image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21 + machineType: m3.medium + maxSize: 1 + minSize: 1 + role: Master + subnets: + - us-test-1a + + diff --git a/tests/integration/update_cluster/jwks/kubernetes.tf b/tests/integration/update_cluster/jwks/kubernetes.tf new file mode 100644 index 0000000000000..5d2e0a6240808 --- /dev/null +++ b/tests/integration/update_cluster/jwks/kubernetes.tf @@ -0,0 +1,548 @@ +locals { + cluster_name = "minimal.example.com" + master_autoscaling_group_ids = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id] + master_security_group_ids = [aws_security_group.masters-minimal-example-com.id] + masters_role_arn = aws_iam_role.masters-minimal-example-com.arn + masters_role_name = aws_iam_role.masters-minimal-example-com.name + node_autoscaling_group_ids = [aws_autoscaling_group.nodes-minimal-example-com.id] + node_security_group_ids = [aws_security_group.nodes-minimal-example-com.id] + node_subnet_ids = [aws_subnet.us-test-1a-minimal-example-com.id] + nodes_role_arn = aws_iam_role.nodes-minimal-example-com.arn + nodes_role_name = aws_iam_role.nodes-minimal-example-com.name + region = "us-test-1" + route_table_public_id = aws_route_table.minimal-example-com.id + subnet_us-test-1a_id = aws_subnet.us-test-1a-minimal-example-com.id + vpc_cidr_block = aws_vpc.minimal-example-com.cidr_block + vpc_id = aws_vpc.minimal-example-com.id +} + +output "cluster_name" { + value = "minimal.example.com" +} + +output "master_autoscaling_group_ids" { + value = [aws_autoscaling_group.master-us-test-1a-masters-minimal-example-com.id] +} + +output "master_security_group_ids" { + value = [aws_security_group.masters-minimal-example-com.id] +} + +output "masters_role_arn" { + value = aws_iam_role.masters-minimal-example-com.arn +} + +output "masters_role_name" { + value = aws_iam_role.masters-minimal-example-com.name +} + +output "node_autoscaling_group_ids" { + value = [aws_autoscaling_group.nodes-minimal-example-com.id] +} + +output "node_security_group_ids" { + value = [aws_security_group.nodes-minimal-example-com.id] +} + +output "node_subnet_ids" { + value = [aws_subnet.us-test-1a-minimal-example-com.id] +} + +output "nodes_role_arn" { + value = aws_iam_role.nodes-minimal-example-com.arn +} + +output "nodes_role_name" { + value = aws_iam_role.nodes-minimal-example-com.name +} + +output "region" { + value = "us-test-1" +} + +output "route_table_public_id" { + value = aws_route_table.minimal-example-com.id +} + +output "subnet_us-test-1a_id" { + value = aws_subnet.us-test-1a-minimal-example-com.id +} + +output "vpc_cidr_block" { + value = aws_vpc.minimal-example-com.cidr_block +} + +output "vpc_id" { + value = aws_vpc.minimal-example-com.id +} + +provider "aws" { + region = "us-test-1" +} + +resource "aws_autoscaling_group" "master-us-test-1a-masters-minimal-example-com" { + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + launch_template { + id = aws_launch_template.master-us-test-1a-masters-minimal-example-com.id + version = aws_launch_template.master-us-test-1a-masters-minimal-example-com.latest_version + } + max_size = 1 + metrics_granularity = "1Minute" + min_size = 1 + name = "master-us-test-1a.masters.minimal.example.com" + tag { + key = "KubernetesCluster" + propagate_at_launch = true + value = "minimal.example.com" + } + tag { + key = "Name" + propagate_at_launch = true + value = "master-us-test-1a.masters.minimal.example.com" + } + tag { + key = "k8s.io/role/master" + propagate_at_launch = true + value = "1" + } + tag { + key = "kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "master-us-test-1a" + } + tag { + key = "kubernetes.io/cluster/minimal.example.com" + propagate_at_launch = true + value = "owned" + } + vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id] +} + +resource "aws_autoscaling_group" "nodes-minimal-example-com" { + enabled_metrics = ["GroupDesiredCapacity", "GroupInServiceInstances", "GroupMaxSize", "GroupMinSize", "GroupPendingInstances", "GroupStandbyInstances", "GroupTerminatingInstances", "GroupTotalInstances"] + launch_template { + id = aws_launch_template.nodes-minimal-example-com.id + version = aws_launch_template.nodes-minimal-example-com.latest_version + } + max_size = 2 + metrics_granularity = "1Minute" + min_size = 2 + name = "nodes.minimal.example.com" + tag { + key = "KubernetesCluster" + propagate_at_launch = true + value = "minimal.example.com" + } + tag { + key = "Name" + propagate_at_launch = true + value = "nodes.minimal.example.com" + } + tag { + key = "k8s.io/role/node" + propagate_at_launch = true + value = "1" + } + tag { + key = "kops.k8s.io/instancegroup" + propagate_at_launch = true + value = "nodes" + } + tag { + key = "kubernetes.io/cluster/minimal.example.com" + propagate_at_launch = true + value = "owned" + } + vpc_zone_identifier = [aws_subnet.us-test-1a-minimal-example-com.id] +} + +resource "aws_ebs_volume" "us-test-1a-etcd-events-minimal-example-com" { + availability_zone = "us-test-1a" + encrypted = false + size = 20 + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.etcd-events.minimal.example.com" + "k8s.io/etcd/events" = "us-test-1a/us-test-1a" + "k8s.io/role/master" = "1" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + type = "gp2" +} + +resource "aws_ebs_volume" "us-test-1a-etcd-main-minimal-example-com" { + availability_zone = "us-test-1a" + encrypted = false + size = 20 + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.etcd-main.minimal.example.com" + "k8s.io/etcd/main" = "us-test-1a/us-test-1a" + "k8s.io/role/master" = "1" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + type = "gp2" +} + +resource "aws_iam_instance_profile" "masters-minimal-example-com" { + name = "masters.minimal.example.com" + role = aws_iam_role.masters-minimal-example-com.name +} + +resource "aws_iam_instance_profile" "nodes-minimal-example-com" { + name = "nodes.minimal.example.com" + role = aws_iam_role.nodes-minimal-example-com.name +} + +resource "aws_iam_openid_connect_provider" "minimal-example-com" { + client_id_list = ["amazonaws.com"] + thumbprint_list = ["d89b37ccc0b574f3e40051ea08a7b60a9db11924"] + url = "https://api.minimal.example.com" +} + +resource "aws_iam_role_policy" "masters-minimal-example-com" { + name = "masters.minimal.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_masters.minimal.example.com_policy") + role = aws_iam_role.masters-minimal-example-com.name +} + +resource "aws_iam_role_policy" "nodes-minimal-example-com" { + name = "nodes.minimal.example.com" + policy = file("${path.module}/data/aws_iam_role_policy_nodes.minimal.example.com_policy") + role = aws_iam_role.nodes-minimal-example-com.name +} + +resource "aws_iam_role" "masters-minimal-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_masters.minimal.example.com_policy") + name = "masters.minimal.example.com" +} + +resource "aws_iam_role" "nodes-minimal-example-com" { + assume_role_policy = file("${path.module}/data/aws_iam_role_nodes.minimal.example.com_policy") + name = "nodes.minimal.example.com" +} + +resource "aws_internet_gateway" "minimal-example-com" { + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_key_pair" "kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157" { + key_name = "kubernetes.minimal.example.com-c4:a6:ed:9a:a8:89:b9:e2:c3:9c:d6:63:eb:9c:71:57" + public_key = file("${path.module}/data/aws_key_pair_kubernetes.minimal.example.com-c4a6ed9aa889b9e2c39cd663eb9c7157_public_key") + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_launch_template" "master-us-test-1a-masters-minimal-example-com" { + block_device_mappings { + device_name = "/dev/xvda" + ebs { + delete_on_termination = true + volume_size = 64 + volume_type = "gp2" + } + } + block_device_mappings { + device_name = "/dev/sdc" + virtual_name = "ephemeral0" + } + iam_instance_profile { + name = aws_iam_instance_profile.masters-minimal-example-com.id + } + image_id = "ami-12345678" + instance_type = "m3.medium" + key_name = aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id + lifecycle { + create_before_destroy = true + } + name_prefix = "master-us-test-1a.masters.minimal.example.com-" + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + security_groups = [aws_security_group.masters-minimal-example-com.id] + } + tag_specifications { + resource_type = "instance" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "master-us-test-1a.masters.minimal.example.com" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + } + tag_specifications { + resource_type = "volume" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "master-us-test-1a.masters.minimal.example.com" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + } + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "master-us-test-1a.masters.minimal.example.com" + "k8s.io/role/master" = "1" + "kops.k8s.io/instancegroup" = "master-us-test-1a" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + user_data = filebase64("${path.module}/data/aws_launch_template_master-us-test-1a.masters.minimal.example.com_user_data") +} + +resource "aws_launch_template" "nodes-minimal-example-com" { + block_device_mappings { + device_name = "/dev/xvda" + ebs { + delete_on_termination = true + volume_size = 128 + volume_type = "gp2" + } + } + iam_instance_profile { + name = aws_iam_instance_profile.nodes-minimal-example-com.id + } + image_id = "ami-12345678" + instance_type = "t2.medium" + key_name = aws_key_pair.kubernetes-minimal-example-com-c4a6ed9aa889b9e2c39cd663eb9c7157.id + lifecycle { + create_before_destroy = true + } + name_prefix = "nodes.minimal.example.com-" + network_interfaces { + associate_public_ip_address = true + delete_on_termination = true + security_groups = [aws_security_group.nodes-minimal-example-com.id] + } + tag_specifications { + resource_type = "instance" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + } + tag_specifications { + resource_type = "volume" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + } + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "k8s.io/role/node" = "1" + "kops.k8s.io/instancegroup" = "nodes" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + user_data = filebase64("${path.module}/data/aws_launch_template_nodes.minimal.example.com_user_data") +} + +resource "aws_route_table_association" "us-test-1a-minimal-example-com" { + route_table_id = aws_route_table.minimal-example-com.id + subnet_id = aws_subnet.us-test-1a-minimal-example-com.id +} + +resource "aws_route_table" "minimal-example-com" { + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/kops/role" = "public" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_route" "route-0-0-0-0--0" { + destination_cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.minimal-example-com.id + route_table_id = aws_route_table.minimal-example-com.id +} + +resource "aws_security_group_rule" "all-master-to-master" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "all-master-to-node" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-minimal-example-com.id + source_security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "all-node-to-node" { + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 0 + type = "ingress" +} + +resource "aws_security_group_rule" "https-external-to-master-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 443 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 443 + type = "ingress" +} + +resource "aws_security_group_rule" "master-egress" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "node-egress" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + protocol = "-1" + security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "node-to-master-tcp-1-2379" { + from_port = 1 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 2379 + type = "ingress" +} + +resource "aws_security_group_rule" "node-to-master-tcp-2382-4000" { + from_port = 2382 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 4000 + type = "ingress" +} + +resource "aws_security_group_rule" "node-to-master-tcp-4003-65535" { + from_port = 4003 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 65535 + type = "ingress" +} + +resource "aws_security_group_rule" "node-to-master-udp-1-65535" { + from_port = 1 + protocol = "udp" + security_group_id = aws_security_group.masters-minimal-example-com.id + source_security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 65535 + type = "ingress" +} + +resource "aws_security_group_rule" "ssh-external-to-master-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.masters-minimal-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group_rule" "ssh-external-to-node-0-0-0-0--0" { + cidr_blocks = ["0.0.0.0/0"] + from_port = 22 + protocol = "tcp" + security_group_id = aws_security_group.nodes-minimal-example-com.id + to_port = 22 + type = "ingress" +} + +resource "aws_security_group" "masters-minimal-example-com" { + description = "Security group for masters" + name = "masters.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "masters.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_security_group" "nodes-minimal-example-com" { + description = "Security group for nodes" + name = "nodes.minimal.example.com" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "nodes.minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_subnet" "us-test-1a-minimal-example-com" { + availability_zone = "us-test-1a" + cidr_block = "172.20.32.0/19" + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "us-test-1a.minimal.example.com" + "SubnetType" = "Public" + "kubernetes.io/cluster/minimal.example.com" = "owned" + "kubernetes.io/role/elb" = "1" + } + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_vpc_dhcp_options_association" "minimal-example-com" { + dhcp_options_id = aws_vpc_dhcp_options.minimal-example-com.id + vpc_id = aws_vpc.minimal-example-com.id +} + +resource "aws_vpc_dhcp_options" "minimal-example-com" { + domain_name = "us-test-1.compute.internal" + domain_name_servers = ["AmazonProvidedDNS"] + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +resource "aws_vpc" "minimal-example-com" { + cidr_block = "172.20.0.0/16" + enable_dns_hostnames = true + enable_dns_support = true + tags = { + "KubernetesCluster" = "minimal.example.com" + "Name" = "minimal.example.com" + "kubernetes.io/cluster/minimal.example.com" = "owned" + } +} + +terraform { + required_version = ">= 0.12.0" +} diff --git a/upup/models/bindata.go b/upup/models/bindata.go index 5810caba08870..bd593d0a55057 100644 --- a/upup/models/bindata.go +++ b/upup/models/bindata.go @@ -1,6 +1,7 @@ // Code generated for package models by go-bindata DO NOT EDIT. (@generated) // sources: // upup/models/cloudup/resources/addons/OWNERS +// upup/models/cloudup/resources/addons/anonymous-access.addons.k8s.io/k8s-1.16.yaml.template // upup/models/cloudup/resources/addons/authentication.aws/k8s-1.10.yaml.template // upup/models/cloudup/resources/addons/authentication.aws/k8s-1.12.yaml.template // upup/models/cloudup/resources/addons/authentication.kope.io/k8s-1.12.yaml @@ -136,6 +137,37 @@ func cloudupResourcesAddonsOwners() (*asset, error) { return a, nil } +var _cloudupResourcesAddonsAnonymousAccessAddonsK8sIoK8s116YamlTemplate = []byte(`apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-addon: anonymous-access.addons.k8s.io + name: anonymous:service-account-issuer-discovery + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:service-account-issuer-discovery +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:anonymous`) + +func cloudupResourcesAddonsAnonymousAccessAddonsK8sIoK8s116YamlTemplateBytes() ([]byte, error) { + return _cloudupResourcesAddonsAnonymousAccessAddonsK8sIoK8s116YamlTemplate, nil +} + +func cloudupResourcesAddonsAnonymousAccessAddonsK8sIoK8s116YamlTemplate() (*asset, error) { + bytes, err := cloudupResourcesAddonsAnonymousAccessAddonsK8sIoK8s116YamlTemplateBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "cloudup/resources/addons/anonymous-access.addons.k8s.io/k8s-1.16.yaml.template", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _cloudupResourcesAddonsAuthenticationAwsK8s110YamlTemplate = []byte(`--- apiVersion: extensions/v1beta1 kind: DaemonSet @@ -20378,6 +20410,7 @@ func AssetNames() []string { // _bindata is a table, holding each asset generator, mapped to its name. var _bindata = map[string]func() (*asset, error){ "cloudup/resources/addons/OWNERS": cloudupResourcesAddonsOwners, + "cloudup/resources/addons/anonymous-access.addons.k8s.io/k8s-1.16.yaml.template": cloudupResourcesAddonsAnonymousAccessAddonsK8sIoK8s116YamlTemplate, "cloudup/resources/addons/authentication.aws/k8s-1.10.yaml.template": cloudupResourcesAddonsAuthenticationAwsK8s110YamlTemplate, "cloudup/resources/addons/authentication.aws/k8s-1.12.yaml.template": cloudupResourcesAddonsAuthenticationAwsK8s112YamlTemplate, "cloudup/resources/addons/authentication.kope.io/k8s-1.12.yaml": cloudupResourcesAddonsAuthenticationKopeIoK8s112Yaml, @@ -20487,6 +20520,9 @@ var _bintree = &bintree{nil, map[string]*bintree{ "resources": {nil, map[string]*bintree{ "addons": {nil, map[string]*bintree{ "OWNERS": {cloudupResourcesAddonsOwners, map[string]*bintree{}}, + "anonymous-access.addons.k8s.io": {nil, map[string]*bintree{ + "k8s-1.16.yaml.template": {cloudupResourcesAddonsAnonymousAccessAddonsK8sIoK8s116YamlTemplate, map[string]*bintree{}}, + }}, "authentication.aws": {nil, map[string]*bintree{ "k8s-1.10.yaml.template": {cloudupResourcesAddonsAuthenticationAwsK8s110YamlTemplate, map[string]*bintree{}}, "k8s-1.12.yaml.template": {cloudupResourcesAddonsAuthenticationAwsK8s112YamlTemplate, map[string]*bintree{}}, diff --git a/upup/models/cloudup/resources/addons/anonymous-access.addons.k8s.io/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/anonymous-access.addons.k8s.io/k8s-1.16.yaml.template new file mode 100644 index 0000000000000..a068456f8b78a --- /dev/null +++ b/upup/models/cloudup/resources/addons/anonymous-access.addons.k8s.io/k8s-1.16.yaml.template @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-addon: anonymous-access.addons.k8s.io + name: anonymous:service-account-issuer-discovery + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:service-account-issuer-discovery +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:anonymous \ No newline at end of file diff --git a/upup/pkg/fi/cloudup/BUILD.bazel b/upup/pkg/fi/cloudup/BUILD.bazel index 5be5cd86329f5..c58229b1df4f8 100644 --- a/upup/pkg/fi/cloudup/BUILD.bazel +++ b/upup/pkg/fi/cloudup/BUILD.bazel @@ -116,6 +116,7 @@ go_test( "//pkg/assets:go_default_library", "//pkg/client/simple/vfsclientset:go_default_library", "//pkg/diff:go_default_library", + "//pkg/featureflag:go_default_library", "//pkg/kopscodecs:go_default_library", "//pkg/model:go_default_library", "//pkg/model/iam:go_default_library", diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index f732a55ac0c04..cca0270a871d2 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -517,6 +517,7 @@ func (c *ApplyClusterCmd) Run(ctx context.Context) error { l.Builders = append(l.Builders, &model.IAMModelBuilder{KopsModelContext: modelContext, Lifecycle: &securityLifecycle}, + &awsmodel.OIDCProviderBuilder{KopsModelContext: modelContext, Lifecycle: &securityLifecycle, KeyStore: keyStore}, ) case kops.CloudProviderDO: doModelContext := &domodel.DOModelContext{ diff --git a/upup/pkg/fi/cloudup/awstasks/iamoidcprovider.go b/upup/pkg/fi/cloudup/awstasks/iamoidcprovider.go index b2ff6d4ce4fdc..3353cb3bd7298 100644 --- a/upup/pkg/fi/cloudup/awstasks/iamoidcprovider.go +++ b/upup/pkg/fi/cloudup/awstasks/iamoidcprovider.go @@ -19,6 +19,7 @@ package awstasks import ( "errors" "fmt" + "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/iam" @@ -33,12 +34,13 @@ import ( type IAMOIDCProvider struct { Lifecycle *fi.Lifecycle - ARN *string ClientIDs []*string - Thumbprints []*string + Thumbprints []fi.Resource URL *string Name *string + + arn *string } var _ fi.CompareWithID = &IAMOIDCProvider{} @@ -64,13 +66,29 @@ func (e *IAMOIDCProvider) Find(c *fi.Context) (*IAMOIDCProvider, error) { if err != nil { return nil, fmt.Errorf("error describing oidc provider: %v", err) } - if fi.StringValue(descResp.Url) == fi.StringValue(e.URL) { + // AWS does not return the https:// in the url + actualURL := aws.StringValue(descResp.Url) + if !strings.Contains(actualURL, "://") { + actualURL = "https://" + actualURL + } + + if actualURL == fi.StringValue(e.URL) { + var actualThumbprints []fi.Resource + for _, thumbprint := range descResp.ThumbprintList { + s := aws.StringValue(thumbprint) + actualThumbprints = append(actualThumbprints, fi.NewStringResource(s)) + } + actual := &IAMOIDCProvider{ ClientIDs: descResp.ClientIDList, - Thumbprints: descResp.ThumbprintList, - URL: descResp.Url, - ARN: arn, + Thumbprints: actualThumbprints, + URL: &actualURL, + arn: arn, } + + actual.Lifecycle = e.Lifecycle + actual.Name = e.Name + klog.V(2).Infof("found matching IAMOIDCProvider %q", aws.StringValue(arn)) return actual, nil } @@ -83,17 +101,17 @@ func (e *IAMOIDCProvider) Run(c *fi.Context) error { } func (s *IAMOIDCProvider) CheckChanges(a, e, changes *IAMOIDCProvider) error { + if e.URL == nil { + return fi.RequiredField("URL") + } + if e.ClientIDs == nil { + return fi.RequiredField("ClientIDs") + } + if len(e.Thumbprints) == 0 { + return fi.RequiredField("Thumbprints") + } + if a != nil { - if e.URL == nil { - return fi.RequiredField("URL") - } - if e.ClientIDs == nil { - return fi.RequiredField("ClientIDs") - } - if len(e.Thumbprints) == 0 { - return fi.RequiredField("Thumbprints") - } - } else { if changes.ClientIDs != nil { return fi.CannotChangeField("ClientIDs") } @@ -105,13 +123,17 @@ func (s *IAMOIDCProvider) CheckChanges(a, e, changes *IAMOIDCProvider) error { } func (p *IAMOIDCProvider) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *IAMOIDCProvider) error { - if a == nil { + thumbprints, err := p.thumbprintsAsStrings() + if err != nil { + return err + } + if a == nil { klog.V(2).Infof("Creating IAMOIDCProvider with Name:%q", *e.Name) request := &iam.CreateOpenIDConnectProviderInput{ ClientIDList: e.ClientIDs, - ThumbprintList: e.Thumbprints, + ThumbprintList: aws.StringSlice(thumbprints), Url: e.URL, } @@ -120,14 +142,14 @@ func (p *IAMOIDCProvider) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *IAMOID return fmt.Errorf("error creating IAMOIDCProvider: %v", err) } - e.ARN = response.OpenIDConnectProviderArn + e.arn = response.OpenIDConnectProviderArn } else { if changes.Thumbprints != nil { - klog.V(2).Infof("Updating IAMOIDCProvider Thumbprints %q", *e.ARN) + klog.V(2).Infof("Updating IAMOIDCProvider Thumbprints %q", fi.StringValue(e.arn)) request := &iam.UpdateOpenIDConnectProviderThumbprintInput{} - request.OpenIDConnectProviderArn = e.ARN - request.ThumbprintList = e.Thumbprints + request.OpenIDConnectProviderArn = a.arn + request.ThumbprintList = aws.StringSlice(thumbprints) _, err := t.Cloud.IAM().UpdateOpenIDConnectProviderThumbprint(request) if err != nil { @@ -138,22 +160,37 @@ func (p *IAMOIDCProvider) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *IAMOID return nil } +func (p *IAMOIDCProvider) thumbprintsAsStrings() ([]string, error) { + var list []string + for _, thumbprint := range p.Thumbprints { + s, err := fi.ResourceAsString(thumbprint) + if err != nil { + return nil, fmt.Errorf("error getting resource as string: %v", err) + } + + list = append(list, s) + } + return list, nil +} + type terraformIAMOIDCProvider struct { URL *string `json:"url" cty:"url"` ClientIDList []*string `json:"client_id_list" cty:"client_id_list"` ThumbprintList []*string `json:"thumbprint_list" cty:"thumbprint_list"` - Name *string `json:"name" cty:"name"` AssumeRolePolicy *terraform.Literal `json:"assume_role_policy" cty:"assume_role_policy"` } func (p *IAMOIDCProvider) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *IAMOIDCProvider) error { + thumbprints, err := p.thumbprintsAsStrings() + if err != nil { + return err + } tf := &terraformIAMOIDCProvider{ - Name: e.Name, URL: e.URL, ClientIDList: e.ClientIDs, - ThumbprintList: e.Thumbprints, + ThumbprintList: aws.StringSlice(thumbprints), } return t.RenderResource("aws_iam_openid_connect_provider", *e.Name, tf) diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index 6ffce6b9c01d0..b044c0a16da74 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -141,6 +141,25 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons { } } + if featureflag.PublicJWKS.Enabled() { + key := "anonymous-access.addons.k8s.io" + version := "1.19.0-alpha.3" + + { + location := key + "/k8s-1.16.yaml" + id := "k8s-1.16" + + addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(version), + Selector: map[string]string{"k8s-addon": key}, + Manifest: fi.String(location), + KubernetesVersion: ">=1.16.0-alpha.0", + Id: id, + }) + } + } + { key := "core.addons.k8s.io" version := "1.4.0" diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go index d9fbde4d3c924..8c7e8bb385684 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go @@ -24,6 +24,7 @@ import ( kopsapi "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/assets" "k8s.io/kops/pkg/client/simple/vfsclientset" + "k8s.io/kops/pkg/featureflag" "k8s.io/kops/pkg/kopscodecs" "k8s.io/kops/pkg/model" "k8s.io/kops/pkg/model/iam" @@ -50,6 +51,20 @@ func TestBootstrapChannelBuilder_BuildTasks(t *testing.T) { runChannelBuilderTest(t, "awsiamauthenticator", []string{"authentication.aws-k8s-1.12"}) } +func TestBootstrapChannelBuilder_JWKS(t *testing.T) { + h := testutils.NewIntegrationTestHarness(t) + defer h.Close() + + h.SetupMockAWS() + + featureflag.ParseFlags("+PublicJWKS") + unsetFeaureFlag := func() { + featureflag.ParseFlags("-PublicJWKS") + } + defer unsetFeaureFlag() + runChannelBuilderTest(t, "jwks", []string{"dns-controller.addons.k8s.io-k8s-1.12", "kops-controller.addons.k8s.io-k8s-1.16", "anonymous-access.addons.k8s.io-k8s-1.16"}) +} + func runChannelBuilderTest(t *testing.T, key string, addonManifests []string) { basedir := path.Join("tests/bootstrapchannelbuilder/", key) diff --git a/upup/pkg/fi/cloudup/populate_cluster_spec.go b/upup/pkg/fi/cloudup/populate_cluster_spec.go index d74abe70ce43b..0251173386dd0 100644 --- a/upup/pkg/fi/cloudup/populate_cluster_spec.go +++ b/upup/pkg/fi/cloudup/populate_cluster_spec.go @@ -283,6 +283,7 @@ func (c *populateClusterSpec) run(clientset simple.Clientset) error { codeModels = append(codeModels, &components.KubeProxyOptionsBuilder{Context: optionsContext}) codeModels = append(codeModels, &components.CiliumOptionsBuilder{Context: optionsContext}) codeModels = append(codeModels, &components.OpenStackOptionsBulder{Context: optionsContext}) + codeModels = append(codeModels, &components.DiscoveryOptionsBuilder{OptionsContext: optionsContext}) } } diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/anonymous-access.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/anonymous-access.addons.k8s.io-k8s-1.16.yaml new file mode 100644 index 0000000000000..01d2200b2eb00 --- /dev/null +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/anonymous-access.addons.k8s.io-k8s-1.16.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-addon: anonymous-access.addons.k8s.io + name: anonymous:service-account-issuer-discovery + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:service-account-issuer-discovery +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:anonymous diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/cluster.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/cluster.yaml new file mode 100644 index 0000000000000..91d03333db679 --- /dev/null +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/cluster.yaml @@ -0,0 +1,42 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2016-12-10T22:42:27Z" + name: minimal.example.com +spec: + addons: + - manifest: s3://somebucket/example.yaml + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/minimal.example.com + etcdClusters: + - etcdMembers: + - instanceGroup: master-us-test-1a + name: master-us-test-1a + name: main + - etcdMembers: + - instanceGroup: master-us-test-1a + name: master-us-test-1a + name: events + iam: {} + kubernetesVersion: v1.14.6 + masterInternalName: api.internal.minimal.example.com + masterPublicName: api.minimal.example.com + additionalSans: + - proxy.api.minimal.example.com + networkCIDR: 172.20.0.0/16 + networking: + kubenet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + topology: + masters: public + nodes: public + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/dns-controller.addons.k8s.io-k8s-1.12.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/dns-controller.addons.k8s.io-k8s-1.12.yaml new file mode 100644 index 0000000000000..62e88a885e1f0 --- /dev/null +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/dns-controller.addons.k8s.io-k8s-1.12.yaml @@ -0,0 +1,109 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + k8s-addon: dns-controller.addons.k8s.io + k8s-app: dns-controller + version: v1.19.0-alpha.3 + name: dns-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: dns-controller + strategy: + type: Recreate + template: + metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + k8s-addon: dns-controller.addons.k8s.io + k8s-app: dns-controller + version: v1.19.0-alpha.3 + spec: + containers: + - command: + - /dns-controller + - --watch-ingress=false + - --dns=aws-route53 + - --zone=*/Z1AFAKE1ZON3YO + - --zone=*/* + - -v=2 + env: + - name: KUBERNETES_SERVICE_HOST + value: 127.0.0.1 + image: kope/dns-controller:1.19.0-alpha.3 + name: dns-controller + resources: + requests: + cpu: 50m + memory: 50Mi + securityContext: + runAsNonRoot: true + dnsPolicy: Default + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + priorityClassName: system-cluster-critical + serviceAccount: dns-controller + tolerations: + - operator: Exists + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: dns-controller.addons.k8s.io + name: dns-controller + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-addon: dns-controller.addons.k8s.io + name: kops:dns-controller +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - ingress + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-addon: dns-controller.addons.k8s.io + name: kops:dns-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kops:dns-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:dns-controller diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/kops-controller.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/kops-controller.addons.k8s.io-k8s-1.16.yaml new file mode 100644 index 0000000000000..76cf026f0e444 --- /dev/null +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/kops-controller.addons.k8s.io-k8s-1.16.yaml @@ -0,0 +1,175 @@ +apiVersion: v1 +data: + config.yaml: | + {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.example.com"} +kind: ConfigMap +metadata: + labels: + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-addon: kops-controller.addons.k8s.io + k8s-app: kops-controller + version: v1.19.0-alpha.3 + name: kops-controller + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kops-controller + template: + metadata: + labels: + k8s-addon: kops-controller.addons.k8s.io + k8s-app: kops-controller + version: v1.19.0-alpha.3 + spec: + containers: + - command: + - /kops-controller + - --v=2 + - --conf=/etc/kubernetes/kops-controller/config/config.yaml + image: kope/kops-controller:1.19.0-alpha.3 + name: kops-controller + resources: + requests: + cpu: 50m + memory: 50Mi + securityContext: + runAsNonRoot: true + volumeMounts: + - mountPath: /etc/kubernetes/kops-controller/config/ + name: kops-controller-config + - mountPath: /etc/kubernetes/kops-controller/pki/ + name: kops-controller-pki + dnsPolicy: Default + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + priorityClassName: system-node-critical + serviceAccount: kops-controller + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + volumes: + - configMap: + name: kops-controller + name: kops-controller-config + - hostPath: + path: /etc/kubernetes/kops-controller/ + type: Directory + name: kops-controller-pki + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kops-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:kops-controller + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - create +- apiGroups: + - "" + resourceNames: + - kops-controller-leader + resources: + - configmaps + verbs: + - get + - list + - watch + - patch + - update + - delete +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + k8s-addon: kops-controller.addons.k8s.io + name: kops-controller + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kops-controller +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:serviceaccount:kube-system:kops-controller diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/manifest.yaml new file mode 100644 index 0000000000000..b64587110792c --- /dev/null +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/jwks/manifest.yaml @@ -0,0 +1,96 @@ +kind: Addons +metadata: + creationTimestamp: null + name: bootstrap +spec: + addons: + - id: k8s-1.16 + kubernetesVersion: '>=1.16.0-alpha.0' + manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml + manifestHash: 32a6a0057d0d1930c0089120b4d0593bb0f109de + name: kops-controller.addons.k8s.io + selector: + k8s-addon: kops-controller.addons.k8s.io + version: 1.19.0-alpha.3 + - id: k8s-1.16 + kubernetesVersion: '>=1.16.0-alpha.0' + manifest: anonymous-access.addons.k8s.io/k8s-1.16.yaml + manifestHash: d01bb2f3c12819e21bf0197624b95fb53dc0951a + name: anonymous-access.addons.k8s.io + selector: + k8s-addon: anonymous-access.addons.k8s.io + version: 1.19.0-alpha.3 + - manifest: core.addons.k8s.io/v1.4.0.yaml + manifestHash: 3ffe9ac576f9eec72e2bdfbd2ea17d56d9b17b90 + name: core.addons.k8s.io + selector: + k8s-addon: core.addons.k8s.io + version: 1.4.0 + - id: k8s-1.6 + kubernetesVersion: <1.12.0 + manifest: kube-dns.addons.k8s.io/k8s-1.6.yaml + manifestHash: a50e6a4c2f800b4af4ac0d80edf7762cfc1de9e3 + name: kube-dns.addons.k8s.io + selector: + k8s-addon: kube-dns.addons.k8s.io + version: 1.15.13-kops.3 + - id: k8s-1.12 + kubernetesVersion: '>=1.12.0' + manifest: kube-dns.addons.k8s.io/k8s-1.12.yaml + manifestHash: db49c98447b9d59dec4fa413461a6614bc6e43e9 + name: kube-dns.addons.k8s.io + selector: + k8s-addon: kube-dns.addons.k8s.io + version: 1.15.13-kops.3 + - id: k8s-1.8 + manifest: rbac.addons.k8s.io/k8s-1.8.yaml + manifestHash: 5d53ce7b920cd1e8d65d2306d80a041420711914 + name: rbac.addons.k8s.io + selector: + k8s-addon: rbac.addons.k8s.io + version: 1.8.0 + - id: k8s-1.9 + manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml + manifestHash: e1508d77cb4e527d7a2939babe36dc350dd83745 + name: kubelet-api.rbac.addons.k8s.io + selector: + k8s-addon: kubelet-api.rbac.addons.k8s.io + version: v0.0.1 + - manifest: limit-range.addons.k8s.io/v1.5.0.yaml + manifestHash: 2ea50e23f1a5aa41df3724630ac25173738cc90c + name: limit-range.addons.k8s.io + selector: + k8s-addon: limit-range.addons.k8s.io + version: 1.5.0 + - id: k8s-1.6 + kubernetesVersion: <1.12.0 + manifest: dns-controller.addons.k8s.io/k8s-1.6.yaml + manifestHash: c844ffd7477e2b1fcb0a5524b3bfea4df8e1fc8b + name: dns-controller.addons.k8s.io + selector: + k8s-addon: dns-controller.addons.k8s.io + version: 1.19.0-alpha.3 + - id: k8s-1.12 + kubernetesVersion: '>=1.12.0' + manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml + manifestHash: a59ae2cef3464a9c7b4a29788e030e5bac6274e7 + name: dns-controller.addons.k8s.io + selector: + k8s-addon: dns-controller.addons.k8s.io + version: 1.19.0-alpha.3 + - id: v1.15.0 + kubernetesVersion: '>=1.15.0' + manifest: storage-aws.addons.k8s.io/v1.15.0.yaml + manifestHash: 00cf6e46e25b736b2da93c6025ce482474d83904 + name: storage-aws.addons.k8s.io + selector: + k8s-addon: storage-aws.addons.k8s.io + version: 1.15.0 + - id: v1.7.0 + kubernetesVersion: <1.15.0 + manifest: storage-aws.addons.k8s.io/v1.7.0.yaml + manifestHash: 62705a596142e6cc283280e8aa973e51536994c5 + name: storage-aws.addons.k8s.io + selector: + k8s-addon: storage-aws.addons.k8s.io + version: 1.15.0 diff --git a/upup/pkg/fi/fitasks/keypair.go b/upup/pkg/fi/fitasks/keypair.go index a3950e2ceb1b9..02b5b68a4d5fc 100644 --- a/upup/pkg/fi/fitasks/keypair.go +++ b/upup/pkg/fi/fitasks/keypair.go @@ -17,6 +17,7 @@ limitations under the License. package fitasks import ( + "crypto/sha1" "crypto/x509/pkix" "fmt" "sort" @@ -44,6 +45,9 @@ type Keypair struct { Type string `json:"type"` // LegacyFormat is whether the keypair is stored in a legacy format. LegacyFormat bool `json:"oldFormat"` + + certificate *fi.TaskDependentResource + certificateSHA1Fingerprint *fi.TaskDependentResource } var _ fi.HasCheckExisting = &Keypair{} @@ -98,6 +102,10 @@ func (e *Keypair) Find(c *fi.Context) (*Keypair, error) { // Avoid spurious changes actual.Lifecycle = e.Lifecycle + if err := e.setResources(cert); err != nil { + return nil, fmt.Errorf("error setting resources: %v", err) + } + return actual, nil } @@ -240,6 +248,17 @@ func (_ *Keypair) Render(c *fi.Context, a, e, changes *Keypair) error { klog.Infof("updated Keypair %q to new format", name) } + // Load the keypair for dependent resources + { + cert, _, _, err := c.Keystore.FindKeypair(name) + if err != nil { + return err + } + + if err := e.setResources(cert); err != nil { + return fmt.Errorf("error setting resources: %v", err) + } + } return nil } @@ -268,3 +287,33 @@ func parsePkixName(s string) (*pkix.Name, error) { return name, nil } + +func (e *Keypair) ensureResources() { + if e.certificate == nil { + e.certificate = &fi.TaskDependentResource{Task: e} + } + if e.certificateSHA1Fingerprint == nil { + e.certificateSHA1Fingerprint = &fi.TaskDependentResource{Task: e} + } +} + +func (e *Keypair) setResources(cert *pki.Certificate) error { + e.ensureResources() + + s, err := cert.AsString() + if err != nil { + return err + } + e.certificate.Resource = fi.NewStringResource(s) + + fingerprint := sha1.Sum(cert.Certificate.Raw) + hex := fmt.Sprintf("%x", fingerprint) + e.certificateSHA1Fingerprint.Resource = fi.NewStringResource(hex) + + return nil +} + +func (e *Keypair) CertificateSHA1Fingerprint() fi.Resource { + e.ensureResources() + return e.certificateSHA1Fingerprint +} diff --git a/upup/pkg/fi/resources.go b/upup/pkg/fi/resources.go index 5d25e55337484..b304b74f9fe2e 100644 --- a/upup/pkg/fi/resources.go +++ b/upup/pkg/fi/resources.go @@ -30,6 +30,11 @@ type Resource interface { Open() (io.Reader, error) } +// HasIsReady is implemented by Resources that are derived (and thus may not be ready at comparison time) +type HasIsReady interface { + IsReady() bool +} + type TemplateResource interface { Resource Curry(args []string) TemplateResource @@ -264,10 +269,11 @@ type TaskDependentResource struct { var _ Resource = &TaskDependentResource{} var _ HasDependencies = &TaskDependentResource{} +var _ HasIsReady = &TaskDependentResource{} func (r *TaskDependentResource) Open() (io.Reader, error) { if r.Resource == nil { - return nil, fmt.Errorf("resource opened before it is ready") + return nil, fmt.Errorf("resource opened before it is ready (task=%v)", r.Task) } return r.Resource.Open() } @@ -275,3 +281,8 @@ func (r *TaskDependentResource) Open() (io.Reader, error) { func (r *TaskDependentResource) GetDependencies(tasks map[string]Task) []Task { return []Task{r.Task} } + +// IsReady implements HasIsReady::IsReady +func (r *TaskDependentResource) IsReady() bool { + return r.Resource != nil +} diff --git a/upup/pkg/fi/topological_sort.go b/upup/pkg/fi/topological_sort.go index 0049519dec189..598d3eba2ad03 100644 --- a/upup/pkg/fi/topological_sort.go +++ b/upup/pkg/fi/topological_sort.go @@ -111,7 +111,7 @@ func getDependencies(tasks map[string]Task, v reflect.Value) []Task { } else if dep, ok := intf.(Task); ok { dependencies = append(dependencies, dep) } else if _, ok := intf.(Resource); ok { - // Ignore: not a dependency (?) + // Ignore: not a dependency, unless we explicitly implement HasDependencies (e.g. TaskDependentResource) } else if _, ok := intf.(*ResourceHolder); ok { // Ignore: not a dependency (?) } else {