From 7d31861f0bc5a7355e42be2008997ad217c9c262 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Wed, 22 Feb 2023 14:58:47 -0800 Subject: [PATCH 01/41] Moved launchtemplate to providers --- pkg/cloudprovider/cloudprovider.go | 88 +-------------- pkg/cloudprovider/cloudprovider_test.go | 7 +- pkg/cloudprovider/instance.go | 5 +- pkg/context/context.go | 101 +++++++++++++++++- .../launchtemplate}/launchtemplate.go | 28 ++--- .../launchtemplate}/launchtemplate_test.go | 2 +- 6 files changed, 122 insertions(+), 109 deletions(-) rename pkg/{cloudprovider => providers/launchtemplate}/launchtemplate.go (87%) rename pkg/{cloudprovider => providers/launchtemplate}/launchtemplate_test.go (99%) diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index 1a1c9bb4f931..0f57d24015f0 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -16,9 +16,7 @@ package cloudprovider import ( "context" - "encoding/base64" "fmt" - "net" "net/http" "strings" @@ -37,22 +35,13 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/eks" - "github.com/aws/aws-sdk-go/service/eks/eksiface" - "github.com/aws/aws-sdk-go/service/ssm" - "github.com/patrickmn/go-cache" "github.com/samber/lo" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "k8s.io/client-go/transport" "knative.dev/pkg/logging" - "knative.dev/pkg/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/cloudprovider/amifamily" awscontext "github.com/aws/karpenter/pkg/context" @@ -81,30 +70,11 @@ type CloudProvider struct { } func New(ctx awscontext.Context) *CloudProvider { - clusterEndpoint, err := resolveClusterEndpoint(ctx, eks.New(ctx.Session)) - if err != nil { - logging.FromContext(ctx).Fatalf("unable to detect the cluster endpoint, %s", err) - } else { - logging.FromContext(ctx).With("cluster-endpoint", clusterEndpoint).Debugf("discovered cluster endpoint") - } - // We perform best-effort on resolving the kube-dns IP - kubeDNSIP, err := kubeDNSIP(ctx, ctx.KubernetesInterface) - if err != nil { - // If we fail to get the kube-dns IP, we don't want to crash because this causes issues with custom DNS setups - // https://github.com/aws/karpenter/issues/2787 - logging.FromContext(ctx).Debugf("unable to detect the IP of the kube-dns service, %s", err) - } else { - logging.FromContext(ctx).With("kube-dns-ip", kubeDNSIP).Debugf("discovered kube dns") - } - instanceTypeProvider := NewInstanceTypeProvider(ctx, ctx.Session, ctx.EC2API, ctx.SubnetProvider, ctx.UnavailableOfferingsCache, ctx.StartAsync) - amiProvider := amifamily.NewAMIProvider(ctx.KubeClient, ctx.KubernetesInterface, ssm.New(ctx.Session), ctx.EC2API, - cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) - amiResolver := amifamily.New(ctx.KubeClient, amiProvider) return &CloudProvider{ kubeClient: ctx.KubeClient, instanceTypeProvider: instanceTypeProvider, - amiProvider: amiProvider, + amiProvider: ctx.AMIProvider, instanceProvider: NewInstanceProvider( ctx, aws.StringValue(ctx.Session.Config.Region), @@ -112,16 +82,7 @@ func New(ctx awscontext.Context) *CloudProvider { ctx.UnavailableOfferingsCache, instanceTypeProvider, ctx.SubnetProvider, - NewLaunchTemplateProvider( - ctx, - ctx.EC2API, - amiResolver, - ctx.SecurityGroupProvider, - lo.Must(getCABundle(ctx.RESTConfig)), - ctx.StartAsync, - kubeDNSIP, - clusterEndpoint, - ), + ctx.LaunchTemplateProvider, ), } } @@ -386,48 +347,3 @@ func (c *CloudProvider) instanceToMachine(ctx context.Context, instance *ec2.Ins machine.Status.ProviderID = fmt.Sprintf("aws:///%s/%s", aws.StringValue(instance.Placement.AvailabilityZone), aws.StringValue(instance.InstanceId)) return machine } - -func resolveClusterEndpoint(ctx context.Context, eksAPI eksiface.EKSAPI) (string, error) { - clusterEndpointFromSettings := settings.FromContext(ctx).ClusterEndpoint - if clusterEndpointFromSettings != "" { - return clusterEndpointFromSettings, nil // cluster endpoint is explicitly set - } - out, err := eksAPI.DescribeCluster(&eks.DescribeClusterInput{ - Name: aws.String(settings.FromContext(ctx).ClusterName), - }) - if err != nil { - return "", fmt.Errorf("failed to resolve cluster endpoint, %w", err) - } - return *out.Cluster.Endpoint, nil -} - -func getCABundle(restConfig *rest.Config) (*string, error) { - // Discover CA Bundle from the REST client. We could alternatively - // have used the simpler client-go InClusterConfig() method. - // However, that only works when Karpenter is running as a Pod - // within the same cluster it's managing. - transportConfig, err := restConfig.TransportConfig() - if err != nil { - return nil, fmt.Errorf("discovering caBundle, loading transport config, %w", err) - } - _, err = transport.TLSConfigFor(transportConfig) // fills in CAData! - if err != nil { - return nil, fmt.Errorf("discovering caBundle, loading TLS config, %w", err) - } - return ptr.String(base64.StdEncoding.EncodeToString(transportConfig.TLS.CAData)), nil -} - -func kubeDNSIP(ctx context.Context, kubernetesInterface kubernetes.Interface) (net.IP, error) { - if kubernetesInterface == nil { - return nil, fmt.Errorf("no K8s client provided") - } - dnsService, err := kubernetesInterface.CoreV1().Services("kube-system").Get(ctx, "kube-dns", metav1.GetOptions{}) - if err != nil { - return nil, err - } - kubeDNSIP := net.ParseIP(dnsService.Spec.ClusterIP) - if kubeDNSIP == nil { - return nil, fmt.Errorf("parsing cluster IP") - } - return kubeDNSIP, nil -} diff --git a/pkg/cloudprovider/cloudprovider_test.go b/pkg/cloudprovider/cloudprovider_test.go index da76ea3588f0..371f1c2d51e2 100644 --- a/pkg/cloudprovider/cloudprovider_test.go +++ b/pkg/cloudprovider/cloudprovider_test.go @@ -23,6 +23,7 @@ import ( "github.com/aws/aws-sdk-go/service/eks" "github.com/aws/karpenter/pkg/apis/settings" + "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/test" . "github.com/onsi/gomega" @@ -37,7 +38,7 @@ var _ = Describe("Cloud Provider", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ ClusterEndpoint: lo.ToPtr("https://api.test-cluster.k8s.local"), })) - endpoint, err := resolveClusterEndpoint(ctx, fakeEKSAPI) + endpoint, err := context.ResolveClusterEndpoint(ctx, fakeEKSAPI) Expect(err).ToNot(HaveOccurred()) Expect(endpoint).To(Equal("https://api.test-cluster.k8s.local")) }) @@ -54,7 +55,7 @@ var _ = Describe("Cloud Provider", func() { }, ) - endpoint, err := resolveClusterEndpoint(ctx, fakeEKSAPI) + endpoint, err := context.ResolveClusterEndpoint(ctx, fakeEKSAPI) Expect(err).ToNot(HaveOccurred()) Expect(endpoint).To(Equal("https://cluster-endpoint.test-cluster.k8s.local")) }) @@ -65,7 +66,7 @@ var _ = Describe("Cloud Provider", func() { })) fakeEKSAPI.DescribeClusterBehaviour.Error.Set(errors.New("test error")) - _, err := resolveClusterEndpoint(ctx, fakeEKSAPI) + _, err := context.ResolveClusterEndpoint(ctx, fakeEKSAPI) Expect(err).To(HaveOccurred()) }) }) diff --git a/pkg/cloudprovider/instance.go b/pkg/cloudprovider/instance.go index 815968fcb966..65d753607246 100644 --- a/pkg/cloudprovider/instance.go +++ b/pkg/cloudprovider/instance.go @@ -38,6 +38,7 @@ import ( "github.com/aws/karpenter/pkg/batcher" "github.com/aws/karpenter/pkg/cache" awserrors "github.com/aws/karpenter/pkg/errors" + "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/subnet" "github.com/aws/karpenter/pkg/utils" @@ -63,11 +64,11 @@ type InstanceProvider struct { unavailableOfferings *cache.UnavailableOfferings instanceTypeProvider *InstanceTypeProvider subnetProvider *subnet.Provider - launchTemplateProvider *LaunchTemplateProvider + launchTemplateProvider *launchtemplate.Provider ec2Batcher *batcher.EC2API } -func NewInstanceProvider(ctx context.Context, region string, ec2api ec2iface.EC2API, unavailableOfferings *cache.UnavailableOfferings, instanceTypeProvider *InstanceTypeProvider, subnetProvider *subnet.Provider, launchTemplateProvider *LaunchTemplateProvider) *InstanceProvider { +func NewInstanceProvider(ctx context.Context, region string, ec2api ec2iface.EC2API, unavailableOfferings *cache.UnavailableOfferings, instanceTypeProvider *InstanceTypeProvider, subnetProvider *subnet.Provider, launchTemplateProvider *launchtemplate.Provider) *InstanceProvider { return &InstanceProvider{ region: region, ec2api: ec2api, diff --git a/pkg/context/context.go b/pkg/context/context.go index 8472df210e51..cd18cfef39e4 100644 --- a/pkg/context/context.go +++ b/pkg/context/context.go @@ -16,8 +16,10 @@ package context import ( "context" + "encoding/base64" "errors" "fmt" + "net" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" @@ -28,10 +30,22 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/aws/aws-sdk-go/service/eks" + "github.com/aws/aws-sdk-go/service/eks/eksiface" + "github.com/aws/aws-sdk-go/service/ssm" + "github.com/patrickmn/go-cache" "github.com/samber/lo" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/transport" "knative.dev/pkg/logging" + "knative.dev/pkg/ptr" - "github.com/aws/karpenter/pkg/cache" + "github.com/aws/karpenter/pkg/apis/settings" + awscache "github.com/aws/karpenter/pkg/cache" + "github.com/aws/karpenter/pkg/cloudprovider/amifamily" + "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/securitygroup" "github.com/aws/karpenter/pkg/providers/subnet" "github.com/aws/karpenter/pkg/utils/project" @@ -44,10 +58,13 @@ type Context struct { cloudprovider.Context Session *session.Session - UnavailableOfferingsCache *cache.UnavailableOfferings + UnavailableOfferingsCache *awscache.UnavailableOfferings EC2API ec2iface.EC2API SubnetProvider *subnet.Provider SecurityGroupProvider *securitygroup.Provider + AMIProvider *amifamily.AMIProvider + AMIResolver *amifamily.Resolver + LaunchTemplateProvider *launchtemplate.Provider } func NewOrDie(ctx cloudprovider.Context) Context { @@ -68,15 +85,48 @@ func NewOrDie(ctx cloudprovider.Context) Context { logging.FromContext(ctx).Fatalf("Checking EC2 API connectivity, %s", err) } logging.FromContext(ctx).With("region", *sess.Config.Region).Debugf("discovered region") + clusterEndpoint, err := ResolveClusterEndpoint(ctx, eks.New(sess)) + if err != nil { + logging.FromContext(ctx).Fatalf("unable to detect the cluster endpoint, %s", err) + } else { + logging.FromContext(ctx).With("cluster-endpoint", clusterEndpoint).Debugf("discovered cluster endpoint") + } + // We perform best-effort on resolving the kube-dns IP + kubeDNSIP, err := kubeDNSIP(ctx, ctx.KubernetesInterface) + if err != nil { + // If we fail to get the kube-dns IP, we don't want to crash because this causes issues with custom DNS setups + // https://github.com/aws/karpenter/issues/2787 + logging.FromContext(ctx).Debugf("unable to detect the IP of the kube-dns service, %s", err) + } else { + logging.FromContext(ctx).With("kube-dns-ip", kubeDNSIP).Debugf("discovered kube dns") + } + subnetProvider := subnet.NewProvider(ec2api) securityGroupProvider := securitygroup.NewProvider(ec2api) + amiProvider := amifamily.NewAMIProvider(ctx.KubeClient, ctx.KubernetesInterface, ssm.New(sess), ec2api, + cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) + amiResolver := amifamily.New(ctx.KubeClient, amiProvider) + launchTemplateProvider := launchtemplate.NewProvider( + ctx, + ec2api, + amiResolver, + securityGroupProvider, + lo.Must(getCABundle(ctx.RESTConfig)), + ctx.StartAsync, + kubeDNSIP, + clusterEndpoint, + ) + return Context{ Context: ctx, Session: sess, - UnavailableOfferingsCache: cache.NewUnavailableOfferings(), + UnavailableOfferingsCache: awscache.NewUnavailableOfferings(), EC2API: ec2api, SubnetProvider: subnetProvider, SecurityGroupProvider: securityGroupProvider, + AMIProvider: amiProvider, + AMIResolver: amiResolver, + LaunchTemplateProvider: launchTemplateProvider, } } @@ -97,3 +147,48 @@ func checkEC2Connectivity(ctx context.Context, api *ec2.EC2) error { } return err } + +func ResolveClusterEndpoint(ctx context.Context, eksAPI eksiface.EKSAPI) (string, error) { + clusterEndpointFromSettings := settings.FromContext(ctx).ClusterEndpoint + if clusterEndpointFromSettings != "" { + return clusterEndpointFromSettings, nil // cluster endpoint is explicitly set + } + out, err := eksAPI.DescribeCluster(&eks.DescribeClusterInput{ + Name: aws.String(settings.FromContext(ctx).ClusterName), + }) + if err != nil { + return "", fmt.Errorf("failed to resolve cluster endpoint, %w", err) + } + return *out.Cluster.Endpoint, nil +} + +func getCABundle(restConfig *rest.Config) (*string, error) { + // Discover CA Bundle from the REST client. We could alternatively + // have used the simpler client-go InClusterConfig() method. + // However, that only works when Karpenter is running as a Pod + // within the same cluster it's managing. + transportConfig, err := restConfig.TransportConfig() + if err != nil { + return nil, fmt.Errorf("discovering caBundle, loading transport config, %w", err) + } + _, err = transport.TLSConfigFor(transportConfig) // fills in CAData! + if err != nil { + return nil, fmt.Errorf("discovering caBundle, loading TLS config, %w", err) + } + return ptr.String(base64.StdEncoding.EncodeToString(transportConfig.TLS.CAData)), nil +} + +func kubeDNSIP(ctx context.Context, kubernetesInterface kubernetes.Interface) (net.IP, error) { + if kubernetesInterface == nil { + return nil, fmt.Errorf("no K8s client provided") + } + dnsService, err := kubernetesInterface.CoreV1().Services("kube-system").Get(ctx, "kube-dns", metav1.GetOptions{}) + if err != nil { + return nil, err + } + kubeDNSIP := net.ParseIP(dnsService.Spec.ClusterIP) + if kubeDNSIP == nil { + return nil, fmt.Errorf("parsing cluster IP") + } + return kubeDNSIP, nil +} diff --git a/pkg/cloudprovider/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go similarity index 87% rename from pkg/cloudprovider/launchtemplate.go rename to pkg/providers/launchtemplate/launchtemplate.go index 979f08937dcb..c6071ad74d7d 100644 --- a/pkg/cloudprovider/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cloudprovider +package launchtemplate import ( "context" @@ -51,7 +51,7 @@ const ( karpenterManagedTagKey = "karpenter.k8s.aws/cluster" ) -type LaunchTemplateProvider struct { +type Provider struct { sync.Mutex ec2api ec2iface.EC2API amiFamily *amifamily.Resolver @@ -63,8 +63,8 @@ type LaunchTemplateProvider struct { clusterEndpoint string } -func NewLaunchTemplateProvider(ctx context.Context, ec2api ec2iface.EC2API, amiFamily *amifamily.Resolver, securityGroupProvider *securitygroup.Provider, caBundle *string, startAsync <-chan struct{}, kubeDNSIP net.IP, clusterEndpoint string) *LaunchTemplateProvider { - l := &LaunchTemplateProvider{ +func NewProvider(ctx context.Context, ec2api ec2iface.EC2API, amiFamily *amifamily.Resolver, securityGroupProvider *securitygroup.Provider, caBundle *string, startAsync <-chan struct{}, kubeDNSIP net.IP, clusterEndpoint string) *Provider { + l := &Provider{ ec2api: ec2api, amiFamily: amiFamily, securityGroupProvider: securityGroupProvider, @@ -87,7 +87,7 @@ func NewLaunchTemplateProvider(ctx context.Context, ec2api ec2iface.EC2API, amiF return l } -func (p *LaunchTemplateProvider) EnsureAll(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate, machine *v1alpha5.Machine, +func (p *Provider) EnsureAll(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate, machine *v1alpha5.Machine, instanceTypes []*cloudprovider.InstanceType, additionalLabels map[string]string) (map[string][]*cloudprovider.InstanceType, error) { p.Lock() @@ -117,7 +117,7 @@ func (p *LaunchTemplateProvider) EnsureAll(ctx context.Context, nodeTemplate *v1 } // Invalidate deletes a launch template from cache if it exists -func (p *LaunchTemplateProvider) Invalidate(ctx context.Context, ltName string, ltID string) { +func (p *Provider) Invalidate(ctx context.Context, ltName string, ltID string) { ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("launch-template-name", ltName, "launch-template-id", ltID)) p.Lock() defer p.Unlock() @@ -135,7 +135,7 @@ func launchTemplateName(options *amifamily.LaunchTemplate) string { return fmt.Sprintf(launchTemplateNameFormat, options.ClusterName, fmt.Sprint(hash)) } -func (p *LaunchTemplateProvider) createAmiOptions(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate, labels map[string]string) (*amifamily.Options, error) { +func (p *Provider) createAmiOptions(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate, labels map[string]string) (*amifamily.Options, error) { instanceProfile, err := p.getInstanceProfile(ctx, nodeTemplate) if err != nil { return nil, err @@ -161,7 +161,7 @@ func (p *LaunchTemplateProvider) createAmiOptions(ctx context.Context, nodeTempl }, nil } -func (p *LaunchTemplateProvider) ensureLaunchTemplate(ctx context.Context, options *amifamily.LaunchTemplate) (*ec2.LaunchTemplate, error) { +func (p *Provider) ensureLaunchTemplate(ctx context.Context, options *amifamily.LaunchTemplate) (*ec2.LaunchTemplate, error) { var launchTemplate *ec2.LaunchTemplate name := launchTemplateName(options) ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("launch-template-name", name)) @@ -194,7 +194,7 @@ func (p *LaunchTemplateProvider) ensureLaunchTemplate(ctx context.Context, optio return launchTemplate, nil } -func (p *LaunchTemplateProvider) createLaunchTemplate(ctx context.Context, options *amifamily.LaunchTemplate) (*ec2.LaunchTemplate, error) { +func (p *Provider) createLaunchTemplate(ctx context.Context, options *amifamily.LaunchTemplate) (*ec2.LaunchTemplate, error) { userData, err := options.UserData.Script() if err != nil { return nil, err @@ -236,7 +236,7 @@ func (p *LaunchTemplateProvider) createLaunchTemplate(ctx context.Context, optio return output.LaunchTemplate, nil } -func (p *LaunchTemplateProvider) blockDeviceMappings(blockDeviceMappings []*v1alpha1.BlockDeviceMapping) []*ec2.LaunchTemplateBlockDeviceMappingRequest { +func (p *Provider) blockDeviceMappings(blockDeviceMappings []*v1alpha1.BlockDeviceMapping) []*ec2.LaunchTemplateBlockDeviceMappingRequest { if len(blockDeviceMappings) == 0 { // The EC2 API fails with empty slices and expects nil. return nil @@ -261,7 +261,7 @@ func (p *LaunchTemplateProvider) blockDeviceMappings(blockDeviceMappings []*v1al } // volumeSize returns a GiB scaled value from a resource quantity or nil if the resource quantity passed in is nil -func (p *LaunchTemplateProvider) volumeSize(quantity *resource.Quantity) *int64 { +func (p *Provider) volumeSize(quantity *resource.Quantity) *int64 { if quantity == nil { return nil } @@ -271,7 +271,7 @@ func (p *LaunchTemplateProvider) volumeSize(quantity *resource.Quantity) *int64 // hydrateCache queries for existing Launch Templates created by Karpenter for the current cluster and adds to the LT cache. // Any error during hydration will result in a panic -func (p *LaunchTemplateProvider) hydrateCache(ctx context.Context) { +func (p *Provider) hydrateCache(ctx context.Context) { clusterName := awssettings.FromContext(ctx).ClusterName ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("tag-key", karpenterManagedTagKey, "tag-value", clusterName)) logging.FromContext(ctx).Debugf("hydrating the launch template cache") @@ -288,7 +288,7 @@ func (p *LaunchTemplateProvider) hydrateCache(ctx context.Context) { logging.FromContext(ctx).With("item-count", p.cache.ItemCount()).Debugf("finished hydrating the launch template cache") } -func (p *LaunchTemplateProvider) cachedEvictedFunc(ctx context.Context) func(string, interface{}) { +func (p *Provider) cachedEvictedFunc(ctx context.Context) func(string, interface{}) { return func(key string, lt interface{}) { p.Lock() defer p.Unlock() @@ -304,7 +304,7 @@ func (p *LaunchTemplateProvider) cachedEvictedFunc(ctx context.Context) func(str } } -func (p *LaunchTemplateProvider) getInstanceProfile(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) (string, error) { +func (p *Provider) getInstanceProfile(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) (string, error) { if nodeTemplate.Spec.InstanceProfile != nil { return aws.StringValue(nodeTemplate.Spec.InstanceProfile), nil } diff --git a/pkg/cloudprovider/launchtemplate_test.go b/pkg/providers/launchtemplate/launchtemplate_test.go similarity index 99% rename from pkg/cloudprovider/launchtemplate_test.go rename to pkg/providers/launchtemplate/launchtemplate_test.go index c6320159bfce..f988ce4caeb5 100644 --- a/pkg/cloudprovider/launchtemplate_test.go +++ b/pkg/providers/launchtemplate/launchtemplate_test.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cloudprovider +package launchtemplate_test import ( "encoding/base64" From 2af2dc4f5f6c81e8a4a9ede138fa39db961b5ed7 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Wed, 22 Feb 2023 17:31:45 -0800 Subject: [PATCH 02/41] First round of fixing tests --- pkg/cloudprovider/suite_test.go | 25 +-- .../launchtemplate/launchtemplate.go | 8 + .../launchtemplate/launchtemplate_test.go | 161 +++++++++++++++++- 3 files changed, 178 insertions(+), 16 deletions(-) diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index df3c09fc4259..9ed3dae86531 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -67,6 +67,7 @@ import ( coretest "github.com/aws/karpenter-core/pkg/test" "github.com/aws/karpenter-core/pkg/utils/pretty" + "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/securitygroup" "github.com/aws/karpenter/pkg/providers/subnet" ) @@ -82,7 +83,7 @@ var kubernetesVersionCache *cache.Cache var unavailableOfferingsCache *awscache.UnavailableOfferings var instanceTypeCache *cache.Cache var instanceTypeProvider *InstanceTypeProvider -var launchTemplateProvider *LaunchTemplateProvider +var launchTemplateProvider *launchtemplate.Provider var amiProvider *amifamily.AMIProvider var fakeEC2API *fake.EC2API var fakeSSMAPI *fake.SSMAPI @@ -133,14 +134,16 @@ var _ = BeforeSuite(func() { cm: pretty.NewChangeMonitor(), } securityGroupProvider = securitygroup.NewProvider(fakeEC2API) - launchTemplateProvider = &LaunchTemplateProvider{ - ec2api: fakeEC2API, - amiFamily: amifamily.New(env.Client, amiProvider), - securityGroupProvider: securityGroupProvider, - cache: launchTemplateCache, - caBundle: ptr.String("ca-bundle"), - cm: pretty.NewChangeMonitor(), - } + launchTemplateProvider = launchtemplate.NewProvider( + ctx, + fakeEC2API, + amifamily.New(env.Client, amiProvider), + securityGroupProvider, + ptr.String("ca-bundle"), + make(chan struct{}), + net.ParseIP("10.0.100.10"), + "https://test-cluster", + ) cloudProvider = &CloudProvider{ instanceTypeProvider: instanceTypeProvider, amiProvider: amiProvider, @@ -204,8 +207,8 @@ var _ = BeforeEach(func() { instanceTypeCache.Flush() subnetProvider.Reset() securityGroupProvider.Reset() - launchTemplateProvider.kubeDNSIP = net.ParseIP("10.0.100.10") - launchTemplateProvider.clusterEndpoint = "https://test-cluster" + launchTemplateProvider.UpdateKubeDNSIP(net.ParseIP("10.0.100.10")) + launchTemplateProvider.UpdateClusterEndpoint("https://test-cluster") // Reset the pricing provider, so we don't cross-pollinate pricing data instanceTypeProvider = &InstanceTypeProvider{ diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index c6071ad74d7d..407848d10555 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -314,3 +314,11 @@ func (p *Provider) getInstanceProfile(ctx context.Context, nodeTemplate *v1alpha } return defaultProfile, nil } + +func (p *Provider) UpdateKubeDNSIP(kDNSIP net.IP) { + p.kubeDNSIP = kDNSIP +} + +func (p *Provider) UpdateClusterEndpoint(endpoint string) { + p.clusterEndpoint = endpoint +} diff --git a/pkg/providers/launchtemplate/launchtemplate_test.go b/pkg/providers/launchtemplate/launchtemplate_test.go index f988ce4caeb5..893f1dcdb416 100644 --- a/pkg/providers/launchtemplate/launchtemplate_test.go +++ b/pkg/providers/launchtemplate/launchtemplate_test.go @@ -15,6 +15,7 @@ limitations under the License. package launchtemplate_test import ( + "context" "encoding/base64" "errors" "fmt" @@ -23,30 +24,180 @@ import ( "os" "sort" "strings" + "testing" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/awstesting/mock" "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/patrickmn/go-cache" "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" + clock "k8s.io/utils/clock/testing" + . "knative.dev/pkg/logging/testing" + "knative.dev/pkg/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" + "github.com/aws/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter/pkg/cloudprovider/amifamily" "github.com/aws/karpenter/pkg/cloudprovider/amifamily/bootstrap" + "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/providers/launchtemplate" + "github.com/aws/karpenter/pkg/providers/securitygroup" + "github.com/aws/karpenter/pkg/providers/subnet" "github.com/aws/karpenter/pkg/test" + coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" + "github.com/aws/karpenter-core/pkg/controllers/provisioning" + "github.com/aws/karpenter-core/pkg/controllers/state" + "github.com/aws/karpenter-core/pkg/events" + "github.com/aws/karpenter-core/pkg/operator/injection" + "github.com/aws/karpenter-core/pkg/operator/options" + "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" + awscache "github.com/aws/karpenter/pkg/cache" + awscontext "github.com/aws/karpenter/pkg/context" ) +var ctx context.Context +var stop context.CancelFunc +var opts options.Options +var env *coretest.Environment +var ssmCache *cache.Cache +var ec2Cache *cache.Cache +var kubernetesVersionCache *cache.Cache +var fakeEC2API *fake.EC2API +var fakeSSMAPI *fake.SSMAPI +var fakeClock *clock.FakeClock +var amiProvider *amifamily.AMIProvider +var cloudProvider *cloudprovider.CloudProvider +var unavailableOfferingsCache *awscache.UnavailableOfferings +var prov *provisioning.Provisioner +var provisioner *v1alpha5.Provisioner +var launchTemplateProvider *launchtemplate.Provider +var nodeTemplate *v1alpha1.AWSNodeTemplate +var cluster *state.Cluster +var securityGroupProvider *securitygroup.Provider + +func TestAWS(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "CloudProvider/AWS") +} + +var _ = BeforeSuite(func() { + env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = settings.ToContext(ctx, test.Settings()) + ctx, stop = context.WithCancel(ctx) + + fakeEC2API = &fake.EC2API{} + fakeSSMAPI = &fake.SSMAPI{} + ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + fakeClock = clock.NewFakeClock(time.Now()) + unavailableOfferingsCache = awscache.NewUnavailableOfferings() + securityGroupProvider = securitygroup.NewProvider(fakeEC2API) + amiProvider = amifamily.NewAMIProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) + launchTemplateProvider = launchtemplate.NewProvider( + ctx, + fakeEC2API, + amifamily.New(env.Client, amiProvider), + securityGroupProvider, + ptr.String("ca-bundle"), + make(chan struct{}), + net.ParseIP("10.0.100.10"), + "https://test-cluster", + ) + + cloudProvider = cloudprovider.New(awscontext.Context{ + Context: corecloudprovider.Context{ + Context: ctx, + RESTConfig: env.Config, + KubernetesInterface: env.KubernetesInterface, + KubeClient: env.Client, + EventRecorder: events.NewRecorder(&record.FakeRecorder{}), + Clock: &clock.FakeClock{}, + StartAsync: nil, + }, + SubnetProvider: subnet.NewProvider(fakeEC2API), + SecurityGroupProvider: securityGroupProvider, + Session: mock.Session, + UnavailableOfferingsCache: unavailableOfferingsCache, + EC2API: fakeEC2API, + }) + cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) +}) + +var _ = AfterSuite(func() { + stop() + Expect(env.Stop()).To(Succeed(), "Failed to stop environment") +}) + +var _ = BeforeEach(func() { + ctx = injection.WithOptions(ctx, opts) + ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = settings.ToContext(ctx, test.Settings()) + nodeTemplate = &v1alpha1.AWSNodeTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: coretest.RandomName(), + }, + Spec: v1alpha1.AWSNodeTemplateSpec{ + AWS: v1alpha1.AWS{ + AMIFamily: aws.String(v1alpha1.AMIFamilyAL2), + SubnetSelector: map[string]string{"*": "*"}, + SecurityGroupSelector: map[string]string{"*": "*"}, + }, + }, + } + nodeTemplate.SetGroupVersionKind(schema.GroupVersionKind{ + Group: v1alpha1.SchemeGroupVersion.Group, + Version: v1alpha1.SchemeGroupVersion.Version, + Kind: "AWSNodeTemplate", + }) + provisioner = test.Provisioner(coretest.ProvisionerOptions{ + Requirements: []v1.NodeSelectorRequirement{{ + Key: v1alpha1.LabelInstanceCategory, + Operator: v1.NodeSelectorOpExists, + }}, + ProviderRef: &v1alpha5.ProviderRef{ + APIVersion: nodeTemplate.APIVersion, + Kind: nodeTemplate.Kind, + Name: nodeTemplate.Name, + }, + }) + + cluster.Reset() + fakeEC2API.Reset() + fakeSSMAPI.Reset() + unavailableOfferingsCache.Flush() + ssmCache.Flush() + ec2Cache.Flush() + kubernetesVersionCache.Flush() + securityGroupProvider.Reset() + launchTemplateProvider.UpdateKubeDNSIP(net.ParseIP("10.0.100.10")) + launchTemplateProvider.UpdateClusterEndpoint("https://test-cluster") +}) + +var _ = AfterEach(func() { + ExpectCleanedUp(ctx, env.Client) +}) + var _ = Describe("LaunchTemplates", func() { It("should default to a generated launch template", func() { ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) @@ -689,7 +840,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := cloudprovider.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1093Mi")) }) @@ -700,7 +851,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := cloudprovider.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1093Mi")) }) @@ -724,7 +875,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := cloudprovider.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1093Mi")) }) @@ -735,7 +886,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := cloudprovider.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1665Mi")) }) @@ -1024,7 +1175,7 @@ var _ = Describe("LaunchTemplates", func() { Expect(string(userData)).To(ContainSubstring("--container-runtime containerd")) }) It("should specify --dns-cluster-ip and --ip-family when running in an ipv6 cluster", func() { - launchTemplateProvider.kubeDNSIP = net.ParseIP("fd4b:121b:812b::a") + launchTemplateProvider.UpdateKubeDNSIP(net.ParseIP("fd4b:121b:812b::a")) ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, prov, pod) From 90b2094fc4f78247aacbb62d64875ee08ed9c68a Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Thu, 23 Feb 2023 10:05:55 -0800 Subject: [PATCH 03/41] Missing on Merge --- pkg/cloudprovider/cloudprovider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index 0f57d24015f0..50ec9d2900cd 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -70,7 +70,7 @@ type CloudProvider struct { } func New(ctx awscontext.Context) *CloudProvider { - instanceTypeProvider := NewInstanceTypeProvider(ctx, ctx.Session, ctx.EC2API, ctx.SubnetProvider, ctx.UnavailableOfferingsCache, ctx.StartAsync) + instanceTypeProvider := NewInstanceTypeProvider(ctx.Session, ctx.EC2API, ctx.SubnetProvider, ctx.UnavailableOfferingsCache, ctx.PricingProvider) return &CloudProvider{ kubeClient: ctx.KubeClient, instanceTypeProvider: instanceTypeProvider, From b177bf192687f759cb99bbe9179f596c52201eed Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Thu, 23 Feb 2023 10:30:48 -0800 Subject: [PATCH 04/41] adjustment to tests --- pkg/providers/launchtemplate/launchtemplate_test.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/providers/launchtemplate/launchtemplate_test.go b/pkg/providers/launchtemplate/launchtemplate_test.go index 893f1dcdb416..1b7f738c587e 100644 --- a/pkg/providers/launchtemplate/launchtemplate_test.go +++ b/pkg/providers/launchtemplate/launchtemplate_test.go @@ -54,6 +54,7 @@ import ( "github.com/aws/karpenter/pkg/cloudprovider/amifamily/bootstrap" "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/providers/launchtemplate" + "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/providers/securitygroup" "github.com/aws/karpenter/pkg/providers/subnet" "github.com/aws/karpenter/pkg/test" @@ -83,7 +84,9 @@ var kubernetesVersionCache *cache.Cache var fakeEC2API *fake.EC2API var fakeSSMAPI *fake.SSMAPI var fakeClock *clock.FakeClock +var fakePricingAPI *fake.PricingAPI var amiProvider *amifamily.AMIProvider +var amiResolver *amifamily.Resolver var cloudProvider *cloudprovider.CloudProvider var unavailableOfferingsCache *awscache.UnavailableOfferings var prov *provisioning.Provisioner @@ -91,6 +94,7 @@ var provisioner *v1alpha5.Provisioner var launchTemplateProvider *launchtemplate.Provider var nodeTemplate *v1alpha1.AWSNodeTemplate var cluster *state.Cluster +var pricingProvider *pricing.Provider var securityGroupProvider *securitygroup.Provider func TestAWS(t *testing.T) { @@ -112,12 +116,15 @@ var _ = BeforeSuite(func() { kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) fakeClock = clock.NewFakeClock(time.Now()) unavailableOfferingsCache = awscache.NewUnavailableOfferings() + fakePricingAPI = &fake.PricingAPI{} + pricingProvider = pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})) securityGroupProvider = securitygroup.NewProvider(fakeEC2API) amiProvider = amifamily.NewAMIProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) + amiResolver = amifamily.New(env.Client, amiProvider) launchTemplateProvider = launchtemplate.NewProvider( ctx, fakeEC2API, - amifamily.New(env.Client, amiProvider), + amiResolver, securityGroupProvider, ptr.String("ca-bundle"), make(chan struct{}), @@ -140,6 +147,10 @@ var _ = BeforeSuite(func() { Session: mock.Session, UnavailableOfferingsCache: unavailableOfferingsCache, EC2API: fakeEC2API, + PricingProvider: pricingProvider, + AMIProvider: amiProvider, + AMIResolver: amiResolver, + LaunchTemplateProvider: launchTemplateProvider, }) cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) }) From 3353793cec3cb924bb00e62a7878b29e5f79cffd Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Fri, 24 Feb 2023 11:41:31 -0800 Subject: [PATCH 05/41] Fix launchTemplateTest --- pkg/cloudprovider/instancetypes.go | 4 ++-- pkg/cloudprovider/instancetypes_test.go | 18 +++++++++--------- pkg/providers/launchtemplate/launchtemplate.go | 12 ++++++++++-- .../launchtemplate/launchtemplate_test.go | 16 ++++++++++++---- .../testdata/al2_no_mime_userdata_input.golden | 0 .../testdata/al2_userdata_input.golden | 0 .../testdata/al2_userdata_merged.golden | 0 .../testdata/al2_userdata_unmerged.golden | 0 .../testdata/br_userdata_input.golden | 0 .../testdata/br_userdata_merged.golden | 0 .../testdata/br_userdata_unmerged.golden | 0 11 files changed, 33 insertions(+), 17 deletions(-) rename pkg/{cloudprovider => providers/launchtemplate}/testdata/al2_no_mime_userdata_input.golden (100%) rename pkg/{cloudprovider => providers/launchtemplate}/testdata/al2_userdata_input.golden (100%) rename pkg/{cloudprovider => providers/launchtemplate}/testdata/al2_userdata_merged.golden (100%) rename pkg/{cloudprovider => providers/launchtemplate}/testdata/al2_userdata_unmerged.golden (100%) rename pkg/{cloudprovider => providers/launchtemplate}/testdata/br_userdata_input.golden (100%) rename pkg/{cloudprovider => providers/launchtemplate}/testdata/br_userdata_merged.golden (100%) rename pkg/{cloudprovider => providers/launchtemplate}/testdata/br_userdata_unmerged.golden (100%) diff --git a/pkg/cloudprovider/instancetypes.go b/pkg/cloudprovider/instancetypes.go index 75a1bc10bceb..b0513601b81e 100644 --- a/pkg/cloudprovider/instancetypes.go +++ b/pkg/cloudprovider/instancetypes.go @@ -84,7 +84,7 @@ func NewInstanceTypeProvider(sess *session.Session, ec2api ec2iface.EC2API, subn func (p *InstanceTypeProvider) List(ctx context.Context, kc *v1alpha5.KubeletConfiguration, nodeTemplate *v1alpha1.AWSNodeTemplate) ([]*cloudprovider.InstanceType, error) { // Get InstanceTypes from EC2 - instanceTypes, err := p.getInstanceTypes(ctx) + instanceTypes, err := p.GetInstanceTypes(ctx) if err != nil { return nil, err } @@ -195,7 +195,7 @@ func (p *InstanceTypeProvider) getInstanceTypeZones(ctx context.Context, nodeTem } // getInstanceTypes retrieves all instance types from the ec2 DescribeInstanceTypes API using some opinionated filters -func (p *InstanceTypeProvider) getInstanceTypes(ctx context.Context) ([]*ec2.InstanceTypeInfo, error) { +func (p *InstanceTypeProvider) GetInstanceTypes(ctx context.Context) ([]*ec2.InstanceTypeInfo, error) { // DO NOT REMOVE THIS LOCK ---------------------------------------------------------------------------- // We lock here so that multiple callers to GetInstanceTypes do not result in cache misses and multiple // calls to EC2 when we could have just made one call. This lock is here because multiple callers to EC2 result diff --git a/pkg/cloudprovider/instancetypes_test.go b/pkg/cloudprovider/instancetypes_test.go index d4ccb0bb5f1b..21df49f44f94 100644 --- a/pkg/cloudprovider/instancetypes_test.go +++ b/pkg/cloudprovider/instancetypes_test.go @@ -400,7 +400,7 @@ var _ = Describe("Instance Types", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), })) - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) for _, info := range instanceInfo { it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) @@ -408,7 +408,7 @@ var _ = Describe("Instance Types", func() { } }) It("should not set pods to 110 if using ENI-based pod density", func() { - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) for _, info := range instanceInfo { it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) @@ -424,7 +424,7 @@ var _ = Describe("Instance Types", func() { })) var ok bool - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) info, ok = lo.Find(instanceInfo, func(i *ec2.InstanceTypeInfo) bool { return aws.StringValue(i.InstanceType) == "m5.xlarge" @@ -686,7 +686,7 @@ var _ = Describe("Instance Types", func() { }) }) It("should set max-pods to user-defined value if specified", func() { - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{MaxPods: ptr.Int32(10)}}) for _, info := range instanceInfo { @@ -699,7 +699,7 @@ var _ = Describe("Instance Types", func() { EnablePodENI: lo.ToPtr(false), })) - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{MaxPods: ptr.Int32(10)}}) for _, info := range instanceInfo { @@ -708,7 +708,7 @@ var _ = Describe("Instance Types", func() { } }) It("should override pods-per-core value", func() { - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}}) for _, info := range instanceInfo { @@ -717,7 +717,7 @@ var _ = Describe("Instance Types", func() { } }) It("should take the minimum of pods-per-core and max-pods", func() { - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(4), MaxPods: ptr.Int32(20)}}) for _, info := range instanceInfo { @@ -726,7 +726,7 @@ var _ = Describe("Instance Types", func() { } }) It("should ignore pods-per-core when using Bottlerocket AMI", func() { - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}}) @@ -740,7 +740,7 @@ var _ = Describe("Instance Types", func() { EnableENILimitedPodDensity: lo.ToPtr(false), })) - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(0)}}) for _, info := range instanceInfo { diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index 407848d10555..05297fd7167b 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -315,10 +315,18 @@ func (p *Provider) getInstanceProfile(ctx context.Context, nodeTemplate *v1alpha return defaultProfile, nil } -func (p *Provider) UpdateKubeDNSIP(kDNSIP net.IP) { - p.kubeDNSIP = kDNSIP +func (p *Provider) UpdateKubeDNSIP(kubeDNSIP net.IP) { + p.kubeDNSIP = kubeDNSIP } func (p *Provider) UpdateClusterEndpoint(endpoint string) { p.clusterEndpoint = endpoint } + +func (p *Provider) GetCache(ltName string) (result interface{}, ok bool) { + return p.cache.Get(ltName) +} + +func (p *Provider) SetCache(ltName string, lt interface{}) { + p.cache.Set(ltName, lt, -1) +} diff --git a/pkg/providers/launchtemplate/launchtemplate_test.go b/pkg/providers/launchtemplate/launchtemplate_test.go index 1b7f738c587e..32b5174a2766 100644 --- a/pkg/providers/launchtemplate/launchtemplate_test.go +++ b/pkg/providers/launchtemplate/launchtemplate_test.go @@ -95,6 +95,8 @@ var launchTemplateProvider *launchtemplate.Provider var nodeTemplate *v1alpha1.AWSNodeTemplate var cluster *state.Cluster var pricingProvider *pricing.Provider +var subnetProvider *subnet.Provider +var instanceTypeProvider *cloudprovider.InstanceTypeProvider var securityGroupProvider *securitygroup.Provider func TestAWS(t *testing.T) { @@ -118,9 +120,12 @@ var _ = BeforeSuite(func() { unavailableOfferingsCache = awscache.NewUnavailableOfferings() fakePricingAPI = &fake.PricingAPI{} pricingProvider = pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})) + subnetProvider = subnet.NewProvider(fakeEC2API) securityGroupProvider = securitygroup.NewProvider(fakeEC2API) amiProvider = amifamily.NewAMIProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) amiResolver = amifamily.New(env.Client, amiProvider) + instanceTypeProvider = cloudprovider.NewInstanceTypeProvider(mock.Session, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) + launchTemplateProvider = launchtemplate.NewProvider( ctx, fakeEC2API, @@ -153,6 +158,8 @@ var _ = BeforeSuite(func() { LaunchTemplateProvider: launchTemplateProvider, }) cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) + prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) + }) var _ = AfterSuite(func() { @@ -213,6 +220,7 @@ var _ = Describe("LaunchTemplates", func() { It("should default to a generated launch template", func() { ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() + fmt.Println(cluster.Nodes()) ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectScheduled(ctx, env.Client, pod) @@ -344,10 +352,10 @@ var _ = Describe("LaunchTemplates", func() { Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) firstLt := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() ltName := aws.StringValue(firstLt.LaunchTemplateName) - lt, ok := launchTemplateCache.Get(ltName) + lt, ok := launchTemplateProvider.GetCache(ltName) Expect(ok).To(Equal(true)) // Remove expiration from cached LT - launchTemplateCache.Set(ltName, lt, -1) + launchTemplateProvider.SetCache(ltName, lt) fakeEC2API.CreateFleetBehavior.Error.Set(awserr.New("InvalidLaunchTemplateName.NotFoundException", "", errors.New(""))) pod = coretest.UnschedulablePod() @@ -836,7 +844,7 @@ var _ = Describe("LaunchTemplates", func() { var info *ec2.InstanceTypeInfo BeforeEach(func() { var ok bool - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) info, ok = lo.Find(instanceInfo, func(i *ec2.InstanceTypeInfo) bool { return aws.StringValue(i.InstanceType) == "m5.xlarge" @@ -871,7 +879,7 @@ var _ = Describe("LaunchTemplates", func() { var info *ec2.InstanceTypeInfo BeforeEach(func() { var ok bool - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) info, ok = lo.Find(instanceInfo, func(i *ec2.InstanceTypeInfo) bool { return aws.StringValue(i.InstanceType) == "m5.xlarge" diff --git a/pkg/cloudprovider/testdata/al2_no_mime_userdata_input.golden b/pkg/providers/launchtemplate/testdata/al2_no_mime_userdata_input.golden similarity index 100% rename from pkg/cloudprovider/testdata/al2_no_mime_userdata_input.golden rename to pkg/providers/launchtemplate/testdata/al2_no_mime_userdata_input.golden diff --git a/pkg/cloudprovider/testdata/al2_userdata_input.golden b/pkg/providers/launchtemplate/testdata/al2_userdata_input.golden similarity index 100% rename from pkg/cloudprovider/testdata/al2_userdata_input.golden rename to pkg/providers/launchtemplate/testdata/al2_userdata_input.golden diff --git a/pkg/cloudprovider/testdata/al2_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/al2_userdata_merged.golden similarity index 100% rename from pkg/cloudprovider/testdata/al2_userdata_merged.golden rename to pkg/providers/launchtemplate/testdata/al2_userdata_merged.golden diff --git a/pkg/cloudprovider/testdata/al2_userdata_unmerged.golden b/pkg/providers/launchtemplate/testdata/al2_userdata_unmerged.golden similarity index 100% rename from pkg/cloudprovider/testdata/al2_userdata_unmerged.golden rename to pkg/providers/launchtemplate/testdata/al2_userdata_unmerged.golden diff --git a/pkg/cloudprovider/testdata/br_userdata_input.golden b/pkg/providers/launchtemplate/testdata/br_userdata_input.golden similarity index 100% rename from pkg/cloudprovider/testdata/br_userdata_input.golden rename to pkg/providers/launchtemplate/testdata/br_userdata_input.golden diff --git a/pkg/cloudprovider/testdata/br_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/br_userdata_merged.golden similarity index 100% rename from pkg/cloudprovider/testdata/br_userdata_merged.golden rename to pkg/providers/launchtemplate/testdata/br_userdata_merged.golden diff --git a/pkg/cloudprovider/testdata/br_userdata_unmerged.golden b/pkg/providers/launchtemplate/testdata/br_userdata_unmerged.golden similarity index 100% rename from pkg/cloudprovider/testdata/br_userdata_unmerged.golden rename to pkg/providers/launchtemplate/testdata/br_userdata_unmerged.golden From 294d0682d21666c00b5c06f4753511b57644afb4 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Fri, 24 Feb 2023 14:00:09 -0800 Subject: [PATCH 06/41] Feedback changes --- pkg/cloudprovider/instancetypes.go | 4 +- pkg/cloudprovider/instancetypes_test.go | 18 ++--- pkg/cloudprovider/suite_test.go | 5 +- pkg/context/context.go | 1 + .../launchtemplate/launchtemplate.go | 73 ++++++++----------- .../launchtemplate/launchtemplate_test.go | 48 ++++++++++-- 6 files changed, 88 insertions(+), 61 deletions(-) diff --git a/pkg/cloudprovider/instancetypes.go b/pkg/cloudprovider/instancetypes.go index b0513601b81e..75a1bc10bceb 100644 --- a/pkg/cloudprovider/instancetypes.go +++ b/pkg/cloudprovider/instancetypes.go @@ -84,7 +84,7 @@ func NewInstanceTypeProvider(sess *session.Session, ec2api ec2iface.EC2API, subn func (p *InstanceTypeProvider) List(ctx context.Context, kc *v1alpha5.KubeletConfiguration, nodeTemplate *v1alpha1.AWSNodeTemplate) ([]*cloudprovider.InstanceType, error) { // Get InstanceTypes from EC2 - instanceTypes, err := p.GetInstanceTypes(ctx) + instanceTypes, err := p.getInstanceTypes(ctx) if err != nil { return nil, err } @@ -195,7 +195,7 @@ func (p *InstanceTypeProvider) getInstanceTypeZones(ctx context.Context, nodeTem } // getInstanceTypes retrieves all instance types from the ec2 DescribeInstanceTypes API using some opinionated filters -func (p *InstanceTypeProvider) GetInstanceTypes(ctx context.Context) ([]*ec2.InstanceTypeInfo, error) { +func (p *InstanceTypeProvider) getInstanceTypes(ctx context.Context) ([]*ec2.InstanceTypeInfo, error) { // DO NOT REMOVE THIS LOCK ---------------------------------------------------------------------------- // We lock here so that multiple callers to GetInstanceTypes do not result in cache misses and multiple // calls to EC2 when we could have just made one call. This lock is here because multiple callers to EC2 result diff --git a/pkg/cloudprovider/instancetypes_test.go b/pkg/cloudprovider/instancetypes_test.go index 21df49f44f94..d4ccb0bb5f1b 100644 --- a/pkg/cloudprovider/instancetypes_test.go +++ b/pkg/cloudprovider/instancetypes_test.go @@ -400,7 +400,7 @@ var _ = Describe("Instance Types", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), })) - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) Expect(err).To(BeNil()) for _, info := range instanceInfo { it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) @@ -408,7 +408,7 @@ var _ = Describe("Instance Types", func() { } }) It("should not set pods to 110 if using ENI-based pod density", func() { - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) Expect(err).To(BeNil()) for _, info := range instanceInfo { it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) @@ -424,7 +424,7 @@ var _ = Describe("Instance Types", func() { })) var ok bool - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) Expect(err).To(BeNil()) info, ok = lo.Find(instanceInfo, func(i *ec2.InstanceTypeInfo) bool { return aws.StringValue(i.InstanceType) == "m5.xlarge" @@ -686,7 +686,7 @@ var _ = Describe("Instance Types", func() { }) }) It("should set max-pods to user-defined value if specified", func() { - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{MaxPods: ptr.Int32(10)}}) for _, info := range instanceInfo { @@ -699,7 +699,7 @@ var _ = Describe("Instance Types", func() { EnablePodENI: lo.ToPtr(false), })) - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{MaxPods: ptr.Int32(10)}}) for _, info := range instanceInfo { @@ -708,7 +708,7 @@ var _ = Describe("Instance Types", func() { } }) It("should override pods-per-core value", func() { - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}}) for _, info := range instanceInfo { @@ -717,7 +717,7 @@ var _ = Describe("Instance Types", func() { } }) It("should take the minimum of pods-per-core and max-pods", func() { - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(4), MaxPods: ptr.Int32(20)}}) for _, info := range instanceInfo { @@ -726,7 +726,7 @@ var _ = Describe("Instance Types", func() { } }) It("should ignore pods-per-core when using Bottlerocket AMI", func() { - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) Expect(err).To(BeNil()) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}}) @@ -740,7 +740,7 @@ var _ = Describe("Instance Types", func() { EnableENILimitedPodDensity: lo.ToPtr(false), })) - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(0)}}) for _, info := range instanceInfo { diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index 5bfd484434f9..80c50f13b9ac 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -137,6 +137,7 @@ var _ = BeforeSuite(func() { securityGroupProvider = securitygroup.NewProvider(fakeEC2API) launchTemplateProvider = launchtemplate.NewProvider( ctx, + launchTemplateCache, fakeEC2API, amifamily.New(env.Client, amiProvider), securityGroupProvider, @@ -208,8 +209,8 @@ var _ = BeforeEach(func() { instanceTypeCache.Flush() subnetProvider.Reset() securityGroupProvider.Reset() - launchTemplateProvider.UpdateKubeDNSIP(net.ParseIP("10.0.100.10")) - launchTemplateProvider.UpdateClusterEndpoint("https://test-cluster") + launchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") + launchTemplateProvider.ClusterEndpoint = "https://test-cluster" // Reset the pricing provider, so we don't cross-pollinate pricing data instanceTypeProvider = &InstanceTypeProvider{ diff --git a/pkg/context/context.go b/pkg/context/context.go index 3abd38b6a1d0..e93fbf3bcf72 100644 --- a/pkg/context/context.go +++ b/pkg/context/context.go @@ -117,6 +117,7 @@ func NewOrDie(ctx cloudprovider.Context) Context { amiResolver := amifamily.New(ctx.KubeClient, amiProvider) launchTemplateProvider := launchtemplate.NewProvider( ctx, + cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), ec2api, amiResolver, securityGroupProvider, diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index 05297fd7167b..a7bbae983c57 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -35,7 +35,6 @@ import ( awssettings "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" - awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/cloudprovider/amifamily" awserrors "github.com/aws/karpenter/pkg/errors" "github.com/aws/karpenter/pkg/providers/securitygroup" @@ -56,27 +55,27 @@ type Provider struct { ec2api ec2iface.EC2API amiFamily *amifamily.Resolver securityGroupProvider *securitygroup.Provider - cache *cache.Cache + Cache *cache.Cache caBundle *string cm *pretty.ChangeMonitor - kubeDNSIP net.IP - clusterEndpoint string + KubeDNSIP net.IP + ClusterEndpoint string } -func NewProvider(ctx context.Context, ec2api ec2iface.EC2API, amiFamily *amifamily.Resolver, securityGroupProvider *securitygroup.Provider, caBundle *string, startAsync <-chan struct{}, kubeDNSIP net.IP, clusterEndpoint string) *Provider { +func NewProvider(ctx context.Context, Cache *cache.Cache, ec2api ec2iface.EC2API, amiFamily *amifamily.Resolver, securityGroupProvider *securitygroup.Provider, caBundle *string, startAsync <-chan struct{}, kubeDNSIP net.IP, clusterEndpoint string) *Provider { l := &Provider{ ec2api: ec2api, amiFamily: amiFamily, securityGroupProvider: securityGroupProvider, - cache: cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), + Cache: Cache, caBundle: caBundle, cm: pretty.NewChangeMonitor(), - kubeDNSIP: kubeDNSIP, - clusterEndpoint: clusterEndpoint, + KubeDNSIP: kubeDNSIP, + ClusterEndpoint: clusterEndpoint, } - l.cache.OnEvicted(l.cachedEvictedFunc(ctx)) + l.Cache.OnEvicted(l.cachedEvictedFunc(ctx)) go func() { - // only hydrate cache once elected leader + // only hydrate Cache once elected leader select { case <-startAsync: case <-ctx.Done(): @@ -116,15 +115,15 @@ func (p *Provider) EnsureAll(ctx context.Context, nodeTemplate *v1alpha1.AWSNode return launchTemplates, nil } -// Invalidate deletes a launch template from cache if it exists +// Invalidate deletes a launch template from Cache if it exists func (p *Provider) Invalidate(ctx context.Context, ltName string, ltID string) { ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("launch-template-name", ltName, "launch-template-id", ltID)) p.Lock() defer p.Unlock() - defer p.cache.OnEvicted(p.cachedEvictedFunc(ctx)) - p.cache.OnEvicted(nil) - logging.FromContext(ctx).Debugf("invalidating launch template in the cache because it no longer exists") - p.cache.Delete(ltName) + defer p.Cache.OnEvicted(p.cachedEvictedFunc(ctx)) + p.Cache.OnEvicted(nil) + logging.FromContext(ctx).Debugf("invalidating launch template in the Cache because it no longer exists") + p.Cache.Delete(ltName) } func launchTemplateName(options *amifamily.LaunchTemplate) string { @@ -150,14 +149,14 @@ func (p *Provider) createAmiOptions(ctx context.Context, nodeTemplate *v1alpha1. } return &amifamily.Options{ ClusterName: awssettings.FromContext(ctx).ClusterName, - ClusterEndpoint: p.clusterEndpoint, + ClusterEndpoint: p.ClusterEndpoint, AWSENILimitedPodDensity: awssettings.FromContext(ctx).EnableENILimitedPodDensity, InstanceProfile: instanceProfile, SecurityGroupsIDs: securityGroupsIDs, Tags: lo.Assign(awssettings.FromContext(ctx).Tags, nodeTemplate.Spec.Tags), Labels: labels, CABundle: p.caBundle, - KubeDNSIP: p.kubeDNSIP, + KubeDNSIP: p.KubeDNSIP, }, nil } @@ -165,9 +164,9 @@ func (p *Provider) ensureLaunchTemplate(ctx context.Context, options *amifamily. var launchTemplate *ec2.LaunchTemplate name := launchTemplateName(options) ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("launch-template-name", name)) - // Read from cache - if launchTemplate, ok := p.cache.Get(name); ok { - p.cache.SetDefault(name, launchTemplate) + // Read from Cache + if launchTemplate, ok := p.Cache.Get(name); ok { + p.Cache.SetDefault(name, launchTemplate) return launchTemplate.(*ec2.LaunchTemplate), nil } // Attempt to find an existing LT. @@ -190,7 +189,7 @@ func (p *Provider) ensureLaunchTemplate(ctx context.Context, options *amifamily. } launchTemplate = output.LaunchTemplates[0] } - p.cache.SetDefault(name, launchTemplate) + p.Cache.SetDefault(name, launchTemplate) return launchTemplate, nil } @@ -269,30 +268,30 @@ func (p *Provider) volumeSize(quantity *resource.Quantity) *int64 { return aws.Int64(int64(math.Ceil(quantity.AsApproximateFloat64() / math.Pow(2, 30)))) } -// hydrateCache queries for existing Launch Templates created by Karpenter for the current cluster and adds to the LT cache. +// hydrateCache queries for existing Launch Templates created by Karpenter for the current cluster and adds to the LT Cache. // Any error during hydration will result in a panic func (p *Provider) hydrateCache(ctx context.Context) { clusterName := awssettings.FromContext(ctx).ClusterName ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("tag-key", karpenterManagedTagKey, "tag-value", clusterName)) - logging.FromContext(ctx).Debugf("hydrating the launch template cache") + logging.FromContext(ctx).Debugf("hydrating the launch template Cache") if err := p.ec2api.DescribeLaunchTemplatesPagesWithContext(ctx, &ec2.DescribeLaunchTemplatesInput{ Filters: []*ec2.Filter{{Name: aws.String(fmt.Sprintf("tag:%s", karpenterManagedTagKey)), Values: []*string{aws.String(clusterName)}}}, }, func(output *ec2.DescribeLaunchTemplatesOutput, _ bool) bool { for _, lt := range output.LaunchTemplates { - p.cache.SetDefault(*lt.LaunchTemplateName, lt) + p.Cache.SetDefault(*lt.LaunchTemplateName, lt) } return true }); err != nil { - logging.FromContext(ctx).Errorf(fmt.Sprintf("Unable to hydrate the AWS launch template cache, %s", err)) + logging.FromContext(ctx).Errorf(fmt.Sprintf("Unable to hydrate the AWS launch template Cache, %s", err)) } - logging.FromContext(ctx).With("item-count", p.cache.ItemCount()).Debugf("finished hydrating the launch template cache") + logging.FromContext(ctx).With("item-count", p.Cache.ItemCount()).Debugf("finished hydrating the launch template Cache") } func (p *Provider) cachedEvictedFunc(ctx context.Context) func(string, interface{}) { return func(key string, lt interface{}) { p.Lock() defer p.Unlock() - if _, expiration, _ := p.cache.GetWithExpiration(key); expiration.After(time.Now()) { + if _, expiration, _ := p.Cache.GetWithExpiration(key); expiration.After(time.Now()) { return } launchTemplate := lt.(*ec2.LaunchTemplate) @@ -315,18 +314,10 @@ func (p *Provider) getInstanceProfile(ctx context.Context, nodeTemplate *v1alpha return defaultProfile, nil } -func (p *Provider) UpdateKubeDNSIP(kubeDNSIP net.IP) { - p.kubeDNSIP = kubeDNSIP -} - -func (p *Provider) UpdateClusterEndpoint(endpoint string) { - p.clusterEndpoint = endpoint -} +// func (p *Provider) GetCache(ltName string) (result interface{}, ok bool) { +// return p.Cache.Get(ltName) +// } -func (p *Provider) GetCache(ltName string) (result interface{}, ok bool) { - return p.cache.Get(ltName) -} - -func (p *Provider) SetCache(ltName string, lt interface{}) { - p.cache.Set(ltName, lt, -1) -} +// func (p *Provider) SetCache(ltName string, lt interface{}) { +// p.Cache.Set(ltName, lt, -1) +// } diff --git a/pkg/providers/launchtemplate/launchtemplate_test.go b/pkg/providers/launchtemplate/launchtemplate_test.go index 32b5174a2766..e219e9404954 100644 --- a/pkg/providers/launchtemplate/launchtemplate_test.go +++ b/pkg/providers/launchtemplate/launchtemplate_test.go @@ -80,6 +80,7 @@ var opts options.Options var env *coretest.Environment var ssmCache *cache.Cache var ec2Cache *cache.Cache +var launchTemplateCache *cache.Cache var kubernetesVersionCache *cache.Cache var fakeEC2API *fake.EC2API var fakeSSMAPI *fake.SSMAPI @@ -115,6 +116,7 @@ var _ = BeforeSuite(func() { fakeSSMAPI = &fake.SSMAPI{} ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) fakeClock = clock.NewFakeClock(time.Now()) unavailableOfferingsCache = awscache.NewUnavailableOfferings() @@ -128,6 +130,7 @@ var _ = BeforeSuite(func() { launchTemplateProvider = launchtemplate.NewProvider( ctx, + launchTemplateCache, fakeEC2API, amiResolver, securityGroupProvider, @@ -203,13 +206,14 @@ var _ = BeforeEach(func() { cluster.Reset() fakeEC2API.Reset() fakeSSMAPI.Reset() + launchTemplateCache.Flush() unavailableOfferingsCache.Flush() ssmCache.Flush() ec2Cache.Flush() kubernetesVersionCache.Flush() securityGroupProvider.Reset() - launchTemplateProvider.UpdateKubeDNSIP(net.ParseIP("10.0.100.10")) - launchTemplateProvider.UpdateClusterEndpoint("https://test-cluster") + launchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") + launchTemplateProvider.ClusterEndpoint = "https://test-cluster" }) var _ = AfterEach(func() { @@ -352,10 +356,10 @@ var _ = Describe("LaunchTemplates", func() { Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) firstLt := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() ltName := aws.StringValue(firstLt.LaunchTemplateName) - lt, ok := launchTemplateProvider.GetCache(ltName) + lt, ok := launchTemplateCache.Get(ltName) Expect(ok).To(Equal(true)) // Remove expiration from cached LT - launchTemplateProvider.SetCache(ltName, lt) + launchTemplateCache.Set(ltName, lt, -1) fakeEC2API.CreateFleetBehavior.Error.Set(awserr.New("InvalidLaunchTemplateName.NotFoundException", "", errors.New(""))) pod = coretest.UnschedulablePod() @@ -844,7 +848,22 @@ var _ = Describe("LaunchTemplates", func() { var info *ec2.InstanceTypeInfo BeforeEach(func() { var ok bool - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + var instanceInfo []*ec2.InstanceTypeInfo + err := fakeEC2API.DescribeInstanceTypesPagesWithContext(ctx, &ec2.DescribeInstanceTypesInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("supported-virtualization-type"), + Values: []*string{aws.String("hvm")}, + }, + { + Name: aws.String("processor-info.supported-architecture"), + Values: aws.StringSlice([]string{"x86_64", "arm64"}), + }, + }, + }, func(page *ec2.DescribeInstanceTypesOutput, lastPage bool) bool { + instanceInfo = append(instanceInfo, page.InstanceTypes...) + return true + }) Expect(err).To(BeNil()) info, ok = lo.Find(instanceInfo, func(i *ec2.InstanceTypeInfo) bool { return aws.StringValue(i.InstanceType) == "m5.xlarge" @@ -879,7 +898,22 @@ var _ = Describe("LaunchTemplates", func() { var info *ec2.InstanceTypeInfo BeforeEach(func() { var ok bool - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + var instanceInfo []*ec2.InstanceTypeInfo + err := fakeEC2API.DescribeInstanceTypesPagesWithContext(ctx, &ec2.DescribeInstanceTypesInput{ + Filters: []*ec2.Filter{ + { + Name: aws.String("supported-virtualization-type"), + Values: []*string{aws.String("hvm")}, + }, + { + Name: aws.String("processor-info.supported-architecture"), + Values: aws.StringSlice([]string{"x86_64", "arm64"}), + }, + }, + }, func(page *ec2.DescribeInstanceTypesOutput, lastPage bool) bool { + instanceInfo = append(instanceInfo, page.InstanceTypes...) + return true + }) Expect(err).To(BeNil()) info, ok = lo.Find(instanceInfo, func(i *ec2.InstanceTypeInfo) bool { return aws.StringValue(i.InstanceType) == "m5.xlarge" @@ -1194,7 +1228,7 @@ var _ = Describe("LaunchTemplates", func() { Expect(string(userData)).To(ContainSubstring("--container-runtime containerd")) }) It("should specify --dns-cluster-ip and --ip-family when running in an ipv6 cluster", func() { - launchTemplateProvider.UpdateKubeDNSIP(net.ParseIP("fd4b:121b:812b::a")) + launchTemplateProvider.KubeDNSIP = net.ParseIP("fd4b:121b:812b::a") ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, prov, pod) From a6433276f227afcbc12b861479e067694038570e Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Mon, 27 Feb 2023 09:55:15 -0800 Subject: [PATCH 07/41] Move Tests into Context --- pkg/context/context.go | 4 +- .../suite_test.go} | 67 +++++++++++++++---- 2 files changed, 55 insertions(+), 16 deletions(-) rename pkg/{cloudprovider/cloudprovider_test.go => context/suite_test.go} (50%) diff --git a/pkg/context/context.go b/pkg/context/context.go index e93fbf3bcf72..cb955d100211 100644 --- a/pkg/context/context.go +++ b/pkg/context/context.go @@ -87,7 +87,7 @@ func NewOrDie(ctx cloudprovider.Context) Context { logging.FromContext(ctx).Fatalf("Checking EC2 API connectivity, %s", err) } logging.FromContext(ctx).With("region", *sess.Config.Region).Debugf("discovered region") - clusterEndpoint, err := ResolveClusterEndpoint(ctx, eks.New(sess)) + clusterEndpoint, err := resolveClusterEndpoint(ctx, eks.New(sess)) if err != nil { logging.FromContext(ctx).Fatalf("unable to detect the cluster endpoint, %s", err) } else { @@ -159,7 +159,7 @@ func checkEC2Connectivity(ctx context.Context, api *ec2.EC2) error { return err } -func ResolveClusterEndpoint(ctx context.Context, eksAPI eksiface.EKSAPI) (string, error) { +func resolveClusterEndpoint(ctx context.Context, eksAPI eksiface.EKSAPI) (string, error) { clusterEndpointFromSettings := settings.FromContext(ctx).ClusterEndpoint if clusterEndpointFromSettings != "" { return clusterEndpointFromSettings, nil // cluster endpoint is explicitly set diff --git a/pkg/cloudprovider/cloudprovider_test.go b/pkg/context/suite_test.go similarity index 50% rename from pkg/cloudprovider/cloudprovider_test.go rename to pkg/context/suite_test.go index 371f1c2d51e2..72e01cc97eca 100644 --- a/pkg/cloudprovider/cloudprovider_test.go +++ b/pkg/context/suite_test.go @@ -12,38 +12,77 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cloudprovider +package context import ( + "context" "errors" + "testing" - . "github.com/onsi/ginkgo/v2" + ginkgov2 "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" "github.com/samber/lo" + . "knative.dev/pkg/logging/testing" + + . "github.com/aws/karpenter-core/pkg/test/expectations" "github.com/aws/aws-sdk-go/service/eks" + "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" - "github.com/aws/karpenter/pkg/context" + "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/test" - . "github.com/onsi/gomega" + coresettings "github.com/aws/karpenter-core/pkg/apis/settings" + "github.com/aws/karpenter-core/pkg/operator/scheme" + coretest "github.com/aws/karpenter-core/pkg/test" ) -var _ = Describe("Cloud Provider", func() { - BeforeEach(func() { - fakeEKSAPI.Reset() - }) +var ctx context.Context +var stop context.CancelFunc +var env *coretest.Environment +var fakeEKSAPI *fake.EKSAPI + +func TestAWS(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(ginkgov2.Fail) + ginkgov2.RunSpecs(t, "CloudProvider/AWS") +} + +var _ = ginkgov2.BeforeSuite(func() { + env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = settings.ToContext(ctx, test.Settings()) + ctx, stop = context.WithCancel(ctx) + + fakeEKSAPI = &fake.EKSAPI{} +}) + +var _ = ginkgov2.AfterSuite(func() { + stop() + Expect(env.Stop()).To(Succeed(), "Failed to stop environment") +}) + +var _ = ginkgov2.BeforeEach(func() { + fakeEKSAPI.Reset() +}) + +var _ = ginkgov2.AfterEach(func() { + ExpectCleanedUp(ctx, env.Client) +}) + +var _ = ginkgov2.Describe("Context", func() { - It("should resolve endpoint if set via configuration", func() { + ginkgov2.It("should resolve endpoint if set via configuration", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ ClusterEndpoint: lo.ToPtr("https://api.test-cluster.k8s.local"), })) - endpoint, err := context.ResolveClusterEndpoint(ctx, fakeEKSAPI) + endpoint, err := resolveClusterEndpoint(ctx, fakeEKSAPI) Expect(err).ToNot(HaveOccurred()) Expect(endpoint).To(Equal("https://api.test-cluster.k8s.local")) }) - It("should resolve endpoint if not set, via call to API", func() { + ginkgov2.It("should resolve endpoint if not set, via call to API", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ ClusterEndpoint: lo.ToPtr(""), })) @@ -55,18 +94,18 @@ var _ = Describe("Cloud Provider", func() { }, ) - endpoint, err := context.ResolveClusterEndpoint(ctx, fakeEKSAPI) + endpoint, err := resolveClusterEndpoint(ctx, fakeEKSAPI) Expect(err).ToNot(HaveOccurred()) Expect(endpoint).To(Equal("https://cluster-endpoint.test-cluster.k8s.local")) }) - It("should propagate error if API fails", func() { + ginkgov2.It("should propagate error if API fails", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ ClusterEndpoint: lo.ToPtr(""), })) fakeEKSAPI.DescribeClusterBehaviour.Error.Set(errors.New("test error")) - _, err := context.ResolveClusterEndpoint(ctx, fakeEKSAPI) + _, err := resolveClusterEndpoint(ctx, fakeEKSAPI) Expect(err).To(HaveOccurred()) }) }) From 92a242e88ee5d87452f3c83fe16332b9d25b84f2 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Mon, 27 Feb 2023 10:48:41 -0800 Subject: [PATCH 08/41] Moved instancetypes to Provider --- pkg/cloudprovider/cloudprovider.go | 13 ++++++- pkg/cloudprovider/instance.go | 5 ++- pkg/cloudprovider/suite_test.go | 37 ++++++++++--------- .../instancetypes}/instancetype.go | 2 +- .../instancetypes}/instancetypes.go | 22 +++++------ .../instancetypes}/instancetypes_test.go | 2 +- .../instancetypes}/zz_generated.bandwidth.go | 2 +- .../instancetypes}/zz_generated.vpclimits.go | 2 +- .../launchtemplate/launchtemplate_test.go | 26 ++++++++++--- 9 files changed, 68 insertions(+), 43 deletions(-) rename pkg/{cloudprovider => providers/instancetypes}/instancetype.go (99%) rename pkg/{cloudprovider => providers/instancetypes}/instancetypes.go (89%) rename pkg/{cloudprovider => providers/instancetypes}/instancetypes_test.go (99%) rename pkg/{cloudprovider => providers/instancetypes}/zz_generated.bandwidth.go (99%) rename pkg/{cloudprovider => providers/instancetypes}/zz_generated.vpclimits.go (99%) diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index 50ec9d2900cd..8953bd869b41 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -20,6 +20,7 @@ import ( "net/http" "strings" + awscache "github.com/aws/karpenter/pkg/cache" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime/schema" @@ -28,7 +29,9 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" + "github.com/aws/karpenter/pkg/providers/instancetypes" "github.com/aws/karpenter/pkg/utils" + "github.com/patrickmn/go-cache" "github.com/aws/karpenter-core/pkg/scheduling" "github.com/aws/karpenter-core/pkg/utils/resources" @@ -63,14 +66,20 @@ func init() { var _ cloudprovider.CloudProvider = (*CloudProvider)(nil) type CloudProvider struct { - instanceTypeProvider *InstanceTypeProvider + instanceTypeProvider *instancetypes.Provider instanceProvider *InstanceProvider kubeClient client.Client amiProvider *amifamily.AMIProvider } func New(ctx awscontext.Context) *CloudProvider { - instanceTypeProvider := NewInstanceTypeProvider(ctx.Session, ctx.EC2API, ctx.SubnetProvider, ctx.UnavailableOfferingsCache, ctx.PricingProvider) + instanceTypeProvider := instancetypes.NewProvider( + ctx.Session, + cache.New(awscache.InstanceTypesAndZonesTTL, awscache.DefaultCleanupInterval), + ctx.EC2API, + ctx.SubnetProvider, + ctx.UnavailableOfferingsCache, + ctx.PricingProvider) return &CloudProvider{ kubeClient: ctx.KubeClient, instanceTypeProvider: instanceTypeProvider, diff --git a/pkg/cloudprovider/instance.go b/pkg/cloudprovider/instance.go index 65d753607246..a0ec5c0bf43e 100644 --- a/pkg/cloudprovider/instance.go +++ b/pkg/cloudprovider/instance.go @@ -38,6 +38,7 @@ import ( "github.com/aws/karpenter/pkg/batcher" "github.com/aws/karpenter/pkg/cache" awserrors "github.com/aws/karpenter/pkg/errors" + "github.com/aws/karpenter/pkg/providers/instancetypes" "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/subnet" "github.com/aws/karpenter/pkg/utils" @@ -62,13 +63,13 @@ type InstanceProvider struct { region string ec2api ec2iface.EC2API unavailableOfferings *cache.UnavailableOfferings - instanceTypeProvider *InstanceTypeProvider + instanceTypeProvider *instancetypes.Provider subnetProvider *subnet.Provider launchTemplateProvider *launchtemplate.Provider ec2Batcher *batcher.EC2API } -func NewInstanceProvider(ctx context.Context, region string, ec2api ec2iface.EC2API, unavailableOfferings *cache.UnavailableOfferings, instanceTypeProvider *InstanceTypeProvider, subnetProvider *subnet.Provider, launchTemplateProvider *launchtemplate.Provider) *InstanceProvider { +func NewInstanceProvider(ctx context.Context, region string, ec2api ec2iface.EC2API, unavailableOfferings *cache.UnavailableOfferings, instanceTypeProvider *instancetypes.Provider, subnetProvider *subnet.Provider, launchTemplateProvider *launchtemplate.Provider) *InstanceProvider { return &InstanceProvider{ region: region, ec2api: ec2api, diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index 80c50f13b9ac..a0d969c7e34a 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -31,6 +31,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/awstesting/mock" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" clock "k8s.io/utils/clock/testing" @@ -65,8 +66,8 @@ import ( "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter-core/pkg/utils/pretty" + "github.com/aws/karpenter/pkg/providers/instancetypes" "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/providers/securitygroup" @@ -83,7 +84,7 @@ var ec2Cache *cache.Cache var kubernetesVersionCache *cache.Cache var unavailableOfferingsCache *awscache.UnavailableOfferings var instanceTypeCache *cache.Cache -var instanceTypeProvider *InstanceTypeProvider +var instanceTypeProvider *instancetypes.Provider var launchTemplateProvider *launchtemplate.Provider var amiProvider *amifamily.AMIProvider var fakeEC2API *fake.EC2API @@ -126,14 +127,14 @@ var _ = BeforeSuite(func() { pricingProvider = pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})) amiProvider = amifamily.NewAMIProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) subnetProvider = subnet.NewProvider(fakeEC2API) - instanceTypeProvider = &InstanceTypeProvider{ - ec2api: fakeEC2API, - subnetProvider: subnetProvider, - cache: instanceTypeCache, - pricingProvider: pricingProvider, - unavailableOfferings: unavailableOfferingsCache, - cm: pretty.NewChangeMonitor(), - } + instanceTypeProvider = instancetypes.NewProvider( + mock.Session, + instanceTypeCache, + fakeEC2API, + subnetProvider, + unavailableOfferingsCache, + pricingProvider, + ) securityGroupProvider = securitygroup.NewProvider(fakeEC2API) launchTemplateProvider = launchtemplate.NewProvider( ctx, @@ -213,14 +214,14 @@ var _ = BeforeEach(func() { launchTemplateProvider.ClusterEndpoint = "https://test-cluster" // Reset the pricing provider, so we don't cross-pollinate pricing data - instanceTypeProvider = &InstanceTypeProvider{ - ec2api: fakeEC2API, - subnetProvider: subnetProvider, - cache: instanceTypeCache, - pricingProvider: pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})), - unavailableOfferings: unavailableOfferingsCache, - cm: pretty.NewChangeMonitor(), - } + instanceTypeProvider = instancetypes.NewProvider( + mock.Session, + instanceTypeCache, + fakeEC2API, + subnetProvider, + unavailableOfferingsCache, + pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})), + ) }) var _ = AfterEach(func() { diff --git a/pkg/cloudprovider/instancetype.go b/pkg/providers/instancetypes/instancetype.go similarity index 99% rename from pkg/cloudprovider/instancetype.go rename to pkg/providers/instancetypes/instancetype.go index 771e1a14fe3b..580846d85959 100644 --- a/pkg/cloudprovider/instancetype.go +++ b/pkg/providers/instancetypes/instancetype.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cloudprovider +package instancetypes import ( "context" diff --git a/pkg/cloudprovider/instancetypes.go b/pkg/providers/instancetypes/instancetypes.go similarity index 89% rename from pkg/cloudprovider/instancetypes.go rename to pkg/providers/instancetypes/instancetypes.go index 75a1bc10bceb..590364ff70c2 100644 --- a/pkg/cloudprovider/instancetypes.go +++ b/pkg/providers/instancetypes/instancetypes.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cloudprovider +package instancetypes import ( "context" @@ -48,7 +48,7 @@ const ( InstanceTypeZonesCacheKeyPrefix = "zones:" ) -type InstanceTypeProvider struct { +type Provider struct { region string ec2api ec2iface.EC2API subnetProvider *subnet.Provider @@ -68,21 +68,21 @@ type InstanceTypeProvider struct { instanceTypesSeqNum uint64 } -func NewInstanceTypeProvider(sess *session.Session, ec2api ec2iface.EC2API, subnetProvider *subnet.Provider, - unavailableOfferingsCache *awscache.UnavailableOfferings, pricingProvider *pricing.Provider) *InstanceTypeProvider { - return &InstanceTypeProvider{ +func NewProvider(sess *session.Session, cache *cache.Cache, ec2api ec2iface.EC2API, subnetProvider *subnet.Provider, + unavailableOfferingsCache *awscache.UnavailableOfferings, pricingProvider *pricing.Provider) *Provider { + return &Provider{ ec2api: ec2api, region: *sess.Config.Region, subnetProvider: subnetProvider, pricingProvider: pricingProvider, - cache: cache.New(awscache.InstanceTypesAndZonesTTL, awscache.DefaultCleanupInterval), + cache: cache, unavailableOfferings: unavailableOfferingsCache, cm: pretty.NewChangeMonitor(), instanceTypesSeqNum: 0, } } -func (p *InstanceTypeProvider) List(ctx context.Context, kc *v1alpha5.KubeletConfiguration, nodeTemplate *v1alpha1.AWSNodeTemplate) ([]*cloudprovider.InstanceType, error) { +func (p *Provider) List(ctx context.Context, kc *v1alpha5.KubeletConfiguration, nodeTemplate *v1alpha1.AWSNodeTemplate) ([]*cloudprovider.InstanceType, error) { // Get InstanceTypes from EC2 instanceTypes, err := p.getInstanceTypes(ctx) if err != nil { @@ -109,7 +109,7 @@ func (p *InstanceTypeProvider) List(ctx context.Context, kc *v1alpha5.KubeletCon return result, nil } -func (p *InstanceTypeProvider) LivenessProbe(req *http.Request) error { +func (p *Provider) LivenessProbe(req *http.Request) error { if err := p.subnetProvider.LivenessProbe(req); err != nil { return err } @@ -119,7 +119,7 @@ func (p *InstanceTypeProvider) LivenessProbe(req *http.Request) error { return nil } -func (p *InstanceTypeProvider) createOfferings(ctx context.Context, instanceType *ec2.InstanceTypeInfo, zones sets.String) []cloudprovider.Offering { +func (p *Provider) createOfferings(ctx context.Context, instanceType *ec2.InstanceTypeInfo, zones sets.String) []cloudprovider.Offering { var offerings []cloudprovider.Offering for zone := range zones { // while usage classes should be a distinct set, there's no guarantee of that @@ -149,7 +149,7 @@ func (p *InstanceTypeProvider) createOfferings(ctx context.Context, instanceType return offerings } -func (p *InstanceTypeProvider) getInstanceTypeZones(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) (map[string]sets.String, error) { +func (p *Provider) getInstanceTypeZones(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate) (map[string]sets.String, error) { subnetSelectorHash, err := hashstructure.Hash(nodeTemplate.Spec.SubnetSelector, hashstructure.FormatV2, nil) if err != nil { return nil, fmt.Errorf("failed to hash the subnet selector: %w", err) @@ -195,7 +195,7 @@ func (p *InstanceTypeProvider) getInstanceTypeZones(ctx context.Context, nodeTem } // getInstanceTypes retrieves all instance types from the ec2 DescribeInstanceTypes API using some opinionated filters -func (p *InstanceTypeProvider) getInstanceTypes(ctx context.Context) ([]*ec2.InstanceTypeInfo, error) { +func (p *Provider) getInstanceTypes(ctx context.Context) ([]*ec2.InstanceTypeInfo, error) { // DO NOT REMOVE THIS LOCK ---------------------------------------------------------------------------- // We lock here so that multiple callers to GetInstanceTypes do not result in cache misses and multiple // calls to EC2 when we could have just made one call. This lock is here because multiple callers to EC2 result diff --git a/pkg/cloudprovider/instancetypes_test.go b/pkg/providers/instancetypes/instancetypes_test.go similarity index 99% rename from pkg/cloudprovider/instancetypes_test.go rename to pkg/providers/instancetypes/instancetypes_test.go index 1053e21b7ccb..a509500993f8 100644 --- a/pkg/cloudprovider/instancetypes_test.go +++ b/pkg/providers/instancetypes/instancetypes_test.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cloudprovider +package instancetypes_test import ( "context" diff --git a/pkg/cloudprovider/zz_generated.bandwidth.go b/pkg/providers/instancetypes/zz_generated.bandwidth.go similarity index 99% rename from pkg/cloudprovider/zz_generated.bandwidth.go rename to pkg/providers/instancetypes/zz_generated.bandwidth.go index 2045fe4bc18b..2a5f3c044e01 100644 --- a/pkg/cloudprovider/zz_generated.bandwidth.go +++ b/pkg/providers/instancetypes/zz_generated.bandwidth.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cloudprovider +package instancetypes // GENERATED FILE. DO NOT EDIT DIRECTLY. // Update hack/code/bandwidth_gen.go and re-generate to edit diff --git a/pkg/cloudprovider/zz_generated.vpclimits.go b/pkg/providers/instancetypes/zz_generated.vpclimits.go similarity index 99% rename from pkg/cloudprovider/zz_generated.vpclimits.go rename to pkg/providers/instancetypes/zz_generated.vpclimits.go index 0b866eebf848..198f3e9bc24b 100644 --- a/pkg/cloudprovider/zz_generated.vpclimits.go +++ b/pkg/providers/instancetypes/zz_generated.vpclimits.go @@ -19,7 +19,7 @@ // Code generated by go generate; DO NOT EDIT. // This file was generated at 2023-01-26T19:39:15Z -package cloudprovider +package instancetypes type VPCLimits struct { Interface int diff --git a/pkg/providers/launchtemplate/launchtemplate_test.go b/pkg/providers/launchtemplate/launchtemplate_test.go index e219e9404954..60c39e3672bc 100644 --- a/pkg/providers/launchtemplate/launchtemplate_test.go +++ b/pkg/providers/launchtemplate/launchtemplate_test.go @@ -53,6 +53,7 @@ import ( "github.com/aws/karpenter/pkg/cloudprovider/amifamily" "github.com/aws/karpenter/pkg/cloudprovider/amifamily/bootstrap" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/providers/instancetypes" "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/providers/securitygroup" @@ -81,6 +82,7 @@ var env *coretest.Environment var ssmCache *cache.Cache var ec2Cache *cache.Cache var launchTemplateCache *cache.Cache +var instanceTypeCache *cache.Cache var kubernetesVersionCache *cache.Cache var fakeEC2API *fake.EC2API var fakeSSMAPI *fake.SSMAPI @@ -97,7 +99,7 @@ var nodeTemplate *v1alpha1.AWSNodeTemplate var cluster *state.Cluster var pricingProvider *pricing.Provider var subnetProvider *subnet.Provider -var instanceTypeProvider *cloudprovider.InstanceTypeProvider +var instanceTypeProvider *instancetypes.Provider var securityGroupProvider *securitygroup.Provider func TestAWS(t *testing.T) { @@ -117,6 +119,7 @@ var _ = BeforeSuite(func() { ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) fakeClock = clock.NewFakeClock(time.Now()) unavailableOfferingsCache = awscache.NewUnavailableOfferings() @@ -126,7 +129,7 @@ var _ = BeforeSuite(func() { securityGroupProvider = securitygroup.NewProvider(fakeEC2API) amiProvider = amifamily.NewAMIProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) amiResolver = amifamily.New(env.Client, amiProvider) - instanceTypeProvider = cloudprovider.NewInstanceTypeProvider(mock.Session, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) + instanceTypeProvider = instancetypes.NewProvider(mock.Session, instanceTypeCache, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) launchTemplateProvider = launchtemplate.NewProvider( ctx, @@ -210,10 +213,21 @@ var _ = BeforeEach(func() { unavailableOfferingsCache.Flush() ssmCache.Flush() ec2Cache.Flush() + instanceTypeCache.Flush() kubernetesVersionCache.Flush() securityGroupProvider.Reset() launchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") launchTemplateProvider.ClusterEndpoint = "https://test-cluster" + + // Reset the pricing provider, so we don't cross-pollinate pricing data + instanceTypeProvider = instancetypes.NewProvider( + mock.Session, + instanceTypeCache, + fakeEC2API, + subnetProvider, + unavailableOfferingsCache, + pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})), + ) }) var _ = AfterEach(func() { @@ -878,7 +892,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 - it := cloudprovider.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1093Mi")) }) @@ -889,7 +903,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 - it := cloudprovider.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1093Mi")) }) @@ -928,7 +942,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - it := cloudprovider.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1093Mi")) }) @@ -939,7 +953,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - it := cloudprovider.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1665Mi")) }) From 97f5242e7b326223291ecc4d90a66ab7b450c55c Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Mon, 27 Feb 2023 12:15:59 -0800 Subject: [PATCH 09/41] Fix tests --- pkg/cloudprovider/cloudprovider.go | 13 +- pkg/context/context.go | 15 +- pkg/providers/instancetypes/instancetype.go | 10 +- pkg/providers/instancetypes/instancetypes.go | 4 +- .../{instancetypes_test.go => suite_test.go} | 297 ++++++++++++++---- .../launchtemplate/launchtemplate_test.go | 6 +- pkg/providers/pricing/suite_test.go | 2 +- pkg/providers/securitygroup/suite_test.go | 2 +- pkg/providers/subnet/suite_test.go | 2 +- 9 files changed, 269 insertions(+), 82 deletions(-) rename pkg/providers/instancetypes/{instancetypes_test.go => suite_test.go} (81%) diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index 8953bd869b41..c6d1b52c1a1b 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -20,7 +20,6 @@ import ( "net/http" "strings" - awscache "github.com/aws/karpenter/pkg/cache" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime/schema" @@ -31,7 +30,6 @@ import ( "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/providers/instancetypes" "github.com/aws/karpenter/pkg/utils" - "github.com/patrickmn/go-cache" "github.com/aws/karpenter-core/pkg/scheduling" "github.com/aws/karpenter-core/pkg/utils/resources" @@ -73,23 +71,16 @@ type CloudProvider struct { } func New(ctx awscontext.Context) *CloudProvider { - instanceTypeProvider := instancetypes.NewProvider( - ctx.Session, - cache.New(awscache.InstanceTypesAndZonesTTL, awscache.DefaultCleanupInterval), - ctx.EC2API, - ctx.SubnetProvider, - ctx.UnavailableOfferingsCache, - ctx.PricingProvider) return &CloudProvider{ kubeClient: ctx.KubeClient, - instanceTypeProvider: instanceTypeProvider, + instanceTypeProvider: ctx.InstanceTypeProvider, amiProvider: ctx.AMIProvider, instanceProvider: NewInstanceProvider( ctx, aws.StringValue(ctx.Session.Config.Region), ctx.EC2API, ctx.UnavailableOfferingsCache, - instanceTypeProvider, + ctx.InstanceTypeProvider, ctx.SubnetProvider, ctx.LaunchTemplateProvider, ), diff --git a/pkg/context/context.go b/pkg/context/context.go index cb955d100211..d2e55fe0673a 100644 --- a/pkg/context/context.go +++ b/pkg/context/context.go @@ -45,6 +45,7 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/cloudprovider/amifamily" + "github.com/aws/karpenter/pkg/providers/instancetypes" "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/providers/securitygroup" @@ -67,6 +68,7 @@ type Context struct { AMIResolver *amifamily.Resolver LaunchTemplateProvider *launchtemplate.Provider PricingProvider *pricing.Provider + InstanceTypeProvider *instancetypes.Provider } func NewOrDie(ctx cloudprovider.Context) Context { @@ -103,6 +105,7 @@ func NewOrDie(ctx cloudprovider.Context) Context { logging.FromContext(ctx).With("kube-dns-ip", kubeDNSIP).Debugf("discovered kube dns") } + unavailableOfferingsCache := awscache.NewUnavailableOfferings() subnetProvider := subnet.NewProvider(ec2api) securityGroupProvider := securitygroup.NewProvider(ec2api) pricingProvider := pricing.NewProvider( @@ -112,6 +115,7 @@ func NewOrDie(ctx cloudprovider.Context) Context { *sess.Config.Region, ctx.StartAsync, ) + amiProvider := amifamily.NewAMIProvider(ctx.KubeClient, ctx.KubernetesInterface, ssm.New(sess), ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) amiResolver := amifamily.New(ctx.KubeClient, amiProvider) @@ -126,11 +130,19 @@ func NewOrDie(ctx cloudprovider.Context) Context { kubeDNSIP, clusterEndpoint, ) + instanceTypeProvider := instancetypes.NewProvider( + sess, + cache.New(awscache.InstanceTypesAndZonesTTL, awscache.DefaultCleanupInterval), + ec2api, + subnetProvider, + unavailableOfferingsCache, + pricingProvider, + ) return Context{ Context: ctx, Session: sess, - UnavailableOfferingsCache: awscache.NewUnavailableOfferings(), + UnavailableOfferingsCache: unavailableOfferingsCache, EC2API: ec2api, SubnetProvider: subnetProvider, SecurityGroupProvider: securityGroupProvider, @@ -138,6 +150,7 @@ func NewOrDie(ctx cloudprovider.Context) Context { AMIResolver: amiResolver, LaunchTemplateProvider: launchTemplateProvider, PricingProvider: pricingProvider, + InstanceTypeProvider: instanceTypeProvider, } } diff --git a/pkg/providers/instancetypes/instancetype.go b/pkg/providers/instancetypes/instancetype.go index 580846d85959..e465d47b953a 100644 --- a/pkg/providers/instancetypes/instancetype.go +++ b/pkg/providers/instancetypes/instancetype.go @@ -40,7 +40,7 @@ import ( ) const ( - memoryAvailable = "memory.available" + MemoryAvailable = "memory.available" ) var ( @@ -57,7 +57,7 @@ func NewInstanceType(ctx context.Context, info *ec2.InstanceTypeInfo, kc *v1alph Offerings: offerings, Capacity: computeCapacity(ctx, info, amiFamily, nodeTemplate.Spec.BlockDeviceMappings, kc), Overhead: &cloudprovider.InstanceTypeOverhead{ - KubeReserved: kubeReservedResources(cpu(info), pods(ctx, info, amiFamily, kc), eniLimitedPods(info), amiFamily, kc), + KubeReserved: kubeReservedResources(cpu(info), pods(ctx, info, amiFamily, kc), EniLimitedPods(info), amiFamily, kc), SystemReserved: systemReservedResources(kc), EvictionThreshold: evictionThreshold(memory(ctx, info), amiFamily, kc), }, @@ -234,7 +234,7 @@ func habanaGaudis(info *ec2.InstanceTypeInfo) *resource.Quantity { // The number of pods per node is calculated using the formula: // max number of ENIs * (IPv4 Addresses per ENI -1) + 2 // https://github.com/awslabs/amazon-eks-ami/blob/master/files/eni-max-pods.txt#L20 -func eniLimitedPods(info *ec2.InstanceTypeInfo) *resource.Quantity { +func EniLimitedPods(info *ec2.InstanceTypeInfo) *resource.Quantity { return resources.Quantity(fmt.Sprint(*info.NetworkInfo.MaximumNetworkInterfaces*(*info.NetworkInfo.Ipv4AddressesPerInterface-1) + 2)) } @@ -306,7 +306,7 @@ func evictionThreshold(memory *resource.Quantity, amiFamily amifamily.AMIFamily, for _, m := range evictionSignals { temp := v1.ResourceList{} - if v, ok := m[memoryAvailable]; ok { + if v, ok := m[MemoryAvailable]; ok { if strings.HasSuffix(v, "%") { p := mustParsePercentage(v) @@ -331,7 +331,7 @@ func pods(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily amifamily.A case !awssettings.FromContext(ctx).EnableENILimitedPodDensity: count = 110 default: - count = eniLimitedPods(info).Value() + count = EniLimitedPods(info).Value() } if kc != nil && ptr.Int32Value(kc.PodsPerCore) > 0 && amiFamily.FeatureFlags().PodsPerCoreEnabled { count = lo.Min([]int64{int64(ptr.Int32Value(kc.PodsPerCore)) * ptr.Int64Value(info.VCpuInfo.DefaultVCpus), count}) diff --git a/pkg/providers/instancetypes/instancetypes.go b/pkg/providers/instancetypes/instancetypes.go index 590364ff70c2..b54b927fdf27 100644 --- a/pkg/providers/instancetypes/instancetypes.go +++ b/pkg/providers/instancetypes/instancetypes.go @@ -84,7 +84,7 @@ func NewProvider(sess *session.Session, cache *cache.Cache, ec2api ec2iface.EC2A func (p *Provider) List(ctx context.Context, kc *v1alpha5.KubeletConfiguration, nodeTemplate *v1alpha1.AWSNodeTemplate) ([]*cloudprovider.InstanceType, error) { // Get InstanceTypes from EC2 - instanceTypes, err := p.getInstanceTypes(ctx) + instanceTypes, err := p.GetInstanceTypes(ctx) if err != nil { return nil, err } @@ -195,7 +195,7 @@ func (p *Provider) getInstanceTypeZones(ctx context.Context, nodeTemplate *v1alp } // getInstanceTypes retrieves all instance types from the ec2 DescribeInstanceTypes API using some opinionated filters -func (p *Provider) getInstanceTypes(ctx context.Context) ([]*ec2.InstanceTypeInfo, error) { +func (p *Provider) GetInstanceTypes(ctx context.Context) ([]*ec2.InstanceTypeInfo, error) { // DO NOT REMOVE THIS LOCK ---------------------------------------------------------------------------- // We lock here so that multiple callers to GetInstanceTypes do not result in cache misses and multiple // calls to EC2 when we could have just made one call. This lock is here because multiple callers to EC2 result diff --git a/pkg/providers/instancetypes/instancetypes_test.go b/pkg/providers/instancetypes/suite_test.go similarity index 81% rename from pkg/providers/instancetypes/instancetypes_test.go rename to pkg/providers/instancetypes/suite_test.go index a509500993f8..563bb682538b 100644 --- a/pkg/providers/instancetypes/instancetypes_test.go +++ b/pkg/providers/instancetypes/suite_test.go @@ -18,33 +18,216 @@ import ( "context" "fmt" "math" + "net" "sort" "strings" + "testing" "time" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/awstesting/mock" "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/patrickmn/go-cache" "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" + clock "k8s.io/utils/clock/testing" + . "knative.dev/pkg/logging/testing" "knative.dev/pkg/ptr" - . "github.com/aws/karpenter-core/pkg/test/expectations" - "github.com/aws/karpenter/pkg/apis/settings" - "github.com/aws/karpenter/pkg/providers/pricing" - "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - "github.com/aws/karpenter-core/pkg/cloudprovider" + corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" + "github.com/aws/karpenter-core/pkg/controllers/provisioning" + "github.com/aws/karpenter-core/pkg/controllers/state" + "github.com/aws/karpenter-core/pkg/events" + "github.com/aws/karpenter-core/pkg/operator/injection" + "github.com/aws/karpenter-core/pkg/operator/options" + "github.com/aws/karpenter-core/pkg/operator/scheme" "github.com/aws/karpenter-core/pkg/scheduling" + . "github.com/aws/karpenter-core/pkg/test/expectations" + + coresettings "github.com/aws/karpenter-core/pkg/apis/settings" coretest "github.com/aws/karpenter-core/pkg/test" + "github.com/aws/karpenter/pkg/apis" + "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" + awscache "github.com/aws/karpenter/pkg/cache" + "github.com/aws/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter/pkg/cloudprovider/amifamily" + awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/providers/instancetypes" + "github.com/aws/karpenter/pkg/providers/launchtemplate" + "github.com/aws/karpenter/pkg/providers/pricing" + "github.com/aws/karpenter/pkg/providers/securitygroup" + "github.com/aws/karpenter/pkg/providers/subnet" "github.com/aws/karpenter/pkg/test" ) +var ctx context.Context +var stop context.CancelFunc +var opts options.Options +var env *coretest.Environment +var ssmCache *cache.Cache +var ec2Cache *cache.Cache +var launchTemplateCache *cache.Cache +var instanceTypeCache *cache.Cache +var kubernetesVersionCache *cache.Cache +var fakeEC2API *fake.EC2API +var fakeSSMAPI *fake.SSMAPI +var fakeClock *clock.FakeClock +var fakePricingAPI *fake.PricingAPI +var amiProvider *amifamily.AMIProvider +var amiResolver *amifamily.Resolver +var cloudProvider *cloudprovider.CloudProvider +var unavailableOfferingsCache *awscache.UnavailableOfferings +var prov *provisioning.Provisioner +var provisioner *v1alpha5.Provisioner +var launchTemplateProvider *launchtemplate.Provider +var nodeTemplate *v1alpha1.AWSNodeTemplate +var cluster *state.Cluster +var pricingProvider *pricing.Provider +var subnetProvider *subnet.Provider +var instanceTypeProvider *instancetypes.Provider +var securityGroupProvider *securitygroup.Provider + +func TestAWS(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "Provider/AWS") +} + +var _ = BeforeSuite(func() { + env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = settings.ToContext(ctx, test.Settings()) + ctx, stop = context.WithCancel(ctx) + + fakeEC2API = &fake.EC2API{} + fakeSSMAPI = &fake.SSMAPI{} + ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + fakeClock = clock.NewFakeClock(time.Now()) + unavailableOfferingsCache = awscache.NewUnavailableOfferings() + fakePricingAPI = &fake.PricingAPI{} + pricingProvider = pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})) + subnetProvider = subnet.NewProvider(fakeEC2API) + securityGroupProvider = securitygroup.NewProvider(fakeEC2API) + amiProvider = amifamily.NewAMIProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) + amiResolver = amifamily.New(env.Client, amiProvider) + instanceTypeProvider = instancetypes.NewProvider(mock.Session, instanceTypeCache, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) + + launchTemplateProvider = launchtemplate.NewProvider( + ctx, + launchTemplateCache, + fakeEC2API, + amiResolver, + securityGroupProvider, + ptr.String("ca-bundle"), + make(chan struct{}), + net.ParseIP("10.0.100.10"), + "https://test-cluster", + ) + + cloudProvider = cloudprovider.New(awscontext.Context{ + Context: corecloudprovider.Context{ + Context: ctx, + RESTConfig: env.Config, + KubernetesInterface: env.KubernetesInterface, + KubeClient: env.Client, + EventRecorder: events.NewRecorder(&record.FakeRecorder{}), + Clock: &clock.FakeClock{}, + StartAsync: nil, + }, + SubnetProvider: subnet.NewProvider(fakeEC2API), + SecurityGroupProvider: securityGroupProvider, + Session: mock.Session, + UnavailableOfferingsCache: unavailableOfferingsCache, + EC2API: fakeEC2API, + PricingProvider: pricingProvider, + AMIProvider: amiProvider, + AMIResolver: amiResolver, + LaunchTemplateProvider: launchTemplateProvider, + InstanceTypeProvider: instanceTypeProvider, + }) + cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) + prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) + +}) + +var _ = AfterSuite(func() { + stop() + Expect(env.Stop()).To(Succeed(), "Failed to stop environment") +}) + +var _ = BeforeEach(func() { + ctx = injection.WithOptions(ctx, opts) + ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = settings.ToContext(ctx, test.Settings()) + nodeTemplate = &v1alpha1.AWSNodeTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: coretest.RandomName(), + }, + Spec: v1alpha1.AWSNodeTemplateSpec{ + AWS: v1alpha1.AWS{ + AMIFamily: aws.String(v1alpha1.AMIFamilyAL2), + SubnetSelector: map[string]string{"*": "*"}, + SecurityGroupSelector: map[string]string{"*": "*"}, + }, + }, + } + nodeTemplate.SetGroupVersionKind(schema.GroupVersionKind{ + Group: v1alpha1.SchemeGroupVersion.Group, + Version: v1alpha1.SchemeGroupVersion.Version, + Kind: "AWSNodeTemplate", + }) + provisioner = test.Provisioner(coretest.ProvisionerOptions{ + Requirements: []v1.NodeSelectorRequirement{{ + Key: v1alpha1.LabelInstanceCategory, + Operator: v1.NodeSelectorOpExists, + }}, + ProviderRef: &v1alpha5.ProviderRef{ + APIVersion: nodeTemplate.APIVersion, + Kind: nodeTemplate.Kind, + Name: nodeTemplate.Name, + }, + }) + + cluster.Reset() + fakeEC2API.Reset() + fakeSSMAPI.Reset() + fakePricingAPI.Reset() + launchTemplateCache.Flush() + unavailableOfferingsCache.Flush() + ssmCache.Flush() + ec2Cache.Flush() + kubernetesVersionCache.Flush() + instanceTypeCache.Flush() + subnetProvider.Reset() + securityGroupProvider.Reset() + launchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") + launchTemplateProvider.ClusterEndpoint = "https://test-cluster" + + // Reset the pricing provider, so we don't cross-pollinate pricing data + instanceTypeProvider = instancetypes.NewProvider( + mock.Session, + instanceTypeCache, + fakeEC2API, + subnetProvider, + unavailableOfferingsCache, + pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})), + ) +}) + var _ = Describe("Instance Types", func() { It("should support instance type labels", func() { ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) @@ -134,14 +317,14 @@ var _ = Describe("Instance Types", func() { return iPrice < jPrice }) // Expect that the launch template overrides gives the 60 cheapest instance types - expected := sets.NewString(lo.Map(its[:MaxInstanceTypes], func(i *cloudprovider.InstanceType, _ int) string { + expected := sets.NewString(lo.Map(its[:cloudprovider.MaxInstanceTypes], func(i *corecloudprovider.InstanceType, _ int) string { return i.Name })...) Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) call := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(call.LaunchTemplateConfigs).To(HaveLen(1)) - Expect(call.LaunchTemplateConfigs[0].Overrides).To(HaveLen(MaxInstanceTypes)) + Expect(call.LaunchTemplateConfigs[0].Overrides).To(HaveLen(cloudprovider.MaxInstanceTypes)) for _, override := range call.LaunchTemplateConfigs[0].Overrides { Expect(expected.Has(aws.StringValue(override.InstanceType))).To(BeTrue(), fmt.Sprintf("expected %s to exist in set", aws.StringValue(override.InstanceType))) } @@ -295,7 +478,7 @@ var _ = Describe("Instance Types", func() { node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKey(v1.LabelInstanceTypeStable)) supportsPodENI := func() bool { - limits, ok := Limits[node.Labels[v1.LabelInstanceTypeStable]] + limits, ok := instancetypes.Limits[node.Labels[v1.LabelInstanceTypeStable]] return ok && limits.IsTrunkingCompatible } Expect(supportsPodENI()).To(Equal(true)) @@ -401,18 +584,18 @@ var _ = Describe("Instance Types", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), })) - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) for _, info := range instanceInfo { - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 110)) } }) It("should not set pods to 110 if using ENI-based pod density", func() { - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) for _, info := range instanceInfo { - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).ToNot(BeNumerically("==", 110)) } }) @@ -425,7 +608,7 @@ var _ = Describe("Instance Types", func() { })) var ok bool - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) info, ok = lo.Find(instanceInfo, func(i *ec2.InstanceTypeInfo) bool { return aws.StringValue(i.InstanceType) == "m5.xlarge" @@ -441,7 +624,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.SystemReserved.Cpu().String()).To(Equal("2")) }) It("should override system reserved memory when specified", func() { @@ -452,7 +635,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.SystemReserved.Memory().String()).To(Equal("20Gi")) }) It("should override kube reserved when specified", func() { @@ -470,7 +653,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.KubeReserved.Cpu().String()).To(Equal("2")) Expect(it.Overhead.KubeReserved.Memory().String()).To(Equal("10Gi")) Expect(it.Overhead.KubeReserved.StorageEphemeral().String()).To(Equal("2Gi")) @@ -492,11 +675,11 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionHard: map[string]string{ - memoryAvailable: "500Mi", + instancetypes.MemoryAvailable: "500Mi", }, }, }) - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("500Mi")) }) It("should override eviction threshold (hard) when specified as a percentage value", func() { @@ -509,11 +692,11 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionHard: map[string]string{ - memoryAvailable: "10%", + instancetypes.MemoryAvailable: "10%", }, }, }) - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) It("should consider the eviction threshold (hard) disabled when specified as 100%", func() { @@ -526,11 +709,11 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionHard: map[string]string{ - memoryAvailable: "100%", + instancetypes.MemoryAvailable: "100%", }, }, }) - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("0")) }) It("should used default eviction threshold (hard) for memory when evictionHard not specified", func() { @@ -543,11 +726,11 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionSoft: map[string]string{ - memoryAvailable: "50Mi", + instancetypes.MemoryAvailable: "50Mi", }, }, }) - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("50Mi")) }) It("should override eviction threshold (soft) when specified as a quantity", func() { @@ -560,11 +743,11 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionSoft: map[string]string{ - memoryAvailable: "500Mi", + instancetypes.MemoryAvailable: "500Mi", }, }, }) - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("500Mi")) }) It("should override eviction threshold (soft) when specified as a percentage value", func() { @@ -577,14 +760,14 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionHard: map[string]string{ - memoryAvailable: "5%", + instancetypes.MemoryAvailable: "5%", }, EvictionSoft: map[string]string{ - memoryAvailable: "10%", + instancetypes.MemoryAvailable: "10%", }, }, }) - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) It("should consider the eviction threshold (soft) disabled when specified as 100%", func() { @@ -597,11 +780,11 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionSoft: map[string]string{ - memoryAvailable: "100%", + instancetypes.MemoryAvailable: "100%", }, }, }) - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("0")) }) It("should ignore eviction threshold (soft) when using Bottlerocket AMI", func() { @@ -615,14 +798,14 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionHard: map[string]string{ - memoryAvailable: "1Gi", + instancetypes.MemoryAvailable: "1Gi", }, EvictionSoft: map[string]string{ - memoryAvailable: "10Gi", + instancetypes.MemoryAvailable: "10Gi", }, }, }) - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("1Gi")) }) It("should take the greater of evictionHard and evictionSoft for overhead as a value", func() { @@ -635,14 +818,14 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionSoft: map[string]string{ - memoryAvailable: "3Gi", + instancetypes.MemoryAvailable: "3Gi", }, EvictionHard: map[string]string{ - memoryAvailable: "1Gi", + instancetypes.MemoryAvailable: "1Gi", }, }, }) - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("3Gi")) }) It("should take the greater of evictionHard and evictionSoft for overhead as a value", func() { @@ -655,14 +838,14 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionSoft: map[string]string{ - memoryAvailable: "2%", + instancetypes.MemoryAvailable: "2%", }, EvictionHard: map[string]string{ - memoryAvailable: "5%", + instancetypes.MemoryAvailable: "5%", }, }, }) - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.05, 10)) }) It("should take the greater of evictionHard and evictionSoft for overhead with mixed percentage/value", func() { @@ -675,23 +858,23 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionSoft: map[string]string{ - memoryAvailable: "10%", + instancetypes.MemoryAvailable: "10%", }, EvictionHard: map[string]string{ - memoryAvailable: "1Gi", + instancetypes.MemoryAvailable: "1Gi", }, }, }) - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) }) It("should set max-pods to user-defined value if specified", func() { - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{MaxPods: ptr.Int32(10)}}) for _, info := range instanceInfo { - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 10)) } }) @@ -700,40 +883,40 @@ var _ = Describe("Instance Types", func() { EnablePodENI: lo.ToPtr(false), })) - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{MaxPods: ptr.Int32(10)}}) for _, info := range instanceInfo { - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 10)) } }) It("should override pods-per-core value", func() { - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}}) for _, info := range instanceInfo { - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", ptr.Int64Value(info.VCpuInfo.DefaultVCpus))) } }) It("should take the minimum of pods-per-core and max-pods", func() { - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(4), MaxPods: ptr.Int32(20)}}) for _, info := range instanceInfo { - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", lo.Min([]int64{20, ptr.Int64Value(info.VCpuInfo.DefaultVCpus) * 4}))) } }) It("should ignore pods-per-core when using Bottlerocket AMI", func() { - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}}) for _, info := range instanceInfo { - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) - Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", eniLimitedPods(info).Value())) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", instancetypes.EniLimitedPods(info).Value())) } }) It("should take 110 to be the default pods number when pods-per-core is 0 and AWSENILimitedPodDensity is unset", func() { @@ -741,11 +924,11 @@ var _ = Describe("Instance Types", func() { EnableENILimitedPodDensity: lo.ToPtr(false), })) - instanceInfo, err := instanceTypeProvider.getInstanceTypes(ctx) + instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(0)}}) for _, info := range instanceInfo { - it := NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 110)) } }) @@ -1068,7 +1251,7 @@ var _ = Describe("Instance Types", func() { // generateSpotPricing creates a spot price history output for use in a mock that has all spot offerings discounted by 50% // vs the on-demand offering. -func generateSpotPricing(cp *CloudProvider, prov *v1alpha5.Provisioner) *ec2.DescribeSpotPriceHistoryOutput { +func generateSpotPricing(cp *cloudprovider.CloudProvider, prov *v1alpha5.Provisioner) *ec2.DescribeSpotPriceHistoryOutput { rsp := &ec2.DescribeSpotPriceHistoryOutput{} instanceTypes, err := cp.GetInstanceTypes(ctx, prov) instanceTypeCache.Flush() diff --git a/pkg/providers/launchtemplate/launchtemplate_test.go b/pkg/providers/launchtemplate/launchtemplate_test.go index 60c39e3672bc..3212b4b64420 100644 --- a/pkg/providers/launchtemplate/launchtemplate_test.go +++ b/pkg/providers/launchtemplate/launchtemplate_test.go @@ -49,9 +49,11 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" + awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/cloudprovider/amifamily" "github.com/aws/karpenter/pkg/cloudprovider/amifamily/bootstrap" + awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/providers/instancetypes" "github.com/aws/karpenter/pkg/providers/launchtemplate" @@ -71,8 +73,6 @@ import ( "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" - awscache "github.com/aws/karpenter/pkg/cache" - awscontext "github.com/aws/karpenter/pkg/context" ) var ctx context.Context @@ -105,7 +105,7 @@ var securityGroupProvider *securitygroup.Provider func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) RegisterFailHandler(Fail) - RunSpecs(t, "CloudProvider/AWS") + RunSpecs(t, "Provider/AWS") } var _ = BeforeSuite(func() { diff --git a/pkg/providers/pricing/suite_test.go b/pkg/providers/pricing/suite_test.go index f6880c15a712..6ce6a219e5a9 100644 --- a/pkg/providers/pricing/suite_test.go +++ b/pkg/providers/pricing/suite_test.go @@ -53,7 +53,7 @@ var pricingProvider *pricing.Provider func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) RegisterFailHandler(Fail) - RunSpecs(t, "CloudProvider/AWS") + RunSpecs(t, "Provider/AWS") } var _ = BeforeSuite(func() { diff --git a/pkg/providers/securitygroup/suite_test.go b/pkg/providers/securitygroup/suite_test.go index ca346039332a..405c3f0ae88a 100644 --- a/pkg/providers/securitygroup/suite_test.go +++ b/pkg/providers/securitygroup/suite_test.go @@ -57,7 +57,7 @@ var securityGroupProvider *securitygroup.Provider func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) RegisterFailHandler(Fail) - RunSpecs(t, "CloudProvider/AWS") + RunSpecs(t, "Provider/AWS") } var _ = BeforeSuite(func() { diff --git a/pkg/providers/subnet/suite_test.go b/pkg/providers/subnet/suite_test.go index df1dd49c42c8..d816a03594b9 100644 --- a/pkg/providers/subnet/suite_test.go +++ b/pkg/providers/subnet/suite_test.go @@ -56,7 +56,7 @@ var subnetProvider *subnet.Provider func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) RegisterFailHandler(Fail) - RunSpecs(t, "CloudProvider/AWS") + RunSpecs(t, "Provider/AWS") } var _ = BeforeSuite(func() { From 33f2857f03e84e1c4247f861c24f4e1175b99f5f Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Mon, 27 Feb 2023 12:49:32 -0800 Subject: [PATCH 10/41] Feedback changes --- pkg/context/context.go | 4 +- pkg/context/suite_test.go | 37 +++++++------- .../launchtemplate/launchtemplate.go | 48 ++++++++----------- 3 files changed, 40 insertions(+), 49 deletions(-) diff --git a/pkg/context/context.go b/pkg/context/context.go index cb955d100211..e93fbf3bcf72 100644 --- a/pkg/context/context.go +++ b/pkg/context/context.go @@ -87,7 +87,7 @@ func NewOrDie(ctx cloudprovider.Context) Context { logging.FromContext(ctx).Fatalf("Checking EC2 API connectivity, %s", err) } logging.FromContext(ctx).With("region", *sess.Config.Region).Debugf("discovered region") - clusterEndpoint, err := resolveClusterEndpoint(ctx, eks.New(sess)) + clusterEndpoint, err := ResolveClusterEndpoint(ctx, eks.New(sess)) if err != nil { logging.FromContext(ctx).Fatalf("unable to detect the cluster endpoint, %s", err) } else { @@ -159,7 +159,7 @@ func checkEC2Connectivity(ctx context.Context, api *ec2.EC2) error { return err } -func resolveClusterEndpoint(ctx context.Context, eksAPI eksiface.EKSAPI) (string, error) { +func ResolveClusterEndpoint(ctx context.Context, eksAPI eksiface.EKSAPI) (string, error) { clusterEndpointFromSettings := settings.FromContext(ctx).ClusterEndpoint if clusterEndpointFromSettings != "" { return clusterEndpointFromSettings, nil // cluster endpoint is explicitly set diff --git a/pkg/context/suite_test.go b/pkg/context/suite_test.go index 72e01cc97eca..ceb34aac050e 100644 --- a/pkg/context/suite_test.go +++ b/pkg/context/suite_test.go @@ -12,30 +12,29 @@ See the License for the specific language governing permissions and limitations under the License. */ -package context +package context_test import ( "context" "errors" "testing" - ginkgov2 "github.com/onsi/ginkgo/v2" + "github.com/aws/aws-sdk-go/service/eks" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/samber/lo" . "knative.dev/pkg/logging/testing" - . "github.com/aws/karpenter-core/pkg/test/expectations" - - "github.com/aws/aws-sdk-go/service/eks" - "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" + awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/test" coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" + . "github.com/aws/karpenter-core/pkg/test/expectations" ) var ctx context.Context @@ -45,11 +44,11 @@ var fakeEKSAPI *fake.EKSAPI func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) - RegisterFailHandler(ginkgov2.Fail) - ginkgov2.RunSpecs(t, "CloudProvider/AWS") + RegisterFailHandler(Fail) + RunSpecs(t, "CloudProvider/AWS") } -var _ = ginkgov2.BeforeSuite(func() { +var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) ctx = coresettings.ToContext(ctx, coretest.Settings()) ctx = settings.ToContext(ctx, test.Settings()) @@ -58,31 +57,31 @@ var _ = ginkgov2.BeforeSuite(func() { fakeEKSAPI = &fake.EKSAPI{} }) -var _ = ginkgov2.AfterSuite(func() { +var _ = AfterSuite(func() { stop() Expect(env.Stop()).To(Succeed(), "Failed to stop environment") }) -var _ = ginkgov2.BeforeEach(func() { +var _ = BeforeEach(func() { fakeEKSAPI.Reset() }) -var _ = ginkgov2.AfterEach(func() { +var _ = AfterEach(func() { ExpectCleanedUp(ctx, env.Client) }) -var _ = ginkgov2.Describe("Context", func() { +var _ = Describe("Context", func() { - ginkgov2.It("should resolve endpoint if set via configuration", func() { + It("should resolve endpoint if set via configuration", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ ClusterEndpoint: lo.ToPtr("https://api.test-cluster.k8s.local"), })) - endpoint, err := resolveClusterEndpoint(ctx, fakeEKSAPI) + endpoint, err := awscontext.ResolveClusterEndpoint(ctx, fakeEKSAPI) Expect(err).ToNot(HaveOccurred()) Expect(endpoint).To(Equal("https://api.test-cluster.k8s.local")) }) - ginkgov2.It("should resolve endpoint if not set, via call to API", func() { + It("should resolve endpoint if not set, via call to API", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ ClusterEndpoint: lo.ToPtr(""), })) @@ -94,18 +93,18 @@ var _ = ginkgov2.Describe("Context", func() { }, ) - endpoint, err := resolveClusterEndpoint(ctx, fakeEKSAPI) + endpoint, err := awscontext.ResolveClusterEndpoint(ctx, fakeEKSAPI) Expect(err).ToNot(HaveOccurred()) Expect(endpoint).To(Equal("https://cluster-endpoint.test-cluster.k8s.local")) }) - ginkgov2.It("should propagate error if API fails", func() { + It("should propagate error if API fails", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ ClusterEndpoint: lo.ToPtr(""), })) fakeEKSAPI.DescribeClusterBehaviour.Error.Set(errors.New("test error")) - _, err := resolveClusterEndpoint(ctx, fakeEKSAPI) + _, err := awscontext.ResolveClusterEndpoint(ctx, fakeEKSAPI) Expect(err).To(HaveOccurred()) }) }) diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index a7bbae983c57..f317381375cd 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -55,27 +55,27 @@ type Provider struct { ec2api ec2iface.EC2API amiFamily *amifamily.Resolver securityGroupProvider *securitygroup.Provider - Cache *cache.Cache + cache *cache.Cache caBundle *string cm *pretty.ChangeMonitor KubeDNSIP net.IP ClusterEndpoint string } -func NewProvider(ctx context.Context, Cache *cache.Cache, ec2api ec2iface.EC2API, amiFamily *amifamily.Resolver, securityGroupProvider *securitygroup.Provider, caBundle *string, startAsync <-chan struct{}, kubeDNSIP net.IP, clusterEndpoint string) *Provider { +func NewProvider(ctx context.Context, cache *cache.Cache, ec2api ec2iface.EC2API, amiFamily *amifamily.Resolver, securityGroupProvider *securitygroup.Provider, caBundle *string, startAsync <-chan struct{}, kubeDNSIP net.IP, clusterEndpoint string) *Provider { l := &Provider{ ec2api: ec2api, amiFamily: amiFamily, securityGroupProvider: securityGroupProvider, - Cache: Cache, + cache: cache, caBundle: caBundle, cm: pretty.NewChangeMonitor(), KubeDNSIP: kubeDNSIP, ClusterEndpoint: clusterEndpoint, } - l.Cache.OnEvicted(l.cachedEvictedFunc(ctx)) + l.cache.OnEvicted(l.cachedEvictedFunc(ctx)) go func() { - // only hydrate Cache once elected leader + // only hydrate cache once elected leader select { case <-startAsync: case <-ctx.Done(): @@ -115,15 +115,15 @@ func (p *Provider) EnsureAll(ctx context.Context, nodeTemplate *v1alpha1.AWSNode return launchTemplates, nil } -// Invalidate deletes a launch template from Cache if it exists +// Invalidate deletes a launch template from cache if it exists func (p *Provider) Invalidate(ctx context.Context, ltName string, ltID string) { ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("launch-template-name", ltName, "launch-template-id", ltID)) p.Lock() defer p.Unlock() - defer p.Cache.OnEvicted(p.cachedEvictedFunc(ctx)) - p.Cache.OnEvicted(nil) - logging.FromContext(ctx).Debugf("invalidating launch template in the Cache because it no longer exists") - p.Cache.Delete(ltName) + defer p.cache.OnEvicted(p.cachedEvictedFunc(ctx)) + p.cache.OnEvicted(nil) + logging.FromContext(ctx).Debugf("invalidating launch template in the cache because it no longer exists") + p.cache.Delete(ltName) } func launchTemplateName(options *amifamily.LaunchTemplate) string { @@ -164,9 +164,9 @@ func (p *Provider) ensureLaunchTemplate(ctx context.Context, options *amifamily. var launchTemplate *ec2.LaunchTemplate name := launchTemplateName(options) ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("launch-template-name", name)) - // Read from Cache - if launchTemplate, ok := p.Cache.Get(name); ok { - p.Cache.SetDefault(name, launchTemplate) + // Read from cache + if launchTemplate, ok := p.cache.Get(name); ok { + p.cache.SetDefault(name, launchTemplate) return launchTemplate.(*ec2.LaunchTemplate), nil } // Attempt to find an existing LT. @@ -189,7 +189,7 @@ func (p *Provider) ensureLaunchTemplate(ctx context.Context, options *amifamily. } launchTemplate = output.LaunchTemplates[0] } - p.Cache.SetDefault(name, launchTemplate) + p.cache.SetDefault(name, launchTemplate) return launchTemplate, nil } @@ -268,30 +268,30 @@ func (p *Provider) volumeSize(quantity *resource.Quantity) *int64 { return aws.Int64(int64(math.Ceil(quantity.AsApproximateFloat64() / math.Pow(2, 30)))) } -// hydrateCache queries for existing Launch Templates created by Karpenter for the current cluster and adds to the LT Cache. +// hydrateCache queries for existing Launch Templates created by Karpenter for the current cluster and adds to the LT cache. // Any error during hydration will result in a panic func (p *Provider) hydrateCache(ctx context.Context) { clusterName := awssettings.FromContext(ctx).ClusterName ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("tag-key", karpenterManagedTagKey, "tag-value", clusterName)) - logging.FromContext(ctx).Debugf("hydrating the launch template Cache") + logging.FromContext(ctx).Debugf("hydrating the launch template cache") if err := p.ec2api.DescribeLaunchTemplatesPagesWithContext(ctx, &ec2.DescribeLaunchTemplatesInput{ Filters: []*ec2.Filter{{Name: aws.String(fmt.Sprintf("tag:%s", karpenterManagedTagKey)), Values: []*string{aws.String(clusterName)}}}, }, func(output *ec2.DescribeLaunchTemplatesOutput, _ bool) bool { for _, lt := range output.LaunchTemplates { - p.Cache.SetDefault(*lt.LaunchTemplateName, lt) + p.cache.SetDefault(*lt.LaunchTemplateName, lt) } return true }); err != nil { - logging.FromContext(ctx).Errorf(fmt.Sprintf("Unable to hydrate the AWS launch template Cache, %s", err)) + logging.FromContext(ctx).Errorf(fmt.Sprintf("Unable to hydrate the AWS launch template cache, %s", err)) } - logging.FromContext(ctx).With("item-count", p.Cache.ItemCount()).Debugf("finished hydrating the launch template Cache") + logging.FromContext(ctx).With("item-count", p.cache.ItemCount()).Debugf("finished hydrating the launch template cache") } func (p *Provider) cachedEvictedFunc(ctx context.Context) func(string, interface{}) { return func(key string, lt interface{}) { p.Lock() defer p.Unlock() - if _, expiration, _ := p.Cache.GetWithExpiration(key); expiration.After(time.Now()) { + if _, expiration, _ := p.cache.GetWithExpiration(key); expiration.After(time.Now()) { return } launchTemplate := lt.(*ec2.LaunchTemplate) @@ -313,11 +313,3 @@ func (p *Provider) getInstanceProfile(ctx context.Context, nodeTemplate *v1alpha } return defaultProfile, nil } - -// func (p *Provider) GetCache(ltName string) (result interface{}, ok bool) { -// return p.Cache.Get(ltName) -// } - -// func (p *Provider) SetCache(ltName string, lt interface{}) { -// p.Cache.Set(ltName, lt, -1) -// } From dcc124967334396509ec901ec8d09b5b5da08791 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Mon, 27 Feb 2023 14:29:39 -0800 Subject: [PATCH 11/41] testing changes --- pkg/providers/instancetypes/suite_test.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/providers/instancetypes/suite_test.go b/pkg/providers/instancetypes/suite_test.go index 563bb682538b..3774ab058e3d 100644 --- a/pkg/providers/instancetypes/suite_test.go +++ b/pkg/providers/instancetypes/suite_test.go @@ -25,6 +25,7 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/awstesting/mock" "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" @@ -46,6 +47,7 @@ import ( "github.com/aws/karpenter-core/pkg/controllers/provisioning" "github.com/aws/karpenter-core/pkg/controllers/state" "github.com/aws/karpenter-core/pkg/events" + "github.com/aws/karpenter-core/pkg/operator/controller" "github.com/aws/karpenter-core/pkg/operator/injection" "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" @@ -74,6 +76,7 @@ var ctx context.Context var stop context.CancelFunc var opts options.Options var env *coretest.Environment +var fakeSession *session.Session var ssmCache *cache.Cache var ec2Cache *cache.Cache var launchTemplateCache *cache.Cache @@ -96,6 +99,7 @@ var pricingProvider *pricing.Provider var subnetProvider *subnet.Provider var instanceTypeProvider *instancetypes.Provider var securityGroupProvider *securitygroup.Provider +var provisioningController controller.Controller func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -109,6 +113,7 @@ var _ = BeforeSuite(func() { ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) + fakeSession = mock.Session fakeEC2API = &fake.EC2API{} fakeSSMAPI = &fake.SSMAPI{} ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) @@ -124,7 +129,7 @@ var _ = BeforeSuite(func() { securityGroupProvider = securitygroup.NewProvider(fakeEC2API) amiProvider = amifamily.NewAMIProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) amiResolver = amifamily.New(env.Client, amiProvider) - instanceTypeProvider = instancetypes.NewProvider(mock.Session, instanceTypeCache, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) + instanceTypeProvider = instancetypes.NewProvider(fakeSession, instanceTypeCache, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) launchTemplateProvider = launchtemplate.NewProvider( ctx, @@ -150,7 +155,7 @@ var _ = BeforeSuite(func() { }, SubnetProvider: subnet.NewProvider(fakeEC2API), SecurityGroupProvider: securityGroupProvider, - Session: mock.Session, + Session: fakeSession, UnavailableOfferingsCache: unavailableOfferingsCache, EC2API: fakeEC2API, PricingProvider: pricingProvider, @@ -161,7 +166,7 @@ var _ = BeforeSuite(func() { }) cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) - + provisioningController = provisioning.NewController(env.Client, prov, events.NewRecorder(&record.FakeRecorder{})) }) var _ = AfterSuite(func() { @@ -219,7 +224,7 @@ var _ = BeforeEach(func() { // Reset the pricing provider, so we don't cross-pollinate pricing data instanceTypeProvider = instancetypes.NewProvider( - mock.Session, + fakeSession, instanceTypeCache, fakeEC2API, subnetProvider, From 6249e4b31347a3bb911b395d2569035400f08a84 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Mon, 27 Feb 2023 16:51:35 -0800 Subject: [PATCH 12/41] Fixed all Tests --- pkg/controllers/machine/link/suite_test.go | 14 +++++++++++++- pkg/providers/instancetypes/suite_test.go | 9 +++++++-- .../launchtemplate/launchtemplate_test.go | 1 + pkg/providers/subnet/suite_test.go | 2 +- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/pkg/controllers/machine/link/suite_test.go b/pkg/controllers/machine/link/suite_test.go index 008c91efff02..89df2a47cc03 100644 --- a/pkg/controllers/machine/link/suite_test.go +++ b/pkg/controllers/machine/link/suite_test.go @@ -25,6 +25,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/patrickmn/go-cache" "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -50,6 +51,7 @@ import ( awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/controllers/machine/link" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/providers/instancetypes" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/providers/securitygroup" "github.com/aws/karpenter/pkg/providers/subnet" @@ -62,7 +64,11 @@ var env *coretest.Environment var unavailableOfferingsCache *awscache.UnavailableOfferings var ec2API *fake.EC2API var cloudProvider *cloudprovider.CloudProvider +var subnetProvider *subnet.Provider +var instanceTypeCache *cache.Cache var linkController controller.Controller +var pricingProvider *pricing.Provider +var instanceTypeProvider *instancetypes.Provider func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -76,6 +82,10 @@ var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) unavailableOfferingsCache = awscache.NewUnavailableOfferings() ec2API = &fake.EC2API{} + instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + subnetProvider = subnet.NewProvider(ec2API) + pricingProvider = pricing.NewProvider(ctx, &fake.PricingAPI{}, ec2API, "", make(chan struct{})) + instanceTypeProvider = instancetypes.NewProvider(mock.Session, instanceTypeCache, ec2API, subnetProvider, unavailableOfferingsCache, pricingProvider) cloudProvider = cloudprovider.New(awscontext.Context{ Context: corecloudprovider.Context{ Context: ctx, @@ -91,7 +101,8 @@ var _ = BeforeSuite(func() { Session: mock.Session, UnavailableOfferingsCache: unavailableOfferingsCache, EC2API: ec2API, - PricingProvider: pricing.NewProvider(ctx, &fake.PricingAPI{}, ec2API, "", make(chan struct{})), + PricingProvider: pricingProvider, + InstanceTypeProvider: instanceTypeProvider, }) linkController = link.NewController(env.Client, cloudProvider) }) @@ -108,6 +119,7 @@ var _ = Describe("MachineLink", func() { BeforeEach(func() { ec2API.Reset() + instanceTypeCache.Flush() instanceID = fake.InstanceID() providerID = fmt.Sprintf("aws:///test-zone-1a/%s", instanceID) nodeTemplate = test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{}) diff --git a/pkg/providers/instancetypes/suite_test.go b/pkg/providers/instancetypes/suite_test.go index 3774ab058e3d..82674dc902d6 100644 --- a/pkg/providers/instancetypes/suite_test.go +++ b/pkg/providers/instancetypes/suite_test.go @@ -114,6 +114,7 @@ var _ = BeforeSuite(func() { ctx, stop = context.WithCancel(ctx) fakeSession = mock.Session + fakeSession.Config.Region = aws.String("") fakeEC2API = &fake.EC2API{} fakeSSMAPI = &fake.SSMAPI{} ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) @@ -233,6 +234,10 @@ var _ = BeforeEach(func() { ) }) +var _ = AfterEach(func() { + ExpectCleanedUp(ctx, env.Client) +}) + var _ = Describe("Instance Types", func() { It("should support instance type labels", func() { ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) @@ -1221,7 +1226,7 @@ var _ = Describe("Instance Types", func() { }) }) Context("Metadata Options", func() { - It("should default metadata options on generated launch template", func() { + FIt("should default metadata options on generated launch template", func() { ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, prov, pod) @@ -1233,7 +1238,7 @@ var _ = Describe("Instance Types", func() { Expect(*input.LaunchTemplateData.MetadataOptions.HttpPutResponseHopLimit).To(Equal(int64(2))) Expect(*input.LaunchTemplateData.MetadataOptions.HttpTokens).To(Equal(ec2.LaunchTemplateHttpTokensStateRequired)) }) - It("should set metadata options on generated launch template from provisioner configuration", func() { + FIt("should set metadata options on generated launch template from provisioner configuration", func() { nodeTemplate.Spec.MetadataOptions = &v1alpha1.MetadataOptions{ HTTPEndpoint: aws.String(ec2.LaunchTemplateInstanceMetadataEndpointStateDisabled), HTTPProtocolIPv6: aws.String(ec2.LaunchTemplateInstanceMetadataProtocolIpv6Enabled), diff --git a/pkg/providers/launchtemplate/launchtemplate_test.go b/pkg/providers/launchtemplate/launchtemplate_test.go index 3212b4b64420..645b91157253 100644 --- a/pkg/providers/launchtemplate/launchtemplate_test.go +++ b/pkg/providers/launchtemplate/launchtemplate_test.go @@ -162,6 +162,7 @@ var _ = BeforeSuite(func() { AMIProvider: amiProvider, AMIResolver: amiResolver, LaunchTemplateProvider: launchTemplateProvider, + InstanceTypeProvider: instanceTypeProvider, }) cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) diff --git a/pkg/providers/subnet/suite_test.go b/pkg/providers/subnet/suite_test.go index d816a03594b9..034b115bddbd 100644 --- a/pkg/providers/subnet/suite_test.go +++ b/pkg/providers/subnet/suite_test.go @@ -113,7 +113,7 @@ var _ = AfterEach(func() { ExpectCleanedUp(ctx, env.Client) }) -var _ = Describe("Security Group Provider", func() { +var _ = Describe("Subnet Provider", func() { It("should discover subnet by ID", func() { nodeTemplate.Spec.SubnetSelector = map[string]string{"aws-ids": "subnet-test1"} ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) From 7ecc6667528a2526a014d918e593e87619bd20aa Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Mon, 27 Feb 2023 17:05:54 -0800 Subject: [PATCH 13/41] Hack File Updates --- hack/api-code-gen.sh | 4 ++-- hack/code/bandwidth_gen.go | 4 ++-- hack/code/vpc_limits_gen.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hack/api-code-gen.sh b/hack/api-code-gen.sh index 59bde8c5f097..efddca8af0ac 100755 --- a/hack/api-code-gen.sh +++ b/hack/api-code-gen.sh @@ -8,7 +8,7 @@ fi echo "api-code-gen running ENABLE_GIT_PUSH: ${ENABLE_GIT_PUSH}" bandwidth() { - GENERATED_FILE="pkg/cloudprovider/zz_generated.bandwidth.go" + GENERATED_FILE="pkg/providers/instancetypes/zz_generated.bandwidth.go" NO_UPDATE='' SUBJECT="Bandwidth" @@ -30,7 +30,7 @@ pricing() { } vpcLimits() { - GENERATED_FILE="pkg/cloudprovider/zz_generated.vpclimits.go" + GENERATED_FILE="pkg/providers/instancetypes/zz_generated.vpclimits.go" NO_UPDATE='' SUBJECT="VPC Limits" diff --git a/hack/code/bandwidth_gen.go b/hack/code/bandwidth_gen.go index ca670407aeb2..dee645521561 100644 --- a/hack/code/bandwidth_gen.go +++ b/hack/code/bandwidth_gen.go @@ -40,7 +40,7 @@ var uriSelectors = map[string]string{ const fileFormat = ` %s -package cloudprovider +package instancetypes // GENERATED FILE. DO NOT EDIT DIRECTLY. // Update hack/code/bandwidth_gen.go and re-generate to edit @@ -56,7 +56,7 @@ var ( func main() { flag.Parse() if flag.NArg() != 1 { - log.Fatalf("Usage: `bandwidth_gen.go pkg/cloudprovider/zz_generated.pricing.go`") + log.Fatalf("Usage: `bandwidth_gen.go pkg/providers/instancetypes/zz_generated.pricing.go`") } bandwidth := map[string]int64{} diff --git a/hack/code/vpc_limits_gen.go b/hack/code/vpc_limits_gen.go index 2255f7ceb3e1..5c0f84146758 100644 --- a/hack/code/vpc_limits_gen.go +++ b/hack/code/vpc_limits_gen.go @@ -35,7 +35,7 @@ func main() { opts := options{} flag.StringVar(&opts.urlInput, "url", "https://raw.githubusercontent.com/aws/amazon-vpc-resource-controller-k8s/master/pkg/aws/vpc/limits.go", "url of the raw vpc/limits.go file in the github.com/aws/amazon-vpc-resource-controller-k8s repo") - flag.StringVar(&opts.sourceOutput, "output", "pkg/cloudprovider/zz_generated.vpclimits.go", "output location for the generated go source file") + flag.StringVar(&opts.sourceOutput, "output", "pkg/providers/instancetypes/zz_generated.vpclimits.go", "output location for the generated go source file") flag.Parse() limitsURL, err := url.Parse(opts.urlInput) @@ -58,7 +58,7 @@ func main() { if err != nil { log.Fatal(err) } - newRespData := strings.Replace(string(respData), "package vpc", "package cloudprovider", 1) + newRespData := strings.Replace(string(respData), "package vpc", "package instancetypes", 1) out.WriteString(newRespData) defer out.Close() From e51c1d4d43ea02cf66ab04950efc578c1ecdbe61 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Tue, 28 Feb 2023 08:25:09 -0800 Subject: [PATCH 14/41] Clean-up tests --- pkg/providers/instancetypes/suite_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/providers/instancetypes/suite_test.go b/pkg/providers/instancetypes/suite_test.go index 82674dc902d6..f94ee5ebfef7 100644 --- a/pkg/providers/instancetypes/suite_test.go +++ b/pkg/providers/instancetypes/suite_test.go @@ -1226,7 +1226,7 @@ var _ = Describe("Instance Types", func() { }) }) Context("Metadata Options", func() { - FIt("should default metadata options on generated launch template", func() { + It("should default metadata options on generated launch template", func() { ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, prov, pod) @@ -1238,7 +1238,7 @@ var _ = Describe("Instance Types", func() { Expect(*input.LaunchTemplateData.MetadataOptions.HttpPutResponseHopLimit).To(Equal(int64(2))) Expect(*input.LaunchTemplateData.MetadataOptions.HttpTokens).To(Equal(ec2.LaunchTemplateHttpTokensStateRequired)) }) - FIt("should set metadata options on generated launch template from provisioner configuration", func() { + It("should set metadata options on generated launch template from provisioner configuration", func() { nodeTemplate.Spec.MetadataOptions = &v1alpha1.MetadataOptions{ HTTPEndpoint: aws.String(ec2.LaunchTemplateInstanceMetadataEndpointStateDisabled), HTTPProtocolIPv6: aws.String(ec2.LaunchTemplateInstanceMetadataProtocolIpv6Enabled), From 4566f5edd9cf7e8fcc3f6178204425a2352252cc Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Tue, 28 Feb 2023 12:15:44 -0800 Subject: [PATCH 15/41] Moved instances to providers --- pkg/cloudprovider/cloudprovider.go | 38 ++++------ pkg/cloudprovider/suite_test.go | 5 +- pkg/context/context.go | 12 +++ .../machine/garbagecollect/suite_test.go | 49 +++++++++++- pkg/controllers/machine/link/suite_test.go | 40 +++++++++- .../instance}/instance.go | 76 ++++++++++--------- pkg/providers/instancetypes/suite_test.go | 10 ++- .../launchtemplate/launchtemplate_test.go | 6 +- 8 files changed, 165 insertions(+), 71 deletions(-) rename pkg/{cloudprovider => providers/instance}/instance.go (84%) diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index c6d1b52c1a1b..26f8c794f5df 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -45,17 +45,13 @@ import ( "github.com/aws/karpenter/pkg/cloudprovider/amifamily" awscontext "github.com/aws/karpenter/pkg/context" + "github.com/aws/karpenter/pkg/providers/instance" coreapis "github.com/aws/karpenter-core/pkg/apis" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/cloudprovider" ) -const ( - // MaxInstanceTypes defines the number of instance type options to pass to CreateFleet - MaxInstanceTypes = 60 -) - func init() { v1alpha5.NormalizedLabels = lo.Assign(v1alpha5.NormalizedLabels, map[string]string{"topology.ebs.csi.aws.com/zone": v1.LabelTopologyZone}) coreapis.Settings = append(coreapis.Settings, apis.Settings...) @@ -65,7 +61,7 @@ var _ cloudprovider.CloudProvider = (*CloudProvider)(nil) type CloudProvider struct { instanceTypeProvider *instancetypes.Provider - instanceProvider *InstanceProvider + instanceProvider *instance.Provider kubeClient client.Client amiProvider *amifamily.AMIProvider } @@ -75,15 +71,7 @@ func New(ctx awscontext.Context) *CloudProvider { kubeClient: ctx.KubeClient, instanceTypeProvider: ctx.InstanceTypeProvider, amiProvider: ctx.AMIProvider, - instanceProvider: NewInstanceProvider( - ctx, - aws.StringValue(ctx.Session.Config.Region), - ctx.EC2API, - ctx.UnavailableOfferingsCache, - ctx.InstanceTypeProvider, - ctx.SubnetProvider, - ctx.LaunchTemplateProvider, - ), + instanceProvider: ctx.InstanceProvider, } } @@ -315,7 +303,7 @@ func (c *CloudProvider) resolveProvisionerFromInstance(ctx context.Context, inst return provisioner, nil } -func (c *CloudProvider) instanceToMachine(ctx context.Context, instance *ec2.Instance, instanceType *cloudprovider.InstanceType) *v1alpha5.Machine { +func (c *CloudProvider) instanceToMachine(ctx context.Context, ec2instance *ec2.Instance, instanceType *cloudprovider.InstanceType) *v1alpha5.Machine { machine := &v1alpha5.Machine{} labels := map[string]string{} @@ -328,22 +316,22 @@ func (c *CloudProvider) instanceToMachine(ctx context.Context, instance *ec2.Ins machine.Status.Capacity = functional.FilterMap(instanceType.Capacity, func(_ v1.ResourceName, v resource.Quantity) bool { return !resources.IsZero(v) }) machine.Status.Allocatable = functional.FilterMap(instanceType.Allocatable(), func(_ v1.ResourceName, v resource.Quantity) bool { return !resources.IsZero(v) }) } - labels[v1alpha1.LabelInstanceAMIID] = aws.StringValue(instance.ImageId) - labels[v1.LabelTopologyZone] = aws.StringValue(instance.Placement.AvailabilityZone) - labels[v1alpha5.LabelCapacityType] = getCapacityType(instance) - if tag, ok := lo.Find(instance.Tags, func(t *ec2.Tag) bool { return aws.StringValue(t.Key) == v1alpha5.ProvisionerNameLabelKey }); ok { + labels[v1alpha1.LabelInstanceAMIID] = aws.StringValue(ec2instance.ImageId) + labels[v1.LabelTopologyZone] = aws.StringValue(ec2instance.Placement.AvailabilityZone) + labels[v1alpha5.LabelCapacityType] = instance.GetCapacityType(ec2instance) + if tag, ok := lo.Find(ec2instance.Tags, func(t *ec2.Tag) bool { return aws.StringValue(t.Key) == v1alpha5.ProvisionerNameLabelKey }); ok { labels[v1alpha5.ProvisionerNameLabelKey] = aws.StringValue(tag.Value) } - if tag, ok := lo.Find(instance.Tags, func(t *ec2.Tag) bool { return aws.StringValue(t.Key) == v1alpha5.ManagedByLabelKey }); ok { + if tag, ok := lo.Find(ec2instance.Tags, func(t *ec2.Tag) bool { return aws.StringValue(t.Key) == v1alpha5.ManagedByLabelKey }); ok { labels[v1alpha5.ManagedByLabelKey] = aws.StringValue(tag.Value) } machine.Name = lo.Ternary( settings.FromContext(ctx).NodeNameConvention == settings.ResourceName, - aws.StringValue(instance.InstanceId), - strings.ToLower(aws.StringValue(instance.PrivateDnsName)), + aws.StringValue(ec2instance.InstanceId), + strings.ToLower(aws.StringValue(ec2instance.PrivateDnsName)), ) machine.Labels = labels - machine.CreationTimestamp = metav1.Time{Time: aws.TimeValue(instance.LaunchTime)} - machine.Status.ProviderID = fmt.Sprintf("aws:///%s/%s", aws.StringValue(instance.Placement.AvailabilityZone), aws.StringValue(instance.InstanceId)) + machine.CreationTimestamp = metav1.Time{Time: aws.TimeValue(ec2instance.LaunchTime)} + machine.Status.ProviderID = fmt.Sprintf("aws:///%s/%s", aws.StringValue(ec2instance.Placement.AvailabilityZone), aws.StringValue(ec2instance.InstanceId)) return machine } diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index a0d969c7e34a..f075cfbff739 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -67,6 +67,7 @@ import ( "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" + "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/providers/instancetypes" "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/pricing" @@ -101,6 +102,7 @@ var nodeTemplate *v1alpha1.AWSNodeTemplate var pricingProvider *pricing.Provider var subnetProvider *subnet.Provider var securityGroupProvider *securitygroup.Provider +var instanceProvider *instance.Provider func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -147,10 +149,11 @@ var _ = BeforeSuite(func() { net.ParseIP("10.0.100.10"), "https://test-cluster", ) + instanceProvider = instance.NewProvider(ctx, "", fakeEC2API, unavailableOfferingsCache, instanceTypeProvider, subnetProvider, launchTemplateProvider) cloudProvider = &CloudProvider{ instanceTypeProvider: instanceTypeProvider, amiProvider: amiProvider, - instanceProvider: NewInstanceProvider(ctx, "", fakeEC2API, unavailableOfferingsCache, instanceTypeProvider, subnetProvider, launchTemplateProvider), + instanceProvider: instanceProvider, kubeClient: env.Client, } fakeClock = clock.NewFakeClock(time.Now()) diff --git a/pkg/context/context.go b/pkg/context/context.go index a325ebf932a0..16d867e90b69 100644 --- a/pkg/context/context.go +++ b/pkg/context/context.go @@ -45,6 +45,7 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/cloudprovider/amifamily" + "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/providers/instancetypes" "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/pricing" @@ -69,6 +70,7 @@ type Context struct { LaunchTemplateProvider *launchtemplate.Provider PricingProvider *pricing.Provider InstanceTypeProvider *instancetypes.Provider + InstanceProvider *instance.Provider } func NewOrDie(ctx cloudprovider.Context) Context { @@ -137,6 +139,15 @@ func NewOrDie(ctx cloudprovider.Context) Context { unavailableOfferingsCache, pricingProvider, ) + instanceProvider := instance.NewProvider( + ctx, + aws.StringValue(sess.Config.Region), + ec2api, + unavailableOfferingsCache, + instanceTypeProvider, + subnetProvider, + launchTemplateProvider, + ) return Context{ Context: ctx, @@ -150,6 +161,7 @@ func NewOrDie(ctx cloudprovider.Context) Context { LaunchTemplateProvider: launchTemplateProvider, PricingProvider: pricingProvider, InstanceTypeProvider: instanceTypeProvider, + InstanceProvider: instanceProvider, } } diff --git a/pkg/controllers/machine/garbagecollect/suite_test.go b/pkg/controllers/machine/garbagecollect/suite_test.go index 0d54824e35a5..63cce4dfa47d 100644 --- a/pkg/controllers/machine/garbagecollect/suite_test.go +++ b/pkg/controllers/machine/garbagecollect/suite_test.go @@ -17,6 +17,7 @@ package garbagecollect_test import ( "context" "fmt" + "net" "sync" "testing" "time" @@ -32,6 +33,7 @@ import ( "k8s.io/client-go/tools/record" clock "k8s.io/utils/clock/testing" . "knative.dev/pkg/logging/testing" + "knative.dev/pkg/ptr" "sigs.k8s.io/controller-runtime/pkg/client" coresettings "github.com/aws/karpenter-core/pkg/apis/settings" @@ -47,10 +49,15 @@ import ( "github.com/aws/karpenter/pkg/apis/v1alpha1" awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter/pkg/cloudprovider/amifamily" awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/controllers/machine/garbagecollect" "github.com/aws/karpenter/pkg/controllers/machine/link" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/providers/instance" + "github.com/aws/karpenter/pkg/providers/instancetypes" + "github.com/aws/karpenter/pkg/providers/launchtemplate" + "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/providers/securitygroup" "github.com/aws/karpenter/pkg/providers/subnet" "github.com/aws/karpenter/pkg/test" @@ -58,11 +65,24 @@ import ( var ctx context.Context var env *coretest.Environment +var instanceTypeCache *cache.Cache var unavailableOfferingsCache *awscache.UnavailableOfferings +var launchTemplateCache *cache.Cache +var ssmCache *cache.Cache +var kubernetesVersionCache *cache.Cache +var ec2Cache *cache.Cache var ec2API *fake.EC2API var cloudProvider *cloudprovider.CloudProvider var garbageCollectController controller.Controller var linkedMachineCache *cache.Cache +var amiProvider *amifamily.AMIProvider +var amiResolver *amifamily.Resolver +var securityGroupProvider *securitygroup.Provider +var pricingProvider *pricing.Provider +var subnetProvider *subnet.Provider +var launchTemplateProvider *launchtemplate.Provider +var instanceTypeProvider *instancetypes.Provider +var instanceProvider *instance.Provider func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -75,7 +95,29 @@ var _ = BeforeSuite(func() { ctx = settings.ToContext(ctx, test.Settings()) env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) unavailableOfferingsCache = awscache.NewUnavailableOfferings() + launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) ec2API = &fake.EC2API{} + securityGroupProvider = securitygroup.NewProvider(ec2API) + amiProvider = amifamily.NewAMIProvider(env.Client, env.KubernetesInterface, &fake.SSMAPI{}, ec2API, ssmCache, ec2Cache, kubernetesVersionCache) + amiResolver = amifamily.New(env.Client, amiProvider) + pricingProvider = pricing.NewProvider(ctx, &fake.PricingAPI{}, ec2API, "", make(chan struct{})) + instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + instanceTypeProvider = instancetypes.NewProvider(mock.Session, instanceTypeCache, ec2API, subnetProvider, unavailableOfferingsCache, pricingProvider) + launchTemplateProvider = launchtemplate.NewProvider( + ctx, + launchTemplateCache, + ec2API, + amiResolver, + securityGroupProvider, + ptr.String("ca-bundle"), + make(chan struct{}), + net.ParseIP("10.0.100.10"), + "https://test-cluster", + ) + instanceProvider = instance.NewProvider(ctx, "", ec2API, unavailableOfferingsCache, instanceTypeProvider, subnetProvider, launchTemplateProvider) cloudProvider = cloudprovider.New(awscontext.Context{ Context: corecloudprovider.Context{ Context: ctx, @@ -87,10 +129,15 @@ var _ = BeforeSuite(func() { StartAsync: nil, }, SubnetProvider: subnet.NewProvider(ec2API), - SecurityGroupProvider: securitygroup.NewProvider(ec2API), + SecurityGroupProvider: securityGroupProvider, Session: mock.Session, UnavailableOfferingsCache: unavailableOfferingsCache, EC2API: ec2API, + InstanceProvider: instanceProvider, + AMIProvider: amiProvider, + AMIResolver: amiResolver, + LaunchTemplateProvider: launchTemplateProvider, + PricingProvider: pricingProvider, }) linkedMachineCache = cache.New(time.Minute*10, time.Second*10) linkController := &link.Controller{ diff --git a/pkg/controllers/machine/link/suite_test.go b/pkg/controllers/machine/link/suite_test.go index 89df2a47cc03..0e65f81cc325 100644 --- a/pkg/controllers/machine/link/suite_test.go +++ b/pkg/controllers/machine/link/suite_test.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "net" "testing" "github.com/aws/aws-sdk-go/aws" @@ -32,6 +33,7 @@ import ( "k8s.io/client-go/tools/record" clock "k8s.io/utils/clock/testing" . "knative.dev/pkg/logging/testing" + "knative.dev/pkg/ptr" "sigs.k8s.io/controller-runtime/pkg/client" coresettings "github.com/aws/karpenter-core/pkg/apis/settings" @@ -48,10 +50,13 @@ import ( "github.com/aws/karpenter/pkg/apis/v1alpha1" awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter/pkg/cloudprovider/amifamily" awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/controllers/machine/link" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/providers/instancetypes" + "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/providers/securitygroup" "github.com/aws/karpenter/pkg/providers/subnet" @@ -61,14 +66,24 @@ import ( var ctx context.Context var env *coretest.Environment +var launchTemplateCache *cache.Cache var unavailableOfferingsCache *awscache.UnavailableOfferings +var ssmCache *cache.Cache +var ec2Cache *cache.Cache +var kubernetesVersionCache *cache.Cache var ec2API *fake.EC2API +var ssmAPI *fake.SSMAPI var cloudProvider *cloudprovider.CloudProvider var subnetProvider *subnet.Provider +var securityGroupProvider *securitygroup.Provider var instanceTypeCache *cache.Cache var linkController controller.Controller var pricingProvider *pricing.Provider +var amiProvider *amifamily.AMIProvider +var amiResolver *amifamily.Resolver var instanceTypeProvider *instancetypes.Provider +var launchTemplateProvider *launchtemplate.Provider +var instanceProvider *instance.Provider func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -81,11 +96,31 @@ var _ = BeforeSuite(func() { ctx = settings.ToContext(ctx, test.Settings()) env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) unavailableOfferingsCache = awscache.NewUnavailableOfferings() - ec2API = &fake.EC2API{} + launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + ec2API = &fake.EC2API{} + ssmAPI = &fake.SSMAPI{} subnetProvider = subnet.NewProvider(ec2API) + securityGroupProvider = securitygroup.NewProvider(ec2API) pricingProvider = pricing.NewProvider(ctx, &fake.PricingAPI{}, ec2API, "", make(chan struct{})) + amiProvider = amifamily.NewAMIProvider(env.Client, env.KubernetesInterface, ssmAPI, ec2API, ssmCache, ec2Cache, kubernetesVersionCache) + amiResolver = amifamily.New(env.Client, amiProvider) instanceTypeProvider = instancetypes.NewProvider(mock.Session, instanceTypeCache, ec2API, subnetProvider, unavailableOfferingsCache, pricingProvider) + launchTemplateProvider = launchtemplate.NewProvider( + ctx, + launchTemplateCache, + ec2API, + amiResolver, + securityGroupProvider, + ptr.String("ca-bundle"), + make(chan struct{}), + net.ParseIP("10.0.100.10"), + "https://test-cluster", + ) + instanceProvider = instance.NewProvider(ctx, "", ec2API, unavailableOfferingsCache, instanceTypeProvider, subnetProvider, launchTemplateProvider) cloudProvider = cloudprovider.New(awscontext.Context{ Context: corecloudprovider.Context{ Context: ctx, @@ -103,6 +138,9 @@ var _ = BeforeSuite(func() { EC2API: ec2API, PricingProvider: pricingProvider, InstanceTypeProvider: instanceTypeProvider, + InstanceProvider: instanceProvider, + AMIProvider: amiProvider, + AMIResolver: amiResolver, }) linkController = link.NewController(env.Client, cloudProvider) }) diff --git a/pkg/cloudprovider/instance.go b/pkg/providers/instance/instance.go similarity index 84% rename from pkg/cloudprovider/instance.go rename to pkg/providers/instance/instance.go index a0ec5c0bf43e..bd65c5a87e12 100644 --- a/pkg/cloudprovider/instance.go +++ b/pkg/providers/instance/instance.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cloudprovider +package instance import ( "context" @@ -46,11 +46,13 @@ import ( "github.com/aws/karpenter-core/pkg/utils/resources" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - "github.com/aws/karpenter-core/pkg/cloudprovider" + corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" "github.com/aws/karpenter-core/pkg/scheduling" ) var ( + // MaxInstanceTypes defines the number of instance type options to pass to CreateFleet + MaxInstanceTypes = 60 instanceTypeFlexibilityThreshold = 5 // falling back to on-demand without flexibility risks insufficient capacity errors instanceStateFilter = &ec2.Filter{ @@ -59,7 +61,7 @@ var ( } ) -type InstanceProvider struct { +type Provider struct { region string ec2api ec2iface.EC2API unavailableOfferings *cache.UnavailableOfferings @@ -69,8 +71,8 @@ type InstanceProvider struct { ec2Batcher *batcher.EC2API } -func NewInstanceProvider(ctx context.Context, region string, ec2api ec2iface.EC2API, unavailableOfferings *cache.UnavailableOfferings, instanceTypeProvider *instancetypes.Provider, subnetProvider *subnet.Provider, launchTemplateProvider *launchtemplate.Provider) *InstanceProvider { - return &InstanceProvider{ +func NewProvider(ctx context.Context, region string, ec2api ec2iface.EC2API, unavailableOfferings *cache.UnavailableOfferings, instanceTypeProvider *instancetypes.Provider, subnetProvider *subnet.Provider, launchTemplateProvider *launchtemplate.Provider) *Provider { + return &Provider{ region: region, ec2api: ec2api, unavailableOfferings: unavailableOfferings, @@ -81,7 +83,7 @@ func NewInstanceProvider(ctx context.Context, region string, ec2api ec2iface.EC2 } } -func (p *InstanceProvider) Create(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate, machine *v1alpha5.Machine, instanceTypes []*cloudprovider.InstanceType) (*ec2.Instance, error) { +func (p *Provider) Create(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate, machine *v1alpha5.Machine, instanceTypes []*corecloudprovider.InstanceType) (*ec2.Instance, error) { instanceTypes = p.filterInstanceTypes(machine, instanceTypes) instanceTypes = orderInstanceTypesByPrice(instanceTypes, scheduling.NewNodeSelectorRequirements(machine.Spec.Requirements...)) if len(instanceTypes) > MaxInstanceTypes { @@ -112,12 +114,12 @@ func (p *InstanceProvider) Create(ctx context.Context, nodeTemplate *v1alpha1.AW "hostname", aws.StringValue(instance.PrivateDnsName), "instance-type", aws.StringValue(instance.InstanceType), "zone", aws.StringValue(instance.Placement.AvailabilityZone), - "capacity-type", getCapacityType(instance)).Infof("launched new instance") + "capacity-type", GetCapacityType(instance)).Infof("launched new instance") return instance, nil } -func (p *InstanceProvider) Link(ctx context.Context, id string) error { +func (p *Provider) Link(ctx context.Context, id string) error { _, err := p.ec2api.CreateTagsWithContext(ctx, &ec2.CreateTagsInput{ Resources: aws.StringSlice([]string{id}), Tags: []*ec2.Tag{ @@ -129,20 +131,20 @@ func (p *InstanceProvider) Link(ctx context.Context, id string) error { }) if err != nil { if awserrors.IsNotFound(err) { - return cloudprovider.NewMachineNotFoundError(fmt.Errorf("linking tags, %w", err)) + return corecloudprovider.NewMachineNotFoundError(fmt.Errorf("linking tags, %w", err)) } return fmt.Errorf("linking tags, %w", err) } return nil } -func (p *InstanceProvider) Get(ctx context.Context, id string) (*ec2.Instance, error) { +func (p *Provider) Get(ctx context.Context, id string) (*ec2.Instance, error) { out, err := p.ec2Batcher.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ InstanceIds: aws.StringSlice([]string{id}), Filters: []*ec2.Filter{instanceStateFilter}, }) if awserrors.IsNotFound(err) { - return nil, cloudprovider.NewMachineNotFoundError(err) + return nil, corecloudprovider.NewMachineNotFoundError(err) } if err != nil { return nil, fmt.Errorf("failed to describe ec2 instances, %w", err) @@ -160,7 +162,7 @@ func (p *InstanceProvider) Get(ctx context.Context, id string) (*ec2.Instance, e return instances[0], nil } -func (p *InstanceProvider) List(ctx context.Context) ([]*ec2.Instance, error) { +func (p *Provider) List(ctx context.Context) ([]*ec2.Instance, error) { // Use the machine name data to determine which instances match this machine out, err := p.ec2api.DescribeInstancesWithContext(ctx, &ec2.DescribeInstancesInput{ Filters: []*ec2.Filter{ @@ -179,18 +181,18 @@ func (p *InstanceProvider) List(ctx context.Context) ([]*ec2.Instance, error) { return nil, fmt.Errorf("describing ec2 instances, %w", err) } instances, err := instancesFromOutput(out) - return instances, cloudprovider.IgnoreMachineNotFoundError(err) + return instances, corecloudprovider.IgnoreMachineNotFoundError(err) } -func (p *InstanceProvider) Delete(ctx context.Context, id string) error { +func (p *Provider) Delete(ctx context.Context, id string) error { if _, err := p.ec2Batcher.TerminateInstances(ctx, &ec2.TerminateInstancesInput{ InstanceIds: []*string{aws.String(id)}, }); err != nil { if awserrors.IsNotFound(err) { - return cloudprovider.NewMachineNotFoundError(fmt.Errorf("instance already terminated")) + return corecloudprovider.NewMachineNotFoundError(fmt.Errorf("instance already terminated")) } if _, e := p.Get(ctx, id); err != nil { - if cloudprovider.IsMachineNotFoundError(e) { + if corecloudprovider.IsMachineNotFoundError(e) { return e } err = multierr.Append(err, e) @@ -200,7 +202,7 @@ func (p *InstanceProvider) Delete(ctx context.Context, id string) error { return nil } -func (p *InstanceProvider) launchInstance(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate, machine *v1alpha5.Machine, instanceTypes []*cloudprovider.InstanceType) (*string, error) { +func (p *Provider) launchInstance(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate, machine *v1alpha5.Machine, instanceTypes []*corecloudprovider.InstanceType) (*string, error) { capacityType := p.getCapacityType(machine, instanceTypes) zonalSubnets, err := p.subnetProvider.ZonalSubnetsForLaunch(ctx, nodeTemplate, instanceTypes, capacityType) if err != nil { @@ -260,7 +262,7 @@ func (p *InstanceProvider) launchInstance(ctx context.Context, nodeTemplate *v1a return createFleetOutput.Instances[0].InstanceIds[0], nil } -func (p *InstanceProvider) checkODFallback(machine *v1alpha5.Machine, instanceTypes []*cloudprovider.InstanceType, launchTemplateConfigs []*ec2.FleetLaunchTemplateConfigRequest) error { +func (p *Provider) checkODFallback(machine *v1alpha5.Machine, instanceTypes []*corecloudprovider.InstanceType, launchTemplateConfigs []*ec2.FleetLaunchTemplateConfigRequest) error { // only evaluate for on-demand fallback if the capacity type for the request is OD and both OD and spot are allowed in requirements if p.getCapacityType(machine, instanceTypes) != v1alpha5.CapacityTypeOnDemand || !scheduling.NewNodeSelectorRequirements(machine.Spec.Requirements...).Get(v1alpha5.LabelCapacityType).Has(v1alpha5.CapacityTypeSpot) { return nil @@ -282,8 +284,8 @@ func (p *InstanceProvider) checkODFallback(machine *v1alpha5.Machine, instanceTy return nil } -func (p *InstanceProvider) getLaunchTemplateConfigs(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate, machine *v1alpha5.Machine, - instanceTypes []*cloudprovider.InstanceType, zonalSubnets map[string]*ec2.Subnet, capacityType string) ([]*ec2.FleetLaunchTemplateConfigRequest, error) { +func (p *Provider) getLaunchTemplateConfigs(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate, machine *v1alpha5.Machine, + instanceTypes []*corecloudprovider.InstanceType, zonalSubnets map[string]*ec2.Subnet, capacityType string) ([]*ec2.FleetLaunchTemplateConfigRequest, error) { var launchTemplateConfigs []*ec2.FleetLaunchTemplateConfigRequest launchTemplates, err := p.launchTemplateProvider.EnsureAll(ctx, nodeTemplate, machine, instanceTypes, map[string]string{v1alpha5.LabelCapacityType: capacityType}) if err != nil { @@ -309,16 +311,16 @@ func (p *InstanceProvider) getLaunchTemplateConfigs(ctx context.Context, nodeTem // getOverrides creates and returns launch template overrides for the cross product of InstanceTypes and subnets (with subnets being constrained by // zones and the offerings in InstanceTypes) -func (p *InstanceProvider) getOverrides(instanceTypes []*cloudprovider.InstanceType, zonalSubnets map[string]*ec2.Subnet, zones *scheduling.Requirement, capacityType string) []*ec2.FleetLaunchTemplateOverridesRequest { +func (p *Provider) getOverrides(instanceTypes []*corecloudprovider.InstanceType, zonalSubnets map[string]*ec2.Subnet, zones *scheduling.Requirement, capacityType string) []*ec2.FleetLaunchTemplateOverridesRequest { // Unwrap all the offerings to a flat slice that includes a pointer // to the parent instance type name type offeringWithParentName struct { - cloudprovider.Offering + corecloudprovider.Offering parentInstanceTypeName string } var unwrappedOfferings []offeringWithParentName for _, it := range instanceTypes { - ofs := lo.Map(it.Offerings.Available(), func(of cloudprovider.Offering, _ int) offeringWithParentName { + ofs := lo.Map(it.Offerings.Available(), func(of corecloudprovider.Offering, _ int) offeringWithParentName { return offeringWithParentName{ Offering: of, parentInstanceTypeName: it.Name, @@ -352,7 +354,7 @@ func (p *InstanceProvider) getOverrides(instanceTypes []*cloudprovider.InstanceT // Update receives a machine and updates the EC2 instance with tags linking it to the machine // Deprecated: This function can be removed when v1alpha6/v1beta1 migration has completed. -func (p *InstanceProvider) Update(ctx context.Context, machine *v1alpha5.Machine) (*ec2.Instance, error) { +func (p *Provider) Update(ctx context.Context, machine *v1alpha5.Machine) (*ec2.Instance, error) { _, err := p.ec2api.CreateTagsWithContext(ctx, &ec2.CreateTagsInput{ Resources: aws.StringSlice([]string{lo.Must(utils.ParseInstanceID(machine.Status.ProviderID))}), Tags: []*ec2.Tag{ @@ -394,7 +396,7 @@ func (p *InstanceProvider) Update(ctx context.Context, machine *v1alpha5.Machine return instance, nil } -func (p *InstanceProvider) updateUnavailableOfferingsCache(ctx context.Context, errors []*ec2.CreateFleetError, capacityType string) { +func (p *Provider) updateUnavailableOfferingsCache(ctx context.Context, errors []*ec2.CreateFleetError, capacityType string) { for _, err := range errors { if awserrors.IsUnfulfillableCapacity(err) { p.unavailableOfferings.MarkUnavailableForFleetErr(ctx, err, capacityType) @@ -405,7 +407,7 @@ func (p *InstanceProvider) updateUnavailableOfferingsCache(ctx context.Context, // getCapacityType selects spot if both constraints are flexible and there is an // available offering. The AWS Cloud Provider defaults to [ on-demand ], so spot // must be explicitly included in capacity type requirements. -func (p *InstanceProvider) getCapacityType(machine *v1alpha5.Machine, instanceTypes []*cloudprovider.InstanceType) string { +func (p *Provider) getCapacityType(machine *v1alpha5.Machine, instanceTypes []*corecloudprovider.InstanceType) string { requirements := scheduling.NewNodeSelectorRequirements(machine. Spec.Requirements...) if requirements.Get(v1alpha5.LabelCapacityType).Has(v1alpha5.CapacityTypeSpot) { @@ -420,7 +422,7 @@ func (p *InstanceProvider) getCapacityType(machine *v1alpha5.Machine, instanceTy return v1alpha5.CapacityTypeOnDemand } -func orderInstanceTypesByPrice(instanceTypes []*cloudprovider.InstanceType, requirements scheduling.Requirements) []*cloudprovider.InstanceType { +func orderInstanceTypesByPrice(instanceTypes []*corecloudprovider.InstanceType, requirements scheduling.Requirements) []*corecloudprovider.InstanceType { // Order instance types so that we get the cheapest instance types of the available offerings sort.Slice(instanceTypes, func(i, j int) bool { iPrice := math.MaxFloat64 @@ -440,8 +442,8 @@ func orderInstanceTypesByPrice(instanceTypes []*cloudprovider.InstanceType, requ } // filterInstanceTypes is used to provide filtering on the list of potential instance types to further limit it to those -// that make the most sense given our specific AWS cloudprovider. -func (p *InstanceProvider) filterInstanceTypes(machine *v1alpha5.Machine, instanceTypes []*cloudprovider.InstanceType) []*cloudprovider.InstanceType { +// that make the most sense given our specific AWS corecloudprovider. +func (p *Provider) filterInstanceTypes(machine *v1alpha5.Machine, instanceTypes []*corecloudprovider.InstanceType) []*corecloudprovider.InstanceType { instanceTypes = filterExoticInstanceTypes(instanceTypes) // If we could potentially launch either a spot or on-demand node, we want to filter out the spot instance types that // are more expensive than the cheapest on-demand type. @@ -453,7 +455,7 @@ func (p *InstanceProvider) filterInstanceTypes(machine *v1alpha5.Machine, instan // isMixedCapacityLaunch returns true if provisioners and available offerings could potentially allow either a spot or // and on-demand node to launch -func (p *InstanceProvider) isMixedCapacityLaunch(machine *v1alpha5.Machine, instanceTypes []*cloudprovider.InstanceType) bool { +func (p *Provider) isMixedCapacityLaunch(machine *v1alpha5.Machine, instanceTypes []*corecloudprovider.InstanceType) bool { requirements := scheduling.NewNodeSelectorRequirements(machine.Spec.Requirements...) // requirements must allow both if !requirements.Get(v1alpha5.LabelCapacityType).Has(v1alpha5.CapacityTypeSpot) || @@ -480,7 +482,7 @@ func (p *InstanceProvider) isMixedCapacityLaunch(machine *v1alpha5.Machine, inst // filterUnwantedSpot is used to filter out spot types that are more expensive than the cheapest on-demand type that we // could launch during mixed capacity-type launches -func filterUnwantedSpot(instanceTypes []*cloudprovider.InstanceType) []*cloudprovider.InstanceType { +func filterUnwantedSpot(instanceTypes []*corecloudprovider.InstanceType) []*corecloudprovider.InstanceType { cheapestOnDemand := math.MaxFloat64 // first, find the price of our cheapest available on-demand instance type that could support this node for _, it := range instanceTypes { @@ -494,7 +496,7 @@ func filterUnwantedSpot(instanceTypes []*cloudprovider.InstanceType) []*cloudpro // Filter out any types where the cheapest offering, which should be spot, is more expensive than the cheapest // on-demand instance type that would have worked. This prevents us from getting a larger more-expensive spot // instance type compared to the cheapest sufficiently large on-demand instance type - instanceTypes = lo.Filter(instanceTypes, func(item *cloudprovider.InstanceType, index int) bool { + instanceTypes = lo.Filter(instanceTypes, func(item *corecloudprovider.InstanceType, index int) bool { available := item.Offerings.Available() if len(available) == 0 { return false @@ -507,8 +509,8 @@ func filterUnwantedSpot(instanceTypes []*cloudprovider.InstanceType) []*cloudpro // filterExoticInstanceTypes is used to eliminate less desirable instance types (like GPUs) from the list of possible instance types when // a set of more appropriate instance types would work. If a set of more desirable instance types is not found, then the original slice // of instance types are returned. -func filterExoticInstanceTypes(instanceTypes []*cloudprovider.InstanceType) []*cloudprovider.InstanceType { - var genericInstanceTypes []*cloudprovider.InstanceType +func filterExoticInstanceTypes(instanceTypes []*corecloudprovider.InstanceType) []*corecloudprovider.InstanceType { + var genericInstanceTypes []*corecloudprovider.InstanceType for _, it := range instanceTypes { // deprioritize metal even if our opinionated filter isn't applied due to something like an instance family // requirement @@ -532,13 +534,13 @@ func filterExoticInstanceTypes(instanceTypes []*cloudprovider.InstanceType) []*c func instancesFromOutput(out *ec2.DescribeInstancesOutput) ([]*ec2.Instance, error) { if len(out.Reservations) == 0 { - return nil, cloudprovider.NewMachineNotFoundError(fmt.Errorf("instance not found")) + return nil, corecloudprovider.NewMachineNotFoundError(fmt.Errorf("instance not found")) } instances := lo.Flatten(lo.Map(out.Reservations, func(r *ec2.Reservation, _ int) []*ec2.Instance { return r.Instances })) if len(instances) == 0 { - return nil, cloudprovider.NewMachineNotFoundError(fmt.Errorf("instance not found")) + return nil, corecloudprovider.NewMachineNotFoundError(fmt.Errorf("instance not found")) } // Get a consistent ordering for instances sort.Slice(instances, func(i, j int) bool { @@ -558,7 +560,7 @@ func combineFleetErrors(errors []*ec2.CreateFleetError) (errs error) { return fmt.Errorf("with fleet error(s), %w", errs) } -func getCapacityType(instance *ec2.Instance) string { +func GetCapacityType(instance *ec2.Instance) string { if instance.SpotInstanceRequestId != nil { return v1alpha5.CapacityTypeSpot } diff --git a/pkg/providers/instancetypes/suite_test.go b/pkg/providers/instancetypes/suite_test.go index f94ee5ebfef7..7bd5357bd60c 100644 --- a/pkg/providers/instancetypes/suite_test.go +++ b/pkg/providers/instancetypes/suite_test.go @@ -64,6 +64,7 @@ import ( "github.com/aws/karpenter/pkg/cloudprovider/amifamily" awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/providers/instancetypes" "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/pricing" @@ -99,6 +100,7 @@ var pricingProvider *pricing.Provider var subnetProvider *subnet.Provider var instanceTypeProvider *instancetypes.Provider var securityGroupProvider *securitygroup.Provider +var instanceProvider *instance.Provider var provisioningController controller.Controller func TestAWS(t *testing.T) { @@ -131,7 +133,6 @@ var _ = BeforeSuite(func() { amiProvider = amifamily.NewAMIProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) amiResolver = amifamily.New(env.Client, amiProvider) instanceTypeProvider = instancetypes.NewProvider(fakeSession, instanceTypeCache, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) - launchTemplateProvider = launchtemplate.NewProvider( ctx, launchTemplateCache, @@ -143,7 +144,7 @@ var _ = BeforeSuite(func() { net.ParseIP("10.0.100.10"), "https://test-cluster", ) - + instanceProvider = instance.NewProvider(ctx, "", fakeEC2API, unavailableOfferingsCache, instanceTypeProvider, subnetProvider, launchTemplateProvider) cloudProvider = cloudprovider.New(awscontext.Context{ Context: corecloudprovider.Context{ Context: ctx, @@ -164,6 +165,7 @@ var _ = BeforeSuite(func() { AMIResolver: amiResolver, LaunchTemplateProvider: launchTemplateProvider, InstanceTypeProvider: instanceTypeProvider, + InstanceProvider: instanceProvider, }) cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) @@ -327,14 +329,14 @@ var _ = Describe("Instance Types", func() { return iPrice < jPrice }) // Expect that the launch template overrides gives the 60 cheapest instance types - expected := sets.NewString(lo.Map(its[:cloudprovider.MaxInstanceTypes], func(i *corecloudprovider.InstanceType, _ int) string { + expected := sets.NewString(lo.Map(its[:instance.MaxInstanceTypes], func(i *corecloudprovider.InstanceType, _ int) string { return i.Name })...) Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) call := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(call.LaunchTemplateConfigs).To(HaveLen(1)) - Expect(call.LaunchTemplateConfigs[0].Overrides).To(HaveLen(cloudprovider.MaxInstanceTypes)) + Expect(call.LaunchTemplateConfigs[0].Overrides).To(HaveLen(instance.MaxInstanceTypes)) for _, override := range call.LaunchTemplateConfigs[0].Overrides { Expect(expected.Has(aws.StringValue(override.InstanceType))).To(BeTrue(), fmt.Sprintf("expected %s to exist in set", aws.StringValue(override.InstanceType))) } diff --git a/pkg/providers/launchtemplate/launchtemplate_test.go b/pkg/providers/launchtemplate/launchtemplate_test.go index 645b91157253..10b36caac4db 100644 --- a/pkg/providers/launchtemplate/launchtemplate_test.go +++ b/pkg/providers/launchtemplate/launchtemplate_test.go @@ -55,6 +55,7 @@ import ( "github.com/aws/karpenter/pkg/cloudprovider/amifamily/bootstrap" awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/providers/instancetypes" "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/pricing" @@ -101,6 +102,7 @@ var pricingProvider *pricing.Provider var subnetProvider *subnet.Provider var instanceTypeProvider *instancetypes.Provider var securityGroupProvider *securitygroup.Provider +var instanceProvider *instance.Provider func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -130,7 +132,6 @@ var _ = BeforeSuite(func() { amiProvider = amifamily.NewAMIProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) amiResolver = amifamily.New(env.Client, amiProvider) instanceTypeProvider = instancetypes.NewProvider(mock.Session, instanceTypeCache, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) - launchTemplateProvider = launchtemplate.NewProvider( ctx, launchTemplateCache, @@ -142,7 +143,7 @@ var _ = BeforeSuite(func() { net.ParseIP("10.0.100.10"), "https://test-cluster", ) - + instanceProvider = instance.NewProvider(ctx, "", fakeEC2API, unavailableOfferingsCache, instanceTypeProvider, subnetProvider, launchTemplateProvider) cloudProvider = cloudprovider.New(awscontext.Context{ Context: corecloudprovider.Context{ Context: ctx, @@ -163,6 +164,7 @@ var _ = BeforeSuite(func() { AMIResolver: amiResolver, LaunchTemplateProvider: launchTemplateProvider, InstanceTypeProvider: instanceTypeProvider, + InstanceProvider: instanceProvider, }) cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) From 598d5f673f982e7e040f210edb4edaf48481362f Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Tue, 28 Feb 2023 14:52:14 -0800 Subject: [PATCH 16/41] Fixed tests --- pkg/providers/instancetypes/suite_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/providers/instancetypes/suite_test.go b/pkg/providers/instancetypes/suite_test.go index f94ee5ebfef7..481f866a80db 100644 --- a/pkg/providers/instancetypes/suite_test.go +++ b/pkg/providers/instancetypes/suite_test.go @@ -61,9 +61,9 @@ import ( "github.com/aws/karpenter/pkg/apis/v1alpha1" awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/cloudprovider" - "github.com/aws/karpenter/pkg/cloudprovider/amifamily" awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/providers/instancetypes" "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/pricing" @@ -86,7 +86,7 @@ var fakeEC2API *fake.EC2API var fakeSSMAPI *fake.SSMAPI var fakeClock *clock.FakeClock var fakePricingAPI *fake.PricingAPI -var amiProvider *amifamily.AMIProvider +var amiProvider *amifamily.Provider var amiResolver *amifamily.Resolver var cloudProvider *cloudprovider.CloudProvider var unavailableOfferingsCache *awscache.UnavailableOfferings @@ -128,7 +128,7 @@ var _ = BeforeSuite(func() { pricingProvider = pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})) subnetProvider = subnet.NewProvider(fakeEC2API) securityGroupProvider = securitygroup.NewProvider(fakeEC2API) - amiProvider = amifamily.NewAMIProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) + amiProvider = amifamily.NewProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) amiResolver = amifamily.New(env.Client, amiProvider) instanceTypeProvider = instancetypes.NewProvider(fakeSession, instanceTypeCache, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) From 2323116d71c018c5598a811b443451c5deeaf0cb Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Wed, 1 Mar 2023 13:36:03 -0800 Subject: [PATCH 17/41] feedback changes --- pkg/cloudprovider/cloudprovider.go | 4 +- pkg/cloudprovider/suite_test.go | 15 +++-- pkg/context/context.go | 6 +- pkg/controllers/machine/link/suite_test.go | 9 +-- pkg/providers/instancetypes/instancetype.go | 8 +-- pkg/providers/instancetypes/instancetypes.go | 8 +-- pkg/providers/instancetypes/suite_test.go | 60 +++++++++---------- .../launchtemplate/launchtemplate_test.go | 18 +++--- 8 files changed, 60 insertions(+), 68 deletions(-) diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index 1c41dd837b3c..ed9d922ecea2 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -73,14 +73,14 @@ type CloudProvider struct { func New(ctx awscontext.Context) *CloudProvider { return &CloudProvider{ kubeClient: ctx.KubeClient, - instanceTypeProvider: ctx.InstanceTypeProvider, + instanceTypeProvider: ctx.InstanceTypesProvider, amiProvider: ctx.AMIProvider, instanceProvider: NewInstanceProvider( ctx, aws.StringValue(ctx.Session.Config.Region), ctx.EC2API, ctx.UnavailableOfferingsCache, - ctx.InstanceTypeProvider, + ctx.InstanceTypesProvider, ctx.SubnetProvider, ctx.LaunchTemplateProvider, ), diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index 817ad2eb2b56..abcbb189e1ac 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -31,7 +31,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/awstesting/mock" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" clock "k8s.io/utils/clock/testing" @@ -84,7 +83,7 @@ var ec2Cache *cache.Cache var kubernetesVersionCache *cache.Cache var unavailableOfferingsCache *awscache.UnavailableOfferings var instanceTypeCache *cache.Cache -var instanceTypeProvider *instancetypes.Provider +var instanceTypesProvider *instancetypes.Provider var launchTemplateProvider *launchtemplate.Provider var amiProvider *amifamily.Provider var fakeEC2API *fake.EC2API @@ -127,8 +126,8 @@ var _ = BeforeSuite(func() { pricingProvider = pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})) amiProvider = amifamily.NewProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) subnetProvider = subnet.NewProvider(fakeEC2API) - instanceTypeProvider = instancetypes.NewProvider( - mock.Session, + instanceTypesProvider = instancetypes.NewProvider( + "", instanceTypeCache, fakeEC2API, subnetProvider, @@ -148,9 +147,9 @@ var _ = BeforeSuite(func() { "https://test-cluster", ) cloudProvider = &CloudProvider{ - instanceTypeProvider: instanceTypeProvider, + instanceTypeProvider: instanceTypesProvider, amiProvider: amiProvider, - instanceProvider: NewInstanceProvider(ctx, "", fakeEC2API, unavailableOfferingsCache, instanceTypeProvider, subnetProvider, launchTemplateProvider), + instanceProvider: NewInstanceProvider(ctx, "", fakeEC2API, unavailableOfferingsCache, instanceTypesProvider, subnetProvider, launchTemplateProvider), kubeClient: env.Client, } fakeClock = clock.NewFakeClock(time.Now()) @@ -214,8 +213,8 @@ var _ = BeforeEach(func() { launchTemplateProvider.ClusterEndpoint = "https://test-cluster" // Reset the pricing provider, so we don't cross-pollinate pricing data - instanceTypeProvider = instancetypes.NewProvider( - mock.Session, + instanceTypesProvider = instancetypes.NewProvider( + "", instanceTypeCache, fakeEC2API, subnetProvider, diff --git a/pkg/context/context.go b/pkg/context/context.go index 9d1a79a1e130..b23591b4c100 100644 --- a/pkg/context/context.go +++ b/pkg/context/context.go @@ -68,7 +68,7 @@ type Context struct { AMIResolver *amifamily.Resolver LaunchTemplateProvider *launchtemplate.Provider PricingProvider *pricing.Provider - InstanceTypeProvider *instancetypes.Provider + InstanceTypesProvider *instancetypes.Provider } func NewOrDie(ctx cloudprovider.Context) Context { @@ -130,7 +130,7 @@ func NewOrDie(ctx cloudprovider.Context) Context { clusterEndpoint, ) instanceTypeProvider := instancetypes.NewProvider( - sess, + *sess.Config.Region, cache.New(awscache.InstanceTypesAndZonesTTL, awscache.DefaultCleanupInterval), ec2api, subnetProvider, @@ -149,7 +149,7 @@ func NewOrDie(ctx cloudprovider.Context) Context { AMIResolver: amiResolver, LaunchTemplateProvider: launchTemplateProvider, PricingProvider: pricingProvider, - InstanceTypeProvider: instanceTypeProvider, + InstanceTypesProvider: instanceTypeProvider, } } diff --git a/pkg/controllers/machine/link/suite_test.go b/pkg/controllers/machine/link/suite_test.go index 89df2a47cc03..1e53a1a0d1b2 100644 --- a/pkg/controllers/machine/link/suite_test.go +++ b/pkg/controllers/machine/link/suite_test.go @@ -65,10 +65,9 @@ var unavailableOfferingsCache *awscache.UnavailableOfferings var ec2API *fake.EC2API var cloudProvider *cloudprovider.CloudProvider var subnetProvider *subnet.Provider -var instanceTypeCache *cache.Cache var linkController controller.Controller var pricingProvider *pricing.Provider -var instanceTypeProvider *instancetypes.Provider +var instanceTypesProvider *instancetypes.Provider func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -82,10 +81,9 @@ var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) unavailableOfferingsCache = awscache.NewUnavailableOfferings() ec2API = &fake.EC2API{} - instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) subnetProvider = subnet.NewProvider(ec2API) pricingProvider = pricing.NewProvider(ctx, &fake.PricingAPI{}, ec2API, "", make(chan struct{})) - instanceTypeProvider = instancetypes.NewProvider(mock.Session, instanceTypeCache, ec2API, subnetProvider, unavailableOfferingsCache, pricingProvider) + instanceTypesProvider = instancetypes.NewProvider("", cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), ec2API, subnetProvider, unavailableOfferingsCache, pricingProvider) cloudProvider = cloudprovider.New(awscontext.Context{ Context: corecloudprovider.Context{ Context: ctx, @@ -102,7 +100,7 @@ var _ = BeforeSuite(func() { UnavailableOfferingsCache: unavailableOfferingsCache, EC2API: ec2API, PricingProvider: pricingProvider, - InstanceTypeProvider: instanceTypeProvider, + InstanceTypesProvider: instanceTypesProvider, }) linkController = link.NewController(env.Client, cloudProvider) }) @@ -119,7 +117,6 @@ var _ = Describe("MachineLink", func() { BeforeEach(func() { ec2API.Reset() - instanceTypeCache.Flush() instanceID = fake.InstanceID() providerID = fmt.Sprintf("aws:///test-zone-1a/%s", instanceID) nodeTemplate = test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{}) diff --git a/pkg/providers/instancetypes/instancetype.go b/pkg/providers/instancetypes/instancetype.go index f16c18f52edf..aac4182a9eb9 100644 --- a/pkg/providers/instancetypes/instancetype.go +++ b/pkg/providers/instancetypes/instancetype.go @@ -47,7 +47,7 @@ var ( instanceTypeScheme = regexp.MustCompile(`(^[a-z]+)(\-[0-9]+tb)?([0-9]+).*\.`) ) -func NewInstanceType(ctx context.Context, info *ec2.InstanceTypeInfo, kc *v1alpha5.KubeletConfiguration, +func New(ctx context.Context, info *ec2.InstanceTypeInfo, kc *v1alpha5.KubeletConfiguration, region string, nodeTemplate *v1alpha1.AWSNodeTemplate, offerings cloudprovider.Offerings) *cloudprovider.InstanceType { amiFamily := amifamily.GetAMIFamily(nodeTemplate.Spec.AMIFamily, &amifamily.Options{}) @@ -57,7 +57,7 @@ func NewInstanceType(ctx context.Context, info *ec2.InstanceTypeInfo, kc *v1alph Offerings: offerings, Capacity: computeCapacity(ctx, info, amiFamily, nodeTemplate.Spec.BlockDeviceMappings, kc), Overhead: &cloudprovider.InstanceTypeOverhead{ - KubeReserved: kubeReservedResources(cpu(info), pods(ctx, info, amiFamily, kc), EniLimitedPods(info), amiFamily, kc), + KubeReserved: kubeReservedResources(cpu(info), pods(ctx, info, amiFamily, kc), eniLimitedPods(info), amiFamily, kc), SystemReserved: systemReservedResources(kc), EvictionThreshold: evictionThreshold(memory(ctx, info), amiFamily, kc), }, @@ -234,7 +234,7 @@ func habanaGaudis(info *ec2.InstanceTypeInfo) *resource.Quantity { // The number of pods per node is calculated using the formula: // max number of ENIs * (IPv4 Addresses per ENI -1) + 2 // https://github.com/awslabs/amazon-eks-ami/blob/master/files/eni-max-pods.txt#L20 -func EniLimitedPods(info *ec2.InstanceTypeInfo) *resource.Quantity { +func eniLimitedPods(info *ec2.InstanceTypeInfo) *resource.Quantity { return resources.Quantity(fmt.Sprint(*info.NetworkInfo.MaximumNetworkInterfaces*(*info.NetworkInfo.Ipv4AddressesPerInterface-1) + 2)) } @@ -331,7 +331,7 @@ func pods(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily amifamily.A case !awssettings.FromContext(ctx).EnableENILimitedPodDensity: count = 110 default: - count = EniLimitedPods(info).Value() + count = eniLimitedPods(info).Value() } if kc != nil && ptr.Int32Value(kc.PodsPerCore) > 0 && amiFamily.FeatureFlags().PodsPerCoreEnabled { count = lo.Min([]int64{int64(ptr.Int32Value(kc.PodsPerCore)) * ptr.Int64Value(info.VCpuInfo.DefaultVCpus), count}) diff --git a/pkg/providers/instancetypes/instancetypes.go b/pkg/providers/instancetypes/instancetypes.go index b54b927fdf27..61f61a7953ef 100644 --- a/pkg/providers/instancetypes/instancetypes.go +++ b/pkg/providers/instancetypes/instancetypes.go @@ -21,8 +21,6 @@ import ( "sync" "sync/atomic" - "github.com/aws/aws-sdk-go/aws/session" - awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/aws-sdk-go/aws" @@ -68,11 +66,11 @@ type Provider struct { instanceTypesSeqNum uint64 } -func NewProvider(sess *session.Session, cache *cache.Cache, ec2api ec2iface.EC2API, subnetProvider *subnet.Provider, +func NewProvider(region string, cache *cache.Cache, ec2api ec2iface.EC2API, subnetProvider *subnet.Provider, unavailableOfferingsCache *awscache.UnavailableOfferings, pricingProvider *pricing.Provider) *Provider { return &Provider{ ec2api: ec2api, - region: *sess.Config.Region, + region: region, subnetProvider: subnetProvider, pricingProvider: pricingProvider, cache: cache, @@ -103,7 +101,7 @@ func (p *Provider) List(ctx context.Context, kc *v1alpha5.KubeletConfiguration, return item.([]*cloudprovider.InstanceType), nil } result := lo.Map(instanceTypes, func(i *ec2.InstanceTypeInfo, _ int) *cloudprovider.InstanceType { - return NewInstanceType(ctx, i, kc, p.region, nodeTemplate, p.createOfferings(ctx, i, instanceTypeZones[aws.StringValue(i.InstanceType)])) + return New(ctx, i, kc, p.region, nodeTemplate, p.createOfferings(ctx, i, instanceTypeZones[aws.StringValue(i.InstanceType)])) }) p.cache.SetDefault(key, result) return result, nil diff --git a/pkg/providers/instancetypes/suite_test.go b/pkg/providers/instancetypes/suite_test.go index 481f866a80db..eda3a7f33e60 100644 --- a/pkg/providers/instancetypes/suite_test.go +++ b/pkg/providers/instancetypes/suite_test.go @@ -25,7 +25,6 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/awstesting/mock" "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" @@ -53,6 +52,7 @@ import ( "github.com/aws/karpenter-core/pkg/operator/scheme" "github.com/aws/karpenter-core/pkg/scheduling" . "github.com/aws/karpenter-core/pkg/test/expectations" + "github.com/aws/karpenter-core/pkg/utils/resources" coresettings "github.com/aws/karpenter-core/pkg/apis/settings" coretest "github.com/aws/karpenter-core/pkg/test" @@ -76,7 +76,6 @@ var ctx context.Context var stop context.CancelFunc var opts options.Options var env *coretest.Environment -var fakeSession *session.Session var ssmCache *cache.Cache var ec2Cache *cache.Cache var launchTemplateCache *cache.Cache @@ -113,8 +112,6 @@ var _ = BeforeSuite(func() { ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) - fakeSession = mock.Session - fakeSession.Config.Region = aws.String("") fakeEC2API = &fake.EC2API{} fakeSSMAPI = &fake.SSMAPI{} ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) @@ -130,7 +127,7 @@ var _ = BeforeSuite(func() { securityGroupProvider = securitygroup.NewProvider(fakeEC2API) amiProvider = amifamily.NewProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) amiResolver = amifamily.New(env.Client, amiProvider) - instanceTypeProvider = instancetypes.NewProvider(fakeSession, instanceTypeCache, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) + instanceTypeProvider = instancetypes.NewProvider("", instanceTypeCache, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) launchTemplateProvider = launchtemplate.NewProvider( ctx, @@ -156,14 +153,14 @@ var _ = BeforeSuite(func() { }, SubnetProvider: subnet.NewProvider(fakeEC2API), SecurityGroupProvider: securityGroupProvider, - Session: fakeSession, + Session: mock.Session, UnavailableOfferingsCache: unavailableOfferingsCache, EC2API: fakeEC2API, PricingProvider: pricingProvider, AMIProvider: amiProvider, AMIResolver: amiResolver, LaunchTemplateProvider: launchTemplateProvider, - InstanceTypeProvider: instanceTypeProvider, + InstanceTypesProvider: instanceTypeProvider, }) cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) @@ -225,7 +222,7 @@ var _ = BeforeEach(func() { // Reset the pricing provider, so we don't cross-pollinate pricing data instanceTypeProvider = instancetypes.NewProvider( - fakeSession, + "", instanceTypeCache, fakeEC2API, subnetProvider, @@ -597,7 +594,7 @@ var _ = Describe("Instance Types", func() { instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) for _, info := range instanceInfo { - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 110)) } }) @@ -605,7 +602,7 @@ var _ = Describe("Instance Types", func() { instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) for _, info := range instanceInfo { - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).ToNot(BeNumerically("==", 110)) } }) @@ -634,7 +631,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.SystemReserved.Cpu().String()).To(Equal("2")) }) It("should override system reserved memory when specified", func() { @@ -645,7 +642,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.SystemReserved.Memory().String()).To(Equal("20Gi")) }) It("should override kube reserved when specified", func() { @@ -663,7 +660,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.KubeReserved.Cpu().String()).To(Equal("2")) Expect(it.Overhead.KubeReserved.Memory().String()).To(Equal("10Gi")) Expect(it.Overhead.KubeReserved.StorageEphemeral().String()).To(Equal("2Gi")) @@ -689,7 +686,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("500Mi")) }) It("should override eviction threshold (hard) when specified as a percentage value", func() { @@ -706,7 +703,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) It("should consider the eviction threshold (hard) disabled when specified as 100%", func() { @@ -723,7 +720,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("0")) }) It("should used default eviction threshold (hard) for memory when evictionHard not specified", func() { @@ -740,7 +737,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("50Mi")) }) It("should override eviction threshold (soft) when specified as a quantity", func() { @@ -757,7 +754,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("500Mi")) }) It("should override eviction threshold (soft) when specified as a percentage value", func() { @@ -777,7 +774,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) It("should consider the eviction threshold (soft) disabled when specified as 100%", func() { @@ -794,7 +791,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("0")) }) It("should ignore eviction threshold (soft) when using Bottlerocket AMI", func() { @@ -815,7 +812,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("1Gi")) }) It("should take the greater of evictionHard and evictionSoft for overhead as a value", func() { @@ -835,7 +832,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("3Gi")) }) It("should take the greater of evictionHard and evictionSoft for overhead as a value", func() { @@ -855,7 +852,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.05, 10)) }) It("should take the greater of evictionHard and evictionSoft for overhead with mixed percentage/value", func() { @@ -875,7 +872,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) }) @@ -884,7 +881,7 @@ var _ = Describe("Instance Types", func() { Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{MaxPods: ptr.Int32(10)}}) for _, info := range instanceInfo { - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 10)) } }) @@ -897,7 +894,7 @@ var _ = Describe("Instance Types", func() { Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{MaxPods: ptr.Int32(10)}}) for _, info := range instanceInfo { - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 10)) } }) @@ -906,7 +903,7 @@ var _ = Describe("Instance Types", func() { Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}}) for _, info := range instanceInfo { - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", ptr.Int64Value(info.VCpuInfo.DefaultVCpus))) } }) @@ -915,7 +912,7 @@ var _ = Describe("Instance Types", func() { Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(4), MaxPods: ptr.Int32(20)}}) for _, info := range instanceInfo { - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", lo.Min([]int64{20, ptr.Int64Value(info.VCpuInfo.DefaultVCpus) * 4}))) } }) @@ -925,8 +922,9 @@ var _ = Describe("Instance Types", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}}) for _, info := range instanceInfo { - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) - Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", instancetypes.EniLimitedPods(info).Value())) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + limitedPods := resources.Quantity(fmt.Sprint(*info.NetworkInfo.MaximumNetworkInterfaces*(*info.NetworkInfo.Ipv4AddressesPerInterface-1) + 2)) + Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", limitedPods.Value())) } }) It("should take 110 to be the default pods number when pods-per-core is 0 and AWSENILimitedPodDensity is unset", func() { @@ -938,7 +936,7 @@ var _ = Describe("Instance Types", func() { Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(0)}}) for _, info := range instanceInfo { - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 110)) } }) diff --git a/pkg/providers/launchtemplate/launchtemplate_test.go b/pkg/providers/launchtemplate/launchtemplate_test.go index 6ab51a5539a4..b3c0d7656448 100644 --- a/pkg/providers/launchtemplate/launchtemplate_test.go +++ b/pkg/providers/launchtemplate/launchtemplate_test.go @@ -99,7 +99,7 @@ var nodeTemplate *v1alpha1.AWSNodeTemplate var cluster *state.Cluster var pricingProvider *pricing.Provider var subnetProvider *subnet.Provider -var instanceTypeProvider *instancetypes.Provider +var instanceTypesProvider *instancetypes.Provider var securityGroupProvider *securitygroup.Provider func TestAWS(t *testing.T) { @@ -129,7 +129,7 @@ var _ = BeforeSuite(func() { securityGroupProvider = securitygroup.NewProvider(fakeEC2API) amiProvider = amifamily.NewProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) amiResolver = amifamily.New(env.Client, amiProvider) - instanceTypeProvider = instancetypes.NewProvider(mock.Session, instanceTypeCache, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) + instanceTypesProvider = instancetypes.NewProvider("", instanceTypeCache, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) launchTemplateProvider = launchtemplate.NewProvider( ctx, @@ -162,7 +162,7 @@ var _ = BeforeSuite(func() { AMIProvider: amiProvider, AMIResolver: amiResolver, LaunchTemplateProvider: launchTemplateProvider, - InstanceTypeProvider: instanceTypeProvider, + InstanceTypesProvider: instanceTypesProvider, }) cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) @@ -221,8 +221,8 @@ var _ = BeforeEach(func() { launchTemplateProvider.ClusterEndpoint = "https://test-cluster" // Reset the pricing provider, so we don't cross-pollinate pricing data - instanceTypeProvider = instancetypes.NewProvider( - mock.Session, + instanceTypesProvider = instancetypes.NewProvider( + "", instanceTypeCache, fakeEC2API, subnetProvider, @@ -893,7 +893,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1093Mi")) }) @@ -904,7 +904,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1093Mi")) }) @@ -943,7 +943,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1093Mi")) }) @@ -954,7 +954,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1665Mi")) }) From 1d190278984647530196592cc1a6f167066cecd4 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Wed, 1 Mar 2023 15:55:20 -0800 Subject: [PATCH 18/41] feedback changes --- pkg/providers/instancetypes/instancetypes.go | 2 +- pkg/providers/instancetypes/suite_test.go | 44 +++++++++---------- .../{instancetype.go => types.go} | 2 +- .../launchtemplate/launchtemplate_test.go | 8 ++-- 4 files changed, 28 insertions(+), 28 deletions(-) rename pkg/providers/instancetypes/{instancetype.go => types.go} (99%) diff --git a/pkg/providers/instancetypes/instancetypes.go b/pkg/providers/instancetypes/instancetypes.go index 61f61a7953ef..6a5e667b3243 100644 --- a/pkg/providers/instancetypes/instancetypes.go +++ b/pkg/providers/instancetypes/instancetypes.go @@ -101,7 +101,7 @@ func (p *Provider) List(ctx context.Context, kc *v1alpha5.KubeletConfiguration, return item.([]*cloudprovider.InstanceType), nil } result := lo.Map(instanceTypes, func(i *ec2.InstanceTypeInfo, _ int) *cloudprovider.InstanceType { - return New(ctx, i, kc, p.region, nodeTemplate, p.createOfferings(ctx, i, instanceTypeZones[aws.StringValue(i.InstanceType)])) + return NewInstanceType(ctx, i, kc, p.region, nodeTemplate, p.createOfferings(ctx, i, instanceTypeZones[aws.StringValue(i.InstanceType)])) }) p.cache.SetDefault(key, result) return result, nil diff --git a/pkg/providers/instancetypes/suite_test.go b/pkg/providers/instancetypes/suite_test.go index eda3a7f33e60..22f4d74481ee 100644 --- a/pkg/providers/instancetypes/suite_test.go +++ b/pkg/providers/instancetypes/suite_test.go @@ -594,7 +594,7 @@ var _ = Describe("Instance Types", func() { instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) for _, info := range instanceInfo { - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 110)) } }) @@ -602,7 +602,7 @@ var _ = Describe("Instance Types", func() { instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) for _, info := range instanceInfo { - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).ToNot(BeNumerically("==", 110)) } }) @@ -631,7 +631,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.SystemReserved.Cpu().String()).To(Equal("2")) }) It("should override system reserved memory when specified", func() { @@ -642,7 +642,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.SystemReserved.Memory().String()).To(Equal("20Gi")) }) It("should override kube reserved when specified", func() { @@ -660,7 +660,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.KubeReserved.Cpu().String()).To(Equal("2")) Expect(it.Overhead.KubeReserved.Memory().String()).To(Equal("10Gi")) Expect(it.Overhead.KubeReserved.StorageEphemeral().String()).To(Equal("2Gi")) @@ -686,7 +686,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("500Mi")) }) It("should override eviction threshold (hard) when specified as a percentage value", func() { @@ -703,7 +703,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) It("should consider the eviction threshold (hard) disabled when specified as 100%", func() { @@ -720,7 +720,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("0")) }) It("should used default eviction threshold (hard) for memory when evictionHard not specified", func() { @@ -737,7 +737,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("50Mi")) }) It("should override eviction threshold (soft) when specified as a quantity", func() { @@ -754,7 +754,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("500Mi")) }) It("should override eviction threshold (soft) when specified as a percentage value", func() { @@ -774,7 +774,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) It("should consider the eviction threshold (soft) disabled when specified as 100%", func() { @@ -791,7 +791,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("0")) }) It("should ignore eviction threshold (soft) when using Bottlerocket AMI", func() { @@ -812,7 +812,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("1Gi")) }) It("should take the greater of evictionHard and evictionSoft for overhead as a value", func() { @@ -832,7 +832,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("3Gi")) }) It("should take the greater of evictionHard and evictionSoft for overhead as a value", func() { @@ -852,7 +852,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.05, 10)) }) It("should take the greater of evictionHard and evictionSoft for overhead with mixed percentage/value", func() { @@ -872,7 +872,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) }) @@ -881,7 +881,7 @@ var _ = Describe("Instance Types", func() { Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{MaxPods: ptr.Int32(10)}}) for _, info := range instanceInfo { - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 10)) } }) @@ -894,7 +894,7 @@ var _ = Describe("Instance Types", func() { Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{MaxPods: ptr.Int32(10)}}) for _, info := range instanceInfo { - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 10)) } }) @@ -903,7 +903,7 @@ var _ = Describe("Instance Types", func() { Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}}) for _, info := range instanceInfo { - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", ptr.Int64Value(info.VCpuInfo.DefaultVCpus))) } }) @@ -912,7 +912,7 @@ var _ = Describe("Instance Types", func() { Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(4), MaxPods: ptr.Int32(20)}}) for _, info := range instanceInfo { - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", lo.Min([]int64{20, ptr.Int64Value(info.VCpuInfo.DefaultVCpus) * 4}))) } }) @@ -922,7 +922,7 @@ var _ = Describe("Instance Types", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}}) for _, info := range instanceInfo { - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) limitedPods := resources.Quantity(fmt.Sprint(*info.NetworkInfo.MaximumNetworkInterfaces*(*info.NetworkInfo.Ipv4AddressesPerInterface-1) + 2)) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", limitedPods.Value())) } @@ -936,7 +936,7 @@ var _ = Describe("Instance Types", func() { Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(0)}}) for _, info := range instanceInfo { - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 110)) } }) diff --git a/pkg/providers/instancetypes/instancetype.go b/pkg/providers/instancetypes/types.go similarity index 99% rename from pkg/providers/instancetypes/instancetype.go rename to pkg/providers/instancetypes/types.go index aac4182a9eb9..b97d1fd65921 100644 --- a/pkg/providers/instancetypes/instancetype.go +++ b/pkg/providers/instancetypes/types.go @@ -47,7 +47,7 @@ var ( instanceTypeScheme = regexp.MustCompile(`(^[a-z]+)(\-[0-9]+tb)?([0-9]+).*\.`) ) -func New(ctx context.Context, info *ec2.InstanceTypeInfo, kc *v1alpha5.KubeletConfiguration, +func NewInstanceType(ctx context.Context, info *ec2.InstanceTypeInfo, kc *v1alpha5.KubeletConfiguration, region string, nodeTemplate *v1alpha1.AWSNodeTemplate, offerings cloudprovider.Offerings) *cloudprovider.InstanceType { amiFamily := amifamily.GetAMIFamily(nodeTemplate.Spec.AMIFamily, &amifamily.Options{}) diff --git a/pkg/providers/launchtemplate/launchtemplate_test.go b/pkg/providers/launchtemplate/launchtemplate_test.go index b3c0d7656448..2adb1af93ccf 100644 --- a/pkg/providers/launchtemplate/launchtemplate_test.go +++ b/pkg/providers/launchtemplate/launchtemplate_test.go @@ -893,7 +893,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1093Mi")) }) @@ -904,7 +904,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1093Mi")) }) @@ -943,7 +943,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1093Mi")) }) @@ -954,7 +954,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - it := instancetypes.New(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1665Mi")) }) From 2af7d8fd42e2650f1d10f47e07c95f1886e37da1 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Wed, 1 Mar 2023 16:09:07 -0800 Subject: [PATCH 19/41] merge missed test --- pkg/controllers/machine/garbagecollect/suite_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controllers/machine/garbagecollect/suite_test.go b/pkg/controllers/machine/garbagecollect/suite_test.go index 0b556f61b8e6..e80c624f4b43 100644 --- a/pkg/controllers/machine/garbagecollect/suite_test.go +++ b/pkg/controllers/machine/garbagecollect/suite_test.go @@ -105,7 +105,7 @@ var _ = BeforeSuite(func() { amiResolver = amifamily.New(env.Client, amiProvider) pricingProvider = pricing.NewProvider(ctx, &fake.PricingAPI{}, ec2API, "", make(chan struct{})) instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - instanceTypeProvider = instancetypes.NewProvider(mock.Session, instanceTypeCache, ec2API, subnetProvider, unavailableOfferingsCache, pricingProvider) + instanceTypeProvider = instancetypes.NewProvider("", instanceTypeCache, ec2API, subnetProvider, unavailableOfferingsCache, pricingProvider) launchTemplateProvider = launchtemplate.NewProvider( ctx, launchTemplateCache, From b66b18d3c9d48d198fc19cf924e13efdcc94f2cf Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Wed, 1 Mar 2023 16:19:22 -0800 Subject: [PATCH 20/41] Folder name change --- pkg/cloudprovider/cloudprovider.go | 4 +- pkg/cloudprovider/instance.go | 6 +- pkg/cloudprovider/suite_test.go | 17 ++-- pkg/context/context.go | 6 +- pkg/controllers/machine/link/suite_test.go | 6 +- .../instancetypes.go | 2 +- .../suite_test.go | 88 +++++++++---------- .../{instancetypes => instancetype}/types.go | 2 +- .../zz_generated.bandwidth.go | 2 +- .../zz_generated.vpclimits.go | 2 +- .../launchtemplate/launchtemplate_test.go | 16 ++-- 11 files changed, 75 insertions(+), 76 deletions(-) rename pkg/providers/{instancetypes => instancetype}/instancetypes.go (99%) rename pkg/providers/{instancetypes => instancetype}/suite_test.go (94%) rename pkg/providers/{instancetypes => instancetype}/types.go (99%) rename pkg/providers/{instancetypes => instancetype}/zz_generated.bandwidth.go (99%) rename pkg/providers/{instancetypes => instancetype}/zz_generated.vpclimits.go (99%) diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index ed9d922ecea2..2c3335193c8e 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -28,7 +28,6 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" - "github.com/aws/karpenter/pkg/providers/instancetypes" "github.com/aws/karpenter/pkg/utils" "github.com/aws/karpenter-core/pkg/scheduling" @@ -45,6 +44,7 @@ import ( awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/providers/amifamily" + "github.com/aws/karpenter/pkg/providers/instancetype" coreapis "github.com/aws/karpenter-core/pkg/apis" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" @@ -64,7 +64,7 @@ func init() { var _ cloudprovider.CloudProvider = (*CloudProvider)(nil) type CloudProvider struct { - instanceTypeProvider *instancetypes.Provider + instanceTypeProvider *instancetype.Provider instanceProvider *InstanceProvider kubeClient client.Client amiProvider *amifamily.Provider diff --git a/pkg/cloudprovider/instance.go b/pkg/cloudprovider/instance.go index d58c116f6bda..cb0d82a8ea00 100644 --- a/pkg/cloudprovider/instance.go +++ b/pkg/cloudprovider/instance.go @@ -38,7 +38,7 @@ import ( "github.com/aws/karpenter/pkg/batcher" "github.com/aws/karpenter/pkg/cache" awserrors "github.com/aws/karpenter/pkg/errors" - "github.com/aws/karpenter/pkg/providers/instancetypes" + "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/subnet" "github.com/aws/karpenter/pkg/utils" @@ -63,13 +63,13 @@ type InstanceProvider struct { region string ec2api ec2iface.EC2API unavailableOfferings *cache.UnavailableOfferings - instanceTypeProvider *instancetypes.Provider + instanceTypeProvider *instancetype.Provider subnetProvider *subnet.Provider launchTemplateProvider *launchtemplate.Provider ec2Batcher *batcher.EC2API } -func NewInstanceProvider(ctx context.Context, region string, ec2api ec2iface.EC2API, unavailableOfferings *cache.UnavailableOfferings, instanceTypeProvider *instancetypes.Provider, subnetProvider *subnet.Provider, launchTemplateProvider *launchtemplate.Provider) *InstanceProvider { +func NewInstanceProvider(ctx context.Context, region string, ec2api ec2iface.EC2API, unavailableOfferings *cache.UnavailableOfferings, instanceTypeProvider *instancetype.Provider, subnetProvider *subnet.Provider, launchTemplateProvider *launchtemplate.Provider) *InstanceProvider { return &InstanceProvider{ region: region, ec2api: ec2api, diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index abcbb189e1ac..028dc4247705 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -45,6 +45,11 @@ import ( "github.com/aws/karpenter/pkg/apis/v1alpha1" awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/providers/amifamily" + "github.com/aws/karpenter/pkg/providers/instancetype" + "github.com/aws/karpenter/pkg/providers/launchtemplate" + "github.com/aws/karpenter/pkg/providers/pricing" + "github.com/aws/karpenter/pkg/providers/securitygroup" + "github.com/aws/karpenter/pkg/providers/subnet" "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter-core/pkg/cloudprovider" @@ -65,12 +70,6 @@ import ( "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" - - "github.com/aws/karpenter/pkg/providers/instancetypes" - "github.com/aws/karpenter/pkg/providers/launchtemplate" - "github.com/aws/karpenter/pkg/providers/pricing" - "github.com/aws/karpenter/pkg/providers/securitygroup" - "github.com/aws/karpenter/pkg/providers/subnet" ) var ctx context.Context @@ -83,7 +82,7 @@ var ec2Cache *cache.Cache var kubernetesVersionCache *cache.Cache var unavailableOfferingsCache *awscache.UnavailableOfferings var instanceTypeCache *cache.Cache -var instanceTypesProvider *instancetypes.Provider +var instanceTypesProvider *instancetype.Provider var launchTemplateProvider *launchtemplate.Provider var amiProvider *amifamily.Provider var fakeEC2API *fake.EC2API @@ -126,7 +125,7 @@ var _ = BeforeSuite(func() { pricingProvider = pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})) amiProvider = amifamily.NewProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) subnetProvider = subnet.NewProvider(fakeEC2API) - instanceTypesProvider = instancetypes.NewProvider( + instanceTypesProvider = instancetype.NewProvider( "", instanceTypeCache, fakeEC2API, @@ -213,7 +212,7 @@ var _ = BeforeEach(func() { launchTemplateProvider.ClusterEndpoint = "https://test-cluster" // Reset the pricing provider, so we don't cross-pollinate pricing data - instanceTypesProvider = instancetypes.NewProvider( + instanceTypesProvider = instancetype.NewProvider( "", instanceTypeCache, fakeEC2API, diff --git a/pkg/context/context.go b/pkg/context/context.go index b23591b4c100..430c2ab1a5fd 100644 --- a/pkg/context/context.go +++ b/pkg/context/context.go @@ -45,7 +45,7 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/providers/amifamily" - "github.com/aws/karpenter/pkg/providers/instancetypes" + "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/providers/securitygroup" @@ -68,7 +68,7 @@ type Context struct { AMIResolver *amifamily.Resolver LaunchTemplateProvider *launchtemplate.Provider PricingProvider *pricing.Provider - InstanceTypesProvider *instancetypes.Provider + InstanceTypesProvider *instancetype.Provider } func NewOrDie(ctx cloudprovider.Context) Context { @@ -129,7 +129,7 @@ func NewOrDie(ctx cloudprovider.Context) Context { kubeDNSIP, clusterEndpoint, ) - instanceTypeProvider := instancetypes.NewProvider( + instanceTypeProvider := instancetype.NewProvider( *sess.Config.Region, cache.New(awscache.InstanceTypesAndZonesTTL, awscache.DefaultCleanupInterval), ec2api, diff --git a/pkg/controllers/machine/link/suite_test.go b/pkg/controllers/machine/link/suite_test.go index 1e53a1a0d1b2..0298af48293c 100644 --- a/pkg/controllers/machine/link/suite_test.go +++ b/pkg/controllers/machine/link/suite_test.go @@ -51,7 +51,7 @@ import ( awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/controllers/machine/link" "github.com/aws/karpenter/pkg/fake" - "github.com/aws/karpenter/pkg/providers/instancetypes" + "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/providers/securitygroup" "github.com/aws/karpenter/pkg/providers/subnet" @@ -67,7 +67,7 @@ var cloudProvider *cloudprovider.CloudProvider var subnetProvider *subnet.Provider var linkController controller.Controller var pricingProvider *pricing.Provider -var instanceTypesProvider *instancetypes.Provider +var instanceTypesProvider *instancetype.Provider func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -83,7 +83,7 @@ var _ = BeforeSuite(func() { ec2API = &fake.EC2API{} subnetProvider = subnet.NewProvider(ec2API) pricingProvider = pricing.NewProvider(ctx, &fake.PricingAPI{}, ec2API, "", make(chan struct{})) - instanceTypesProvider = instancetypes.NewProvider("", cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), ec2API, subnetProvider, unavailableOfferingsCache, pricingProvider) + instanceTypesProvider = instancetype.NewProvider("", cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), ec2API, subnetProvider, unavailableOfferingsCache, pricingProvider) cloudProvider = cloudprovider.New(awscontext.Context{ Context: corecloudprovider.Context{ Context: ctx, diff --git a/pkg/providers/instancetypes/instancetypes.go b/pkg/providers/instancetype/instancetypes.go similarity index 99% rename from pkg/providers/instancetypes/instancetypes.go rename to pkg/providers/instancetype/instancetypes.go index 6a5e667b3243..4d2df86f8bda 100644 --- a/pkg/providers/instancetypes/instancetypes.go +++ b/pkg/providers/instancetype/instancetypes.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package instancetypes +package instancetype import ( "context" diff --git a/pkg/providers/instancetypes/suite_test.go b/pkg/providers/instancetype/suite_test.go similarity index 94% rename from pkg/providers/instancetypes/suite_test.go rename to pkg/providers/instancetype/suite_test.go index 22f4d74481ee..e30e43c9a643 100644 --- a/pkg/providers/instancetypes/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package instancetypes_test +package instancetype_test import ( "context" @@ -64,7 +64,7 @@ import ( awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/providers/amifamily" - "github.com/aws/karpenter/pkg/providers/instancetypes" + "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/providers/securitygroup" @@ -96,7 +96,7 @@ var nodeTemplate *v1alpha1.AWSNodeTemplate var cluster *state.Cluster var pricingProvider *pricing.Provider var subnetProvider *subnet.Provider -var instanceTypeProvider *instancetypes.Provider +var instanceTypeProvider *instancetype.Provider var securityGroupProvider *securitygroup.Provider var provisioningController controller.Controller @@ -127,7 +127,7 @@ var _ = BeforeSuite(func() { securityGroupProvider = securitygroup.NewProvider(fakeEC2API) amiProvider = amifamily.NewProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) amiResolver = amifamily.New(env.Client, amiProvider) - instanceTypeProvider = instancetypes.NewProvider("", instanceTypeCache, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) + instanceTypeProvider = instancetype.NewProvider("", instanceTypeCache, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) launchTemplateProvider = launchtemplate.NewProvider( ctx, @@ -221,7 +221,7 @@ var _ = BeforeEach(func() { launchTemplateProvider.ClusterEndpoint = "https://test-cluster" // Reset the pricing provider, so we don't cross-pollinate pricing data - instanceTypeProvider = instancetypes.NewProvider( + instanceTypeProvider = instancetype.NewProvider( "", instanceTypeCache, fakeEC2API, @@ -485,7 +485,7 @@ var _ = Describe("Instance Types", func() { node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKey(v1.LabelInstanceTypeStable)) supportsPodENI := func() bool { - limits, ok := instancetypes.Limits[node.Labels[v1.LabelInstanceTypeStable]] + limits, ok := instancetype.Limits[node.Labels[v1.LabelInstanceTypeStable]] return ok && limits.IsTrunkingCompatible } Expect(supportsPodENI()).To(Equal(true)) @@ -594,7 +594,7 @@ var _ = Describe("Instance Types", func() { instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) for _, info := range instanceInfo { - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 110)) } }) @@ -602,7 +602,7 @@ var _ = Describe("Instance Types", func() { instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) for _, info := range instanceInfo { - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).ToNot(BeNumerically("==", 110)) } }) @@ -631,7 +631,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.SystemReserved.Cpu().String()).To(Equal("2")) }) It("should override system reserved memory when specified", func() { @@ -642,7 +642,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.SystemReserved.Memory().String()).To(Equal("20Gi")) }) It("should override kube reserved when specified", func() { @@ -660,7 +660,7 @@ var _ = Describe("Instance Types", func() { }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.KubeReserved.Cpu().String()).To(Equal("2")) Expect(it.Overhead.KubeReserved.Memory().String()).To(Equal("10Gi")) Expect(it.Overhead.KubeReserved.StorageEphemeral().String()).To(Equal("2Gi")) @@ -682,11 +682,11 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionHard: map[string]string{ - instancetypes.MemoryAvailable: "500Mi", + instancetype.MemoryAvailable: "500Mi", }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("500Mi")) }) It("should override eviction threshold (hard) when specified as a percentage value", func() { @@ -699,11 +699,11 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionHard: map[string]string{ - instancetypes.MemoryAvailable: "10%", + instancetype.MemoryAvailable: "10%", }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) It("should consider the eviction threshold (hard) disabled when specified as 100%", func() { @@ -716,11 +716,11 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionHard: map[string]string{ - instancetypes.MemoryAvailable: "100%", + instancetype.MemoryAvailable: "100%", }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("0")) }) It("should used default eviction threshold (hard) for memory when evictionHard not specified", func() { @@ -733,11 +733,11 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionSoft: map[string]string{ - instancetypes.MemoryAvailable: "50Mi", + instancetype.MemoryAvailable: "50Mi", }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("50Mi")) }) It("should override eviction threshold (soft) when specified as a quantity", func() { @@ -750,11 +750,11 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionSoft: map[string]string{ - instancetypes.MemoryAvailable: "500Mi", + instancetype.MemoryAvailable: "500Mi", }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("500Mi")) }) It("should override eviction threshold (soft) when specified as a percentage value", func() { @@ -767,14 +767,14 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionHard: map[string]string{ - instancetypes.MemoryAvailable: "5%", + instancetype.MemoryAvailable: "5%", }, EvictionSoft: map[string]string{ - instancetypes.MemoryAvailable: "10%", + instancetype.MemoryAvailable: "10%", }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) It("should consider the eviction threshold (soft) disabled when specified as 100%", func() { @@ -787,11 +787,11 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionSoft: map[string]string{ - instancetypes.MemoryAvailable: "100%", + instancetype.MemoryAvailable: "100%", }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("0")) }) It("should ignore eviction threshold (soft) when using Bottlerocket AMI", func() { @@ -805,14 +805,14 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionHard: map[string]string{ - instancetypes.MemoryAvailable: "1Gi", + instancetype.MemoryAvailable: "1Gi", }, EvictionSoft: map[string]string{ - instancetypes.MemoryAvailable: "10Gi", + instancetype.MemoryAvailable: "10Gi", }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("1Gi")) }) It("should take the greater of evictionHard and evictionSoft for overhead as a value", func() { @@ -825,14 +825,14 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionSoft: map[string]string{ - instancetypes.MemoryAvailable: "3Gi", + instancetype.MemoryAvailable: "3Gi", }, EvictionHard: map[string]string{ - instancetypes.MemoryAvailable: "1Gi", + instancetype.MemoryAvailable: "1Gi", }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("3Gi")) }) It("should take the greater of evictionHard and evictionSoft for overhead as a value", func() { @@ -845,14 +845,14 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionSoft: map[string]string{ - instancetypes.MemoryAvailable: "2%", + instancetype.MemoryAvailable: "2%", }, EvictionHard: map[string]string{ - instancetypes.MemoryAvailable: "5%", + instancetype.MemoryAvailable: "5%", }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.05, 10)) }) It("should take the greater of evictionHard and evictionSoft for overhead with mixed percentage/value", func() { @@ -865,14 +865,14 @@ var _ = Describe("Instance Types", func() { v1.ResourceMemory: resource.MustParse("10Gi"), }, EvictionSoft: map[string]string{ - instancetypes.MemoryAvailable: "10%", + instancetype.MemoryAvailable: "10%", }, EvictionHard: map[string]string{ - instancetypes.MemoryAvailable: "1Gi", + instancetype.MemoryAvailable: "1Gi", }, }, }) - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) }) @@ -881,7 +881,7 @@ var _ = Describe("Instance Types", func() { Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{MaxPods: ptr.Int32(10)}}) for _, info := range instanceInfo { - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 10)) } }) @@ -894,7 +894,7 @@ var _ = Describe("Instance Types", func() { Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{MaxPods: ptr.Int32(10)}}) for _, info := range instanceInfo { - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 10)) } }) @@ -903,7 +903,7 @@ var _ = Describe("Instance Types", func() { Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}}) for _, info := range instanceInfo { - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", ptr.Int64Value(info.VCpuInfo.DefaultVCpus))) } }) @@ -912,7 +912,7 @@ var _ = Describe("Instance Types", func() { Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(4), MaxPods: ptr.Int32(20)}}) for _, info := range instanceInfo { - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", lo.Min([]int64{20, ptr.Int64Value(info.VCpuInfo.DefaultVCpus) * 4}))) } }) @@ -922,7 +922,7 @@ var _ = Describe("Instance Types", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}}) for _, info := range instanceInfo { - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) limitedPods := resources.Quantity(fmt.Sprint(*info.NetworkInfo.MaximumNetworkInterfaces*(*info.NetworkInfo.Ipv4AddressesPerInterface-1) + 2)) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", limitedPods.Value())) } @@ -936,7 +936,7 @@ var _ = Describe("Instance Types", func() { Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(0)}}) for _, info := range instanceInfo { - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 110)) } }) diff --git a/pkg/providers/instancetypes/types.go b/pkg/providers/instancetype/types.go similarity index 99% rename from pkg/providers/instancetypes/types.go rename to pkg/providers/instancetype/types.go index b97d1fd65921..5b2d30bb8998 100644 --- a/pkg/providers/instancetypes/types.go +++ b/pkg/providers/instancetype/types.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package instancetypes +package instancetype import ( "context" diff --git a/pkg/providers/instancetypes/zz_generated.bandwidth.go b/pkg/providers/instancetype/zz_generated.bandwidth.go similarity index 99% rename from pkg/providers/instancetypes/zz_generated.bandwidth.go rename to pkg/providers/instancetype/zz_generated.bandwidth.go index 2a5f3c044e01..a39c92faa235 100644 --- a/pkg/providers/instancetypes/zz_generated.bandwidth.go +++ b/pkg/providers/instancetype/zz_generated.bandwidth.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package instancetypes +package instancetype // GENERATED FILE. DO NOT EDIT DIRECTLY. // Update hack/code/bandwidth_gen.go and re-generate to edit diff --git a/pkg/providers/instancetypes/zz_generated.vpclimits.go b/pkg/providers/instancetype/zz_generated.vpclimits.go similarity index 99% rename from pkg/providers/instancetypes/zz_generated.vpclimits.go rename to pkg/providers/instancetype/zz_generated.vpclimits.go index 198f3e9bc24b..26df09625feb 100644 --- a/pkg/providers/instancetypes/zz_generated.vpclimits.go +++ b/pkg/providers/instancetype/zz_generated.vpclimits.go @@ -19,7 +19,7 @@ // Code generated by go generate; DO NOT EDIT. // This file was generated at 2023-01-26T19:39:15Z -package instancetypes +package instancetype type VPCLimits struct { Interface int diff --git a/pkg/providers/launchtemplate/launchtemplate_test.go b/pkg/providers/launchtemplate/launchtemplate_test.go index 2adb1af93ccf..07f9087991e5 100644 --- a/pkg/providers/launchtemplate/launchtemplate_test.go +++ b/pkg/providers/launchtemplate/launchtemplate_test.go @@ -55,7 +55,7 @@ import ( "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/providers/amifamily/bootstrap" - "github.com/aws/karpenter/pkg/providers/instancetypes" + "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/providers/securitygroup" @@ -99,7 +99,7 @@ var nodeTemplate *v1alpha1.AWSNodeTemplate var cluster *state.Cluster var pricingProvider *pricing.Provider var subnetProvider *subnet.Provider -var instanceTypesProvider *instancetypes.Provider +var instanceTypesProvider *instancetype.Provider var securityGroupProvider *securitygroup.Provider func TestAWS(t *testing.T) { @@ -129,7 +129,7 @@ var _ = BeforeSuite(func() { securityGroupProvider = securitygroup.NewProvider(fakeEC2API) amiProvider = amifamily.NewProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) amiResolver = amifamily.New(env.Client, amiProvider) - instanceTypesProvider = instancetypes.NewProvider("", instanceTypeCache, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) + instanceTypesProvider = instancetype.NewProvider("", instanceTypeCache, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) launchTemplateProvider = launchtemplate.NewProvider( ctx, @@ -221,7 +221,7 @@ var _ = BeforeEach(func() { launchTemplateProvider.ClusterEndpoint = "https://test-cluster" // Reset the pricing provider, so we don't cross-pollinate pricing data - instanceTypesProvider = instancetypes.NewProvider( + instanceTypesProvider = instancetype.NewProvider( "", instanceTypeCache, fakeEC2API, @@ -893,7 +893,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1093Mi")) }) @@ -904,7 +904,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1093Mi")) }) @@ -943,7 +943,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1093Mi")) }) @@ -954,7 +954,7 @@ var _ = Describe("LaunchTemplates", func() { })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - it := instancetypes.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) + it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1665Mi")) }) From 98415d3a007f0e1f52b5e47696e5cfded9731048 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Wed, 1 Mar 2023 16:21:35 -0800 Subject: [PATCH 21/41] hack folder change --- hack/api-code-gen.sh | 4 ++-- hack/code/bandwidth_gen.go | 2 +- hack/code/vpc_limits_gen.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hack/api-code-gen.sh b/hack/api-code-gen.sh index efddca8af0ac..b58ad2aba4f7 100755 --- a/hack/api-code-gen.sh +++ b/hack/api-code-gen.sh @@ -8,7 +8,7 @@ fi echo "api-code-gen running ENABLE_GIT_PUSH: ${ENABLE_GIT_PUSH}" bandwidth() { - GENERATED_FILE="pkg/providers/instancetypes/zz_generated.bandwidth.go" + GENERATED_FILE="pkg/providers/instancetype/zz_generated.bandwidth.go" NO_UPDATE='' SUBJECT="Bandwidth" @@ -30,7 +30,7 @@ pricing() { } vpcLimits() { - GENERATED_FILE="pkg/providers/instancetypes/zz_generated.vpclimits.go" + GENERATED_FILE="pkg/providers/instancetype/zz_generated.vpclimits.go" NO_UPDATE='' SUBJECT="VPC Limits" diff --git a/hack/code/bandwidth_gen.go b/hack/code/bandwidth_gen.go index dee645521561..1ac616f8caf9 100644 --- a/hack/code/bandwidth_gen.go +++ b/hack/code/bandwidth_gen.go @@ -56,7 +56,7 @@ var ( func main() { flag.Parse() if flag.NArg() != 1 { - log.Fatalf("Usage: `bandwidth_gen.go pkg/providers/instancetypes/zz_generated.pricing.go`") + log.Fatalf("Usage: `bandwidth_gen.go pkg/providers/instancetype/zz_generated.pricing.go`") } bandwidth := map[string]int64{} diff --git a/hack/code/vpc_limits_gen.go b/hack/code/vpc_limits_gen.go index 5c0f84146758..478a2ba20593 100644 --- a/hack/code/vpc_limits_gen.go +++ b/hack/code/vpc_limits_gen.go @@ -35,7 +35,7 @@ func main() { opts := options{} flag.StringVar(&opts.urlInput, "url", "https://raw.githubusercontent.com/aws/amazon-vpc-resource-controller-k8s/master/pkg/aws/vpc/limits.go", "url of the raw vpc/limits.go file in the github.com/aws/amazon-vpc-resource-controller-k8s repo") - flag.StringVar(&opts.sourceOutput, "output", "pkg/providers/instancetypes/zz_generated.vpclimits.go", "output location for the generated go source file") + flag.StringVar(&opts.sourceOutput, "output", "pkg/providers/instancetype/zz_generated.vpclimits.go", "output location for the generated go source file") flag.Parse() limitsURL, err := url.Parse(opts.urlInput) From 3fec83c2b4f91783b6efcc89349f1329802bf471 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Wed, 1 Mar 2023 16:23:56 -0800 Subject: [PATCH 22/41] Missing changes --- hack/code/bandwidth_gen.go | 2 +- hack/code/vpc_limits_gen.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/code/bandwidth_gen.go b/hack/code/bandwidth_gen.go index 1ac616f8caf9..5279c18d64a2 100644 --- a/hack/code/bandwidth_gen.go +++ b/hack/code/bandwidth_gen.go @@ -40,7 +40,7 @@ var uriSelectors = map[string]string{ const fileFormat = ` %s -package instancetypes +package instancetype // GENERATED FILE. DO NOT EDIT DIRECTLY. // Update hack/code/bandwidth_gen.go and re-generate to edit diff --git a/hack/code/vpc_limits_gen.go b/hack/code/vpc_limits_gen.go index 478a2ba20593..773777fc567c 100644 --- a/hack/code/vpc_limits_gen.go +++ b/hack/code/vpc_limits_gen.go @@ -58,7 +58,7 @@ func main() { if err != nil { log.Fatal(err) } - newRespData := strings.Replace(string(respData), "package vpc", "package instancetypes", 1) + newRespData := strings.Replace(string(respData), "package vpc", "package instancetype", 1) out.WriteString(newRespData) defer out.Close() From 674cea730a5a3d0b5058cd9fdbf78345232275c8 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Wed, 1 Mar 2023 16:25:38 -0800 Subject: [PATCH 23/41] file name change --- pkg/providers/instancetype/{instancetypes.go => instancetype.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/providers/instancetype/{instancetypes.go => instancetype.go} (100%) diff --git a/pkg/providers/instancetype/instancetypes.go b/pkg/providers/instancetype/instancetype.go similarity index 100% rename from pkg/providers/instancetype/instancetypes.go rename to pkg/providers/instancetype/instancetype.go From 46e0e280d1d51d6cbbb246c2fd3b7f2a17819cdb Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Thu, 2 Mar 2023 09:26:43 -0800 Subject: [PATCH 24/41] feedback changes --- pkg/cloudprovider/cloudprovider.go | 4 +-- pkg/providers/instance/instance.go | 51 +++++++++++++++--------------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index 4da091682b94..bc3504414cc2 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -68,10 +68,10 @@ type CloudProvider struct { func New(ctx awscontext.Context) *CloudProvider { return &CloudProvider{ - kubeClient: ctx.KubeClient, instanceTypeProvider: ctx.InstanceTypesProvider, - amiProvider: ctx.AMIProvider, instanceProvider: ctx.InstanceProvider, + kubeClient: ctx.KubeClient, + amiProvider: ctx.AMIProvider, } } diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go index 0805efc29aef..98a40b7722fe 100644 --- a/pkg/providers/instance/instance.go +++ b/pkg/providers/instance/instance.go @@ -46,7 +46,7 @@ import ( "github.com/aws/karpenter-core/pkg/utils/resources" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" + cloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" "github.com/aws/karpenter-core/pkg/scheduling" ) @@ -71,7 +71,8 @@ type Provider struct { ec2Batcher *batcher.EC2API } -func NewProvider(ctx context.Context, region string, ec2api ec2iface.EC2API, unavailableOfferings *cache.UnavailableOfferings, instanceTypeProvider *instancetype.Provider, subnetProvider *subnet.Provider, launchTemplateProvider *launchtemplate.Provider) *Provider { +func NewProvider(ctx context.Context, region string, ec2api ec2iface.EC2API, unavailableOfferings *cache.UnavailableOfferings, + instanceTypeProvider *instancetype.Provider, subnetProvider *subnet.Provider, launchTemplateProvider *launchtemplate.Provider) *Provider { return &Provider{ region: region, ec2api: ec2api, @@ -83,7 +84,7 @@ func NewProvider(ctx context.Context, region string, ec2api ec2iface.EC2API, una } } -func (p *Provider) Create(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate, machine *v1alpha5.Machine, instanceTypes []*corecloudprovider.InstanceType) (*ec2.Instance, error) { +func (p *Provider) Create(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate, machine *v1alpha5.Machine, instanceTypes []*cloudprovider.InstanceType) (*ec2.Instance, error) { instanceTypes = p.filterInstanceTypes(machine, instanceTypes) instanceTypes = orderInstanceTypesByPrice(instanceTypes, scheduling.NewNodeSelectorRequirements(machine.Spec.Requirements...)) if len(instanceTypes) > MaxInstanceTypes { @@ -131,7 +132,7 @@ func (p *Provider) Link(ctx context.Context, id string) error { }) if err != nil { if awserrors.IsNotFound(err) { - return corecloudprovider.NewMachineNotFoundError(fmt.Errorf("linking tags, %w", err)) + return cloudprovider.NewMachineNotFoundError(fmt.Errorf("linking tags, %w", err)) } return fmt.Errorf("linking tags, %w", err) } @@ -144,7 +145,7 @@ func (p *Provider) Get(ctx context.Context, id string) (*ec2.Instance, error) { Filters: []*ec2.Filter{instanceStateFilter}, }) if awserrors.IsNotFound(err) { - return nil, corecloudprovider.NewMachineNotFoundError(err) + return nil, cloudprovider.NewMachineNotFoundError(err) } if err != nil { return nil, fmt.Errorf("failed to describe ec2 instances, %w", err) @@ -181,7 +182,7 @@ func (p *Provider) List(ctx context.Context) ([]*ec2.Instance, error) { return nil, fmt.Errorf("describing ec2 instances, %w", err) } instances, err := instancesFromOutput(out) - return instances, corecloudprovider.IgnoreMachineNotFoundError(err) + return instances, cloudprovider.IgnoreMachineNotFoundError(err) } func (p *Provider) Delete(ctx context.Context, id string) error { @@ -189,10 +190,10 @@ func (p *Provider) Delete(ctx context.Context, id string) error { InstanceIds: []*string{aws.String(id)}, }); err != nil { if awserrors.IsNotFound(err) { - return corecloudprovider.NewMachineNotFoundError(fmt.Errorf("instance already terminated")) + return cloudprovider.NewMachineNotFoundError(fmt.Errorf("instance already terminated")) } if _, e := p.Get(ctx, id); err != nil { - if corecloudprovider.IsMachineNotFoundError(e) { + if cloudprovider.IsMachineNotFoundError(e) { return e } err = multierr.Append(err, e) @@ -202,7 +203,7 @@ func (p *Provider) Delete(ctx context.Context, id string) error { return nil } -func (p *Provider) launchInstance(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate, machine *v1alpha5.Machine, instanceTypes []*corecloudprovider.InstanceType) (*string, error) { +func (p *Provider) launchInstance(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate, machine *v1alpha5.Machine, instanceTypes []*cloudprovider.InstanceType) (*string, error) { capacityType := p.getCapacityType(machine, instanceTypes) zonalSubnets, err := p.subnetProvider.ZonalSubnetsForLaunch(ctx, nodeTemplate, instanceTypes, capacityType) if err != nil { @@ -262,7 +263,7 @@ func (p *Provider) launchInstance(ctx context.Context, nodeTemplate *v1alpha1.AW return createFleetOutput.Instances[0].InstanceIds[0], nil } -func (p *Provider) checkODFallback(machine *v1alpha5.Machine, instanceTypes []*corecloudprovider.InstanceType, launchTemplateConfigs []*ec2.FleetLaunchTemplateConfigRequest) error { +func (p *Provider) checkODFallback(machine *v1alpha5.Machine, instanceTypes []*cloudprovider.InstanceType, launchTemplateConfigs []*ec2.FleetLaunchTemplateConfigRequest) error { // only evaluate for on-demand fallback if the capacity type for the request is OD and both OD and spot are allowed in requirements if p.getCapacityType(machine, instanceTypes) != v1alpha5.CapacityTypeOnDemand || !scheduling.NewNodeSelectorRequirements(machine.Spec.Requirements...).Get(v1alpha5.LabelCapacityType).Has(v1alpha5.CapacityTypeSpot) { return nil @@ -285,7 +286,7 @@ func (p *Provider) checkODFallback(machine *v1alpha5.Machine, instanceTypes []*c } func (p *Provider) getLaunchTemplateConfigs(ctx context.Context, nodeTemplate *v1alpha1.AWSNodeTemplate, machine *v1alpha5.Machine, - instanceTypes []*corecloudprovider.InstanceType, zonalSubnets map[string]*ec2.Subnet, capacityType string) ([]*ec2.FleetLaunchTemplateConfigRequest, error) { + instanceTypes []*cloudprovider.InstanceType, zonalSubnets map[string]*ec2.Subnet, capacityType string) ([]*ec2.FleetLaunchTemplateConfigRequest, error) { var launchTemplateConfigs []*ec2.FleetLaunchTemplateConfigRequest launchTemplates, err := p.launchTemplateProvider.EnsureAll(ctx, nodeTemplate, machine, instanceTypes, map[string]string{v1alpha5.LabelCapacityType: capacityType}) if err != nil { @@ -311,16 +312,16 @@ func (p *Provider) getLaunchTemplateConfigs(ctx context.Context, nodeTemplate *v // getOverrides creates and returns launch template overrides for the cross product of InstanceTypes and subnets (with subnets being constrained by // zones and the offerings in InstanceTypes) -func (p *Provider) getOverrides(instanceTypes []*corecloudprovider.InstanceType, zonalSubnets map[string]*ec2.Subnet, zones *scheduling.Requirement, capacityType string) []*ec2.FleetLaunchTemplateOverridesRequest { +func (p *Provider) getOverrides(instanceTypes []*cloudprovider.InstanceType, zonalSubnets map[string]*ec2.Subnet, zones *scheduling.Requirement, capacityType string) []*ec2.FleetLaunchTemplateOverridesRequest { // Unwrap all the offerings to a flat slice that includes a pointer // to the parent instance type name type offeringWithParentName struct { - corecloudprovider.Offering + cloudprovider.Offering parentInstanceTypeName string } var unwrappedOfferings []offeringWithParentName for _, it := range instanceTypes { - ofs := lo.Map(it.Offerings.Available(), func(of corecloudprovider.Offering, _ int) offeringWithParentName { + ofs := lo.Map(it.Offerings.Available(), func(of cloudprovider.Offering, _ int) offeringWithParentName { return offeringWithParentName{ Offering: of, parentInstanceTypeName: it.Name, @@ -407,7 +408,7 @@ func (p *Provider) updateUnavailableOfferingsCache(ctx context.Context, errors [ // getCapacityType selects spot if both constraints are flexible and there is an // available offering. The AWS Cloud Provider defaults to [ on-demand ], so spot // must be explicitly included in capacity type requirements. -func (p *Provider) getCapacityType(machine *v1alpha5.Machine, instanceTypes []*corecloudprovider.InstanceType) string { +func (p *Provider) getCapacityType(machine *v1alpha5.Machine, instanceTypes []*cloudprovider.InstanceType) string { requirements := scheduling.NewNodeSelectorRequirements(machine. Spec.Requirements...) if requirements.Get(v1alpha5.LabelCapacityType).Has(v1alpha5.CapacityTypeSpot) { @@ -422,7 +423,7 @@ func (p *Provider) getCapacityType(machine *v1alpha5.Machine, instanceTypes []*c return v1alpha5.CapacityTypeOnDemand } -func orderInstanceTypesByPrice(instanceTypes []*corecloudprovider.InstanceType, requirements scheduling.Requirements) []*corecloudprovider.InstanceType { +func orderInstanceTypesByPrice(instanceTypes []*cloudprovider.InstanceType, requirements scheduling.Requirements) []*cloudprovider.InstanceType { // Order instance types so that we get the cheapest instance types of the available offerings sort.Slice(instanceTypes, func(i, j int) bool { iPrice := math.MaxFloat64 @@ -442,8 +443,8 @@ func orderInstanceTypesByPrice(instanceTypes []*corecloudprovider.InstanceType, } // filterInstanceTypes is used to provide filtering on the list of potential instance types to further limit it to those -// that make the most sense given our specific AWS corecloudprovider. -func (p *Provider) filterInstanceTypes(machine *v1alpha5.Machine, instanceTypes []*corecloudprovider.InstanceType) []*corecloudprovider.InstanceType { +// that make the most sense given our specific AWS cloudprovider. +func (p *Provider) filterInstanceTypes(machine *v1alpha5.Machine, instanceTypes []*cloudprovider.InstanceType) []*cloudprovider.InstanceType { instanceTypes = filterExoticInstanceTypes(instanceTypes) // If we could potentially launch either a spot or on-demand node, we want to filter out the spot instance types that // are more expensive than the cheapest on-demand type. @@ -455,7 +456,7 @@ func (p *Provider) filterInstanceTypes(machine *v1alpha5.Machine, instanceTypes // isMixedCapacityLaunch returns true if provisioners and available offerings could potentially allow either a spot or // and on-demand node to launch -func (p *Provider) isMixedCapacityLaunch(machine *v1alpha5.Machine, instanceTypes []*corecloudprovider.InstanceType) bool { +func (p *Provider) isMixedCapacityLaunch(machine *v1alpha5.Machine, instanceTypes []*cloudprovider.InstanceType) bool { requirements := scheduling.NewNodeSelectorRequirements(machine.Spec.Requirements...) // requirements must allow both if !requirements.Get(v1alpha5.LabelCapacityType).Has(v1alpha5.CapacityTypeSpot) || @@ -482,7 +483,7 @@ func (p *Provider) isMixedCapacityLaunch(machine *v1alpha5.Machine, instanceType // filterUnwantedSpot is used to filter out spot types that are more expensive than the cheapest on-demand type that we // could launch during mixed capacity-type launches -func filterUnwantedSpot(instanceTypes []*corecloudprovider.InstanceType) []*corecloudprovider.InstanceType { +func filterUnwantedSpot(instanceTypes []*cloudprovider.InstanceType) []*cloudprovider.InstanceType { cheapestOnDemand := math.MaxFloat64 // first, find the price of our cheapest available on-demand instance type that could support this node for _, it := range instanceTypes { @@ -496,7 +497,7 @@ func filterUnwantedSpot(instanceTypes []*corecloudprovider.InstanceType) []*core // Filter out any types where the cheapest offering, which should be spot, is more expensive than the cheapest // on-demand instance type that would have worked. This prevents us from getting a larger more-expensive spot // instance type compared to the cheapest sufficiently large on-demand instance type - instanceTypes = lo.Filter(instanceTypes, func(item *corecloudprovider.InstanceType, index int) bool { + instanceTypes = lo.Filter(instanceTypes, func(item *cloudprovider.InstanceType, index int) bool { available := item.Offerings.Available() if len(available) == 0 { return false @@ -509,8 +510,8 @@ func filterUnwantedSpot(instanceTypes []*corecloudprovider.InstanceType) []*core // filterExoticInstanceTypes is used to eliminate less desirable instance types (like GPUs) from the list of possible instance types when // a set of more appropriate instance types would work. If a set of more desirable instance types is not found, then the original slice // of instance types are returned. -func filterExoticInstanceTypes(instanceTypes []*corecloudprovider.InstanceType) []*corecloudprovider.InstanceType { - var genericInstanceTypes []*corecloudprovider.InstanceType +func filterExoticInstanceTypes(instanceTypes []*cloudprovider.InstanceType) []*cloudprovider.InstanceType { + var genericInstanceTypes []*cloudprovider.InstanceType for _, it := range instanceTypes { // deprioritize metal even if our opinionated filter isn't applied due to something like an instance family // requirement @@ -534,13 +535,13 @@ func filterExoticInstanceTypes(instanceTypes []*corecloudprovider.InstanceType) func instancesFromOutput(out *ec2.DescribeInstancesOutput) ([]*ec2.Instance, error) { if len(out.Reservations) == 0 { - return nil, corecloudprovider.NewMachineNotFoundError(fmt.Errorf("instance not found")) + return nil, cloudprovider.NewMachineNotFoundError(fmt.Errorf("instance not found")) } instances := lo.Flatten(lo.Map(out.Reservations, func(r *ec2.Reservation, _ int) []*ec2.Instance { return r.Instances })) if len(instances) == 0 { - return nil, corecloudprovider.NewMachineNotFoundError(fmt.Errorf("instance not found")) + return nil, cloudprovider.NewMachineNotFoundError(fmt.Errorf("instance not found")) } // Get a consistent ordering for instances sort.Slice(instances, func(i, j int) bool { From 216647b2c106a677c6e5bd5fe587f58283a19560 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Thu, 2 Mar 2023 13:54:26 -0800 Subject: [PATCH 25/41] Added test.context --- pkg/test/context.go | 127 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 pkg/test/context.go diff --git a/pkg/test/context.go b/pkg/test/context.go new file mode 100644 index 000000000000..c6512d898be6 --- /dev/null +++ b/pkg/test/context.go @@ -0,0 +1,127 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "context" + "net" + + clock "k8s.io/utils/clock/testing" + "knative.dev/pkg/ptr" + + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/awstesting/mock" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + coresettings "github.com/aws/karpenter-core/pkg/apis/settings" + "github.com/aws/karpenter-core/pkg/cloudprovider" + "github.com/aws/karpenter-core/pkg/events" + "github.com/aws/karpenter-core/pkg/operator/scheme" + coretest "github.com/aws/karpenter-core/pkg/test" + "github.com/patrickmn/go-cache" + "github.com/samber/lo" + "k8s.io/client-go/tools/record" + + "github.com/aws/karpenter/pkg/apis" + "github.com/aws/karpenter/pkg/apis/settings" + awscache "github.com/aws/karpenter/pkg/cache" + awscontext "github.com/aws/karpenter/pkg/context" + "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/providers/amifamily" + "github.com/aws/karpenter/pkg/providers/instance" + "github.com/aws/karpenter/pkg/providers/instancetype" + "github.com/aws/karpenter/pkg/providers/launchtemplate" + "github.com/aws/karpenter/pkg/providers/pricing" + "github.com/aws/karpenter/pkg/providers/securitygroup" + "github.com/aws/karpenter/pkg/providers/subnet" +) + +type ContextOptions struct { + Session *session.Session + UnavailableOfferingsCache *awscache.UnavailableOfferings + SubnetProvider *subnet.Provider + SecurityGroupProvider *securitygroup.Provider + AMIProvider *amifamily.Provider + AMIResolver *amifamily.Resolver + LaunchTemplateProvider *launchtemplate.Provider + PricingProvider *pricing.Provider + InstanceTypesProvider *instancetype.Provider + InstanceProvider *instance.Provider +} + +func Context(ec2api ec2iface.EC2API, overrides ContextOptions) awscontext.Context { + env := coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + var ctx context.Context + ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = settings.ToContext(ctx, Settings()) + // ctx, stop := context.WithCancel(ctx) + + // cache + ssmCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + ec2Cache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + kubernetesVersionCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + instanceTypeCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + unavailableOfferingsCache := awscache.NewUnavailableOfferings() + launchTemplateCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + + // Providers + pricingProvider := pricing.NewProvider(ctx, &fake.PricingAPI{}, ec2api, "", make(chan struct{})) + subnetProvider := subnet.NewProvider(ec2api) + securityGroupProvider := securitygroup.NewProvider(ec2api) + amiProvider := amifamily.NewProvider(env.Client, env.KubernetesInterface, &fake.SSMAPI{}, ec2api, ssmCache, ec2Cache, kubernetesVersionCache) + amiResolver := amifamily.New(env.Client, amiProvider) + instanceTypesProvider := instancetype.NewProvider("", instanceTypeCache, ec2api, subnetProvider, unavailableOfferingsCache, pricingProvider) + launchTemplateProvider := launchtemplate.NewProvider( + ctx, + launchTemplateCache, + ec2api, + amiResolver, + securityGroupProvider, + ptr.String("ca-bundle"), + make(chan struct{}), + net.ParseIP("10.0.100.10"), + "https://test-cluster", + ) + instanceProvider := instance.NewProvider(ctx, + "", + ec2api, + unavailableOfferingsCache, + instanceTypesProvider, + subnetProvider, + launchTemplateProvider, + ) + + return awscontext.Context{ + Context: cloudprovider.Context{ + Context: ctx, + RESTConfig: env.Config, + KubernetesInterface: env.KubernetesInterface, + KubeClient: env.Client, + EventRecorder: events.NewRecorder(&record.FakeRecorder{}), + Clock: &clock.FakeClock{}, + StartAsync: nil, + }, + Session: lo.FromPtrOr(&overrides.Session, mock.Session), + EC2API: ec2api, + UnavailableOfferingsCache: lo.FromPtrOr(&overrides.UnavailableOfferingsCache, unavailableOfferingsCache), + InstanceTypesProvider: lo.FromPtrOr(&overrides.InstanceTypesProvider, instanceTypesProvider), + InstanceProvider: lo.FromPtrOr(&overrides.InstanceProvider, instanceProvider), + SubnetProvider: lo.FromPtrOr(&overrides.SubnetProvider, subnetProvider), + SecurityGroupProvider: lo.FromPtrOr(&overrides.SecurityGroupProvider, securityGroupProvider), + PricingProvider: lo.FromPtrOr(&overrides.PricingProvider, pricingProvider), + AMIProvider: lo.FromPtrOr(&overrides.AMIProvider, amiProvider), + AMIResolver: lo.FromPtrOr(&overrides.AMIResolver, amiResolver), + LaunchTemplateProvider: lo.FromPtrOr(&overrides.LaunchTemplateProvider, launchTemplateProvider), + } +} From 63d5f3ff952a8c7272a32fe92ac0c3cca4523108 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Thu, 2 Mar 2023 16:31:56 -0800 Subject: [PATCH 26/41] changes --- pkg/context/context.go | 9 + pkg/providers/amifamily/ami.go | 6 + pkg/providers/instancetype/instancetype.go | 4 + .../launchtemplate/launchtemplate.go | 44 +- .../launchtemplate/launchtemplate_test.go | 573 ++++++++---------- pkg/test/context.go | 146 +++-- test/go.mod | 1 + test/go.sum | 7 + 8 files changed, 411 insertions(+), 379 deletions(-) diff --git a/pkg/context/context.go b/pkg/context/context.go index 0bb352cb5272..d0c970237872 100644 --- a/pkg/context/context.go +++ b/pkg/context/context.go @@ -227,3 +227,12 @@ func kubeDNSIP(ctx context.Context, kubernetesInterface kubernetes.Interface) (n } return kubeDNSIP, nil } + +func (c *Context) RestProviderCache() { + c.SecurityGroupProvider.Reset() + c.SubnetProvider.Reset() + c.InstanceTypesProvider.Reset() + c.LaunchTemplateProvider.Reset() + c.AMIProvider.Reset() + c.UnavailableOfferingsCache.Flush() +} diff --git a/pkg/providers/amifamily/ami.go b/pkg/providers/amifamily/ami.go index 9bf1054ee4b2..b56b4cfa6b6c 100644 --- a/pkg/providers/amifamily/ami.go +++ b/pkg/providers/amifamily/ami.go @@ -255,3 +255,9 @@ func (p *Provider) getRequirementsFromImage(ec2Image *ec2.Image) scheduling.Requ requirements.Add(scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, architecture)) return requirements } + +func (p *Provider) Reset() { + p.ssmCache.Flush() + p.ec2Cache.Flush() + p.kubernetesVersionCache.Flush() +} diff --git a/pkg/providers/instancetype/instancetype.go b/pkg/providers/instancetype/instancetype.go index 4d2df86f8bda..af8a80566bda 100644 --- a/pkg/providers/instancetype/instancetype.go +++ b/pkg/providers/instancetype/instancetype.go @@ -231,3 +231,7 @@ func (p *Provider) GetInstanceTypes(ctx context.Context) ([]*ec2.InstanceTypeInf p.cache.SetDefault(InstanceTypesCacheKey, instanceTypes) return instanceTypes, nil } + +func (p *Provider) Reset() { + p.cache.Flush() +} diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index 9e7d76def6ce..3cd7655881d7 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -55,27 +55,27 @@ type Provider struct { ec2api ec2iface.EC2API amiFamily *amifamily.Resolver securityGroupProvider *securitygroup.Provider - cache *cache.Cache + Cache *cache.Cache caBundle *string cm *pretty.ChangeMonitor KubeDNSIP net.IP ClusterEndpoint string } -func NewProvider(ctx context.Context, cache *cache.Cache, ec2api ec2iface.EC2API, amiFamily *amifamily.Resolver, securityGroupProvider *securitygroup.Provider, caBundle *string, startAsync <-chan struct{}, kubeDNSIP net.IP, clusterEndpoint string) *Provider { +func NewProvider(ctx context.Context, Cache *cache.Cache, ec2api ec2iface.EC2API, amiFamily *amifamily.Resolver, securityGroupProvider *securitygroup.Provider, caBundle *string, startAsync <-chan struct{}, kubeDNSIP net.IP, clusterEndpoint string) *Provider { l := &Provider{ ec2api: ec2api, amiFamily: amiFamily, securityGroupProvider: securityGroupProvider, - cache: cache, + Cache: Cache, caBundle: caBundle, cm: pretty.NewChangeMonitor(), KubeDNSIP: kubeDNSIP, ClusterEndpoint: clusterEndpoint, } - l.cache.OnEvicted(l.cachedEvictedFunc(ctx)) + l.Cache.OnEvicted(l.cachedEvictedFunc(ctx)) go func() { - // only hydrate cache once elected leader + // only hydrate Cache once elected leader select { case <-startAsync: case <-ctx.Done(): @@ -115,15 +115,15 @@ func (p *Provider) EnsureAll(ctx context.Context, nodeTemplate *v1alpha1.AWSNode return launchTemplates, nil } -// Invalidate deletes a launch template from cache if it exists +// Invalidate deletes a launch template from Cache if it exists func (p *Provider) Invalidate(ctx context.Context, ltName string, ltID string) { ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("launch-template-name", ltName, "launch-template-id", ltID)) p.Lock() defer p.Unlock() - defer p.cache.OnEvicted(p.cachedEvictedFunc(ctx)) - p.cache.OnEvicted(nil) - logging.FromContext(ctx).Debugf("invalidating launch template in the cache because it no longer exists") - p.cache.Delete(ltName) + defer p.Cache.OnEvicted(p.cachedEvictedFunc(ctx)) + p.Cache.OnEvicted(nil) + logging.FromContext(ctx).Debugf("invalidating launch template in the Cache because it no longer exists") + p.Cache.Delete(ltName) } func launchTemplateName(options *amifamily.LaunchTemplate) string { @@ -164,9 +164,9 @@ func (p *Provider) ensureLaunchTemplate(ctx context.Context, options *amifamily. var launchTemplate *ec2.LaunchTemplate name := launchTemplateName(options) ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("launch-template-name", name)) - // Read from cache - if launchTemplate, ok := p.cache.Get(name); ok { - p.cache.SetDefault(name, launchTemplate) + // Read from Cache + if launchTemplate, ok := p.Cache.Get(name); ok { + p.Cache.SetDefault(name, launchTemplate) return launchTemplate.(*ec2.LaunchTemplate), nil } // Attempt to find an existing LT. @@ -189,7 +189,7 @@ func (p *Provider) ensureLaunchTemplate(ctx context.Context, options *amifamily. } launchTemplate = output.LaunchTemplates[0] } - p.cache.SetDefault(name, launchTemplate) + p.Cache.SetDefault(name, launchTemplate) return launchTemplate, nil } @@ -268,30 +268,30 @@ func (p *Provider) volumeSize(quantity *resource.Quantity) *int64 { return aws.Int64(int64(math.Ceil(quantity.AsApproximateFloat64() / math.Pow(2, 30)))) } -// hydrateCache queries for existing Launch Templates created by Karpenter for the current cluster and adds to the LT cache. +// hydrateCache queries for existing Launch Templates created by Karpenter for the current cluster and adds to the LT Cache. // Any error during hydration will result in a panic func (p *Provider) hydrateCache(ctx context.Context) { clusterName := awssettings.FromContext(ctx).ClusterName ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("tag-key", karpenterManagedTagKey, "tag-value", clusterName)) - logging.FromContext(ctx).Debugf("hydrating the launch template cache") + logging.FromContext(ctx).Debugf("hydrating the launch template Cache") if err := p.ec2api.DescribeLaunchTemplatesPagesWithContext(ctx, &ec2.DescribeLaunchTemplatesInput{ Filters: []*ec2.Filter{{Name: aws.String(fmt.Sprintf("tag:%s", karpenterManagedTagKey)), Values: []*string{aws.String(clusterName)}}}, }, func(output *ec2.DescribeLaunchTemplatesOutput, _ bool) bool { for _, lt := range output.LaunchTemplates { - p.cache.SetDefault(*lt.LaunchTemplateName, lt) + p.Cache.SetDefault(*lt.LaunchTemplateName, lt) } return true }); err != nil { - logging.FromContext(ctx).Errorf(fmt.Sprintf("Unable to hydrate the AWS launch template cache, %s", err)) + logging.FromContext(ctx).Errorf(fmt.Sprintf("Unable to hydrate the AWS launch template Cache, %s", err)) } - logging.FromContext(ctx).With("item-count", p.cache.ItemCount()).Debugf("finished hydrating the launch template cache") + logging.FromContext(ctx).With("item-count", p.Cache.ItemCount()).Debugf("finished hydrating the launch template Cache") } func (p *Provider) cachedEvictedFunc(ctx context.Context) func(string, interface{}) { return func(key string, lt interface{}) { p.Lock() defer p.Unlock() - if _, expiration, _ := p.cache.GetWithExpiration(key); expiration.After(time.Now()) { + if _, expiration, _ := p.Cache.GetWithExpiration(key); expiration.After(time.Now()) { return } launchTemplate := lt.(*ec2.LaunchTemplate) @@ -313,3 +313,7 @@ func (p *Provider) getInstanceProfile(ctx context.Context, nodeTemplate *v1alpha } return defaultProfile, nil } + +func (p *Provider) Reset() { + p.Cache.Flush() +} diff --git a/pkg/providers/launchtemplate/launchtemplate_test.go b/pkg/providers/launchtemplate/launchtemplate_test.go index 589899a4d03e..e27201e0e608 100644 --- a/pkg/providers/launchtemplate/launchtemplate_test.go +++ b/pkg/providers/launchtemplate/launchtemplate_test.go @@ -29,7 +29,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/awstesting/mock" "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -53,19 +52,14 @@ import ( "github.com/aws/karpenter/pkg/cloudprovider" awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/fake" - "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/providers/amifamily/bootstrap" - "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/pricing" - "github.com/aws/karpenter/pkg/providers/securitygroup" - "github.com/aws/karpenter/pkg/providers/subnet" "github.com/aws/karpenter/pkg/test" coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" "github.com/aws/karpenter-core/pkg/controllers/provisioning" "github.com/aws/karpenter-core/pkg/controllers/state" "github.com/aws/karpenter-core/pkg/events" @@ -77,32 +71,25 @@ import ( ) var ctx context.Context +var awsCtx *awscontext.Context var stop context.CancelFunc var opts options.Options var env *coretest.Environment -var ssmCache *cache.Cache -var ec2Cache *cache.Cache -var launchTemplateCache *cache.Cache -var instanceTypeCache *cache.Cache -var kubernetesVersionCache *cache.Cache var fakeEC2API *fake.EC2API var fakeSSMAPI *fake.SSMAPI var fakeClock *clock.FakeClock -var fakePricingAPI *fake.PricingAPI -var amiProvider *amifamily.Provider -var amiResolver *amifamily.Resolver -var cloudProvider *cloudprovider.CloudProvider -var unavailableOfferingsCache *awscache.UnavailableOfferings var prov *provisioning.Provisioner var provisioner *v1alpha5.Provisioner -var launchTemplateProvider *launchtemplate.Provider var nodeTemplate *v1alpha1.AWSNodeTemplate var cluster *state.Cluster + +// cache +var launchTemplateCache *cache.Cache + +// providers var pricingProvider *pricing.Provider -var subnetProvider *subnet.Provider -var instanceTypesProvider *instancetype.Provider -var securityGroupProvider *securitygroup.Provider -var instanceProvider *instance.Provider +var launchTemplateProvider *launchtemplate.Provider +var cloudProvider *cloudprovider.CloudProvider func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -118,57 +105,31 @@ var _ = BeforeSuite(func() { fakeEC2API = &fake.EC2API{} fakeSSMAPI = &fake.SSMAPI{} - ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + fakeClock = &clock.FakeClock{} + awsCtx = test.Context(ctx, fakeEC2API, fakeSSMAPI, env, fakeClock) + pricingProvider = pricing.NewProvider(ctx, &fake.PricingAPI{}, fakeEC2API, "", make(chan struct{})) launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - fakeClock = clock.NewFakeClock(time.Now()) - unavailableOfferingsCache = awscache.NewUnavailableOfferings() - fakePricingAPI = &fake.PricingAPI{} - pricingProvider = pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})) - subnetProvider = subnet.NewProvider(fakeEC2API) - securityGroupProvider = securitygroup.NewProvider(fakeEC2API) - amiProvider = amifamily.NewProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) - amiResolver = amifamily.New(env.Client, amiProvider) - instanceTypesProvider = instancetype.NewProvider("", instanceTypeCache, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) - launchTemplateProvider = launchtemplate.NewProvider( ctx, launchTemplateCache, fakeEC2API, - amiResolver, - securityGroupProvider, + awsCtx.AMIResolver, + awsCtx.SecurityGroupProvider, ptr.String("ca-bundle"), make(chan struct{}), net.ParseIP("10.0.100.10"), "https://test-cluster", ) - instanceProvider = instance.NewProvider(ctx, "", fakeEC2API, unavailableOfferingsCache, instanceTypesProvider, subnetProvider, launchTemplateProvider) - cloudProvider = cloudprovider.New(awscontext.Context{ - Context: corecloudprovider.Context{ - Context: ctx, - RESTConfig: env.Config, - KubernetesInterface: env.KubernetesInterface, - KubeClient: env.Client, - EventRecorder: events.NewRecorder(&record.FakeRecorder{}), - Clock: &clock.FakeClock{}, - StartAsync: nil, - }, - SubnetProvider: subnet.NewProvider(fakeEC2API), - SecurityGroupProvider: securityGroupProvider, - Session: mock.Session, - UnavailableOfferingsCache: unavailableOfferingsCache, - EC2API: fakeEC2API, - PricingProvider: pricingProvider, - AMIProvider: amiProvider, - AMIResolver: amiResolver, - LaunchTemplateProvider: launchTemplateProvider, - InstanceProvider: instanceProvider, - InstanceTypesProvider: instanceTypesProvider, + + test.UpdateContext(awsCtx, test.ContextOptions{ + LaunchTemplateProvider: launchTemplateProvider, + PricingProvider: pricingProvider, }) - cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) - prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) + + fmt.Println(launchTemplateCache == awsCtx.LaunchTemplateProvider.Cache) + cloudProvider = cloudprovider.New(*awsCtx) + cluster = state.NewCluster(fakeClock, awsCtx.KubeClient, cloudProvider) + prov = provisioning.NewProvisioner(ctx, awsCtx.KubeClient, awsCtx.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) }) @@ -209,42 +170,36 @@ var _ = BeforeEach(func() { Name: nodeTemplate.Name, }, }) - + fmt.Println(launchTemplateCache == awsCtx.LaunchTemplateProvider.Cache) cluster.Reset() fakeEC2API.Reset() fakeSSMAPI.Reset() - launchTemplateCache.Flush() - unavailableOfferingsCache.Flush() - ssmCache.Flush() - ec2Cache.Flush() - instanceTypeCache.Flush() - kubernetesVersionCache.Flush() - securityGroupProvider.Reset() - launchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") - launchTemplateProvider.ClusterEndpoint = "https://test-cluster" + awsCtx.RestProviderCache() + awsCtx.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") + awsCtx.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" // Reset the pricing provider, so we don't cross-pollinate pricing data - instanceTypesProvider = instancetype.NewProvider( + awsCtx.InstanceTypesProvider = instancetype.NewProvider( "", - instanceTypeCache, + cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), fakeEC2API, - subnetProvider, - unavailableOfferingsCache, - pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})), + awsCtx.SubnetProvider, + awsCtx.UnavailableOfferingsCache, + pricing.NewProvider(ctx, &fake.PricingAPI{}, fakeEC2API, "", make(chan struct{})), ) }) var _ = AfterEach(func() { - ExpectCleanedUp(ctx, env.Client) + ExpectCleanedUp(ctx, awsCtx.KubeClient) }) var _ = Describe("LaunchTemplates", func() { It("should default to a generated launch template", func() { - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() fmt.Println(cluster.Nodes()) - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) @@ -276,10 +231,10 @@ var _ = Describe("LaunchTemplates", func() { Name: nodeTemplate.Name, }, }) - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) @@ -302,10 +257,10 @@ var _ = Describe("LaunchTemplates", func() { It("should allow a launch template to be specified", func() { nodeTemplate.Spec.LaunchTemplateName = aws.String("test-launch-template") nodeTemplate.Spec.SecurityGroupSelector = nil - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) input := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(input.LaunchTemplateConfigs).To(HaveLen(1)) @@ -344,13 +299,13 @@ var _ = Describe("LaunchTemplates", func() { Limits: v1.ResourceList{v1alpha1.ResourceNVIDIAGPU: resource.MustParse("1")}, } - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod1 := coretest.UnschedulablePod(coretest.PodOptions{ Tolerations: []v1.Toleration{t1, t2, t3}, ResourceRequirements: rr, }) - ExpectProvisioned(ctx, env.Client, cluster, prov, pod1) - ExpectScheduled(ctx, env.Client, pod1) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod1) + ExpectScheduled(ctx, awsCtx.KubeClient, pod1) Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) name1 := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop().LaunchTemplateConfigs[0].LaunchTemplateSpecification.LaunchTemplateName @@ -358,18 +313,18 @@ var _ = Describe("LaunchTemplates", func() { Tolerations: []v1.Toleration{t2, t3, t1}, ResourceRequirements: rr, }) - ExpectProvisioned(ctx, env.Client, cluster, prov, pod2) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod2) - ExpectScheduled(ctx, env.Client, pod2) + ExpectScheduled(ctx, awsCtx.KubeClient, pod2) Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) name2 := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop().LaunchTemplateConfigs[0].LaunchTemplateSpecification.LaunchTemplateName Expect(name1).To(Equal(name2)) }) - It("should recover from an out-of-sync launch template cache", func() { - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + FIt("should recover from an out-of-sync launch template cache", func() { + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) firstLt := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() @@ -381,19 +336,19 @@ var _ = Describe("LaunchTemplates", func() { fakeEC2API.CreateFleetBehavior.Error.Set(awserr.New("InvalidLaunchTemplateName.NotFoundException", "", errors.New(""))) pod = coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) // should call fleet twice. Once will fail on invalid LT and the next will succeed fleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(aws.StringValue(fleetInput.LaunchTemplateConfigs[0].LaunchTemplateSpecification.LaunchTemplateName)).To(Equal(ltName)) - ExpectScheduled(ctx, env.Client, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) }) }) Context("Labels", func() { It("should apply labels to the node", func() { - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + node := ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(node.Labels).To(HaveKey(v1.LabelOSStable)) Expect(node.Labels).To(HaveKey(v1.LabelArchStable)) Expect(node.Labels).To(HaveKey(v1.LabelInstanceTypeStable)) @@ -412,13 +367,13 @@ var _ = Describe("LaunchTemplates", func() { }, }}) nodeTemplate.Spec.AMISelector = map[string]string{"karpenter.sh/discovery": "my-cluster"} - ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, env.Client, newProvisioner) - Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(newProvisioner), newProvisioner)).To(Succeed()) + ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) + Expect(awsCtx.KubeClient.Get(ctx, client.ObjectKeyFromObject(newProvisioner), newProvisioner)).To(Succeed()) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - node := ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + node := ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(node.Labels).To(HaveKeyWithValue(v1alpha1.LabelInstanceAMIID, "ami-123")) }) }) @@ -426,10 +381,10 @@ var _ = Describe("LaunchTemplates", func() { It("should tag with provisioner name", func() { provisionerName := "the-provisioner" provisioner.Name = provisionerName - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(createFleetInput.TagSpecifications).To(HaveLen(3)) @@ -453,10 +408,10 @@ var _ = Describe("LaunchTemplates", func() { "tag1": "tag1value", "tag2": "tag2value", } - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(createFleetInput.TagSpecifications).To(HaveLen(3)) @@ -477,10 +432,10 @@ var _ = Describe("LaunchTemplates", func() { v1alpha5.ProvisionerNameLabelKey: "myprovisioner", "Name": "myname", } - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(createFleetInput.TagSpecifications).To(HaveLen(3)) @@ -508,10 +463,10 @@ var _ = Describe("LaunchTemplates", func() { Tags: settingsTags, })) - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(createFleetInput.TagSpecifications).To(HaveLen(3)) @@ -538,10 +493,10 @@ var _ = Describe("LaunchTemplates", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ Tags: settingsTags, })) - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(createFleetInput.TagSpecifications).To(HaveLen(3)) @@ -563,10 +518,10 @@ var _ = Describe("LaunchTemplates", func() { Context("Block Device Mappings", func() { It("should default AL2 block device mappings", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(len(input.LaunchTemplateData.BlockDeviceMappings)).To(Equal(1)) @@ -600,10 +555,10 @@ var _ = Describe("LaunchTemplates", func() { }, }, } - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(input.LaunchTemplateData.BlockDeviceMappings[0].Ebs).To(Equal(&ec2.LaunchTemplateEbsBlockDeviceRequest{ @@ -649,10 +604,10 @@ var _ = Describe("LaunchTemplates", func() { }, }, } - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() @@ -662,10 +617,10 @@ var _ = Describe("LaunchTemplates", func() { }) It("should default bottlerocket second volume with root volume size", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(len(input.LaunchTemplateData.BlockDeviceMappings)).To(Equal(2)) @@ -680,10 +635,10 @@ var _ = Describe("LaunchTemplates", func() { }) It("should not default block device mappings for custom AMIFamilies", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyCustom - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(len(input.LaunchTemplateData.BlockDeviceMappings)).To(Equal(0)) @@ -703,10 +658,10 @@ var _ = Describe("LaunchTemplates", func() { }, }, } - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(len(input.LaunchTemplateData.BlockDeviceMappings)).To(Equal(1)) @@ -720,7 +675,7 @@ var _ = Describe("LaunchTemplates", func() { }) Context("Ephemeral Storage", func() { It("should pack pods when a daemonset has an ephemeral-storage request", func() { - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate, coretest.DaemonSet( + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate, coretest.DaemonSet( coretest.DaemonSetOptions{PodOptions: coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), @@ -729,39 +684,39 @@ var _ = Describe("LaunchTemplates", func() { }}, )) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) }) It("should pack pods with any ephemeral-storage request", func() { - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ v1.ResourceEphemeralStorage: resource.MustParse("1G"), }}}) - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) }) It("should pack pods with large ephemeral-storage request", func() { - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ v1.ResourceEphemeralStorage: resource.MustParse("10Gi"), }}}) - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) }) It("should not pack pods if the sum of pod ephemeral-storage and overhead exceeds node capacity", func() { - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ v1.ResourceEphemeralStorage: resource.MustParse("19Gi"), }}}) - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectNotScheduled(ctx, awsCtx.KubeClient, pod) }) It("should launch multiple nodes if sum of pod ephemeral-storage requests exceeds a single nodes capacity", func() { var nodes []*v1.Node - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pods := []*v1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ @@ -776,14 +731,14 @@ var _ = Describe("LaunchTemplates", func() { }, }), } - ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pods...) for _, pod := range pods { - nodes = append(nodes, ExpectScheduled(ctx, env.Client, pod)) + nodes = append(nodes, ExpectScheduled(ctx, awsCtx.KubeClient, pod)) } Expect(nodes).To(HaveLen(2)) }) It("should only pack pods with ephemeral-storage requests that will fit on an available node", func() { - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pods := []*v1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ @@ -798,20 +753,20 @@ var _ = Describe("LaunchTemplates", func() { }, }), } - ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) - ExpectScheduled(ctx, env.Client, pods[0]) - ExpectNotScheduled(ctx, env.Client, pods[1]) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pods...) + ExpectScheduled(ctx, awsCtx.KubeClient, pods[0]) + ExpectNotScheduled(ctx, awsCtx.KubeClient, pods[1]) }) It("should not pack pod if no available instance types have enough storage", func() { - ExpectApplied(ctx, env.Client, provisioner) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ v1.ResourceEphemeralStorage: resource.MustParse("150Gi"), }, }, }) - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectNotScheduled(ctx, awsCtx.KubeClient, pod) }) It("should pack pods using the blockdevicemappings from the provider spec when defined", func() { nodeTemplate.Spec.BlockDeviceMappings = []*v1alpha1.BlockDeviceMapping{{ @@ -820,17 +775,17 @@ var _ = Describe("LaunchTemplates", func() { VolumeSize: resource.NewScaledQuantity(50, resource.Giga), }, }} - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ v1.ResourceEphemeralStorage: resource.MustParse("25Gi"), }, }, }) - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) // capacity isn't recorded on the node any longer, but we know the pod should schedule - ExpectScheduled(ctx, env.Client, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) }) It("should pack pods using blockdevicemappings for Custom AMIFamily", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyCustom @@ -848,7 +803,7 @@ var _ = Describe("LaunchTemplates", func() { }, }, } - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ // this pod can only be satisfied if `/dev/xvdb` will house all the pods. @@ -856,10 +811,10 @@ var _ = Describe("LaunchTemplates", func() { }, }, }) - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) // capacity isn't recorded on the node any longer, but we know the pod should schedule - ExpectScheduled(ctx, env.Client, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) }) }) Context("AL2", func() { @@ -964,10 +919,10 @@ var _ = Describe("LaunchTemplates", func() { }) Context("User Data", func() { It("should not specify --use-max-pods=false when using ENI-based pod density", func() { - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -979,10 +934,10 @@ var _ = Describe("LaunchTemplates", func() { EnableENILimitedPodDensity: lo.ToPtr(false), })) - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -992,10 +947,10 @@ var _ = Describe("LaunchTemplates", func() { }) It("should specify --use-max-pods=false and --max-pods user value when user specifies maxPods in Provisioner", func() { provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{MaxPods: aws.Int32(10)} - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1011,10 +966,10 @@ var _ = Describe("LaunchTemplates", func() { v1.ResourceEphemeralStorage: resource.MustParse("2Gi"), }, } - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1037,10 +992,10 @@ var _ = Describe("LaunchTemplates", func() { v1.ResourceEphemeralStorage: resource.MustParse("2Gi"), }, } - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1063,10 +1018,10 @@ var _ = Describe("LaunchTemplates", func() { "nodefs.inodesFree": "5%", }, } - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1089,10 +1044,10 @@ var _ = Describe("LaunchTemplates", func() { "nodefs.inodesFree": "5%", }, } - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1115,10 +1070,10 @@ var _ = Describe("LaunchTemplates", func() { "nodefs.inodesFree": {Duration: time.Minute * 5}, }, } - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1137,10 +1092,10 @@ var _ = Describe("LaunchTemplates", func() { provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{ EvictionMaxPodGracePeriod: aws.Int32(300), } - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1152,10 +1107,10 @@ var _ = Describe("LaunchTemplates", func() { provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{ PodsPerCore: aws.Int32(2), } - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1167,10 +1122,10 @@ var _ = Describe("LaunchTemplates", func() { PodsPerCore: aws.Int32(2), MaxPods: aws.Int32(100), } - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1179,10 +1134,10 @@ var _ = Describe("LaunchTemplates", func() { Expect(string(userData)).To(ContainSubstring(fmt.Sprintf("--max-pods=%d", 100))) }) It("should specify --container-runtime containerd by default", func() { - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1191,10 +1146,10 @@ var _ = Describe("LaunchTemplates", func() { }) It("should specify dockerd if specified in the provisionerSpec", func() { provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{ContainerRuntime: aws.String("dockerd")} - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1203,7 +1158,7 @@ var _ = Describe("LaunchTemplates", func() { }) It("should specify --container-runtime containerd when using Neuron GPUs", func() { provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{Key: v1alpha1.LabelInstanceCategory, Operator: v1.NodeSelectorOpExists}} - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod(coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ @@ -1215,8 +1170,8 @@ var _ = Describe("LaunchTemplates", func() { }, }, }) - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1225,7 +1180,7 @@ var _ = Describe("LaunchTemplates", func() { }) It("should specify --container-runtime containerd when using Nvidia GPUs", func() { provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{Key: v1alpha1.LabelInstanceCategory, Operator: v1.NodeSelectorOpExists}} - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod(coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ @@ -1237,8 +1192,8 @@ var _ = Describe("LaunchTemplates", func() { }, }, }) - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1247,10 +1202,10 @@ var _ = Describe("LaunchTemplates", func() { }) It("should specify --dns-cluster-ip and --ip-family when running in an ipv6 cluster", func() { launchTemplateProvider.KubeDNSIP = net.ParseIP("fd4b:121b:812b::a") - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1263,10 +1218,10 @@ var _ = Describe("LaunchTemplates", func() { provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{ ImageGCHighThresholdPercent: aws.Int32(50), } - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1277,10 +1232,10 @@ var _ = Describe("LaunchTemplates", func() { provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{ ImageGCLowThresholdPercent: aws.Int32(50), } - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1299,13 +1254,13 @@ var _ = Describe("LaunchTemplates", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket provisioner.Spec.Taints = []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectNoExecute}} provisioner.Spec.StartupTaints = []v1.Taint{{Key: "baz", Value: "bin", Effect: v1.TaintEffectNoExecute}} - ExpectApplied(ctx, env.Client, nodeTemplate, provisioner) - Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(provisioner), provisioner)).To(Succeed()) + ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate, provisioner) + Expect(awsCtx.KubeClient.Get(ctx, client.ObjectKeyFromObject(provisioner), provisioner)).To(Succeed()) pod := coretest.UnschedulablePod(coretest.PodOptions{ Tolerations: []v1.Toleration{{Operator: v1.TolerationOpExists}}, }) - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1324,13 +1279,13 @@ var _ = Describe("LaunchTemplates", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket provisioner.Spec.Taints = []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectNoExecute}} provisioner.Spec.StartupTaints = []v1.Taint{{Key: "baz", Value: "bin", Effect: v1.TaintEffectNoExecute}} - ExpectApplied(ctx, env.Client, nodeTemplate, provisioner) - Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(provisioner), provisioner)).To(Succeed()) + ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate, provisioner) + Expect(awsCtx.KubeClient.Get(ctx, client.ObjectKeyFromObject(provisioner), provisioner)).To(Succeed()) pod := coretest.UnschedulablePod(coretest.PodOptions{ Tolerations: []v1.Toleration{{Operator: v1.TolerationOpExists}}, }) - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1347,26 +1302,26 @@ var _ = Describe("LaunchTemplates", func() { })) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: "doesnotexist"}}) - ExpectApplied(ctx, env.Client, newProvisioner) + ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) // This will not be scheduled since we were pointed to a non-existent awsnodetemplate resource. - ExpectNotScheduled(ctx, env.Client, pod) + ExpectNotScheduled(ctx, awsCtx.KubeClient, pod) }) It("should not bootstrap on invalid toml user data", func() { nodeTemplate.Spec.UserData = aws.String("#/bin/bash\n ./not-toml.sh") nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, env.Client, newProvisioner) + ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) // This will not be scheduled since userData cannot be generated for the prospective node. - ExpectNotScheduled(ctx, env.Client, pod) + ExpectNotScheduled(ctx, awsCtx.KubeClient, pod) }) It("should override system reserved values in user data", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) provisioner = test.Provisioner(coretest.ProvisionerOptions{ ProviderRef: &v1alpha5.ProviderRef{ Name: nodeTemplate.Name, @@ -1379,10 +1334,10 @@ var _ = Describe("LaunchTemplates", func() { }, }, }) - ExpectApplied(ctx, env.Client, provisioner) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1396,7 +1351,7 @@ var _ = Describe("LaunchTemplates", func() { }) It("should override kube reserved values in user data", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) provisioner = test.Provisioner(coretest.ProvisionerOptions{ ProviderRef: &v1alpha5.ProviderRef{ Name: nodeTemplate.Name, @@ -1409,10 +1364,10 @@ var _ = Describe("LaunchTemplates", func() { }, }, }) - ExpectApplied(ctx, env.Client, provisioner) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1426,7 +1381,7 @@ var _ = Describe("LaunchTemplates", func() { }) It("should override kube reserved values in user data", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) provisioner = test.Provisioner(coretest.ProvisionerOptions{ ProviderRef: &v1alpha5.ProviderRef{ Name: nodeTemplate.Name, @@ -1439,10 +1394,10 @@ var _ = Describe("LaunchTemplates", func() { }, }, }) - ExpectApplied(ctx, env.Client, provisioner) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1464,10 +1419,10 @@ var _ = Describe("LaunchTemplates", func() { MaxPods: aws.Int32(10), }, }) - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1487,12 +1442,12 @@ var _ = Describe("LaunchTemplates", func() { content, err := os.ReadFile("testdata/al2_userdata_input.golden") Expect(err).To(BeNil()) nodeTemplate.Spec.UserData = aws.String(string(content)) - ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, env.Client, newProvisioner) + ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1510,12 +1465,12 @@ var _ = Describe("LaunchTemplates", func() { content, err := os.ReadFile("testdata/al2_no_mime_userdata_input.golden") Expect(err).To(BeNil()) nodeTemplate.Spec.UserData = aws.String(string(content)) - ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, env.Client, newProvisioner) + ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1530,12 +1485,12 @@ var _ = Describe("LaunchTemplates", func() { EnableENILimitedPodDensity: lo.ToPtr(false), })) nodeTemplate.Spec.UserData = nil - ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, env.Client, newProvisioner) + ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1555,12 +1510,12 @@ var _ = Describe("LaunchTemplates", func() { Architecture: aws.String("x86_64"), CreationDate: aws.String("2022-08-15T12:00:00Z")}, }}) - ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, env.Client, newProvisioner) + ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect("ami-123").To(Equal(*input.LaunchTemplateData.ImageId)) @@ -1575,12 +1530,12 @@ var _ = Describe("LaunchTemplates", func() { Architecture: aws.String("x86_64"), CreationDate: aws.String("2022-08-15T12:00:00Z")}, }}) - ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, env.Client, newProvisioner) + ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1603,12 +1558,12 @@ var _ = Describe("LaunchTemplates", func() { CreationDate: aws.String("2022-08-15T12:00:00Z"), }, }}) - ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, env.Client, newProvisioner) + ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(2)) actualFilter := fakeEC2API.CalledWithDescribeImagesInput.Pop().Filters expectedFilter := []*ec2.Filter{ @@ -1635,12 +1590,12 @@ var _ = Describe("LaunchTemplates", func() { }, }}) nodeTemplate.Spec.AMISelector = map[string]string{"karpenter.sh/discovery": "my-cluster"} - ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, env.Client, newProvisioner) + ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(2)) expectedImageIds := sets.NewString("ami-123", "ami-456") actualImageIds := sets.NewString( @@ -1669,7 +1624,7 @@ var _ = Describe("LaunchTemplates", func() { }, }}) nodeTemplate.Spec.AMISelector = map[string]string{"karpenter.sh/discovery": "my-cluster"} - ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}, Requirements: []v1.NodeSelectorRequirement{ @@ -1680,10 +1635,10 @@ var _ = Describe("LaunchTemplates", func() { }, }, }) - ExpectApplied(ctx, env.Client, newProvisioner) + ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect("ami-456").To(Equal(*input.LaunchTemplateData.ImageId)) @@ -1692,12 +1647,12 @@ var _ = Describe("LaunchTemplates", func() { It("should fail if no amis match selector.", func() { fakeEC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{}}) nodeTemplate.Spec.AMISelector = map[string]string{"karpenter.sh/discovery": "my-cluster"} - ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, env.Client, newProvisioner) + ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectNotScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(0)) }) It("should fail if no instanceType matches ami requirements.", func() { @@ -1705,21 +1660,21 @@ var _ = Describe("LaunchTemplates", func() { {ImageId: aws.String("ami-123"), Architecture: aws.String("newnew"), CreationDate: aws.String("2022-01-01T12:00:00Z")}, }}) nodeTemplate.Spec.AMISelector = map[string]string{"karpenter.sh/discovery": "my-cluster"} - ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, env.Client, newProvisioner) + ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectNotScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectNotScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(0)) }) It("should choose amis from SSM if no selector specified in AWSNodeTemplate", func() { - ExpectApplied(ctx, env.Client, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, env.Client, newProvisioner) + ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(*input.LaunchTemplateData.ImageId).To(ContainSubstring("test-ami")) }) @@ -1727,10 +1682,10 @@ var _ = Describe("LaunchTemplates", func() { Context("Kubelet Args", func() { It("should specify the --dns-cluster-ip flag when clusterDNSIP is set", func() { provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{ClusterDNS: []string{"10.0.10.100"}} - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) @@ -1740,20 +1695,20 @@ var _ = Describe("LaunchTemplates", func() { }) Context("Instance Profile", func() { It("should use the default instance profile if none specified on the Provisioner", func() { - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(*input.LaunchTemplateData.IamInstanceProfile.Name).To(Equal("test-instance-profile")) }) It("should use the instance profile on the Provisioner when specified", func() { nodeTemplate.Spec.InstanceProfile = aws.String("overridden-profile") - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(*input.LaunchTemplateData.IamInstanceProfile.Name).To(Equal("overridden-profile")) @@ -1763,10 +1718,10 @@ var _ = Describe("LaunchTemplates", func() { Context("Detailed Monitoring", func() { It("should default detailed monitoring to off", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(aws.BoolValue(input.LaunchTemplateData.Monitoring.Enabled)).To(BeFalse()) @@ -1774,10 +1729,10 @@ var _ = Describe("LaunchTemplates", func() { It("should pass detailed monitoring setting to the launch template at creation", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 nodeTemplate.Spec.DetailedMonitoring = aws.Bool(true) - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(aws.BoolValue(input.LaunchTemplateData.Monitoring.Enabled)).To(BeTrue()) diff --git a/pkg/test/context.go b/pkg/test/context.go index c6512d898be6..2c4da8a5c88b 100644 --- a/pkg/test/context.go +++ b/pkg/test/context.go @@ -16,25 +16,23 @@ package test import ( "context" + "fmt" "net" - clock "k8s.io/utils/clock/testing" + "k8s.io/utils/clock" "knative.dev/pkg/ptr" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/awstesting/mock" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" - "github.com/aws/karpenter-core/pkg/cloudprovider" - "github.com/aws/karpenter-core/pkg/events" - "github.com/aws/karpenter-core/pkg/operator/scheme" - coretest "github.com/aws/karpenter-core/pkg/test" + "github.com/aws/aws-sdk-go/service/ssm/ssmiface" + "github.com/imdario/mergo" "github.com/patrickmn/go-cache" - "github.com/samber/lo" "k8s.io/client-go/tools/record" - "github.com/aws/karpenter/pkg/apis" - "github.com/aws/karpenter/pkg/apis/settings" + "github.com/aws/karpenter-core/pkg/cloudprovider" + "github.com/aws/karpenter-core/pkg/events" + awscache "github.com/aws/karpenter/pkg/cache" awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/fake" @@ -45,6 +43,8 @@ import ( "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/providers/securitygroup" "github.com/aws/karpenter/pkg/providers/subnet" + + coretest "github.com/aws/karpenter-core/pkg/test" ) type ContextOptions struct { @@ -60,12 +60,14 @@ type ContextOptions struct { InstanceProvider *instance.Provider } -func Context(ec2api ec2iface.EC2API, overrides ContextOptions) awscontext.Context { - env := coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - var ctx context.Context - ctx = coresettings.ToContext(ctx, coretest.Settings()) - ctx = settings.ToContext(ctx, Settings()) - // ctx, stop := context.WithCancel(ctx) +func Context(ctx context.Context, ec2api ec2iface.EC2API, ssmapi ssmiface.SSMAPI, + env *coretest.Environment, clock clock.Clock, overrides ...ContextOptions) *awscontext.Context { + options := ContextOptions{} + for _, override := range overrides { + if err := mergo.Merge(&options, override, mergo.WithOverride); err != nil { + panic(fmt.Sprintf("Failed to merge settings: %s", err)) + } + } // cache ssmCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) @@ -76,52 +78,96 @@ func Context(ec2api ec2iface.EC2API, overrides ContextOptions) awscontext.Contex launchTemplateCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) // Providers - pricingProvider := pricing.NewProvider(ctx, &fake.PricingAPI{}, ec2api, "", make(chan struct{})) - subnetProvider := subnet.NewProvider(ec2api) - securityGroupProvider := securitygroup.NewProvider(ec2api) - amiProvider := amifamily.NewProvider(env.Client, env.KubernetesInterface, &fake.SSMAPI{}, ec2api, ssmCache, ec2Cache, kubernetesVersionCache) - amiResolver := amifamily.New(env.Client, amiProvider) - instanceTypesProvider := instancetype.NewProvider("", instanceTypeCache, ec2api, subnetProvider, unavailableOfferingsCache, pricingProvider) - launchTemplateProvider := launchtemplate.NewProvider( - ctx, - launchTemplateCache, - ec2api, - amiResolver, - securityGroupProvider, - ptr.String("ca-bundle"), - make(chan struct{}), - net.ParseIP("10.0.100.10"), - "https://test-cluster", + sess := OptionOR(options.Session, mock.Session) + pricingProvider := OptionOR( + options.PricingProvider, + pricing.NewProvider(ctx, &fake.PricingAPI{}, ec2api, "", make(chan struct{})), + ) + subnetProvider := OptionOR(options.SubnetProvider, subnet.NewProvider(ec2api)) + securityGroupProvider := OptionOR(options.SecurityGroupProvider, securitygroup.NewProvider(ec2api)) + amiProvider := OptionOR( + options.AMIProvider, + amifamily.NewProvider(env.Client, env.KubernetesInterface, &fake.SSMAPI{}, ec2api, ssmCache, ec2Cache, kubernetesVersionCache), ) - instanceProvider := instance.NewProvider(ctx, - "", - ec2api, - unavailableOfferingsCache, - instanceTypesProvider, - subnetProvider, - launchTemplateProvider, + amiResolver := OptionOR(options.AMIResolver, amifamily.New(env.Client, amiProvider)) + instanceTypesProvider := OptionOR( + options.InstanceTypesProvider, + instancetype.NewProvider("", instanceTypeCache, ec2api, subnetProvider, unavailableOfferingsCache, pricingProvider), + ) + launchTemplateProvider := OptionOR( + options.LaunchTemplateProvider, + launchtemplate.NewProvider( + ctx, + launchTemplateCache, + ec2api, + amiResolver, + securityGroupProvider, + ptr.String("ca-bundle"), + make(chan struct{}), + net.ParseIP("10.0.100.10"), + "https://test-cluster", + ), + ) + instanceProvider := OptionOR( + options.InstanceProvider, + instance.NewProvider(ctx, + "", + ec2api, + unavailableOfferingsCache, + instanceTypesProvider, + subnetProvider, + launchTemplateProvider, + ), ) - return awscontext.Context{ + return &awscontext.Context{ Context: cloudprovider.Context{ Context: ctx, RESTConfig: env.Config, KubernetesInterface: env.KubernetesInterface, KubeClient: env.Client, EventRecorder: events.NewRecorder(&record.FakeRecorder{}), - Clock: &clock.FakeClock{}, + Clock: clock, StartAsync: nil, }, - Session: lo.FromPtrOr(&overrides.Session, mock.Session), + Session: sess, EC2API: ec2api, - UnavailableOfferingsCache: lo.FromPtrOr(&overrides.UnavailableOfferingsCache, unavailableOfferingsCache), - InstanceTypesProvider: lo.FromPtrOr(&overrides.InstanceTypesProvider, instanceTypesProvider), - InstanceProvider: lo.FromPtrOr(&overrides.InstanceProvider, instanceProvider), - SubnetProvider: lo.FromPtrOr(&overrides.SubnetProvider, subnetProvider), - SecurityGroupProvider: lo.FromPtrOr(&overrides.SecurityGroupProvider, securityGroupProvider), - PricingProvider: lo.FromPtrOr(&overrides.PricingProvider, pricingProvider), - AMIProvider: lo.FromPtrOr(&overrides.AMIProvider, amiProvider), - AMIResolver: lo.FromPtrOr(&overrides.AMIResolver, amiResolver), - LaunchTemplateProvider: lo.FromPtrOr(&overrides.LaunchTemplateProvider, launchTemplateProvider), + UnavailableOfferingsCache: unavailableOfferingsCache, + InstanceTypesProvider: instanceTypesProvider, + InstanceProvider: instanceProvider, + SubnetProvider: subnetProvider, + SecurityGroupProvider: securityGroupProvider, + PricingProvider: pricingProvider, + AMIProvider: amiProvider, + AMIResolver: amiResolver, + LaunchTemplateProvider: launchTemplateProvider, + } +} + +func UpdateContext(ctx *awscontext.Context, overrides ...ContextOptions) { + options := ContextOptions{} + for _, override := range overrides { + if err := mergo.Merge(&options, override, mergo.WithOverride); err != nil { + panic(fmt.Sprintf("Failed to merge settings: %s", err)) + } } + + ctx.Session = OptionOR(options.Session, ctx.Session) + ctx.UnavailableOfferingsCache = OptionOR(options.UnavailableOfferingsCache, ctx.UnavailableOfferingsCache) + ctx.InstanceTypesProvider = OptionOR(options.InstanceTypesProvider, ctx.InstanceTypesProvider) + ctx.InstanceProvider = OptionOR(options.InstanceProvider, ctx.InstanceProvider) + ctx.SubnetProvider = OptionOR(options.SubnetProvider, ctx.SubnetProvider) + ctx.SecurityGroupProvider = OptionOR(options.SecurityGroupProvider, ctx.SecurityGroupProvider) + ctx.PricingProvider = OptionOR(options.PricingProvider, ctx.PricingProvider) + ctx.AMIProvider = OptionOR(options.AMIProvider, ctx.AMIProvider) + ctx.AMIResolver = OptionOR(options.AMIResolver, ctx.AMIResolver) + ctx.LaunchTemplateProvider = OptionOR(options.LaunchTemplateProvider, ctx.LaunchTemplateProvider) +} + +func OptionOR[T any](x *T, fallback *T) *T { + if x == nil { + return fallback + } + + return x } diff --git a/test/go.mod b/test/go.mod index 414e80ef88ef..03f7198a590a 100644 --- a/test/go.mod +++ b/test/go.mod @@ -72,6 +72,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect diff --git a/test/go.sum b/test/go.sum index a1753a9a7026..a7d164140f92 100644 --- a/test/go.sum +++ b/test/go.sum @@ -284,6 +284,8 @@ github.com/onsi/gomega v1.27.1 h1:rfztXRbg6nv/5f+Raen9RcGoSecHIFgBBLQK3Wdj754= github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -328,11 +330,16 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= From da76fb50b340aaa9e0062d72793306460c962de0 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Fri, 3 Mar 2023 11:15:33 -0800 Subject: [PATCH 27/41] adjusting tests --- .../machine/garbagecollect/suite_test.go | 77 ++--------------- .../launchtemplate/launchtemplate.go | 42 ++++----- .../{launchtemplate_test.go => suite_test.go} | 36 ++------ pkg/test/context.go | 85 ++++++------------- 4 files changed, 56 insertions(+), 184 deletions(-) rename pkg/providers/launchtemplate/{launchtemplate_test.go => suite_test.go} (98%) diff --git a/pkg/controllers/machine/garbagecollect/suite_test.go b/pkg/controllers/machine/garbagecollect/suite_test.go index 1ddfa1ad3087..00bb7be4355d 100644 --- a/pkg/controllers/machine/garbagecollect/suite_test.go +++ b/pkg/controllers/machine/garbagecollect/suite_test.go @@ -17,72 +17,47 @@ package garbagecollect_test import ( "context" "fmt" - "net" "sync" "testing" "time" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/awstesting/mock" "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/patrickmn/go-cache" "github.com/samber/lo" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/record" clock "k8s.io/utils/clock/testing" . "knative.dev/pkg/logging/testing" - "knative.dev/pkg/ptr" "sigs.k8s.io/controller-runtime/pkg/client" coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" - "github.com/aws/karpenter-core/pkg/events" "github.com/aws/karpenter-core/pkg/operator/controller" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" + "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" - awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/cloudprovider" awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/controllers/machine/garbagecollect" "github.com/aws/karpenter/pkg/controllers/machine/link" "github.com/aws/karpenter/pkg/fake" - "github.com/aws/karpenter/pkg/providers/amifamily" - "github.com/aws/karpenter/pkg/providers/instance" - "github.com/aws/karpenter/pkg/providers/instancetype" - "github.com/aws/karpenter/pkg/providers/launchtemplate" - "github.com/aws/karpenter/pkg/providers/pricing" - "github.com/aws/karpenter/pkg/providers/securitygroup" - "github.com/aws/karpenter/pkg/providers/subnet" "github.com/aws/karpenter/pkg/test" ) var ctx context.Context +var awsCtx *awscontext.Context var env *coretest.Environment -var instanceTypeCache *cache.Cache -var unavailableOfferingsCache *awscache.UnavailableOfferings -var launchTemplateCache *cache.Cache -var ssmCache *cache.Cache -var kubernetesVersionCache *cache.Cache -var ec2Cache *cache.Cache var ec2API *fake.EC2API var cloudProvider *cloudprovider.CloudProvider var garbageCollectController controller.Controller var linkedMachineCache *cache.Cache -var amiProvider *amifamily.Provider -var amiResolver *amifamily.Resolver -var securityGroupProvider *securitygroup.Provider -var pricingProvider *pricing.Provider -var subnetProvider *subnet.Provider -var launchTemplateProvider *launchtemplate.Provider -var instanceTypeProvider *instancetype.Provider -var instanceProvider *instance.Provider func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -94,51 +69,11 @@ var _ = BeforeSuite(func() { ctx = coresettings.ToContext(ctx, coretest.Settings()) ctx = settings.ToContext(ctx, test.Settings()) env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - unavailableOfferingsCache = awscache.NewUnavailableOfferings() - launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) ec2API = &fake.EC2API{} - securityGroupProvider = securitygroup.NewProvider(ec2API) - amiProvider = amifamily.NewProvider(env.Client, env.KubernetesInterface, &fake.SSMAPI{}, ec2API, ssmCache, ec2Cache, kubernetesVersionCache) - amiResolver = amifamily.New(env.Client, amiProvider) - pricingProvider = pricing.NewProvider(ctx, &fake.PricingAPI{}, ec2API, "", make(chan struct{})) - instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - instanceTypeProvider = instancetype.NewProvider("", instanceTypeCache, ec2API, subnetProvider, unavailableOfferingsCache, pricingProvider) - launchTemplateProvider = launchtemplate.NewProvider( - ctx, - launchTemplateCache, - ec2API, - amiResolver, - securityGroupProvider, - ptr.String("ca-bundle"), - make(chan struct{}), - net.ParseIP("10.0.100.10"), - "https://test-cluster", - ) - instanceProvider = instance.NewProvider(ctx, "", ec2API, unavailableOfferingsCache, instanceTypeProvider, subnetProvider, launchTemplateProvider) - cloudProvider = cloudprovider.New(awscontext.Context{ - Context: corecloudprovider.Context{ - Context: ctx, - RESTConfig: env.Config, - KubernetesInterface: env.KubernetesInterface, - KubeClient: env.Client, - EventRecorder: events.NewRecorder(&record.FakeRecorder{}), - Clock: &clock.FakeClock{}, - StartAsync: nil, - }, - SubnetProvider: subnet.NewProvider(ec2API), - SecurityGroupProvider: securityGroupProvider, - Session: mock.Session, - UnavailableOfferingsCache: unavailableOfferingsCache, - EC2API: ec2API, - InstanceProvider: instanceProvider, - AMIProvider: amiProvider, - AMIResolver: amiResolver, - LaunchTemplateProvider: launchTemplateProvider, - PricingProvider: pricingProvider, - }) + awsCtx = test.Context(ctx, ec2API, &fake.SSMAPI{}, env, &clock.FakeClock{}) + + cloudProvider = cloudprovider.New(*awsCtx) + linkedMachineCache = cache.New(time.Minute*10, time.Second*10) linkController := &link.Controller{ Cache: linkedMachineCache, diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index 3cd7655881d7..eedd5683577b 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -55,27 +55,27 @@ type Provider struct { ec2api ec2iface.EC2API amiFamily *amifamily.Resolver securityGroupProvider *securitygroup.Provider - Cache *cache.Cache + cache *cache.Cache caBundle *string cm *pretty.ChangeMonitor KubeDNSIP net.IP ClusterEndpoint string } -func NewProvider(ctx context.Context, Cache *cache.Cache, ec2api ec2iface.EC2API, amiFamily *amifamily.Resolver, securityGroupProvider *securitygroup.Provider, caBundle *string, startAsync <-chan struct{}, kubeDNSIP net.IP, clusterEndpoint string) *Provider { +func NewProvider(ctx context.Context, cache *cache.Cache, ec2api ec2iface.EC2API, amiFamily *amifamily.Resolver, securityGroupProvider *securitygroup.Provider, caBundle *string, startAsync <-chan struct{}, kubeDNSIP net.IP, clusterEndpoint string) *Provider { l := &Provider{ ec2api: ec2api, amiFamily: amiFamily, securityGroupProvider: securityGroupProvider, - Cache: Cache, + cache: cache, caBundle: caBundle, cm: pretty.NewChangeMonitor(), KubeDNSIP: kubeDNSIP, ClusterEndpoint: clusterEndpoint, } - l.Cache.OnEvicted(l.cachedEvictedFunc(ctx)) + l.cache.OnEvicted(l.cachedEvictedFunc(ctx)) go func() { - // only hydrate Cache once elected leader + // only hydrate cache once elected leader select { case <-startAsync: case <-ctx.Done(): @@ -115,15 +115,15 @@ func (p *Provider) EnsureAll(ctx context.Context, nodeTemplate *v1alpha1.AWSNode return launchTemplates, nil } -// Invalidate deletes a launch template from Cache if it exists +// Invalidate deletes a launch template from cache if it exists func (p *Provider) Invalidate(ctx context.Context, ltName string, ltID string) { ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("launch-template-name", ltName, "launch-template-id", ltID)) p.Lock() defer p.Unlock() - defer p.Cache.OnEvicted(p.cachedEvictedFunc(ctx)) - p.Cache.OnEvicted(nil) - logging.FromContext(ctx).Debugf("invalidating launch template in the Cache because it no longer exists") - p.Cache.Delete(ltName) + defer p.cache.OnEvicted(p.cachedEvictedFunc(ctx)) + p.cache.OnEvicted(nil) + logging.FromContext(ctx).Debugf("invalidating launch template in the cache because it no longer exists") + p.cache.Delete(ltName) } func launchTemplateName(options *amifamily.LaunchTemplate) string { @@ -164,9 +164,9 @@ func (p *Provider) ensureLaunchTemplate(ctx context.Context, options *amifamily. var launchTemplate *ec2.LaunchTemplate name := launchTemplateName(options) ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("launch-template-name", name)) - // Read from Cache - if launchTemplate, ok := p.Cache.Get(name); ok { - p.Cache.SetDefault(name, launchTemplate) + // Read from cache + if launchTemplate, ok := p.cache.Get(name); ok { + p.cache.SetDefault(name, launchTemplate) return launchTemplate.(*ec2.LaunchTemplate), nil } // Attempt to find an existing LT. @@ -189,7 +189,7 @@ func (p *Provider) ensureLaunchTemplate(ctx context.Context, options *amifamily. } launchTemplate = output.LaunchTemplates[0] } - p.Cache.SetDefault(name, launchTemplate) + p.cache.SetDefault(name, launchTemplate) return launchTemplate, nil } @@ -268,30 +268,30 @@ func (p *Provider) volumeSize(quantity *resource.Quantity) *int64 { return aws.Int64(int64(math.Ceil(quantity.AsApproximateFloat64() / math.Pow(2, 30)))) } -// hydrateCache queries for existing Launch Templates created by Karpenter for the current cluster and adds to the LT Cache. +// hydrateCache queries for existing Launch Templates created by Karpenter for the current cluster and adds to the LT cache. // Any error during hydration will result in a panic func (p *Provider) hydrateCache(ctx context.Context) { clusterName := awssettings.FromContext(ctx).ClusterName ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("tag-key", karpenterManagedTagKey, "tag-value", clusterName)) - logging.FromContext(ctx).Debugf("hydrating the launch template Cache") + logging.FromContext(ctx).Debugf("hydrating the launch template cache") if err := p.ec2api.DescribeLaunchTemplatesPagesWithContext(ctx, &ec2.DescribeLaunchTemplatesInput{ Filters: []*ec2.Filter{{Name: aws.String(fmt.Sprintf("tag:%s", karpenterManagedTagKey)), Values: []*string{aws.String(clusterName)}}}, }, func(output *ec2.DescribeLaunchTemplatesOutput, _ bool) bool { for _, lt := range output.LaunchTemplates { - p.Cache.SetDefault(*lt.LaunchTemplateName, lt) + p.cache.SetDefault(*lt.LaunchTemplateName, lt) } return true }); err != nil { - logging.FromContext(ctx).Errorf(fmt.Sprintf("Unable to hydrate the AWS launch template Cache, %s", err)) + logging.FromContext(ctx).Errorf(fmt.Sprintf("Unable to hydrate the AWS launch template cache, %s", err)) } - logging.FromContext(ctx).With("item-count", p.Cache.ItemCount()).Debugf("finished hydrating the launch template Cache") + logging.FromContext(ctx).With("item-count", p.cache.ItemCount()).Debugf("finished hydrating the launch template cache") } func (p *Provider) cachedEvictedFunc(ctx context.Context) func(string, interface{}) { return func(key string, lt interface{}) { p.Lock() defer p.Unlock() - if _, expiration, _ := p.Cache.GetWithExpiration(key); expiration.After(time.Now()) { + if _, expiration, _ := p.cache.GetWithExpiration(key); expiration.After(time.Now()) { return } launchTemplate := lt.(*ec2.LaunchTemplate) @@ -315,5 +315,5 @@ func (p *Provider) getInstanceProfile(ctx context.Context, nodeTemplate *v1alpha } func (p *Provider) Reset() { - p.Cache.Flush() + p.cache.Flush() } diff --git a/pkg/providers/launchtemplate/launchtemplate_test.go b/pkg/providers/launchtemplate/suite_test.go similarity index 98% rename from pkg/providers/launchtemplate/launchtemplate_test.go rename to pkg/providers/launchtemplate/suite_test.go index e27201e0e608..57a3b44bc545 100644 --- a/pkg/providers/launchtemplate/launchtemplate_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -42,7 +42,6 @@ import ( "k8s.io/client-go/tools/record" clock "k8s.io/utils/clock/testing" . "knative.dev/pkg/logging/testing" - "knative.dev/pkg/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aws/karpenter/pkg/apis" @@ -54,7 +53,6 @@ import ( "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/providers/amifamily/bootstrap" "github.com/aws/karpenter/pkg/providers/instancetype" - "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/test" @@ -82,13 +80,7 @@ var prov *provisioning.Provisioner var provisioner *v1alpha5.Provisioner var nodeTemplate *v1alpha1.AWSNodeTemplate var cluster *state.Cluster - -// cache var launchTemplateCache *cache.Cache - -// providers -var pricingProvider *pricing.Provider -var launchTemplateProvider *launchtemplate.Provider var cloudProvider *cloudprovider.CloudProvider func TestAWS(t *testing.T) { @@ -106,31 +98,14 @@ var _ = BeforeSuite(func() { fakeEC2API = &fake.EC2API{} fakeSSMAPI = &fake.SSMAPI{} fakeClock = &clock.FakeClock{} - awsCtx = test.Context(ctx, fakeEC2API, fakeSSMAPI, env, fakeClock) - pricingProvider = pricing.NewProvider(ctx, &fake.PricingAPI{}, fakeEC2API, "", make(chan struct{})) launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - launchTemplateProvider = launchtemplate.NewProvider( - ctx, - launchTemplateCache, - fakeEC2API, - awsCtx.AMIResolver, - awsCtx.SecurityGroupProvider, - ptr.String("ca-bundle"), - make(chan struct{}), - net.ParseIP("10.0.100.10"), - "https://test-cluster", - ) - - test.UpdateContext(awsCtx, test.ContextOptions{ - LaunchTemplateProvider: launchTemplateProvider, - PricingProvider: pricingProvider, + awsCtx = test.Context(ctx, fakeEC2API, fakeSSMAPI, env, fakeClock, test.ContextOptions{ + LaunchTemplateCache: launchTemplateCache, }) - fmt.Println(launchTemplateCache == awsCtx.LaunchTemplateProvider.Cache) cloudProvider = cloudprovider.New(*awsCtx) cluster = state.NewCluster(fakeClock, awsCtx.KubeClient, cloudProvider) prov = provisioning.NewProvisioner(ctx, awsCtx.KubeClient, awsCtx.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) - }) var _ = AfterSuite(func() { @@ -170,7 +145,6 @@ var _ = BeforeEach(func() { Name: nodeTemplate.Name, }, }) - fmt.Println(launchTemplateCache == awsCtx.LaunchTemplateProvider.Cache) cluster.Reset() fakeEC2API.Reset() fakeSSMAPI.Reset() @@ -247,7 +221,7 @@ var _ = Describe("LaunchTemplates", func() { }) lastPrice := -math.MaxFloat64 for _, override := range overrides { - offeringPrice, ok := pricingProvider.SpotPrice(*override.InstanceType, *override.AvailabilityZone) + offeringPrice, ok := awsCtx.PricingProvider.SpotPrice(*override.InstanceType, *override.AvailabilityZone) Expect(ok).To(BeTrue()) Expect(offeringPrice).To(BeNumerically(">=", lastPrice)) lastPrice = offeringPrice @@ -320,7 +294,7 @@ var _ = Describe("LaunchTemplates", func() { name2 := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop().LaunchTemplateConfigs[0].LaunchTemplateSpecification.LaunchTemplateName Expect(name1).To(Equal(name2)) }) - FIt("should recover from an out-of-sync launch template cache", func() { + It("should recover from an out-of-sync launch template cache", func() { ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) @@ -1201,7 +1175,7 @@ var _ = Describe("LaunchTemplates", func() { Expect(string(userData)).To(ContainSubstring("--container-runtime containerd")) }) It("should specify --dns-cluster-ip and --ip-family when running in an ipv6 cluster", func() { - launchTemplateProvider.KubeDNSIP = net.ParseIP("fd4b:121b:812b::a") + awsCtx.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("fd4b:121b:812b::a") ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) diff --git a/pkg/test/context.go b/pkg/test/context.go index 2c4da8a5c88b..d66a5c9cae20 100644 --- a/pkg/test/context.go +++ b/pkg/test/context.go @@ -22,7 +22,6 @@ import ( "k8s.io/utils/clock" "knative.dev/pkg/ptr" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/awstesting/mock" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/aws/aws-sdk-go/service/ssm/ssmiface" @@ -48,16 +47,13 @@ import ( ) type ContextOptions struct { - Session *session.Session + SSMCache *cache.Cache + EC2Cache *cache.Cache + KubernetesVersionCache *cache.Cache + InstanceTypeCache *cache.Cache UnavailableOfferingsCache *awscache.UnavailableOfferings - SubnetProvider *subnet.Provider - SecurityGroupProvider *securitygroup.Provider - AMIProvider *amifamily.Provider - AMIResolver *amifamily.Resolver - LaunchTemplateProvider *launchtemplate.Provider - PricingProvider *pricing.Provider - InstanceTypesProvider *instancetype.Provider - InstanceProvider *instance.Provider + LaunchTemplateCache *cache.Cache + PricingAPI *fake.PricingAPI } func Context(ctx context.Context, ec2api ec2iface.EC2API, ssmapi ssmiface.SSMAPI, @@ -70,32 +66,22 @@ func Context(ctx context.Context, ec2api ec2iface.EC2API, ssmapi ssmiface.SSMAPI } // cache - ssmCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - ec2Cache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - kubernetesVersionCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - instanceTypeCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - unavailableOfferingsCache := awscache.NewUnavailableOfferings() - launchTemplateCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + ssmCache := OptionOR(options.SSMCache, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) + ec2Cache := OptionOR(options.EC2Cache, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) + kubernetesVersionCache := OptionOR(options.KubernetesVersionCache, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) + instanceTypeCache := OptionOR(options.InstanceTypeCache, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) + unavailableOfferingsCache := OptionOR(options.UnavailableOfferingsCache, awscache.NewUnavailableOfferings()) + launchTemplateCache := OptionOR(options.LaunchTemplateCache, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) + pricingAPI := OptionOR(options.PricingAPI, &fake.PricingAPI{}) // Providers - sess := OptionOR(options.Session, mock.Session) - pricingProvider := OptionOR( - options.PricingProvider, - pricing.NewProvider(ctx, &fake.PricingAPI{}, ec2api, "", make(chan struct{})), - ) - subnetProvider := OptionOR(options.SubnetProvider, subnet.NewProvider(ec2api)) - securityGroupProvider := OptionOR(options.SecurityGroupProvider, securitygroup.NewProvider(ec2api)) - amiProvider := OptionOR( - options.AMIProvider, - amifamily.NewProvider(env.Client, env.KubernetesInterface, &fake.SSMAPI{}, ec2api, ssmCache, ec2Cache, kubernetesVersionCache), - ) - amiResolver := OptionOR(options.AMIResolver, amifamily.New(env.Client, amiProvider)) - instanceTypesProvider := OptionOR( - options.InstanceTypesProvider, - instancetype.NewProvider("", instanceTypeCache, ec2api, subnetProvider, unavailableOfferingsCache, pricingProvider), - ) - launchTemplateProvider := OptionOR( - options.LaunchTemplateProvider, + pricingProvider := pricing.NewProvider(ctx, pricingAPI, ec2api, "", make(chan struct{})) + subnetProvider := subnet.NewProvider(ec2api) + securityGroupProvider := securitygroup.NewProvider(ec2api) + amiProvider := amifamily.NewProvider(env.Client, env.KubernetesInterface, &fake.SSMAPI{}, ec2api, ssmCache, ec2Cache, kubernetesVersionCache) + amiResolver := amifamily.New(env.Client, amiProvider) + instanceTypesProvider := instancetype.NewProvider("", instanceTypeCache, ec2api, subnetProvider, unavailableOfferingsCache, pricingProvider) + launchTemplateProvider := launchtemplate.NewProvider( ctx, launchTemplateCache, @@ -106,10 +92,8 @@ func Context(ctx context.Context, ec2api ec2iface.EC2API, ssmapi ssmiface.SSMAPI make(chan struct{}), net.ParseIP("10.0.100.10"), "https://test-cluster", - ), - ) - instanceProvider := OptionOR( - options.InstanceProvider, + ) + instanceProvider := instance.NewProvider(ctx, "", ec2api, @@ -117,8 +101,7 @@ func Context(ctx context.Context, ec2api ec2iface.EC2API, ssmapi ssmiface.SSMAPI instanceTypesProvider, subnetProvider, launchTemplateProvider, - ), - ) + ) return &awscontext.Context{ Context: cloudprovider.Context{ @@ -130,7 +113,7 @@ func Context(ctx context.Context, ec2api ec2iface.EC2API, ssmapi ssmiface.SSMAPI Clock: clock, StartAsync: nil, }, - Session: sess, + Session: mock.Session, EC2API: ec2api, UnavailableOfferingsCache: unavailableOfferingsCache, InstanceTypesProvider: instanceTypesProvider, @@ -144,26 +127,6 @@ func Context(ctx context.Context, ec2api ec2iface.EC2API, ssmapi ssmiface.SSMAPI } } -func UpdateContext(ctx *awscontext.Context, overrides ...ContextOptions) { - options := ContextOptions{} - for _, override := range overrides { - if err := mergo.Merge(&options, override, mergo.WithOverride); err != nil { - panic(fmt.Sprintf("Failed to merge settings: %s", err)) - } - } - - ctx.Session = OptionOR(options.Session, ctx.Session) - ctx.UnavailableOfferingsCache = OptionOR(options.UnavailableOfferingsCache, ctx.UnavailableOfferingsCache) - ctx.InstanceTypesProvider = OptionOR(options.InstanceTypesProvider, ctx.InstanceTypesProvider) - ctx.InstanceProvider = OptionOR(options.InstanceProvider, ctx.InstanceProvider) - ctx.SubnetProvider = OptionOR(options.SubnetProvider, ctx.SubnetProvider) - ctx.SecurityGroupProvider = OptionOR(options.SecurityGroupProvider, ctx.SecurityGroupProvider) - ctx.PricingProvider = OptionOR(options.PricingProvider, ctx.PricingProvider) - ctx.AMIProvider = OptionOR(options.AMIProvider, ctx.AMIProvider) - ctx.AMIResolver = OptionOR(options.AMIResolver, ctx.AMIResolver) - ctx.LaunchTemplateProvider = OptionOR(options.LaunchTemplateProvider, ctx.LaunchTemplateProvider) -} - func OptionOR[T any](x *T, fallback *T) *T { if x == nil { return fallback From 7d56ac7dfe57c688497cb06ea2ab37a524ac75c5 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Fri, 3 Mar 2023 11:20:42 -0800 Subject: [PATCH 28/41] Adjusting more tests --- pkg/controllers/machine/link/suite_test.go | 82 ++-------------------- 1 file changed, 5 insertions(+), 77 deletions(-) diff --git a/pkg/controllers/machine/link/suite_test.go b/pkg/controllers/machine/link/suite_test.go index 7e7ed0db7be2..f32ebb50ce3b 100644 --- a/pkg/controllers/machine/link/suite_test.go +++ b/pkg/controllers/machine/link/suite_test.go @@ -18,28 +18,21 @@ import ( "context" "encoding/json" "fmt" - "net" "testing" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/awstesting/mock" "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/patrickmn/go-cache" "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/tools/record" clock "k8s.io/utils/clock/testing" . "knative.dev/pkg/logging/testing" - "knative.dev/pkg/ptr" "sigs.k8s.io/controller-runtime/pkg/client" coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" - "github.com/aws/karpenter-core/pkg/events" "github.com/aws/karpenter-core/pkg/operator/controller" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" @@ -48,43 +41,20 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" - awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/cloudprovider" awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/controllers/machine/link" "github.com/aws/karpenter/pkg/fake" - "github.com/aws/karpenter/pkg/providers/amifamily" - "github.com/aws/karpenter/pkg/providers/instance" - "github.com/aws/karpenter/pkg/providers/instancetype" - "github.com/aws/karpenter/pkg/providers/launchtemplate" - "github.com/aws/karpenter/pkg/providers/pricing" - "github.com/aws/karpenter/pkg/providers/securitygroup" - "github.com/aws/karpenter/pkg/providers/subnet" "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/pkg/utils" ) var ctx context.Context +var awsCtx *awscontext.Context var env *coretest.Environment -var launchTemplateCache *cache.Cache -var unavailableOfferingsCache *awscache.UnavailableOfferings -var ssmCache *cache.Cache -var ec2Cache *cache.Cache -var kubernetesVersionCache *cache.Cache var ec2API *fake.EC2API -var ssmAPI *fake.SSMAPI var cloudProvider *cloudprovider.CloudProvider -var subnetProvider *subnet.Provider -var securityGroupProvider *securitygroup.Provider -var instanceTypeCache *cache.Cache var linkController controller.Controller -var pricingProvider *pricing.Provider -var amiProvider *amifamily.Provider -var amiResolver *amifamily.Resolver -var instanceTypeProvider *instancetype.Provider -var launchTemplateProvider *launchtemplate.Provider -var instanceProvider *instance.Provider -var instanceTypesProvider *instancetype.Provider func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -96,53 +66,11 @@ var _ = BeforeSuite(func() { ctx = coresettings.ToContext(ctx, coretest.Settings()) ctx = settings.ToContext(ctx, test.Settings()) env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - unavailableOfferingsCache = awscache.NewUnavailableOfferings() - launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) ec2API = &fake.EC2API{} - ssmAPI = &fake.SSMAPI{} - subnetProvider = subnet.NewProvider(ec2API) - securityGroupProvider = securitygroup.NewProvider(ec2API) - pricingProvider = pricing.NewProvider(ctx, &fake.PricingAPI{}, ec2API, "", make(chan struct{})) - amiProvider = amifamily.NewProvider(env.Client, env.KubernetesInterface, ssmAPI, ec2API, ssmCache, ec2Cache, kubernetesVersionCache) - amiResolver = amifamily.New(env.Client, amiProvider) - launchTemplateProvider = launchtemplate.NewProvider( - ctx, - launchTemplateCache, - ec2API, - amiResolver, - securityGroupProvider, - ptr.String("ca-bundle"), - make(chan struct{}), - net.ParseIP("10.0.100.10"), - "https://test-cluster", - ) - instanceProvider = instance.NewProvider(ctx, "", ec2API, unavailableOfferingsCache, instanceTypeProvider, subnetProvider, launchTemplateProvider) - instanceTypesProvider = instancetype.NewProvider("", cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), ec2API, subnetProvider, unavailableOfferingsCache, pricingProvider) - cloudProvider = cloudprovider.New(awscontext.Context{ - Context: corecloudprovider.Context{ - Context: ctx, - RESTConfig: env.Config, - KubernetesInterface: env.KubernetesInterface, - KubeClient: env.Client, - EventRecorder: events.NewRecorder(&record.FakeRecorder{}), - Clock: &clock.FakeClock{}, - StartAsync: nil, - }, - SubnetProvider: subnet.NewProvider(ec2API), - SecurityGroupProvider: securitygroup.NewProvider(ec2API), - Session: mock.Session, - UnavailableOfferingsCache: unavailableOfferingsCache, - EC2API: ec2API, - PricingProvider: pricingProvider, - InstanceProvider: instanceProvider, - AMIProvider: amiProvider, - AMIResolver: amiResolver, - InstanceTypesProvider: instanceTypesProvider, - }) + awsCtx = test.Context(ctx, ec2API, &fake.SSMAPI{}, env, &clock.FakeClock{}) + + cloudProvider = cloudprovider.New(*awsCtx) + linkController = link.NewController(env.Client, cloudProvider) }) From b1e3e58a1d38af45b4e1fd24f630d17cb539f242 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Fri, 3 Mar 2023 12:32:40 -0800 Subject: [PATCH 29/41] Only one failing test --- pkg/providers/instancetype/suite_test.go | 135 ++++++----------------- 1 file changed, 34 insertions(+), 101 deletions(-) diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index 48be9462e01a..a0b685e2715b 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -25,7 +25,6 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/awstesting/mock" "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -63,13 +62,9 @@ import ( "github.com/aws/karpenter/pkg/cloudprovider" awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/fake" - "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/providers/instancetype" - "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/pricing" - "github.com/aws/karpenter/pkg/providers/securitygroup" - "github.com/aws/karpenter/pkg/providers/subnet" "github.com/aws/karpenter/pkg/test" ) @@ -77,29 +72,16 @@ var ctx context.Context var stop context.CancelFunc var opts options.Options var env *coretest.Environment -var ssmCache *cache.Cache -var ec2Cache *cache.Cache -var launchTemplateCache *cache.Cache -var instanceTypeCache *cache.Cache -var kubernetesVersionCache *cache.Cache +var awsCtx *awscontext.Context var fakeEC2API *fake.EC2API var fakeSSMAPI *fake.SSMAPI var fakeClock *clock.FakeClock -var fakePricingAPI *fake.PricingAPI -var amiProvider *amifamily.Provider -var amiResolver *amifamily.Resolver -var cloudProvider *cloudprovider.CloudProvider -var unavailableOfferingsCache *awscache.UnavailableOfferings +var instanceTypeCache *cache.Cache var prov *provisioning.Provisioner var provisioner *v1alpha5.Provisioner -var launchTemplateProvider *launchtemplate.Provider var nodeTemplate *v1alpha1.AWSNodeTemplate var cluster *state.Cluster -var pricingProvider *pricing.Provider -var subnetProvider *subnet.Provider -var instanceTypeProvider *instancetype.Provider -var securityGroupProvider *securitygroup.Provider -var instanceProvider *instance.Provider +var cloudProvider *cloudprovider.CloudProvider var provisioningController controller.Controller func TestAWS(t *testing.T) { @@ -116,55 +98,13 @@ var _ = BeforeSuite(func() { fakeEC2API = &fake.EC2API{} fakeSSMAPI = &fake.SSMAPI{} - ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + fakeClock = &clock.FakeClock{} instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - fakeClock = clock.NewFakeClock(time.Now()) - unavailableOfferingsCache = awscache.NewUnavailableOfferings() - fakePricingAPI = &fake.PricingAPI{} - pricingProvider = pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})) - subnetProvider = subnet.NewProvider(fakeEC2API) - securityGroupProvider = securitygroup.NewProvider(fakeEC2API) - amiProvider = amifamily.NewProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) - amiResolver = amifamily.New(env.Client, amiProvider) - instanceTypeProvider = instancetype.NewProvider("", instanceTypeCache, fakeEC2API, subnetProvider, unavailableOfferingsCache, pricingProvider) - - launchTemplateProvider = launchtemplate.NewProvider( - ctx, - launchTemplateCache, - fakeEC2API, - amiResolver, - securityGroupProvider, - ptr.String("ca-bundle"), - make(chan struct{}), - net.ParseIP("10.0.100.10"), - "https://test-cluster", - ) - instanceProvider = instance.NewProvider(ctx, "", fakeEC2API, unavailableOfferingsCache, instanceTypeProvider, subnetProvider, launchTemplateProvider) - cloudProvider = cloudprovider.New(awscontext.Context{ - Context: corecloudprovider.Context{ - Context: ctx, - RESTConfig: env.Config, - KubernetesInterface: env.KubernetesInterface, - KubeClient: env.Client, - EventRecorder: events.NewRecorder(&record.FakeRecorder{}), - Clock: &clock.FakeClock{}, - StartAsync: nil, - }, - SubnetProvider: subnet.NewProvider(fakeEC2API), - SecurityGroupProvider: securityGroupProvider, - Session: mock.Session, - UnavailableOfferingsCache: unavailableOfferingsCache, - EC2API: fakeEC2API, - PricingProvider: pricingProvider, - AMIProvider: amiProvider, - AMIResolver: amiResolver, - LaunchTemplateProvider: launchTemplateProvider, - InstanceProvider: instanceProvider, - InstanceTypesProvider: instanceTypeProvider, + awsCtx = test.Context(ctx, fakeEC2API, fakeSSMAPI, env, fakeClock, test.ContextOptions{ + InstanceTypeCache: instanceTypeCache, }) + + cloudProvider = cloudprovider.New(*awsCtx) cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) provisioningController = provisioning.NewController(env.Client, prov, events.NewRecorder(&record.FakeRecorder{})) @@ -211,26 +151,19 @@ var _ = BeforeEach(func() { cluster.Reset() fakeEC2API.Reset() fakeSSMAPI.Reset() - fakePricingAPI.Reset() - launchTemplateCache.Flush() - unavailableOfferingsCache.Flush() - ssmCache.Flush() - ec2Cache.Flush() - kubernetesVersionCache.Flush() - instanceTypeCache.Flush() - subnetProvider.Reset() - securityGroupProvider.Reset() - launchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") - launchTemplateProvider.ClusterEndpoint = "https://test-cluster" + awsCtx.RestProviderCache() + awsCtx.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") + awsCtx.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" // Reset the pricing provider, so we don't cross-pollinate pricing data - instanceTypeProvider = instancetype.NewProvider( + awsCtx.PricingProvider = pricing.NewProvider(ctx, &fake.PricingAPI{}, fakeEC2API, "", make(chan struct{})) + awsCtx.InstanceTypesProvider = instancetype.NewProvider( "", instanceTypeCache, fakeEC2API, - subnetProvider, - unavailableOfferingsCache, - pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})), + awsCtx.SubnetProvider, + awsCtx.UnavailableOfferingsCache, + awsCtx.PricingProvider, ) }) @@ -360,7 +293,7 @@ var _ = Describe("Instance Types", func() { } ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) fakeEC2API.DescribeSpotPriceHistoryOutput.Set(generateSpotPricing(cloudProvider, provisioner)) - Expect(pricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) + Expect(awsCtx.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) pod := coretest.UnschedulablePod(coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ @@ -392,7 +325,7 @@ var _ = Describe("Instance Types", func() { // find the cheapest OD price that works cheapestODPrice := math.MaxFloat64 for _, override := range call.LaunchTemplateConfigs[0].Overrides { - odPrice, ok := pricingProvider.OnDemandPrice(*override.InstanceType) + odPrice, ok := awsCtx.PricingProvider.OnDemandPrice(*override.InstanceType) Expect(ok).To(BeTrue()) if odPrice < cheapestODPrice { cheapestODPrice = odPrice @@ -400,7 +333,7 @@ var _ = Describe("Instance Types", func() { } // and our spot prices should be cheaper than the OD price for _, override := range call.LaunchTemplateConfigs[0].Overrides { - spotPrice, ok := pricingProvider.SpotPrice(*override.InstanceType, *override.AvailabilityZone) + spotPrice, ok := awsCtx.PricingProvider.SpotPrice(*override.InstanceType, *override.AvailabilityZone) Expect(ok).To(BeTrue()) Expect(spotPrice).To(BeNumerically("<", cheapestODPrice)) } @@ -594,7 +527,7 @@ var _ = Describe("Instance Types", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), })) - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsCtx.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) for _, info := range instanceInfo { it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) @@ -602,7 +535,7 @@ var _ = Describe("Instance Types", func() { } }) It("should not set pods to 110 if using ENI-based pod density", func() { - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsCtx.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) for _, info := range instanceInfo { it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) @@ -618,7 +551,7 @@ var _ = Describe("Instance Types", func() { })) var ok bool - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsCtx.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) info, ok = lo.Find(instanceInfo, func(i *ec2.InstanceTypeInfo) bool { return aws.StringValue(i.InstanceType) == "m5.xlarge" @@ -880,7 +813,7 @@ var _ = Describe("Instance Types", func() { }) }) It("should set max-pods to user-defined value if specified", func() { - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsCtx.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{MaxPods: ptr.Int32(10)}}) for _, info := range instanceInfo { @@ -893,7 +826,7 @@ var _ = Describe("Instance Types", func() { EnablePodENI: lo.ToPtr(false), })) - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsCtx.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{MaxPods: ptr.Int32(10)}}) for _, info := range instanceInfo { @@ -902,7 +835,7 @@ var _ = Describe("Instance Types", func() { } }) It("should override pods-per-core value", func() { - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsCtx.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}}) for _, info := range instanceInfo { @@ -911,7 +844,7 @@ var _ = Describe("Instance Types", func() { } }) It("should take the minimum of pods-per-core and max-pods", func() { - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsCtx.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(4), MaxPods: ptr.Int32(20)}}) for _, info := range instanceInfo { @@ -920,7 +853,7 @@ var _ = Describe("Instance Types", func() { } }) It("should ignore pods-per-core when using Bottlerocket AMI", func() { - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsCtx.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}}) @@ -935,7 +868,7 @@ var _ = Describe("Instance Types", func() { EnableENILimitedPodDensity: lo.ToPtr(false), })) - instanceInfo, err := instanceTypeProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsCtx.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(0)}}) for _, info := range instanceInfo { @@ -1051,7 +984,7 @@ var _ = Describe("Instance Types", func() { ExpectNotScheduled(ctx, env.Client, pod) // capacity shortage is over - expire the item from the cache and try again fakeEC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{}) - unavailableOfferingsCache.Delete("inf1.6xlarge", "test-zone-1a", v1alpha5.CapacityTypeOnDemand) + awsCtx.UnavailableOfferingsCache.Delete("inf1.6xlarge", "test-zone-1a", v1alpha5.CapacityTypeOnDemand) ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "inf1.6xlarge")) @@ -1171,7 +1104,7 @@ var _ = Describe("Instance Types", func() { node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.LabelCapacityType, v1alpha5.CapacityTypeSpot)) }) - It("should fail to launch capacity when there is no zonal availability for spot", func() { + FIt("should fail to launch capacity when there is no zonal availability for spot", func() { now := time.Now() fakeEC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ SpotPriceHistory: []*ec2.SpotPrice{ @@ -1183,8 +1116,8 @@ var _ = Describe("Instance Types", func() { }, }, }) - Expect(pricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) - Eventually(func() bool { return pricingProvider.SpotLastUpdated().After(now) }).Should(BeTrue()) + Expect(awsCtx.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) + Eventually(func() bool { return awsCtx.PricingProvider.SpotLastUpdated().After(now) }).Should(BeTrue()) provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeSpot}}, @@ -1210,8 +1143,8 @@ var _ = Describe("Instance Types", func() { }, }, }) - Expect(pricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) - Eventually(func() bool { return pricingProvider.SpotLastUpdated().After(now) }).Should(BeTrue()) + Expect(awsCtx.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) + Eventually(func() bool { return awsCtx.PricingProvider.SpotLastUpdated().After(now) }).Should(BeTrue()) // not restricting to the zone so we can get any zone provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ From edb0c3db5c9d0b899fc8268c3e935382338d210a Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Fri, 3 Mar 2023 12:58:55 -0800 Subject: [PATCH 30/41] Fixed All tests --- .../machine/garbagecollect/suite_test.go | 4 ++-- pkg/controllers/machine/link/suite_test.go | 4 ++-- pkg/providers/instancetype/suite_test.go | 17 +++-------------- pkg/providers/launchtemplate/suite_test.go | 15 ++------------- pkg/test/context.go | 4 ++-- 5 files changed, 11 insertions(+), 33 deletions(-) diff --git a/pkg/controllers/machine/garbagecollect/suite_test.go b/pkg/controllers/machine/garbagecollect/suite_test.go index 00bb7be4355d..cc3ed715c0e3 100644 --- a/pkg/controllers/machine/garbagecollect/suite_test.go +++ b/pkg/controllers/machine/garbagecollect/suite_test.go @@ -52,7 +52,7 @@ import ( ) var ctx context.Context -var awsCtx *awscontext.Context +var awsCtx awscontext.Context var env *coretest.Environment var ec2API *fake.EC2API var cloudProvider *cloudprovider.CloudProvider @@ -72,7 +72,7 @@ var _ = BeforeSuite(func() { ec2API = &fake.EC2API{} awsCtx = test.Context(ctx, ec2API, &fake.SSMAPI{}, env, &clock.FakeClock{}) - cloudProvider = cloudprovider.New(*awsCtx) + cloudProvider = cloudprovider.New(awsCtx) linkedMachineCache = cache.New(time.Minute*10, time.Second*10) linkController := &link.Controller{ diff --git a/pkg/controllers/machine/link/suite_test.go b/pkg/controllers/machine/link/suite_test.go index f32ebb50ce3b..db8bdd6b17e8 100644 --- a/pkg/controllers/machine/link/suite_test.go +++ b/pkg/controllers/machine/link/suite_test.go @@ -50,7 +50,7 @@ import ( ) var ctx context.Context -var awsCtx *awscontext.Context +var awsCtx awscontext.Context var env *coretest.Environment var ec2API *fake.EC2API var cloudProvider *cloudprovider.CloudProvider @@ -69,7 +69,7 @@ var _ = BeforeSuite(func() { ec2API = &fake.EC2API{} awsCtx = test.Context(ctx, ec2API, &fake.SSMAPI{}, env, &clock.FakeClock{}) - cloudProvider = cloudprovider.New(*awsCtx) + cloudProvider = cloudprovider.New(awsCtx) linkController = link.NewController(env.Client, cloudProvider) }) diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index a0b685e2715b..047bcd827ecd 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -72,7 +72,7 @@ var ctx context.Context var stop context.CancelFunc var opts options.Options var env *coretest.Environment -var awsCtx *awscontext.Context +var awsCtx awscontext.Context var fakeEC2API *fake.EC2API var fakeSSMAPI *fake.SSMAPI var fakeClock *clock.FakeClock @@ -104,7 +104,7 @@ var _ = BeforeSuite(func() { InstanceTypeCache: instanceTypeCache, }) - cloudProvider = cloudprovider.New(*awsCtx) + cloudProvider = cloudprovider.New(awsCtx) cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) provisioningController = provisioning.NewController(env.Client, prov, events.NewRecorder(&record.FakeRecorder{})) @@ -154,17 +154,6 @@ var _ = BeforeEach(func() { awsCtx.RestProviderCache() awsCtx.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") awsCtx.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" - - // Reset the pricing provider, so we don't cross-pollinate pricing data - awsCtx.PricingProvider = pricing.NewProvider(ctx, &fake.PricingAPI{}, fakeEC2API, "", make(chan struct{})) - awsCtx.InstanceTypesProvider = instancetype.NewProvider( - "", - instanceTypeCache, - fakeEC2API, - awsCtx.SubnetProvider, - awsCtx.UnavailableOfferingsCache, - awsCtx.PricingProvider, - ) }) var _ = AfterEach(func() { @@ -1104,7 +1093,7 @@ var _ = Describe("Instance Types", func() { node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.LabelCapacityType, v1alpha5.CapacityTypeSpot)) }) - FIt("should fail to launch capacity when there is no zonal availability for spot", func() { + It("should fail to launch capacity when there is no zonal availability for spot", func() { now := time.Now() fakeEC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ SpotPriceHistory: []*ec2.SpotPrice{ diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 57a3b44bc545..210a6c3bc0f6 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -53,7 +53,6 @@ import ( "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/providers/amifamily/bootstrap" "github.com/aws/karpenter/pkg/providers/instancetype" - "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/test" coresettings "github.com/aws/karpenter-core/pkg/apis/settings" @@ -69,7 +68,7 @@ import ( ) var ctx context.Context -var awsCtx *awscontext.Context +var awsCtx awscontext.Context var stop context.CancelFunc var opts options.Options var env *coretest.Environment @@ -103,7 +102,7 @@ var _ = BeforeSuite(func() { LaunchTemplateCache: launchTemplateCache, }) - cloudProvider = cloudprovider.New(*awsCtx) + cloudProvider = cloudprovider.New(awsCtx) cluster = state.NewCluster(fakeClock, awsCtx.KubeClient, cloudProvider) prov = provisioning.NewProvisioner(ctx, awsCtx.KubeClient, awsCtx.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) }) @@ -151,16 +150,6 @@ var _ = BeforeEach(func() { awsCtx.RestProviderCache() awsCtx.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") awsCtx.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" - - // Reset the pricing provider, so we don't cross-pollinate pricing data - awsCtx.InstanceTypesProvider = instancetype.NewProvider( - "", - cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), - fakeEC2API, - awsCtx.SubnetProvider, - awsCtx.UnavailableOfferingsCache, - pricing.NewProvider(ctx, &fake.PricingAPI{}, fakeEC2API, "", make(chan struct{})), - ) }) var _ = AfterEach(func() { diff --git a/pkg/test/context.go b/pkg/test/context.go index d66a5c9cae20..88cdf38d6efd 100644 --- a/pkg/test/context.go +++ b/pkg/test/context.go @@ -57,7 +57,7 @@ type ContextOptions struct { } func Context(ctx context.Context, ec2api ec2iface.EC2API, ssmapi ssmiface.SSMAPI, - env *coretest.Environment, clock clock.Clock, overrides ...ContextOptions) *awscontext.Context { + env *coretest.Environment, clock clock.Clock, overrides ...ContextOptions) awscontext.Context { options := ContextOptions{} for _, override := range overrides { if err := mergo.Merge(&options, override, mergo.WithOverride); err != nil { @@ -103,7 +103,7 @@ func Context(ctx context.Context, ec2api ec2iface.EC2API, ssmapi ssmiface.SSMAPI launchTemplateProvider, ) - return &awscontext.Context{ + return awscontext.Context{ Context: cloudprovider.Context{ Context: ctx, RESTConfig: env.Config, From 763b0eb837119ff6c195d6ba33e93112b9f3ac9f Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Fri, 3 Mar 2023 13:23:17 -0800 Subject: [PATCH 31/41] One Failing test --- pkg/cloudprovider/suite_test.go | 218 ++++++++++---------------------- 1 file changed, 69 insertions(+), 149 deletions(-) diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index 4a94d686d29d..a89cde15a03a 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -12,7 +12,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cloudprovider +package cloudprovider_test import ( "context" @@ -22,20 +22,18 @@ import ( "time" "github.com/Pallinder/go-randomdata" - "github.com/patrickmn/go-cache" "github.com/samber/lo" "k8s.io/client-go/tools/record" - "github.com/aws/karpenter-core/pkg/events" - . "github.com/aws/karpenter-core/pkg/test/expectations" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/aws/aws-sdk-go/aws" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" clock "k8s.io/utils/clock/testing" - "knative.dev/pkg/ptr" + + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ssm" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -44,64 +42,40 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" - awscache "github.com/aws/karpenter/pkg/cache" - "github.com/aws/karpenter/pkg/providers/amifamily" - "github.com/aws/karpenter/pkg/providers/instance" - "github.com/aws/karpenter/pkg/providers/instancetype" - "github.com/aws/karpenter/pkg/providers/launchtemplate" - "github.com/aws/karpenter/pkg/providers/pricing" - "github.com/aws/karpenter/pkg/providers/securitygroup" - "github.com/aws/karpenter/pkg/providers/subnet" - "github.com/aws/karpenter/pkg/test" - - "github.com/aws/karpenter-core/pkg/cloudprovider" - machineutil "github.com/aws/karpenter-core/pkg/utils/machine" - - "github.com/aws/karpenter-core/pkg/operator/controller" - - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/ssm" - + "github.com/aws/karpenter/pkg/cloudprovider" + awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/test" coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" "github.com/aws/karpenter-core/pkg/controllers/provisioning" "github.com/aws/karpenter-core/pkg/controllers/state" + "github.com/aws/karpenter-core/pkg/events" + "github.com/aws/karpenter-core/pkg/operator/controller" "github.com/aws/karpenter-core/pkg/operator/injection" "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" + . "github.com/aws/karpenter-core/pkg/test/expectations" + machineutil "github.com/aws/karpenter-core/pkg/utils/machine" ) var ctx context.Context var stop context.CancelFunc var opts options.Options var env *coretest.Environment -var launchTemplateCache *cache.Cache -var ssmCache *cache.Cache -var ec2Cache *cache.Cache -var kubernetesVersionCache *cache.Cache -var unavailableOfferingsCache *awscache.UnavailableOfferings -var instanceTypeCache *cache.Cache -var instanceTypesProvider *instancetype.Provider -var launchTemplateProvider *launchtemplate.Provider -var amiProvider *amifamily.Provider +var awsCtx awscontext.Context var fakeEC2API *fake.EC2API var fakeSSMAPI *fake.SSMAPI -var fakeEKSAPI *fake.EKSAPI -var fakePricingAPI *fake.PricingAPI var prov *provisioning.Provisioner var provisioningController controller.Controller -var cloudProvider *CloudProvider +var cloudProvider *cloudprovider.CloudProvider var cluster *state.Cluster var fakeClock *clock.FakeClock var provisioner *v1alpha5.Provisioner var nodeTemplate *v1alpha1.AWSNodeTemplate -var pricingProvider *pricing.Provider -var subnetProvider *subnet.Provider -var securityGroupProvider *securitygroup.Provider -var instanceProvider *instance.Provider func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -115,50 +89,15 @@ var _ = BeforeSuite(func() { ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) - launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - unavailableOfferingsCache = awscache.NewUnavailableOfferings() - ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - instanceTypeCache = cache.New(awscache.InstanceTypesAndZonesTTL, awscache.DefaultCleanupInterval) fakeEC2API = &fake.EC2API{} fakeSSMAPI = &fake.SSMAPI{} - fakeEKSAPI = &fake.EKSAPI{} - fakePricingAPI = &fake.PricingAPI{} - pricingProvider = pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})) - amiProvider = amifamily.NewProvider(env.Client, env.KubernetesInterface, fakeSSMAPI, fakeEC2API, ssmCache, ec2Cache, kubernetesVersionCache) - subnetProvider = subnet.NewProvider(fakeEC2API) - instanceTypesProvider = instancetype.NewProvider( - "", - instanceTypeCache, - fakeEC2API, - subnetProvider, - unavailableOfferingsCache, - pricingProvider, - ) - securityGroupProvider = securitygroup.NewProvider(fakeEC2API) - launchTemplateProvider = launchtemplate.NewProvider( - ctx, - launchTemplateCache, - fakeEC2API, - amifamily.New(env.Client, amiProvider), - securityGroupProvider, - ptr.String("ca-bundle"), - make(chan struct{}), - net.ParseIP("10.0.100.10"), - "https://test-cluster", - ) - instanceProvider = instance.NewProvider(ctx, "", fakeEC2API, unavailableOfferingsCache, instanceTypesProvider, subnetProvider, launchTemplateProvider) - cloudProvider = &CloudProvider{ - instanceTypeProvider: instanceTypesProvider, - amiProvider: amiProvider, - instanceProvider: instanceProvider, - kubeClient: env.Client, - } fakeClock = clock.NewFakeClock(time.Now()) - cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) - prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) - provisioningController = provisioning.NewController(env.Client, prov, events.NewRecorder(&record.FakeRecorder{})) + awsCtx = test.Context(ctx, fakeEC2API, fakeSSMAPI, env, fakeClock, test.ContextOptions{}) + + cloudProvider = cloudprovider.New(awsCtx) + cluster = state.NewCluster(fakeClock, awsCtx.KubeClient, cloudProvider) + prov = provisioning.NewProvisioner(ctx, awsCtx.KubeClient, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) + provisioningController = provisioning.NewController(awsCtx.KubeClient, prov, events.NewRecorder(&record.FakeRecorder{})) }) var _ = AfterSuite(func() { @@ -202,35 +141,16 @@ var _ = BeforeEach(func() { cluster.Reset() fakeEC2API.Reset() fakeSSMAPI.Reset() - fakeEKSAPI.Reset() - fakePricingAPI.Reset() - launchTemplateCache.Flush() - unavailableOfferingsCache.Flush() - ssmCache.Flush() - ec2Cache.Flush() - kubernetesVersionCache.Flush() - instanceTypeCache.Flush() - subnetProvider.Reset() - securityGroupProvider.Reset() - launchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") - launchTemplateProvider.ClusterEndpoint = "https://test-cluster" - - // Reset the pricing provider, so we don't cross-pollinate pricing data - instanceTypesProvider = instancetype.NewProvider( - "", - instanceTypeCache, - fakeEC2API, - subnetProvider, - unavailableOfferingsCache, - pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})), - ) + awsCtx.RestProviderCache() + awsCtx.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") + awsCtx.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" }) var _ = AfterEach(func() { - ExpectCleanedUp(ctx, env.Client) + ExpectCleanedUp(ctx, awsCtx.KubeClient) }) -var _ = Describe("Allocation", func() { +var _ = Describe("CloudProvider", func() { Context("Defaulting", func() { // Intent here is that if updates occur on the provisioningController, the Provisioner doesn't need to be recreated It("should not set the InstanceProfile with the default if none provided in Provisioner", func() { @@ -262,20 +182,20 @@ var _ = Describe("Allocation", func() { provider.SecurityGroupSelector = map[string]string{"*": "*"} provisioner = coretest.Provisioner(coretest.ProvisionerOptions{Provider: provider}) provisioner.SetDefaults(ctx) - ExpectApplied(ctx, env.Client, provisioner) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(aws.StringValue(createFleetInput.Context)).To(Equal("context-1234")) }) It("should default to no EC2 Context", func() { provisioner.SetDefaults(ctx) - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(createFleetInput.Context).To(BeNil()) @@ -283,7 +203,7 @@ var _ = Describe("Allocation", func() { }) Context("Node Drift", func() { var validAMI string - var selectedInstanceType *cloudprovider.InstanceType + var selectedInstanceType *corecloudprovider.InstanceType var instance *ec2.Instance BeforeEach(func() { validAMI = fake.ImageID() @@ -293,7 +213,7 @@ var _ = Describe("Allocation", func() { fakeEC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ Images: []*ec2.Image{{ImageId: aws.String(validAMI)}}, }) - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) instanceTypes, err := cloudProvider.GetInstanceTypes(ctx, provisioner) Expect(err).ToNot(HaveOccurred()) selectedInstanceType = instanceTypes[0] @@ -314,7 +234,7 @@ var _ = Describe("Allocation", func() { }) }) It("should not fail if node template does not exist", func() { - ExpectDeleted(ctx, env.Client, nodeTemplate) + ExpectDeleted(ctx, awsCtx.KubeClient, nodeTemplate) node := coretest.Node(coretest.NodeOptions{ ProviderID: fake.ProviderID(lo.FromPtr(instance.InstanceId)), ObjectMeta: metav1.ObjectMeta{ @@ -330,7 +250,7 @@ var _ = Describe("Allocation", func() { }) It("should return false if providerRef is not defined", func() { provisioner.Spec.ProviderRef = nil - ExpectApplied(ctx, env.Client, provisioner) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner) node := coretest.Node(coretest.NodeOptions{ ProviderID: fake.ProviderID(lo.FromPtr(instance.InstanceId)), ObjectMeta: metav1.ObjectMeta{ @@ -345,7 +265,7 @@ var _ = Describe("Allocation", func() { Expect(drifted).To(BeFalse()) }) It("should not fail if provisioner does not exist", func() { - ExpectDeleted(ctx, env.Client, provisioner) + ExpectDeleted(ctx, awsCtx.KubeClient, provisioner) node := coretest.Node(coretest.NodeOptions{ ProviderID: fake.ProviderID(lo.FromPtr(instance.InstanceId)), ObjectMeta: metav1.ObjectMeta{ @@ -401,7 +321,7 @@ var _ = Describe("Allocation", func() { _, err := cloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) Expect(err).To(HaveOccurred()) }) - It("should error drift if node doesn't have provider id", func() { + FIt("should error drift if node doesn't have provider id", func() { node := coretest.Node(coretest.NodeOptions{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ @@ -428,10 +348,10 @@ var _ = Describe("Allocation", func() { Operator: v1.NodeSelectorOpExists, }}, }) - ExpectApplied(ctx, env.Client, provisioner) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) firstLt := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() @@ -458,10 +378,10 @@ var _ = Describe("Allocation", func() { Operator: v1.NodeSelectorOpExists, }}, }) - ExpectApplied(ctx, env.Client, provisioner) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(aws.StringValueSlice(input.LaunchTemplateData.SecurityGroupIds)).To(ConsistOf( @@ -480,10 +400,10 @@ var _ = Describe("Allocation", func() { Operator: v1.NodeSelectorOpExists, }}, }) - ExpectApplied(ctx, env.Client, provisioner) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("subnet-test1")) }) @@ -500,10 +420,10 @@ var _ = Describe("Allocation", func() { Operator: v1.NodeSelectorOpExists, }}, }) - ExpectApplied(ctx, env.Client, provisioner) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(*input.LaunchTemplateData.IamInstanceProfile.Name).To(Equal("overridden-profile")) @@ -513,11 +433,11 @@ var _ = Describe("Allocation", func() { // Note when debugging these tests - // hard coded fixture data (ex. what the aws api will return) is maintained in fake/ec2api.go It("should default to the cluster's subnets", func() { - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod( coretest.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: v1alpha5.ArchitectureAmd64}}) - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) input := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(input.LaunchTemplateConfigs).To(HaveLen(1)) @@ -544,10 +464,10 @@ var _ = Describe("Allocation", func() { {SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(100), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}}, }}) - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}}) - ExpectProvisioned(ctx, env.Client, cluster, prov, pod) - ExpectScheduled(ctx, env.Client, pod) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectScheduled(ctx, awsCtx.KubeClient, pod) createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("test-subnet-2")) }) @@ -559,18 +479,18 @@ var _ = Describe("Allocation", func() { Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}}, }}) provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{MaxPods: aws.Int32(1)} - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) pod1 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}}) pod2 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}}) - ExpectProvisioned(ctx, env.Client, cluster, prov, pod1, pod2) - ExpectScheduled(ctx, env.Client, pod1) - ExpectScheduled(ctx, env.Client, pod2) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod1, pod2) + ExpectScheduled(ctx, awsCtx.KubeClient, pod1) + ExpectScheduled(ctx, awsCtx.KubeClient, pod2) createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("test-subnet-2")) // Provision for another pod that should now use the other subnet since we've consumed some from the first launch. pod3 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}}) - ExpectProvisioned(ctx, env.Client, cluster, prov, pod3) - ExpectScheduled(ctx, env.Client, pod3) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod3) + ExpectScheduled(ctx, awsCtx.KubeClient, pod3) createFleetInput = fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("test-subnet-1")) }) @@ -580,9 +500,9 @@ var _ = Describe("Allocation", func() { Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}}, }}) pod1 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}}) - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate, pod1) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate, pod1) fakeEC2API.CreateFleetBehavior.Error.Set(fmt.Errorf("CreateFleet synthetic error")) - bindings := ExpectProvisioned(ctx, env.Client, cluster, prov, pod1) + bindings := ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod1) Expect(len(bindings)).To(Equal(0)) }) It("should launch instances into subnets that are excluded by another provisioner", func() { @@ -593,10 +513,10 @@ var _ = Describe("Allocation", func() { Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}}, }}) nodeTemplate.Spec.SubnetSelector = map[string]string{"Name": "test-subnet-1"} - ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) podSubnet1 := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, prov, podSubnet1) - ExpectScheduled(ctx, env.Client, podSubnet1) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, podSubnet1) + ExpectScheduled(ctx, awsCtx.KubeClient, podSubnet1) createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("test-subnet-1")) @@ -604,10 +524,10 @@ var _ = Describe("Allocation", func() { SubnetSelector: map[string]string{"Name": "test-subnet-2"}, SecurityGroupSelector: map[string]string{"*": "*"}, }}) - ExpectApplied(ctx, env.Client, provisioner) + ExpectApplied(ctx, awsCtx.KubeClient, provisioner) podSubnet2 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1alpha5.ProvisionerNameLabelKey: provisioner.Name}}) - ExpectProvisioned(ctx, env.Client, cluster, prov, podSubnet2) - ExpectScheduled(ctx, env.Client, podSubnet2) + ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, podSubnet2) + ExpectScheduled(ctx, awsCtx.KubeClient, podSubnet2) createFleetInput = fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("test-subnet-2")) }) From 596b35b307423b2d24f79c77c88b2aff0e1fe8ca Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Fri, 3 Mar 2023 13:55:42 -0800 Subject: [PATCH 32/41] Fixed all tests --- pkg/cloudprovider/suite_test.go | 2 +- pkg/test/context.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index a89cde15a03a..5e10f3655f51 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -321,7 +321,7 @@ var _ = Describe("CloudProvider", func() { _, err := cloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) Expect(err).To(HaveOccurred()) }) - FIt("should error drift if node doesn't have provider id", func() { + It("should error drift if node doesn't have provider id", func() { node := coretest.Node(coretest.NodeOptions{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ diff --git a/pkg/test/context.go b/pkg/test/context.go index 88cdf38d6efd..71643dfec565 100644 --- a/pkg/test/context.go +++ b/pkg/test/context.go @@ -78,7 +78,7 @@ func Context(ctx context.Context, ec2api ec2iface.EC2API, ssmapi ssmiface.SSMAPI pricingProvider := pricing.NewProvider(ctx, pricingAPI, ec2api, "", make(chan struct{})) subnetProvider := subnet.NewProvider(ec2api) securityGroupProvider := securitygroup.NewProvider(ec2api) - amiProvider := amifamily.NewProvider(env.Client, env.KubernetesInterface, &fake.SSMAPI{}, ec2api, ssmCache, ec2Cache, kubernetesVersionCache) + amiProvider := amifamily.NewProvider(env.Client, env.KubernetesInterface, ssmapi, ec2api, ssmCache, ec2Cache, kubernetesVersionCache) amiResolver := amifamily.New(env.Client, amiProvider) instanceTypesProvider := instancetype.NewProvider("", instanceTypeCache, ec2api, subnetProvider, unavailableOfferingsCache, pricingProvider) launchTemplateProvider := From 4dfbab290a37d442e792668519381f090e1247c7 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Fri, 3 Mar 2023 14:05:16 -0800 Subject: [PATCH 33/41] cloudprovider params --- cmd/controller/main.go | 8 +++++++- pkg/cloudprovider/cloudprovider.go | 12 ++++++------ pkg/cloudprovider/suite_test.go | 2 +- pkg/controllers/machine/garbagecollect/suite_test.go | 2 +- pkg/controllers/machine/link/suite_test.go | 2 +- pkg/providers/instancetype/suite_test.go | 2 +- pkg/providers/launchtemplate/suite_test.go | 2 +- 7 files changed, 18 insertions(+), 12 deletions(-) diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 06f0366d0227..e41c110ca1e0 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -41,7 +41,13 @@ func main() { EventRecorder: operator.EventRecorder, StartAsync: operator.Elected(), }) - awsCloudProvider := cloudprovider.New(awsCtx) + awsCloudProvider := cloudprovider.New( + awsCtx, + awsCtx.InstanceTypesProvider, + awsCtx.InstanceProvider, + awsCtx.KubeClient, + awsCtx.AMIProvider, + ) lo.Must0(operator.AddHealthzCheck("cloud-provider", awsCloudProvider.LivenessProbe)) cloudProvider := metrics.Decorate(awsCloudProvider) diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index bc3504414cc2..c01b553a094b 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -42,7 +42,6 @@ import ( "knative.dev/pkg/logging" "sigs.k8s.io/controller-runtime/pkg/client" - awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/providers/instancetype" @@ -66,12 +65,13 @@ type CloudProvider struct { amiProvider *amifamily.Provider } -func New(ctx awscontext.Context) *CloudProvider { +func New(ctx context.Context, instanceTypeProvider *instancetype.Provider, + instanceProvider *instance.Provider, kubeClient client.Client, amiProvider *amifamily.Provider) *CloudProvider { return &CloudProvider{ - instanceTypeProvider: ctx.InstanceTypesProvider, - instanceProvider: ctx.InstanceProvider, - kubeClient: ctx.KubeClient, - amiProvider: ctx.AMIProvider, + instanceTypeProvider: instanceTypeProvider, + instanceProvider: instanceProvider, + kubeClient: kubeClient, + amiProvider: amiProvider, } } diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index 5e10f3655f51..03a964a88e69 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -94,7 +94,7 @@ var _ = BeforeSuite(func() { fakeClock = clock.NewFakeClock(time.Now()) awsCtx = test.Context(ctx, fakeEC2API, fakeSSMAPI, env, fakeClock, test.ContextOptions{}) - cloudProvider = cloudprovider.New(awsCtx) + cloudProvider = cloudprovider.New(awsCtx, awsCtx.InstanceTypesProvider, awsCtx.InstanceProvider, awsCtx.KubeClient, awsCtx.AMIProvider) cluster = state.NewCluster(fakeClock, awsCtx.KubeClient, cloudProvider) prov = provisioning.NewProvisioner(ctx, awsCtx.KubeClient, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) provisioningController = provisioning.NewController(awsCtx.KubeClient, prov, events.NewRecorder(&record.FakeRecorder{})) diff --git a/pkg/controllers/machine/garbagecollect/suite_test.go b/pkg/controllers/machine/garbagecollect/suite_test.go index cc3ed715c0e3..b01a87da3729 100644 --- a/pkg/controllers/machine/garbagecollect/suite_test.go +++ b/pkg/controllers/machine/garbagecollect/suite_test.go @@ -72,7 +72,7 @@ var _ = BeforeSuite(func() { ec2API = &fake.EC2API{} awsCtx = test.Context(ctx, ec2API, &fake.SSMAPI{}, env, &clock.FakeClock{}) - cloudProvider = cloudprovider.New(awsCtx) + cloudProvider = cloudprovider.New(awsCtx, awsCtx.InstanceTypesProvider, awsCtx.InstanceProvider, awsCtx.KubeClient, awsCtx.AMIProvider) linkedMachineCache = cache.New(time.Minute*10, time.Second*10) linkController := &link.Controller{ diff --git a/pkg/controllers/machine/link/suite_test.go b/pkg/controllers/machine/link/suite_test.go index db8bdd6b17e8..2627f4f09a09 100644 --- a/pkg/controllers/machine/link/suite_test.go +++ b/pkg/controllers/machine/link/suite_test.go @@ -69,7 +69,7 @@ var _ = BeforeSuite(func() { ec2API = &fake.EC2API{} awsCtx = test.Context(ctx, ec2API, &fake.SSMAPI{}, env, &clock.FakeClock{}) - cloudProvider = cloudprovider.New(awsCtx) + cloudProvider = cloudprovider.New(awsCtx, awsCtx.InstanceTypesProvider, awsCtx.InstanceProvider, awsCtx.KubeClient, awsCtx.AMIProvider) linkController = link.NewController(env.Client, cloudProvider) }) diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index 047bcd827ecd..c3966b66dc5d 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -104,7 +104,7 @@ var _ = BeforeSuite(func() { InstanceTypeCache: instanceTypeCache, }) - cloudProvider = cloudprovider.New(awsCtx) + cloudProvider = cloudprovider.New(awsCtx, awsCtx.InstanceTypesProvider, awsCtx.InstanceProvider, awsCtx.KubeClient, awsCtx.AMIProvider) cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) provisioningController = provisioning.NewController(env.Client, prov, events.NewRecorder(&record.FakeRecorder{})) diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 210a6c3bc0f6..96ea2a4b4f91 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -102,7 +102,7 @@ var _ = BeforeSuite(func() { LaunchTemplateCache: launchTemplateCache, }) - cloudProvider = cloudprovider.New(awsCtx) + cloudProvider = cloudprovider.New(awsCtx, awsCtx.InstanceTypesProvider, awsCtx.InstanceProvider, awsCtx.KubeClient, awsCtx.AMIProvider) cluster = state.NewCluster(fakeClock, awsCtx.KubeClient, cloudProvider) prov = provisioning.NewProvisioner(ctx, awsCtx.KubeClient, awsCtx.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) }) From 95b1b989a1ae2346b9908dc87a37e7196655ac0a Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Fri, 3 Mar 2023 15:56:58 -0800 Subject: [PATCH 34/41] Feedback changes --- pkg/cloudprovider/suite_test.go | 46 ++++++++++++++++- pkg/context/context.go | 13 +---- .../machine/garbagecollect/suite_test.go | 47 ++++++++++++++++- pkg/controllers/machine/link/suite_test.go | 50 ++++++++++++++++++- pkg/controllers/nodetemplate/suite_test.go | 12 ++++- pkg/providers/instancetype/instancetype.go | 4 -- pkg/providers/instancetype/suite_test.go | 42 ++++++++++++++-- .../launchtemplate/launchtemplate.go | 4 -- pkg/providers/launchtemplate/suite_test.go | 42 ++++++++++++++-- pkg/providers/securitygroup/securitygroup.go | 9 +--- pkg/providers/securitygroup/suite_test.go | 8 ++- pkg/providers/subnet/subnet.go | 10 +--- pkg/providers/subnet/suite_test.go | 8 ++- pkg/test/context.go | 27 ++++------ 14 files changed, 254 insertions(+), 68 deletions(-) diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index 03a964a88e69..2dc40275c6f0 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -22,6 +22,7 @@ import ( "time" "github.com/Pallinder/go-randomdata" + "github.com/patrickmn/go-cache" "github.com/samber/lo" "k8s.io/client-go/tools/record" @@ -42,6 +43,7 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" + awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/cloudprovider" awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/fake" @@ -69,6 +71,7 @@ var env *coretest.Environment var awsCtx awscontext.Context var fakeEC2API *fake.EC2API var fakeSSMAPI *fake.SSMAPI +var fakePricingAPI *fake.PricingAPI var prov *provisioning.Provisioner var provisioningController controller.Controller var cloudProvider *cloudprovider.CloudProvider @@ -77,6 +80,16 @@ var fakeClock *clock.FakeClock var provisioner *v1alpha5.Provisioner var nodeTemplate *v1alpha1.AWSNodeTemplate +// Cache +var ec2Cache *cache.Cache +var ssmCache *cache.Cache +var kubernetesVersionCache *cache.Cache +var instanceTypeCache *cache.Cache +var unavailableOfferingsCache *awscache.UnavailableOfferings +var launchTemplateCache *cache.Cache +var subnetCache *cache.Cache +var securityGroupCache *cache.Cache + func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) RegisterFailHandler(Fail) @@ -92,7 +105,28 @@ var _ = BeforeSuite(func() { fakeEC2API = &fake.EC2API{} fakeSSMAPI = &fake.SSMAPI{} fakeClock = clock.NewFakeClock(time.Now()) - awsCtx = test.Context(ctx, fakeEC2API, fakeSSMAPI, env, fakeClock, test.ContextOptions{}) + fakePricingAPI = &fake.PricingAPI{} + + ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + unavailableOfferingsCache = awscache.NewUnavailableOfferings() + launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + subnetCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + securityGroupCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + + awsCtx = test.Context(ctx, fakeEC2API, fakeSSMAPI, env, fakeClock, test.ContextOptions{ + SSMCache: ssmCache, + EC2Cache: ec2Cache, + KubernetesVersionCache: kubernetesVersionCache, + UnavailableOfferingsCache: unavailableOfferingsCache, + LaunchTemplateCache: launchTemplateCache, + InstanceTypeCache: instanceTypeCache, + SubnetCache: subnetCache, + SecurityGroupCache: securityGroupCache, + PricingAPI: fakePricingAPI, + }) cloudProvider = cloudprovider.New(awsCtx, awsCtx.InstanceTypesProvider, awsCtx.InstanceProvider, awsCtx.KubeClient, awsCtx.AMIProvider) cluster = state.NewCluster(fakeClock, awsCtx.KubeClient, cloudProvider) @@ -141,7 +175,15 @@ var _ = BeforeEach(func() { cluster.Reset() fakeEC2API.Reset() fakeSSMAPI.Reset() - awsCtx.RestProviderCache() + ec2Cache.Flush() + ssmCache.Flush() + kubernetesVersionCache.Flush() + instanceTypeCache.Flush() + unavailableOfferingsCache.Flush() + launchTemplateCache.Flush() + subnetCache.Flush() + securityGroupCache.Flush() + awsCtx.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") awsCtx.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" }) diff --git a/pkg/context/context.go b/pkg/context/context.go index d0c970237872..cad118905089 100644 --- a/pkg/context/context.go +++ b/pkg/context/context.go @@ -108,8 +108,8 @@ func NewOrDie(ctx cloudprovider.Context) Context { } unavailableOfferingsCache := awscache.NewUnavailableOfferings() - subnetProvider := subnet.NewProvider(ec2api) - securityGroupProvider := securitygroup.NewProvider(ec2api) + subnetProvider := subnet.NewProvider(ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) + securityGroupProvider := securitygroup.NewProvider(ec2api, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) pricingProvider := pricing.NewProvider( ctx, pricing.NewAPI(sess, *sess.Config.Region), @@ -227,12 +227,3 @@ func kubeDNSIP(ctx context.Context, kubernetesInterface kubernetes.Interface) (n } return kubeDNSIP, nil } - -func (c *Context) RestProviderCache() { - c.SecurityGroupProvider.Reset() - c.SubnetProvider.Reset() - c.InstanceTypesProvider.Reset() - c.LaunchTemplateProvider.Reset() - c.AMIProvider.Reset() - c.UnavailableOfferingsCache.Flush() -} diff --git a/pkg/controllers/machine/garbagecollect/suite_test.go b/pkg/controllers/machine/garbagecollect/suite_test.go index b01a87da3729..f4a362f2b49a 100644 --- a/pkg/controllers/machine/garbagecollect/suite_test.go +++ b/pkg/controllers/machine/garbagecollect/suite_test.go @@ -43,6 +43,7 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" + awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/cloudprovider" awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/controllers/machine/garbagecollect" @@ -58,6 +59,17 @@ var ec2API *fake.EC2API var cloudProvider *cloudprovider.CloudProvider var garbageCollectController controller.Controller var linkedMachineCache *cache.Cache +var fakePricingAPI *fake.PricingAPI + +// Cache +var ec2Cache *cache.Cache +var ssmCache *cache.Cache +var kubernetesVersionCache *cache.Cache +var instanceTypeCache *cache.Cache +var unavailableOfferingsCache *awscache.UnavailableOfferings +var launchTemplateCache *cache.Cache +var subnetCache *cache.Cache +var securityGroupCache *cache.Cache func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -70,7 +82,29 @@ var _ = BeforeSuite(func() { ctx = settings.ToContext(ctx, test.Settings()) env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) ec2API = &fake.EC2API{} - awsCtx = test.Context(ctx, ec2API, &fake.SSMAPI{}, env, &clock.FakeClock{}) + + fakePricingAPI = &fake.PricingAPI{} + + ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + unavailableOfferingsCache = awscache.NewUnavailableOfferings() + launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + subnetCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + securityGroupCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + + awsCtx = test.Context(ctx, ec2API, fake.SSMAPI{}, env, &clock.FakeClock{}, test.ContextOptions{ + SSMCache: ssmCache, + EC2Cache: ec2Cache, + KubernetesVersionCache: kubernetesVersionCache, + UnavailableOfferingsCache: unavailableOfferingsCache, + LaunchTemplateCache: launchTemplateCache, + InstanceTypeCache: instanceTypeCache, + SubnetCache: subnetCache, + SecurityGroupCache: securityGroupCache, + PricingAPI: fakePricingAPI, + }) cloudProvider = cloudprovider.New(awsCtx, awsCtx.InstanceTypesProvider, awsCtx.InstanceProvider, awsCtx.KubeClient, awsCtx.AMIProvider) @@ -85,6 +119,17 @@ var _ = AfterSuite(func() { Expect(env.Stop()).To(Succeed(), "Failed to stop environment") }) +var _ = BeforeEach(func() { + ec2Cache.Flush() + ssmCache.Flush() + kubernetesVersionCache.Flush() + instanceTypeCache.Flush() + unavailableOfferingsCache.Flush() + launchTemplateCache.Flush() + subnetCache.Flush() + securityGroupCache.Flush() +}) + var _ = Describe("MachineGarbageCollect", func() { var instance *ec2.Instance var providerID string diff --git a/pkg/controllers/machine/link/suite_test.go b/pkg/controllers/machine/link/suite_test.go index 2627f4f09a09..15f7e1b8461f 100644 --- a/pkg/controllers/machine/link/suite_test.go +++ b/pkg/controllers/machine/link/suite_test.go @@ -24,6 +24,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/patrickmn/go-cache" "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -38,9 +39,11 @@ import ( coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" "github.com/aws/karpenter-core/pkg/utils/sets" + "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" + awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/cloudprovider" awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/controllers/machine/link" @@ -55,6 +58,17 @@ var env *coretest.Environment var ec2API *fake.EC2API var cloudProvider *cloudprovider.CloudProvider var linkController controller.Controller +var fakePricingAPI *fake.PricingAPI + +// Cache +var ec2Cache *cache.Cache +var ssmCache *cache.Cache +var kubernetesVersionCache *cache.Cache +var instanceTypeCache *cache.Cache +var unavailableOfferingsCache *awscache.UnavailableOfferings +var launchTemplateCache *cache.Cache +var subnetCache *cache.Cache +var securityGroupCache *cache.Cache func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -66,18 +80,50 @@ var _ = BeforeSuite(func() { ctx = coresettings.ToContext(ctx, coretest.Settings()) ctx = settings.ToContext(ctx, test.Settings()) env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + ec2API = &fake.EC2API{} - awsCtx = test.Context(ctx, ec2API, &fake.SSMAPI{}, env, &clock.FakeClock{}) + fakePricingAPI = &fake.PricingAPI{} + + ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + unavailableOfferingsCache = awscache.NewUnavailableOfferings() + launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + subnetCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + securityGroupCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + + awsCtx = test.Context(ctx, ec2API, fake.SSMAPI{}, env, &clock.FakeClock{}, test.ContextOptions{ + SSMCache: ssmCache, + EC2Cache: ec2Cache, + KubernetesVersionCache: kubernetesVersionCache, + UnavailableOfferingsCache: unavailableOfferingsCache, + LaunchTemplateCache: launchTemplateCache, + InstanceTypeCache: instanceTypeCache, + SubnetCache: subnetCache, + SecurityGroupCache: securityGroupCache, + PricingAPI: fakePricingAPI, + }) cloudProvider = cloudprovider.New(awsCtx, awsCtx.InstanceTypesProvider, awsCtx.InstanceProvider, awsCtx.KubeClient, awsCtx.AMIProvider) linkController = link.NewController(env.Client, cloudProvider) }) - var _ = AfterSuite(func() { Expect(env.Stop()).To(Succeed(), "Failed to stop environment") }) +var _ = BeforeEach(func() { + ec2Cache.Flush() + ssmCache.Flush() + kubernetesVersionCache.Flush() + instanceTypeCache.Flush() + unavailableOfferingsCache.Flush() + launchTemplateCache.Flush() + subnetCache.Flush() + securityGroupCache.Flush() +}) + var _ = Describe("MachineLink", func() { var instanceID string var providerID string diff --git a/pkg/controllers/nodetemplate/suite_test.go b/pkg/controllers/nodetemplate/suite_test.go index 717832d3ea62..a4f7e30160fe 100644 --- a/pkg/controllers/nodetemplate/suite_test.go +++ b/pkg/controllers/nodetemplate/suite_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/patrickmn/go-cache" "github.com/samber/lo" . "knative.dev/pkg/logging/testing" _ "knative.dev/pkg/system/testing" @@ -39,6 +40,7 @@ import ( . "github.com/aws/karpenter-core/pkg/test/expectations" "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/v1alpha1" + awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/controllers/nodetemplate" "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/providers/securitygroup" @@ -53,6 +55,8 @@ var subnetProvider *subnet.Provider var securityGroupProvider *securitygroup.Provider var nodeTemplate *v1alpha1.AWSNodeTemplate var controller corecontroller.Controller +var subnetCache *cache.Cache +var securityGroupCache *cache.Cache func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -64,8 +68,10 @@ var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) fakeEC2API = &fake.EC2API{} - subnetProvider = subnet.NewProvider(fakeEC2API) - securityGroupProvider = securitygroup.NewProvider(fakeEC2API) + subnetCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + securityGroupCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + subnetProvider = subnet.NewProvider(fakeEC2API, subnetCache) + securityGroupProvider = securitygroup.NewProvider(fakeEC2API, securityGroupCache) controller = nodetemplate.NewController(env.Client, subnetProvider, securityGroupProvider) }) @@ -89,6 +95,8 @@ var _ = BeforeEach(func() { } fakeEC2API.Reset() + subnetCache.Flush() + securityGroupCache.Flush() }) var _ = AfterEach(func() { diff --git a/pkg/providers/instancetype/instancetype.go b/pkg/providers/instancetype/instancetype.go index af8a80566bda..4d2df86f8bda 100644 --- a/pkg/providers/instancetype/instancetype.go +++ b/pkg/providers/instancetype/instancetype.go @@ -231,7 +231,3 @@ func (p *Provider) GetInstanceTypes(ctx context.Context) ([]*ec2.InstanceTypeInf p.cache.SetDefault(InstanceTypesCacheKey, instanceTypes) return instanceTypes, nil } - -func (p *Provider) Reset() { - p.cache.Flush() -} diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index c3966b66dc5d..eeaa66d6b11e 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -76,7 +76,7 @@ var awsCtx awscontext.Context var fakeEC2API *fake.EC2API var fakeSSMAPI *fake.SSMAPI var fakeClock *clock.FakeClock -var instanceTypeCache *cache.Cache +var fakePricingAPI *fake.PricingAPI var prov *provisioning.Provisioner var provisioner *v1alpha5.Provisioner var nodeTemplate *v1alpha1.AWSNodeTemplate @@ -84,6 +84,16 @@ var cluster *state.Cluster var cloudProvider *cloudprovider.CloudProvider var provisioningController controller.Controller +// Cache +var ec2Cache *cache.Cache +var ssmCache *cache.Cache +var kubernetesVersionCache *cache.Cache +var instanceTypeCache *cache.Cache +var unavailableOfferingsCache *awscache.UnavailableOfferings +var launchTemplateCache *cache.Cache +var subnetCache *cache.Cache +var securityGroupCache *cache.Cache + func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) RegisterFailHandler(Fail) @@ -99,9 +109,27 @@ var _ = BeforeSuite(func() { fakeEC2API = &fake.EC2API{} fakeSSMAPI = &fake.SSMAPI{} fakeClock = &clock.FakeClock{} + fakePricingAPI = &fake.PricingAPI{} + + ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + unavailableOfferingsCache = awscache.NewUnavailableOfferings() + launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + subnetCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + securityGroupCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + awsCtx = test.Context(ctx, fakeEC2API, fakeSSMAPI, env, fakeClock, test.ContextOptions{ - InstanceTypeCache: instanceTypeCache, + SSMCache: ssmCache, + EC2Cache: ec2Cache, + KubernetesVersionCache: kubernetesVersionCache, + UnavailableOfferingsCache: unavailableOfferingsCache, + LaunchTemplateCache: launchTemplateCache, + InstanceTypeCache: instanceTypeCache, + SubnetCache: subnetCache, + SecurityGroupCache: securityGroupCache, + PricingAPI: fakePricingAPI, }) cloudProvider = cloudprovider.New(awsCtx, awsCtx.InstanceTypesProvider, awsCtx.InstanceProvider, awsCtx.KubeClient, awsCtx.AMIProvider) @@ -151,7 +179,15 @@ var _ = BeforeEach(func() { cluster.Reset() fakeEC2API.Reset() fakeSSMAPI.Reset() - awsCtx.RestProviderCache() + ec2Cache.Flush() + ssmCache.Flush() + kubernetesVersionCache.Flush() + instanceTypeCache.Flush() + unavailableOfferingsCache.Flush() + launchTemplateCache.Flush() + subnetCache.Flush() + securityGroupCache.Flush() + awsCtx.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") awsCtx.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" }) diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index eedd5683577b..9e7d76def6ce 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -313,7 +313,3 @@ func (p *Provider) getInstanceProfile(ctx context.Context, nodeTemplate *v1alpha } return defaultProfile, nil } - -func (p *Provider) Reset() { - p.cache.Flush() -} diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 96ea2a4b4f91..1e08b8f6d550 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -75,13 +75,23 @@ var env *coretest.Environment var fakeEC2API *fake.EC2API var fakeSSMAPI *fake.SSMAPI var fakeClock *clock.FakeClock +var fakePricingAPI *fake.PricingAPI var prov *provisioning.Provisioner var provisioner *v1alpha5.Provisioner var nodeTemplate *v1alpha1.AWSNodeTemplate var cluster *state.Cluster -var launchTemplateCache *cache.Cache var cloudProvider *cloudprovider.CloudProvider +// Cache +var ec2Cache *cache.Cache +var ssmCache *cache.Cache +var kubernetesVersionCache *cache.Cache +var instanceTypeCache *cache.Cache +var unavailableOfferingsCache *awscache.UnavailableOfferings +var launchTemplateCache *cache.Cache +var subnetCache *cache.Cache +var securityGroupCache *cache.Cache + func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) RegisterFailHandler(Fail) @@ -97,9 +107,27 @@ var _ = BeforeSuite(func() { fakeEC2API = &fake.EC2API{} fakeSSMAPI = &fake.SSMAPI{} fakeClock = &clock.FakeClock{} + fakePricingAPI = &fake.PricingAPI{} + + ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + unavailableOfferingsCache = awscache.NewUnavailableOfferings() launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + subnetCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + securityGroupCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + awsCtx = test.Context(ctx, fakeEC2API, fakeSSMAPI, env, fakeClock, test.ContextOptions{ - LaunchTemplateCache: launchTemplateCache, + SSMCache: ssmCache, + EC2Cache: ec2Cache, + KubernetesVersionCache: kubernetesVersionCache, + UnavailableOfferingsCache: unavailableOfferingsCache, + LaunchTemplateCache: launchTemplateCache, + InstanceTypeCache: instanceTypeCache, + SubnetCache: subnetCache, + SecurityGroupCache: securityGroupCache, + PricingAPI: fakePricingAPI, }) cloudProvider = cloudprovider.New(awsCtx, awsCtx.InstanceTypesProvider, awsCtx.InstanceProvider, awsCtx.KubeClient, awsCtx.AMIProvider) @@ -147,7 +175,15 @@ var _ = BeforeEach(func() { cluster.Reset() fakeEC2API.Reset() fakeSSMAPI.Reset() - awsCtx.RestProviderCache() + ec2Cache.Flush() + ssmCache.Flush() + kubernetesVersionCache.Flush() + instanceTypeCache.Flush() + unavailableOfferingsCache.Flush() + launchTemplateCache.Flush() + subnetCache.Flush() + securityGroupCache.Flush() + awsCtx.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") awsCtx.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" }) diff --git a/pkg/providers/securitygroup/securitygroup.go b/pkg/providers/securitygroup/securitygroup.go index 0ea8eb47a27a..178f5e71a24e 100644 --- a/pkg/providers/securitygroup/securitygroup.go +++ b/pkg/providers/securitygroup/securitygroup.go @@ -30,7 +30,6 @@ import ( "github.com/aws/karpenter-core/pkg/utils/functional" "github.com/aws/karpenter-core/pkg/utils/pretty" "github.com/aws/karpenter/pkg/apis/v1alpha1" - awscache "github.com/aws/karpenter/pkg/cache" ) type Provider struct { @@ -42,12 +41,12 @@ type Provider struct { const TTL = 5 * time.Minute -func NewProvider(ec2api ec2iface.EC2API) *Provider { +func NewProvider(ec2api ec2iface.EC2API, cache *cache.Cache) *Provider { return &Provider{ ec2api: ec2api, cm: pretty.NewChangeMonitor(), // TODO: Remove cache for v1beta1, utilize resolved security groups from the AWSNodeTemplate.status - cache: cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), + cache: cache, } } @@ -122,7 +121,3 @@ func (p *Provider) securityGroupIds(securityGroups []*ec2.SecurityGroup) []strin } return names } - -func (p *Provider) Reset() { - p.cache.Flush() -} diff --git a/pkg/providers/securitygroup/suite_test.go b/pkg/providers/securitygroup/suite_test.go index 405c3f0ae88a..b922a57fdf3f 100644 --- a/pkg/providers/securitygroup/suite_test.go +++ b/pkg/providers/securitygroup/suite_test.go @@ -22,6 +22,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/patrickmn/go-cache" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -32,6 +33,7 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" + awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/providers/securitygroup" "github.com/aws/karpenter/pkg/test" @@ -53,6 +55,7 @@ var fakeEC2API *fake.EC2API var provisioner *corev1alpha5.Provisioner var nodeTemplate *v1alpha1.AWSNodeTemplate var securityGroupProvider *securitygroup.Provider +var securityGroupCache *cache.Cache func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -66,7 +69,8 @@ var _ = BeforeSuite(func() { ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) fakeEC2API = &fake.EC2API{} - securityGroupProvider = securitygroup.NewProvider(fakeEC2API) + securityGroupCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + securityGroupProvider = securitygroup.NewProvider(fakeEC2API, securityGroupCache) }) var _ = AfterSuite(func() { @@ -108,7 +112,7 @@ var _ = BeforeEach(func() { }) fakeEC2API.Reset() - securityGroupProvider.Reset() + securityGroupCache.Flush() }) var _ = AfterEach(func() { diff --git a/pkg/providers/subnet/subnet.go b/pkg/providers/subnet/subnet.go index f72f238e5a04..553f8f11be30 100644 --- a/pkg/providers/subnet/subnet.go +++ b/pkg/providers/subnet/subnet.go @@ -34,7 +34,6 @@ import ( "github.com/aws/karpenter-core/pkg/cloudprovider" "github.com/aws/karpenter-core/pkg/utils/functional" "github.com/aws/karpenter-core/pkg/utils/pretty" - awscache "github.com/aws/karpenter/pkg/cache" ) type Provider struct { @@ -45,13 +44,13 @@ type Provider struct { inflightIPs map[string]int64 } -func NewProvider(ec2api ec2iface.EC2API) *Provider { +func NewProvider(ec2api ec2iface.EC2API, cache *cache.Cache) *Provider { return &Provider{ ec2api: ec2api, cm: pretty.NewChangeMonitor(), // TODO: Remove cache for v1beta1, utilize resolved subnet from the AWSNodeTemplate.status // Subnets are sorted on AvailableIpAddressCount, descending order - cache: cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), + cache: cache, // inflightIPs is used to track IPs from known launched instances inflightIPs: map[string]int64{}, } @@ -242,8 +241,3 @@ func Pretty(subnets []*ec2.Subnet) []string { } return names } - -func (p *Provider) Reset() { - p.cache.Flush() - p.inflightIPs = map[string]int64{} -} diff --git a/pkg/providers/subnet/suite_test.go b/pkg/providers/subnet/suite_test.go index 034b115bddbd..41dfa7d4ab76 100644 --- a/pkg/providers/subnet/suite_test.go +++ b/pkg/providers/subnet/suite_test.go @@ -21,6 +21,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/aws/aws-sdk-go/aws" + "github.com/patrickmn/go-cache" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -31,6 +32,7 @@ import ( "github.com/aws/karpenter/pkg/apis" awssettings "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" + awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/providers/subnet" "github.com/aws/karpenter/pkg/test" @@ -52,6 +54,7 @@ var fakeEC2API *fake.EC2API var provisioner *corev1alpha5.Provisioner var nodeTemplate *v1alpha1.AWSNodeTemplate var subnetProvider *subnet.Provider +var subnetCache *cache.Cache func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -64,7 +67,8 @@ var _ = BeforeSuite(func() { ctx, stop = context.WithCancel(ctx) fakeEC2API = &fake.EC2API{} - subnetProvider = subnet.NewProvider(fakeEC2API) + subnetCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + subnetProvider = subnet.NewProvider(fakeEC2API, subnetCache) }) var _ = AfterSuite(func() { @@ -106,7 +110,7 @@ var _ = BeforeEach(func() { }) fakeEC2API.Reset() - subnetProvider.Reset() + subnetCache.Flush() }) var _ = AfterEach(func() { diff --git a/pkg/test/context.go b/pkg/test/context.go index 71643dfec565..7e866f9d93c7 100644 --- a/pkg/test/context.go +++ b/pkg/test/context.go @@ -53,6 +53,8 @@ type ContextOptions struct { InstanceTypeCache *cache.Cache UnavailableOfferingsCache *awscache.UnavailableOfferings LaunchTemplateCache *cache.Cache + SubnetCache *cache.Cache + SecurityGroupCache *cache.Cache PricingAPI *fake.PricingAPI } @@ -65,26 +67,17 @@ func Context(ctx context.Context, ec2api ec2iface.EC2API, ssmapi ssmiface.SSMAPI } } - // cache - ssmCache := OptionOR(options.SSMCache, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) - ec2Cache := OptionOR(options.EC2Cache, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) - kubernetesVersionCache := OptionOR(options.KubernetesVersionCache, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) - instanceTypeCache := OptionOR(options.InstanceTypeCache, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) - unavailableOfferingsCache := OptionOR(options.UnavailableOfferingsCache, awscache.NewUnavailableOfferings()) - launchTemplateCache := OptionOR(options.LaunchTemplateCache, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval)) - pricingAPI := OptionOR(options.PricingAPI, &fake.PricingAPI{}) - // Providers - pricingProvider := pricing.NewProvider(ctx, pricingAPI, ec2api, "", make(chan struct{})) - subnetProvider := subnet.NewProvider(ec2api) - securityGroupProvider := securitygroup.NewProvider(ec2api) - amiProvider := amifamily.NewProvider(env.Client, env.KubernetesInterface, ssmapi, ec2api, ssmCache, ec2Cache, kubernetesVersionCache) + pricingProvider := pricing.NewProvider(ctx, options.PricingAPI, ec2api, "", make(chan struct{})) + subnetProvider := subnet.NewProvider(ec2api, options.SubnetCache) + securityGroupProvider := securitygroup.NewProvider(ec2api, options.SecurityGroupCache) + amiProvider := amifamily.NewProvider(env.Client, env.KubernetesInterface, ssmapi, ec2api, options.SSMCache, options.EC2Cache, options.KubernetesVersionCache) amiResolver := amifamily.New(env.Client, amiProvider) - instanceTypesProvider := instancetype.NewProvider("", instanceTypeCache, ec2api, subnetProvider, unavailableOfferingsCache, pricingProvider) + instanceTypesProvider := instancetype.NewProvider("", options.InstanceTypeCache, ec2api, subnetProvider, options.UnavailableOfferingsCache, pricingProvider) launchTemplateProvider := launchtemplate.NewProvider( ctx, - launchTemplateCache, + options.LaunchTemplateCache, ec2api, amiResolver, securityGroupProvider, @@ -97,7 +90,7 @@ func Context(ctx context.Context, ec2api ec2iface.EC2API, ssmapi ssmiface.SSMAPI instance.NewProvider(ctx, "", ec2api, - unavailableOfferingsCache, + options.UnavailableOfferingsCache, instanceTypesProvider, subnetProvider, launchTemplateProvider, @@ -115,7 +108,7 @@ func Context(ctx context.Context, ec2api ec2iface.EC2API, ssmapi ssmiface.SSMAPI }, Session: mock.Session, EC2API: ec2api, - UnavailableOfferingsCache: unavailableOfferingsCache, + UnavailableOfferingsCache: options.UnavailableOfferingsCache, InstanceTypesProvider: instanceTypesProvider, InstanceProvider: instanceProvider, SubnetProvider: subnetProvider, From f7f7f4665de12ce6ee7409f79e3ad663a2c3faa6 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Mon, 6 Mar 2023 13:46:33 -0800 Subject: [PATCH 35/41] Added environment for testing --- pkg/cloudprovider/suite_test.go | 234 ++---- .../machine/garbagecollect/suite_test.go | 102 +-- pkg/controllers/machine/link/suite_test.go | 90 +- pkg/controllers/nodetemplate/suite_test.go | 74 +- pkg/providers/instancetype/suite_test.go | 173 ++-- pkg/providers/launchtemplate/suite_test.go | 795 ++++++++---------- pkg/providers/pricing/suite_test.go | 54 +- pkg/providers/securitygroup/suite_test.go | 31 +- pkg/providers/subnet/suite_test.go | 37 +- pkg/test/context.go | 129 --- pkg/test/environment.go | 152 ++++ 11 files changed, 801 insertions(+), 1070 deletions(-) delete mode 100644 pkg/test/context.go create mode 100644 pkg/test/environment.go diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index 2dc40275c6f0..1db30da426f8 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -22,7 +22,6 @@ import ( "time" "github.com/Pallinder/go-randomdata" - "github.com/patrickmn/go-cache" "github.com/samber/lo" "k8s.io/client-go/tools/record" @@ -43,15 +42,12 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" - awscache "github.com/aws/karpenter/pkg/cache" - "github.com/aws/karpenter/pkg/cloudprovider" - awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/test" coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" + corecloudproivder "github.com/aws/karpenter-core/pkg/cloudprovider" "github.com/aws/karpenter-core/pkg/controllers/provisioning" "github.com/aws/karpenter-core/pkg/controllers/state" "github.com/aws/karpenter-core/pkg/events" @@ -68,32 +64,18 @@ var ctx context.Context var stop context.CancelFunc var opts options.Options var env *coretest.Environment -var awsCtx awscontext.Context -var fakeEC2API *fake.EC2API -var fakeSSMAPI *fake.SSMAPI -var fakePricingAPI *fake.PricingAPI +var awsEnv *test.Environment var prov *provisioning.Provisioner var provisioningController controller.Controller -var cloudProvider *cloudprovider.CloudProvider var cluster *state.Cluster var fakeClock *clock.FakeClock var provisioner *v1alpha5.Provisioner var nodeTemplate *v1alpha1.AWSNodeTemplate -// Cache -var ec2Cache *cache.Cache -var ssmCache *cache.Cache -var kubernetesVersionCache *cache.Cache -var instanceTypeCache *cache.Cache -var unavailableOfferingsCache *awscache.UnavailableOfferings -var launchTemplateCache *cache.Cache -var subnetCache *cache.Cache -var securityGroupCache *cache.Cache - func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) RegisterFailHandler(Fail) - RunSpecs(t, "CloudProvider/AWS") + RunSpecs(t, "awsEnv.CloudProvider/AWS") } var _ = BeforeSuite(func() { @@ -101,37 +83,12 @@ var _ = BeforeSuite(func() { ctx = coresettings.ToContext(ctx, coretest.Settings()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) + awsEnv = test.NewEnvironment(ctx, env) - fakeEC2API = &fake.EC2API{} - fakeSSMAPI = &fake.SSMAPI{} fakeClock = clock.NewFakeClock(time.Now()) - fakePricingAPI = &fake.PricingAPI{} - - ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - unavailableOfferingsCache = awscache.NewUnavailableOfferings() - launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - subnetCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - securityGroupCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - - awsCtx = test.Context(ctx, fakeEC2API, fakeSSMAPI, env, fakeClock, test.ContextOptions{ - SSMCache: ssmCache, - EC2Cache: ec2Cache, - KubernetesVersionCache: kubernetesVersionCache, - UnavailableOfferingsCache: unavailableOfferingsCache, - LaunchTemplateCache: launchTemplateCache, - InstanceTypeCache: instanceTypeCache, - SubnetCache: subnetCache, - SecurityGroupCache: securityGroupCache, - PricingAPI: fakePricingAPI, - }) - - cloudProvider = cloudprovider.New(awsCtx, awsCtx.InstanceTypesProvider, awsCtx.InstanceProvider, awsCtx.KubeClient, awsCtx.AMIProvider) - cluster = state.NewCluster(fakeClock, awsCtx.KubeClient, cloudProvider) - prov = provisioning.NewProvisioner(ctx, awsCtx.KubeClient, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) - provisioningController = provisioning.NewController(awsCtx.KubeClient, prov, events.NewRecorder(&record.FakeRecorder{})) + cluster = state.NewCluster(fakeClock, env.Client, awsEnv.CloudProvider) + prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), awsEnv.CloudProvider, cluster) + provisioningController = provisioning.NewController(env.Client, prov, events.NewRecorder(&record.FakeRecorder{})) }) var _ = AfterSuite(func() { @@ -173,26 +130,17 @@ var _ = BeforeEach(func() { }) cluster.Reset() - fakeEC2API.Reset() - fakeSSMAPI.Reset() - ec2Cache.Flush() - ssmCache.Flush() - kubernetesVersionCache.Flush() - instanceTypeCache.Flush() - unavailableOfferingsCache.Flush() - launchTemplateCache.Flush() - subnetCache.Flush() - securityGroupCache.Flush() + awsEnv.ResetCache() - awsCtx.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") - awsCtx.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" + awsEnv.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") + awsEnv.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" }) var _ = AfterEach(func() { - ExpectCleanedUp(ctx, awsCtx.KubeClient) + ExpectCleanedUp(ctx, env.Client) }) -var _ = Describe("CloudProvider", func() { +var _ = Describe("awsEnv.CloudProvider", func() { Context("Defaulting", func() { // Intent here is that if updates occur on the provisioningController, the Provisioner doesn't need to be recreated It("should not set the InstanceProfile with the default if none provided in Provisioner", func() { @@ -224,39 +172,39 @@ var _ = Describe("CloudProvider", func() { provider.SecurityGroupSelector = map[string]string{"*": "*"} provisioner = coretest.Provisioner(coretest.ProvisionerOptions{Provider: provider}) provisioner.SetDefaults(ctx) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner) + ExpectApplied(ctx, env.Client, provisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) - createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(aws.StringValue(createFleetInput.Context)).To(Equal("context-1234")) }) It("should default to no EC2 Context", func() { provisioner.SetDefaults(ctx) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) - createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(createFleetInput.Context).To(BeNil()) }) }) Context("Node Drift", func() { var validAMI string - var selectedInstanceType *corecloudprovider.InstanceType + var selectedInstanceType *corecloudproivder.InstanceType var instance *ec2.Instance BeforeEach(func() { validAMI = fake.ImageID() - fakeSSMAPI.GetParameterOutput = &ssm.GetParameterOutput{ + awsEnv.SSMAPI.GetParameterOutput = &ssm.GetParameterOutput{ Parameter: &ssm.Parameter{Value: aws.String(validAMI)}, } - fakeEC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ + awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ Images: []*ec2.Image{{ImageId: aws.String(validAMI)}}, }) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) - instanceTypes, err := cloudProvider.GetInstanceTypes(ctx, provisioner) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) + instanceTypes, err := awsEnv.CloudProvider.GetInstanceTypes(ctx, provisioner) Expect(err).ToNot(HaveOccurred()) selectedInstanceType = instanceTypes[0] @@ -271,12 +219,12 @@ var _ = Describe("CloudProvider", func() { }, InstanceId: aws.String(fake.InstanceID()), } - fakeEC2API.DescribeInstancesBehavior.Output.Set(&ec2.DescribeInstancesOutput{ + awsEnv.EC2API.DescribeInstancesBehavior.Output.Set(&ec2.DescribeInstancesOutput{ Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{instance}}}, }) }) It("should not fail if node template does not exist", func() { - ExpectDeleted(ctx, awsCtx.KubeClient, nodeTemplate) + ExpectDeleted(ctx, env.Client, nodeTemplate) node := coretest.Node(coretest.NodeOptions{ ProviderID: fake.ProviderID(lo.FromPtr(instance.InstanceId)), ObjectMeta: metav1.ObjectMeta{ @@ -286,13 +234,13 @@ var _ = Describe("CloudProvider", func() { }, }, }) - drifted, err := cloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) + drifted, err := awsEnv.CloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) Expect(err).ToNot(HaveOccurred()) Expect(drifted).To(BeFalse()) }) It("should return false if providerRef is not defined", func() { provisioner.Spec.ProviderRef = nil - ExpectApplied(ctx, awsCtx.KubeClient, provisioner) + ExpectApplied(ctx, env.Client, provisioner) node := coretest.Node(coretest.NodeOptions{ ProviderID: fake.ProviderID(lo.FromPtr(instance.InstanceId)), ObjectMeta: metav1.ObjectMeta{ @@ -302,12 +250,12 @@ var _ = Describe("CloudProvider", func() { }, }, }) - drifted, err := cloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) + drifted, err := awsEnv.CloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) Expect(err).ToNot(HaveOccurred()) Expect(drifted).To(BeFalse()) }) It("should not fail if provisioner does not exist", func() { - ExpectDeleted(ctx, awsCtx.KubeClient, provisioner) + ExpectDeleted(ctx, env.Client, provisioner) node := coretest.Node(coretest.NodeOptions{ ProviderID: fake.ProviderID(lo.FromPtr(instance.InstanceId)), ObjectMeta: metav1.ObjectMeta{ @@ -317,7 +265,7 @@ var _ = Describe("CloudProvider", func() { }, }, }) - drifted, err := cloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) + drifted, err := awsEnv.CloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) Expect(err).ToNot(HaveOccurred()) Expect(drifted).To(BeFalse()) }) @@ -333,7 +281,7 @@ var _ = Describe("CloudProvider", func() { }) // Instance is a reference to what we return in the GetInstances call instance.ImageId = aws.String(fake.ImageID()) - isDrifted, err := cloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) + isDrifted, err := awsEnv.CloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) Expect(err).ToNot(HaveOccurred()) Expect(isDrifted).To(BeTrue()) }) @@ -347,7 +295,7 @@ var _ = Describe("CloudProvider", func() { }, }, }) - isDrifted, err := cloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) + isDrifted, err := awsEnv.CloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) Expect(err).ToNot(HaveOccurred()) Expect(isDrifted).To(BeFalse()) }) @@ -360,7 +308,7 @@ var _ = Describe("CloudProvider", func() { }, }, }) - _, err := cloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) + _, err := awsEnv.CloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) Expect(err).To(HaveOccurred()) }) It("should error drift if node doesn't have provider id", func() { @@ -372,7 +320,7 @@ var _ = Describe("CloudProvider", func() { }, }, }) - isDrifted, err := cloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) + isDrifted, err := awsEnv.CloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) Expect(err).To(HaveOccurred()) Expect(isDrifted).To(BeFalse()) }) @@ -390,16 +338,16 @@ var _ = Describe("CloudProvider", func() { Operator: v1.NodeSelectorOpExists, }}, }) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner) + ExpectApplied(ctx, env.Client, provisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - firstLt := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() - Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + firstLt := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) - createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() launchTemplate := createFleetInput.LaunchTemplateConfigs[0].LaunchTemplateSpecification Expect(createFleetInput.LaunchTemplateConfigs).To(HaveLen(1)) @@ -420,12 +368,12 @@ var _ = Describe("CloudProvider", func() { Operator: v1.NodeSelectorOpExists, }}, }) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner) + ExpectApplied(ctx, env.Client, provisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(aws.StringValueSlice(input.LaunchTemplateData.SecurityGroupIds)).To(ConsistOf( "sg-test1", )) @@ -442,11 +390,11 @@ var _ = Describe("CloudProvider", func() { Operator: v1.NodeSelectorOpExists, }}, }) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner) + ExpectApplied(ctx, env.Client, provisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("subnet-test1")) }) It("should use the instance profile on the Provisioner when specified", func() { @@ -462,12 +410,12 @@ var _ = Describe("CloudProvider", func() { Operator: v1.NodeSelectorOpExists, }}, }) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner) + ExpectApplied(ctx, env.Client, provisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(*input.LaunchTemplateData.IamInstanceProfile.Name).To(Equal("overridden-profile")) }) }) @@ -475,13 +423,13 @@ var _ = Describe("CloudProvider", func() { // Note when debugging these tests - // hard coded fixture data (ex. what the aws api will return) is maintained in fake/ec2api.go It("should default to the cluster's subnets", func() { - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod( coretest.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: v1alpha5.ArchitectureAmd64}}) - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) - input := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(input.LaunchTemplateConfigs).To(HaveLen(1)) foundNonGPULT := false @@ -500,77 +448,77 @@ var _ = Describe("CloudProvider", func() { Expect(foundNonGPULT).To(BeTrue()) }) It("should launch instances into subnet with the most available IP addresses", func() { - fakeEC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{ + awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{ {SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(10), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}}, {SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(100), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}}, }}) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}}) - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("test-subnet-2")) }) It("should launch instances into subnet with the most available IP addresses in-between cache refreshes", func() { - fakeEC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{ + awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{ {SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(10), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}}, {SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(11), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}}, }}) provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{MaxPods: aws.Int32(1)} - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod1 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}}) pod2 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}}) - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod1, pod2) - ExpectScheduled(ctx, awsCtx.KubeClient, pod1) - ExpectScheduled(ctx, awsCtx.KubeClient, pod2) - createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod1, pod2) + ExpectScheduled(ctx, env.Client, pod1) + ExpectScheduled(ctx, env.Client, pod2) + createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("test-subnet-2")) // Provision for another pod that should now use the other subnet since we've consumed some from the first launch. pod3 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}}) - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod3) - ExpectScheduled(ctx, awsCtx.KubeClient, pod3) - createFleetInput = fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod3) + ExpectScheduled(ctx, env.Client, pod3) + createFleetInput = awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("test-subnet-1")) }) It("should update in-flight IPs when a CreateFleet error occurs", func() { - fakeEC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{ + awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{ {SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(10), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}}, }}) pod1 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1a"}}) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate, pod1) - fakeEC2API.CreateFleetBehavior.Error.Set(fmt.Errorf("CreateFleet synthetic error")) - bindings := ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod1) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate, pod1) + awsEnv.EC2API.CreateFleetBehavior.Error.Set(fmt.Errorf("CreateFleet synthetic error")) + bindings := ExpectProvisioned(ctx, env.Client, cluster, prov, pod1) Expect(len(bindings)).To(Equal(0)) }) It("should launch instances into subnets that are excluded by another provisioner", func() { - fakeEC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{ + awsEnv.EC2API.DescribeSubnetsOutput.Set(&ec2.DescribeSubnetsOutput{Subnets: []*ec2.Subnet{ {SubnetId: aws.String("test-subnet-1"), AvailabilityZone: aws.String("test-zone-1a"), AvailableIpAddressCount: aws.Int64(10), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-1")}}}, {SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1b"), AvailableIpAddressCount: aws.Int64(100), Tags: []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}}, }}) nodeTemplate.Spec.SubnetSelector = map[string]string{"Name": "test-subnet-1"} - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) podSubnet1 := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, podSubnet1) - ExpectScheduled(ctx, awsCtx.KubeClient, podSubnet1) - createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, podSubnet1) + ExpectScheduled(ctx, env.Client, podSubnet1) + createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("test-subnet-1")) provisioner = test.Provisioner(coretest.ProvisionerOptions{Provider: &v1alpha1.AWS{ SubnetSelector: map[string]string{"Name": "test-subnet-2"}, SecurityGroupSelector: map[string]string{"*": "*"}, }}) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner) + ExpectApplied(ctx, env.Client, provisioner) podSubnet2 := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{v1alpha5.ProvisionerNameLabelKey: provisioner.Name}}) - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, podSubnet2) - ExpectScheduled(ctx, awsCtx.KubeClient, podSubnet2) - createFleetInput = fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, podSubnet2) + ExpectScheduled(ctx, env.Client, podSubnet2) + createFleetInput = awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(fake.SubnetsFromFleetRequest(createFleetInput)).To(ConsistOf("test-subnet-2")) }) }) diff --git a/pkg/controllers/machine/garbagecollect/suite_test.go b/pkg/controllers/machine/garbagecollect/suite_test.go index f4a362f2b49a..06f64c07fbc5 100644 --- a/pkg/controllers/machine/garbagecollect/suite_test.go +++ b/pkg/controllers/machine/garbagecollect/suite_test.go @@ -28,13 +28,12 @@ import ( "github.com/patrickmn/go-cache" "github.com/samber/lo" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clock "k8s.io/utils/clock/testing" . "knative.dev/pkg/logging/testing" "sigs.k8s.io/controller-runtime/pkg/client" coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" + "github.com/aws/karpenter-core/pkg/cloudprovider" "github.com/aws/karpenter-core/pkg/operator/controller" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" @@ -43,9 +42,6 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" - awscache "github.com/aws/karpenter/pkg/cache" - "github.com/aws/karpenter/pkg/cloudprovider" - awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/controllers/machine/garbagecollect" "github.com/aws/karpenter/pkg/controllers/machine/link" "github.com/aws/karpenter/pkg/fake" @@ -53,24 +49,12 @@ import ( ) var ctx context.Context -var awsCtx awscontext.Context +var awsEnv *test.Environment var env *coretest.Environment -var ec2API *fake.EC2API -var cloudProvider *cloudprovider.CloudProvider var garbageCollectController controller.Controller var linkedMachineCache *cache.Cache var fakePricingAPI *fake.PricingAPI -// Cache -var ec2Cache *cache.Cache -var ssmCache *cache.Cache -var kubernetesVersionCache *cache.Cache -var instanceTypeCache *cache.Cache -var unavailableOfferingsCache *awscache.UnavailableOfferings -var launchTemplateCache *cache.Cache -var subnetCache *cache.Cache -var securityGroupCache *cache.Cache - func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) RegisterFailHandler(Fail) @@ -81,38 +65,13 @@ var _ = BeforeSuite(func() { ctx = coresettings.ToContext(ctx, coretest.Settings()) ctx = settings.ToContext(ctx, test.Settings()) env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ec2API = &fake.EC2API{} - - fakePricingAPI = &fake.PricingAPI{} - - ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - unavailableOfferingsCache = awscache.NewUnavailableOfferings() - launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - subnetCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - securityGroupCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - - awsCtx = test.Context(ctx, ec2API, fake.SSMAPI{}, env, &clock.FakeClock{}, test.ContextOptions{ - SSMCache: ssmCache, - EC2Cache: ec2Cache, - KubernetesVersionCache: kubernetesVersionCache, - UnavailableOfferingsCache: unavailableOfferingsCache, - LaunchTemplateCache: launchTemplateCache, - InstanceTypeCache: instanceTypeCache, - SubnetCache: subnetCache, - SecurityGroupCache: securityGroupCache, - PricingAPI: fakePricingAPI, - }) - - cloudProvider = cloudprovider.New(awsCtx, awsCtx.InstanceTypesProvider, awsCtx.InstanceProvider, awsCtx.KubeClient, awsCtx.AMIProvider) + awsEnv = test.NewEnvironment(ctx, env) linkedMachineCache = cache.New(time.Minute*10, time.Second*10) linkController := &link.Controller{ Cache: linkedMachineCache, } - garbageCollectController = garbagecollect.NewController(env.Client, cloudProvider, linkController) + garbageCollectController = garbagecollect.NewController(env.Client, awsEnv.CloudProvider, linkController) }) var _ = AfterSuite(func() { @@ -120,14 +79,7 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ec2Cache.Flush() - ssmCache.Flush() - kubernetesVersionCache.Flush() - instanceTypeCache.Flush() - unavailableOfferingsCache.Flush() - launchTemplateCache.Flush() - subnetCache.Flush() - securityGroupCache.Flush() + awsEnv.ResetCache() }) var _ = Describe("MachineGarbageCollect", func() { @@ -135,7 +87,6 @@ var _ = Describe("MachineGarbageCollect", func() { var providerID string BeforeEach(func() { - ec2API.Reset() instanceID := fake.InstanceID() providerID = fmt.Sprintf("aws:///test-zone-1a/%s", instanceID) nodeTemplate := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{}) @@ -171,6 +122,7 @@ var _ = Describe("MachineGarbageCollect", func() { InstanceId: aws.String(instanceID), InstanceType: aws.String("m5.large"), } + awsEnv.ResetCache() }) AfterEach(func() { ExpectCleanedUp(ctx, env.Client) @@ -180,17 +132,17 @@ var _ = Describe("MachineGarbageCollect", func() { It("should delete an instance if there is no machine owner", func() { // Launch time was 10m ago instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute * 10)) - ec2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) + awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) ExpectReconcileSucceeded(ctx, garbageCollectController, client.ObjectKey{}) - _, err := cloudProvider.Get(ctx, providerID) + _, err := awsEnv.CloudProvider.Get(ctx, providerID) Expect(err).To(HaveOccurred()) - Expect(corecloudprovider.IsMachineNotFoundError(err)).To(BeTrue()) + Expect(cloudprovider.IsMachineNotFoundError(err)).To(BeTrue()) }) It("should delete an instance along with the node if there is no machine owner (to quicken scheduling)", func() { // Launch time was 10m ago instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute * 10)) - ec2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) + awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) node := coretest.Node(coretest.NodeOptions{ ProviderID: providerID, @@ -198,9 +150,9 @@ var _ = Describe("MachineGarbageCollect", func() { ExpectApplied(ctx, env.Client, node) ExpectReconcileSucceeded(ctx, garbageCollectController, client.ObjectKey{}) - _, err := cloudProvider.Get(ctx, providerID) + _, err := awsEnv.CloudProvider.Get(ctx, providerID) Expect(err).To(HaveOccurred()) - Expect(corecloudprovider.IsMachineNotFoundError(err)).To(BeTrue()) + Expect(cloudprovider.IsMachineNotFoundError(err)).To(BeTrue()) ExpectNotFound(ctx, env.Client, node) }) @@ -209,7 +161,7 @@ var _ = Describe("MachineGarbageCollect", func() { var ids []string for i := 0; i < 500; i++ { instanceID := fake.InstanceID() - ec2API.Instances.Store( + awsEnv.EC2API.Instances.Store( instanceID, &ec2.Instance{ State: &ec2.InstanceState{ @@ -250,9 +202,9 @@ var _ = Describe("MachineGarbageCollect", func() { defer GinkgoRecover() defer wg.Done() - _, err := cloudProvider.Get(ctx, fmt.Sprintf("aws:///test-zone-1a/%s", id)) + _, err := awsEnv.CloudProvider.Get(ctx, fmt.Sprintf("aws:///test-zone-1a/%s", id)) Expect(err).To(HaveOccurred()) - Expect(corecloudprovider.IsMachineNotFoundError(err)).To(BeTrue()) + Expect(cloudprovider.IsMachineNotFoundError(err)).To(BeTrue()) }(id) } wg.Wait() @@ -263,7 +215,7 @@ var _ = Describe("MachineGarbageCollect", func() { var machines []*v1alpha5.Machine for i := 0; i < 500; i++ { instanceID := fake.InstanceID() - ec2API.Instances.Store( + awsEnv.EC2API.Instances.Store( instanceID, &ec2.Instance{ State: &ec2.InstanceState{ @@ -311,7 +263,7 @@ var _ = Describe("MachineGarbageCollect", func() { defer GinkgoRecover() defer wg.Done() - _, err := cloudProvider.Get(ctx, fmt.Sprintf("aws:///test-zone-1a/%s", id)) + _, err := awsEnv.CloudProvider.Get(ctx, fmt.Sprintf("aws:///test-zone-1a/%s", id)) Expect(err).ToNot(HaveOccurred()) }(id) } @@ -324,10 +276,10 @@ var _ = Describe("MachineGarbageCollect", func() { It("should not delete an instance if it is within the machine resolution window (1m)", func() { // Launch time just happened instance.LaunchTime = aws.Time(time.Now()) - ec2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) + awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) ExpectReconcileSucceeded(ctx, garbageCollectController, client.ObjectKey{}) - _, err := cloudProvider.Get(ctx, providerID) + _, err := awsEnv.CloudProvider.Get(ctx, providerID) Expect(err).NotTo(HaveOccurred()) }) It("should not delete an instance if it was not launched by a machine", func() { @@ -338,16 +290,16 @@ var _ = Describe("MachineGarbageCollect", func() { // Launch time was 10m ago instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute * 10)) - ec2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) + awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) ExpectReconcileSucceeded(ctx, garbageCollectController, client.ObjectKey{}) - _, err := cloudProvider.Get(ctx, providerID) + _, err := awsEnv.CloudProvider.Get(ctx, providerID) Expect(err).NotTo(HaveOccurred()) }) It("should not delete the instance or node if it already has a machine that matches it", func() { // Launch time was 10m ago instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute * 10)) - ec2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) + awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) machine := coretest.Machine(v1alpha5.Machine{ Status: v1alpha5.MachineStatus{ @@ -360,14 +312,14 @@ var _ = Describe("MachineGarbageCollect", func() { ExpectApplied(ctx, env.Client, machine, node) ExpectReconcileSucceeded(ctx, garbageCollectController, client.ObjectKey{}) - _, err := cloudProvider.Get(ctx, providerID) + _, err := awsEnv.CloudProvider.Get(ctx, providerID) Expect(err).ToNot(HaveOccurred()) ExpectExists(ctx, env.Client, node) }) It("should not delete an instance if it is linked", func() { // Launch time was 10m ago instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute * 10)) - ec2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) + awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) // Create a machine that is actively linking machine := coretest.Machine(v1alpha5.Machine{ @@ -380,19 +332,19 @@ var _ = Describe("MachineGarbageCollect", func() { ExpectApplied(ctx, env.Client, machine) ExpectReconcileSucceeded(ctx, garbageCollectController, client.ObjectKey{}) - _, err := cloudProvider.Get(ctx, providerID) + _, err := awsEnv.CloudProvider.Get(ctx, providerID) Expect(err).NotTo(HaveOccurred()) }) It("should not delete an instance if it is recently linked but the machine doesn't exist", func() { // Launch time was 10m ago instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute * 10)) - ec2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) + awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) // Add a provider id to the recently linked cache linkedMachineCache.SetDefault(providerID, nil) ExpectReconcileSucceeded(ctx, garbageCollectController, client.ObjectKey{}) - _, err := cloudProvider.Get(ctx, providerID) + _, err := awsEnv.CloudProvider.Get(ctx, providerID) Expect(err).NotTo(HaveOccurred()) }) }) diff --git a/pkg/controllers/machine/link/suite_test.go b/pkg/controllers/machine/link/suite_test.go index 15f7e1b8461f..56cfa29b93f4 100644 --- a/pkg/controllers/machine/link/suite_test.go +++ b/pkg/controllers/machine/link/suite_test.go @@ -24,11 +24,9 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/patrickmn/go-cache" "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" - clock "k8s.io/utils/clock/testing" . "knative.dev/pkg/logging/testing" "sigs.k8s.io/controller-runtime/pkg/client" @@ -43,9 +41,6 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" - awscache "github.com/aws/karpenter/pkg/cache" - "github.com/aws/karpenter/pkg/cloudprovider" - awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/controllers/machine/link" "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/test" @@ -53,22 +48,9 @@ import ( ) var ctx context.Context -var awsCtx awscontext.Context +var awsEnv *test.Environment var env *coretest.Environment -var ec2API *fake.EC2API -var cloudProvider *cloudprovider.CloudProvider var linkController controller.Controller -var fakePricingAPI *fake.PricingAPI - -// Cache -var ec2Cache *cache.Cache -var ssmCache *cache.Cache -var kubernetesVersionCache *cache.Cache -var instanceTypeCache *cache.Cache -var unavailableOfferingsCache *awscache.UnavailableOfferings -var launchTemplateCache *cache.Cache -var subnetCache *cache.Cache -var securityGroupCache *cache.Cache func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -80,48 +62,16 @@ var _ = BeforeSuite(func() { ctx = coresettings.ToContext(ctx, coretest.Settings()) ctx = settings.ToContext(ctx, test.Settings()) env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + awsEnv = test.NewEnvironment(ctx, env) - ec2API = &fake.EC2API{} - fakePricingAPI = &fake.PricingAPI{} - - ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - unavailableOfferingsCache = awscache.NewUnavailableOfferings() - launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - subnetCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - securityGroupCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - - awsCtx = test.Context(ctx, ec2API, fake.SSMAPI{}, env, &clock.FakeClock{}, test.ContextOptions{ - SSMCache: ssmCache, - EC2Cache: ec2Cache, - KubernetesVersionCache: kubernetesVersionCache, - UnavailableOfferingsCache: unavailableOfferingsCache, - LaunchTemplateCache: launchTemplateCache, - InstanceTypeCache: instanceTypeCache, - SubnetCache: subnetCache, - SecurityGroupCache: securityGroupCache, - PricingAPI: fakePricingAPI, - }) - - cloudProvider = cloudprovider.New(awsCtx, awsCtx.InstanceTypesProvider, awsCtx.InstanceProvider, awsCtx.KubeClient, awsCtx.AMIProvider) - - linkController = link.NewController(env.Client, cloudProvider) + linkController = link.NewController(env.Client, awsEnv.CloudProvider) }) var _ = AfterSuite(func() { Expect(env.Stop()).To(Succeed(), "Failed to stop environment") }) var _ = BeforeEach(func() { - ec2Cache.Flush() - ssmCache.Flush() - kubernetesVersionCache.Flush() - instanceTypeCache.Flush() - unavailableOfferingsCache.Flush() - launchTemplateCache.Flush() - subnetCache.Flush() - securityGroupCache.Flush() + awsEnv.ResetCache() }) var _ = Describe("MachineLink", func() { @@ -131,7 +81,6 @@ var _ = Describe("MachineLink", func() { var nodeTemplate *v1alpha1.AWSNodeTemplate BeforeEach(func() { - ec2API.Reset() instanceID = fake.InstanceID() providerID = fmt.Sprintf("aws:///test-zone-1a/%s", instanceID) nodeTemplate = test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{}) @@ -144,7 +93,7 @@ var _ = Describe("MachineLink", func() { }) // Store the instance as existing at DescribeInstances - ec2API.Instances.Store( + awsEnv.EC2API.Instances.Store( instanceID, &ec2.Instance{ State: &ec2.InstanceState{ @@ -168,6 +117,7 @@ var _ = Describe("MachineLink", func() { InstanceType: aws.String("m5.large"), }, ) + awsEnv.ResetCache() }) AfterEach(func() { ExpectCleanedUp(ctx, env.Client) @@ -190,7 +140,7 @@ var _ = Describe("MachineLink", func() { }, } ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - ExpectInstanceExists(ec2API, instanceID) + ExpectInstanceExists(awsEnv.EC2API, instanceID) ExpectReconcileSucceeded(ctx, linkController, client.ObjectKey{}) machineList := &v1alpha5.MachineList{} @@ -206,7 +156,7 @@ var _ = Describe("MachineLink", func() { // Expect machine has linking annotation to get machine details Expect(machine.Annotations).To(HaveKeyWithValue(v1alpha5.MachineLinkedAnnotationKey, providerID)) - instance := ExpectInstanceExists(ec2API, instanceID) + instance := ExpectInstanceExists(awsEnv.EC2API, instanceID) ExpectManagedByTagExists(instance) }) It("should link and instance with expected requirements and labels", func() { @@ -228,7 +178,7 @@ var _ = Describe("MachineLink", func() { }, } ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - ExpectInstanceExists(ec2API, instanceID) + ExpectInstanceExists(awsEnv.EC2API, instanceID) ExpectReconcileSucceeded(ctx, linkController, client.ObjectKey{}) machineList := &v1alpha5.MachineList{} @@ -257,7 +207,7 @@ var _ = Describe("MachineLink", func() { // Expect machine has linking annotation to get machine details Expect(machine.Annotations).To(HaveKeyWithValue(v1alpha5.MachineLinkedAnnotationKey, providerID)) - instance := ExpectInstanceExists(ec2API, instanceID) + instance := ExpectInstanceExists(awsEnv.EC2API, instanceID) ExpectManagedByTagExists(instance) }) It("should link an instance with expected kubelet from provisioner kubelet configuration", func() { @@ -281,18 +231,18 @@ var _ = Describe("MachineLink", func() { // Expect machine has linking annotation to get machine details Expect(machine.Annotations).To(HaveKeyWithValue(v1alpha5.MachineLinkedAnnotationKey, providerID)) - instance := ExpectInstanceExists(ec2API, instanceID) + instance := ExpectInstanceExists(awsEnv.EC2API, instanceID) ExpectManagedByTagExists(instance) }) It("should link many instances to many machines", func() { - ec2API.Reset() // Reset so we don't store the extra instance + awsEnv.EC2API.Reset() // Reset so we don't store the extra instance ExpectApplied(ctx, env.Client, provisioner) // Generate 500 instances that have different instanceIDs var ids []string for i := 0; i < 500; i++ { instanceID = fake.InstanceID() - ec2API.EC2Behavior.Instances.Store( + awsEnv.EC2API.EC2Behavior.Instances.Store( instanceID, &ec2.Instance{ State: &ec2.InstanceState{ @@ -333,7 +283,7 @@ var _ = Describe("MachineLink", func() { Expect(machineInstanceIDs).To(HaveLen(len(ids))) for _, id := range ids { Expect(machineInstanceIDs.Has(id)).To(BeTrue()) - instance := ExpectInstanceExists(ec2API, id) + instance := ExpectInstanceExists(awsEnv.EC2API, id) ExpectManagedByTagExists(instance) } }) @@ -358,7 +308,7 @@ var _ = Describe("MachineLink", func() { // Expect machine has linking annotation to get machine details Expect(machine.Annotations).To(HaveKeyWithValue(v1alpha5.MachineLinkedAnnotationKey, providerID)) - instance := ExpectInstanceExists(ec2API, instanceID) + instance := ExpectInstanceExists(awsEnv.EC2API, instanceID) ExpectManagedByTagExists(instance) }) It("should link an instance without node template existence", func() { @@ -375,17 +325,17 @@ var _ = Describe("MachineLink", func() { // Expect machine has linking annotation to get machine details Expect(machine.Annotations).To(HaveKeyWithValue(v1alpha5.MachineLinkedAnnotationKey, providerID)) - instance := ExpectInstanceExists(ec2API, instanceID) + instance := ExpectInstanceExists(awsEnv.EC2API, instanceID) ExpectManagedByTagExists(instance) }) }) Context("Failed", func() { It("should not link an instance without a provisioner tag", func() { - instance := ExpectInstanceExists(ec2API, instanceID) + instance := ExpectInstanceExists(awsEnv.EC2API, instanceID) instance.Tags = lo.Reject(instance.Tags, func(t *ec2.Tag, _ int) bool { return aws.StringValue(t.Key) == v1alpha5.ProvisionerNameLabelKey }) - ec2API.Instances.Store(instanceID, instance) + awsEnv.EC2API.Instances.Store(instanceID, instance) ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) ExpectReconcileSucceeded(ctx, linkController, client.ObjectKey{}) @@ -422,9 +372,9 @@ var _ = Describe("MachineLink", func() { }) It("should not link an instance that is terminated", func() { // Update the state of the existing instance - instance := ExpectInstanceExists(ec2API, instanceID) + instance := ExpectInstanceExists(awsEnv.EC2API, instanceID) instance.State.Name = aws.String(ec2.InstanceStateNameTerminated) - ec2API.Instances.Store(instanceID, instance) + awsEnv.EC2API.Instances.Store(instanceID, instance) ExpectReconcileSucceeded(ctx, linkController, client.ObjectKey{}) machineList := &v1alpha5.MachineList{} diff --git a/pkg/controllers/nodetemplate/suite_test.go b/pkg/controllers/nodetemplate/suite_test.go index a4f7e30160fe..e780f677f7c6 100644 --- a/pkg/controllers/nodetemplate/suite_test.go +++ b/pkg/controllers/nodetemplate/suite_test.go @@ -19,44 +19,36 @@ import ( "sort" "testing" + "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/patrickmn/go-cache" "github.com/samber/lo" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "knative.dev/pkg/logging/testing" _ "knative.dev/pkg/system/testing" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/aws/aws-sdk-go/service/ec2" - + coresettings "github.com/aws/karpenter-core/pkg/apis/settings" + corecontroller "github.com/aws/karpenter-core/pkg/operator/controller" "github.com/aws/karpenter-core/pkg/operator/injection" "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - corecontroller "github.com/aws/karpenter-core/pkg/operator/controller" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" + "github.com/aws/karpenter/pkg/apis" + "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" - awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/controllers/nodetemplate" - "github.com/aws/karpenter/pkg/fake" - "github.com/aws/karpenter/pkg/providers/securitygroup" - "github.com/aws/karpenter/pkg/providers/subnet" + "github.com/aws/karpenter/pkg/test" ) var ctx context.Context var env *coretest.Environment -var fakeEC2API *fake.EC2API +var awsEnv *test.Environment var opts options.Options -var subnetProvider *subnet.Provider -var securityGroupProvider *securitygroup.Provider var nodeTemplate *v1alpha1.AWSNodeTemplate var controller corecontroller.Controller -var subnetCache *cache.Cache -var securityGroupCache *cache.Cache func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -66,13 +58,11 @@ func TestAPIs(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = settings.ToContext(ctx, test.Settings()) + awsEnv = test.NewEnvironment(ctx, env) - fakeEC2API = &fake.EC2API{} - subnetCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - securityGroupCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - subnetProvider = subnet.NewProvider(fakeEC2API, subnetCache) - securityGroupProvider = securitygroup.NewProvider(fakeEC2API, securityGroupCache) - controller = nodetemplate.NewController(env.Client, subnetProvider, securityGroupProvider) + controller = nodetemplate.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider) }) var _ = AfterSuite(func() { @@ -94,9 +84,7 @@ var _ = BeforeEach(func() { }, } - fakeEC2API.Reset() - subnetCache.Flush() - securityGroupCache.Flush() + awsEnv.ResetCache() }) var _ = AfterEach(func() { @@ -109,7 +97,7 @@ var _ = Describe("AWSNodeTemplateController", func() { ExpectApplied(ctx, env.Client, nodeTemplate) ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - subnet, _ := subnetProvider.List(ctx, nodeTemplate) + subnet, _ := awsEnv.SubnetProvider.List(ctx, nodeTemplate) subnetIDs := lo.Map(subnet, func(ec2subnet *ec2.Subnet, _ int) string { return *ec2subnet.SubnetId }) @@ -124,7 +112,7 @@ var _ = Describe("AWSNodeTemplateController", func() { ExpectApplied(ctx, env.Client, nodeTemplate) ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - subnet, _ := subnetProvider.List(ctx, nodeTemplate) + subnet, _ := awsEnv.SubnetProvider.List(ctx, nodeTemplate) sort.Slice(subnet, func(i, j int) bool { return int(*subnet[i].AvailableIpAddressCount) > int(*subnet[j].AvailableIpAddressCount) }) @@ -141,7 +129,7 @@ var _ = Describe("AWSNodeTemplateController", func() { ExpectApplied(ctx, env.Client, nodeTemplate) ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - subnet, _ := subnetProvider.List(ctx, nodeTemplate) + subnet, _ := awsEnv.SubnetProvider.List(ctx, nodeTemplate) sort.Slice(subnet, func(i, j int) bool { return int(*subnet[i].AvailableIpAddressCount) > int(*subnet[j].AvailableIpAddressCount) }) @@ -158,7 +146,7 @@ var _ = Describe("AWSNodeTemplateController", func() { ExpectApplied(ctx, env.Client, nodeTemplate) ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - subnet, _ := subnetProvider.List(ctx, nodeTemplate) + subnet, _ := awsEnv.SubnetProvider.List(ctx, nodeTemplate) correctSubnetIDs := lo.Map(subnet, func(ec2subnet *ec2.Subnet, _ int) v1alpha1.SubnetStatus { return v1alpha1.SubnetStatus{ ID: *ec2subnet.SubnetId, @@ -172,7 +160,7 @@ var _ = Describe("AWSNodeTemplateController", func() { ExpectApplied(ctx, env.Client, nodeTemplate) ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - subnet, _ := subnetProvider.List(ctx, nodeTemplate) + subnet, _ := awsEnv.SubnetProvider.List(ctx, nodeTemplate) subnetIDs := lo.Map(subnet, func(ec2subnet *ec2.Subnet, _ int) string { return *ec2subnet.SubnetId }) @@ -187,7 +175,7 @@ var _ = Describe("AWSNodeTemplateController", func() { ExpectApplied(ctx, env.Client, nodeTemplate) ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - subnet, _ = subnetProvider.List(ctx, nodeTemplate) + subnet, _ = awsEnv.SubnetProvider.List(ctx, nodeTemplate) sort.Slice(subnet, func(i, j int) bool { return int(*subnet[i].AvailableIpAddressCount) > int(*subnet[j].AvailableIpAddressCount) }) @@ -203,7 +191,7 @@ var _ = Describe("AWSNodeTemplateController", func() { ExpectApplied(ctx, env.Client, nodeTemplate) ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - subnet, _ := subnetProvider.List(ctx, nodeTemplate) + subnet, _ := awsEnv.SubnetProvider.List(ctx, nodeTemplate) subnetIDs := lo.Map(subnet, func(ec2subnet *ec2.Subnet, _ int) string { return *ec2subnet.SubnetId }) @@ -218,7 +206,7 @@ var _ = Describe("AWSNodeTemplateController", func() { ExpectApplied(ctx, env.Client, nodeTemplate) ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - subnet, _ = subnetProvider.List(ctx, nodeTemplate) + subnet, _ = awsEnv.SubnetProvider.List(ctx, nodeTemplate) correctSubnetIDs := lo.Map(subnet, func(ec2subnet *ec2.Subnet, _ int) v1alpha1.SubnetStatus { return v1alpha1.SubnetStatus{ ID: *ec2subnet.SubnetId, @@ -239,7 +227,7 @@ var _ = Describe("AWSNodeTemplateController", func() { ExpectApplied(ctx, env.Client, nodeTemplate) ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - subnet, _ := subnetProvider.List(ctx, nodeTemplate) + subnet, _ := awsEnv.SubnetProvider.List(ctx, nodeTemplate) subnetIDs := lo.Map(subnet, func(ec2subnet *ec2.Subnet, _ int) string { return *ec2subnet.SubnetId }) @@ -264,7 +252,7 @@ var _ = Describe("AWSNodeTemplateController", func() { ExpectApplied(ctx, env.Client, nodeTemplate) ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - securityGroupsIDs, _ := securityGroupProvider.List(ctx, nodeTemplate) + securityGroupsIDs, _ := awsEnv.SecurityGroupProvider.List(ctx, nodeTemplate) securityGroupsIDInStatus := lo.Map(nodeTemplate.Status.SecurityGroups, func(securitygroup v1alpha1.SecurityGroupStatus, _ int) string { return securitygroup.ID }) @@ -274,7 +262,7 @@ var _ = Describe("AWSNodeTemplateController", func() { ExpectApplied(ctx, env.Client, nodeTemplate) ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - securityGroupsIDs, _ := securityGroupProvider.List(ctx, nodeTemplate) + securityGroupsIDs, _ := awsEnv.SecurityGroupProvider.List(ctx, nodeTemplate) securityGroupsIDInStatus := lo.Map(nodeTemplate.Status.SecurityGroups, func(securitygroup v1alpha1.SecurityGroupStatus, _ int) string { return securitygroup.ID }) @@ -285,7 +273,7 @@ var _ = Describe("AWSNodeTemplateController", func() { ExpectApplied(ctx, env.Client, nodeTemplate) ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - securityGroupsIDs, _ := securityGroupProvider.List(ctx, nodeTemplate) + securityGroupsIDs, _ := awsEnv.SecurityGroupProvider.List(ctx, nodeTemplate) correctSecurityGroupsIDs := lo.Map(securityGroupsIDs, func(securitygroup string, _ int) v1alpha1.SecurityGroupStatus { return v1alpha1.SecurityGroupStatus{ ID: securitygroup, @@ -298,7 +286,7 @@ var _ = Describe("AWSNodeTemplateController", func() { ExpectApplied(ctx, env.Client, nodeTemplate) ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - securityGroupsIDs, _ := securityGroupProvider.List(ctx, nodeTemplate) + securityGroupsIDs, _ := awsEnv.SecurityGroupProvider.List(ctx, nodeTemplate) correctSecurityGroupsIDs := lo.Map(securityGroupsIDs, func(securitygroup string, _ int) v1alpha1.SecurityGroupStatus { return v1alpha1.SecurityGroupStatus{ ID: securitygroup, @@ -310,7 +298,7 @@ var _ = Describe("AWSNodeTemplateController", func() { ExpectApplied(ctx, env.Client, nodeTemplate) ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - securityGroupsIDs, _ := securityGroupProvider.List(ctx, nodeTemplate) + securityGroupsIDs, _ := awsEnv.SecurityGroupProvider.List(ctx, nodeTemplate) securityGroupsIDInStatus := lo.Map(nodeTemplate.Status.SecurityGroups, func(securitygroup v1alpha1.SecurityGroupStatus, _ int) string { return securitygroup.ID }) @@ -320,7 +308,7 @@ var _ = Describe("AWSNodeTemplateController", func() { ExpectApplied(ctx, env.Client, nodeTemplate) ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - securityGroupsIDs, _ = securityGroupProvider.List(ctx, nodeTemplate) + securityGroupsIDs, _ = awsEnv.SecurityGroupProvider.List(ctx, nodeTemplate) correctSecurityGroupsIDs := lo.Map(securityGroupsIDs, func(securitygroup string, _ int) v1alpha1.SecurityGroupStatus { return v1alpha1.SecurityGroupStatus{ ID: securitygroup, @@ -332,7 +320,7 @@ var _ = Describe("AWSNodeTemplateController", func() { ExpectApplied(ctx, env.Client, nodeTemplate) ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - securityGroupsIDs, _ := securityGroupProvider.List(ctx, nodeTemplate) + securityGroupsIDs, _ := awsEnv.SecurityGroupProvider.List(ctx, nodeTemplate) securityGroupsIDInStatus := lo.Map(nodeTemplate.Status.SecurityGroups, func(securitygroup v1alpha1.SecurityGroupStatus, _ int) string { return securitygroup.ID }) @@ -342,7 +330,7 @@ var _ = Describe("AWSNodeTemplateController", func() { ExpectApplied(ctx, env.Client, nodeTemplate) ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - securityGroupsIDs, _ = securityGroupProvider.List(ctx, nodeTemplate) + securityGroupsIDs, _ = awsEnv.SecurityGroupProvider.List(ctx, nodeTemplate) correctSecurityGroupsIDs := lo.Map(securityGroupsIDs, func(securitygroup string, _ int) v1alpha1.SecurityGroupStatus { return v1alpha1.SecurityGroupStatus{ ID: securitygroup, @@ -361,7 +349,7 @@ var _ = Describe("AWSNodeTemplateController", func() { ExpectApplied(ctx, env.Client, nodeTemplate) ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(nodeTemplate)) nodeTemplate = ExpectExists(ctx, env.Client, nodeTemplate) - securityGroupsIDs, _ := securityGroupProvider.List(ctx, nodeTemplate) + securityGroupsIDs, _ := awsEnv.SecurityGroupProvider.List(ctx, nodeTemplate) securityGroupsIDInStatus := lo.Map(nodeTemplate.Status.SecurityGroups, func(securitygroup v1alpha1.SecurityGroupStatus, _ int) string { return securitygroup.ID }) diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index eeaa66d6b11e..beaac3da7833 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -28,7 +28,6 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/patrickmn/go-cache" "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -40,8 +39,9 @@ import ( . "knative.dev/pkg/logging/testing" "knative.dev/pkg/ptr" + coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" + corecloudproivder "github.com/aws/karpenter-core/pkg/cloudprovider" "github.com/aws/karpenter-core/pkg/controllers/provisioning" "github.com/aws/karpenter-core/pkg/controllers/state" "github.com/aws/karpenter-core/pkg/events" @@ -50,17 +50,14 @@ import ( "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" "github.com/aws/karpenter-core/pkg/scheduling" + coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" "github.com/aws/karpenter-core/pkg/utils/resources" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" - coretest "github.com/aws/karpenter-core/pkg/test" "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" - awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/cloudprovider" - awscontext "github.com/aws/karpenter/pkg/context" "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/providers/instancetype" @@ -72,28 +69,14 @@ var ctx context.Context var stop context.CancelFunc var opts options.Options var env *coretest.Environment -var awsCtx awscontext.Context -var fakeEC2API *fake.EC2API -var fakeSSMAPI *fake.SSMAPI +var awsEnv *test.Environment var fakeClock *clock.FakeClock -var fakePricingAPI *fake.PricingAPI var prov *provisioning.Provisioner var provisioner *v1alpha5.Provisioner var nodeTemplate *v1alpha1.AWSNodeTemplate var cluster *state.Cluster -var cloudProvider *cloudprovider.CloudProvider var provisioningController controller.Controller -// Cache -var ec2Cache *cache.Cache -var ssmCache *cache.Cache -var kubernetesVersionCache *cache.Cache -var instanceTypeCache *cache.Cache -var unavailableOfferingsCache *awscache.UnavailableOfferings -var launchTemplateCache *cache.Cache -var subnetCache *cache.Cache -var securityGroupCache *cache.Cache - func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) RegisterFailHandler(Fail) @@ -105,36 +88,11 @@ var _ = BeforeSuite(func() { ctx = coresettings.ToContext(ctx, coretest.Settings()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) + awsEnv = test.NewEnvironment(ctx, env) - fakeEC2API = &fake.EC2API{} - fakeSSMAPI = &fake.SSMAPI{} fakeClock = &clock.FakeClock{} - fakePricingAPI = &fake.PricingAPI{} - - ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - unavailableOfferingsCache = awscache.NewUnavailableOfferings() - launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - subnetCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - securityGroupCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - - awsCtx = test.Context(ctx, fakeEC2API, fakeSSMAPI, env, fakeClock, test.ContextOptions{ - SSMCache: ssmCache, - EC2Cache: ec2Cache, - KubernetesVersionCache: kubernetesVersionCache, - UnavailableOfferingsCache: unavailableOfferingsCache, - LaunchTemplateCache: launchTemplateCache, - InstanceTypeCache: instanceTypeCache, - SubnetCache: subnetCache, - SecurityGroupCache: securityGroupCache, - PricingAPI: fakePricingAPI, - }) - - cloudProvider = cloudprovider.New(awsCtx, awsCtx.InstanceTypesProvider, awsCtx.InstanceProvider, awsCtx.KubeClient, awsCtx.AMIProvider) - cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) - prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) + cluster = state.NewCluster(fakeClock, env.Client, awsEnv.CloudProvider) + prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), awsEnv.CloudProvider, cluster) provisioningController = provisioning.NewController(env.Client, prov, events.NewRecorder(&record.FakeRecorder{})) }) @@ -177,19 +135,10 @@ var _ = BeforeEach(func() { }) cluster.Reset() - fakeEC2API.Reset() - fakeSSMAPI.Reset() - ec2Cache.Flush() - ssmCache.Flush() - kubernetesVersionCache.Flush() - instanceTypeCache.Flush() - unavailableOfferingsCache.Flush() - launchTemplateCache.Flush() - subnetCache.Flush() - securityGroupCache.Flush() + awsEnv.ResetCache() - awsCtx.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") - awsCtx.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" + awsEnv.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") + awsEnv.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" }) var _ = AfterEach(func() { @@ -256,10 +205,10 @@ var _ = Describe("Instance Types", func() { }) It("should order the instance types by price and only consider the cheapest ones", func() { instances := makeFakeInstances() - fakeEC2API.DescribeInstanceTypesOutput.Set(&ec2.DescribeInstanceTypesOutput{ + awsEnv.EC2API.DescribeInstanceTypesOutput.Set(&ec2.DescribeInstanceTypesOutput{ InstanceTypes: makeFakeInstances(), }) - fakeEC2API.DescribeInstanceTypeOfferingsOutput.Set(&ec2.DescribeInstanceTypeOfferingsOutput{ + awsEnv.EC2API.DescribeInstanceTypeOfferingsOutput.Set(&ec2.DescribeInstanceTypeOfferingsOutput{ InstanceTypeOfferings: makeFakeInstanceOfferings(instances), }) ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) @@ -271,7 +220,7 @@ var _ = Describe("Instance Types", func() { }) ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectScheduled(ctx, env.Client, pod) - its, err := cloudProvider.GetInstanceTypes(ctx, provisioner) + its, err := awsEnv.CloudProvider.GetInstanceTypes(ctx, provisioner) Expect(err).To(BeNil()) // Order all the instances by their price // We need some way to deterministically order them if their prices match @@ -285,11 +234,11 @@ var _ = Describe("Instance Types", func() { return iPrice < jPrice }) // Expect that the launch template overrides gives the 60 cheapest instance types - expected := sets.NewString(lo.Map(its[:instance.MaxInstanceTypes], func(i *corecloudprovider.InstanceType, _ int) string { + expected := sets.NewString(lo.Map(its[:instance.MaxInstanceTypes], func(i *corecloudproivder.InstanceType, _ int) string { return i.Name })...) - Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) - call := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + call := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(call.LaunchTemplateConfigs).To(HaveLen(1)) Expect(call.LaunchTemplateConfigs[0].Overrides).To(HaveLen(instance.MaxInstanceTypes)) @@ -299,10 +248,10 @@ var _ = Describe("Instance Types", func() { }) It("should order the instance types by price and only consider the spot types that are cheaper than the cheapest on-demand", func() { instances := makeFakeInstances() - fakeEC2API.DescribeInstanceTypesOutput.Set(&ec2.DescribeInstanceTypesOutput{ + awsEnv.EC2API.DescribeInstanceTypesOutput.Set(&ec2.DescribeInstanceTypesOutput{ InstanceTypes: makeFakeInstances(), }) - fakeEC2API.DescribeInstanceTypeOfferingsOutput.Set(&ec2.DescribeInstanceTypeOfferingsOutput{ + awsEnv.EC2API.DescribeInstanceTypeOfferingsOutput.Set(&ec2.DescribeInstanceTypeOfferingsOutput{ InstanceTypeOfferings: makeFakeInstanceOfferings(instances), }) @@ -317,8 +266,8 @@ var _ = Describe("Instance Types", func() { }, } ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - fakeEC2API.DescribeSpotPriceHistoryOutput.Set(generateSpotPricing(cloudProvider, provisioner)) - Expect(awsCtx.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) + awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(generateSpotPricing(awsEnv.CloudProvider, provisioner)) + Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) pod := coretest.UnschedulablePod(coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ @@ -329,7 +278,7 @@ var _ = Describe("Instance Types", func() { ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectScheduled(ctx, env.Client, pod) - its, err := cloudProvider.GetInstanceTypes(ctx, provisioner) + its, err := awsEnv.CloudProvider.GetInstanceTypes(ctx, provisioner) Expect(err).To(BeNil()) // Order all the instances by their price // We need some way to deterministically order them if their prices match @@ -343,14 +292,14 @@ var _ = Describe("Instance Types", func() { return iPrice < jPrice }) - Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) - call := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + call := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(call.LaunchTemplateConfigs).To(HaveLen(1)) // find the cheapest OD price that works cheapestODPrice := math.MaxFloat64 for _, override := range call.LaunchTemplateConfigs[0].Overrides { - odPrice, ok := awsCtx.PricingProvider.OnDemandPrice(*override.InstanceType) + odPrice, ok := awsEnv.PricingProvider.OnDemandPrice(*override.InstanceType) Expect(ok).To(BeTrue()) if odPrice < cheapestODPrice { cheapestODPrice = odPrice @@ -358,7 +307,7 @@ var _ = Describe("Instance Types", func() { } // and our spot prices should be cheaper than the OD price for _, override := range call.LaunchTemplateConfigs[0].Overrides { - spotPrice, ok := awsCtx.PricingProvider.SpotPrice(*override.InstanceType, *override.AvailabilityZone) + spotPrice, ok := awsEnv.PricingProvider.SpotPrice(*override.InstanceType, *override.AvailabilityZone) Expect(ok).To(BeTrue()) Expect(spotPrice).To(BeNumerically("<", cheapestODPrice)) } @@ -374,8 +323,8 @@ var _ = Describe("Instance Types", func() { ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectScheduled(ctx, env.Client, pod) - Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) - call := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + call := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() for _, ltc := range call.LaunchTemplateConfigs { for _, ovr := range ltc.Overrides { Expect(strings.HasSuffix(aws.StringValue(ovr.InstanceType), "metal")).To(BeFalse()) @@ -393,8 +342,8 @@ var _ = Describe("Instance Types", func() { ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectScheduled(ctx, env.Client, pod) - Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) - call := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + call := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() for _, ltc := range call.LaunchTemplateConfigs { for _, ovr := range ltc.Overrides { Expect(strings.HasPrefix(aws.StringValue(ovr.InstanceType), "g")).To(BeFalse()) @@ -552,7 +501,7 @@ var _ = Describe("Instance Types", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), })) - instanceInfo, err := awsCtx.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) for _, info := range instanceInfo { it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) @@ -560,7 +509,7 @@ var _ = Describe("Instance Types", func() { } }) It("should not set pods to 110 if using ENI-based pod density", func() { - instanceInfo, err := awsCtx.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) for _, info := range instanceInfo { it := instancetype.NewInstanceType(ctx, info, provisioner.Spec.KubeletConfiguration, "", nodeTemplate, nil) @@ -576,7 +525,7 @@ var _ = Describe("Instance Types", func() { })) var ok bool - instanceInfo, err := awsCtx.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) info, ok = lo.Find(instanceInfo, func(i *ec2.InstanceTypeInfo) bool { return aws.StringValue(i.InstanceType) == "m5.xlarge" @@ -838,7 +787,7 @@ var _ = Describe("Instance Types", func() { }) }) It("should set max-pods to user-defined value if specified", func() { - instanceInfo, err := awsCtx.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{MaxPods: ptr.Int32(10)}}) for _, info := range instanceInfo { @@ -851,7 +800,7 @@ var _ = Describe("Instance Types", func() { EnablePodENI: lo.ToPtr(false), })) - instanceInfo, err := awsCtx.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{MaxPods: ptr.Int32(10)}}) for _, info := range instanceInfo { @@ -860,7 +809,7 @@ var _ = Describe("Instance Types", func() { } }) It("should override pods-per-core value", func() { - instanceInfo, err := awsCtx.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}}) for _, info := range instanceInfo { @@ -869,7 +818,7 @@ var _ = Describe("Instance Types", func() { } }) It("should take the minimum of pods-per-core and max-pods", func() { - instanceInfo, err := awsCtx.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(4), MaxPods: ptr.Int32(20)}}) for _, info := range instanceInfo { @@ -878,7 +827,7 @@ var _ = Describe("Instance Types", func() { } }) It("should ignore pods-per-core when using Bottlerocket AMI", func() { - instanceInfo, err := awsCtx.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(1)}}) @@ -893,7 +842,7 @@ var _ = Describe("Instance Types", func() { EnableENILimitedPodDensity: lo.ToPtr(false), })) - instanceInfo, err := awsCtx.InstanceTypesProvider.GetInstanceTypes(ctx) + instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) provisioner = test.Provisioner(coretest.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{PodsPerCore: ptr.Int32(0)}}) for _, info := range instanceInfo { @@ -904,7 +853,7 @@ var _ = Describe("Instance Types", func() { }) Context("Insufficient Capacity Error Cache", func() { It("should launch instances of different type on second reconciliation attempt with Insufficient Capacity Error Cache fallback", func() { - fakeEC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{{CapacityType: v1alpha5.CapacityTypeOnDemand, InstanceType: "inf1.6xlarge", Zone: "test-zone-1a"}}) + awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{{CapacityType: v1alpha5.CapacityTypeOnDemand, InstanceType: "inf1.6xlarge", Zone: "test-zone-1a"}}) ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pods := []*v1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ @@ -937,7 +886,7 @@ var _ = Describe("Instance Types", func() { Expect(nodeNames.Len()).To(Equal(2)) }) It("should launch instances in a different zone on second reconciliation attempt with Insufficient Capacity Error Cache fallback", func() { - fakeEC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{{CapacityType: v1alpha5.CapacityTypeOnDemand, InstanceType: "p3.8xlarge", Zone: "test-zone-1a"}}) + awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{{CapacityType: v1alpha5.CapacityTypeOnDemand, InstanceType: "p3.8xlarge", Zone: "test-zone-1a"}}) pod := coretest.UnschedulablePod(coretest.PodOptions{ NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "p3.8xlarge"}, ResourceRequirements: v1.ResourceRequirements{ @@ -964,7 +913,7 @@ var _ = Describe("Instance Types", func() { HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-1b"))) }) It("should launch smaller instances than optimal if larger instance launch results in Insufficient Capacity Error", func() { - fakeEC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{ + awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{ {CapacityType: v1alpha5.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, }) provisioner.Spec.Requirements = append(provisioner.Spec.Requirements, v1.NodeSelectorRequirement{ @@ -996,7 +945,7 @@ var _ = Describe("Instance Types", func() { } }) It("should launch instances on later reconciliation attempt with Insufficient Capacity Error Cache expiry", func() { - fakeEC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{{CapacityType: v1alpha5.CapacityTypeOnDemand, InstanceType: "inf1.6xlarge", Zone: "test-zone-1a"}}) + awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{{CapacityType: v1alpha5.CapacityTypeOnDemand, InstanceType: "inf1.6xlarge", Zone: "test-zone-1a"}}) ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod(coretest.PodOptions{ NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "inf1.6xlarge"}, @@ -1008,14 +957,14 @@ var _ = Describe("Instance Types", func() { ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) // capacity shortage is over - expire the item from the cache and try again - fakeEC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{}) - awsCtx.UnavailableOfferingsCache.Delete("inf1.6xlarge", "test-zone-1a", v1alpha5.CapacityTypeOnDemand) + awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{}) + awsEnv.UnavailableOfferingsCache.Delete("inf1.6xlarge", "test-zone-1a", v1alpha5.CapacityTypeOnDemand) ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "inf1.6xlarge")) }) It("should launch instances in a different zone on second reconciliation attempt with Insufficient Capacity Error Cache fallback (Habana)", func() { - fakeEC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{{CapacityType: v1alpha5.CapacityTypeOnDemand, InstanceType: "dl1.24xlarge", Zone: "test-zone-1a"}}) + awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{{CapacityType: v1alpha5.CapacityTypeOnDemand, InstanceType: "dl1.24xlarge", Zone: "test-zone-1a"}}) pod := coretest.UnschedulablePod(coretest.PodOptions{ NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "dl1.24xlarge"}, ResourceRequirements: v1.ResourceRequirements{ @@ -1042,9 +991,9 @@ var _ = Describe("Instance Types", func() { HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-1b"))) }) It("should launch on-demand capacity if flexible to both spot and on-demand, but spot is unavailable", func() { - Expect(fakeEC2API.DescribeInstanceTypesPagesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}, func(dito *ec2.DescribeInstanceTypesOutput, b bool) bool { + Expect(awsEnv.EC2API.DescribeInstanceTypesPagesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}, func(dito *ec2.DescribeInstanceTypesOutput, b bool) bool { for _, it := range dito.InstanceTypes { - fakeEC2API.InsufficientCapacityPools.Add(fake.CapacityPool{CapacityType: v1alpha5.CapacityTypeSpot, InstanceType: aws.StringValue(it.InstanceType), Zone: "test-zone-1a"}) + awsEnv.EC2API.InsufficientCapacityPools.Add(fake.CapacityPool{CapacityType: v1alpha5.CapacityTypeSpot, InstanceType: aws.StringValue(it.InstanceType), Zone: "test-zone-1a"}) } return true })).To(Succeed()) @@ -1064,7 +1013,7 @@ var _ = Describe("Instance Types", func() { Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.LabelCapacityType, v1alpha5.CapacityTypeOnDemand)) }) It("should return all instance types, even though with no offerings due to Insufficient Capacity Error", func() { - fakeEC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{ + awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{ {CapacityType: v1alpha5.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, {CapacityType: v1alpha5.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1b"}, {CapacityType: v1alpha5.CapacityTypeSpot, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, @@ -1098,8 +1047,8 @@ var _ = Describe("Instance Types", func() { } } - instanceTypeCache.Flush() - instanceTypes, err := cloudProvider.GetInstanceTypes(ctx, provisioner) + awsEnv.InstanceTypeCache.Flush() + instanceTypes, err := awsEnv.CloudProvider.GetInstanceTypes(ctx, provisioner) Expect(err).To(BeNil()) instanceTypeNames := sets.NewString() for _, it := range instanceTypes { @@ -1131,7 +1080,7 @@ var _ = Describe("Instance Types", func() { }) It("should fail to launch capacity when there is no zonal availability for spot", func() { now := time.Now() - fakeEC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ + awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ SpotPriceHistory: []*ec2.SpotPrice{ { AvailabilityZone: aws.String("test-zone-1a"), @@ -1141,8 +1090,8 @@ var _ = Describe("Instance Types", func() { }, }, }) - Expect(awsCtx.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) - Eventually(func() bool { return awsCtx.PricingProvider.SpotLastUpdated().After(now) }).Should(BeTrue()) + Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) + Eventually(func() bool { return awsEnv.PricingProvider.SpotLastUpdated().After(now) }).Should(BeTrue()) provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeSpot}}, @@ -1158,7 +1107,7 @@ var _ = Describe("Instance Types", func() { }) It("should succeed to launch spot instance when zonal availability exists", func() { now := time.Now() - fakeEC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ + awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ SpotPriceHistory: []*ec2.SpotPrice{ { AvailabilityZone: aws.String("test-zone-1a"), @@ -1168,8 +1117,8 @@ var _ = Describe("Instance Types", func() { }, }, }) - Expect(awsCtx.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) - Eventually(func() bool { return awsCtx.PricingProvider.SpotLastUpdated().After(now) }).Should(BeTrue()) + Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) + Eventually(func() bool { return awsEnv.PricingProvider.SpotLastUpdated().After(now) }).Should(BeTrue()) // not restricting to the zone so we can get any zone provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ @@ -1190,8 +1139,8 @@ var _ = Describe("Instance Types", func() { pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectScheduled(ctx, env.Client, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(*input.LaunchTemplateData.MetadataOptions.HttpEndpoint).To(Equal(ec2.LaunchTemplateInstanceMetadataEndpointStateEnabled)) Expect(*input.LaunchTemplateData.MetadataOptions.HttpProtocolIpv6).To(Equal(ec2.LaunchTemplateInstanceMetadataProtocolIpv6Disabled)) Expect(*input.LaunchTemplateData.MetadataOptions.HttpPutResponseHopLimit).To(Equal(int64(2))) @@ -1208,8 +1157,8 @@ var _ = Describe("Instance Types", func() { pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectScheduled(ctx, env.Client, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(*input.LaunchTemplateData.MetadataOptions.HttpEndpoint).To(Equal(ec2.LaunchTemplateInstanceMetadataEndpointStateDisabled)) Expect(*input.LaunchTemplateData.MetadataOptions.HttpProtocolIpv6).To(Equal(ec2.LaunchTemplateInstanceMetadataProtocolIpv6Enabled)) Expect(*input.LaunchTemplateData.MetadataOptions.HttpPutResponseHopLimit).To(Equal(int64(1))) @@ -1223,7 +1172,7 @@ var _ = Describe("Instance Types", func() { func generateSpotPricing(cp *cloudprovider.CloudProvider, prov *v1alpha5.Provisioner) *ec2.DescribeSpotPriceHistoryOutput { rsp := &ec2.DescribeSpotPriceHistoryOutput{} instanceTypes, err := cp.GetInstanceTypes(ctx, prov) - instanceTypeCache.Flush() + awsEnv.InstanceTypeCache.Flush() Expect(err).To(Succeed()) t := fakeClock.Now() diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 1e08b8f6d550..d0df5f9e48a8 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -32,7 +32,6 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/patrickmn/go-cache" "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -47,10 +46,6 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" - awscache "github.com/aws/karpenter/pkg/cache" - "github.com/aws/karpenter/pkg/cloudprovider" - awscontext "github.com/aws/karpenter/pkg/context" - "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/providers/amifamily/bootstrap" "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/test" @@ -68,29 +63,15 @@ import ( ) var ctx context.Context -var awsCtx awscontext.Context var stop context.CancelFunc var opts options.Options var env *coretest.Environment -var fakeEC2API *fake.EC2API -var fakeSSMAPI *fake.SSMAPI +var awsEnv *test.Environment var fakeClock *clock.FakeClock -var fakePricingAPI *fake.PricingAPI var prov *provisioning.Provisioner var provisioner *v1alpha5.Provisioner var nodeTemplate *v1alpha1.AWSNodeTemplate var cluster *state.Cluster -var cloudProvider *cloudprovider.CloudProvider - -// Cache -var ec2Cache *cache.Cache -var ssmCache *cache.Cache -var kubernetesVersionCache *cache.Cache -var instanceTypeCache *cache.Cache -var unavailableOfferingsCache *awscache.UnavailableOfferings -var launchTemplateCache *cache.Cache -var subnetCache *cache.Cache -var securityGroupCache *cache.Cache func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -103,36 +84,11 @@ var _ = BeforeSuite(func() { ctx = coresettings.ToContext(ctx, coretest.Settings()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) + awsEnv = test.NewEnvironment(ctx, env) - fakeEC2API = &fake.EC2API{} - fakeSSMAPI = &fake.SSMAPI{} fakeClock = &clock.FakeClock{} - fakePricingAPI = &fake.PricingAPI{} - - ssmCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - ec2Cache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - kubernetesVersionCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - instanceTypeCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - unavailableOfferingsCache = awscache.NewUnavailableOfferings() - launchTemplateCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - subnetCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - securityGroupCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - - awsCtx = test.Context(ctx, fakeEC2API, fakeSSMAPI, env, fakeClock, test.ContextOptions{ - SSMCache: ssmCache, - EC2Cache: ec2Cache, - KubernetesVersionCache: kubernetesVersionCache, - UnavailableOfferingsCache: unavailableOfferingsCache, - LaunchTemplateCache: launchTemplateCache, - InstanceTypeCache: instanceTypeCache, - SubnetCache: subnetCache, - SecurityGroupCache: securityGroupCache, - PricingAPI: fakePricingAPI, - }) - - cloudProvider = cloudprovider.New(awsCtx, awsCtx.InstanceTypesProvider, awsCtx.InstanceProvider, awsCtx.KubeClient, awsCtx.AMIProvider) - cluster = state.NewCluster(fakeClock, awsCtx.KubeClient, cloudProvider) - prov = provisioning.NewProvisioner(ctx, awsCtx.KubeClient, awsCtx.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) + cluster = state.NewCluster(fakeClock, env.Client, awsEnv.CloudProvider) + prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), awsEnv.CloudProvider, cluster) }) var _ = AfterSuite(func() { @@ -173,40 +129,31 @@ var _ = BeforeEach(func() { }, }) cluster.Reset() - fakeEC2API.Reset() - fakeSSMAPI.Reset() - ec2Cache.Flush() - ssmCache.Flush() - kubernetesVersionCache.Flush() - instanceTypeCache.Flush() - unavailableOfferingsCache.Flush() - launchTemplateCache.Flush() - subnetCache.Flush() - securityGroupCache.Flush() - - awsCtx.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") - awsCtx.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" + awsEnv.ResetCache() + + awsEnv.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") + awsEnv.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" }) var _ = AfterEach(func() { - ExpectCleanedUp(ctx, awsCtx.KubeClient) + ExpectCleanedUp(ctx, env.Client) }) var _ = Describe("LaunchTemplates", func() { It("should default to a generated launch template", func() { - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() fmt.Println(cluster.Nodes()) - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - firstLt := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + firstLt := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() - Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) - createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() launchTemplate := createFleetInput.LaunchTemplateConfigs[0].LaunchTemplateSpecification Expect(createFleetInput.LaunchTemplateConfigs).To(HaveLen(1)) @@ -230,14 +177,14 @@ var _ = Describe("LaunchTemplates", func() { Name: nodeTemplate.Name, }, }) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) - createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() // Expect these values to be correctly ordered by price overrides := createFleetInput.LaunchTemplateConfigs[0].Overrides @@ -246,7 +193,7 @@ var _ = Describe("LaunchTemplates", func() { }) lastPrice := -math.MaxFloat64 for _, override := range overrides { - offeringPrice, ok := awsCtx.PricingProvider.SpotPrice(*override.InstanceType, *override.AvailabilityZone) + offeringPrice, ok := awsEnv.PricingProvider.SpotPrice(*override.InstanceType, *override.AvailabilityZone) Expect(ok).To(BeTrue()) Expect(offeringPrice).To(BeNumerically(">=", lastPrice)) lastPrice = offeringPrice @@ -256,12 +203,12 @@ var _ = Describe("LaunchTemplates", func() { It("should allow a launch template to be specified", func() { nodeTemplate.Spec.LaunchTemplateName = aws.String("test-launch-template") nodeTemplate.Spec.SecurityGroupSelector = nil - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) - input := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(input.LaunchTemplateConfigs).To(HaveLen(1)) launchTemplate := input.LaunchTemplateConfigs[0].LaunchTemplateSpecification Expect(*launchTemplate.LaunchTemplateName).To(Equal("test-launch-template")) @@ -298,62 +245,62 @@ var _ = Describe("LaunchTemplates", func() { Limits: v1.ResourceList{v1alpha1.ResourceNVIDIAGPU: resource.MustParse("1")}, } - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod1 := coretest.UnschedulablePod(coretest.PodOptions{ Tolerations: []v1.Toleration{t1, t2, t3}, ResourceRequirements: rr, }) - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod1) - ExpectScheduled(ctx, awsCtx.KubeClient, pod1) - Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) - name1 := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop().LaunchTemplateConfigs[0].LaunchTemplateSpecification.LaunchTemplateName + ExpectProvisioned(ctx, env.Client, cluster, prov, pod1) + ExpectScheduled(ctx, env.Client, pod1) + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + name1 := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop().LaunchTemplateConfigs[0].LaunchTemplateSpecification.LaunchTemplateName pod2 := coretest.UnschedulablePod(coretest.PodOptions{ Tolerations: []v1.Toleration{t2, t3, t1}, ResourceRequirements: rr, }) - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod2) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod2) - ExpectScheduled(ctx, awsCtx.KubeClient, pod2) - Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) - name2 := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop().LaunchTemplateConfigs[0].LaunchTemplateSpecification.LaunchTemplateName + ExpectScheduled(ctx, env.Client, pod2) + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + name2 := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop().LaunchTemplateConfigs[0].LaunchTemplateSpecification.LaunchTemplateName Expect(name1).To(Equal(name2)) }) It("should recover from an out-of-sync launch template cache", func() { - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - firstLt := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + firstLt := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() ltName := aws.StringValue(firstLt.LaunchTemplateName) - lt, ok := launchTemplateCache.Get(ltName) + lt, ok := awsEnv.LaunchTemplateCache.Get(ltName) Expect(ok).To(Equal(true)) // Remove expiration from cached LT - launchTemplateCache.Set(ltName, lt, -1) + awsEnv.LaunchTemplateCache.Set(ltName, lt, -1) - fakeEC2API.CreateFleetBehavior.Error.Set(awserr.New("InvalidLaunchTemplateName.NotFoundException", "", errors.New(""))) + awsEnv.EC2API.CreateFleetBehavior.Error.Set(awserr.New("InvalidLaunchTemplateName.NotFoundException", "", errors.New(""))) pod = coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) // should call fleet twice. Once will fail on invalid LT and the next will succeed - fleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + fleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(aws.StringValue(fleetInput.LaunchTemplateConfigs[0].LaunchTemplateSpecification.LaunchTemplateName)).To(Equal(ltName)) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) + ExpectScheduled(ctx, env.Client, pod) }) }) Context("Labels", func() { It("should apply labels to the node", func() { - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - node := ExpectScheduled(ctx, awsCtx.KubeClient, pod) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKey(v1.LabelOSStable)) Expect(node.Labels).To(HaveKey(v1.LabelArchStable)) Expect(node.Labels).To(HaveKey(v1.LabelInstanceTypeStable)) }) It("should apply provider labels to the node", func() { - fakeEC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ { ImageId: aws.String("ami-123"), Architecture: aws.String("x86_64"), @@ -366,13 +313,13 @@ var _ = Describe("LaunchTemplates", func() { }, }}) nodeTemplate.Spec.AMISelector = map[string]string{"karpenter.sh/discovery": "my-cluster"} - ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) + ExpectApplied(ctx, env.Client, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) - Expect(awsCtx.KubeClient.Get(ctx, client.ObjectKeyFromObject(newProvisioner), newProvisioner)).To(Succeed()) + ExpectApplied(ctx, env.Client, newProvisioner) + Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(newProvisioner), newProvisioner)).To(Succeed()) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - node := ExpectScheduled(ctx, awsCtx.KubeClient, pod) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1alpha1.LabelInstanceAMIID, "ami-123")) }) }) @@ -380,12 +327,12 @@ var _ = Describe("LaunchTemplates", func() { It("should tag with provisioner name", func() { provisionerName := "the-provisioner" provisioner.Name = provisionerName - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) - createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(createFleetInput.TagSpecifications).To(HaveLen(3)) tags := map[string]string{ @@ -407,12 +354,12 @@ var _ = Describe("LaunchTemplates", func() { "tag1": "tag1value", "tag2": "tag2value", } - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) - createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(createFleetInput.TagSpecifications).To(HaveLen(3)) // tags should be included in instance, volume, and fleet tag specification @@ -431,12 +378,12 @@ var _ = Describe("LaunchTemplates", func() { v1alpha5.ProvisionerNameLabelKey: "myprovisioner", "Name": "myname", } - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) - createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(createFleetInput.TagSpecifications).To(HaveLen(3)) // tags should be included in instance, volume, and fleet tag specification @@ -462,12 +409,12 @@ var _ = Describe("LaunchTemplates", func() { Tags: settingsTags, })) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) - createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(createFleetInput.TagSpecifications).To(HaveLen(3)) // tags should be included in instance, volume, and fleet tag specification @@ -492,12 +439,12 @@ var _ = Describe("LaunchTemplates", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ Tags: settingsTags, })) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) - createFleetInput := fakeEC2API.CreateFleetBehavior.CalledWithInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(createFleetInput.TagSpecifications).To(HaveLen(3)) // tags should be included in instance, volume, and fleet tag specification @@ -517,12 +464,12 @@ var _ = Describe("LaunchTemplates", func() { Context("Block Device Mappings", func() { It("should default AL2 block device mappings", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(len(input.LaunchTemplateData.BlockDeviceMappings)).To(Equal(1)) Expect(*input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeSize).To(Equal(int64(20))) Expect(*input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeType).To(Equal("gp3")) @@ -554,12 +501,12 @@ var _ = Describe("LaunchTemplates", func() { }, }, } - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(input.LaunchTemplateData.BlockDeviceMappings[0].Ebs).To(Equal(&ec2.LaunchTemplateEbsBlockDeviceRequest{ VolumeSize: aws.Int64(187), VolumeType: aws.String("io2"), @@ -603,12 +550,12 @@ var _ = Describe("LaunchTemplates", func() { }, }, } - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() // Both of these values are rounded up when converting to Gibibytes Expect(aws.Int64Value(input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeSize)).To(BeNumerically("==", 4)) @@ -616,12 +563,12 @@ var _ = Describe("LaunchTemplates", func() { }) It("should default bottlerocket second volume with root volume size", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(len(input.LaunchTemplateData.BlockDeviceMappings)).To(Equal(2)) // Bottlerocket control volume Expect(*input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeSize).To(Equal(int64(4))) @@ -634,12 +581,12 @@ var _ = Describe("LaunchTemplates", func() { }) It("should not default block device mappings for custom AMIFamilies", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyCustom - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(len(input.LaunchTemplateData.BlockDeviceMappings)).To(Equal(0)) }) It("should use custom block device mapping for custom AMIFamilies", func() { @@ -657,12 +604,12 @@ var _ = Describe("LaunchTemplates", func() { }, }, } - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(len(input.LaunchTemplateData.BlockDeviceMappings)).To(Equal(1)) Expect(*input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeSize).To(Equal(int64(40))) Expect(*input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeType).To(Equal("io2")) @@ -674,7 +621,7 @@ var _ = Describe("LaunchTemplates", func() { }) Context("Ephemeral Storage", func() { It("should pack pods when a daemonset has an ephemeral-storage request", func() { - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate, coretest.DaemonSet( + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate, coretest.DaemonSet( coretest.DaemonSetOptions{PodOptions: coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), @@ -683,39 +630,39 @@ var _ = Describe("LaunchTemplates", func() { }}, )) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) }) It("should pack pods with any ephemeral-storage request", func() { - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ v1.ResourceEphemeralStorage: resource.MustParse("1G"), }}}) - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) }) It("should pack pods with large ephemeral-storage request", func() { - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ v1.ResourceEphemeralStorage: resource.MustParse("10Gi"), }}}) - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) }) It("should not pack pods if the sum of pod ephemeral-storage and overhead exceeds node capacity", func() { - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ v1.ResourceEphemeralStorage: resource.MustParse("19Gi"), }}}) - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectNotScheduled(ctx, awsCtx.KubeClient, pod) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) }) It("should launch multiple nodes if sum of pod ephemeral-storage requests exceeds a single nodes capacity", func() { var nodes []*v1.Node - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pods := []*v1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ @@ -730,14 +677,14 @@ var _ = Describe("LaunchTemplates", func() { }, }), } - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pods...) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) for _, pod := range pods { - nodes = append(nodes, ExpectScheduled(ctx, awsCtx.KubeClient, pod)) + nodes = append(nodes, ExpectScheduled(ctx, env.Client, pod)) } Expect(nodes).To(HaveLen(2)) }) It("should only pack pods with ephemeral-storage requests that will fit on an available node", func() { - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pods := []*v1.Pod{ coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ @@ -752,20 +699,20 @@ var _ = Describe("LaunchTemplates", func() { }, }), } - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pods...) - ExpectScheduled(ctx, awsCtx.KubeClient, pods[0]) - ExpectNotScheduled(ctx, awsCtx.KubeClient, pods[1]) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + ExpectScheduled(ctx, env.Client, pods[0]) + ExpectNotScheduled(ctx, env.Client, pods[1]) }) It("should not pack pod if no available instance types have enough storage", func() { - ExpectApplied(ctx, awsCtx.KubeClient, provisioner) + ExpectApplied(ctx, env.Client, provisioner) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ v1.ResourceEphemeralStorage: resource.MustParse("150Gi"), }, }, }) - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectNotScheduled(ctx, awsCtx.KubeClient, pod) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) }) It("should pack pods using the blockdevicemappings from the provider spec when defined", func() { nodeTemplate.Spec.BlockDeviceMappings = []*v1alpha1.BlockDeviceMapping{{ @@ -774,17 +721,17 @@ var _ = Describe("LaunchTemplates", func() { VolumeSize: resource.NewScaledQuantity(50, resource.Giga), }, }} - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ v1.ResourceEphemeralStorage: resource.MustParse("25Gi"), }, }, }) - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) // capacity isn't recorded on the node any longer, but we know the pod should schedule - ExpectScheduled(ctx, awsCtx.KubeClient, pod) + ExpectScheduled(ctx, env.Client, pod) }) It("should pack pods using blockdevicemappings for Custom AMIFamily", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyCustom @@ -802,7 +749,7 @@ var _ = Describe("LaunchTemplates", func() { }, }, } - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod(coretest.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ // this pod can only be satisfied if `/dev/xvdb` will house all the pods. @@ -810,10 +757,10 @@ var _ = Describe("LaunchTemplates", func() { }, }, }) - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) // capacity isn't recorded on the node any longer, but we know the pod should schedule - ExpectScheduled(ctx, awsCtx.KubeClient, pod) + ExpectScheduled(ctx, env.Client, pod) }) }) Context("AL2", func() { @@ -821,7 +768,7 @@ var _ = Describe("LaunchTemplates", func() { BeforeEach(func() { var ok bool var instanceInfo []*ec2.InstanceTypeInfo - err := fakeEC2API.DescribeInstanceTypesPagesWithContext(ctx, &ec2.DescribeInstanceTypesInput{ + err := awsEnv.EC2API.DescribeInstanceTypesPagesWithContext(ctx, &ec2.DescribeInstanceTypesInput{ Filters: []*ec2.Filter{ { Name: aws.String("supported-virtualization-type"), @@ -871,7 +818,7 @@ var _ = Describe("LaunchTemplates", func() { BeforeEach(func() { var ok bool var instanceInfo []*ec2.InstanceTypeInfo - err := fakeEC2API.DescribeInstanceTypesPagesWithContext(ctx, &ec2.DescribeInstanceTypesInput{ + err := awsEnv.EC2API.DescribeInstanceTypesPagesWithContext(ctx, &ec2.DescribeInstanceTypesInput{ Filters: []*ec2.Filter{ { Name: aws.String("supported-virtualization-type"), @@ -918,12 +865,12 @@ var _ = Describe("LaunchTemplates", func() { }) Context("User Data", func() { It("should not specify --use-max-pods=false when using ENI-based pod density", func() { - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) Expect(string(userData)).NotTo(ContainSubstring("--use-max-pods false")) @@ -933,12 +880,12 @@ var _ = Describe("LaunchTemplates", func() { EnableENILimitedPodDensity: lo.ToPtr(false), })) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) Expect(string(userData)).To(ContainSubstring("--use-max-pods false")) @@ -946,12 +893,12 @@ var _ = Describe("LaunchTemplates", func() { }) It("should specify --use-max-pods=false and --max-pods user value when user specifies maxPods in Provisioner", func() { provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{MaxPods: aws.Int32(10)} - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) Expect(string(userData)).To(ContainSubstring("--use-max-pods false")) @@ -965,12 +912,12 @@ var _ = Describe("LaunchTemplates", func() { v1.ResourceEphemeralStorage: resource.MustParse("2Gi"), }, } - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) @@ -991,12 +938,12 @@ var _ = Describe("LaunchTemplates", func() { v1.ResourceEphemeralStorage: resource.MustParse("2Gi"), }, } - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) @@ -1017,12 +964,12 @@ var _ = Describe("LaunchTemplates", func() { "nodefs.inodesFree": "5%", }, } - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) @@ -1043,12 +990,12 @@ var _ = Describe("LaunchTemplates", func() { "nodefs.inodesFree": "5%", }, } - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) @@ -1069,12 +1016,12 @@ var _ = Describe("LaunchTemplates", func() { "nodefs.inodesFree": {Duration: time.Minute * 5}, }, } - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) @@ -1091,12 +1038,12 @@ var _ = Describe("LaunchTemplates", func() { provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{ EvictionMaxPodGracePeriod: aws.Int32(300), } - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) @@ -1106,12 +1053,12 @@ var _ = Describe("LaunchTemplates", func() { provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{ PodsPerCore: aws.Int32(2), } - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) Expect(string(userData)).To(ContainSubstring(fmt.Sprintf("--pods-per-core=%d", 2))) @@ -1121,43 +1068,43 @@ var _ = Describe("LaunchTemplates", func() { PodsPerCore: aws.Int32(2), MaxPods: aws.Int32(100), } - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) Expect(string(userData)).To(ContainSubstring(fmt.Sprintf("--pods-per-core=%d", 2))) Expect(string(userData)).To(ContainSubstring(fmt.Sprintf("--max-pods=%d", 100))) }) It("should specify --container-runtime containerd by default", func() { - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) Expect(string(userData)).To(ContainSubstring("--container-runtime containerd")) }) It("should specify dockerd if specified in the provisionerSpec", func() { provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{ContainerRuntime: aws.String("dockerd")} - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) Expect(string(userData)).To(ContainSubstring("--container-runtime dockerd")) }) It("should specify --container-runtime containerd when using Neuron GPUs", func() { provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{Key: v1alpha1.LabelInstanceCategory, Operator: v1.NodeSelectorOpExists}} - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod(coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ @@ -1169,17 +1116,17 @@ var _ = Describe("LaunchTemplates", func() { }, }, }) - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) Expect(string(userData)).To(ContainSubstring("--container-runtime containerd")) }) It("should specify --container-runtime containerd when using Nvidia GPUs", func() { provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{Key: v1alpha1.LabelInstanceCategory, Operator: v1.NodeSelectorOpExists}} - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod(coretest.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ @@ -1191,22 +1138,22 @@ var _ = Describe("LaunchTemplates", func() { }, }, }) - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) Expect(string(userData)).To(ContainSubstring("--container-runtime containerd")) }) It("should specify --dns-cluster-ip and --ip-family when running in an ipv6 cluster", func() { - awsCtx.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("fd4b:121b:812b::a") - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + awsEnv.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("fd4b:121b:812b::a") + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) Expect(string(userData)).To(ContainSubstring("--dns-cluster-ip 'fd4b:121b:812b::a'")) @@ -1217,12 +1164,12 @@ var _ = Describe("LaunchTemplates", func() { provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{ ImageGCHighThresholdPercent: aws.Int32(50), } - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) Expect(string(userData)).To(ContainSubstring("--image-gc-high-threshold=50")) @@ -1231,12 +1178,12 @@ var _ = Describe("LaunchTemplates", func() { provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{ ImageGCLowThresholdPercent: aws.Int32(50), } - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) Expect(string(userData)).To(ContainSubstring("--image-gc-low-threshold=50")) @@ -1253,15 +1200,15 @@ var _ = Describe("LaunchTemplates", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket provisioner.Spec.Taints = []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectNoExecute}} provisioner.Spec.StartupTaints = []v1.Taint{{Key: "baz", Value: "bin", Effect: v1.TaintEffectNoExecute}} - ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate, provisioner) - Expect(awsCtx.KubeClient.Get(ctx, client.ObjectKeyFromObject(provisioner), provisioner)).To(Succeed()) + ExpectApplied(ctx, env.Client, nodeTemplate, provisioner) + Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(provisioner), provisioner)).To(Succeed()) pod := coretest.UnschedulablePod(coretest.PodOptions{ Tolerations: []v1.Toleration{{Operator: v1.TolerationOpExists}}, }) - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) content, err = os.ReadFile("testdata/br_userdata_merged.golden") @@ -1278,15 +1225,15 @@ var _ = Describe("LaunchTemplates", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket provisioner.Spec.Taints = []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectNoExecute}} provisioner.Spec.StartupTaints = []v1.Taint{{Key: "baz", Value: "bin", Effect: v1.TaintEffectNoExecute}} - ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate, provisioner) - Expect(awsCtx.KubeClient.Get(ctx, client.ObjectKeyFromObject(provisioner), provisioner)).To(Succeed()) + ExpectApplied(ctx, env.Client, nodeTemplate, provisioner) + Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(provisioner), provisioner)).To(Succeed()) pod := coretest.UnschedulablePod(coretest.PodOptions{ Tolerations: []v1.Toleration{{Operator: v1.TolerationOpExists}}, }) - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) content, err := os.ReadFile("testdata/br_userdata_unmerged.golden") @@ -1301,26 +1248,26 @@ var _ = Describe("LaunchTemplates", func() { })) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: "doesnotexist"}}) - ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) + ExpectApplied(ctx, env.Client, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) // This will not be scheduled since we were pointed to a non-existent awsnodetemplate resource. - ExpectNotScheduled(ctx, awsCtx.KubeClient, pod) + ExpectNotScheduled(ctx, env.Client, pod) }) It("should not bootstrap on invalid toml user data", func() { nodeTemplate.Spec.UserData = aws.String("#/bin/bash\n ./not-toml.sh") nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) + ExpectApplied(ctx, env.Client, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) + ExpectApplied(ctx, env.Client, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) // This will not be scheduled since userData cannot be generated for the prospective node. - ExpectNotScheduled(ctx, awsCtx.KubeClient, pod) + ExpectNotScheduled(ctx, env.Client, pod) }) It("should override system reserved values in user data", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) + ExpectApplied(ctx, env.Client, nodeTemplate) provisioner = test.Provisioner(coretest.ProvisionerOptions{ ProviderRef: &v1alpha5.ProviderRef{ Name: nodeTemplate.Name, @@ -1333,12 +1280,12 @@ var _ = Describe("LaunchTemplates", func() { }, }, }) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner) + ExpectApplied(ctx, env.Client, provisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) config := &bootstrap.BottlerocketConfig{} @@ -1350,7 +1297,7 @@ var _ = Describe("LaunchTemplates", func() { }) It("should override kube reserved values in user data", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) + ExpectApplied(ctx, env.Client, nodeTemplate) provisioner = test.Provisioner(coretest.ProvisionerOptions{ ProviderRef: &v1alpha5.ProviderRef{ Name: nodeTemplate.Name, @@ -1363,12 +1310,12 @@ var _ = Describe("LaunchTemplates", func() { }, }, }) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner) + ExpectApplied(ctx, env.Client, provisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) config := &bootstrap.BottlerocketConfig{} @@ -1380,7 +1327,7 @@ var _ = Describe("LaunchTemplates", func() { }) It("should override kube reserved values in user data", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket - ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) + ExpectApplied(ctx, env.Client, nodeTemplate) provisioner = test.Provisioner(coretest.ProvisionerOptions{ ProviderRef: &v1alpha5.ProviderRef{ Name: nodeTemplate.Name, @@ -1393,12 +1340,12 @@ var _ = Describe("LaunchTemplates", func() { }, }, }) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner) + ExpectApplied(ctx, env.Client, provisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) config := &bootstrap.BottlerocketConfig{} @@ -1418,12 +1365,12 @@ var _ = Describe("LaunchTemplates", func() { MaxPods: aws.Int32(10), }, }) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) config := &bootstrap.BottlerocketConfig{} @@ -1441,14 +1388,14 @@ var _ = Describe("LaunchTemplates", func() { content, err := os.ReadFile("testdata/al2_userdata_input.golden") Expect(err).To(BeNil()) nodeTemplate.Spec.UserData = aws.String(string(content)) - ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) + ExpectApplied(ctx, env.Client, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) + ExpectApplied(ctx, env.Client, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) content, err = os.ReadFile("testdata/al2_userdata_merged.golden") @@ -1464,14 +1411,14 @@ var _ = Describe("LaunchTemplates", func() { content, err := os.ReadFile("testdata/al2_no_mime_userdata_input.golden") Expect(err).To(BeNil()) nodeTemplate.Spec.UserData = aws.String(string(content)) - ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) + ExpectApplied(ctx, env.Client, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) + ExpectApplied(ctx, env.Client, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) content, err = os.ReadFile("testdata/al2_userdata_merged.golden") @@ -1484,14 +1431,14 @@ var _ = Describe("LaunchTemplates", func() { EnableENILimitedPodDensity: lo.ToPtr(false), })) nodeTemplate.Spec.UserData = nil - ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) + ExpectApplied(ctx, env.Client, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) + ExpectApplied(ctx, env.Client, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) content, err := os.ReadFile("testdata/al2_userdata_unmerged.golden") @@ -1503,47 +1450,47 @@ var _ = Describe("LaunchTemplates", func() { Context("Custom AMI Selector", func() { It("should use ami selector specified in AWSNodeTemplate", func() { nodeTemplate.Spec.AMISelector = map[string]string{"karpenter.sh/discovery": "my-cluster"} - fakeEC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ { ImageId: aws.String("ami-123"), Architecture: aws.String("x86_64"), CreationDate: aws.String("2022-08-15T12:00:00Z")}, }}) - ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) + ExpectApplied(ctx, env.Client, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) + ExpectApplied(ctx, env.Client, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect("ami-123").To(Equal(*input.LaunchTemplateData.ImageId)) }) It("should copy over userData untouched when AMIFamily is Custom", func() { nodeTemplate.Spec.UserData = aws.String("special user data") nodeTemplate.Spec.AMISelector = map[string]string{"karpenter.sh/discovery": "my-cluster"} nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyCustom - fakeEC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ { ImageId: aws.String("ami-123"), Architecture: aws.String("x86_64"), CreationDate: aws.String("2022-08-15T12:00:00Z")}, }}) - ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) + ExpectApplied(ctx, env.Client, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) + ExpectApplied(ctx, env.Client, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) Expect("special user data").To(Equal(string(userData))) }) It("should correctly use ami selector with specific IDs in AWSNodeTemplate", func() { nodeTemplate.Spec.AMISelector = map[string]string{"aws-ids": "ami-123,ami-456"} - fakeEC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ { ImageId: aws.String("ami-123"), Architecture: aws.String("x86_64"), @@ -1557,14 +1504,14 @@ var _ = Describe("LaunchTemplates", func() { CreationDate: aws.String("2022-08-15T12:00:00Z"), }, }}) - ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) + ExpectApplied(ctx, env.Client, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) + ExpectApplied(ctx, env.Client, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(2)) - actualFilter := fakeEC2API.CalledWithDescribeImagesInput.Pop().Filters + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(2)) + actualFilter := awsEnv.EC2API.CalledWithDescribeImagesInput.Pop().Filters expectedFilter := []*ec2.Filter{ { Name: aws.String("image-id"), @@ -1574,7 +1521,7 @@ var _ = Describe("LaunchTemplates", func() { Expect(actualFilter).To(Equal(expectedFilter)) }) It("should create multiple launch templates when multiple amis are discovered with non-equivalent requirements", func() { - fakeEC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ { ImageId: aws.String("ami-123"), Architecture: aws.String("x86_64"), @@ -1589,22 +1536,22 @@ var _ = Describe("LaunchTemplates", func() { }, }}) nodeTemplate.Spec.AMISelector = map[string]string{"karpenter.sh/discovery": "my-cluster"} - ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) + ExpectApplied(ctx, env.Client, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) + ExpectApplied(ctx, env.Client, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(2)) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(2)) expectedImageIds := sets.NewString("ami-123", "ami-456") actualImageIds := sets.NewString( - *fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().LaunchTemplateData.ImageId, - *fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().LaunchTemplateData.ImageId, + *awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop().LaunchTemplateData.ImageId, + *awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop().LaunchTemplateData.ImageId, ) Expect(expectedImageIds.Equal(actualImageIds)).To(BeTrue()) }) It("should create a launch template with the newest compatible AMI when multiple amis are discovered", func() { - fakeEC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ { ImageId: aws.String("ami-123"), Architecture: aws.String("x86_64"), @@ -1623,7 +1570,7 @@ var _ = Describe("LaunchTemplates", func() { }, }}) nodeTemplate.Spec.AMISelector = map[string]string{"karpenter.sh/discovery": "my-cluster"} - ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) + ExpectApplied(ctx, env.Client, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}, Requirements: []v1.NodeSelectorRequirement{ @@ -1634,59 +1581,59 @@ var _ = Describe("LaunchTemplates", func() { }, }, }) - ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) + ExpectApplied(ctx, env.Client, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect("ami-456").To(Equal(*input.LaunchTemplateData.ImageId)) }) It("should fail if no amis match selector.", func() { - fakeEC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{}}) + awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{}}) nodeTemplate.Spec.AMISelector = map[string]string{"karpenter.sh/discovery": "my-cluster"} - ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) + ExpectApplied(ctx, env.Client, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) + ExpectApplied(ctx, env.Client, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectNotScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(0)) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(0)) }) It("should fail if no instanceType matches ami requirements.", func() { - fakeEC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ + awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{Images: []*ec2.Image{ {ImageId: aws.String("ami-123"), Architecture: aws.String("newnew"), CreationDate: aws.String("2022-01-01T12:00:00Z")}, }}) nodeTemplate.Spec.AMISelector = map[string]string{"karpenter.sh/discovery": "my-cluster"} - ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) + ExpectApplied(ctx, env.Client, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) + ExpectApplied(ctx, env.Client, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectNotScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(0)) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(0)) }) It("should choose amis from SSM if no selector specified in AWSNodeTemplate", func() { - ExpectApplied(ctx, awsCtx.KubeClient, nodeTemplate) + ExpectApplied(ctx, env.Client, nodeTemplate) newProvisioner := test.Provisioner(coretest.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: nodeTemplate.Name}}) - ExpectApplied(ctx, awsCtx.KubeClient, newProvisioner) + ExpectApplied(ctx, env.Client, newProvisioner) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(*input.LaunchTemplateData.ImageId).To(ContainSubstring("test-ami")) }) }) Context("Kubelet Args", func() { It("should specify the --dns-cluster-ip flag when clusterDNSIP is set", func() { provisioner.Spec.KubeletConfiguration = &v1alpha5.KubeletConfiguration{ClusterDNS: []string{"10.0.10.100"}} - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) Expect(err).To(BeNil()) Expect(string(userData)).To(ContainSubstring("--dns-cluster-ip '10.0.10.100'")) @@ -1694,22 +1641,22 @@ var _ = Describe("LaunchTemplates", func() { }) Context("Instance Profile", func() { It("should use the default instance profile if none specified on the Provisioner", func() { - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(*input.LaunchTemplateData.IamInstanceProfile.Name).To(Equal("test-instance-profile")) }) It("should use the instance profile on the Provisioner when specified", func() { nodeTemplate.Spec.InstanceProfile = aws.String("overridden-profile") - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(*input.LaunchTemplateData.IamInstanceProfile.Name).To(Equal("overridden-profile")) }) }) @@ -1717,23 +1664,23 @@ var _ = Describe("LaunchTemplates", func() { Context("Detailed Monitoring", func() { It("should default detailed monitoring to off", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(aws.BoolValue(input.LaunchTemplateData.Monitoring.Enabled)).To(BeFalse()) }) It("should pass detailed monitoring setting to the launch template at creation", func() { nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 nodeTemplate.Spec.DetailedMonitoring = aws.Bool(true) - ExpectApplied(ctx, awsCtx.KubeClient, provisioner, nodeTemplate) + ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, awsCtx.KubeClient, cluster, prov, pod) - ExpectScheduled(ctx, awsCtx.KubeClient, pod) - Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) - input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(1)) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(aws.BoolValue(input.LaunchTemplateData.Monitoring.Enabled)).To(BeTrue()) }) }) diff --git a/pkg/providers/pricing/suite_test.go b/pkg/providers/pricing/suite_test.go index 6ce6a219e5a9..8be6355b062c 100644 --- a/pkg/providers/pricing/suite_test.go +++ b/pkg/providers/pricing/suite_test.go @@ -46,9 +46,7 @@ var ctx context.Context var stop context.CancelFunc var opts options.Options var env *coretest.Environment -var fakePricingAPI *fake.PricingAPI -var fakeEC2API *fake.EC2API -var pricingProvider *pricing.Provider +var awsEnv *test.Environment func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -61,10 +59,7 @@ var _ = BeforeSuite(func() { ctx = coresettings.ToContext(ctx, coretest.Settings()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) - - fakeEC2API = &fake.EC2API{} - fakePricingAPI = &fake.PricingAPI{} - pricingProvider = pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})) + awsEnv = test.NewEnvironment(ctx, env) }) var _ = AfterSuite(func() { @@ -77,8 +72,7 @@ var _ = BeforeEach(func() { ctx = coresettings.ToContext(ctx, coretest.Settings()) ctx = settings.ToContext(ctx, test.Settings()) - fakeEC2API.Reset() - fakePricingAPI.Reset() + awsEnv.ResetCache() }) var _ = AfterEach(func() { @@ -86,20 +80,16 @@ var _ = AfterEach(func() { }) var _ = Describe("Pricing", func() { - BeforeEach(func() { - fakeEC2API.Reset() - fakePricingAPI.Reset() - }) It("should return static on-demand data if pricing API fails", func() { - fakePricingAPI.NextError.Set(fmt.Errorf("failed")) - p := pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})) + awsEnv.PricingAPI.NextError.Set(fmt.Errorf("failed")) + p := pricing.NewProvider(ctx, awsEnv.PricingAPI, awsEnv.EC2API, "", make(chan struct{})) price, ok := p.OnDemandPrice("c5.large") Expect(ok).To(BeTrue()) Expect(price).To(BeNumerically(">", 0)) }) It("should return static spot data if EC2 describeSpotPriceHistory API fails", func() { - fakePricingAPI.NextError.Set(fmt.Errorf("failed")) - p := pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})) + awsEnv.PricingAPI.NextError.Set(fmt.Errorf("failed")) + p := pricing.NewProvider(ctx, awsEnv.PricingAPI, awsEnv.EC2API, "", make(chan struct{})) price, ok := p.SpotPrice("c5.large", "test-zone-1a") Expect(ok).To(BeTrue()) Expect(price).To(BeNumerically(">", 0)) @@ -107,14 +97,14 @@ var _ = Describe("Pricing", func() { It("should update on-demand pricing with response from the pricing API", func() { // modify our API before creating the pricing provider as it performs an initial update on creation. The pricing // API provides on-demand prices, the ec2 API provides spot prices - fakePricingAPI.GetProductsOutput.Set(&awspricing.GetProductsOutput{ + awsEnv.PricingAPI.GetProductsOutput.Set(&awspricing.GetProductsOutput{ PriceList: []aws.JSONValue{ fake.NewOnDemandPrice("c98.large", 1.20), fake.NewOnDemandPrice("c99.large", 1.23), }, }) updateStart := time.Now() - p := pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})) + p := pricing.NewProvider(ctx, awsEnv.PricingAPI, awsEnv.EC2API, "", make(chan struct{})) Eventually(func() bool { return p.OnDemandLastUpdated().After(updateStart) }).Should(BeTrue()) price, ok := p.OnDemandPrice("c98.large") @@ -127,7 +117,7 @@ var _ = Describe("Pricing", func() { }) It("should update spot pricing with response from the pricing API", func() { now := time.Now() - fakeEC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ + awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ SpotPriceHistory: []*ec2.SpotPrice{ { AvailabilityZone: aws.String("test-zone-1a"), @@ -155,14 +145,14 @@ var _ = Describe("Pricing", func() { }, }, }) - fakePricingAPI.GetProductsOutput.Set(&awspricing.GetProductsOutput{ + awsEnv.PricingAPI.GetProductsOutput.Set(&awspricing.GetProductsOutput{ PriceList: []aws.JSONValue{ fake.NewOnDemandPrice("c98.large", 1.20), fake.NewOnDemandPrice("c99.large", 1.23), }, }) updateStart := time.Now() - p := pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})) + p := pricing.NewProvider(ctx, awsEnv.PricingAPI, awsEnv.EC2API, "", make(chan struct{})) Eventually(func() bool { return p.SpotLastUpdated().After(updateStart) }).Should(BeTrue()) price, ok := p.SpotPrice("c98.large", "test-zone-1b") @@ -175,7 +165,7 @@ var _ = Describe("Pricing", func() { }) It("should update zonal pricing with data from the spot pricing API", func() { now := time.Now() - fakeEC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ + awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ SpotPriceHistory: []*ec2.SpotPrice{ { AvailabilityZone: aws.String("test-zone-1a"), @@ -191,14 +181,14 @@ var _ = Describe("Pricing", func() { }, }, }) - fakePricingAPI.GetProductsOutput.Set(&awspricing.GetProductsOutput{ + awsEnv.PricingAPI.GetProductsOutput.Set(&awspricing.GetProductsOutput{ PriceList: []aws.JSONValue{ fake.NewOnDemandPrice("c98.large", 1.20), fake.NewOnDemandPrice("c99.large", 1.23), }, }) updateStart := time.Now() - p := pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})) + p := pricing.NewProvider(ctx, awsEnv.PricingAPI, awsEnv.EC2API, "", make(chan struct{})) Eventually(func() bool { return p.SpotLastUpdated().After(updateStart) }).Should(BeTrue()) price, ok := p.SpotPrice("c98.large", "test-zone-1a") @@ -210,7 +200,7 @@ var _ = Describe("Pricing", func() { }) It("should respond with false if price doesn't exist in zone", func() { now := time.Now() - fakeEC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ + awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ SpotPriceHistory: []*ec2.SpotPrice{ { AvailabilityZone: aws.String("test-zone-1a"), @@ -220,14 +210,14 @@ var _ = Describe("Pricing", func() { }, }, }) - fakePricingAPI.GetProductsOutput.Set(&awspricing.GetProductsOutput{ + awsEnv.PricingAPI.GetProductsOutput.Set(&awspricing.GetProductsOutput{ PriceList: []aws.JSONValue{ fake.NewOnDemandPrice("c98.large", 1.20), fake.NewOnDemandPrice("c99.large", 1.23), }, }) updateStart := time.Now() - p := pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})) + p := pricing.NewProvider(ctx, awsEnv.PricingAPI, awsEnv.EC2API, "", make(chan struct{})) Eventually(func() bool { return p.SpotLastUpdated().After(updateStart) }).Should(BeTrue()) _, ok := p.SpotPrice("c99.large", "test-zone-1b") @@ -239,7 +229,7 @@ var _ = Describe("Pricing", func() { // If it doesn't, they have a product description of Linux/UNIX. To work in both cases, we // need to search for both values. updateStart := time.Now() - fakeEC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ + awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ SpotPriceHistory: []*ec2.SpotPrice{ { AvailabilityZone: aws.String("test-zone-1a"), @@ -249,15 +239,15 @@ var _ = Describe("Pricing", func() { }, }, }) - fakePricingAPI.GetProductsOutput.Set(&awspricing.GetProductsOutput{ + awsEnv.PricingAPI.GetProductsOutput.Set(&awspricing.GetProductsOutput{ PriceList: []aws.JSONValue{ fake.NewOnDemandPrice("c98.large", 1.20), fake.NewOnDemandPrice("c99.large", 1.23), }, }) - p := pricing.NewProvider(ctx, fakePricingAPI, fakeEC2API, "", make(chan struct{})) + p := pricing.NewProvider(ctx, awsEnv.PricingAPI, awsEnv.EC2API, "", make(chan struct{})) Eventually(func() bool { return p.SpotLastUpdated().After(updateStart) }, 5*time.Second).Should(BeTrue()) - inp := fakeEC2API.DescribeSpotPriceHistoryInput.Clone() + inp := awsEnv.EC2API.DescribeSpotPriceHistoryInput.Clone() Expect(lo.Map(inp.ProductDescriptions, func(x *string, _ int) string { return *x })). To(ContainElements("Linux/UNIX", "Linux/UNIX (Amazon VPC)")) }) diff --git a/pkg/providers/securitygroup/suite_test.go b/pkg/providers/securitygroup/suite_test.go index b922a57fdf3f..7c50fe1dca32 100644 --- a/pkg/providers/securitygroup/suite_test.go +++ b/pkg/providers/securitygroup/suite_test.go @@ -22,7 +22,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/patrickmn/go-cache" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -33,8 +32,6 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" - awscache "github.com/aws/karpenter/pkg/cache" - "github.com/aws/karpenter/pkg/providers/securitygroup" "github.com/aws/karpenter/pkg/test" coresettings "github.com/aws/karpenter-core/pkg/apis/settings" @@ -44,18 +41,15 @@ import ( "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" - "github.com/aws/karpenter/pkg/fake" ) var ctx context.Context var stop context.CancelFunc var opts options.Options var env *coretest.Environment -var fakeEC2API *fake.EC2API +var awsEnv *test.Environment var provisioner *corev1alpha5.Provisioner var nodeTemplate *v1alpha1.AWSNodeTemplate -var securityGroupProvider *securitygroup.Provider -var securityGroupCache *cache.Cache func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -68,9 +62,7 @@ var _ = BeforeSuite(func() { ctx = coresettings.ToContext(ctx, coretest.Settings()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) - fakeEC2API = &fake.EC2API{} - securityGroupCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - securityGroupProvider = securitygroup.NewProvider(fakeEC2API, securityGroupCache) + awsEnv = test.NewEnvironment(ctx, env) }) var _ = AfterSuite(func() { @@ -111,8 +103,7 @@ var _ = BeforeEach(func() { }, }) - fakeEC2API.Reset() - securityGroupCache.Flush() + awsEnv.ResetCache() }) var _ = AfterEach(func() { @@ -122,7 +113,7 @@ var _ = AfterEach(func() { var _ = Describe("Security Group Provider", func() { It("should default to the clusters security groups", func() { ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - resolvedSecurityGroups, err := securityGroupProvider.List(ctx, nodeTemplate) + resolvedSecurityGroups, err := awsEnv.SecurityGroupProvider.List(ctx, nodeTemplate) Expect(err).To(BeNil()) Expect(len(resolvedSecurityGroups)).To(Equal(3)) Expect(resolvedSecurityGroups).To(ConsistOf( @@ -132,12 +123,12 @@ var _ = Describe("Security Group Provider", func() { )) }) It("should discover security groups by tag", func() { - fakeEC2API.DescribeSecurityGroupsOutput.Set(&ec2.DescribeSecurityGroupsOutput{SecurityGroups: []*ec2.SecurityGroup{ + awsEnv.EC2API.DescribeSecurityGroupsOutput.Set(&ec2.DescribeSecurityGroupsOutput{SecurityGroups: []*ec2.SecurityGroup{ {GroupId: aws.String("test-sg-1"), Tags: []*ec2.Tag{{Key: aws.String("kubernetes.io/cluster/test-cluster"), Value: aws.String("test-sg-1")}}}, {GroupId: aws.String("test-sg-2"), Tags: []*ec2.Tag{{Key: aws.String("kubernetes.io/cluster/test-cluster"), Value: aws.String("test-sg-2")}}}, }}) ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - resolvedSecurityGroups, err := securityGroupProvider.List(ctx, nodeTemplate) + resolvedSecurityGroups, err := awsEnv.SecurityGroupProvider.List(ctx, nodeTemplate) Expect(err).To(BeNil()) Expect(len(resolvedSecurityGroups)).To(Equal(2)) Expect(resolvedSecurityGroups).To(ConsistOf( @@ -148,7 +139,7 @@ var _ = Describe("Security Group Provider", func() { It("should discover security groups by multiple tag values", func() { nodeTemplate.Spec.SecurityGroupSelector = map[string]string{"Name": "test-security-group-1,test-security-group-2"} ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - resolvedSecurityGroups, err := securityGroupProvider.List(ctx, nodeTemplate) + resolvedSecurityGroups, err := awsEnv.SecurityGroupProvider.List(ctx, nodeTemplate) Expect(err).To(BeNil()) Expect(len(resolvedSecurityGroups)).To(Equal(2)) Expect(resolvedSecurityGroups).To(ConsistOf( @@ -159,7 +150,7 @@ var _ = Describe("Security Group Provider", func() { It("should discover security groups by ID", func() { nodeTemplate.Spec.SecurityGroupSelector = map[string]string{"aws-ids": "sg-test1"} ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - resolvedSecurityGroups, err := securityGroupProvider.List(ctx, nodeTemplate) + resolvedSecurityGroups, err := awsEnv.SecurityGroupProvider.List(ctx, nodeTemplate) Expect(err).To(BeNil()) Expect(len(resolvedSecurityGroups)).To(Equal(1)) Expect(resolvedSecurityGroups).To(ConsistOf( @@ -169,7 +160,7 @@ var _ = Describe("Security Group Provider", func() { It("should discover security groups by IDs", func() { nodeTemplate.Spec.SecurityGroupSelector = map[string]string{"aws-ids": "sg-test1,sg-test2"} ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - resolvedSecurityGroups, err := securityGroupProvider.List(ctx, nodeTemplate) + resolvedSecurityGroups, err := awsEnv.SecurityGroupProvider.List(ctx, nodeTemplate) Expect(err).To(BeNil()) Expect(len(resolvedSecurityGroups)).To(Equal(2)) Expect(resolvedSecurityGroups).To(ConsistOf( @@ -180,7 +171,7 @@ var _ = Describe("Security Group Provider", func() { It("should discover security groups by IDs and tags", func() { nodeTemplate.Spec.SecurityGroupSelector = map[string]string{"aws-ids": "sg-test1,sg-test2", "foo": "bar"} ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - resolvedSecurityGroups, err := securityGroupProvider.List(ctx, nodeTemplate) + resolvedSecurityGroups, err := awsEnv.SecurityGroupProvider.List(ctx, nodeTemplate) Expect(err).To(BeNil()) Expect(len(resolvedSecurityGroups)).To(Equal(2)) Expect(resolvedSecurityGroups).To(ConsistOf( @@ -191,7 +182,7 @@ var _ = Describe("Security Group Provider", func() { It("should discover security groups by IDs intersected with tags", func() { nodeTemplate.Spec.SecurityGroupSelector = map[string]string{"aws-ids": "sg-test2", "foo": "bar"} ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - resolvedSecurityGroups, err := securityGroupProvider.List(ctx, nodeTemplate) + resolvedSecurityGroups, err := awsEnv.SecurityGroupProvider.List(ctx, nodeTemplate) Expect(err).To(BeNil()) Expect(len(resolvedSecurityGroups)).To(Equal(1)) Expect(resolvedSecurityGroups).To(ConsistOf( diff --git a/pkg/providers/subnet/suite_test.go b/pkg/providers/subnet/suite_test.go index 41dfa7d4ab76..d7429e0713b4 100644 --- a/pkg/providers/subnet/suite_test.go +++ b/pkg/providers/subnet/suite_test.go @@ -21,7 +21,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/aws/aws-sdk-go/aws" - "github.com/patrickmn/go-cache" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -30,31 +29,27 @@ import ( . "knative.dev/pkg/logging/testing" "github.com/aws/karpenter/pkg/apis" - awssettings "github.com/aws/karpenter/pkg/apis/settings" + "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" - awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/providers/subnet" "github.com/aws/karpenter/pkg/test" - "github.com/aws/karpenter-core/pkg/apis/settings" + coresettings "github.com/aws/karpenter-core/pkg/apis/settings" corev1alpha5 "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/operator/injection" "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" - "github.com/aws/karpenter/pkg/fake" ) var ctx context.Context var stop context.CancelFunc var opts options.Options var env *coretest.Environment -var fakeEC2API *fake.EC2API +var awsEnv *test.Environment var provisioner *corev1alpha5.Provisioner var nodeTemplate *v1alpha1.AWSNodeTemplate -var subnetProvider *subnet.Provider -var subnetCache *cache.Cache func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -64,11 +59,10 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) - - fakeEC2API = &fake.EC2API{} - subnetCache = cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) - subnetProvider = subnet.NewProvider(fakeEC2API, subnetCache) + awsEnv = test.NewEnvironment(ctx, env) }) var _ = AfterSuite(func() { @@ -78,8 +72,8 @@ var _ = AfterSuite(func() { var _ = BeforeEach(func() { ctx = injection.WithOptions(ctx, opts) - ctx = settings.ToContext(ctx, coretest.Settings()) - ctx = awssettings.ToContext(ctx, test.Settings()) + ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = settings.ToContext(ctx, test.Settings()) nodeTemplate = &v1alpha1.AWSNodeTemplate{ ObjectMeta: metav1.ObjectMeta{ Name: coretest.RandomName(), @@ -109,8 +103,7 @@ var _ = BeforeEach(func() { }, }) - fakeEC2API.Reset() - subnetCache.Flush() + awsEnv.ResetCache() }) var _ = AfterEach(func() { @@ -121,7 +114,7 @@ var _ = Describe("Subnet Provider", func() { It("should discover subnet by ID", func() { nodeTemplate.Spec.SubnetSelector = map[string]string{"aws-ids": "subnet-test1"} ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - resolvedSubnetProvider, err := subnetProvider.List(ctx, nodeTemplate) + resolvedSubnetProvider, err := awsEnv.SubnetProvider.List(ctx, nodeTemplate) resolvedSubnet := subnet.Pretty(resolvedSubnetProvider) Expect(err).To(BeNil()) Expect(len(resolvedSubnet)).To(Equal(1)) @@ -132,7 +125,7 @@ var _ = Describe("Subnet Provider", func() { It("should discover subnets by IDs", func() { nodeTemplate.Spec.SubnetSelector = map[string]string{"aws-ids": "subnet-test1,subnet-test2"} ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - resolvedSubnetProvider, err := subnetProvider.List(ctx, nodeTemplate) + resolvedSubnetProvider, err := awsEnv.SubnetProvider.List(ctx, nodeTemplate) resolvedSubnet := subnet.Pretty(resolvedSubnetProvider) Expect(err).To(BeNil()) Expect(len(resolvedSubnet)).To(Equal(2)) @@ -144,7 +137,7 @@ var _ = Describe("Subnet Provider", func() { It("should discover subnets by IDs and tags", func() { nodeTemplate.Spec.SubnetSelector = map[string]string{"aws-ids": "subnet-test1,subnet-test2", "foo": "bar"} ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - resolvedSubnetProvider, err := subnetProvider.List(ctx, nodeTemplate) + resolvedSubnetProvider, err := awsEnv.SubnetProvider.List(ctx, nodeTemplate) resolvedSubnet := subnet.Pretty(resolvedSubnetProvider) Expect(err).To(BeNil()) Expect(len(resolvedSubnet)).To(Equal(2)) @@ -156,7 +149,7 @@ var _ = Describe("Subnet Provider", func() { It("should discover subnets by a single tag", func() { nodeTemplate.Spec.SubnetSelector = map[string]string{"Name": "test-subnet-1"} ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - resolvedSubnetProvider, err := subnetProvider.List(ctx, nodeTemplate) + resolvedSubnetProvider, err := awsEnv.SubnetProvider.List(ctx, nodeTemplate) resolvedSubnet := subnet.Pretty(resolvedSubnetProvider) Expect(err).To(BeNil()) Expect(len(resolvedSubnet)).To(Equal(1)) @@ -167,7 +160,7 @@ var _ = Describe("Subnet Provider", func() { It("should discover subnets by multiple tag values", func() { nodeTemplate.Spec.SubnetSelector = map[string]string{"Name": "test-subnet-1,test-subnet-2"} ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - resolvedSubnetProvider, err := subnetProvider.List(ctx, nodeTemplate) + resolvedSubnetProvider, err := awsEnv.SubnetProvider.List(ctx, nodeTemplate) resolvedSubnet := subnet.Pretty(resolvedSubnetProvider) Expect(err).To(BeNil()) Expect(len(resolvedSubnet)).To(Equal(2)) @@ -179,7 +172,7 @@ var _ = Describe("Subnet Provider", func() { It("should discover subnets by IDs intersected with tags", func() { nodeTemplate.Spec.SubnetSelector = map[string]string{"aws-ids": "subnet-test2", "foo": "bar"} ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - resolvedSubnetProvider, err := subnetProvider.List(ctx, nodeTemplate) + resolvedSubnetProvider, err := awsEnv.SubnetProvider.List(ctx, nodeTemplate) resolvedSubnet := subnet.Pretty(resolvedSubnetProvider) Expect(err).To(BeNil()) Expect(len(resolvedSubnet)).To(Equal(1)) diff --git a/pkg/test/context.go b/pkg/test/context.go deleted file mode 100644 index 7e866f9d93c7..000000000000 --- a/pkg/test/context.go +++ /dev/null @@ -1,129 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package test - -import ( - "context" - "fmt" - "net" - - "k8s.io/utils/clock" - "knative.dev/pkg/ptr" - - "github.com/aws/aws-sdk-go/awstesting/mock" - "github.com/aws/aws-sdk-go/service/ec2/ec2iface" - "github.com/aws/aws-sdk-go/service/ssm/ssmiface" - "github.com/imdario/mergo" - "github.com/patrickmn/go-cache" - "k8s.io/client-go/tools/record" - - "github.com/aws/karpenter-core/pkg/cloudprovider" - "github.com/aws/karpenter-core/pkg/events" - - awscache "github.com/aws/karpenter/pkg/cache" - awscontext "github.com/aws/karpenter/pkg/context" - "github.com/aws/karpenter/pkg/fake" - "github.com/aws/karpenter/pkg/providers/amifamily" - "github.com/aws/karpenter/pkg/providers/instance" - "github.com/aws/karpenter/pkg/providers/instancetype" - "github.com/aws/karpenter/pkg/providers/launchtemplate" - "github.com/aws/karpenter/pkg/providers/pricing" - "github.com/aws/karpenter/pkg/providers/securitygroup" - "github.com/aws/karpenter/pkg/providers/subnet" - - coretest "github.com/aws/karpenter-core/pkg/test" -) - -type ContextOptions struct { - SSMCache *cache.Cache - EC2Cache *cache.Cache - KubernetesVersionCache *cache.Cache - InstanceTypeCache *cache.Cache - UnavailableOfferingsCache *awscache.UnavailableOfferings - LaunchTemplateCache *cache.Cache - SubnetCache *cache.Cache - SecurityGroupCache *cache.Cache - PricingAPI *fake.PricingAPI -} - -func Context(ctx context.Context, ec2api ec2iface.EC2API, ssmapi ssmiface.SSMAPI, - env *coretest.Environment, clock clock.Clock, overrides ...ContextOptions) awscontext.Context { - options := ContextOptions{} - for _, override := range overrides { - if err := mergo.Merge(&options, override, mergo.WithOverride); err != nil { - panic(fmt.Sprintf("Failed to merge settings: %s", err)) - } - } - - // Providers - pricingProvider := pricing.NewProvider(ctx, options.PricingAPI, ec2api, "", make(chan struct{})) - subnetProvider := subnet.NewProvider(ec2api, options.SubnetCache) - securityGroupProvider := securitygroup.NewProvider(ec2api, options.SecurityGroupCache) - amiProvider := amifamily.NewProvider(env.Client, env.KubernetesInterface, ssmapi, ec2api, options.SSMCache, options.EC2Cache, options.KubernetesVersionCache) - amiResolver := amifamily.New(env.Client, amiProvider) - instanceTypesProvider := instancetype.NewProvider("", options.InstanceTypeCache, ec2api, subnetProvider, options.UnavailableOfferingsCache, pricingProvider) - launchTemplateProvider := - launchtemplate.NewProvider( - ctx, - options.LaunchTemplateCache, - ec2api, - amiResolver, - securityGroupProvider, - ptr.String("ca-bundle"), - make(chan struct{}), - net.ParseIP("10.0.100.10"), - "https://test-cluster", - ) - instanceProvider := - instance.NewProvider(ctx, - "", - ec2api, - options.UnavailableOfferingsCache, - instanceTypesProvider, - subnetProvider, - launchTemplateProvider, - ) - - return awscontext.Context{ - Context: cloudprovider.Context{ - Context: ctx, - RESTConfig: env.Config, - KubernetesInterface: env.KubernetesInterface, - KubeClient: env.Client, - EventRecorder: events.NewRecorder(&record.FakeRecorder{}), - Clock: clock, - StartAsync: nil, - }, - Session: mock.Session, - EC2API: ec2api, - UnavailableOfferingsCache: options.UnavailableOfferingsCache, - InstanceTypesProvider: instanceTypesProvider, - InstanceProvider: instanceProvider, - SubnetProvider: subnetProvider, - SecurityGroupProvider: securityGroupProvider, - PricingProvider: pricingProvider, - AMIProvider: amiProvider, - AMIResolver: amiResolver, - LaunchTemplateProvider: launchTemplateProvider, - } -} - -func OptionOR[T any](x *T, fallback *T) *T { - if x == nil { - return fallback - } - - return x -} diff --git a/pkg/test/environment.go b/pkg/test/environment.go new file mode 100644 index 000000000000..d5dab7a83553 --- /dev/null +++ b/pkg/test/environment.go @@ -0,0 +1,152 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "context" + "net" + + "knative.dev/pkg/ptr" + + "github.com/patrickmn/go-cache" + + awscache "github.com/aws/karpenter/pkg/cache" + "github.com/aws/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/providers/amifamily" + "github.com/aws/karpenter/pkg/providers/instance" + "github.com/aws/karpenter/pkg/providers/instancetype" + "github.com/aws/karpenter/pkg/providers/launchtemplate" + "github.com/aws/karpenter/pkg/providers/pricing" + "github.com/aws/karpenter/pkg/providers/securitygroup" + "github.com/aws/karpenter/pkg/providers/subnet" + + coretest "github.com/aws/karpenter-core/pkg/test" +) + +type Environment struct { + // API + EC2API *fake.EC2API + SSMAPI *fake.SSMAPI + PricingAPI *fake.PricingAPI + + // Cache + SSMCache *cache.Cache + EC2Cache *cache.Cache + KubernetesVersionCache *cache.Cache + InstanceTypeCache *cache.Cache + UnavailableOfferingsCache *awscache.UnavailableOfferings + LaunchTemplateCache *cache.Cache + SubnetCache *cache.Cache + SecurityGroupCache *cache.Cache + + // Providers + InstanceTypesProvider *instancetype.Provider + InstanceProvider *instance.Provider + SubnetProvider *subnet.Provider + SecurityGroupProvider *securitygroup.Provider + PricingProvider *pricing.Provider + AMIProvider *amifamily.Provider + AMIResolver *amifamily.Resolver + LaunchTemplateProvider *launchtemplate.Provider + CloudProvider *cloudprovider.CloudProvider +} + +func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment { + // + ec2api := &fake.EC2API{} + ssmapi := &fake.SSMAPI{} + + // cache + ssmCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + ec2Cache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + kubernetesVersionCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + instanceTypeCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + unavailableOfferingsCache := awscache.NewUnavailableOfferings() + launchTemplateCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + subnetCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + securityGroupCache := cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval) + fakePricingAPI := &fake.PricingAPI{} + + // Providers + pricingProvider := pricing.NewProvider(ctx, fakePricingAPI, ec2api, "", make(chan struct{})) + subnetProvider := subnet.NewProvider(ec2api, subnetCache) + securityGroupProvider := securitygroup.NewProvider(ec2api, securityGroupCache) + amiProvider := amifamily.NewProvider(env.Client, env.KubernetesInterface, ssmapi, ec2api, ssmCache, ec2Cache, kubernetesVersionCache) + amiResolver := amifamily.New(env.Client, amiProvider) + instanceTypesProvider := instancetype.NewProvider("", instanceTypeCache, ec2api, subnetProvider, unavailableOfferingsCache, pricingProvider) + launchTemplateProvider := + launchtemplate.NewProvider( + ctx, + launchTemplateCache, + ec2api, + amiResolver, + securityGroupProvider, + ptr.String("ca-bundle"), + make(chan struct{}), + net.ParseIP("10.0.100.10"), + "https://test-cluster", + ) + instanceProvider := + instance.NewProvider(ctx, + "", + ec2api, + unavailableOfferingsCache, + instanceTypesProvider, + subnetProvider, + launchTemplateProvider, + ) + cloudProvider := cloudprovider.New(ctx, instanceTypesProvider, instanceProvider, env.Client, amiProvider) + + return &Environment{ + EC2API: ec2api, + SSMAPI: ssmapi, + PricingAPI: fakePricingAPI, + + SSMCache: ssmCache, + EC2Cache: ec2Cache, + KubernetesVersionCache: kubernetesVersionCache, + InstanceTypeCache: instanceTypeCache, + LaunchTemplateCache: launchTemplateCache, + SubnetCache: subnetCache, + SecurityGroupCache: securityGroupCache, + UnavailableOfferingsCache: unavailableOfferingsCache, + + InstanceTypesProvider: instanceTypesProvider, + InstanceProvider: instanceProvider, + SubnetProvider: subnetProvider, + SecurityGroupProvider: securityGroupProvider, + PricingProvider: pricingProvider, + AMIProvider: amiProvider, + AMIResolver: amiResolver, + LaunchTemplateProvider: launchTemplateProvider, + CloudProvider: cloudProvider, + } +} + +func (env *Environment) ResetCache() { + env.EC2API.Reset() + env.SSMAPI.Reset() + env.PricingAPI.Reset() + + env.SSMCache.Flush() + env.EC2Cache.Flush() + env.KubernetesVersionCache.Flush() + env.InstanceTypeCache.Flush() + env.UnavailableOfferingsCache.Flush() + env.LaunchTemplateCache.Flush() + env.SubnetCache.Flush() + env.SecurityGroupCache.Flush() +} From 91a05c8fed5416614ea9224daec2c4095fe57d9e Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Mon, 6 Mar 2023 13:48:19 -0800 Subject: [PATCH 36/41] clean-up --- pkg/controllers/machine/garbagecollect/suite_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/controllers/machine/garbagecollect/suite_test.go b/pkg/controllers/machine/garbagecollect/suite_test.go index 06f64c07fbc5..cd6ee18b7f8c 100644 --- a/pkg/controllers/machine/garbagecollect/suite_test.go +++ b/pkg/controllers/machine/garbagecollect/suite_test.go @@ -53,7 +53,6 @@ var awsEnv *test.Environment var env *coretest.Environment var garbageCollectController controller.Controller var linkedMachineCache *cache.Cache -var fakePricingAPI *fake.PricingAPI func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) From 8d4dba698894c047272b3e1fa17a814b5b363e8c Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Mon, 6 Mar 2023 14:03:47 -0800 Subject: [PATCH 37/41] fixing tests --- pkg/controllers/machine/garbagecollect/suite_test.go | 2 +- pkg/controllers/machine/link/suite_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/machine/garbagecollect/suite_test.go b/pkg/controllers/machine/garbagecollect/suite_test.go index cd6ee18b7f8c..ec96a412fb56 100644 --- a/pkg/controllers/machine/garbagecollect/suite_test.go +++ b/pkg/controllers/machine/garbagecollect/suite_test.go @@ -86,6 +86,7 @@ var _ = Describe("MachineGarbageCollect", func() { var providerID string BeforeEach(func() { + awsEnv.ResetCache() instanceID := fake.InstanceID() providerID = fmt.Sprintf("aws:///test-zone-1a/%s", instanceID) nodeTemplate := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{}) @@ -121,7 +122,6 @@ var _ = Describe("MachineGarbageCollect", func() { InstanceId: aws.String(instanceID), InstanceType: aws.String("m5.large"), } - awsEnv.ResetCache() }) AfterEach(func() { ExpectCleanedUp(ctx, env.Client) diff --git a/pkg/controllers/machine/link/suite_test.go b/pkg/controllers/machine/link/suite_test.go index 56cfa29b93f4..5843f6590ac6 100644 --- a/pkg/controllers/machine/link/suite_test.go +++ b/pkg/controllers/machine/link/suite_test.go @@ -81,6 +81,7 @@ var _ = Describe("MachineLink", func() { var nodeTemplate *v1alpha1.AWSNodeTemplate BeforeEach(func() { + awsEnv.ResetCache() instanceID = fake.InstanceID() providerID = fmt.Sprintf("aws:///test-zone-1a/%s", instanceID) nodeTemplate = test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{}) @@ -117,7 +118,6 @@ var _ = Describe("MachineLink", func() { InstanceType: aws.String("m5.large"), }, ) - awsEnv.ResetCache() }) AfterEach(func() { ExpectCleanedUp(ctx, env.Client) From f758354ad2f83182cb50ecb1be99ee0d417eeeb7 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Mon, 6 Mar 2023 14:13:23 -0800 Subject: [PATCH 38/41] name change --- pkg/cloudprovider/suite_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index 1db30da426f8..005898dc0789 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -140,7 +140,7 @@ var _ = AfterEach(func() { ExpectCleanedUp(ctx, env.Client) }) -var _ = Describe("awsEnv.CloudProvider", func() { +var _ = Describe("CloudProvider", func() { Context("Defaulting", func() { // Intent here is that if updates occur on the provisioningController, the Provisioner doesn't need to be recreated It("should not set the InstanceProfile with the default if none provided in Provisioner", func() { From 35bde79e6dfefed1362d415a95a6b682fc7fd6d9 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Mon, 6 Mar 2023 14:57:07 -0800 Subject: [PATCH 39/41] Feedback changes --- pkg/cloudprovider/suite_test.go | 2 +- pkg/controllers/machine/garbagecollect/suite_test.go | 4 ++-- pkg/controllers/machine/link/suite_test.go | 4 ++-- pkg/controllers/nodetemplate/suite_test.go | 2 +- pkg/providers/amifamily/ami.go | 6 ------ pkg/providers/instancetype/suite_test.go | 2 +- pkg/providers/launchtemplate/suite_test.go | 2 +- pkg/providers/pricing/suite_test.go | 2 +- pkg/providers/securitygroup/suite_test.go | 2 +- pkg/providers/subnet/suite_test.go | 2 +- pkg/test/environment.go | 4 ++-- 11 files changed, 13 insertions(+), 19 deletions(-) diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index 005898dc0789..04330cd7b9f8 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -130,7 +130,7 @@ var _ = BeforeEach(func() { }) cluster.Reset() - awsEnv.ResetCache() + awsEnv.Reset() awsEnv.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") awsEnv.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" diff --git a/pkg/controllers/machine/garbagecollect/suite_test.go b/pkg/controllers/machine/garbagecollect/suite_test.go index ec96a412fb56..1386305343e6 100644 --- a/pkg/controllers/machine/garbagecollect/suite_test.go +++ b/pkg/controllers/machine/garbagecollect/suite_test.go @@ -78,7 +78,7 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - awsEnv.ResetCache() + awsEnv.Reset() }) var _ = Describe("MachineGarbageCollect", func() { @@ -86,7 +86,7 @@ var _ = Describe("MachineGarbageCollect", func() { var providerID string BeforeEach(func() { - awsEnv.ResetCache() + awsEnv.Reset() instanceID := fake.InstanceID() providerID = fmt.Sprintf("aws:///test-zone-1a/%s", instanceID) nodeTemplate := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{}) diff --git a/pkg/controllers/machine/link/suite_test.go b/pkg/controllers/machine/link/suite_test.go index 5843f6590ac6..61bf8008fc50 100644 --- a/pkg/controllers/machine/link/suite_test.go +++ b/pkg/controllers/machine/link/suite_test.go @@ -71,7 +71,7 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - awsEnv.ResetCache() + awsEnv.Reset() }) var _ = Describe("MachineLink", func() { @@ -81,7 +81,7 @@ var _ = Describe("MachineLink", func() { var nodeTemplate *v1alpha1.AWSNodeTemplate BeforeEach(func() { - awsEnv.ResetCache() + awsEnv.Reset() instanceID = fake.InstanceID() providerID = fmt.Sprintf("aws:///test-zone-1a/%s", instanceID) nodeTemplate = test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{}) diff --git a/pkg/controllers/nodetemplate/suite_test.go b/pkg/controllers/nodetemplate/suite_test.go index e780f677f7c6..f3f20bffd9dc 100644 --- a/pkg/controllers/nodetemplate/suite_test.go +++ b/pkg/controllers/nodetemplate/suite_test.go @@ -84,7 +84,7 @@ var _ = BeforeEach(func() { }, } - awsEnv.ResetCache() + awsEnv.Reset() }) var _ = AfterEach(func() { diff --git a/pkg/providers/amifamily/ami.go b/pkg/providers/amifamily/ami.go index eedc268f9fe0..96b92c57b646 100644 --- a/pkg/providers/amifamily/ami.go +++ b/pkg/providers/amifamily/ami.go @@ -259,9 +259,3 @@ func (p *Provider) getRequirementsFromImage(ec2Image *ec2.Image) scheduling.Requ requirements.Add(scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, architecture)) return requirements } - -func (p *Provider) Reset() { - p.ssmCache.Flush() - p.ec2Cache.Flush() - p.kubernetesVersionCache.Flush() -} diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index beaac3da7833..3e9c4a72ae9b 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -135,7 +135,7 @@ var _ = BeforeEach(func() { }) cluster.Reset() - awsEnv.ResetCache() + awsEnv.Reset() awsEnv.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") awsEnv.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index d0df5f9e48a8..92261abb2561 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -129,7 +129,7 @@ var _ = BeforeEach(func() { }, }) cluster.Reset() - awsEnv.ResetCache() + awsEnv.Reset() awsEnv.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") awsEnv.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" diff --git a/pkg/providers/pricing/suite_test.go b/pkg/providers/pricing/suite_test.go index 8be6355b062c..60511eaf2d58 100644 --- a/pkg/providers/pricing/suite_test.go +++ b/pkg/providers/pricing/suite_test.go @@ -72,7 +72,7 @@ var _ = BeforeEach(func() { ctx = coresettings.ToContext(ctx, coretest.Settings()) ctx = settings.ToContext(ctx, test.Settings()) - awsEnv.ResetCache() + awsEnv.Reset() }) var _ = AfterEach(func() { diff --git a/pkg/providers/securitygroup/suite_test.go b/pkg/providers/securitygroup/suite_test.go index 7c50fe1dca32..f6183bdf88c5 100644 --- a/pkg/providers/securitygroup/suite_test.go +++ b/pkg/providers/securitygroup/suite_test.go @@ -103,7 +103,7 @@ var _ = BeforeEach(func() { }, }) - awsEnv.ResetCache() + awsEnv.Reset() }) var _ = AfterEach(func() { diff --git a/pkg/providers/subnet/suite_test.go b/pkg/providers/subnet/suite_test.go index d7429e0713b4..20203ca2b2e6 100644 --- a/pkg/providers/subnet/suite_test.go +++ b/pkg/providers/subnet/suite_test.go @@ -103,7 +103,7 @@ var _ = BeforeEach(func() { }, }) - awsEnv.ResetCache() + awsEnv.Reset() }) var _ = AfterEach(func() { diff --git a/pkg/test/environment.go b/pkg/test/environment.go index d5dab7a83553..295a8004069e 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -65,7 +65,7 @@ type Environment struct { } func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment { - // + // API ec2api := &fake.EC2API{} ssmapi := &fake.SSMAPI{} @@ -136,7 +136,7 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment } } -func (env *Environment) ResetCache() { +func (env *Environment) Reset() { env.EC2API.Reset() env.SSMAPI.Reset() env.PricingAPI.Reset() From 23255755b24468bea3c4057f0fb029b1172ed8a5 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Mon, 6 Mar 2023 15:09:24 -0800 Subject: [PATCH 40/41] Removed cloudProvider from environment --- pkg/cloudprovider/suite_test.go | 25 ++++++++------- .../machine/garbagecollect/suite_test.go | 31 ++++++++++--------- pkg/controllers/machine/link/suite_test.go | 5 ++- pkg/providers/instancetype/suite_test.go | 14 +++++---- pkg/providers/launchtemplate/suite_test.go | 7 +++-- pkg/test/environment.go | 4 --- 6 files changed, 48 insertions(+), 38 deletions(-) diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index 04330cd7b9f8..e0180651c273 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -42,6 +42,7 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" + "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/test" @@ -71,11 +72,12 @@ var cluster *state.Cluster var fakeClock *clock.FakeClock var provisioner *v1alpha5.Provisioner var nodeTemplate *v1alpha1.AWSNodeTemplate +var cloudProvider *cloudprovider.CloudProvider func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) RegisterFailHandler(Fail) - RunSpecs(t, "awsEnv.CloudProvider/AWS") + RunSpecs(t, "cloudProvider/AWS") } var _ = BeforeSuite(func() { @@ -86,8 +88,9 @@ var _ = BeforeSuite(func() { awsEnv = test.NewEnvironment(ctx, env) fakeClock = clock.NewFakeClock(time.Now()) - cluster = state.NewCluster(fakeClock, env.Client, awsEnv.CloudProvider) - prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), awsEnv.CloudProvider, cluster) + cloudProvider = cloudprovider.New(ctx, awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, env.Client, awsEnv.AMIProvider) + cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) + prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) provisioningController = provisioning.NewController(env.Client, prov, events.NewRecorder(&record.FakeRecorder{})) }) @@ -204,7 +207,7 @@ var _ = Describe("CloudProvider", func() { Images: []*ec2.Image{{ImageId: aws.String(validAMI)}}, }) ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - instanceTypes, err := awsEnv.CloudProvider.GetInstanceTypes(ctx, provisioner) + instanceTypes, err := cloudProvider.GetInstanceTypes(ctx, provisioner) Expect(err).ToNot(HaveOccurred()) selectedInstanceType = instanceTypes[0] @@ -234,7 +237,7 @@ var _ = Describe("CloudProvider", func() { }, }, }) - drifted, err := awsEnv.CloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) + drifted, err := cloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) Expect(err).ToNot(HaveOccurred()) Expect(drifted).To(BeFalse()) }) @@ -250,7 +253,7 @@ var _ = Describe("CloudProvider", func() { }, }, }) - drifted, err := awsEnv.CloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) + drifted, err := cloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) Expect(err).ToNot(HaveOccurred()) Expect(drifted).To(BeFalse()) }) @@ -265,7 +268,7 @@ var _ = Describe("CloudProvider", func() { }, }, }) - drifted, err := awsEnv.CloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) + drifted, err := cloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) Expect(err).ToNot(HaveOccurred()) Expect(drifted).To(BeFalse()) }) @@ -281,7 +284,7 @@ var _ = Describe("CloudProvider", func() { }) // Instance is a reference to what we return in the GetInstances call instance.ImageId = aws.String(fake.ImageID()) - isDrifted, err := awsEnv.CloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) + isDrifted, err := cloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) Expect(err).ToNot(HaveOccurred()) Expect(isDrifted).To(BeTrue()) }) @@ -295,7 +298,7 @@ var _ = Describe("CloudProvider", func() { }, }, }) - isDrifted, err := awsEnv.CloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) + isDrifted, err := cloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) Expect(err).ToNot(HaveOccurred()) Expect(isDrifted).To(BeFalse()) }) @@ -308,7 +311,7 @@ var _ = Describe("CloudProvider", func() { }, }, }) - _, err := awsEnv.CloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) + _, err := cloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) Expect(err).To(HaveOccurred()) }) It("should error drift if node doesn't have provider id", func() { @@ -320,7 +323,7 @@ var _ = Describe("CloudProvider", func() { }, }, }) - isDrifted, err := awsEnv.CloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) + isDrifted, err := cloudProvider.IsMachineDrifted(ctx, machineutil.NewFromNode(node)) Expect(err).To(HaveOccurred()) Expect(isDrifted).To(BeFalse()) }) diff --git a/pkg/controllers/machine/garbagecollect/suite_test.go b/pkg/controllers/machine/garbagecollect/suite_test.go index 1386305343e6..6630aec2a403 100644 --- a/pkg/controllers/machine/garbagecollect/suite_test.go +++ b/pkg/controllers/machine/garbagecollect/suite_test.go @@ -33,7 +33,7 @@ import ( coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - "github.com/aws/karpenter-core/pkg/cloudprovider" + corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" "github.com/aws/karpenter-core/pkg/operator/controller" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" @@ -42,6 +42,7 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" + "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/controllers/machine/garbagecollect" "github.com/aws/karpenter/pkg/controllers/machine/link" "github.com/aws/karpenter/pkg/fake" @@ -53,6 +54,7 @@ var awsEnv *test.Environment var env *coretest.Environment var garbageCollectController controller.Controller var linkedMachineCache *cache.Cache +var cloudProvider *cloudprovider.CloudProvider func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -66,11 +68,12 @@ var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) awsEnv = test.NewEnvironment(ctx, env) + cloudProvider = cloudprovider.New(ctx, awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, env.Client, awsEnv.AMIProvider) linkedMachineCache = cache.New(time.Minute*10, time.Second*10) linkController := &link.Controller{ Cache: linkedMachineCache, } - garbageCollectController = garbagecollect.NewController(env.Client, awsEnv.CloudProvider, linkController) + garbageCollectController = garbagecollect.NewController(env.Client, cloudProvider, linkController) }) var _ = AfterSuite(func() { @@ -134,9 +137,9 @@ var _ = Describe("MachineGarbageCollect", func() { awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) ExpectReconcileSucceeded(ctx, garbageCollectController, client.ObjectKey{}) - _, err := awsEnv.CloudProvider.Get(ctx, providerID) + _, err := cloudProvider.Get(ctx, providerID) Expect(err).To(HaveOccurred()) - Expect(cloudprovider.IsMachineNotFoundError(err)).To(BeTrue()) + Expect(corecloudprovider.IsMachineNotFoundError(err)).To(BeTrue()) }) It("should delete an instance along with the node if there is no machine owner (to quicken scheduling)", func() { // Launch time was 10m ago @@ -149,9 +152,9 @@ var _ = Describe("MachineGarbageCollect", func() { ExpectApplied(ctx, env.Client, node) ExpectReconcileSucceeded(ctx, garbageCollectController, client.ObjectKey{}) - _, err := awsEnv.CloudProvider.Get(ctx, providerID) + _, err := cloudProvider.Get(ctx, providerID) Expect(err).To(HaveOccurred()) - Expect(cloudprovider.IsMachineNotFoundError(err)).To(BeTrue()) + Expect(corecloudprovider.IsMachineNotFoundError(err)).To(BeTrue()) ExpectNotFound(ctx, env.Client, node) }) @@ -201,9 +204,9 @@ var _ = Describe("MachineGarbageCollect", func() { defer GinkgoRecover() defer wg.Done() - _, err := awsEnv.CloudProvider.Get(ctx, fmt.Sprintf("aws:///test-zone-1a/%s", id)) + _, err := cloudProvider.Get(ctx, fmt.Sprintf("aws:///test-zone-1a/%s", id)) Expect(err).To(HaveOccurred()) - Expect(cloudprovider.IsMachineNotFoundError(err)).To(BeTrue()) + Expect(corecloudprovider.IsMachineNotFoundError(err)).To(BeTrue()) }(id) } wg.Wait() @@ -262,7 +265,7 @@ var _ = Describe("MachineGarbageCollect", func() { defer GinkgoRecover() defer wg.Done() - _, err := awsEnv.CloudProvider.Get(ctx, fmt.Sprintf("aws:///test-zone-1a/%s", id)) + _, err := cloudProvider.Get(ctx, fmt.Sprintf("aws:///test-zone-1a/%s", id)) Expect(err).ToNot(HaveOccurred()) }(id) } @@ -278,7 +281,7 @@ var _ = Describe("MachineGarbageCollect", func() { awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) ExpectReconcileSucceeded(ctx, garbageCollectController, client.ObjectKey{}) - _, err := awsEnv.CloudProvider.Get(ctx, providerID) + _, err := cloudProvider.Get(ctx, providerID) Expect(err).NotTo(HaveOccurred()) }) It("should not delete an instance if it was not launched by a machine", func() { @@ -292,7 +295,7 @@ var _ = Describe("MachineGarbageCollect", func() { awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) ExpectReconcileSucceeded(ctx, garbageCollectController, client.ObjectKey{}) - _, err := awsEnv.CloudProvider.Get(ctx, providerID) + _, err := cloudProvider.Get(ctx, providerID) Expect(err).NotTo(HaveOccurred()) }) It("should not delete the instance or node if it already has a machine that matches it", func() { @@ -311,7 +314,7 @@ var _ = Describe("MachineGarbageCollect", func() { ExpectApplied(ctx, env.Client, machine, node) ExpectReconcileSucceeded(ctx, garbageCollectController, client.ObjectKey{}) - _, err := awsEnv.CloudProvider.Get(ctx, providerID) + _, err := cloudProvider.Get(ctx, providerID) Expect(err).ToNot(HaveOccurred()) ExpectExists(ctx, env.Client, node) }) @@ -331,7 +334,7 @@ var _ = Describe("MachineGarbageCollect", func() { ExpectApplied(ctx, env.Client, machine) ExpectReconcileSucceeded(ctx, garbageCollectController, client.ObjectKey{}) - _, err := awsEnv.CloudProvider.Get(ctx, providerID) + _, err := cloudProvider.Get(ctx, providerID) Expect(err).NotTo(HaveOccurred()) }) It("should not delete an instance if it is recently linked but the machine doesn't exist", func() { @@ -343,7 +346,7 @@ var _ = Describe("MachineGarbageCollect", func() { linkedMachineCache.SetDefault(providerID, nil) ExpectReconcileSucceeded(ctx, garbageCollectController, client.ObjectKey{}) - _, err := awsEnv.CloudProvider.Get(ctx, providerID) + _, err := cloudProvider.Get(ctx, providerID) Expect(err).NotTo(HaveOccurred()) }) }) diff --git a/pkg/controllers/machine/link/suite_test.go b/pkg/controllers/machine/link/suite_test.go index 61bf8008fc50..96a222af4337 100644 --- a/pkg/controllers/machine/link/suite_test.go +++ b/pkg/controllers/machine/link/suite_test.go @@ -41,6 +41,7 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" + "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/controllers/machine/link" "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/test" @@ -51,6 +52,7 @@ var ctx context.Context var awsEnv *test.Environment var env *coretest.Environment var linkController controller.Controller +var cloudProvider *cloudprovider.CloudProvider func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -64,7 +66,8 @@ var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) awsEnv = test.NewEnvironment(ctx, env) - linkController = link.NewController(env.Client, awsEnv.CloudProvider) + cloudProvider = cloudprovider.New(ctx, awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, env.Client, awsEnv.AMIProvider) + linkController = link.NewController(env.Client, cloudProvider) }) var _ = AfterSuite(func() { Expect(env.Stop()).To(Succeed(), "Failed to stop environment") diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index 3e9c4a72ae9b..500804c2d073 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -76,6 +76,7 @@ var provisioner *v1alpha5.Provisioner var nodeTemplate *v1alpha1.AWSNodeTemplate var cluster *state.Cluster var provisioningController controller.Controller +var cloudProvider *cloudprovider.CloudProvider func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -91,8 +92,9 @@ var _ = BeforeSuite(func() { awsEnv = test.NewEnvironment(ctx, env) fakeClock = &clock.FakeClock{} - cluster = state.NewCluster(fakeClock, env.Client, awsEnv.CloudProvider) - prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), awsEnv.CloudProvider, cluster) + cloudProvider = cloudprovider.New(ctx, awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, env.Client, awsEnv.AMIProvider) + cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) + prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) provisioningController = provisioning.NewController(env.Client, prov, events.NewRecorder(&record.FakeRecorder{})) }) @@ -220,7 +222,7 @@ var _ = Describe("Instance Types", func() { }) ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectScheduled(ctx, env.Client, pod) - its, err := awsEnv.CloudProvider.GetInstanceTypes(ctx, provisioner) + its, err := cloudProvider.GetInstanceTypes(ctx, provisioner) Expect(err).To(BeNil()) // Order all the instances by their price // We need some way to deterministically order them if their prices match @@ -266,7 +268,7 @@ var _ = Describe("Instance Types", func() { }, } ExpectApplied(ctx, env.Client, provisioner, nodeTemplate) - awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(generateSpotPricing(awsEnv.CloudProvider, provisioner)) + awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(generateSpotPricing(cloudProvider, provisioner)) Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) pod := coretest.UnschedulablePod(coretest.PodOptions{ @@ -278,7 +280,7 @@ var _ = Describe("Instance Types", func() { ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectScheduled(ctx, env.Client, pod) - its, err := awsEnv.CloudProvider.GetInstanceTypes(ctx, provisioner) + its, err := cloudProvider.GetInstanceTypes(ctx, provisioner) Expect(err).To(BeNil()) // Order all the instances by their price // We need some way to deterministically order them if their prices match @@ -1048,7 +1050,7 @@ var _ = Describe("Instance Types", func() { } awsEnv.InstanceTypeCache.Flush() - instanceTypes, err := awsEnv.CloudProvider.GetInstanceTypes(ctx, provisioner) + instanceTypes, err := cloudProvider.GetInstanceTypes(ctx, provisioner) Expect(err).To(BeNil()) instanceTypeNames := sets.NewString() for _, it := range instanceTypes { diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 92261abb2561..e31c2cbfc285 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -46,6 +46,7 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" + "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/providers/amifamily/bootstrap" "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/test" @@ -72,6 +73,7 @@ var prov *provisioning.Provisioner var provisioner *v1alpha5.Provisioner var nodeTemplate *v1alpha1.AWSNodeTemplate var cluster *state.Cluster +var cloudProvider *cloudprovider.CloudProvider func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -87,8 +89,9 @@ var _ = BeforeSuite(func() { awsEnv = test.NewEnvironment(ctx, env) fakeClock = &clock.FakeClock{} - cluster = state.NewCluster(fakeClock, env.Client, awsEnv.CloudProvider) - prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), awsEnv.CloudProvider, cluster) + cloudProvider = cloudprovider.New(ctx, awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, env.Client, awsEnv.AMIProvider) + cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) + prov = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) }) var _ = AfterSuite(func() { diff --git a/pkg/test/environment.go b/pkg/test/environment.go index 295a8004069e..36bcc528f675 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -23,7 +23,6 @@ import ( "github.com/patrickmn/go-cache" awscache "github.com/aws/karpenter/pkg/cache" - "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/providers/instance" @@ -61,7 +60,6 @@ type Environment struct { AMIProvider *amifamily.Provider AMIResolver *amifamily.Resolver LaunchTemplateProvider *launchtemplate.Provider - CloudProvider *cloudprovider.CloudProvider } func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment { @@ -108,7 +106,6 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment subnetProvider, launchTemplateProvider, ) - cloudProvider := cloudprovider.New(ctx, instanceTypesProvider, instanceProvider, env.Client, amiProvider) return &Environment{ EC2API: ec2api, @@ -132,7 +129,6 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment AMIProvider: amiProvider, AMIResolver: amiResolver, LaunchTemplateProvider: launchTemplateProvider, - CloudProvider: cloudProvider, } } From 3e14492f5350d09791468e97421c5c5dd7f60329 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Mon, 6 Mar 2023 15:30:21 -0800 Subject: [PATCH 41/41] Feedback changes --- hack/docs/instancetypes_gen_docs.go | 5 +++-- pkg/controllers/machine/garbagecollect/suite_test.go | 1 - pkg/controllers/machine/link/suite_test.go | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/hack/docs/instancetypes_gen_docs.go b/hack/docs/instancetypes_gen_docs.go index 826ee621297a..9b9d4b671172 100644 --- a/hack/docs/instancetypes_gen_docs.go +++ b/hack/docs/instancetypes_gen_docs.go @@ -233,9 +233,10 @@ func (f kubeDnsTransport) RoundTrip(request *http.Request) (*http.Response, erro } func NewAWSCloudProviderForCodeGen(ctx context.Context) *awscloudprovider.CloudProvider { - return awscloudprovider.New(awscontext.NewOrDie(cloudprovider.Context{ + context := awscontext.NewOrDie(cloudprovider.Context{ Context: ctx, RESTConfig: &rest.Config{}, KubernetesInterface: lo.Must(kubernetes.NewForConfigAndClient(&rest.Config{}, &http.Client{Transport: &kubeDnsTransport{}})), - })) + }) + return awscloudprovider.New(context, context.InstanceTypesProvider, context.InstanceProvider, context.KubeClient, context.AMIProvider) } diff --git a/pkg/controllers/machine/garbagecollect/suite_test.go b/pkg/controllers/machine/garbagecollect/suite_test.go index 6630aec2a403..975819ca1ef6 100644 --- a/pkg/controllers/machine/garbagecollect/suite_test.go +++ b/pkg/controllers/machine/garbagecollect/suite_test.go @@ -89,7 +89,6 @@ var _ = Describe("MachineGarbageCollect", func() { var providerID string BeforeEach(func() { - awsEnv.Reset() instanceID := fake.InstanceID() providerID = fmt.Sprintf("aws:///test-zone-1a/%s", instanceID) nodeTemplate := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{}) diff --git a/pkg/controllers/machine/link/suite_test.go b/pkg/controllers/machine/link/suite_test.go index 96a222af4337..2ece7472cdb4 100644 --- a/pkg/controllers/machine/link/suite_test.go +++ b/pkg/controllers/machine/link/suite_test.go @@ -84,7 +84,6 @@ var _ = Describe("MachineLink", func() { var nodeTemplate *v1alpha1.AWSNodeTemplate BeforeEach(func() { - awsEnv.Reset() instanceID = fake.InstanceID() providerID = fmt.Sprintf("aws:///test-zone-1a/%s", instanceID) nodeTemplate = test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{})