From d7bddc6c1ed4c25bbeeb93dc23c0822b59fac962 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Wed, 1 Feb 2023 12:55:27 -0800 Subject: [PATCH] Update provisioning in testing across all files --- pkg/controllers/deprovisioning/suite_test.go | 23 +- pkg/controllers/inflightchecks/suite_test.go | 48 +- pkg/controllers/machine/suite_test.go | 7 +- pkg/controllers/provisioning/provisioner.go | 1 + .../provisioning/scheduling/existingnode.go | 2 +- .../scheduling/instance_selection_test.go | 1098 ++-- .../provisioning/scheduling/scheduler.go | 2 +- .../scheduling/scheduling_benchmark_test.go | 4 +- .../provisioning/scheduling/suite_test.go | 4194 ++++++++------- .../provisioning/scheduling/topology_test.go | 4502 +++++++++-------- pkg/controllers/provisioning/suite_test.go | 335 +- pkg/controllers/state/node.go | 8 +- pkg/controllers/termination/suite_test.go | 7 +- pkg/events/suite_test.go | 37 +- pkg/test/expectations/expectations.go | 27 +- pkg/utils/pod/scheduling.go | 2 +- 16 files changed, 5210 insertions(+), 5087 deletions(-) diff --git a/pkg/controllers/deprovisioning/suite_test.go b/pkg/controllers/deprovisioning/suite_test.go index 7d12e30c1d..efbdcb498f 100644 --- a/pkg/controllers/deprovisioning/suite_test.go +++ b/pkg/controllers/deprovisioning/suite_test.go @@ -32,6 +32,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" clock "k8s.io/utils/clock/testing" . "knative.dev/pkg/logging/testing" "knative.dev/pkg/ptr" @@ -47,6 +48,7 @@ import ( "github.com/aws/karpenter-core/pkg/controllers/provisioning" "github.com/aws/karpenter-core/pkg/controllers/state" "github.com/aws/karpenter-core/pkg/controllers/state/informer" + "github.com/aws/karpenter-core/pkg/events" "github.com/aws/karpenter-core/pkg/operator/controller" "github.com/aws/karpenter-core/pkg/operator/scheme" "github.com/aws/karpenter-core/pkg/test" @@ -60,7 +62,6 @@ var deprovisioningController *deprovisioning.Controller var provisioningController controller.Controller var provisioner *provisioning.Provisioner var cloudProvider *fake.CloudProvider -var recorder *test.EventRecorder var nodeStateController controller.Controller var fakeClock *clock.FakeClock var onDemandInstances []*cloudprovider.InstanceType @@ -82,9 +83,8 @@ var _ = BeforeSuite(func() { fakeClock = clock.NewFakeClock(time.Now()) cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) nodeStateController = informer.NewNodeController(env.Client, cluster) - recorder = test.NewEventRecorder() - provisioner = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), recorder, cloudProvider, cluster) - provisioningController = provisioning.NewController(env.Client, provisioner, recorder) + provisioner = provisioning.NewProvisioner(ctx, env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) + provisioningController = provisioning.NewController(env.Client, provisioner, events.NewRecorder(&record.FakeRecorder{})) }) var _ = AfterSuite(func() { @@ -122,13 +122,12 @@ var _ = BeforeEach(func() { mostExpensiveInstance = onDemandInstances[len(onDemandInstances)-1] mostExpensiveOffering = mostExpensiveInstance.Offerings[0] - recorder.Reset() // ensure any waiters on our clock are allowed to proceed before resetting our clock time for fakeClock.HasWaiters() { fakeClock.Step(1 * time.Minute) } fakeClock.SetTime(time.Now()) - deprovisioningController = deprovisioning.NewController(fakeClock, env.Client, provisioner, cloudProvider, recorder, cluster) + deprovisioningController = deprovisioning.NewController(fakeClock, env.Client, provisioner, cloudProvider, events.NewRecorder(&record.FakeRecorder{}), cluster) // Reset Feature Flags to test defaults ctx = settings.ToContext(ctx, test.Settings(test.SettingsOptions{DriftEnabled: true})) }) @@ -1793,11 +1792,12 @@ var _ = Describe("Parallelization", func() { }, time.Second*10).Should(Succeed()) wg.Wait() // Add a new pending pod that should schedule while node is not yet deleted - pods := ExpectProvisionedNoBinding(ctx, env.Client, provisioningController, provisioner, test.UnschedulablePod()) + pod = test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, provisioner, pod) nodes := &v1.NodeList{} Expect(env.Client.List(ctx, nodes)).To(Succeed()) Expect(len(nodes.Items)).To(Equal(2)) - Expect(pods[0].Spec.NodeName).NotTo(Equal(node.Name)) + ExpectScheduled(ctx, env.Client, pod) }) It("should not consolidate a node that is launched for pods on a deleting node", func() { labels := map[string]string{ @@ -1838,7 +1838,7 @@ var _ = Describe("Parallelization", func() { pods = append(pods, pod) } ExpectApplied(ctx, env.Client, rs, prov) - ExpectProvisionedNoBinding(ctx, env.Client, provisioningController, provisioner, lo.Map(pods, func(p *v1.Pod, _ int) *v1.Pod { return p.DeepCopy() })...) + ExpectProvisionedNoBinding(ctx, env.Client, provisioner, lo.Map(pods, func(p *v1.Pod, _ int) *v1.Pod { return p.DeepCopy() })...) nodeList := &v1.NodeList{} Expect(env.Client.List(ctx, nodeList)).To(Succeed()) @@ -1847,13 +1847,10 @@ var _ = Describe("Parallelization", func() { // Update cluster state with new node ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(&nodeList.Items[0])) - // Reset the bindings so we can re-record bindings - recorder.ResetBindings() - // Mark the node for deletion and re-trigger reconciliation oldNodeName := nodeList.Items[0].Name cluster.MarkForDeletion(nodeList.Items[0].Name) - ExpectProvisionedNoBinding(ctx, env.Client, provisioningController, provisioner, lo.Map(pods, func(p *v1.Pod, _ int) *v1.Pod { return p.DeepCopy() })...) + ExpectProvisionedNoBinding(ctx, env.Client, provisioner, lo.Map(pods, func(p *v1.Pod, _ int) *v1.Pod { return p.DeepCopy() })...) // Make sure that the cluster state is aware of the current node state Expect(env.Client.List(ctx, nodeList)).To(Succeed()) diff --git a/pkg/controllers/inflightchecks/suite_test.go b/pkg/controllers/inflightchecks/suite_test.go index 724751b177..eaff4b577a 100644 --- a/pkg/controllers/inflightchecks/suite_test.go +++ b/pkg/controllers/inflightchecks/suite_test.go @@ -17,6 +17,7 @@ package inflightchecks_test import ( "context" "fmt" + "sync" "testing" "time" @@ -47,7 +48,7 @@ var inflightController controller.Controller var env *test.Environment var fakeClock *clock.FakeClock var cp *fake.CloudProvider -var recorder *test.EventRecorder +var recorder *FakeEventRecorder func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -60,7 +61,7 @@ var _ = BeforeSuite(func() { env = test.NewEnvironment(scheme.Scheme, test.WithCRDs(apis.CRDs...)) ctx = settings.ToContext(ctx, test.Settings()) cp = &fake.CloudProvider{} - recorder = test.NewEventRecorder() + recorder = NewFakeEventRecorder() inflightController = inflightchecks.NewController(fakeClock, env.Client, recorder, cp) }) @@ -183,6 +184,49 @@ var _ = Describe("Controller", func() { }) }) +var _ events.Recorder = (*FakeEventRecorder)(nil) + +// FakeEventRecorder is a mock event recorder that is used to facilitate testing. +type FakeEventRecorder struct { + mu sync.RWMutex + calls map[string]int + events []events.Event +} + +func NewFakeEventRecorder() *FakeEventRecorder { + return &FakeEventRecorder{ + calls: map[string]int{}, + } +} + +func (e *FakeEventRecorder) Publish(evt events.Event) { + e.mu.Lock() + defer e.mu.Unlock() + e.events = append(e.events, evt) + e.calls[evt.Reason]++ +} + +func (e *FakeEventRecorder) Calls(reason string) int { + e.mu.RLock() + defer e.mu.RUnlock() + return e.calls[reason] +} + +func (e *FakeEventRecorder) Reset() { + e.mu.Lock() + defer e.mu.Unlock() + e.events = nil + e.calls = map[string]int{} +} + +func (e *FakeEventRecorder) ForEachEvent(f func(evt events.Event)) { + e.mu.RLock() + defer e.mu.RUnlock() + for _, e := range e.events { + f(e) + } +} + func ExpectDetectedEvent(msg string) { foundEvent := false recorder.ForEachEvent(func(evt events.Event) { diff --git a/pkg/controllers/machine/suite_test.go b/pkg/controllers/machine/suite_test.go index 5b55df5fa9..eff87c041b 100644 --- a/pkg/controllers/machine/suite_test.go +++ b/pkg/controllers/machine/suite_test.go @@ -25,6 +25,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" clock "k8s.io/utils/clock/testing" . "knative.dev/pkg/logging/testing" "sigs.k8s.io/controller-runtime/pkg/cache" @@ -36,6 +37,7 @@ import ( "github.com/aws/karpenter-core/pkg/cloudprovider/fake" "github.com/aws/karpenter-core/pkg/controllers/machine" "github.com/aws/karpenter-core/pkg/controllers/machine/terminator" + "github.com/aws/karpenter-core/pkg/events" "github.com/aws/karpenter-core/pkg/operator/controller" "github.com/aws/karpenter-core/pkg/operator/scheme" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -65,9 +67,8 @@ var _ = BeforeSuite(func() { ctx = settings.ToContext(ctx, test.Settings()) cloudProvider = fake.NewCloudProvider() - recorder := test.NewEventRecorder() - terminator := terminator.NewTerminator(fakeClock, env.Client, cloudProvider, terminator.NewEvictionQueue(ctx, env.KubernetesInterface.CoreV1(), recorder)) - machineController = machine.NewController(fakeClock, env.Client, cloudProvider, terminator, recorder) + terminator := terminator.NewTerminator(fakeClock, env.Client, cloudProvider, terminator.NewEvictionQueue(ctx, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}))) + machineController = machine.NewController(fakeClock, env.Client, cloudProvider, terminator, events.NewRecorder(&record.FakeRecorder{})) }) var _ = AfterSuite(func() { diff --git a/pkg/controllers/provisioning/provisioner.go b/pkg/controllers/provisioning/provisioner.go index cc5b85f0a4..9189549ef2 100644 --- a/pkg/controllers/provisioning/provisioner.go +++ b/pkg/controllers/provisioning/provisioner.go @@ -194,6 +194,7 @@ func (p *Provisioner) consolidationWarnings(ctx context.Context, po v1.Pod) { } } +//nolint:gocyclo func (p *Provisioner) NewScheduler(ctx context.Context, pods []*v1.Pod, stateNodes []*state.Node, opts scheduler.SchedulerOptions) (*scheduler.Scheduler, error) { // Build node templates var machines []*scheduler.MachineTemplate diff --git a/pkg/controllers/provisioning/scheduling/existingnode.go b/pkg/controllers/provisioning/scheduling/existingnode.go index 602959d768..800e3f2f47 100644 --- a/pkg/controllers/provisioning/scheduling/existingnode.go +++ b/pkg/controllers/provisioning/scheduling/existingnode.go @@ -88,7 +88,7 @@ func (n *ExistingNode) Add(ctx context.Context, pod *v1.Pod) error { nodeRequirements := scheduling.NewRequirements(n.requirements.Values()...) podRequirements := scheduling.NewPodRequirements(pod) - // Check Machine Affinity Requirements + // Check Node Affinity Requirements if err = nodeRequirements.Compatible(podRequirements); err != nil { return err } diff --git a/pkg/controllers/provisioning/scheduling/instance_selection_test.go b/pkg/controllers/provisioning/scheduling/instance_selection_test.go index bf42a44c6a..3c7598828b 100644 --- a/pkg/controllers/provisioning/scheduling/instance_selection_test.go +++ b/pkg/controllers/provisioning/scheduling/instance_selection_test.go @@ -17,549 +17,583 @@ package scheduling_test import ( "fmt" "math" + "math/rand" + "github.com/mitchellh/hashstructure/v2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/samber/lo" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/sets" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/cloudprovider" + "github.com/aws/karpenter-core/pkg/cloudprovider/fake" "github.com/aws/karpenter-core/pkg/scheduling" + "github.com/aws/karpenter-core/pkg/test" + . "github.com/aws/karpenter-core/pkg/test/expectations" + "github.com/aws/karpenter-core/pkg/utils/resources" ) -//var _ = Describe("Instance Type Selection", func() { -// var minPrice float64 -// var instanceTypeMap map[string]*cloudprovider.InstanceType -// nodePrice := func(n *v1.Node) float64 { -// of, _ := instanceTypeMap[n.Labels[v1.LabelInstanceTypeStable]].Offerings.Get(n.Labels[v1alpha5.LabelCapacityType], n.Labels[v1.LabelTopologyZone]) -// return of.Price -// } -// -// BeforeEach(func() { -// // open up the provisioner to any instance types -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.ArchitectureArm64, v1alpha5.ArchitectureAmd64}, -// }, -// } -// cloudProv.CreateCalls = nil -// cloudProv.InstanceTypes = fake.InstanceTypesAssorted() -// -// instanceTypeMap = getInstanceTypeMap(cloudProv.InstanceTypes) -// minPrice = getMinPrice(cloudProv.InstanceTypes) -// -// // add some randomness to instance type ordering to ensure we sort everywhere we need to -// rand.Shuffle(len(cloudProv.InstanceTypes), func(i, j int) { -// cloudProv.InstanceTypes[i], cloudProv.InstanceTypes[j] = cloudProv.InstanceTypes[j], cloudProv.InstanceTypes[i] -// }) -// }) -// -// // This set of tests ensure that we schedule on the cheapest valid instance type while also ensuring that all of the -// // instance types passed to the cloud provider are also valid per provisioner and node selector requirements. In some -// // ways they repeat some other tests, but the testing regarding checking against all possible instance types -// // passed to the cloud provider is unique. -// It("should schedule on one of the cheapest instances", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// }) -// It("should schedule on one of the cheapest instances (pod arch = amd64)", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.ArchitectureAmd64}, -// }}})) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// // ensure that the entire list of instance types match the label -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelArchStable, v1alpha5.ArchitectureAmd64) -// }) -// It("should schedule on one of the cheapest instances (pod arch = arm64)", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.ArchitectureArm64}, -// }}})) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelArchStable, v1alpha5.ArchitectureArm64) -// }) -// It("should schedule on one of the cheapest instances (prov arch = amd64)", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.ArchitectureAmd64}, -// }, -// } -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelArchStable, v1alpha5.ArchitectureAmd64) -// }) -// It("should schedule on one of the cheapest instances (prov arch = arm64)", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.ArchitectureArm64}, -// }, -// } -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelArchStable, v1alpha5.ArchitectureArm64) -// }) -// It("should schedule on one of the cheapest instances (prov os = windows)", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelOSStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{string(v1.Windows)}, -// }, -// } -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelOSStable, string(v1.Windows)) -// }) -// It("should schedule on one of the cheapest instances (pod os = windows)", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ -// Key: v1.LabelOSStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{string(v1.Windows)}, -// }}})) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelOSStable, string(v1.Windows)) -// }) -// It("should schedule on one of the cheapest instances (prov os = windows)", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelOSStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{string(v1.Windows)}, -// }, -// } -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelOSStable, string(v1.Windows)) -// }) -// It("should schedule on one of the cheapest instances (pod os = linux)", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ -// Key: v1.LabelOSStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{string(v1.Linux)}, -// }}})) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelOSStable, string(v1.Linux)) -// }) -// It("should schedule on one of the cheapest instances (pod os = linux)", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ -// Key: v1.LabelOSStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{string(v1.Linux)}, -// }}})) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelOSStable, string(v1.Linux)) -// }) -// It("should schedule on one of the cheapest instances (prov zone = test-zone-2)", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelTopologyZone, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"test-zone-2"}, -// }, -// } -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelTopologyZone, "test-zone-2") -// }) -// It("should schedule on one of the cheapest instances (pod zone = test-zone-2)", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ -// Key: v1.LabelTopologyZone, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"test-zone-2"}, -// }}})) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelTopologyZone, "test-zone-2") -// }) -// It("should schedule on one of the cheapest instances (prov ct = spot)", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// { -// Key: v1alpha5.LabelCapacityType, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.CapacityTypeSpot}, -// }, -// } -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1alpha5.LabelCapacityType, v1alpha5.CapacityTypeSpot) -// }) -// It("should schedule on one of the cheapest instances (pod ct = spot)", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ -// Key: v1alpha5.LabelCapacityType, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.CapacityTypeSpot}, -// }}})) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1alpha5.LabelCapacityType, v1alpha5.CapacityTypeSpot) -// }) -// It("should schedule on one of the cheapest instances (prov ct = ondemand, prov zone = test-zone-1)", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// { -// Key: v1alpha5.LabelCapacityType, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.CapacityTypeOnDemand}, -// }, -// { -// Key: v1.LabelTopologyZone, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"test-zone-1"}, -// }, -// } -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// ExpectInstancesWithOffering(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1alpha5.CapacityTypeOnDemand, "test-zone-1") -// }) -// It("should schedule on one of the cheapest instances (pod ct = spot, pod zone = test-zone-1)", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ -// Key: v1alpha5.LabelCapacityType, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.CapacityTypeSpot}, -// }, -// { -// Key: v1.LabelTopologyZone, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"test-zone-1"}, -// }, -// }})) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// ExpectInstancesWithOffering(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1alpha5.CapacityTypeSpot, "test-zone-1") -// }) -// It("should schedule on one of the cheapest instances (prov ct = spot, pod zone = test-zone-2)", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// { -// Key: v1alpha5.LabelCapacityType, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.CapacityTypeSpot}, -// }, -// } -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ -// Key: v1.LabelTopologyZone, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"test-zone-2"}, -// }}})) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// ExpectInstancesWithOffering(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1alpha5.CapacityTypeSpot, "test-zone-2") -// }) -// It("should schedule on one of the cheapest instances (prov ct = ondemand/test-zone-1/arm64/windows)", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.ArchitectureArm64}, -// }, -// { -// Key: v1.LabelOSStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{string(v1.Windows)}, -// }, -// { -// Key: v1alpha5.LabelCapacityType, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.CapacityTypeOnDemand}, -// }, -// { -// Key: v1.LabelTopologyZone, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"test-zone-1"}, -// }, -// } -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// ExpectInstancesWithOffering(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1alpha5.CapacityTypeOnDemand, "test-zone-1") -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelOSStable, string(v1.Windows)) -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelArchStable, "arm64") -// }) -// It("should schedule on one of the cheapest instances (prov = spot/test-zone-2, pod = amd64/linux)", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.ArchitectureAmd64}, -// }, -// { -// Key: v1.LabelOSStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{string(v1.Linux)}, -// }, -// } -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ -// { -// Key: v1alpha5.LabelCapacityType, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.CapacityTypeSpot}, -// }, -// { -// Key: v1.LabelTopologyZone, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"test-zone-2"}, -// }, -// }})) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// ExpectInstancesWithOffering(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1alpha5.CapacityTypeSpot, "test-zone-2") -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelOSStable, string(v1.Linux)) -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelArchStable, "amd64") -// }) -// It("should schedule on one of the cheapest instances (pod ct = spot/test-zone-2/amd64/linux)", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.ArchitectureAmd64}, -// }, -// { -// Key: v1.LabelOSStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{string(v1.Linux)}, -// }, -// { -// Key: v1alpha5.LabelCapacityType, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.CapacityTypeSpot}, -// }, -// { -// Key: v1.LabelTopologyZone, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"test-zone-2"}, -// }, -// }})) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(nodePrice(node)).To(Equal(minPrice)) -// ExpectInstancesWithOffering(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1alpha5.CapacityTypeSpot, "test-zone-2") -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelOSStable, string(v1.Linux)) -// ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelArchStable, "amd64") -// }) -// It("should not schedule if no instance type matches selector (pod arch = arm)", func() { -// // remove all Arm instance types -// cloudProv.InstanceTypes = filterInstanceTypes(cloudProv.InstanceTypes, func(i *cloudprovider.InstanceType) bool { -// return i.Requirements.Get(v1.LabelArchStable).Has(v1alpha5.ArchitectureAmd64) -// }) -// -// Expect(len(cloudProv.InstanceTypes)).To(BeNumerically(">", 0)) -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.ArchitectureArm64}, -// }, -// }})) -// ExpectNotScheduled(ctx, env.Client, pod[0]) -// Expect(cloudProv.CreateCalls).To(HaveLen(0)) -// }) -// It("should not schedule if no instance type matches selector (pod arch = arm zone=test-zone-2)", func() { -// // remove all Arm instance types in zone-2 -// cloudProv.InstanceTypes = filterInstanceTypes(cloudProv.InstanceTypes, func(i *cloudprovider.InstanceType) bool { -// for _, off := range i.Offerings { -// if off.Zone == "test-zone-2" { -// return i.Requirements.Get(v1.LabelArchStable).Has(v1alpha5.ArchitectureAmd64) -// } -// } -// return true -// }) -// Expect(len(cloudProv.InstanceTypes)).To(BeNumerically(">", 0)) -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.ArchitectureArm64}, -// }, -// { -// Key: v1.LabelTopologyZone, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"test-zone-2"}, -// }, -// }})) -// ExpectNotScheduled(ctx, env.Client, pod[0]) -// Expect(cloudProv.CreateCalls).To(HaveLen(0)) -// }) -// It("should not schedule if no instance type matches selector (prov arch = arm / pod zone=test-zone-2)", func() { -// // remove all Arm instance types in zone-2 -// cloudProv.InstanceTypes = filterInstanceTypes(cloudProv.InstanceTypes, func(i *cloudprovider.InstanceType) bool { -// for _, off := range i.Offerings { -// if off.Zone == "test-zone-2" { -// return i.Requirements.Get(v1.LabelArchStable).Has(v1alpha5.ArchitectureAmd64) -// } -// } -// return true -// }) -// -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.ArchitectureArm64}, -// }, -// } -// Expect(len(cloudProv.InstanceTypes)).To(BeNumerically(">", 0)) -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelTopologyZone, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"test-zone-2"}, -// }, -// }})) -// ExpectNotScheduled(ctx, env.Client, pod[0]) -// Expect(cloudProv.CreateCalls).To(HaveLen(0)) -// }) -// It("should schedule on an instance with enough resources", func() { -// // this is a pretty thorough exercise of scheduling, so we also check an invariant that scheduling doesn't -// // modify the instance type's Overhead() or Resources() maps so they can return the same map every time instead -// // of re-alllocating a new one per call -// resourceHashes := map[string]uint64{} -// overheadHashes := map[string]uint64{} -// for _, it := range cloudProv.InstanceTypes { -// var err error -// resourceHashes[it.Name], err = hashstructure.Hash(it.Capacity, hashstructure.FormatV2, nil) -// Expect(err).To(BeNil()) -// overheadHashes[it.Name], err = hashstructure.Hash(it.Overhead.Total(), hashstructure.FormatV2, nil) -// Expect(err).To(BeNil()) -// } -// ExpectApplied(ctx, env.Client, provisioner) -// // these values are constructed so that three of these pods can always fit on at least one of our instance types -// for _, cpu := range []float64{0.1, 1.0, 2, 2.5, 4, 8, 16} { -// for _, mem := range []float64{0.1, 1.0, 2, 4, 8, 16, 32} { -// cluster.Reset() -// cloudProv.CreateCalls = nil -// opts := test.PodOptions{ -// ResourceRequirements: v1.ResourceRequirements{Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%0.1f", cpu)), -// v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%0.1fGi", mem)), -// }}} -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(opts), test.UnschedulablePod(opts), test.UnschedulablePod(opts)) -// nodeNames := sets.NewString() -// for _, p := range pods { -// node := ExpectScheduled(ctx, env.Client, p) -// nodeNames.Insert(node.Name) -// } -// // should fit on one node -// Expect(nodeNames).To(HaveLen(1)) -// totalPodResources := resources.RequestsForPods(pods...) -// for _, it := range supportedInstanceTypes(cloudProv.CreateCalls[0]) { -// totalReserved := resources.Merge(totalPodResources, it.Overhead.Total()) -// // the total pod resources in CPU and memory + instance overhead should always be less than the -// // resources available on every viable instance has -// Expect(totalReserved.Cpu().Cmp(it.Capacity[v1.ResourceCPU])).To(Equal(-1)) -// Expect(totalReserved.Memory().Cmp(it.Capacity[v1.ResourceMemory])).To(Equal(-1)) -// } -// } -// } -// for _, it := range cloudProv.InstanceTypes { -// resourceHash, err := hashstructure.Hash(it.Capacity, hashstructure.FormatV2, nil) -// Expect(err).To(BeNil()) -// overheadHash, err := hashstructure.Hash(it.Overhead.Total(), hashstructure.FormatV2, nil) -// Expect(err).To(BeNil()) -// Expect(resourceHash).To(Equal(resourceHashes[it.Name]), fmt.Sprintf("expected %s Resources() to not be modified by scheduling", it.Name)) -// Expect(overheadHash).To(Equal(overheadHashes[it.Name]), fmt.Sprintf("expected %s Overhead() to not be modified by scheduling", it.Name)) -// } -// }) -// It("should schedule on cheaper on-demand instance even when spot price ordering would place other instance types first", func() { -// cloudProv.InstanceTypes = []*cloudprovider.InstanceType{ -// fake.NewInstanceType(fake.InstanceTypeOptions{ -// Name: "test-instance1", -// Architecture: "amd64", -// OperatingSystems: sets.NewString(string(v1.Linux)), -// Resources: v1.ResourceList{ -// v1.ResourceCPU: resource.MustParse("1"), -// v1.ResourceMemory: resource.MustParse("1Gi"), -// }, -// Offerings: []cloudprovider.Offering{ -// {CapacityType: v1alpha5.CapacityTypeOnDemand, Zone: "test-zone-1a", Price: 1.0, Available: true}, -// {CapacityType: v1alpha5.CapacityTypeSpot, Zone: "test-zone-1a", Price: 0.2, Available: true}, -// }, -// }), -// fake.NewInstanceType(fake.InstanceTypeOptions{ -// Name: "test-instance2", -// Architecture: "amd64", -// OperatingSystems: sets.NewString(string(v1.Linux)), -// Resources: v1.ResourceList{ -// v1.ResourceCPU: resource.MustParse("1"), -// v1.ResourceMemory: resource.MustParse("1Gi"), -// }, -// Offerings: []cloudprovider.Offering{ -// {CapacityType: v1alpha5.CapacityTypeOnDemand, Zone: "test-zone-1a", Price: 1.3, Available: true}, -// {CapacityType: v1alpha5.CapacityTypeSpot, Zone: "test-zone-1a", Price: 0.1, Available: true}, -// }, -// }), -// } -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// { -// Key: v1alpha5.LabelCapacityType, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"on-demand"}, -// }, -// } -// -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("test-instance1")) -// }) -//}) +var _ = Describe("Instance Type Selection", func() { + var minPrice float64 + var instanceTypeMap map[string]*cloudprovider.InstanceType + nodePrice := func(n *v1.Node) float64 { + of, _ := instanceTypeMap[n.Labels[v1.LabelInstanceTypeStable]].Offerings.Get(n.Labels[v1alpha5.LabelCapacityType], n.Labels[v1.LabelTopologyZone]) + return of.Price + } + + BeforeEach(func() { + // open up the provisioner to any instance types + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureArm64, v1alpha5.ArchitectureAmd64}, + }, + } + cloudProv.CreateCalls = nil + cloudProv.InstanceTypes = fake.InstanceTypesAssorted() + + instanceTypeMap = getInstanceTypeMap(cloudProv.InstanceTypes) + minPrice = getMinPrice(cloudProv.InstanceTypes) + + // add some randomness to instance type ordering to ensure we sort everywhere we need to + rand.Shuffle(len(cloudProv.InstanceTypes), func(i, j int) { + cloudProv.InstanceTypes[i], cloudProv.InstanceTypes[j] = cloudProv.InstanceTypes[j], cloudProv.InstanceTypes[i] + }) + }) + + // This set of tests ensure that we schedule on the cheapest valid instance type while also ensuring that all of the + // instance types passed to the cloud provider are also valid per provisioner and node selector requirements. In some + // ways they repeat some other tests, but the testing regarding checking against all possible instance types + // passed to the cloud provider is unique. + It("should schedule on one of the cheapest instances", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + }) + It("should schedule on one of the cheapest instances (pod arch = amd64)", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureAmd64}, + }}}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + // ensure that the entire list of instance types match the label + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelArchStable, v1alpha5.ArchitectureAmd64) + }) + It("should schedule on one of the cheapest instances (pod arch = arm64)", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureArm64}, + }}}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelArchStable, v1alpha5.ArchitectureArm64) + }) + It("should schedule on one of the cheapest instances (prov arch = amd64)", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureAmd64}, + }, + } + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelArchStable, v1alpha5.ArchitectureAmd64) + }) + It("should schedule on one of the cheapest instances (prov arch = arm64)", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureArm64}, + }, + } + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelArchStable, v1alpha5.ArchitectureArm64) + }) + It("should schedule on one of the cheapest instances (prov os = windows)", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + { + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{string(v1.Windows)}, + }, + } + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelOSStable, string(v1.Windows)) + }) + It("should schedule on one of the cheapest instances (pod os = windows)", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{string(v1.Windows)}, + }}}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelOSStable, string(v1.Windows)) + }) + It("should schedule on one of the cheapest instances (prov os = windows)", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + { + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{string(v1.Windows)}, + }, + } + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelOSStable, string(v1.Windows)) + }) + It("should schedule on one of the cheapest instances (pod os = linux)", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{string(v1.Linux)}, + }}}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelOSStable, string(v1.Linux)) + }) + It("should schedule on one of the cheapest instances (pod os = linux)", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{string(v1.Linux)}, + }}}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelOSStable, string(v1.Linux)) + }) + It("should schedule on one of the cheapest instances (prov zone = test-zone-2)", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + { + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-2"}, + }, + } + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelTopologyZone, "test-zone-2") + }) + It("should schedule on one of the cheapest instances (pod zone = test-zone-2)", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-2"}, + }}}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelTopologyZone, "test-zone-2") + }) + It("should schedule on one of the cheapest instances (prov ct = spot)", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + { + Key: v1alpha5.LabelCapacityType, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.CapacityTypeSpot}, + }, + } + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1alpha5.LabelCapacityType, v1alpha5.CapacityTypeSpot) + }) + It("should schedule on one of the cheapest instances (pod ct = spot)", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1alpha5.LabelCapacityType, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.CapacityTypeSpot}, + }}}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1alpha5.LabelCapacityType, v1alpha5.CapacityTypeSpot) + }) + It("should schedule on one of the cheapest instances (prov ct = ondemand, prov zone = test-zone-1)", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + { + Key: v1alpha5.LabelCapacityType, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.CapacityTypeOnDemand}, + }, + { + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-1"}, + }, + } + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + ExpectInstancesWithOffering(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1alpha5.CapacityTypeOnDemand, "test-zone-1") + }) + It("should schedule on one of the cheapest instances (pod ct = spot, pod zone = test-zone-1)", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1alpha5.LabelCapacityType, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.CapacityTypeSpot}, + }, + { + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-1"}, + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + ExpectInstancesWithOffering(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1alpha5.CapacityTypeSpot, "test-zone-1") + }) + It("should schedule on one of the cheapest instances (prov ct = spot, pod zone = test-zone-2)", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + { + Key: v1alpha5.LabelCapacityType, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.CapacityTypeSpot}, + }, + } + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-2"}, + }}}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + ExpectInstancesWithOffering(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1alpha5.CapacityTypeSpot, "test-zone-2") + }) + It("should schedule on one of the cheapest instances (prov ct = ondemand/test-zone-1/arm64/windows)", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureArm64}, + }, + { + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{string(v1.Windows)}, + }, + { + Key: v1alpha5.LabelCapacityType, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.CapacityTypeOnDemand}, + }, + { + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-1"}, + }, + } + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + ExpectInstancesWithOffering(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1alpha5.CapacityTypeOnDemand, "test-zone-1") + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelOSStable, string(v1.Windows)) + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelArchStable, "arm64") + }) + It("should schedule on one of the cheapest instances (prov = spot/test-zone-2, pod = amd64/linux)", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureAmd64}, + }, + { + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{string(v1.Linux)}, + }, + } + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1alpha5.LabelCapacityType, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.CapacityTypeSpot}, + }, + { + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-2"}, + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + ExpectInstancesWithOffering(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1alpha5.CapacityTypeSpot, "test-zone-2") + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelOSStable, string(v1.Linux)) + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelArchStable, "amd64") + }) + It("should schedule on one of the cheapest instances (pod ct = spot/test-zone-2/amd64/linux)", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureAmd64}, + }, + { + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{string(v1.Linux)}, + }, + { + Key: v1alpha5.LabelCapacityType, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.CapacityTypeSpot}, + }, + { + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-2"}, + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(nodePrice(node)).To(Equal(minPrice)) + ExpectInstancesWithOffering(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1alpha5.CapacityTypeSpot, "test-zone-2") + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelOSStable, string(v1.Linux)) + ExpectInstancesWithLabel(supportedInstanceTypes(cloudProv.CreateCalls[0]), v1.LabelArchStable, "amd64") + }) + It("should not schedule if no instance type matches selector (pod arch = arm)", func() { + // remove all Arm instance types + cloudProv.InstanceTypes = filterInstanceTypes(cloudProv.InstanceTypes, func(i *cloudprovider.InstanceType) bool { + return i.Requirements.Get(v1.LabelArchStable).Has(v1alpha5.ArchitectureAmd64) + }) + + Expect(len(cloudProv.InstanceTypes)).To(BeNumerically(">", 0)) + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureArm64}, + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + Expect(cloudProv.CreateCalls).To(HaveLen(0)) + }) + It("should not schedule if no instance type matches selector (pod arch = arm zone=test-zone-2)", func() { + // remove all Arm instance types in zone-2 + cloudProv.InstanceTypes = filterInstanceTypes(cloudProv.InstanceTypes, func(i *cloudprovider.InstanceType) bool { + for _, off := range i.Offerings { + if off.Zone == "test-zone-2" { + return i.Requirements.Get(v1.LabelArchStable).Has(v1alpha5.ArchitectureAmd64) + } + } + return true + }) + Expect(len(cloudProv.InstanceTypes)).To(BeNumerically(">", 0)) + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureArm64}, + }, + { + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-2"}, + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + Expect(cloudProv.CreateCalls).To(HaveLen(0)) + }) + It("should not schedule if no instance type matches selector (prov arch = arm / pod zone=test-zone-2)", func() { + // remove all Arm instance types in zone-2 + cloudProv.InstanceTypes = filterInstanceTypes(cloudProv.InstanceTypes, func(i *cloudprovider.InstanceType) bool { + for _, off := range i.Offerings { + if off.Zone == "test-zone-2" { + return i.Requirements.Get(v1.LabelArchStable).Has(v1alpha5.ArchitectureAmd64) + } + } + return true + }) + + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureArm64}, + }, + } + Expect(len(cloudProv.InstanceTypes)).To(BeNumerically(">", 0)) + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-2"}, + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + Expect(cloudProv.CreateCalls).To(HaveLen(0)) + }) + It("should schedule on an instance with enough resources", func() { + // this is a pretty thorough exercise of scheduling, so we also check an invariant that scheduling doesn't + // modify the instance type's Overhead() or Resources() maps so they can return the same map every time instead + // of re-alllocating a new one per call + resourceHashes := map[string]uint64{} + overheadHashes := map[string]uint64{} + for _, it := range cloudProv.InstanceTypes { + var err error + resourceHashes[it.Name], err = hashstructure.Hash(it.Capacity, hashstructure.FormatV2, nil) + Expect(err).To(BeNil()) + overheadHashes[it.Name], err = hashstructure.Hash(it.Overhead.Total(), hashstructure.FormatV2, nil) + Expect(err).To(BeNil()) + } + ExpectApplied(ctx, env.Client, provisioner) + // these values are constructed so that three of these pods can always fit on at least one of our instance types + for _, cpu := range []float64{0.1, 1.0, 2, 2.5, 4, 8, 16} { + for _, mem := range []float64{0.1, 1.0, 2, 4, 8, 16, 32} { + cluster.Reset() + cloudProv.CreateCalls = nil + opts := test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse(fmt.Sprintf("%0.1f", cpu)), + v1.ResourceMemory: resource.MustParse(fmt.Sprintf("%0.1fGi", mem)), + }}} + pods := []*v1.Pod{ + test.UnschedulablePod(opts), test.UnschedulablePod(opts), test.UnschedulablePod(opts), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + nodeNames := sets.NewString() + for _, p := range pods { + node := ExpectScheduled(ctx, env.Client, p) + nodeNames.Insert(node.Name) + } + // should fit on one node + Expect(nodeNames).To(HaveLen(1)) + totalPodResources := resources.RequestsForPods(pods...) + for _, it := range supportedInstanceTypes(cloudProv.CreateCalls[0]) { + totalReserved := resources.Merge(totalPodResources, it.Overhead.Total()) + // the total pod resources in CPU and memory + instance overhead should always be less than the + // resources available on every viable instance has + Expect(totalReserved.Cpu().Cmp(it.Capacity[v1.ResourceCPU])).To(Equal(-1)) + Expect(totalReserved.Memory().Cmp(it.Capacity[v1.ResourceMemory])).To(Equal(-1)) + } + } + } + for _, it := range cloudProv.InstanceTypes { + resourceHash, err := hashstructure.Hash(it.Capacity, hashstructure.FormatV2, nil) + Expect(err).To(BeNil()) + overheadHash, err := hashstructure.Hash(it.Overhead.Total(), hashstructure.FormatV2, nil) + Expect(err).To(BeNil()) + Expect(resourceHash).To(Equal(resourceHashes[it.Name]), fmt.Sprintf("expected %s Resources() to not be modified by scheduling", it.Name)) + Expect(overheadHash).To(Equal(overheadHashes[it.Name]), fmt.Sprintf("expected %s Overhead() to not be modified by scheduling", it.Name)) + } + }) + It("should schedule on cheaper on-demand instance even when spot price ordering would place other instance types first", func() { + cloudProv.InstanceTypes = []*cloudprovider.InstanceType{ + fake.NewInstanceType(fake.InstanceTypeOptions{ + Name: "test-instance1", + Architecture: "amd64", + OperatingSystems: sets.NewString(string(v1.Linux)), + Resources: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("1Gi"), + }, + Offerings: []cloudprovider.Offering{ + {CapacityType: v1alpha5.CapacityTypeOnDemand, Zone: "test-zone-1a", Price: 1.0, Available: true}, + {CapacityType: v1alpha5.CapacityTypeSpot, Zone: "test-zone-1a", Price: 0.2, Available: true}, + }, + }), + fake.NewInstanceType(fake.InstanceTypeOptions{ + Name: "test-instance2", + Architecture: "amd64", + OperatingSystems: sets.NewString(string(v1.Linux)), + Resources: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("1Gi"), + }, + Offerings: []cloudprovider.Offering{ + {CapacityType: v1alpha5.CapacityTypeOnDemand, Zone: "test-zone-1a", Price: 1.3, Available: true}, + {CapacityType: v1alpha5.CapacityTypeSpot, Zone: "test-zone-1a", Price: 0.1, Available: true}, + }, + }), + } + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + { + Key: v1alpha5.LabelCapacityType, + Operator: v1.NodeSelectorOpIn, + Values: []string{"on-demand"}, + }, + } + + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("test-instance1")) + }) +}) func supportedInstanceTypes(machine *v1alpha5.Machine) (res []*cloudprovider.InstanceType) { reqs := scheduling.NewNodeSelectorRequirements(machine.Spec.Requirements...) diff --git a/pkg/controllers/provisioning/scheduling/scheduler.go b/pkg/controllers/provisioning/scheduling/scheduler.go index 78f9b180fd..011336241c 100644 --- a/pkg/controllers/provisioning/scheduling/scheduler.go +++ b/pkg/controllers/provisioning/scheduling/scheduler.go @@ -243,7 +243,7 @@ func (s *Scheduler) calculateExistingMachines(stateNodes []*state.Node, daemonSe // We don't use the status field and instead recompute the remaining resources to ensure we have a consistent view // of the cluster during scheduling. Depending on how node creation falls out, this will also work for cases where - // we don't create Machine resources. + // we don't create Node resources. if _, ok := s.remainingResources[node.Labels()[v1alpha5.ProvisionerNameLabelKey]]; ok { s.remainingResources[node.Labels()[v1alpha5.ProvisionerNameLabelKey]] = resources.Subtract(s.remainingResources[node.Labels()[v1alpha5.ProvisionerNameLabelKey]], node.Capacity()) } diff --git a/pkg/controllers/provisioning/scheduling/scheduling_benchmark_test.go b/pkg/controllers/provisioning/scheduling/scheduling_benchmark_test.go index 8bb5b2eabb..8b03b55322 100644 --- a/pkg/controllers/provisioning/scheduling/scheduling_benchmark_test.go +++ b/pkg/controllers/provisioning/scheduling/scheduling_benchmark_test.go @@ -29,6 +29,7 @@ import ( "github.com/samber/lo" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/client-go/tools/record" "k8s.io/utils/clock" "github.com/aws/karpenter-core/pkg/apis/settings" @@ -36,6 +37,7 @@ import ( "github.com/aws/karpenter-core/pkg/cloudprovider/fake" "github.com/aws/karpenter-core/pkg/controllers/provisioning/scheduling" "github.com/aws/karpenter-core/pkg/controllers/state" + "github.com/aws/karpenter-core/pkg/events" "github.com/aws/karpenter-core/pkg/test" "go.uber.org/zap" @@ -122,7 +124,7 @@ func benchmarkScheduler(b *testing.B, instanceCount, podCount int) { scheduler := scheduling.NewScheduler(ctx, nil, []*scheduling.MachineTemplate{scheduling.NewMachineTemplate(provisioner)}, nil, state.NewCluster(&clock.RealClock{}, nil, cloudProv), nil, &scheduling.Topology{}, map[string][]*cloudprovider.InstanceType{provisioner.Name: instanceTypes}, nil, - test.NewEventRecorder(), + events.NewRecorder(&record.FakeRecorder{}), scheduling.SchedulerOptions{}) pods := makeDiversePods(podCount) diff --git a/pkg/controllers/provisioning/scheduling/suite_test.go b/pkg/controllers/provisioning/scheduling/suite_test.go index 34f0fb926e..194f2c108d 100644 --- a/pkg/controllers/provisioning/scheduling/suite_test.go +++ b/pkg/controllers/provisioning/scheduling/suite_test.go @@ -16,12 +16,19 @@ package scheduling_test import ( "context" + "fmt" "math" + "math/rand" "testing" "time" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" clock "k8s.io/utils/clock/testing" + "knative.dev/pkg/ptr" v1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -29,6 +36,7 @@ import ( "github.com/aws/karpenter-core/pkg/apis" "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + "github.com/aws/karpenter-core/pkg/cloudprovider" "github.com/aws/karpenter-core/pkg/cloudprovider/fake" "github.com/aws/karpenter-core/pkg/controllers/provisioning" "github.com/aws/karpenter-core/pkg/controllers/provisioning/scheduling" @@ -37,6 +45,7 @@ import ( "github.com/aws/karpenter-core/pkg/events" "github.com/aws/karpenter-core/pkg/operator/controller" "github.com/aws/karpenter-core/pkg/operator/scheme" + pscheduling "github.com/aws/karpenter-core/pkg/scheduling" "github.com/aws/karpenter-core/pkg/test" . "github.com/onsi/ginkgo/v2" @@ -138,2131 +147,2072 @@ var _ = Describe("Custom Constraints", func() { node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) }) - //It("should not schedule pods that have conflicting requirements", func() { - // provisioner.Spec.Labels = map[string]string{"test-key": "test-value"} - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, - // }}, - // ))[0] - // ExpectNotScheduled(ctx, env.Client, pod) - //}) + It("should not schedule pods that have conflicting requirements", func() { + provisioner.Spec.Labels = map[string]string{"test-key": "test-value"} + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + }) + Context("Well Known Labels", func() { + It("should use provisioner constraints", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}} + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) + }) + It("should use node selectors", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) + }) + It("should not schedule nodes with a hostname selector", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeSelector: map[string]string{v1.LabelHostname: "red-node"}}, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should not schedule the pod if nodeselector unknown", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "unknown"}}, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should not schedule if node selector outside of provisioner constraints", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule compatible requirements with Operator=In", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + }) + It("should schedule compatible requirements with Operator=Gt", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: fake.IntegerInstanceLabelKey, Operator: v1.NodeSelectorOpGt, Values: []string{"8"}, + }} + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(fake.IntegerInstanceLabelKey, "16")) + }) + It("should schedule compatible requirements with Operator=Lt", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: fake.IntegerInstanceLabelKey, Operator: v1.NodeSelectorOpLt, Values: []string{"8"}, + }} + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(fake.IntegerInstanceLabelKey, "2")) + }) + It("should not schedule incompatible preferences and requirements with Operator=In", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule compatible requirements with Operator=NotIn", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "unknown"}}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + }) + It("should not schedule incompatible preferences and requirements with Operator=NotIn", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule compatible preferences and requirements with Operator=In", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, + NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2", "unknown"}}}, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) + }) + It("should schedule incompatible preferences and requirements with Operator=In", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, + NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should schedule compatible preferences and requirements with Operator=NotIn", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, + NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-3"}}}, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) + }) + It("should schedule incompatible preferences and requirements with Operator=NotIn", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, + NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should schedule compatible node selectors, preferences and requirements", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}, + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, + NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + }) + It("should combine multidimensional node selectors, preferences and requirements", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeSelector: map[string]string{ + v1.LabelTopologyZone: "test-zone-3", + v1.LabelInstanceTypeStable: "arm-instance-type", + }, + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-3"}}, + {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"default-instance-type", "arm-instance-type"}}, + }, + NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"unknown"}}, + {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpNotIn, Values: []string{"unknown"}}, + }, + }, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "arm-instance-type")) + }) + }) + Context("Constraints Validation", func() { + It("should not schedule pods that have node selectors with restricted labels", func() { + ExpectApplied(ctx, env.Client, provisioner) + for label := range v1alpha5.RestrictedLabels { + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: label, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + } + }) + It("should not schedule pods that have node selectors with restricted domains", func() { + ExpectApplied(ctx, env.Client, provisioner) + for domain := range v1alpha5.RestrictedLabelDomains { + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + } + }) + It("should schedule pods that have node selectors with label in restricted domains exceptions list", func() { + var requirements []v1.NodeSelectorRequirement + for domain := range v1alpha5.LabelDomainExceptions { + requirements = append(requirements, v1.NodeSelectorRequirement{Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}) + } + provisioner.Spec.Requirements = requirements + ExpectApplied(ctx, env.Client, provisioner) + for domain := range v1alpha5.LabelDomainExceptions { + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(domain+"/test", "test-value")) + } + }) + It("should schedule pods that have node selectors with label in wellknown label list", func() { + schedulable := []*v1.Pod{ + // Constrained by zone + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}), + // Constrained by instanceType + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "default-instance-type"}}), + // Constrained by architecture + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "arm64"}}), + // Constrained by operatingSystem + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Linux)}}), + // Constrained by capacity type + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1alpha5.LabelCapacityType: "spot"}}), + } + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, schedulable...) + for _, pod := range schedulable { + ExpectScheduled(ctx, env.Client, pod) + } + }) + }) + Context("Scheduling Logic", func() { + It("should not schedule pods that have node selectors with In operator and undefined key", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule pods that have node selectors with NotIn operator and undefined key", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).ToNot(HaveKeyWithValue("test-key", "test-value")) + }) + It("should not schedule pods that have node selectors with Exists operator and undefined key", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpExists}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule pods that with DoesNotExists operator and undefined key", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpDoesNotExist}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).ToNot(HaveKey("test-key")) + }) + It("should schedule unconstrained pods that don't have matching node selectors", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) + }) + It("should schedule pods that have node selectors with matching value and In operator", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) + }) + It("should not schedule pods that have node selectors with matching value and NotIn operator", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule the pod with Exists operator and defined key", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpExists}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should not schedule the pod with DoesNotExists operator and defined key", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpDoesNotExist}, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should not schedule pods that have node selectors with different value and In operator", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should schedule pods that have node selectors with different value and NotIn operator", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"another-value"}}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) + }) + It("should schedule compatible pods to the same node", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}} + ExpectApplied(ctx, env.Client, provisioner) + pods := []*v1.Pod{ + test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, + }}), + test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"another-value"}}, + }}), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + node1 := ExpectScheduled(ctx, env.Client, pods[0]) + node2 := ExpectScheduled(ctx, env.Client, pods[1]) + Expect(node1.Labels).To(HaveKeyWithValue("test-key", "test-value")) + Expect(node2.Labels).To(HaveKeyWithValue("test-key", "test-value")) + Expect(node1.Name).To(Equal(node2.Name)) + }) + It("should schedule incompatible pods to the different node", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}} + ExpectApplied(ctx, env.Client, provisioner) + pods := []*v1.Pod{ + test.UnschedulablePod( + test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, + }}), + test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, + }}), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + node1 := ExpectScheduled(ctx, env.Client, pods[0]) + node2 := ExpectScheduled(ctx, env.Client, pods[1]) + Expect(node1.Labels).To(HaveKeyWithValue("test-key", "test-value")) + Expect(node2.Labels).To(HaveKeyWithValue("test-key", "another-value")) + Expect(node1.Name).ToNot(Equal(node2.Name)) + }) + It("Exists operator should not overwrite the existing value", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"non-existent-zone"}}, + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpExists}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + }) +}) + +var _ = Describe("Preferential Fallback", func() { + Context("Required", func() { + It("should not relax the final term", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, + {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"default-instance-type"}}, + } + pod := test.UnschedulablePod() + pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ + {MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, // Should not be relaxed + }}, + }}}} + // Don't relax + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should relax multiple terms", func() { + pod := test.UnschedulablePod() + pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ + {MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, + }}, + {MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, + }}, + {MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, + }}, + {MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, // OR operator, never get to this one + }}, + }}}} + // Success + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-1")) + }) + }) + Context("Preferred", func() { + It("should relax all terms", func() { + pod := test.UnschedulablePod() + pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ + { + Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, + }}, + }, + { + Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, + }}, + }, + }}} + // Success + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should relax to use lighter weights", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} + pod := test.UnschedulablePod() + pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ + { + Weight: 100, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, + }}, + }, + { + Weight: 50, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, + }}, + }, + { + Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ // OR operator, never get to this one + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, + }}, + }, + }}} + // Success + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) + }) + It("should schedule even if preference is conflicting with requirement", func() { + pod := test.UnschedulablePod() + pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ + { + Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-3"}}, + }}, + }, + }, + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ + {MatchExpressions: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, // Should not be relaxed + }}, + }}, + }} + // Success + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) + }) + It("should schedule even if preference requirements are conflicting", func() { + pod := test.UnschedulablePod(test.PodOptions{NodePreferences: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"invalid"}}, + }}) + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + }) +}) + +var _ = Describe("Instance Type Compatibility", func() { + It("should not schedule if requesting more resources than any instance type has", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("512"), + }}, + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should launch pods with different archs on different instances", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureArm64, v1alpha5.ArchitectureAmd64}, + }} + nodeNames := sets.NewString() + ExpectApplied(ctx, env.Client, provisioner) + pods := []*v1.Pod{ + test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{v1.LabelArchStable: v1alpha5.ArchitectureAmd64}, + }), + test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{v1.LabelArchStable: v1alpha5.ArchitectureArm64}, + }), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + for _, pod := range pods { + node := ExpectScheduled(ctx, env.Client, pod) + nodeNames.Insert(node.Name) + } + Expect(nodeNames.Len()).To(Equal(2)) + }) + It("should exclude instance types that are not supported by the pod constraints (node affinity/instance type)", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureAmd64}, + }} + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod(test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"arm-instance-type"}, + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + // arm instance type conflicts with the provisioner limitation of AMD only + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should exclude instance types that are not supported by the pod constraints (node affinity/operating system)", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureAmd64}, + }} + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod(test.PodOptions{ + NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"ios"}, + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + // there's an instance with an OS of ios, but it has an arm processor so the provider requirements will + // exclude it + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should exclude instance types that are not supported by the provider constraints (arch)", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureAmd64}, + }} + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{v1.ResourceCPU: resource.MustParse("14")}}}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + // only the ARM instance has enough CPU, but it's not allowed per the provisioner + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should launch pods with different operating systems on different instances", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureArm64, v1alpha5.ArchitectureAmd64}, + }} + nodeNames := sets.NewString() + ExpectApplied(ctx, env.Client, provisioner) + pods := []*v1.Pod{ + test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Linux)}, + }), + test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Windows)}, + }), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + for _, pod := range pods { + node := ExpectScheduled(ctx, env.Client, pod) + nodeNames.Insert(node.Name) + } + Expect(nodeNames.Len()).To(Equal(2)) + }) + It("should launch pods with different instance type node selectors on different instances", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureArm64, v1alpha5.ArchitectureAmd64}, + }} + nodeNames := sets.NewString() + ExpectApplied(ctx, env.Client, provisioner) + pods := []*v1.Pod{ + test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{v1.LabelInstanceType: "small-instance-type"}, + }), + test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "default-instance-type"}, + }), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + for _, pod := range pods { + node := ExpectScheduled(ctx, env.Client, pod) + nodeNames.Insert(node.Name) + } + Expect(nodeNames.Len()).To(Equal(2)) + }) + It("should launch pods with different zone selectors on different instances", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{v1alpha5.ArchitectureArm64, v1alpha5.ArchitectureAmd64}, + }} + nodeNames := sets.NewString() + ExpectApplied(ctx, env.Client, provisioner) + pods := []*v1.Pod{ + test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}, + }), + test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}, + }), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + for _, pod := range pods { + node := ExpectScheduled(ctx, env.Client, pod) + nodeNames.Insert(node.Name) + } + Expect(nodeNames.Len()).To(Equal(2)) + }) + It("should launch pods with resources that aren't on any single instance type on different instances", func() { + cloudProv.InstanceTypes = fake.InstanceTypes(5) + const fakeGPU1 = "karpenter.sh/super-great-gpu" + const fakeGPU2 = "karpenter.sh/even-better-gpu" + cloudProv.InstanceTypes[0].Capacity[fakeGPU1] = resource.MustParse("25") + cloudProv.InstanceTypes[1].Capacity[fakeGPU2] = resource.MustParse("25") + + nodeNames := sets.NewString() + ExpectApplied(ctx, env.Client, provisioner) + pods := []*v1.Pod{ + test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{fakeGPU1: resource.MustParse("1")}, + }, + }), + // Should pack onto a different instance since no instance type has both GPUs + test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{fakeGPU2: resource.MustParse("1")}, + }, + }), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + for _, pod := range pods { + node := ExpectScheduled(ctx, env.Client, pod) + nodeNames.Insert(node.Name) + } + Expect(nodeNames.Len()).To(Equal(2)) + }) + It("should fail to schedule a pod with resources requests that aren't on a single instance type", func() { + cloudProv.InstanceTypes = fake.InstanceTypes(5) + const fakeGPU1 = "karpenter.sh/super-great-gpu" + const fakeGPU2 = "karpenter.sh/even-better-gpu" + cloudProv.InstanceTypes[0].Capacity[fakeGPU1] = resource.MustParse("25") + cloudProv.InstanceTypes[1].Capacity[fakeGPU2] = resource.MustParse("25") + + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + fakeGPU1: resource.MustParse("1"), + fakeGPU2: resource.MustParse("1")}, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + Context("Provider Specific Labels", func() { + It("should filter instance types that match labels", func() { + cloudProv.InstanceTypes = fake.InstanceTypes(5) + ExpectApplied(ctx, env.Client, provisioner) + pods := []*v1.Pod{ + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{fake.LabelInstanceSize: "large"}}), + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{fake.LabelInstanceSize: "small"}}), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + node := ExpectScheduled(ctx, env.Client, pods[0]) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "fake-it-4")) + node = ExpectScheduled(ctx, env.Client, pods[1]) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "fake-it-0")) + }) + It("should not schedule with incompatible labels", func() { + cloudProv.InstanceTypes = fake.InstanceTypes(5) + ExpectApplied(ctx, env.Client, provisioner) + pods := []*v1.Pod{ + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{ + fake.LabelInstanceSize: "large", + v1.LabelInstanceTypeStable: cloudProv.InstanceTypes[0].Name, + }}), + test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{ + fake.LabelInstanceSize: "small", + v1.LabelInstanceTypeStable: cloudProv.InstanceTypes[4].Name, + }}), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + ExpectNotScheduled(ctx, env.Client, pods[0]) + ExpectNotScheduled(ctx, env.Client, pods[1]) + }) + It("should schedule optional labels", func() { + cloudProv.InstanceTypes = fake.InstanceTypes(5) + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + // Only some instance types have this key + {Key: fake.ExoticInstanceLabelKey, Operator: v1.NodeSelectorOpExists}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKey(fake.ExoticInstanceLabelKey)) + Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, cloudProv.InstanceTypes[4].Name)) + }) + It("should schedule without optional labels if disallowed", func() { + cloudProv.InstanceTypes = fake.InstanceTypes(5) + ExpectApplied(ctx, env.Client, test.Provisioner()) + pod := test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ + // Only some instance types have this key + {Key: fake.ExoticInstanceLabelKey, Operator: v1.NodeSelectorOpDoesNotExist}, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).ToNot(HaveKey(fake.ExoticInstanceLabelKey)) + }) + }) +}) + +var _ = Describe("Binpacking", func() { + It("should schedule a small pod on the smallest instance", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("100M"), + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) + }) + It("should schedule a small pod on the smallest possible instance type", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("2000M"), + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) + }) + It("should schedule multiple small pods on the smallest possible instance type", func() { + opts := test.PodOptions{ + Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, + ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("10M"), + }, + }} + pods := test.Pods(5, opts) + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + nodeNames := sets.NewString() + for _, p := range pods { + node := ExpectScheduled(ctx, env.Client, p) + nodeNames.Insert(node.Name) + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) + } + Expect(nodeNames).To(HaveLen(1)) + }) + It("should create new nodes when a node is at capacity", func() { + opts := test.PodOptions{ + NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, + Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, + ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("1.8G"), + }, + }} + ExpectApplied(ctx, env.Client, provisioner) + pods := test.Pods(40, opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + nodeNames := sets.NewString() + for _, p := range pods { + node := ExpectScheduled(ctx, env.Client, p) + nodeNames.Insert(node.Name) + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("default-instance-type")) + } + Expect(nodeNames).To(HaveLen(20)) + }) + It("should pack small and large pods together", func() { + largeOpts := test.PodOptions{ + NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, + Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, + ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("1.8G"), + }, + }} + smallOpts := test.PodOptions{ + NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, + Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, + ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("400M"), + }, + }} + + // Two large pods are all that will fit on the default-instance type (the largest instance type) which will create + // twenty nodes. This leaves just enough room on each of those newNodes for one additional small pod per node, so we + // should only end up with 20 newNodes total. + provPods := append(test.Pods(40, largeOpts), test.Pods(20, smallOpts)...) + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, provPods...) + nodeNames := sets.NewString() + for _, p := range provPods { + node := ExpectScheduled(ctx, env.Client, p) + nodeNames.Insert(node.Name) + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("default-instance-type")) + } + Expect(nodeNames).To(HaveLen(20)) + }) + It("should pack newNodes tightly", func() { + cloudProv.InstanceTypes = fake.InstanceTypes(5) + var nodes []*v1.Node + ExpectApplied(ctx, env.Client, provisioner) + pods := []*v1.Pod{ + test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.5")}, + }, + }), + test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + }, + }), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + for _, pod := range pods { + node := ExpectScheduled(ctx, env.Client, pod) + nodes = append(nodes, node) + } + Expect(nodes).To(HaveLen(2)) + // the first pod consumes nearly all CPU of the largest instance type with no room for the second pod, the + // second pod is much smaller in terms of resources and should get a smaller node + Expect(nodes[0].Labels[v1.LabelInstanceTypeStable]).ToNot(Equal(nodes[1].Labels[v1.LabelInstanceTypeStable])) + }) + It("should handle zero-quantity resource requests", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{"foo.com/weird-resources": resource.MustParse("0")}, + Limits: v1.ResourceList{"foo.com/weird-resources": resource.MustParse("0")}, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + // requesting a resource of quantity zero of a type unsupported by any instance is fine + ExpectScheduled(ctx, env.Client, pod) + }) + It("should not schedule pods that exceed every instance type's capacity", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("2Ti"), + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should create new newNodes when a node is at capacity due to pod limits per node", func() { + opts := test.PodOptions{ + NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, + Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, + ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("1m"), + v1.ResourceCPU: resource.MustParse("1m"), + }, + }} + ExpectApplied(ctx, env.Client, provisioner) + pods := test.Pods(25, opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + nodeNames := sets.NewString() + // all of the test instance types support 5 pods each, so we use the 5 instances of the smallest one for our 25 pods + for _, p := range pods { + node := ExpectScheduled(ctx, env.Client, p) + nodeNames.Insert(node.Name) + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) + } + Expect(nodeNames).To(HaveLen(5)) + }) + It("should take into account initContainer resource requests when binpacking", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("1Gi"), + v1.ResourceCPU: resource.MustParse("1"), + }, + }, + InitImage: "pause", + InitResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("1Gi"), + v1.ResourceCPU: resource.MustParse("2"), + }, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("default-instance-type")) + }) + It("should not schedule pods when initContainer resource requests are greater than available instance types", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("1Gi"), + v1.ResourceCPU: resource.MustParse("1"), + }, + }, + InitImage: "pause", + InitResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceMemory: resource.MustParse("1Ti"), + v1.ResourceCPU: resource.MustParse("2"), + }, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should select for valid instance types, regardless of price", func() { + // capacity sizes and prices don't correlate here, regardless we should filter and see that all three instance types + // are valid before preferring the cheapest one 'large' + cloudProv.InstanceTypes = []*cloudprovider.InstanceType{ + fake.NewInstanceType(fake.InstanceTypeOptions{ + Name: "medium", + Resources: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("2Gi"), + }, + Offerings: []cloudprovider.Offering{ + { + CapacityType: v1alpha5.CapacityTypeOnDemand, + Zone: "test-zone-1a", + Price: 3.00, + Available: true, + }, + }, + }), + fake.NewInstanceType(fake.InstanceTypeOptions{ + Name: "small", + Resources: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("1Gi"), + }, + Offerings: []cloudprovider.Offering{ + { + CapacityType: v1alpha5.CapacityTypeOnDemand, + Zone: "test-zone-1a", + Price: 2.00, + Available: true, + }, + }, + }), + fake.NewInstanceType(fake.InstanceTypeOptions{ + Name: "large", + Resources: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("4"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }, + Offerings: []cloudprovider.Offering{ + { + CapacityType: v1alpha5.CapacityTypeOnDemand, + Zone: "test-zone-1a", + Price: 1.00, + Available: true, + }, + }, + }), + } + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1m"), + v1.ResourceMemory: resource.MustParse("1Mi"), + }, + }}, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + // large is the cheapest, so we should pick it, but the other two types are also valid options + Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("large")) + // all three options should be passed to the cloud provider + possibleInstanceType := sets.NewString(pscheduling.NewNodeSelectorRequirements(cloudProv.CreateCalls[0].Spec.Requirements...).Get(v1.LabelInstanceTypeStable).Values()...) + Expect(possibleInstanceType).To(Equal(sets.NewString("small", "medium", "large"))) }) - //Context("Well Known Labels", func() { - // It("should use provisioner constraints", func() { - // provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}} - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod())[0] - // node := ExpectScheduled(ctx, env.Client, pod) - // Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - // }) - // It("should use node selectors", func() { - // provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}, - // ))[0] - // node := ExpectScheduled(ctx, env.Client, pod) - // Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - // }) - // It("should not schedule nodes with a hostname selector", func() { - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeSelector: map[string]string{v1.LabelHostname: "red-node"}}, - // ))[0] - // ExpectNotScheduled(ctx, env.Client, pod) - // }) - // It("should not schedule the pod if nodeselector unknown", func() { - // provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "unknown"}}, - // ))[0] - // ExpectNotScheduled(ctx, env.Client, pod) - // }) - // It("should not schedule if node selector outside of provisioner constraints", func() { - // provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}, - // ))[0] - // ExpectNotScheduled(ctx, env.Client, pod) - // }) - // It("should schedule compatible requirements with Operator=In", func() { - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, - // }}, - // ))[0] - // node := ExpectScheduled(ctx, env.Client, pod) - // Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - // }) - // It("should schedule compatible requirements with Operator=Gt", func() { - // provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ - // Key: fake.IntegerInstanceLabelKey, Operator: v1.NodeSelectorOpGt, Values: []string{"8"}, - // }} - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod())[0] - // node := ExpectScheduled(ctx, env.Client, pod) - // Expect(node.Labels).To(HaveKeyWithValue(fake.IntegerInstanceLabelKey, "16")) - // }) - // It("should schedule compatible requirements with Operator=Lt", func() { - // provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ - // Key: fake.IntegerInstanceLabelKey, Operator: v1.NodeSelectorOpLt, Values: []string{"8"}, - // }} - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod())[0] - // node := ExpectScheduled(ctx, env.Client, pod) - // Expect(node.Labels).To(HaveKeyWithValue(fake.IntegerInstanceLabelKey, "2")) - // }) - // It("should not schedule incompatible preferences and requirements with Operator=In", func() { - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}, - // }}, - // ))[0] - // ExpectNotScheduled(ctx, env.Client, pod) - // }) - // It("should schedule compatible requirements with Operator=NotIn", func() { - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "unknown"}}, - // }}, - // ))[0] - // node := ExpectScheduled(ctx, env.Client, pod) - // Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - // }) - // It("should not schedule incompatible preferences and requirements with Operator=NotIn", func() { - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{ - // NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}, - // }}, - // ))[0] - // ExpectNotScheduled(ctx, env.Client, pod) - // }) - // It("should schedule compatible preferences and requirements with Operator=In", func() { - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{ - // NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - // NodePreferences: []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2", "unknown"}}}, - // }, - // ))[0] - // node := ExpectScheduled(ctx, env.Client, pod) - // Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - // }) - // It("should schedule incompatible preferences and requirements with Operator=In", func() { - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{ - // NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - // NodePreferences: []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"unknown"}}}, - // }, - // ))[0] - // ExpectScheduled(ctx, env.Client, pod) - // }) - // It("should schedule compatible preferences and requirements with Operator=NotIn", func() { - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{ - // NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - // NodePreferences: []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-3"}}}, - // }, - // ))[0] - // node := ExpectScheduled(ctx, env.Client, pod) - // Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) - // }) - // It("should schedule incompatible preferences and requirements with Operator=NotIn", func() { - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{ - // NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3", "unknown"}}}, - // NodePreferences: []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, - // }, - // ))[0] - // ExpectScheduled(ctx, env.Client, pod) - // }) - // It("should schedule compatible node selectors, preferences and requirements", func() { - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{ - // NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}, - // NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, - // NodePreferences: []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}}, - // }, - // ))[0] - // node := ExpectScheduled(ctx, env.Client, pod) - // Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - // }) - // It("should combine multidimensional node selectors, preferences and requirements", func() { - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{ - // NodeSelector: map[string]string{ - // v1.LabelTopologyZone: "test-zone-3", - // v1.LabelInstanceTypeStable: "arm-instance-type", - // }, - // NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-3"}}, - // {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"default-instance-type", "arm-instance-type"}}, - // }, - // NodePreferences: []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"unknown"}}, - // {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpNotIn, Values: []string{"unknown"}}, - // }, - // }, - // ))[0] - // node := ExpectScheduled(ctx, env.Client, pod) - // Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) - // Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "arm-instance-type")) - // }) - //}) - //Context("Constraints Validation", func() { - // It("should not schedule pods that have node selectors with restricted labels", func() { - // ExpectApplied(ctx, env.Client, provisioner) - // for label := range v1alpha5.RestrictedLabels { - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: label, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, - // }}))[0] - // ExpectNotScheduled(ctx, env.Client, pod) - // } - // }) - // It("should not schedule pods that have node selectors with restricted domains", func() { - // ExpectApplied(ctx, env.Client, provisioner) - // for domain := range v1alpha5.RestrictedLabelDomains { - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, - // }}))[0] - // ExpectNotScheduled(ctx, env.Client, pod) - // } - // }) - // It("should schedule pods that have node selectors with label in restricted domains exceptions list", func() { - // var requirements []v1.NodeSelectorRequirement - // for domain := range v1alpha5.LabelDomainExceptions { - // requirements = append(requirements, v1.NodeSelectorRequirement{Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}) - // } - // provisioner.Spec.Requirements = requirements - // ExpectApplied(ctx, env.Client, provisioner) - // for domain := range v1alpha5.LabelDomainExceptions { - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod())[0] - // node := ExpectScheduled(ctx, env.Client, pod) - // Expect(node.Labels).To(HaveKeyWithValue(domain+"/test", "test-value")) - // } - // }) - // It("should schedule pods that have node selectors with label in wellknown label list", func() { - // schedulable := []*v1.Pod{ - // // Constrained by zone - // test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}), - // // Constrained by instanceType - // test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "default-instance-type"}}), - // // Constrained by architecture - // test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "arm64"}}), - // // Constrained by operatingSystem - // test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Linux)}}), - // // Constrained by capacity type - // test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1alpha5.LabelCapacityType: "spot"}}), - // } - // ExpectApplied(ctx, env.Client, provisioner) - // for _, pod := range ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, schedulable...) { - // ExpectScheduled(ctx, env.Client, pod) - // } - // }) - //}) - //Context("Scheduling Logic", func() { - // It("should not schedule pods that have node selectors with In operator and undefined key", func() { - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - // }}))[0] - // ExpectNotScheduled(ctx, env.Client, pod) - // }) - // It("should schedule pods that have node selectors with NotIn operator and undefined key", func() { - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value"}}, - // }}))[0] - // node := ExpectScheduled(ctx, env.Client, pod) - // Expect(node.Labels).ToNot(HaveKeyWithValue("test-key", "test-value")) - // }) - // It("should not schedule pods that have node selectors with Exists operator and undefined key", func() { - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpExists}, - // }}))[0] - // ExpectNotScheduled(ctx, env.Client, pod) - // }) - // It("should schedule pods that with DoesNotExists operator and undefined key", func() { - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpDoesNotExist}, - // }}))[0] - // node := ExpectScheduled(ctx, env.Client, pod) - // Expect(node.Labels).ToNot(HaveKey("test-key")) - // }) - // It("should schedule unconstrained pods that don't have matching node selectors", func() { - // provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod())[0] - // node := ExpectScheduled(ctx, env.Client, pod) - // Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - // }) - // It("should schedule pods that have node selectors with matching value and In operator", func() { - // provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - // }}))[0] - // node := ExpectScheduled(ctx, env.Client, pod) - // Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - // }) - // It("should not schedule pods that have node selectors with matching value and NotIn operator", func() { - // provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-value"}}, - // }}))[0] - // ExpectNotScheduled(ctx, env.Client, pod) - // }) - // It("should schedule the pod with Exists operator and defined key", func() { - // provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpExists}, - // }}, - // ))[0] - // ExpectScheduled(ctx, env.Client, pod) - // }) - // It("should not schedule the pod with DoesNotExists operator and defined key", func() { - // provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpDoesNotExist}, - // }}, - // ))[0] - // ExpectNotScheduled(ctx, env.Client, pod) - // }) - // It("should not schedule pods that have node selectors with different value and In operator", func() { - // provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, - // }}))[0] - // ExpectNotScheduled(ctx, env.Client, pod) - // }) - // It("should schedule pods that have node selectors with different value and NotIn operator", func() { - // provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}} - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"another-value"}}, - // }}))[0] - // node := ExpectScheduled(ctx, env.Client, pod) - // Expect(node.Labels).To(HaveKeyWithValue("test-key", "test-value")) - // }) - // It("should schedule compatible pods to the same node", func() { - // provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}} - // ExpectApplied(ctx, env.Client, provisioner) - // pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - // }}), - // test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpNotIn, Values: []string{"another-value"}}, - // }})) - // node1 := ExpectScheduled(ctx, env.Client, pods[0]) - // node2 := ExpectScheduled(ctx, env.Client, pods[1]) - // Expect(node1.Labels).To(HaveKeyWithValue("test-key", "test-value")) - // Expect(node2.Labels).To(HaveKeyWithValue("test-key", "test-value")) - // Expect(node1.Name).To(Equal(node2.Name)) - // }) - // It("should schedule incompatible pods to the different node", func() { - // provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value", "another-value"}}} - // ExpectApplied(ctx, env.Client, provisioner) - // pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}, - // }}), - // test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: "test-key", Operator: v1.NodeSelectorOpIn, Values: []string{"another-value"}}, - // }})) - // node1 := ExpectScheduled(ctx, env.Client, pods[0]) - // node2 := ExpectScheduled(ctx, env.Client, pods[1]) - // Expect(node1.Labels).To(HaveKeyWithValue("test-key", "test-value")) - // Expect(node2.Labels).To(HaveKeyWithValue("test-key", "another-value")) - // Expect(node1.Name).ToNot(Equal(node2.Name)) - // }) - // It("Exists operator should not overwrite the existing value", func() { - // ExpectApplied(ctx, env.Client, provisioner) - // pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( - // test.PodOptions{ - // NodeRequirements: []v1.NodeSelectorRequirement{ - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"non-existent-zone"}}, - // {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpExists}, - // }}, - // ))[0] - // ExpectNotScheduled(ctx, env.Client, pod) - // }) - //}) }) -//var _ = Describe("Preferential Fallback", func() { -// Context("Required", func() { -// It("should not relax the final term", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, -// {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"default-instance-type"}}, -// } -// pod := test.UnschedulablePod() -// pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ -// {MatchExpressions: []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, // Should not be relaxed -// }}, -// }}}} -// // Don't relax -// ExpectApplied(ctx, env.Client, provisioner) -// pod = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod)[0] -// ExpectNotScheduled(ctx, env.Client, pod) -// }) -// It("should relax multiple terms", func() { -// pod := test.UnschedulablePod() -// pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ -// {MatchExpressions: []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, -// }}, -// {MatchExpressions: []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, -// }}, -// {MatchExpressions: []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, -// }}, -// {MatchExpressions: []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, // OR operator, never get to this one -// }}, -// }}}} -// // Success -// ExpectApplied(ctx, env.Client, provisioner) -// pod = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod)[0] -// node := ExpectScheduled(ctx, env.Client, pod) -// Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-1")) -// }) -// }) -// Context("Preferred", func() { -// It("should relax all terms", func() { -// pod := test.UnschedulablePod() -// pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ -// { -// Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, -// }}, -// }, -// { -// Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ -// {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, -// }}, -// }, -// }}} -// // Success -// ExpectApplied(ctx, env.Client, provisioner) -// pod = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod)[0] -// ExpectScheduled(ctx, env.Client, pod) -// }) -// It("should relax to use lighter weights", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} -// pod := test.UnschedulablePod() -// pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ -// { -// Weight: 100, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ -// {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, -// }}, -// }, -// { -// Weight: 50, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, -// }}, -// }, -// { -// Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ // OR operator, never get to this one -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, -// }}, -// }, -// }}} -// // Success -// ExpectApplied(ctx, env.Client, provisioner) -// pod = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod)[0] -// node := ExpectScheduled(ctx, env.Client, pod) -// Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) -// }) -// It("should schedule even if preference is conflicting with requirement", func() { -// pod := test.UnschedulablePod() -// pod.Spec.Affinity = &v1.Affinity{NodeAffinity: &v1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []v1.PreferredSchedulingTerm{ -// { -// Weight: 1, Preference: v1.NodeSelectorTerm{MatchExpressions: []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"test-zone-3"}}, -// }}, -// }, -// }, -// RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ -// {MatchExpressions: []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}, // Should not be relaxed -// }}, -// }}, -// }} -// // Success -// ExpectApplied(ctx, env.Client, provisioner) -// pod = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod)[0] -// node := ExpectScheduled(ctx, env.Client, pod) -// Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) -// }) -// It("should schedule even if preference requirements are conflicting", func() { -// pod := test.UnschedulablePod(test.PodOptions{NodePreferences: []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"invalid"}}, -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpNotIn, Values: []string{"invalid"}}, -// }}) -// ExpectApplied(ctx, env.Client, provisioner) -// pod = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod)[0] -// ExpectScheduled(ctx, env.Client, pod) -// }) -// }) -//}) -// -//var _ = Describe("Instance Type Compatibility", func() { -// It("should not schedule if requesting more resources than any instance type has", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("512"), -// }}, -// })) -// ExpectNotScheduled(ctx, env.Client, pod[0]) -// }) -// It("should launch pods with different archs on different instances", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.ArchitectureArm64, v1alpha5.ArchitectureAmd64}, -// }} -// nodeNames := sets.NewString() -// ExpectApplied(ctx, env.Client, provisioner) -// for _, pod := range ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ -// NodeSelector: map[string]string{v1.LabelArchStable: v1alpha5.ArchitectureAmd64}, -// }), -// test.UnschedulablePod(test.PodOptions{ -// NodeSelector: map[string]string{v1.LabelArchStable: v1alpha5.ArchitectureArm64}, -// })) { -// node := ExpectScheduled(ctx, env.Client, pod) -// nodeNames.Insert(node.Name) -// } -// Expect(nodeNames.Len()).To(Equal(2)) -// }) -// It("should exclude instance types that are not supported by the pod constraints (node affinity/instance type)", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.ArchitectureAmd64}, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ -// NodeRequirements: []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelInstanceTypeStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"arm-instance-type"}, -// }, -// }})) -// // arm instance type conflicts with the provisioner limitation of AMD only -// ExpectNotScheduled(ctx, env.Client, pod[0]) -// }) -// It("should exclude instance types that are not supported by the pod constraints (node affinity/operating system)", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.ArchitectureAmd64}, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ -// NodeRequirements: []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelOSStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"ios"}, -// }, -// }})) -// // there's an instance with an OS of ios, but it has an arm processor so the provider requirements will -// // exclude it -// ExpectNotScheduled(ctx, env.Client, pod[0]) -// }) -// It("should exclude instance types that are not supported by the provider constraints (arch)", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.ArchitectureAmd64}, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{v1.ResourceCPU: resource.MustParse("14")}}})) -// // only the ARM instance has enough CPU, but it's not allowed per the provisioner -// ExpectNotScheduled(ctx, env.Client, pod[0]) -// }) -// It("should launch pods with different operating systems on different instances", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.ArchitectureArm64, v1alpha5.ArchitectureAmd64}, -// }} -// nodeNames := sets.NewString() -// ExpectApplied(ctx, env.Client, provisioner) -// for _, pod := range ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ -// NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Linux)}, -// }), -// test.UnschedulablePod(test.PodOptions{ -// NodeSelector: map[string]string{v1.LabelOSStable: string(v1.Windows)}, -// })) { -// node := ExpectScheduled(ctx, env.Client, pod) -// nodeNames.Insert(node.Name) -// } -// Expect(nodeNames.Len()).To(Equal(2)) -// }) -// It("should launch pods with different instance type node selectors on different instances", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.ArchitectureArm64, v1alpha5.ArchitectureAmd64}, -// }} -// nodeNames := sets.NewString() -// ExpectApplied(ctx, env.Client, provisioner) -// for _, pod := range ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ -// NodeSelector: map[string]string{v1.LabelInstanceType: "small-instance-type"}, -// }), -// test.UnschedulablePod(test.PodOptions{ -// NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "default-instance-type"}, -// })) { -// node := ExpectScheduled(ctx, env.Client, pod) -// nodeNames.Insert(node.Name) -// } -// Expect(nodeNames.Len()).To(Equal(2)) -// }) -// It("should launch pods with different zone selectors on different instances", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{{ -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{v1alpha5.ArchitectureArm64, v1alpha5.ArchitectureAmd64}, -// }} -// nodeNames := sets.NewString() -// ExpectApplied(ctx, env.Client, provisioner) -// for _, pod := range ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}, -// }), -// test.UnschedulablePod(test.PodOptions{ -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}, -// })) { -// node := ExpectScheduled(ctx, env.Client, pod) -// nodeNames.Insert(node.Name) -// } -// Expect(nodeNames.Len()).To(Equal(2)) -// }) -// It("should launch pods with resources that aren't on any single instance type on different instances", func() { -// cloudProv.InstanceTypes = fake.InstanceTypes(5) -// const fakeGPU1 = "karpenter.sh/super-great-gpu" -// const fakeGPU2 = "karpenter.sh/even-better-gpu" -// cloudProv.InstanceTypes[0].Capacity[fakeGPU1] = resource.MustParse("25") -// cloudProv.InstanceTypes[1].Capacity[fakeGPU2] = resource.MustParse("25") -// -// nodeNames := sets.NewString() -// ExpectApplied(ctx, env.Client, provisioner) -// for _, pod := range ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: v1.ResourceRequirements{ -// Limits: v1.ResourceList{fakeGPU1: resource.MustParse("1")}, -// }, -// }), -// // Should pack onto a different instance since no instance type has both GPUs -// test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: v1.ResourceRequirements{ -// Limits: v1.ResourceList{fakeGPU2: resource.MustParse("1")}, -// }, -// })) { -// node := ExpectScheduled(ctx, env.Client, pod) -// nodeNames.Insert(node.Name) -// } -// Expect(nodeNames.Len()).To(Equal(2)) -// }) -// It("should fail to schedule a pod with resources requests that aren't on a single instance type", func() { -// cloudProv.InstanceTypes = fake.InstanceTypes(5) -// const fakeGPU1 = "karpenter.sh/super-great-gpu" -// const fakeGPU2 = "karpenter.sh/even-better-gpu" -// cloudProv.InstanceTypes[0].Capacity[fakeGPU1] = resource.MustParse("25") -// cloudProv.InstanceTypes[1].Capacity[fakeGPU2] = resource.MustParse("25") -// -// ExpectApplied(ctx, env.Client, provisioner) -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: v1.ResourceRequirements{ -// Limits: v1.ResourceList{ -// fakeGPU1: resource.MustParse("1"), -// fakeGPU2: resource.MustParse("1")}, -// }, -// })) -// ExpectNotScheduled(ctx, env.Client, pods[0]) -// }) -// Context("Provider Specific Labels", func() { -// It("should filter instance types that match labels", func() { -// cloudProv.InstanceTypes = fake.InstanceTypes(5) -// ExpectApplied(ctx, env.Client, provisioner) -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{fake.LabelInstanceSize: "large"}}), -// test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{fake.LabelInstanceSize: "small"}}), -// ) -// node := ExpectScheduled(ctx, env.Client, pods[0]) -// Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "fake-it-4")) -// node = ExpectScheduled(ctx, env.Client, pods[1]) -// Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, "fake-it-0")) -// }) -// It("should not schedule with incompatible labels", func() { -// cloudProv.InstanceTypes = fake.InstanceTypes(5) -// ExpectApplied(ctx, env.Client, provisioner) -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{ -// fake.LabelInstanceSize: "large", -// v1.LabelInstanceTypeStable: cloudProv.InstanceTypes[0].Name, -// }}), -// test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{ -// fake.LabelInstanceSize: "small", -// v1.LabelInstanceTypeStable: cloudProv.InstanceTypes[4].Name, -// }}), -// ) -// ExpectNotScheduled(ctx, env.Client, pods[0]) -// ExpectNotScheduled(ctx, env.Client, pods[1]) -// }) -// It("should schedule optional labels", func() { -// cloudProv.InstanceTypes = fake.InstanceTypes(5) -// ExpectApplied(ctx, env.Client, provisioner) -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ -// // Only some instance types have this key -// {Key: fake.ExoticInstanceLabelKey, Operator: v1.NodeSelectorOpExists}, -// }}), -// ) -// node := ExpectScheduled(ctx, env.Client, pods[0]) -// Expect(node.Labels).To(HaveKey(fake.ExoticInstanceLabelKey)) -// Expect(node.Labels).To(HaveKeyWithValue(v1.LabelInstanceTypeStable, cloudProv.InstanceTypes[4].Name)) -// }) -// It("should schedule without optional labels if disallowed", func() { -// cloudProv.InstanceTypes = fake.InstanceTypes(5) -// ExpectApplied(ctx, env.Client, test.Provisioner()) -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ -// // Only some instance types have this key -// {Key: fake.ExoticInstanceLabelKey, Operator: v1.NodeSelectorOpDoesNotExist}, -// }}), -// ) -// node := ExpectScheduled(ctx, env.Client, pods[0]) -// Expect(node.Labels).ToNot(HaveKey(fake.ExoticInstanceLabelKey)) -// }) -// }) -//}) -// -//var _ = Describe("Networking constraints", func() { -// Context("HostPort", func() { -// It("shouldn't co-locate pods that use the same HostPort and protocol (default protocol)", func() { -// port := v1.ContainerPort{ -// Name: "test-port", -// HostPort: 80, -// ContainerPort: 1234, -// } -// pod1 := test.UnschedulablePod() -// pod1.Spec.Containers[0].Ports = append(pod1.Spec.Containers[0].Ports, port) -// pod2 := test.UnschedulablePod() -// pod2.Spec.Containers[0].Ports = append(pod2.Spec.Containers[0].Ports, port) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod1, pod2) -// node1 := ExpectScheduled(ctx, env.Client, pod1) -// node2 := ExpectScheduled(ctx, env.Client, pod2) -// Expect(node1.Name).ToNot(Equal(node2.Name)) -// }) -// It("shouldn't co-locate pods that use the same HostPort and protocol (specific protocol)", func() { -// port := v1.ContainerPort{ -// Name: "test-port", -// HostPort: 80, -// ContainerPort: 1234, -// Protocol: "UDP", -// } -// pod1 := test.UnschedulablePod() -// pod1.Spec.Containers[0].Ports = append(pod1.Spec.Containers[0].Ports, port) -// pod2 := test.UnschedulablePod() -// pod2.Spec.Containers[0].Ports = append(pod2.Spec.Containers[0].Ports, port) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod1, pod2) -// node1 := ExpectScheduled(ctx, env.Client, pod1) -// node2 := ExpectScheduled(ctx, env.Client, pod2) -// Expect(node1.Name).ToNot(Equal(node2.Name)) -// }) -// It("shouldn't co-locate pods that use the same HostPort and IP (default (_))", func() { -// port := v1.ContainerPort{ -// Name: "test-port", -// HostPort: 80, -// ContainerPort: 1234, -// } -// pod1 := test.UnschedulablePod() -// pod1.Spec.Containers[0].Ports = append(pod1.Spec.Containers[0].Ports, port) -// port.HostIP = "1.2.3.4" // Defaulted "0.0.0.0" on pod1 should conflict -// pod2 := test.UnschedulablePod() -// pod2.Spec.Containers[0].Ports = append(pod2.Spec.Containers[0].Ports, port) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod1, pod2) -// node1 := ExpectScheduled(ctx, env.Client, pod1) -// node2 := ExpectScheduled(ctx, env.Client, pod2) -// Expect(node1.Name).ToNot(Equal(node2.Name)) -// }) -// It("shouldn't co-locate pods that use the same HostPort but a different IP, where one ip is 0.0.0.0", func() { -// port := v1.ContainerPort{ -// Name: "test-port", -// HostPort: 80, -// ContainerPort: 1234, -// Protocol: "TCP", -// HostIP: "1.2.3.4", -// } -// pod1 := test.UnschedulablePod() -// pod1.Spec.Containers[0].Ports = append(pod1.Spec.Containers[0].Ports, port) -// pod2 := test.UnschedulablePod() -// port.HostIP = "0.0.0.0" // all interfaces -// pod2.Spec.Containers[0].Ports = append(pod2.Spec.Containers[0].Ports, port) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod1, pod2) -// node1 := ExpectScheduled(ctx, env.Client, pod1) -// node2 := ExpectScheduled(ctx, env.Client, pod2) -// Expect(node1.Name).ToNot(Equal(node2.Name)) -// }) -// It("shouldn't co-locate pods that use the same HostPort but a different IP, where one ip is 0.0.0.0 (existingNodes)", func() { -// port := v1.ContainerPort{ -// Name: "test-port", -// HostPort: 80, -// ContainerPort: 1234, -// Protocol: "TCP", -// HostIP: "1.2.3.4", -// } -// pod1 := test.UnschedulablePod() -// pod1.Spec.Containers[0].Ports = append(pod1.Spec.Containers[0].Ports, port) -// pod2 := test.UnschedulablePod() -// port.HostIP = "0.0.0.0" // all interfaces -// pod2.Spec.Containers[0].Ports = append(pod2.Spec.Containers[0].Ports, port) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod1) -// node1 := ExpectScheduled(ctx, env.Client, pod1) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod2) -// node2 := ExpectScheduled(ctx, env.Client, pod2) -// Expect(node1.Name).ToNot(Equal(node2.Name)) -// }) -// It("should co-locate pods that use the same HostPort but a different protocol", func() { -// port := v1.ContainerPort{ -// Name: "test-port", -// HostPort: 80, -// ContainerPort: 1234, -// Protocol: "TCP", -// } -// pod1 := test.UnschedulablePod() -// pod1.Spec.Containers[0].Ports = append(pod1.Spec.Containers[0].Ports, port) -// pod2 := test.UnschedulablePod() -// port.Protocol = "UDP" -// pod2.Spec.Containers[0].Ports = append(pod2.Spec.Containers[0].Ports, port) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod1, pod2) -// node1 := ExpectScheduled(ctx, env.Client, pod1) -// node2 := ExpectScheduled(ctx, env.Client, pod2) -// Expect(node1.Name).To(Equal(node2.Name)) -// }) -// It("should co-locate pods that use the same HostPort but a different IP", func() { -// port := v1.ContainerPort{ -// Name: "test-port", -// HostPort: 80, -// ContainerPort: 1234, -// Protocol: "TCP", -// HostIP: "1.2.3.4", -// } -// pod1 := test.UnschedulablePod() -// pod1.Spec.Containers[0].Ports = append(pod1.Spec.Containers[0].Ports, port) -// pod2 := test.UnschedulablePod() -// port.HostIP = "4.5.6.7" -// pod2.Spec.Containers[0].Ports = append(pod2.Spec.Containers[0].Ports, port) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod1, pod2) -// node1 := ExpectScheduled(ctx, env.Client, pod1) -// node2 := ExpectScheduled(ctx, env.Client, pod2) -// Expect(node1.Name).To(Equal(node2.Name)) -// }) -// It("should co-locate pods that don't use HostPort", func() { -// port := v1.ContainerPort{ -// Name: "test-port", -// ContainerPort: 1234, -// Protocol: "TCP", -// } -// pod1 := test.UnschedulablePod() -// pod1.Spec.Containers[0].Ports = append(pod1.Spec.Containers[0].Ports, port) -// pod2 := test.UnschedulablePod() -// pod2.Spec.Containers[0].Ports = append(pod2.Spec.Containers[0].Ports, port) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod1, pod2) -// node1 := ExpectScheduled(ctx, env.Client, pod1) -// node2 := ExpectScheduled(ctx, env.Client, pod2) -// Expect(node1.Name).To(Equal(node2.Name)) -// }) -// }) -//}) -// -//var _ = Describe("Binpacking", func() { -// It("should schedule a small pod on the smallest instance", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceMemory: resource.MustParse("100M"), -// }, -// }}))[0] -// node := ExpectScheduled(ctx, env.Client, pod) -// Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) -// }) -// It("should schedule a small pod on the smallest possible instance type", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceMemory: resource.MustParse("2000M"), -// }, -// }}))[0] -// node := ExpectScheduled(ctx, env.Client, pod) -// Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) -// }) -// It("should schedule multiple small pods on the smallest possible instance type", func() { -// opts := test.PodOptions{ -// Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceMemory: resource.MustParse("10M"), -// }, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.Pods(5, opts)...) -// nodeNames := sets.NewString() -// for _, p := range pods { -// node := ExpectScheduled(ctx, env.Client, p) -// nodeNames.Insert(node.Name) -// Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) -// } -// Expect(nodeNames).To(HaveLen(1)) -// }) -// It("should create new nodes when a node is at capacity", func() { -// opts := test.PodOptions{ -// NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, -// Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceMemory: resource.MustParse("1.8G"), -// }, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.Pods(40, opts)...) -// nodeNames := sets.NewString() -// for _, p := range pods { -// node := ExpectScheduled(ctx, env.Client, p) -// nodeNames.Insert(node.Name) -// Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("default-instance-type")) -// } -// Expect(nodeNames).To(HaveLen(20)) -// }) -// It("should pack small and large pods together", func() { -// largeOpts := test.PodOptions{ -// NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, -// Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceMemory: resource.MustParse("1.8G"), -// }, -// }} -// smallOpts := test.PodOptions{ -// NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, -// Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceMemory: resource.MustParse("400M"), -// }, -// }} -// -// // Two large pods are all that will fit on the default-instance type (the largest instance type) which will create -// // twenty nodes. This leaves just enough room on each of those newNodes for one additional small pod per node, so we -// // should only end up with 20 newNodes total. -// provPods := append(test.Pods(40, largeOpts), test.Pods(20, smallOpts)...) -// ExpectApplied(ctx, env.Client, provisioner) -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, provPods...) -// nodeNames := sets.NewString() -// for _, p := range pods { -// node := ExpectScheduled(ctx, env.Client, p) -// nodeNames.Insert(node.Name) -// Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("default-instance-type")) -// } -// Expect(nodeNames).To(HaveLen(20)) -// }) -// It("should pack newNodes tightly", func() { -// cloudProv.InstanceTypes = fake.InstanceTypes(5) -// var nodes []*v1.Node -// ExpectApplied(ctx, env.Client, provisioner) -// for _, pod := range ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("4.5")}, -// }, -// }), -// test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, -// }, -// })) { -// node := ExpectScheduled(ctx, env.Client, pod) -// nodes = append(nodes, node) -// } -// Expect(nodes).To(HaveLen(2)) -// // the first pod consumes nearly all CPU of the largest instance type with no room for the second pod, the -// // second pod is much smaller in terms of resources and should get a smaller node -// Expect(nodes[0].Labels[v1.LabelInstanceTypeStable]).ToNot(Equal(nodes[1].Labels[v1.LabelInstanceTypeStable])) -// }) -// It("should handle zero-quantity resource requests", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: v1.ResourceList{"foo.com/weird-resources": resource.MustParse("0")}, -// Limits: v1.ResourceList{"foo.com/weird-resources": resource.MustParse("0")}, -// }, -// })) -// // requesting a resource of quantity zero of a type unsupported by any instance is fine -// ExpectScheduled(ctx, env.Client, pod[0]) -// }) -// It("should not schedule pods that exceed every instance type's capacity", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceMemory: resource.MustParse("2Ti"), -// }, -// }}))[0] -// ExpectNotScheduled(ctx, env.Client, pod) -// }) -// It("should create new newNodes when a node is at capacity due to pod limits per node", func() { -// opts := test.PodOptions{ -// NodeSelector: map[string]string{v1.LabelArchStable: "amd64"}, -// Conditions: []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}}, -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceMemory: resource.MustParse("1m"), -// v1.ResourceCPU: resource.MustParse("1m"), -// }, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.Pods(25, opts)...) -// nodeNames := sets.NewString() -// // all of the test instance types support 5 pods each, so we use the 5 instances of the smallest one for our 25 pods -// for _, p := range pods { -// node := ExpectScheduled(ctx, env.Client, p) -// nodeNames.Insert(node.Name) -// Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("small-instance-type")) -// } -// Expect(nodeNames).To(HaveLen(5)) -// }) -// It("should take into account initContainer resource requests when binpacking", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceMemory: resource.MustParse("1Gi"), -// v1.ResourceCPU: resource.MustParse("1"), -// }, -// }, -// InitImage: "pause", -// InitResourceRequirements: v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceMemory: resource.MustParse("1Gi"), -// v1.ResourceCPU: resource.MustParse("2"), -// }, -// }, -// }))[0] -// node := ExpectScheduled(ctx, env.Client, pod) -// Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("default-instance-type")) -// }) -// It("should not schedule pods when initContainer resource requests are greater than available instance types", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceMemory: resource.MustParse("1Gi"), -// v1.ResourceCPU: resource.MustParse("1"), -// }, -// }, -// InitImage: "pause", -// InitResourceRequirements: v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceMemory: resource.MustParse("1Ti"), -// v1.ResourceCPU: resource.MustParse("2"), -// }, -// }, -// }))[0] -// ExpectNotScheduled(ctx, env.Client, pod) -// }) -// It("should select for valid instance types, regardless of price", func() { -// // capacity sizes and prices don't correlate here, regardless we should filter and see that all three instance types -// // are valid before preferring the cheapest one 'large' -// cloudProv.InstanceTypes = []*cloudprovider.InstanceType{ -// fake.NewInstanceType(fake.InstanceTypeOptions{ -// Name: "medium", -// Resources: v1.ResourceList{ -// v1.ResourceCPU: resource.MustParse("2"), -// v1.ResourceMemory: resource.MustParse("2Gi"), -// }, -// Offerings: []cloudprovider.Offering{ -// { -// CapacityType: v1alpha5.CapacityTypeOnDemand, -// Zone: "test-zone-1a", -// Price: 3.00, -// Available: true, -// }, -// }, -// }), -// fake.NewInstanceType(fake.InstanceTypeOptions{ -// Name: "small", -// Resources: v1.ResourceList{ -// v1.ResourceCPU: resource.MustParse("1"), -// v1.ResourceMemory: resource.MustParse("1Gi"), -// }, -// Offerings: []cloudprovider.Offering{ -// { -// CapacityType: v1alpha5.CapacityTypeOnDemand, -// Zone: "test-zone-1a", -// Price: 2.00, -// Available: true, -// }, -// }, -// }), -// fake.NewInstanceType(fake.InstanceTypeOptions{ -// Name: "large", -// Resources: v1.ResourceList{ -// v1.ResourceCPU: resource.MustParse("4"), -// v1.ResourceMemory: resource.MustParse("4Gi"), -// }, -// Offerings: []cloudprovider.Offering{ -// { -// CapacityType: v1alpha5.CapacityTypeOnDemand, -// Zone: "test-zone-1a", -// Price: 1.00, -// Available: true, -// }, -// }, -// }), -// } -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("1m"), -// v1.ResourceMemory: resource.MustParse("1Mi"), -// }, -// }}, -// )) -// node := ExpectScheduled(ctx, env.Client, pod[0]) -// // large is the cheapest, so we should pick it, but the other two types are also valid options -// Expect(node.Labels[v1.LabelInstanceTypeStable]).To(Equal("large")) -// // all three options should be passed to the cloud provider -// possibleInstanceType := sets.NewString(pscheduling.NewNodeSelectorRequirements(cloudProv.CreateCalls[0].Spec.Requirements...).Get(v1.LabelInstanceTypeStable).Values()...) -// Expect(possibleInstanceType).To(Equal(sets.NewString("small", "medium", "large"))) -// }) -//}) -// -//var _ = Describe("In-Flight Nodes", func() { -// It("should not launch a second node if there is an in-flight node that can support the pod", func() { -// opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("10m"), -// }, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// initialPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(opts)) -// node1 := ExpectScheduled(ctx, env.Client, initialPod[0]) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// -// secondPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(opts)) -// node2 := ExpectScheduled(ctx, env.Client, secondPod[0]) -// Expect(node1.Name).To(Equal(node2.Name)) -// }) -// It("should not launch a second node if there is an in-flight node that can support the pod (node selectors)", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// initialPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("10m"), -// }, -// }, -// NodeRequirements: []v1.NodeSelectorRequirement{{ -// Key: v1.LabelTopologyZone, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"test-zone-2"}, -// }}})) -// node1 := ExpectScheduled(ctx, env.Client, initialPod[0]) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// -// // the node gets created in test-zone-2 -// secondPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("10m"), -// }, -// }, -// NodeRequirements: []v1.NodeSelectorRequirement{{ -// Key: v1.LabelTopologyZone, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"test-zone-1", "test-zone-2"}, -// }}})) -// // test-zone-2 is in the intersection of their node selectors and the node has capacity, so we shouldn't create a new node -// node2 := ExpectScheduled(ctx, env.Client, secondPod[0]) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// Expect(node1.Name).To(Equal(node2.Name)) -// -// // the node gets created in test-zone-2 -// thirdPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("10m"), -// }, -// }, -// NodeRequirements: []v1.NodeSelectorRequirement{{ -// Key: v1.LabelTopologyZone, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"test-zone-1", "test-zone-3"}, -// }}})) -// // node is in test-zone-2, so this pod needs a new node -// node3 := ExpectScheduled(ctx, env.Client, thirdPod[0]) -// Expect(node1.Name).ToNot(Equal(node3.Name)) -// }) -// It("should launch a second node if a pod won't fit on the existingNodes node", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("1001m"), -// }, -// }} -// initialPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(opts)) -// node1 := ExpectScheduled(ctx, env.Client, initialPod[0]) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// -// // the node will have 2000m CPU, so these two pods can't both fit on it -// opts.ResourceRequirements.Limits[v1.ResourceCPU] = resource.MustParse("1") -// secondPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(opts)) -// node2 := ExpectScheduled(ctx, env.Client, secondPod[0]) -// Expect(node1.Name).ToNot(Equal(node2.Name)) -// }) -// It("should launch a second node if a pod isn't compatible with the existingNodes node (node selector)", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("10m"), -// }, -// }} -// initialPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(opts)) -// node1 := ExpectScheduled(ctx, env.Client, initialPod[0]) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// -// secondPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "arm64"}})) -// node2 := ExpectScheduled(ctx, env.Client, secondPod[0]) -// Expect(node1.Name).ToNot(Equal(node2.Name)) -// }) -// It("should launch a second node if an in-flight node is terminating", func() { -// opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("10m"), -// }, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// initialPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(opts)) -// node1 := ExpectScheduled(ctx, env.Client, initialPod[0]) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// -// // delete the node -// node1.Finalizers = nil -// ExpectApplied(ctx, env.Client, node1) -// ExpectDeleted(ctx, env.Client, node1) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// -// secondPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(opts)) -// node2 := ExpectScheduled(ctx, env.Client, secondPod[0]) -// Expect(node1.Name).ToNot(Equal(node2.Name)) -// }) -// Context("Topology", func() { -// It("should balance pods across zones with in-flight newNodes", func() { -// labels := map[string]string{"foo": "bar"} -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 2)) -// -// // reconcile our newNodes with the cluster state so they'll show up as in-flight -// var nodeList v1.NodeList -// Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) -// for _, node := range nodeList.Items { -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKey{Name: node.Name}) -// } -// -// firstRoundNumNodes := len(nodeList.Items) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 3, 3)) -// Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) -// -// // shouldn't create any new newNodes as the in-flight ones can support the pods -// Expect(nodeList.Items).To(HaveLen(firstRoundNumNodes)) -// }) -// It("should balance pods across hostnames with in-flight newNodes", func() { -// labels := map[string]string{"foo": "bar"} -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 1, 1)) -// -// // reconcile our newNodes with the cluster state so they'll show up as in-flight -// var nodeList v1.NodeList -// Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) -// for _, node := range nodeList.Items { -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKey{Name: node.Name}) -// } -// -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// ) -// // we prefer to launch new newNodes to satisfy the topology spread even though we could technnically schedule against existingNodes -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 1, 1, 1, 1, 1, 1, 1)) -// }) -// }) -// Context("Taints", func() { -// It("should assume pod will schedule to a tainted node with no taints", func() { -// opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("8"), -// }, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// initialPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(opts)) -// node1 := ExpectScheduled(ctx, env.Client, initialPod[0]) -// -// // delete the pod so that the node is empty -// ExpectDeleted(ctx, env.Client, initialPod[0]) -// node1.Spec.Taints = nil -// ExpectApplied(ctx, env.Client, node1) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// -// secondPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node2 := ExpectScheduled(ctx, env.Client, secondPod[0]) -// Expect(node1.Name).To(Equal(node2.Name)) -// }) -// It("should not assume pod will schedule to a tainted node", func() { -// opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("8"), -// }, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// initialPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(opts)) -// node1 := ExpectScheduled(ctx, env.Client, initialPod[0]) -// -// // delete the pod so that the node is empty -// ExpectDeleted(ctx, env.Client, initialPod[0]) -// // and taint it -// node1.Spec.Taints = append(node1.Spec.Taints, v1.Taint{ -// Key: "foo.com/taint", -// Value: "tainted", -// Effect: v1.TaintEffectNoSchedule, -// }) -// ExpectApplied(ctx, env.Client, node1) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// -// secondPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node2 := ExpectScheduled(ctx, env.Client, secondPod[0]) -// Expect(node1.Name).ToNot(Equal(node2.Name)) -// }) -// It("should assume pod will schedule to a tainted node with a custom startup taint", func() { -// opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("8"), -// }, -// }} -// provisioner.Spec.StartupTaints = append(provisioner.Spec.StartupTaints, v1.Taint{ -// Key: "foo.com/taint", -// Value: "tainted", -// Effect: v1.TaintEffectNoSchedule, -// }) -// ExpectApplied(ctx, env.Client, provisioner) -// initialPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(opts)) -// node1 := ExpectScheduled(ctx, env.Client, initialPod[0]) -// -// // delete the pod so that the node is empty -// ExpectDeleted(ctx, env.Client, initialPod[0]) -// // startup taint + node not ready taint = 2 -// Expect(node1.Spec.Taints).To(HaveLen(2)) -// Expect(node1.Spec.Taints).To(ContainElement(v1.Taint{ -// Key: "foo.com/taint", -// Value: "tainted", -// Effect: v1.TaintEffectNoSchedule, -// })) -// ExpectApplied(ctx, env.Client, node1) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// -// secondPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node2 := ExpectScheduled(ctx, env.Client, secondPod[0]) -// Expect(node1.Name).To(Equal(node2.Name)) -// }) -// It("should not assume pod will schedule to a node with startup taints after initialization", func() { -// startupTaint := v1.Taint{Key: "ignore-me", Value: "nothing-to-see-here", Effect: v1.TaintEffectNoSchedule} -// provisioner.Spec.StartupTaints = []v1.Taint{startupTaint} -// ExpectApplied(ctx, env.Client, provisioner) -// initialPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node1 := ExpectScheduled(ctx, env.Client, initialPod[0]) -// -// // delete the pod so that the node is empty -// ExpectDeleted(ctx, env.Client, initialPod[0]) -// -// // Mark it initialized which only occurs once the startup taint was removed and re-apply only the startup taint. -// // We also need to add resource capacity as after initialization we assume that kubelet has recorded them. -// node1.Labels[v1alpha5.LabelNodeInitialized] = "true" -// node1.Spec.Taints = []v1.Taint{startupTaint} -// node1.Status.Capacity = v1.ResourceList{v1.ResourcePods: resource.MustParse("10")} -// ExpectApplied(ctx, env.Client, node1) -// -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// -// // we should launch a new node since the startup taint is there, but was gone at some point -// secondPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node2 := ExpectScheduled(ctx, env.Client, secondPod[0]) -// Expect(node1.Name).ToNot(Equal(node2.Name)) -// }) -// It("should consider a tainted NotReady node as in-flight even if initialized", func() { -// opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{v1.ResourceCPU: resource.MustParse("10m")}, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// -// // Schedule to New Machine -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(opts))[0] -// node1 := ExpectScheduled(ctx, env.Client, pod) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// // Mark Initialized -// node1.Labels[v1alpha5.LabelNodeInitialized] = "true" -// node1.Spec.Taints = []v1.Taint{ -// {Key: v1.TaintNodeNotReady, Effect: v1.TaintEffectNoSchedule}, -// {Key: v1.TaintNodeUnreachable, Effect: v1.TaintEffectNoSchedule}, -// } -// ExpectApplied(ctx, env.Client, node1) -// // Schedule to In Flight Machine -// pod = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(opts))[0] -// node2 := ExpectScheduled(ctx, env.Client, pod) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node2)) -// -// Expect(node1.Name).To(Equal(node2.Name)) -// }) -// }) -// Context("Daemonsets", func() { -// It("should track daemonset usage separately so we know how many DS resources are remaining to be scheduled", func() { -// ds := test.DaemonSet( -// test.DaemonSetOptions{PodOptions: test.PodOptions{ -// ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ -// v1.ResourceCPU: resource.MustParse("1"), -// v1.ResourceMemory: resource.MustParse("1Gi")}}, -// }}, -// ) -// ExpectApplied(ctx, env.Client, provisioner, ds) -// Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(ds), ds)).To(Succeed()) -// -// opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("8"), -// }, -// }} -// initialPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(opts)) -// node1 := ExpectScheduled(ctx, env.Client, initialPod[0]) -// -// // create our daemonset pod and manually bind it to the node -// dsPod := test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("1"), -// v1.ResourceMemory: resource.MustParse("2Gi"), -// }}, -// }) -// dsPod.OwnerReferences = append(dsPod.OwnerReferences, metav1.OwnerReference{ -// APIVersion: "apps/v1", -// Kind: "DaemonSet", -// Name: ds.Name, -// UID: ds.UID, -// Controller: ptr.Bool(true), -// BlockOwnerDeletion: ptr.Bool(true), -// }) -// -// // delete the pod so that the node is empty -// ExpectDeleted(ctx, env.Client, initialPod[0]) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// -// ExpectApplied(ctx, env.Client, provisioner, dsPod) -// cluster.ForEachNode(func(f *state.Node) bool { -// dsRequests := f.DaemonSetRequests() -// available := f.Available() -// Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 0)) -// // no pods so we have the full (16 cpu - 100m overhead) -// Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 15.9)) -// return true -// }) -// ExpectManualBinding(ctx, env.Client, dsPod, node1) -// ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(dsPod)) -// -// cluster.ForEachNode(func(f *state.Node) bool { -// dsRequests := f.DaemonSetRequests() -// available := f.Available() -// Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 1)) -// // only the DS pod is bound, so available is reduced by one and the DS requested is incremented by one -// Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 14.9)) -// return true -// }) -// -// opts = test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("14.9"), -// }, -// }} -// // this pod should schedule on the existingNodes node as the daemonset pod has already bound, meaning that the -// // remaining daemonset resources should be zero leaving 14.9 CPUs for the pod -// secondPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(opts)) -// node2 := ExpectScheduled(ctx, env.Client, secondPod[0]) -// Expect(node1.Name).To(Equal(node2.Name)) -// }) -// It("should handle unexpected daemonset pods binding to the node", func() { -// ds1 := test.DaemonSet( -// test.DaemonSetOptions{PodOptions: test.PodOptions{ -// NodeSelector: map[string]string{ -// "my-node-label": "value", -// }, -// ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ -// v1.ResourceCPU: resource.MustParse("1"), -// v1.ResourceMemory: resource.MustParse("1Gi")}}, -// }}, -// ) -// ds2 := test.DaemonSet( -// test.DaemonSetOptions{PodOptions: test.PodOptions{ -// ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ -// v1.ResourceCPU: resource.MustParse("1m"), -// }}}}) -// ExpectApplied(ctx, env.Client, provisioner, ds1, ds2) -// Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(ds1), ds1)).To(Succeed()) -// -// opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("8"), -// }, -// }} -// initialPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(opts)) -// node1 := ExpectScheduled(ctx, env.Client, initialPod[0]) -// // this label appears on the node for some reason that Karpenter can't track -// node1.Labels["my-node-label"] = "value" -// ExpectApplied(ctx, env.Client, node1) -// -// // create our daemonset pod and manually bind it to the node -// dsPod := test.UnschedulablePod(test.PodOptions{ -// NodeSelector: map[string]string{ -// "my-node-label": "value", -// }, -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("1"), -// v1.ResourceMemory: resource.MustParse("2Gi"), -// }}, -// }) -// dsPod.OwnerReferences = append(dsPod.OwnerReferences, metav1.OwnerReference{ -// APIVersion: "apps/v1", -// Kind: "DaemonSet", -// Name: ds1.Name, -// UID: ds1.UID, -// Controller: ptr.Bool(true), -// BlockOwnerDeletion: ptr.Bool(true), -// }) -// -// // delete the pod so that the node is empty -// ExpectDeleted(ctx, env.Client, initialPod[0]) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// -// ExpectApplied(ctx, env.Client, provisioner, dsPod) -// cluster.ForEachNode(func(f *state.Node) bool { -// dsRequests := f.DaemonSetRequests() -// available := f.Available() -// Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 0)) -// // no pods, so we have the full (16 CPU - 100m overhead) -// Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 15.9)) -// return true -// }) -// ExpectManualBinding(ctx, env.Client, dsPod, node1) -// ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(dsPod)) -// -// cluster.ForEachNode(func(f *state.Node) bool { -// dsRequests := f.DaemonSetRequests() -// available := f.Available() -// Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 1)) -// // only the DS pod is bound, so available is reduced by one and the DS requested is incremented by one -// Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 14.9)) -// return true -// }) -// -// opts = test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("15.5"), -// }, -// }} -// // This pod should not schedule on the inflight node as it requires more CPU than we have. This verifies -// // we don't reintroduce a bug where more daemonsets scheduled than anticipated due to unexepected labels -// // appearing on the node which caused us to compute a negative amount of resources remaining for daemonsets -// // which in turn caused us to mis-calculate the amount of resources that were free on the node. -// secondPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(opts)) -// node2 := ExpectScheduled(ctx, env.Client, secondPod[0]) -// // must create a new node -// Expect(node1.Name).ToNot(Equal(node2.Name)) -// }) -// -// }) -// // nolint:gosec -// It("should pack in-flight newNodes before launching new newNodes", func() { -// cloudProv.InstanceTypes = []*cloudprovider.InstanceType{ -// fake.NewInstanceType(fake.InstanceTypeOptions{ -// Name: "medium", -// Resources: v1.ResourceList{ -// // enough CPU for four pods + a bit of overhead -// v1.ResourceCPU: resource.MustParse("4.25"), -// v1.ResourcePods: resource.MustParse("4"), -// }, -// }), -// } -// opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("1"), -// }, -// }} -// -// ExpectApplied(ctx, env.Client, provisioner) -// -// // scheduling in multiple batches random sets of pods -// for i := 0; i < 10; i++ { -// initialPods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, MakePods(rand.Intn(10), opts)...) -// for _, pod := range initialPods { -// node := ExpectScheduled(ctx, env.Client, pod) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) -// } -// } -// -// // due to the in-flight node support, we should pack existing newNodes before launching new node. The end result -// // is that we should only have some spare capacity on our final node -// nodesWithCPUFree := 0 -// cluster.ForEachNode(func(n *state.Node) bool { -// available := n.Available() -// if available.Cpu().AsApproximateFloat64() >= 1 { -// nodesWithCPUFree++ -// } -// return true -// }) -// Expect(nodesWithCPUFree).To(BeNumerically("<=", 1)) -// }) -// It("should not launch a second node if there is an in-flight node that can support the pod (#2011)", func() { -// opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("10m"), -// }, -// }} -// -// // there was a bug in cluster state where we failed to identify the instance type resources when using a -// // ProviderRef so modify our provisioner to use the ProviderRef and ensure that the second pod schedules -// // to the existingNodes node -// provisioner.Spec.Provider = nil -// provisioner.Spec.ProviderRef = &v1alpha5.ProviderRef{} -// -// ExpectApplied(ctx, env.Client, provisioner) -// pod := test.UnschedulablePod(opts) -// ExpectProvisionedNoBinding(ctx, env.Client, provisioningController, prov, pod) -// var nodes v1.NodeList -// Expect(env.Client.List(ctx, &nodes)).To(Succeed()) -// Expect(nodes.Items).To(HaveLen(1)) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(&nodes.Items[0])) -// -// pod.Status.Conditions = []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}} -// ExpectApplied(ctx, env.Client, pod) -// ExpectProvisionedNoBinding(ctx, env.Client, provisioningController, prov, pod) -// Expect(env.Client.List(ctx, &nodes)).To(Succeed()) -// // shouldn't create a second node -// Expect(nodes.Items).To(HaveLen(1)) -// }) -//}) -// -//var _ = Describe("No Pre-Binding", func() { -// It("should not bind pods to newNodes", func() { -// opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("10m"), -// }, -// }} -// -// var nodeList v1.NodeList -// // shouldn't have any newNodes -// Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) -// Expect(nodeList.Items).To(HaveLen(0)) -// -// ExpectApplied(ctx, env.Client, provisioner) -// initialPod := ExpectProvisionedNoBinding(ctx, env.Client, provisioningController, prov, test.UnschedulablePod(opts)) -// ExpectNotScheduled(ctx, env.Client, initialPod[0]) -// -// // should launch a single node -// Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) -// Expect(nodeList.Items).To(HaveLen(1)) -// node1 := &nodeList.Items[0] -// -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// secondPod := ExpectProvisionedNoBinding(ctx, env.Client, provisioningController, prov, test.UnschedulablePod(opts)) -// ExpectNotScheduled(ctx, env.Client, secondPod[0]) -// // shouldn't create a second node as it can bind to the existingNodes node -// Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) -// Expect(nodeList.Items).To(HaveLen(1)) -// }) -// It("should handle resource zeroing of extended resources by kubelet", func() { -// // Issue #1459 -// opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ -// Limits: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("10m"), -// fake.ResourceGPUVendorA: resource.MustParse("1"), -// }, -// }} -// -// var nodeList v1.NodeList -// // shouldn't have any newNodes -// Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) -// Expect(nodeList.Items).To(HaveLen(0)) -// -// ExpectApplied(ctx, env.Client, provisioner) -// initialPod := ExpectProvisionedNoBinding(ctx, env.Client, provisioningController, prov, test.UnschedulablePod(opts)) -// ExpectNotScheduled(ctx, env.Client, initialPod[0]) -// -// // should launch a single node -// Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) -// Expect(nodeList.Items).To(HaveLen(1)) -// node1 := &nodeList.Items[0] -// -// // simulate kubelet zeroing out the extended resources on the node at startup -// node1.Status.Capacity = map[v1.ResourceName]resource.Quantity{ -// fake.ResourceGPUVendorA: resource.MustParse("0"), -// } -// node1.Status.Allocatable = map[v1.ResourceName]resource.Quantity{ -// fake.ResourceGPUVendorB: resource.MustParse("0"), -// } -// -// ExpectApplied(ctx, env.Client, node1) -// -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// secondPod := ExpectProvisionedNoBinding(ctx, env.Client, provisioningController, prov, test.UnschedulablePod(opts)) -// ExpectNotScheduled(ctx, env.Client, secondPod[0]) -// // shouldn't create a second node as it can bind to the existingNodes node -// Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) -// Expect(nodeList.Items).To(HaveLen(1)) -// }) -// It("should respect self pod affinity without pod binding (zone)", func() { -// // Issue #1975 -// affLabels := map[string]string{"security": "s2"} -// -// pods := MakePods(2, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{ -// Labels: affLabels, -// }, -// PodRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelTopologyZone, -// }}, -// }) -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisionedNoBinding(ctx, env.Client, provisioningController, prov, pods[0]) -// var nodeList v1.NodeList -// Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) -// for i := range nodeList.Items { -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(&nodeList.Items[i])) -// } -// // the second pod can schedule against the in-flight node, but for that to work we need to be careful -// // in how we fulfill the self-affinity by taking the existing node's domain as a preference over any -// // random viable domain -// ExpectProvisionedNoBinding(ctx, env.Client, provisioningController, prov, pods[1]) -// Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) -// Expect(nodeList.Items).To(HaveLen(1)) -// }) -//}) -// -//var _ = Describe("VolumeUsage", func() { -// It("should launch multiple newNodes if required due to volume limits", func() { -// const csiProvider = "fake.csi.provider" -// cloudProv.InstanceTypes = []*cloudprovider.InstanceType{ -// fake.NewInstanceType( -// fake.InstanceTypeOptions{ -// Name: "instance-type", -// Resources: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("1024"), -// v1.ResourcePods: resource.MustParse("1024"), -// }, -// }), -// } -// -// provisioner.Spec.Limits = nil -// ExpectApplied(ctx, env.Client, provisioner) -// initialPods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node := ExpectScheduled(ctx, env.Client, initialPods[0]) -// csiNode := &storagev1.CSINode{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: node.Name, -// }, -// Spec: storagev1.CSINodeSpec{ -// Drivers: []storagev1.CSINodeDriver{ -// { -// Name: csiProvider, -// NodeID: "fake-node-id", -// Allocatable: &storagev1.VolumeNodeResources{ -// Count: ptr.Int32(10), -// }, -// }, -// }, -// }, -// } -// ExpectApplied(ctx, env.Client, csiNode) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) -// -// sc := test.StorageClass(test.StorageClassOptions{ -// ObjectMeta: metav1.ObjectMeta{Name: "my-storage-class"}, -// Provisioner: ptr.String(csiProvider), -// Zones: []string{"test-zone-1"}}) -// ExpectApplied(ctx, env.Client, sc) -// -// var pods []*v1.Pod -// for i := 0; i < 6; i++ { -// pvcA := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ -// StorageClassName: ptr.String("my-storage-class"), -// ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("my-claim-a-%d", i)}, -// }) -// pvcB := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ -// StorageClassName: ptr.String("my-storage-class"), -// ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("my-claim-b-%d", i)}, -// }) -// ExpectApplied(ctx, env.Client, pvcA, pvcB) -// pods = append(pods, test.UnschedulablePod(test.PodOptions{ -// PersistentVolumeClaims: []string{pvcA.Name, pvcB.Name}, -// })) -// } -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pods...) -// var nodeList v1.NodeList -// Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) -// // we need to create a new node as the in-flight one can only contain 5 pods due to the CSINode volume limit -// Expect(nodeList.Items).To(HaveLen(2)) -// }) -// It("should launch a single node if all pods use the same PVC", func() { -// const csiProvider = "fake.csi.provider" -// cloudProv.InstanceTypes = []*cloudprovider.InstanceType{ -// fake.NewInstanceType( -// fake.InstanceTypeOptions{ -// Name: "instance-type", -// Resources: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("1024"), -// v1.ResourcePods: resource.MustParse("1024"), -// }, -// }), -// } -// -// provisioner.Spec.Limits = nil -// ExpectApplied(ctx, env.Client, provisioner) -// initialPods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node := ExpectScheduled(ctx, env.Client, initialPods[0]) -// csiNode := &storagev1.CSINode{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: node.Name, -// }, -// Spec: storagev1.CSINodeSpec{ -// Drivers: []storagev1.CSINodeDriver{ -// { -// Name: csiProvider, -// NodeID: "fake-node-id", -// Allocatable: &storagev1.VolumeNodeResources{ -// Count: ptr.Int32(10), -// }, -// }, -// }, -// }, -// } -// ExpectApplied(ctx, env.Client, csiNode) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) -// -// sc := test.StorageClass(test.StorageClassOptions{ -// ObjectMeta: metav1.ObjectMeta{Name: "my-storage-class"}, -// Provisioner: ptr.String(csiProvider), -// Zones: []string{"test-zone-1"}}) -// ExpectApplied(ctx, env.Client, sc) -// -// pv := test.PersistentVolume(test.PersistentVolumeOptions{ -// ObjectMeta: metav1.ObjectMeta{Name: "my-volume"}, -// Zones: []string{"test-zone-1"}}) -// -// pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ -// ObjectMeta: metav1.ObjectMeta{Name: "my-claim"}, -// StorageClassName: ptr.String("my-storage-class"), -// VolumeName: pv.Name, -// }) -// ExpectApplied(ctx, env.Client, pv, pvc) -// -// var pods []*v1.Pod -// for i := 0; i < 100; i++ { -// pods = append(pods, test.UnschedulablePod(test.PodOptions{ -// PersistentVolumeClaims: []string{pvc.Name, pvc.Name}, -// })) -// } -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pods...) -// var nodeList v1.NodeList -// Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) -// // 100 of the same PVC should all be schedulable on the same node -// Expect(nodeList.Items).To(HaveLen(1)) -// }) -// It("should not fail for non-dynamic PVCs", func() { -// const csiProvider = "fake.csi.provider" -// cloudProv.InstanceTypes = []*cloudprovider.InstanceType{ -// fake.NewInstanceType( -// fake.InstanceTypeOptions{ -// Name: "instance-type", -// Resources: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("1024"), -// v1.ResourcePods: resource.MustParse("1024"), -// }, -// }), -// } -// -// provisioner.Spec.Limits = nil -// ExpectApplied(ctx, env.Client, provisioner) -// initialPods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node := ExpectScheduled(ctx, env.Client, initialPods[0]) -// csiNode := &storagev1.CSINode{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: node.Name, -// }, -// Spec: storagev1.CSINodeSpec{ -// Drivers: []storagev1.CSINodeDriver{ -// { -// Name: csiProvider, -// NodeID: "fake-node-id", -// Allocatable: &storagev1.VolumeNodeResources{ -// Count: ptr.Int32(10), -// }, -// }, -// }, -// }, -// } -// ExpectApplied(ctx, env.Client, csiNode) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) -// -// sc := test.StorageClass(test.StorageClassOptions{ -// ObjectMeta: metav1.ObjectMeta{Name: "my-storage-class"}, -// Provisioner: ptr.String(csiProvider), -// Zones: []string{"test-zone-1"}}) -// ExpectApplied(ctx, env.Client, sc) -// -// pv := test.PersistentVolume(test.PersistentVolumeOptions{ -// ObjectMeta: metav1.ObjectMeta{Name: "my-volume"}, -// Driver: csiProvider, -// Zones: []string{"test-zone-1"}}) -// -// pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ -// ObjectMeta: metav1.ObjectMeta{Name: "my-claim"}, -// VolumeName: pv.Name, -// StorageClassName: ptr.String(""), -// }) -// ExpectApplied(ctx, env.Client, pv, pvc) -// -// var pods []*v1.Pod -// for i := 0; i < 5; i++ { -// pods = append(pods, test.UnschedulablePod(test.PodOptions{ -// PersistentVolumeClaims: []string{pvc.Name, pvc.Name}, -// })) -// } -// ExpectApplied(ctx, env.Client, provisioner) -// _ = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pods...) -// -// var nodeList v1.NodeList -// Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) -// // 5 of the same PVC should all be schedulable on the same node -// Expect(nodeList.Items).To(HaveLen(1)) -// }) -// It("should not fail for NFS volumes", func() { -// cloudProv.InstanceTypes = []*cloudprovider.InstanceType{ -// fake.NewInstanceType( -// fake.InstanceTypeOptions{ -// Name: "instance-type", -// Resources: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("1024"), -// v1.ResourcePods: resource.MustParse("1024"), -// }, -// }), -// } -// -// provisioner.Spec.Limits = nil -// ExpectApplied(ctx, env.Client, provisioner) -// initialPods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) -// node := ExpectScheduled(ctx, env.Client, initialPods[0]) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) -// -// pv := test.PersistentVolume(test.PersistentVolumeOptions{ -// ObjectMeta: metav1.ObjectMeta{Name: "my-volume"}, -// StorageClassName: "nfs", -// Zones: []string{"test-zone-1"}}) -// pv.Spec.NFS = &v1.NFSVolumeSource{ -// Server: "fake.server", -// Path: "/some/path", -// } -// pv.Spec.CSI = nil -// -// pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ -// ObjectMeta: metav1.ObjectMeta{Name: "my-claim"}, -// VolumeName: pv.Name, -// StorageClassName: ptr.String(""), -// }) -// ExpectApplied(ctx, env.Client, pv, pvc) -// -// var pods []*v1.Pod -// for i := 0; i < 5; i++ { -// pods = append(pods, test.UnschedulablePod(test.PodOptions{ -// PersistentVolumeClaims: []string{pvc.Name, pvc.Name}, -// })) -// } -// ExpectApplied(ctx, env.Client, provisioner) -// _ = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pods...) -// -// var nodeList v1.NodeList -// Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) -// // 5 of the same PVC should all be schedulable on the same node -// Expect(nodeList.Items).To(HaveLen(1)) -// }) -// It("should not launch nodes for pods with ephemeral volume using a non-existent storage classes", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := test.UnschedulablePod(test.PodOptions{}) -// pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ -// Name: "tmp-ephemeral", -// VolumeSource: v1.VolumeSource{ -// Ephemeral: &v1.EphemeralVolumeSource{ -// VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ -// Spec: v1.PersistentVolumeClaimSpec{ -// StorageClassName: ptr.String("non-existent"), -// AccessModes: []v1.PersistentVolumeAccessMode{ -// v1.ReadWriteOnce, -// }, -// Resources: v1.ResourceRequirements{ -// Requests: v1.ResourceList{ -// v1.ResourceStorage: resource.MustParse("1Gi"), -// }, -// }, -// }, -// }, -// }, -// }, -// }) -// ExpectApplied(ctx, env.Client, provisioner) -// _ = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod) -// -// var nodeList v1.NodeList -// Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) -// // no nodes should be created as the storage class doesn't eixst -// Expect(nodeList.Items).To(HaveLen(0)) -// }) -//}) +var _ = Describe("In-Flight Nodes", func() { + It("should not launch a second node if there is an in-flight node that can support the pod", func() { + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + }} + ExpectApplied(ctx, env.Client, provisioner) + initialPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + secondPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + Expect(node1.Name).To(Equal(node2.Name)) + }) + It("should not launch a second node if there is an in-flight node that can support the pod (node selectors)", func() { + ExpectApplied(ctx, env.Client, provisioner) + initialPod := test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + }, + NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-2"}, + }}}) + ExpectProvisioned(ctx, env.Client, cluster, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + // the node gets created in test-zone-2 + secondPod := test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + }, + NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-1", "test-zone-2"}, + }}}) + ExpectProvisioned(ctx, env.Client, cluster, prov, secondPod) + // test-zone-2 is in the intersection of their node selectors and the node has capacity, so we shouldn't create a new node + node2 := ExpectScheduled(ctx, env.Client, secondPod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + Expect(node1.Name).To(Equal(node2.Name)) + + // the node gets created in test-zone-2 + thirdPod := test.UnschedulablePod(test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + }, + NodeRequirements: []v1.NodeSelectorRequirement{{ + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-1", "test-zone-3"}, + }}}) + ExpectProvisioned(ctx, env.Client, cluster, prov, thirdPod) + // node is in test-zone-2, so this pod needs a new node + node3 := ExpectScheduled(ctx, env.Client, thirdPod) + Expect(node1.Name).ToNot(Equal(node3.Name)) + }) + It("should launch a second node if a pod won't fit on the existingNodes node", func() { + ExpectApplied(ctx, env.Client, provisioner) + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1001m"), + }, + }} + initialPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + // the node will have 2000m CPU, so these two pods can't both fit on it + opts.ResourceRequirements.Limits[v1.ResourceCPU] = resource.MustParse("1") + secondPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + Expect(node1.Name).ToNot(Equal(node2.Name)) + }) + It("should launch a second node if a pod isn't compatible with the existingNodes node (node selector)", func() { + ExpectApplied(ctx, env.Client, provisioner) + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + }} + initialPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + secondPod := test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1.LabelArchStable: "arm64"}}) + ExpectProvisioned(ctx, env.Client, cluster, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + Expect(node1.Name).ToNot(Equal(node2.Name)) + }) + It("should launch a second node if an in-flight node is terminating", func() { + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + }} + ExpectApplied(ctx, env.Client, provisioner) + initialPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + // delete the node + node1.Finalizers = nil + ExpectApplied(ctx, env.Client, node1) + ExpectDeleted(ctx, env.Client, node1) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + secondPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + Expect(node1.Name).ToNot(Equal(node2.Name)) + }) + Context("Topology", func() { + It("should balance pods across zones with in-flight newNodes", func() { + labels := map[string]string{"foo": "bar"} + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 2)) + + // reconcile our newNodes with the cluster state so they'll show up as in-flight + var nodeList v1.NodeList + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + for _, node := range nodeList.Items { + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKey{Name: node.Name}) + } + + firstRoundNumNodes := len(nodeList.Items) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 3, 3)) + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + + // shouldn't create any new newNodes as the in-flight ones can support the pods + Expect(nodeList.Items).To(HaveLen(firstRoundNumNodes)) + }) + It("should balance pods across hostnames with in-flight newNodes", func() { + labels := map[string]string{"foo": "bar"} + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 1, 1)) + + // reconcile our newNodes with the cluster state so they'll show up as in-flight + var nodeList v1.NodeList + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + for _, node := range nodeList.Items { + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKey{Name: node.Name}) + } + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + ) + // we prefer to launch new newNodes to satisfy the topology spread even though we could technnically schedule against existingNodes + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 1, 1, 1, 1, 1, 1, 1)) + }) + }) + Context("Taints", func() { + It("should assume pod will schedule to a tainted node with no taints", func() { + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("8"), + }, + }} + ExpectApplied(ctx, env.Client, provisioner) + initialPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + + // delete the pod so that the node is empty + ExpectDeleted(ctx, env.Client, initialPod) + node1.Spec.Taints = nil + ExpectApplied(ctx, env.Client, node1) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + secondPod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + Expect(node1.Name).To(Equal(node2.Name)) + }) + It("should not assume pod will schedule to a tainted node", func() { + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("8"), + }, + }} + ExpectApplied(ctx, env.Client, provisioner) + initialPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + + // delete the pod so that the node is empty + ExpectDeleted(ctx, env.Client, initialPod) + // and taint it + node1.Spec.Taints = append(node1.Spec.Taints, v1.Taint{ + Key: "foo.com/taint", + Value: "tainted", + Effect: v1.TaintEffectNoSchedule, + }) + ExpectApplied(ctx, env.Client, node1) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + secondPod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + Expect(node1.Name).ToNot(Equal(node2.Name)) + }) + It("should assume pod will schedule to a tainted node with a custom startup taint", func() { + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("8"), + }, + }} + provisioner.Spec.StartupTaints = append(provisioner.Spec.StartupTaints, v1.Taint{ + Key: "foo.com/taint", + Value: "tainted", + Effect: v1.TaintEffectNoSchedule, + }) + ExpectApplied(ctx, env.Client, provisioner) + initialPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + + // delete the pod so that the node is empty + ExpectDeleted(ctx, env.Client, initialPod) + // startup taint + node not ready taint = 2 + Expect(node1.Spec.Taints).To(HaveLen(2)) + Expect(node1.Spec.Taints).To(ContainElement(v1.Taint{ + Key: "foo.com/taint", + Value: "tainted", + Effect: v1.TaintEffectNoSchedule, + })) + ExpectApplied(ctx, env.Client, node1) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + secondPod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + Expect(node1.Name).To(Equal(node2.Name)) + }) + It("should not assume pod will schedule to a node with startup taints after initialization", func() { + startupTaint := v1.Taint{Key: "ignore-me", Value: "nothing-to-see-here", Effect: v1.TaintEffectNoSchedule} + provisioner.Spec.StartupTaints = []v1.Taint{startupTaint} + ExpectApplied(ctx, env.Client, provisioner) + initialPod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + + // delete the pod so that the node is empty + ExpectDeleted(ctx, env.Client, initialPod) + + // Mark it initialized which only occurs once the startup taint was removed and re-apply only the startup taint. + // We also need to add resource capacity as after initialization we assume that kubelet has recorded them. + node1.Labels[v1alpha5.LabelNodeInitialized] = "true" + node1.Spec.Taints = []v1.Taint{startupTaint} + node1.Status.Capacity = v1.ResourceList{v1.ResourcePods: resource.MustParse("10")} + ExpectApplied(ctx, env.Client, node1) + + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + // we should launch a new node since the startup taint is there, but was gone at some point + secondPod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + Expect(node1.Name).ToNot(Equal(node2.Name)) + }) + It("should consider a tainted NotReady node as in-flight even if initialized", func() { + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{v1.ResourceCPU: resource.MustParse("10m")}, + }} + ExpectApplied(ctx, env.Client, provisioner) + + // Schedule to New Machine + pod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node1 := ExpectScheduled(ctx, env.Client, pod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + // Mark Initialized + node1.Labels[v1alpha5.LabelNodeInitialized] = "true" + node1.Spec.Taints = []v1.Taint{ + {Key: v1.TaintNodeNotReady, Effect: v1.TaintEffectNoSchedule}, + {Key: v1.TaintNodeUnreachable, Effect: v1.TaintEffectNoSchedule}, + } + ExpectApplied(ctx, env.Client, node1) + // Schedule to In Flight Machine + pod = test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node2 := ExpectScheduled(ctx, env.Client, pod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node2)) + + Expect(node1.Name).To(Equal(node2.Name)) + }) + }) + Context("Daemonsets", func() { + It("should track daemonset usage separately so we know how many DS resources are remaining to be scheduled", func() { + ds := test.DaemonSet( + test.DaemonSetOptions{PodOptions: test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("1Gi")}}, + }}, + ) + ExpectApplied(ctx, env.Client, provisioner, ds) + Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(ds), ds)).To(Succeed()) + + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("8"), + }, + }} + initialPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + + // create our daemonset pod and manually bind it to the node + dsPod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("2Gi"), + }}, + }) + dsPod.OwnerReferences = append(dsPod.OwnerReferences, metav1.OwnerReference{ + APIVersion: "apps/v1", + Kind: "DaemonSet", + Name: ds.Name, + UID: ds.UID, + Controller: ptr.Bool(true), + BlockOwnerDeletion: ptr.Bool(true), + }) + + // delete the pod so that the node is empty + ExpectDeleted(ctx, env.Client, initialPod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + ExpectApplied(ctx, env.Client, provisioner, dsPod) + cluster.ForEachNode(func(f *state.Node) bool { + dsRequests := f.DaemonSetRequests() + available := f.Available() + Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 0)) + // no pods so we have the full (16 cpu - 100m overhead) + Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 15.9)) + return true + }) + ExpectManualBinding(ctx, env.Client, dsPod, node1) + ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(dsPod)) + + cluster.ForEachNode(func(f *state.Node) bool { + dsRequests := f.DaemonSetRequests() + available := f.Available() + Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 1)) + // only the DS pod is bound, so available is reduced by one and the DS requested is incremented by one + Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 14.9)) + return true + }) + + opts = test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("14.9"), + }, + }} + // this pod should schedule on the existingNodes node as the daemonset pod has already bound, meaning that the + // remaining daemonset resources should be zero leaving 14.9 CPUs for the pod + secondPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + Expect(node1.Name).To(Equal(node2.Name)) + }) + It("should handle unexpected daemonset pods binding to the node", func() { + ds1 := test.DaemonSet( + test.DaemonSetOptions{PodOptions: test.PodOptions{ + NodeSelector: map[string]string{ + "my-node-label": "value", + }, + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("1Gi")}}, + }}, + ) + ds2 := test.DaemonSet( + test.DaemonSetOptions{PodOptions: test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1m"), + }}}}) + ExpectApplied(ctx, env.Client, provisioner, ds1, ds2) + Expect(env.Client.Get(ctx, client.ObjectKeyFromObject(ds1), ds1)).To(Succeed()) + + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("8"), + }, + }} + initialPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, initialPod) + node1 := ExpectScheduled(ctx, env.Client, initialPod) + // this label appears on the node for some reason that Karpenter can't track + node1.Labels["my-node-label"] = "value" + ExpectApplied(ctx, env.Client, node1) + + // create our daemonset pod and manually bind it to the node + dsPod := test.UnschedulablePod(test.PodOptions{ + NodeSelector: map[string]string{ + "my-node-label": "value", + }, + ResourceRequirements: v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("2Gi"), + }}, + }) + dsPod.OwnerReferences = append(dsPod.OwnerReferences, metav1.OwnerReference{ + APIVersion: "apps/v1", + Kind: "DaemonSet", + Name: ds1.Name, + UID: ds1.UID, + Controller: ptr.Bool(true), + BlockOwnerDeletion: ptr.Bool(true), + }) + + // delete the pod so that the node is empty + ExpectDeleted(ctx, env.Client, initialPod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + + ExpectApplied(ctx, env.Client, provisioner, dsPod) + cluster.ForEachNode(func(f *state.Node) bool { + dsRequests := f.DaemonSetRequests() + available := f.Available() + Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 0)) + // no pods, so we have the full (16 CPU - 100m overhead) + Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 15.9)) + return true + }) + ExpectManualBinding(ctx, env.Client, dsPod, node1) + ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(dsPod)) + + cluster.ForEachNode(func(f *state.Node) bool { + dsRequests := f.DaemonSetRequests() + available := f.Available() + Expect(dsRequests.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 1)) + // only the DS pod is bound, so available is reduced by one and the DS requested is incremented by one + Expect(available.Cpu().AsApproximateFloat64()).To(BeNumerically("~", 14.9)) + return true + }) + + opts = test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("15.5"), + }, + }} + // This pod should not schedule on the inflight node as it requires more CPU than we have. This verifies + // we don't reintroduce a bug where more daemonsets scheduled than anticipated due to unexepected labels + // appearing on the node which caused us to compute a negative amount of resources remaining for daemonsets + // which in turn caused us to mis-calculate the amount of resources that were free on the node. + secondPod := test.UnschedulablePod(opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, secondPod) + node2 := ExpectScheduled(ctx, env.Client, secondPod) + // must create a new node + Expect(node1.Name).ToNot(Equal(node2.Name)) + }) + + }) + // nolint:gosec + It("should pack in-flight newNodes before launching new newNodes", func() { + cloudProv.InstanceTypes = []*cloudprovider.InstanceType{ + fake.NewInstanceType(fake.InstanceTypeOptions{ + Name: "medium", + Resources: v1.ResourceList{ + // enough CPU for four pods + a bit of overhead + v1.ResourceCPU: resource.MustParse("4.25"), + v1.ResourcePods: resource.MustParse("4"), + }, + }), + } + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1"), + }, + }} + + ExpectApplied(ctx, env.Client, provisioner) + + // scheduling in multiple batches random sets of pods + for i := 0; i < 10; i++ { + initialPods := MakePods(rand.Intn(10), opts) + ExpectProvisioned(ctx, env.Client, cluster, prov, initialPods...) + for _, pod := range initialPods { + node := ExpectScheduled(ctx, env.Client, pod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + } + } + + // due to the in-flight node support, we should pack existing newNodes before launching new node. The end result + // is that we should only have some spare capacity on our final node + nodesWithCPUFree := 0 + cluster.ForEachNode(func(n *state.Node) bool { + available := n.Available() + if available.Cpu().AsApproximateFloat64() >= 1 { + nodesWithCPUFree++ + } + return true + }) + Expect(nodesWithCPUFree).To(BeNumerically("<=", 1)) + }) + It("should not launch a second node if there is an in-flight node that can support the pod (#2011)", func() { + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + }} + + // there was a bug in cluster state where we failed to identify the instance type resources when using a + // ProviderRef so modify our provisioner to use the ProviderRef and ensure that the second pod schedules + // to the existingNodes node + provisioner.Spec.Provider = nil + provisioner.Spec.ProviderRef = &v1alpha5.ProviderRef{} + + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod(opts) + ExpectProvisionedNoBinding(ctx, env.Client, prov, pod) + var nodes v1.NodeList + Expect(env.Client.List(ctx, &nodes)).To(Succeed()) + Expect(nodes.Items).To(HaveLen(1)) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(&nodes.Items[0])) + + pod.Status.Conditions = []v1.PodCondition{{Type: v1.PodScheduled, Reason: v1.PodReasonUnschedulable, Status: v1.ConditionFalse}} + ExpectApplied(ctx, env.Client, pod) + ExpectProvisionedNoBinding(ctx, env.Client, prov, pod) + Expect(env.Client.List(ctx, &nodes)).To(Succeed()) + // shouldn't create a second node + Expect(nodes.Items).To(HaveLen(1)) + }) +}) + +var _ = Describe("No Pre-Binding", func() { + It("should not bind pods to newNodes", func() { + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + }, + }} + + var nodeList v1.NodeList + // shouldn't have any newNodes + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + Expect(nodeList.Items).To(HaveLen(0)) + + ExpectApplied(ctx, env.Client, provisioner) + initialPod := test.UnschedulablePod(opts) + ExpectProvisionedNoBinding(ctx, env.Client, prov, initialPod) + ExpectNotScheduled(ctx, env.Client, initialPod) + + // should launch a single node + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + Expect(nodeList.Items).To(HaveLen(1)) + node1 := &nodeList.Items[0] + + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + secondPod := test.UnschedulablePod(opts) + ExpectProvisionedNoBinding(ctx, env.Client, prov, secondPod) + ExpectNotScheduled(ctx, env.Client, secondPod) + // shouldn't create a second node as it can bind to the existingNodes node + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + Expect(nodeList.Items).To(HaveLen(1)) + }) + It("should handle resource zeroing of extended resources by kubelet", func() { + // Issue #1459 + opts := test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("10m"), + fake.ResourceGPUVendorA: resource.MustParse("1"), + }, + }} + + var nodeList v1.NodeList + // shouldn't have any newNodes + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + Expect(nodeList.Items).To(HaveLen(0)) + + ExpectApplied(ctx, env.Client, provisioner) + initialPod := test.UnschedulablePod(opts) + ExpectProvisionedNoBinding(ctx, env.Client, prov, initialPod) + ExpectNotScheduled(ctx, env.Client, initialPod) + + // should launch a single node + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + Expect(nodeList.Items).To(HaveLen(1)) + node1 := &nodeList.Items[0] + + // simulate kubelet zeroing out the extended resources on the node at startup + node1.Status.Capacity = map[v1.ResourceName]resource.Quantity{ + fake.ResourceGPUVendorA: resource.MustParse("0"), + } + node1.Status.Allocatable = map[v1.ResourceName]resource.Quantity{ + fake.ResourceGPUVendorB: resource.MustParse("0"), + } + + ExpectApplied(ctx, env.Client, node1) + + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + secondPod := test.UnschedulablePod(opts) + ExpectProvisionedNoBinding(ctx, env.Client, prov, secondPod) + ExpectNotScheduled(ctx, env.Client, secondPod) + // shouldn't create a second node as it can bind to the existingNodes node + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + Expect(nodeList.Items).To(HaveLen(1)) + }) + It("should respect self pod affinity without pod binding (zone)", func() { + // Issue #1975 + affLabels := map[string]string{"security": "s2"} + + pods := MakePods(2, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: affLabels, + }, + PodRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelTopologyZone, + }}, + }) + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisionedNoBinding(ctx, env.Client, prov, pods[0]) + var nodeList v1.NodeList + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + for i := range nodeList.Items { + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(&nodeList.Items[i])) + } + // the second pod can schedule against the in-flight node, but for that to work we need to be careful + // in how we fulfill the self-affinity by taking the existing node's domain as a preference over any + // random viable domain + ExpectProvisionedNoBinding(ctx, env.Client, prov, pods[1]) + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + Expect(nodeList.Items).To(HaveLen(1)) + }) +}) + +var _ = Describe("VolumeUsage", func() { + It("should launch multiple newNodes if required due to volume limits", func() { + const csiProvider = "fake.csi.provider" + cloudProv.InstanceTypes = []*cloudprovider.InstanceType{ + fake.NewInstanceType( + fake.InstanceTypeOptions{ + Name: "instance-type", + Resources: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1024"), + v1.ResourcePods: resource.MustParse("1024"), + }, + }), + } + + provisioner.Spec.Limits = nil + ExpectApplied(ctx, env.Client, provisioner) + initialPod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, initialPod) + node := ExpectScheduled(ctx, env.Client, initialPod) + csiNode := &storagev1.CSINode{ + ObjectMeta: metav1.ObjectMeta{ + Name: node.Name, + }, + Spec: storagev1.CSINodeSpec{ + Drivers: []storagev1.CSINodeDriver{ + { + Name: csiProvider, + NodeID: "fake-node-id", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.Int32(10), + }, + }, + }, + }, + } + ExpectApplied(ctx, env.Client, csiNode) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + + sc := test.StorageClass(test.StorageClassOptions{ + ObjectMeta: metav1.ObjectMeta{Name: "my-storage-class"}, + Provisioner: ptr.String(csiProvider), + Zones: []string{"test-zone-1"}}) + ExpectApplied(ctx, env.Client, sc) + + var pods []*v1.Pod + for i := 0; i < 6; i++ { + pvcA := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + StorageClassName: ptr.String("my-storage-class"), + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("my-claim-a-%d", i)}, + }) + pvcB := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + StorageClassName: ptr.String("my-storage-class"), + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("my-claim-b-%d", i)}, + }) + ExpectApplied(ctx, env.Client, pvcA, pvcB) + pods = append(pods, test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{pvcA.Name, pvcB.Name}, + })) + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + var nodeList v1.NodeList + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + // we need to create a new node as the in-flight one can only contain 5 pods due to the CSINode volume limit + Expect(nodeList.Items).To(HaveLen(2)) + }) + It("should launch a single node if all pods use the same PVC", func() { + const csiProvider = "fake.csi.provider" + cloudProv.InstanceTypes = []*cloudprovider.InstanceType{ + fake.NewInstanceType( + fake.InstanceTypeOptions{ + Name: "instance-type", + Resources: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1024"), + v1.ResourcePods: resource.MustParse("1024"), + }, + }), + } + + provisioner.Spec.Limits = nil + ExpectApplied(ctx, env.Client, provisioner) + initialPod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, initialPod) + node := ExpectScheduled(ctx, env.Client, initialPod) + csiNode := &storagev1.CSINode{ + ObjectMeta: metav1.ObjectMeta{ + Name: node.Name, + }, + Spec: storagev1.CSINodeSpec{ + Drivers: []storagev1.CSINodeDriver{ + { + Name: csiProvider, + NodeID: "fake-node-id", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.Int32(10), + }, + }, + }, + }, + } + ExpectApplied(ctx, env.Client, csiNode) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + + sc := test.StorageClass(test.StorageClassOptions{ + ObjectMeta: metav1.ObjectMeta{Name: "my-storage-class"}, + Provisioner: ptr.String(csiProvider), + Zones: []string{"test-zone-1"}}) + ExpectApplied(ctx, env.Client, sc) + + pv := test.PersistentVolume(test.PersistentVolumeOptions{ + ObjectMeta: metav1.ObjectMeta{Name: "my-volume"}, + Zones: []string{"test-zone-1"}}) + + pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + ObjectMeta: metav1.ObjectMeta{Name: "my-claim"}, + StorageClassName: ptr.String("my-storage-class"), + VolumeName: pv.Name, + }) + ExpectApplied(ctx, env.Client, pv, pvc) + + var pods []*v1.Pod + for i := 0; i < 100; i++ { + pods = append(pods, test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{pvc.Name, pvc.Name}, + })) + } + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + var nodeList v1.NodeList + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + // 100 of the same PVC should all be schedulable on the same node + Expect(nodeList.Items).To(HaveLen(1)) + }) + It("should not fail for non-dynamic PVCs", func() { + const csiProvider = "fake.csi.provider" + cloudProv.InstanceTypes = []*cloudprovider.InstanceType{ + fake.NewInstanceType( + fake.InstanceTypeOptions{ + Name: "instance-type", + Resources: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1024"), + v1.ResourcePods: resource.MustParse("1024"), + }, + }), + } + + provisioner.Spec.Limits = nil + ExpectApplied(ctx, env.Client, provisioner) + initialPod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, initialPod) + node := ExpectScheduled(ctx, env.Client, initialPod) + csiNode := &storagev1.CSINode{ + ObjectMeta: metav1.ObjectMeta{ + Name: node.Name, + }, + Spec: storagev1.CSINodeSpec{ + Drivers: []storagev1.CSINodeDriver{ + { + Name: csiProvider, + NodeID: "fake-node-id", + Allocatable: &storagev1.VolumeNodeResources{ + Count: ptr.Int32(10), + }, + }, + }, + }, + } + ExpectApplied(ctx, env.Client, csiNode) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + + sc := test.StorageClass(test.StorageClassOptions{ + ObjectMeta: metav1.ObjectMeta{Name: "my-storage-class"}, + Provisioner: ptr.String(csiProvider), + Zones: []string{"test-zone-1"}}) + ExpectApplied(ctx, env.Client, sc) + + pv := test.PersistentVolume(test.PersistentVolumeOptions{ + ObjectMeta: metav1.ObjectMeta{Name: "my-volume"}, + Driver: csiProvider, + Zones: []string{"test-zone-1"}}) + + pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + ObjectMeta: metav1.ObjectMeta{Name: "my-claim"}, + VolumeName: pv.Name, + StorageClassName: ptr.String(""), + }) + ExpectApplied(ctx, env.Client, pv, pvc) + + var pods []*v1.Pod + for i := 0; i < 5; i++ { + pods = append(pods, test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{pvc.Name, pvc.Name}, + })) + } + ExpectApplied(ctx, env.Client, provisioner) + _ = ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + + var nodeList v1.NodeList + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + // 5 of the same PVC should all be schedulable on the same node + Expect(nodeList.Items).To(HaveLen(1)) + }) + It("should not fail for NFS volumes", func() { + cloudProv.InstanceTypes = []*cloudprovider.InstanceType{ + fake.NewInstanceType( + fake.InstanceTypeOptions{ + Name: "instance-type", + Resources: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1024"), + v1.ResourcePods: resource.MustParse("1024"), + }, + }), + } + + provisioner.Spec.Limits = nil + ExpectApplied(ctx, env.Client, provisioner) + initialPod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, initialPod) + node := ExpectScheduled(ctx, env.Client, initialPod) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node)) + + pv := test.PersistentVolume(test.PersistentVolumeOptions{ + ObjectMeta: metav1.ObjectMeta{Name: "my-volume"}, + StorageClassName: "nfs", + Zones: []string{"test-zone-1"}}) + pv.Spec.NFS = &v1.NFSVolumeSource{ + Server: "fake.server", + Path: "/some/path", + } + pv.Spec.CSI = nil + + pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + ObjectMeta: metav1.ObjectMeta{Name: "my-claim"}, + VolumeName: pv.Name, + StorageClassName: ptr.String(""), + }) + ExpectApplied(ctx, env.Client, pv, pvc) + + var pods []*v1.Pod + for i := 0; i < 5; i++ { + pods = append(pods, test.UnschedulablePod(test.PodOptions{ + PersistentVolumeClaims: []string{pvc.Name, pvc.Name}, + })) + } + ExpectApplied(ctx, env.Client, provisioner) + _ = ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + + var nodeList v1.NodeList + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + // 5 of the same PVC should all be schedulable on the same node + Expect(nodeList.Items).To(HaveLen(1)) + }) + It("should not launch nodes for pods with ephemeral volume using a non-existent storage classes", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod(test.PodOptions{}) + pod.Spec.Volumes = append(pod.Spec.Volumes, v1.Volume{ + Name: "tmp-ephemeral", + VolumeSource: v1.VolumeSource{ + Ephemeral: &v1.EphemeralVolumeSource{ + VolumeClaimTemplate: &v1.PersistentVolumeClaimTemplate{ + Spec: v1.PersistentVolumeClaimSpec{ + StorageClassName: ptr.String("non-existent"), + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + }, + }, + }, + }) + ExpectApplied(ctx, env.Client, provisioner) + _ = ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + + var nodeList v1.NodeList + Expect(env.Client.List(ctx, &nodeList)).To(Succeed()) + // no nodes should be created as the storage class doesn't eixst + Expect(nodeList.Items).To(HaveLen(0)) + }) +}) func MakePods(count int, options test.PodOptions) (pods []*v1.Pod) { for i := 0; i < count; i++ { diff --git a/pkg/controllers/provisioning/scheduling/topology_test.go b/pkg/controllers/provisioning/scheduling/topology_test.go index 9448fa71b1..8e72fa5ba0 100644 --- a/pkg/controllers/provisioning/scheduling/topology_test.go +++ b/pkg/controllers/provisioning/scheduling/topology_test.go @@ -16,2202 +16,2225 @@ package scheduling_test import ( "context" + "time" . "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" + . "github.com/onsi/ginkgo/v2" + + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + "github.com/aws/karpenter-core/pkg/cloudprovider/fake" + "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" ) -//var _ = Describe("Topology", func() { -// labels := map[string]string{"test": "test"} -// -// It("should ignore unknown topology keys", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: []v1.TopologySpreadConstraint{{ -// TopologyKey: "unknown", -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }}}, -// ), test.UnschedulablePod()) -// ExpectNotScheduled(ctx, env.Client, pods[0]) -// ExpectScheduled(ctx, env.Client, pods[1]) -// }) -// -// It("should not spread an invalid label selector", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app.kubernetes.io/name": "{{ zqfmgb }}"}}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(2, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})...) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2)) -// }) -// -// Context("Zonal", func() { -// It("should balance pods across zones (match labels)", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 2)) -// }) -// It("should balance pods across zones (match expressions)", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{ -// MatchExpressions: []metav1.LabelSelectorRequirement{ -// { -// Key: "test", -// Operator: metav1.LabelSelectorOpIn, -// Values: []string{"test"}, -// }, -// }, -// }, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 2)) -// }) -// It("should respect provisioner zonal constraints", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}} -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 2)) -// }) -// It("should respect provisioner zonal constraints (subset)", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// ) -// // should spread the two pods evenly across the only valid zones in our universe (the two zones from our single provisioner) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2)) -// }) -// It("should respect provisioner zonal constraints (existing pod)", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// // need enough resource requests that the first node we create fills a node and can't act as an in-flight -// // node for the other pods -// rr := v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("1.1"), -// }, -// } -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// ResourceRequirements: rr, -// NodeSelector: map[string]string{ -// v1.LabelTopologyZone: "test-zone-3", -// }, -// })) -// ExpectScheduled(ctx, env.Client, pods[0]) -// -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, ResourceRequirements: rr, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, ResourceRequirements: rr, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, ResourceRequirements: rr, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, ResourceRequirements: rr, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, ResourceRequirements: rr, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, ResourceRequirements: rr, TopologySpreadConstraints: topology}), -// ) -// // we should have unschedulable pods now, the provisioner can only schedule to zone-1/zone-2, but because of the existing -// // pod in zone-3 it can put a max of two per zone before it would violate max skew -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 2, 2)) -// }) -// It("should schedule to the non-minimum domain if its all that's available", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 5, -// }} -// rr := v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("1.1"), -// }, -// } -// // force this pod onto zone-1 -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// ResourceRequirements: rr, TopologySpreadConstraints: topology})) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1)) -// -// // force this pod onto zone-2 -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// ResourceRequirements: rr, TopologySpreadConstraints: topology})) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1)) -// -// // now only allow scheduling pods on zone-3 -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(10, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// ResourceRequirements: rr, TopologySpreadConstraints: topology})..., -// ) -// -// // max skew of 5, so test-zone-1/2 will have 1 pod each, test-zone-3 will have 6, and the rest will fail to schedule -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 6)) -// }) -// It("should only schedule to minimum domains if already violating max skew", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// rr := v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("1.1"), -// }, -// } -// createPods := func(count int) []*v1.Pod { -// var pods []*v1.Pod -// for i := 0; i < count; i++ { -// pods = append(pods, test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// ResourceRequirements: rr, TopologySpreadConstraints: topology})) -// } -// return pods -// } -// // Spread 9 pods -// ExpectApplied(ctx, env.Client, provisioner) -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, createPods(9)...) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 3, 3)) -// -// // Delete pods to create a skew -// for _, pod := range pods { -// node := ExpectScheduled(ctx, env.Client, pod) -// if node.Labels[v1.LabelTopologyZone] != "test-zone-1" { -// ExpectDeleted(ctx, env.Client, pod) -// } -// } -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3)) -// -// // Create 3 more pods, skew should recover -// _ = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, createPods(3)...) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 1, 2)) -// }) -// It("should not violate max-skew when unsat = do not schedule", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// rr := v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("1.1"), -// }, -// } -// // force this pod onto zone-1 -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// ResourceRequirements: rr, TopologySpreadConstraints: topology})) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1)) -// -// // now only allow scheduling pods on zone-2 and zone-3 -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2", "test-zone-3"}}} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(10, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// ResourceRequirements: rr, TopologySpreadConstraints: topology})..., -// ) -// -// // max skew of 1, so test-zone-2/3 will have 2 nodes each and the rest of the pods will fail to schedule -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 2, 2)) -// }) -// It("should not violate max-skew when unsat = do not schedule (discover domains)", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// rr := v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("1.1"), -// }, -// } -// // force this pod onto zone-1 -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, ResourceRequirements: rr})) -// -// // now only allow scheduling pods on zone-2 and zone-3 -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2", "test-zone-3"}}} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(10, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: topology, ResourceRequirements: rr})..., -// ) -// -// // max skew of 1, so test-zone-2/3 will have 2 nodes each and the rest of the pods will fail to schedule since -// // test-zone-1 has 1 pods in it. -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 2, 2)) -// }) -// It("should only count running/scheduled pods with matching labels scheduled to nodes with a corresponding domain", func() { -// wrongNamespace := test.RandomName() -// firstNode := test.Node(test.NodeOptions{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}}) -// secondNode := test.Node(test.NodeOptions{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}}) -// thirdNode := test.Node(test.NodeOptions{}) // missing topology domain -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner, firstNode, secondNode, thirdNode, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: wrongNamespace}}) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(firstNode)) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(secondNode)) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(thirdNode)) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.Pod(test.PodOptions{NodeName: firstNode.Name}), // ignored, missing labels -// test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}}), // ignored, pending -// test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: thirdNode.Name}), // ignored, no domain on node -// test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels, Namespace: wrongNamespace}, NodeName: firstNode.Name}), // ignored, wrong namespace -// test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels, DeletionTimestamp: &metav1.Time{Time: time.Now().Add(10 * time.Second)}}}), // ignored, terminating -// test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name, Phase: v1.PodFailed}), // ignored, phase=Failed -// test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name, Phase: v1.PodSucceeded}), // ignored, phase=Succeeded -// test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name}), -// test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name}), -// test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: secondNode.Name}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// ) -// nodes := v1.NodeList{} -// Expect(env.Client.List(ctx, &nodes)).To(Succeed()) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2, 1)) -// }) -// It("should match all pods when labelSelector is not specified", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(), -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1)) -// }) -// It("should handle interdependent selectors", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(5, test.PodOptions{TopologySpreadConstraints: topology})..., -// ) -// // This is weird, but the topology label selector is used for determining domain counts. The pod that -// // owns the topology is what the spread actually applies to. In this test case, there are no pods matching -// // the label selector, so the max skew is zero. This means we can pack all the pods onto the same node since -// // it doesn't violate the topology spread constraint (i.e. adding new pods doesn't increase skew since the -// // pods we are adding don't count toward skew). This behavior is called out at -// // https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ , though it's not -// // recommended for users. -// nodeNames := sets.NewString() -// for _, p := range pods { -// nodeNames.Insert(p.Spec.NodeName) -// } -// Expect(nodeNames).To(HaveLen(1)) -// }) -// }) -// -// Context("Hostname", func() { -// It("should balance pods across nodes", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 1, 1)) -// }) -// It("should balance pods on the same hostname up to maxskew", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 4, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(4)) -// }) -// It("balance multiple deployments with hostname topology spread", func() { -// // Issue #1425 -// spreadPod := func(appName string) test.PodOptions { -// return test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{ -// Labels: map[string]string{ -// "app": appName, -// }, -// }, -// TopologySpreadConstraints: []v1.TopologySpreadConstraint{ -// { -// MaxSkew: 1, -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: map[string]string{"app": appName}, -// }, -// }, -// }, -// } -// } -// -// ExpectApplied(ctx, env.Client, provisioner) -// scheduled := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(spreadPod("app1")), test.UnschedulablePod(spreadPod("app1")), -// test.UnschedulablePod(spreadPod("app2")), test.UnschedulablePod(spreadPod("app2"))) -// -// for _, p := range scheduled { -// ExpectScheduled(ctx, env.Client, p) -// } -// nodes := v1.NodeList{} -// Expect(env.Client.List(ctx, &nodes)).To(Succeed()) -// // this wasn't part of #1425, but ensures that we launch the minimum number of nodes -// Expect(nodes.Items).To(HaveLen(2)) -// }) -// It("balance multiple deployments with hostname topology spread & varying arch", func() { -// // Issue #1425 -// spreadPod := func(appName, arch string) test.PodOptions { -// return test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{ -// Labels: map[string]string{ -// "app": appName, -// }, -// }, -// NodeRequirements: []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{arch}, -// }, -// }, -// TopologySpreadConstraints: []v1.TopologySpreadConstraint{ -// { -// MaxSkew: 1, -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: map[string]string{"app": appName}, -// }, -// }, -// }, -// } -// } -// -// ExpectApplied(ctx, env.Client, provisioner) -// scheduled := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(spreadPod("app1", v1alpha5.ArchitectureAmd64)), test.UnschedulablePod(spreadPod("app1", v1alpha5.ArchitectureAmd64)), -// test.UnschedulablePod(spreadPod("app2", v1alpha5.ArchitectureArm64)), test.UnschedulablePod(spreadPod("app2", v1alpha5.ArchitectureArm64))) -// -// for _, p := range scheduled { -// ExpectScheduled(ctx, env.Client, p) -// } -// nodes := v1.NodeList{} -// Expect(env.Client.List(ctx, &nodes)).To(Succeed()) -// // same test as the previous one, but now the architectures are different so we need four nodes in total -// Expect(nodes.Items).To(HaveLen(4)) -// }) -// }) -// -// Context("CapacityType", func() { -// It("should balance pods across capacity types", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1alpha5.LabelCapacityType, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2)) -// }) -// It("should respect provisioner capacity type constraints", func() { -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeSpot, v1alpha5.CapacityTypeOnDemand}}} -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1alpha5.LabelCapacityType, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2)) -// }) -// It("should not violate max-skew when unsat = do not schedule (capacity type)", func() { -// // this test can pass in a flaky manner if we don't restrict our min domain selection to valid choices -// // per the provisioner spec -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1alpha5.LabelCapacityType, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// rr := v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("1.1"), -// }, -// } -// // force this pod onto spot -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeSpot}}} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// ResourceRequirements: rr, TopologySpreadConstraints: topology})) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1)) -// -// // now only allow scheduling pods on on-demand -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeOnDemand}}} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(5, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// ResourceRequirements: rr, TopologySpreadConstraints: topology})..., -// ) -// -// // max skew of 1, so on-demand will have 2 pods and the rest of the pods will fail to schedule -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 2)) -// }) -// It("should violate max-skew when unsat = schedule anyway (capacity type)", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1alpha5.LabelCapacityType, -// WhenUnsatisfiable: v1.ScheduleAnyway, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// rr := v1.ResourceRequirements{ -// Requests: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("1.1"), -// }, -// } -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeSpot}}} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// ResourceRequirements: rr, TopologySpreadConstraints: topology})) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1)) -// -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeOnDemand}}} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(5, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// ResourceRequirements: rr, TopologySpreadConstraints: topology})..., -// ) -// -// // max skew of 1, on-demand will end up with 5 pods even though spot has a single pod -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 5)) -// }) -// It("should only count running/scheduled pods with matching labels scheduled to nodes with a corresponding domain", func() { -// wrongNamespace := test.RandomName() -// firstNode := test.Node(test.NodeOptions{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{v1alpha5.LabelCapacityType: v1alpha5.CapacityTypeSpot}}}) -// secondNode := test.Node(test.NodeOptions{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{v1alpha5.LabelCapacityType: v1alpha5.CapacityTypeOnDemand}}}) -// thirdNode := test.Node(test.NodeOptions{}) // missing topology capacity type -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1alpha5.LabelCapacityType, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner, firstNode, secondNode, thirdNode, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: wrongNamespace}}) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(firstNode)) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(secondNode)) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(thirdNode)) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.Pod(test.PodOptions{NodeName: firstNode.Name}), // ignored, missing labels -// test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}}), // ignored, pending -// test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: thirdNode.Name}), // ignored, no domain on node -// test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels, Namespace: wrongNamespace}, NodeName: firstNode.Name}), // ignored, wrong namespace -// test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels, DeletionTimestamp: &metav1.Time{Time: time.Now().Add(10 * time.Second)}}}), // ignored, terminating -// test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name, Phase: v1.PodFailed}), // ignored, phase=Failed -// test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name, Phase: v1.PodSucceeded}), // ignored, phase=Succeeded -// test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name}), -// test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name}), -// test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: secondNode.Name}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), -// ) -// nodes := v1.NodeList{} -// Expect(env.Client.List(ctx, &nodes)).To(Succeed()) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 3)) -// }) -// It("should match all pods when labelSelector is not specified", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1alpha5.LabelCapacityType, -// WhenUnsatisfiable: v1.DoNotSchedule, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(), -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1)) -// }) -// It("should handle interdependent selectors", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(5, test.PodOptions{TopologySpreadConstraints: topology})..., -// ) -// // This is weird, but the topology label selector is used for determining domain counts. The pod that -// // owns the topology is what the spread actually applies to. In this test case, there are no pods matching -// // the label selector, so the max skew is zero. This means we can pack all the pods onto the same node since -// // it doesn't violate the topology spread constraint (i.e. adding new pods doesn't increase skew since the -// // pods we are adding don't count toward skew). This behavior is called out at -// // https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ , though it's not -// // recommended for users. -// nodeNames := sets.NewString() -// for _, p := range pods { -// nodeNames.Insert(p.Spec.NodeName) -// } -// Expect(nodeNames).To(HaveLen(1)) -// }) -// It("should balance pods across capacity-types (node required affinity constrained)", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, MakePods(1, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// NodeRequirements: []v1.NodeSelectorRequirement{ -// // launch this on-demand pod in zone-1 -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, -// {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{"on-demand"}}, -// }, -// })...) -// ExpectScheduled(ctx, env.Client, pod[0]) -// -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1alpha5.LabelCapacityType, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// -// // Try to run 5 pods, with a node selector restricted to test-zone-2, they should all schedule on the same -// // spot node. This doesn't violate the max-skew of 1 as the node selector requirement here excludes the -// // existing on-demand pod from counting within this topology. -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(5, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// // limit our provisioner to only creating spot nodes -// NodeRequirements: []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, -// {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{"spot"}}, -// }, -// TopologySpreadConstraints: topology, -// })..., -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 5)) -// }) -// It("should balance pods across capacity-types (no constraints)", func() { -// rr := v1.ResourceRequirements{ -// Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, -// } -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "single-pod-instance-type"}, -// NodeRequirements: []v1.NodeSelectorRequirement{ -// { -// Key: v1alpha5.LabelCapacityType, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"on-demand"}, -// }, -// }, -// }))[0] -// -// ExpectScheduled(ctx, env.Client, pod) -// -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1alpha5.LabelCapacityType, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// -// // limit our provisioner to only creating spot nodes -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{"spot"}}, -// } -// -// // since there is no node selector on this pod, the topology can see the single on-demand node that already -// // exists and that limits us to scheduling 2 more spot pods before we would violate max-skew -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(5, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// ResourceRequirements: rr, -// TopologySpreadConstraints: topology, -// })..., -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 2)) -// }) -// It("should balance pods across arch (no constraints)", func() { -// rr := v1.ResourceRequirements{ -// Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, -// } -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "single-pod-instance-type"}, -// NodeRequirements: []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelArchStable, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"amd64"}, -// }, -// }, -// })) -// -// ExpectScheduled(ctx, env.Client, pod[0]) -// -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelArchStable, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// -// // limit our provisioner to only creating arm64 nodes -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{"arm64"}}} -// -// // since there is no node selector on this pod, the topology can see the single arm64 node that already -// // exists and that limits us to scheduling 2 more spot pods before we would violate max-skew -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(5, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// ResourceRequirements: rr, -// TopologySpreadConstraints: topology, -// })..., -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 2)) -// }) -// }) -// -// Context("Combined Hostname and Zonal Topology", func() { -// It("should spread pods while respecting both constraints (hostname and zonal)", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }, { -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 3, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(2, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1)) -// ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(3, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2, 1)) -// ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(5, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(4, 3, 3)) -// ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(11, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(7, 7, 7)) -// ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) -// }) -// It("should balance pods across provisioner requirements", func() { -// spotProv := test.Provisioner(test.ProvisionerOptions{ -// Requirements: []v1.NodeSelectorRequirement{ -// { -// Key: v1alpha5.LabelCapacityType, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"spot"}, -// }, -// { -// Key: "capacity.spread.4-1", -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"2", "3", "4", "5"}, -// }, -// }, -// }) -// onDemandProv := test.Provisioner(test.ProvisionerOptions{ -// Requirements: []v1.NodeSelectorRequirement{ -// { -// Key: v1alpha5.LabelCapacityType, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"on-demand"}, -// }, -// { -// Key: "capacity.spread.4-1", -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"1"}, -// }, -// }, -// }) -// -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: "capacity.spread.4-1", -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, spotProv, onDemandProv) -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, MakePods(20, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: topology, -// })...) -// for _, p := range pods { -// ExpectScheduled(ctx, env.Client, p) -// } -// -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(4, 4, 4, 4, 4)) -// // due to the spread across provisioners, we've forced a 4:1 spot to on-demand spread -// ExpectSkew(ctx, env.Client, "default", &v1.TopologySpreadConstraint{ -// TopologyKey: v1alpha5.LabelCapacityType, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }).To(ConsistOf(4, 16)) -// }) -// -// It("should spread pods while respecting both constraints", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }, { -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.ScheduleAnyway, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} -// -// // create a second provisioner that can't provision at all -// provisionerB := test.Provisioner() -// provisionerB.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}} -// provisionerB.Spec.Limits = &v1alpha5.Limits{ -// Resources: map[v1.ResourceName]resource.Quantity{ -// v1.ResourceCPU: resource.MustParse("0"), -// }, -// } -// -// ExpectApplied(ctx, env.Client, provisioner, provisionerB) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(10, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., -// ) -// -// // should get one pod per zone, can't schedule to test-zone-3 since that provisioner is effectively disabled -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1)) -// // and one pod per node -// ExpectSkew(ctx, env.Client, "default", &topology[1]).To(ConsistOf(1, 1)) -// }) -// -// It("should spread pods while respecting both constraints", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1alpha5.LabelCapacityType, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }, { -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 3, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(2, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1)) -// ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(3, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 2)) -// ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(5, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(5, 5)) -// ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(11, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(11, 10)) -// ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) -// }) -// }) -// -// Context("Combined Zonal and Capacity Type Topology", func() { -// It("should spread pods while respecting both constraints", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1alpha5.LabelCapacityType, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }, { -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(2, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).ToNot(ContainElements(BeNumerically(">", 1))) -// ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 1))) -// -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(3, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).ToNot(ContainElements(BeNumerically(">", 3))) -// ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 2))) -// -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(5, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).ToNot(ContainElements(BeNumerically(">", 5))) -// ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 4))) -// -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(11, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).ToNot(ContainElements(BeNumerically(">", 11))) -// ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 7))) -// }) -// }) -// -// Context("Combined Hostname, Zonal, and Capacity Type Topology", func() { -// It("should spread pods while respecting all constraints", func() { -// // ensure we've got an instance type for every zone/capacity-type pair -// cloudProv.InstanceTypes = fake.InstanceTypesAssorted() -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1alpha5.LabelCapacityType, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }, { -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 2, -// }, { -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 3, -// }} -// -// // add varying numbers of pods, checking after each scheduling to ensure that our max required max skew -// // has not been violated for each constraint -// for i := 1; i < 15; i++ { -// pods := MakePods(i, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}) -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pods...) -// ExpectMaxSkew(ctx, env.Client, "default", &topology[0]).To(BeNumerically("<=", 1)) -// ExpectMaxSkew(ctx, env.Client, "default", &topology[1]).To(BeNumerically("<=", 2)) -// ExpectMaxSkew(ctx, env.Client, "default", &topology[2]).To(BeNumerically("<=", 3)) -// for _, pod := range pods { -// ExpectScheduled(ctx, env.Client, pod) -// } -// } -// }) -// }) -// -// // https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#interaction-with-node-affinity-and-node-selectors -// Context("Combined Zonal Topology and Machine Affinity", func() { -// It("should limit spread options by nodeSelector", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// append( -// MakePods(5, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: topology, -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}, -// }), -// MakePods(10, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: topology, -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}, -// })..., -// )..., -// ) -// // we limit the zones of each pod via node selectors, which causes the topology spreads to only consider -// // the single zone as the only valid domain for the topology spread allowing us to schedule multiple pods per domain -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(5, 10)) -// }) -// It("should limit spread options by node requirements", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(10, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: topology, -// NodeRequirements: []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelTopologyZone, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"test-zone-1", "test-zone-2"}, -// }, -// }, -// })...) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(5, 5)) -// }) -// It("should limit spread options by node affinity", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(6, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: topology, -// NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{ -// "test-zone-1", "test-zone-2", -// }}}, -// })...) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 3)) -// -// // open the provisioner back to up so it can see all zones again -// provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ -// {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}} -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, MakePods(1, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: topology, -// NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{ -// "test-zone-2", "test-zone-3", -// }}}, -// })...) -// -// // it will schedule on the currently empty zone-3 even though max-skew is violated as it improves max-skew -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 3, 1)) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(5, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: topology, -// })..., -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(4, 4, 4)) -// }) -// }) -// -// // https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#interaction-with-node-affinity-and-node-selectors -// Context("Combined Capacity Type Topology and Machine Affinity", func() { -// It("should limit spread options by nodeSelector", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1alpha5.LabelCapacityType, -// WhenUnsatisfiable: v1.ScheduleAnyway, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// append( -// MakePods(5, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: topology, -// NodeSelector: map[string]string{v1alpha5.LabelCapacityType: v1alpha5.CapacityTypeSpot}, -// }), -// MakePods(5, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: topology, -// NodeSelector: map[string]string{v1alpha5.LabelCapacityType: v1alpha5.CapacityTypeOnDemand}, -// })..., -// )..., -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(5, 5)) -// }) -// It("should limit spread options by node affinity (capacity type)", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1alpha5.LabelCapacityType, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// -// // need to limit the rules to spot or else it will know that on-demand has 0 pods and won't violate the max-skew -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(3, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: topology, -// NodeRequirements: []v1.NodeSelectorRequirement{ -// {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeSpot}}, -// }, -// })...) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3)) -// -// // open the rules back to up so it can see all capacity types -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, MakePods(1, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: topology, -// NodeRequirements: []v1.NodeSelectorRequirement{ -// {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeOnDemand, v1alpha5.CapacityTypeSpot}}, -// }, -// })...) -// -// // it will schedule on the currently empty on-demand even though max-skew is violated as it improves max-skew -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 1)) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// MakePods(5, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: topology, -// })..., -// ) -// ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(5, 4)) -// }) -// }) -// -// Context("Pod Affinity/Anti-Affinity", func() { -// It("should schedule a pod with empty pod affinity and anti-affinity", func() { -// ExpectApplied(ctx, env.Client) -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{ -// PodRequirements: []v1.PodAffinityTerm{}, -// PodAntiRequirements: []v1.PodAffinityTerm{}, -// }))[0] -// ExpectScheduled(ctx, env.Client, pod) -// }) -// It("should respect pod affinity (hostname)", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// -// affLabels := map[string]string{"security": "s2"} -// -// affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) -// // affPod2 will try to get scheduled with affPod1 -// affPod2 := test.UnschedulablePod(test.PodOptions{PodRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelHostname, -// }}}) -// -// var pods []*v1.Pod -// pods = append(pods, MakePods(10, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: topology, -// })...) -// pods = append(pods, affPod1) -// pods = append(pods, affPod2) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pods...) -// n1 := ExpectScheduled(ctx, env.Client, affPod1) -// n2 := ExpectScheduled(ctx, env.Client, affPod2) -// // should be scheduled on the same node -// Expect(n1.Name).To(Equal(n2.Name)) -// }) -// It("should respect pod affinity (arch)", func() { -// affLabels := map[string]string{"security": "s2"} -// tsc := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: affLabels}, -// MaxSkew: 1, -// }} -// -// affPod1 := test.UnschedulablePod(test.PodOptions{ -// TopologySpreadConstraints: tsc, -// ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, -// }, -// NodeSelector: map[string]string{ -// v1.LabelArchStable: "arm64", -// }}) -// // affPod2 will try to get scheduled with affPod1 -// affPod2 := test.UnschedulablePod(test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, -// TopologySpreadConstraints: tsc, -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, -// }, -// PodRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelArchStable, -// }}}) -// -// pods := []*v1.Pod{affPod1, affPod2} -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pods...) -// n1 := ExpectScheduled(ctx, env.Client, affPod1) -// n2 := ExpectScheduled(ctx, env.Client, affPod2) -// // should be scheduled on a node with the same arch -// Expect(n1.Labels[v1.LabelArchStable]).To(Equal(n2.Labels[v1.LabelArchStable])) -// // but due to TSC, not on the same node -// Expect(n1.Name).ToNot(Equal(n2.Name)) -// }) -// It("should respect self pod affinity (hostname)", func() { -// affLabels := map[string]string{"security": "s2"} -// -// pods := MakePods(3, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{ -// Labels: affLabels, -// }, -// PodRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelHostname, -// }}, -// }) -// -// ExpectApplied(ctx, env.Client, provisioner) -// pods = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pods...) -// nodeNames := map[string]struct{}{} -// for _, p := range pods { -// n := ExpectScheduled(ctx, env.Client, p) -// nodeNames[n.Name] = struct{}{} -// } -// Expect(len(nodeNames)).To(Equal(1)) -// }) -// It("should respect self pod affinity for first empty topology domain only (hostname)", func() { -// affLabels := map[string]string{"security": "s2"} -// createPods := func() []*v1.Pod { -// return MakePods(10, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{ -// Labels: affLabels, -// }, -// PodRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelHostname, -// }}, -// }) -// } -// ExpectApplied(ctx, env.Client, provisioner) -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, createPods()...) -// nodeNames := map[string]struct{}{} -// unscheduledCount := 0 -// scheduledCount := 0 -// for _, p := range pods { -// p = ExpectPodExists(ctx, env.Client, p.Name, p.Namespace) -// if p.Spec.NodeName == "" { -// unscheduledCount++ -// } else { -// nodeNames[p.Spec.NodeName] = struct{}{} -// scheduledCount++ -// } -// } -// // the node can only hold 5 pods, so we should get a single node with 5 pods and 5 unschedulable pods from that batch -// Expect(len(nodeNames)).To(Equal(1)) -// Expect(scheduledCount).To(BeNumerically("==", 5)) -// Expect(unscheduledCount).To(BeNumerically("==", 5)) -// -// // and pods in a different batch should not schedule as well even if the node is not ready yet -// pods = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, createPods()...) -// for _, p := range pods { -// ExpectNotScheduled(ctx, env.Client, p) -// } -// }) -// It("should respect self pod affinity for first empty topology domain only (hostname/constrained zones)", func() { -// affLabels := map[string]string{"security": "s2"} -// // put one pod in test-zone-1, this does affect pod affinity even though we have different node selectors. -// // The node selector and required node affinity restrictions to topology counting only apply to topology spread. -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{ -// Labels: affLabels, -// }, -// NodeSelector: map[string]string{ -// v1.LabelTopologyZone: "test-zone-1", -// }, -// PodRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelHostname, -// }}, -// })) -// -// pods := MakePods(10, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{ -// Labels: affLabels, -// }, -// NodeRequirements: []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelTopologyZone, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"test-zone-2", "test-zone-3"}, -// }, -// }, -// PodRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelHostname, -// }}, -// }) -// pods = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pods...) -// for _, p := range pods { -// // none of this should schedule -// ExpectNotScheduled(ctx, env.Client, p) -// } -// }) -// It("should respect self pod affinity (zone)", func() { -// affLabels := map[string]string{"security": "s2"} -// -// pods := MakePods(3, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{ -// Labels: affLabels, -// }, -// PodRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelTopologyZone, -// }}, -// }) -// -// ExpectApplied(ctx, env.Client, provisioner) -// pods = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pods...) -// nodeNames := map[string]struct{}{} -// for _, p := range pods { -// n := ExpectScheduled(ctx, env.Client, p) -// nodeNames[n.Name] = struct{}{} -// } -// Expect(len(nodeNames)).To(Equal(1)) -// }) -// It("should respect self pod affinity (zone w/ constraint)", func() { -// affLabels := map[string]string{"security": "s2"} -// // the pod needs to provide it's own zonal affinity, but we further limit it to only being on test-zone-3 -// pods := MakePods(3, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{ -// Labels: affLabels, -// }, -// PodRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelTopologyZone, -// }}, -// NodeRequirements: []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelTopologyZone, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"test-zone-3"}, -// }, -// }, -// }) -// ExpectApplied(ctx, env.Client, provisioner) -// pods = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pods...) -// nodeNames := map[string]struct{}{} -// for _, p := range pods { -// n := ExpectScheduled(ctx, env.Client, p) -// nodeNames[n.Name] = struct{}{} -// Expect(n.Labels[v1.LabelTopologyZone]).To(Equal("test-zone-3")) -// } -// Expect(len(nodeNames)).To(Equal(1)) -// }) -// It("should allow violation of preferred pod affinity", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// -// affPod2 := test.UnschedulablePod(test.PodOptions{PodPreferences: []v1.WeightedPodAffinityTerm{{ -// Weight: 50, -// PodAffinityTerm: v1.PodAffinityTerm{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: map[string]string{"security": "s2"}, -// }, -// TopologyKey: v1.LabelHostname, -// }, -// }}}) -// -// var pods []*v1.Pod -// pods = append(pods, MakePods(10, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: topology, -// })...) -// -// pods = append(pods, affPod2) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pods...) -// // should be scheduled as the pod it has affinity to doesn't exist, but it's only a preference and not a -// // hard constraints -// ExpectScheduled(ctx, env.Client, affPod2) -// -// }) -// It("should allow violation of preferred pod anti-affinity", func() { -// affPods := MakePods(10, test.PodOptions{PodAntiPreferences: []v1.WeightedPodAffinityTerm{ -// { -// Weight: 50, -// PodAffinityTerm: v1.PodAffinityTerm{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: labels, -// }, -// TopologyKey: v1.LabelTopologyZone, -// }, -// }, -// }}) -// -// var pods []*v1.Pod -// pods = append(pods, MakePods(3, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelTopologyZone, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }}, -// })...) -// -// pods = append(pods, affPods...) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pods...) -// for _, aff := range affPods { -// ExpectScheduled(ctx, env.Client, aff) -// } -// -// }) -// It("should separate nodes using simple pod anti-affinity on hostname", func() { -// affLabels := map[string]string{"security": "s2"} -// // pod affinity/anti-affinity are bidirectional, so run this a few times to ensure we handle it regardless -// // of pod scheduling order -// ExpectApplied(ctx, env.Client, provisioner) -// for i := 0; i < 10; i++ { -// affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) -// // affPod2 will avoid affPod1 -// affPod2 := test.UnschedulablePod(test.PodOptions{PodAntiRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelHostname, -// }}}) -// -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, affPod2, affPod1) -// n1 := ExpectScheduled(ctx, env.Client, affPod1) -// n2 := ExpectScheduled(ctx, env.Client, affPod2) -// // should not be scheduled on the same node -// Expect(n1.Name).ToNot(Equal(n2.Name)) -// } -// }) -// It("should not violate pod anti-affinity on zone", func() { -// affLabels := map[string]string{"security": "s2"} -// zone1Pod := test.UnschedulablePod(test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, -// }, -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}) -// zone2Pod := test.UnschedulablePod(test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, -// }, -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}) -// zone3Pod := test.UnschedulablePod(test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, -// }, -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}}) -// -// affPod := test.UnschedulablePod(test.PodOptions{ -// PodAntiRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelTopologyZone, -// }}}) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, zone1Pod, zone2Pod, zone3Pod, affPod) -// // the three larger zone specific pods should get scheduled first due to first fit descending onto one -// // node per zone. -// ExpectScheduled(ctx, env.Client, zone1Pod) -// ExpectScheduled(ctx, env.Client, zone2Pod) -// ExpectScheduled(ctx, env.Client, zone3Pod) -// // the pod with anti-affinity -// ExpectNotScheduled(ctx, env.Client, affPod) -// }) -// It("should not violate pod anti-affinity on zone (other schedules first)", func() { -// affLabels := map[string]string{"security": "s2"} -// pod := test.UnschedulablePod(test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, -// }}) -// affPod := test.UnschedulablePod(test.PodOptions{ -// PodAntiRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelTopologyZone, -// }}}) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod, affPod) -// // the pod we need to avoid schedules first, but we don't know where. -// ExpectScheduled(ctx, env.Client, pod) -// // the pod with anti-affinity -// ExpectNotScheduled(ctx, env.Client, affPod) -// }) -// It("should not violate pod anti-affinity (arch)", func() { -// affLabels := map[string]string{"security": "s2"} -// tsc := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: affLabels}, -// MaxSkew: 1, -// }} -// -// affPod1 := test.UnschedulablePod(test.PodOptions{ -// TopologySpreadConstraints: tsc, -// ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, -// }, -// NodeSelector: map[string]string{ -// v1.LabelArchStable: "arm64", -// }}) -// -// // affPod2 will try to get scheduled on a node with a different archi from affPod1. Due to resource -// // requests we try to schedule it last -// affPod2 := test.UnschedulablePod(test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, -// TopologySpreadConstraints: tsc, -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, -// }, -// PodAntiRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelArchStable, -// }}}) -// -// pods := []*v1.Pod{affPod1, affPod2} -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pods...) -// n1 := ExpectScheduled(ctx, env.Client, affPod1) -// n2 := ExpectScheduled(ctx, env.Client, affPod2) -// // should not be scheduled on nodes with the same arch -// Expect(n1.Labels[v1.LabelArchStable]).ToNot(Equal(n2.Labels[v1.LabelArchStable])) -// }) -// It("should violate preferred pod anti-affinity on zone (inverse)", func() { -// affLabels := map[string]string{"security": "s2"} -// anti := []v1.WeightedPodAffinityTerm{ -// { -// Weight: 10, -// PodAffinityTerm: v1.PodAffinityTerm{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelTopologyZone, -// }, -// }, -// } -// rr := v1.ResourceRequirements{ -// Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, -// } -// zone1Pod := test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: rr, -// PodAntiPreferences: anti, -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}) -// zone2Pod := test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: rr, -// PodAntiPreferences: anti, -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}) -// zone3Pod := test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: rr, -// PodAntiPreferences: anti, -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}}) -// -// affPod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, zone1Pod, zone2Pod, zone3Pod, affPod) -// // three pods with anti-affinity will schedule first due to first fit-descending -// ExpectScheduled(ctx, env.Client, zone1Pod) -// ExpectScheduled(ctx, env.Client, zone2Pod) -// ExpectScheduled(ctx, env.Client, zone3Pod) -// // the anti-affinity was a preference, so this can schedule -// ExpectScheduled(ctx, env.Client, affPod) -// }) -// It("should not violate pod anti-affinity on zone (inverse)", func() { -// affLabels := map[string]string{"security": "s2"} -// anti := []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelTopologyZone, -// }} -// rr := v1.ResourceRequirements{ -// Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, -// } -// zone1Pod := test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: rr, -// PodAntiRequirements: anti, -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}) -// zone2Pod := test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: rr, -// PodAntiRequirements: anti, -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}) -// zone3Pod := test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: rr, -// PodAntiRequirements: anti, -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}}) -// -// affPod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, zone1Pod, zone2Pod, zone3Pod, affPod) -// // three pods with anti-affinity will schedule first due to first fit-descending -// ExpectScheduled(ctx, env.Client, zone1Pod) -// ExpectScheduled(ctx, env.Client, zone2Pod) -// ExpectScheduled(ctx, env.Client, zone3Pod) -// // this pod with no anti-affinity rules can't schedule. It has no anti-affinity rules, but every zone has a -// // pod with anti-affinity rules that prevent it from scheduling -// ExpectNotScheduled(ctx, env.Client, affPod) -// }) -// It("should not violate pod anti-affinity on zone (Schrödinger)", func() { -// affLabels := map[string]string{"security": "s2"} -// anti := []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelTopologyZone, -// }} -// zoneAnywherePod := test.UnschedulablePod(test.PodOptions{ -// PodAntiRequirements: anti, -// ResourceRequirements: v1.ResourceRequirements{ -// Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, -// }, -// }) -// -// affPod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, zoneAnywherePod, affPod) -// // the pod with anti-affinity will schedule first due to first fit-descending, but we don't know which zone it landed in -// node1 := ExpectScheduled(ctx, env.Client, zoneAnywherePod) -// -// // this pod cannot schedule since the pod with anti-affinity could potentially be in any zone -// affPod = ExpectNotScheduled(ctx, env.Client, affPod) -// -// // a second batching will now allow the pod to schedule as the zoneAnywherePod has been committed to a zone -// // by the actual node creation -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, affPod) -// node2 := ExpectScheduled(ctx, env.Client, affPod) -// Expect(node1.Labels[v1.LabelTopologyZone]).ToNot(Equal(node2.Labels[v1.LabelTopologyZone])) -// -// }) -// It("should not violate pod anti-affinity on zone (inverse w/existing nodes)", func() { -// affLabels := map[string]string{"security": "s2"} -// anti := []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelTopologyZone, -// }} -// rr := v1.ResourceRequirements{ -// Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, -// } -// zone1Pod := test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: rr, -// PodAntiRequirements: anti, -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}) -// zone2Pod := test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: rr, -// PodAntiRequirements: anti, -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}) -// zone3Pod := test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: rr, -// PodAntiRequirements: anti, -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}}) -// -// affPod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) -// -// // provision these so we get three nodes that exist in the cluster with anti-affinity to a pod that we will -// // then try to schedule -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, zone1Pod, zone2Pod, zone3Pod) -// node1 := ExpectScheduled(ctx, env.Client, zone1Pod) -// node2 := ExpectScheduled(ctx, env.Client, zone2Pod) -// node3 := ExpectScheduled(ctx, env.Client, zone3Pod) -// -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node2)) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node3)) -// ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone1Pod)) -// ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone2Pod)) -// ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone3Pod)) -// -// ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone1Pod)) -// ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone2Pod)) -// ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone3Pod)) -// -// // this pod with no anti-affinity rules can't schedule. It has no anti-affinity rules, but every zone has an -// // existing pod (not from this batch) with anti-affinity rules that prevent it from scheduling -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, affPod) -// ExpectNotScheduled(ctx, env.Client, affPod) -// }) -// It("should violate preferred pod anti-affinity on zone (inverse w/existing nodes)", func() { -// affLabels := map[string]string{"security": "s2"} -// anti := []v1.WeightedPodAffinityTerm{ -// { -// Weight: 10, -// PodAffinityTerm: v1.PodAffinityTerm{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelTopologyZone, -// }, -// }, -// } -// rr := v1.ResourceRequirements{ -// Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, -// } -// zone1Pod := test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: rr, -// PodAntiPreferences: anti, -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}) -// zone2Pod := test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: rr, -// PodAntiPreferences: anti, -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}) -// zone3Pod := test.UnschedulablePod(test.PodOptions{ -// ResourceRequirements: rr, -// PodAntiPreferences: anti, -// NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}}) -// -// affPod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) -// -// // provision these so we get three nodes that exist in the cluster with anti-affinity to a pod that we will -// // then try to schedule -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, zone1Pod, zone2Pod, zone3Pod) -// node1 := ExpectScheduled(ctx, env.Client, zone1Pod) -// node2 := ExpectScheduled(ctx, env.Client, zone2Pod) -// node3 := ExpectScheduled(ctx, env.Client, zone3Pod) -// -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node2)) -// ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node3)) -// ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone1Pod)) -// ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone2Pod)) -// ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone3Pod)) -// -// // this pod with no anti-affinity rules can schedule, though it couldn't if the anti-affinity were required -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, affPod) -// ExpectScheduled(ctx, env.Client, affPod) -// }) -// It("should allow violation of a pod affinity preference with a conflicting required constraint", func() { -// affLabels := map[string]string{"security": "s2"} -// constraint := v1.TopologySpreadConstraint{ -// MaxSkew: 1, -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: labels, -// }, -// } -// affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) -// affPods := MakePods(3, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// // limit these pods to one per host -// TopologySpreadConstraints: []v1.TopologySpreadConstraint{constraint}, -// // with a preference to the other pod -// PodPreferences: []v1.WeightedPodAffinityTerm{{ -// Weight: 50, -// PodAffinityTerm: v1.PodAffinityTerm{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelHostname, -// }, -// }}}) -// ExpectApplied(ctx, env.Client, provisioner) -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, append(affPods, affPod1)...) -// // all pods should be scheduled since the affinity term is just a preference -// for _, pod := range pods { -// ExpectScheduled(ctx, env.Client, pod) -// } -// // and we'll get three nodes due to the topology spread -// ExpectSkew(ctx, env.Client, "", &constraint).To(ConsistOf(1, 1, 1)) -// }) -// It("should support pod anti-affinity with a zone topology", func() { -// affLabels := map[string]string{"security": "s2"} -// -// // affPods will avoid being scheduled in the same zone -// createPods := func() []*v1.Pod { -// return MakePods(3, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, -// PodAntiRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelTopologyZone, -// }}}) -// } -// -// top := &v1.TopologySpreadConstraint{TopologyKey: v1.LabelTopologyZone} -// -// // One of the downsides of late committal is that absent other constraints, it takes multiple batches of -// // scheduling for zonal anti-affinities to work themselves out. The first schedule, we know that the pod -// // will land in test-zone-1, test-zone-2, or test-zone-3, but don't know which it collapses to until the -// // node is actually created. -// -// // one pod pod will schedule -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, createPods()...) -// ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(1)) -// // delete all of the unscheduled ones as provisioning will only bind pods passed into the provisioning call -// // the scheduler looks at all pods though, so it may assume a pod from this batch schedules and no others do -// ExpectDeleteAllUnscheduledPods(ctx, env.Client) -// -// // second pod in a second zone -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, createPods()...) -// ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(1, 1)) -// ExpectDeleteAllUnscheduledPods(ctx, env.Client) -// -// // third pod in the last zone -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, createPods()...) -// ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(1, 1, 1)) -// ExpectDeleteAllUnscheduledPods(ctx, env.Client) -// -// // and nothing else can schedule -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, createPods()...) -// ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(1, 1, 1)) -// ExpectDeleteAllUnscheduledPods(ctx, env.Client) -// }) -// It("should not schedule pods with affinity to a non-existent pod", func() { -// affLabels := map[string]string{"security": "s2"} -// affPods := MakePods(10, test.PodOptions{ -// PodRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelTopologyZone, -// }}}) -// -// ExpectApplied(ctx, env.Client, provisioner) -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, affPods...) -// // the pod we have affinity to is not in the cluster, so all of these pods are unschedulable -// for _, p := range pods { -// ExpectNotScheduled(ctx, env.Client, p) -// } -// }) -// It("should support pod affinity with zone topology (unconstrained target)", func() { -// affLabels := map[string]string{"security": "s2"} -// -// // the pod that the others have an affinity to -// targetPod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) -// -// // affPods all want to schedule in the same zone as targetPod, but can't as it's zone is undetermined -// affPods := MakePods(10, test.PodOptions{ -// PodRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelTopologyZone, -// }}}) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, append(affPods, targetPod)...) -// top := &v1.TopologySpreadConstraint{TopologyKey: v1.LabelTopologyZone} -// // these pods can't schedule as the pod they have affinity to isn't limited to any particular zone -// for i := range affPods { -// ExpectNotScheduled(ctx, env.Client, affPods[i]) -// affPods[i] = ExpectPodExists(ctx, env.Client, affPods[i].Name, affPods[i].Namespace) -// } -// ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(1)) -// -// // now that targetPod has been scheduled to a node, it's zone is committed and the pods with affinity to it -// // should schedule in the same zone -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, affPods...) -// for _, pod := range affPods { -// ExpectScheduled(ctx, env.Client, pod) -// } -// ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(11)) -// }) -// It("should support pod affinity with zone topology (constrained target)", func() { -// affLabels := map[string]string{"security": "s2"} -// -// // the pod that the others have an affinity to -// affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, -// NodeRequirements: []v1.NodeSelectorRequirement{ -// { -// Key: v1.LabelTopologyZone, -// Operator: v1.NodeSelectorOpIn, -// Values: []string{"test-zone-1"}, -// }, -// }}) -// -// // affPods will all be scheduled in the same zone as affPod1 -// affPods := MakePods(10, test.PodOptions{ -// PodRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelTopologyZone, -// }}}) -// -// affPods = append(affPods, affPod1) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, affPods...) -// top := &v1.TopologySpreadConstraint{TopologyKey: v1.LabelTopologyZone} -// ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(11)) -// }) -// It("should handle multiple dependent affinities", func() { -// dbLabels := map[string]string{"type": "db", "spread": "spread"} -// webLabels := map[string]string{"type": "web", "spread": "spread"} -// cacheLabels := map[string]string{"type": "cache", "spread": "spread"} -// uiLabels := map[string]string{"type": "ui", "spread": "spread"} -// for i := 0; i < 50; i++ { -// ExpectApplied(ctx, env.Client, provisioner.DeepCopy()) -// // we have to schedule DB -> Web -> Cache -> UI in that order or else there are pod affinity violations -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: dbLabels}}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: webLabels}, -// PodRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{MatchLabels: dbLabels}, -// TopologyKey: v1.LabelHostname}, -// }}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: cacheLabels}, -// PodRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{MatchLabels: webLabels}, -// TopologyKey: v1.LabelHostname}, -// }}), -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: uiLabels}, -// PodRequirements: []v1.PodAffinityTerm{ -// { -// LabelSelector: &metav1.LabelSelector{MatchLabels: cacheLabels}, -// TopologyKey: v1.LabelHostname}, -// }}), -// ) -// for i := range pods { -// ExpectScheduled(ctx, env.Client, pods[i]) -// } -// ExpectCleanedUp(ctx, env.Client) -// cluster.Reset() -// } -// }) -// It("should fail to schedule pods with unsatisfiable dependencies", func() { -// dbLabels := map[string]string{"type": "db", "spread": "spread"} -// webLabels := map[string]string{"type": "web", "spread": "spread"} -// ExpectApplied(ctx, env.Client, provisioner) -// // this pods wants to schedule with a non-existent pod, this test just ensures that the scheduling loop -// // doesn't infinite loop -// pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: dbLabels}, -// PodRequirements: []v1.PodAffinityTerm{ -// { -// LabelSelector: &metav1.LabelSelector{MatchLabels: webLabels}, -// TopologyKey: v1.LabelHostname, -// }, -// }}), -// ) -// ExpectNotScheduled(ctx, env.Client, pods[0]) -// }) -// It("should filter pod affinity topologies by namespace, no matching pods", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// -// ExpectApplied(ctx, env.Client, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "other-ns-no-match"}}) -// affLabels := map[string]string{"security": "s2"} -// -// affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels, Namespace: "other-ns-no-match"}}) -// // affPod2 will try to get scheduled with affPod1 -// affPod2 := test.UnschedulablePod(test.PodOptions{PodRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// TopologyKey: v1.LabelHostname, -// }}}) -// -// var pods []*v1.Pod -// // creates 10 nodes due to topo spread -// pods = append(pods, MakePods(10, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: topology, -// })...) -// pods = append(pods, affPod1) -// pods = append(pods, affPod2) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pods...) -// -// // the target pod gets scheduled -// ExpectScheduled(ctx, env.Client, affPod1) -// // but the one with affinity does not since the target pod is not in the same namespace and doesn't -// // match the namespace list or namespace selector -// ExpectNotScheduled(ctx, env.Client, affPod2) -// }) -// It("should filter pod affinity topologies by namespace, matching pods namespace list", func() { -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// -// ExpectApplied(ctx, env.Client, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "other-ns-list"}}) -// affLabels := map[string]string{"security": "s2"} -// -// affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels, Namespace: "other-ns-list"}}) -// // affPod2 will try to get scheduled with affPod1 -// affPod2 := test.UnschedulablePod(test.PodOptions{PodRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// Namespaces: []string{"other-ns-list"}, -// TopologyKey: v1.LabelHostname, -// }}}) -// -// var pods []*v1.Pod -// // create 10 nodes -// pods = append(pods, MakePods(10, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: topology, -// })...) -// // put our target pod on one of them -// pods = append(pods, affPod1) -// // and our pod with affinity should schedule on the same node -// pods = append(pods, affPod2) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pods...) -// n1 := ExpectScheduled(ctx, env.Client, affPod1) -// n2 := ExpectScheduled(ctx, env.Client, affPod2) -// // should be scheduled on the same node -// Expect(n1.Name).To(Equal(n2.Name)) -// }) -// It("should filter pod affinity topologies by namespace, empty namespace selector", func() { -// if env.Version.Minor() < 21 { -// Skip("namespace selector is only supported on K8s >= 1.21.x") -// } -// topology := []v1.TopologySpreadConstraint{{ -// TopologyKey: v1.LabelHostname, -// WhenUnsatisfiable: v1.DoNotSchedule, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// MaxSkew: 1, -// }} -// -// ExpectApplied(ctx, env.Client, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "empty-ns-selector", Labels: map[string]string{"foo": "bar"}}}) -// affLabels := map[string]string{"security": "s2"} -// -// affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels, Namespace: "empty-ns-selector"}}) -// // affPod2 will try to get scheduled with affPod1 -// affPod2 := test.UnschedulablePod(test.PodOptions{PodRequirements: []v1.PodAffinityTerm{{ -// LabelSelector: &metav1.LabelSelector{ -// MatchLabels: affLabels, -// }, -// // select all pods in all namespaces since the selector is empty -// NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{}}, -// TopologyKey: v1.LabelHostname, -// }}}) -// -// var pods []*v1.Pod -// // create 10 nodes -// pods = append(pods, MakePods(10, test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: topology, -// })...) -// // put our target pod on one of them -// pods = append(pods, affPod1) -// // and our pod with affinity should schedule on the same node -// pods = append(pods, affPod2) -// -// ExpectApplied(ctx, env.Client, provisioner) -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pods...) -// n1 := ExpectScheduled(ctx, env.Client, affPod1) -// n2 := ExpectScheduled(ctx, env.Client, affPod2) -// // should be scheduled on the same node due to the empty namespace selector -// Expect(n1.Name).To(Equal(n2.Name)) -// }) -// It("should count topology across multiple provisioners", func() { -// ExpectApplied(ctx, env.Client, -// test.Provisioner(test.ProvisionerOptions{ -// Requirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}}, -// }), -// test.Provisioner(test.ProvisionerOptions{ -// Requirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2", "test-zone-3"}}}, -// }), -// ) -// labels := map[string]string{"foo": "bar"} -// topology := v1.TopologySpreadConstraint{ -// TopologyKey: v1.LabelTopologyZone, -// MaxSkew: 1, -// LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, -// WhenUnsatisfiable: v1.DoNotSchedule, -// } -// ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.Pods(10, test.UnscheduleablePodOptions(test.PodOptions{ -// ObjectMeta: metav1.ObjectMeta{Labels: labels}, -// TopologySpreadConstraints: []v1.TopologySpreadConstraint{topology}, -// }))...) -// ExpectSkew(ctx, env.Client, "default", &topology).To(ConsistOf(3, 3, 4)) -// }) -// }) -//}) +var _ = Describe("Topology", func() { + labels := map[string]string{"test": "test"} + + It("should ignore unknown topology keys", func() { + ExpectApplied(ctx, env.Client, provisioner) + pods := []*v1.Pod{ + test.UnschedulablePod( + test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: []v1.TopologySpreadConstraint{{ + TopologyKey: "unknown", + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }}}, + ), + test.UnschedulablePod(), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + ExpectNotScheduled(ctx, env.Client, pods[0]) + ExpectScheduled(ctx, env.Client, pods[1]) + }) + + It("should not spread an invalid label selector", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app.kubernetes.io/name": "{{ zqfmgb }}"}}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(2, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})...) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2)) + }) + + Context("Zonal", func() { + It("should balance pods across zones (match labels)", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 2)) + }) + It("should balance pods across zones (match expressions)", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "test", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"test"}, + }, + }, + }, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 2)) + }) + It("should respect provisioner zonal constraints", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}} + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 2)) + }) + It("should respect provisioner zonal constraints (subset)", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + ) + // should spread the two pods evenly across the only valid zones in our universe (the two zones from our single provisioner) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2)) + }) + It("should respect provisioner zonal constraints (existing pod)", func() { + ExpectApplied(ctx, env.Client, provisioner) + // need enough resource requests that the first node we create fills a node and can't act as an in-flight + // node for the other pods + rr := v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1.1"), + }, + } + pod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, + ResourceRequirements: rr, + NodeSelector: map[string]string{ + v1.LabelTopologyZone: "test-zone-3", + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, ResourceRequirements: rr, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, ResourceRequirements: rr, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, ResourceRequirements: rr, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, ResourceRequirements: rr, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, ResourceRequirements: rr, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, ResourceRequirements: rr, TopologySpreadConstraints: topology}), + ) + // we should have unschedulable pods now, the provisioner can only schedule to zone-1/zone-2, but because of the existing + // pod in zone-3 it can put a max of two per zone before it would violate max skew + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 2, 2)) + }) + It("should schedule to the non-minimum domain if its all that's available", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 5, + }} + rr := v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1.1"), + }, + } + // force this pod onto zone-1 + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, + ResourceRequirements: rr, TopologySpreadConstraints: topology})) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1)) + + // force this pod onto zone-2 + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, + ResourceRequirements: rr, TopologySpreadConstraints: topology})) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1)) + + // now only allow scheduling pods on zone-3 + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(10, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, + ResourceRequirements: rr, TopologySpreadConstraints: topology})..., + ) + + // max skew of 5, so test-zone-1/2 will have 1 pod each, test-zone-3 will have 6, and the rest will fail to schedule + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 6)) + }) + It("should only schedule to minimum domains if already violating max skew", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + rr := v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1.1"), + }, + } + createPods := func(count int) []*v1.Pod { + var pods []*v1.Pod + for i := 0; i < count; i++ { + pods = append(pods, test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, + ResourceRequirements: rr, TopologySpreadConstraints: topology})) + } + return pods + } + // Spread 9 pods + ExpectApplied(ctx, env.Client, provisioner) + pods := createPods(9) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 3, 3)) + + // Delete pods to create a skew + for _, pod := range pods { + node := ExpectScheduled(ctx, env.Client, pod) + if node.Labels[v1.LabelTopologyZone] != "test-zone-1" { + ExpectDeleted(ctx, env.Client, pod) + } + } + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3)) + + // Create 3 more pods, skew should recover + ExpectProvisioned(ctx, env.Client, cluster, prov, createPods(3)...) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 1, 2)) + }) + It("should not violate max-skew when unsat = do not schedule", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + rr := v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1.1"), + }, + } + // force this pod onto zone-1 + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, + ResourceRequirements: rr, TopologySpreadConstraints: topology})) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1)) + + // now only allow scheduling pods on zone-2 and zone-3 + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2", "test-zone-3"}}} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(10, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, + ResourceRequirements: rr, TopologySpreadConstraints: topology})..., + ) + + // max skew of 1, so test-zone-2/3 will have 2 nodes each and the rest of the pods will fail to schedule + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 2, 2)) + }) + It("should not violate max-skew when unsat = do not schedule (discover domains)", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + rr := v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1.1"), + }, + } + // force this pod onto zone-1 + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, ResourceRequirements: rr})) + + // now only allow scheduling pods on zone-2 and zone-3 + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2", "test-zone-3"}}} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(10, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: topology, ResourceRequirements: rr})..., + ) + + // max skew of 1, so test-zone-2/3 will have 2 nodes each and the rest of the pods will fail to schedule since + // test-zone-1 has 1 pods in it. + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 2, 2)) + }) + It("should only count running/scheduled pods with matching labels scheduled to nodes with a corresponding domain", func() { + wrongNamespace := test.RandomName() + firstNode := test.Node(test.NodeOptions{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}}) + secondNode := test.Node(test.NodeOptions{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}}) + thirdNode := test.Node(test.NodeOptions{}) // missing topology domain + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner, firstNode, secondNode, thirdNode, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: wrongNamespace}}) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(firstNode)) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(secondNode)) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(thirdNode)) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.Pod(test.PodOptions{NodeName: firstNode.Name}), // ignored, missing labels + test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}}), // ignored, pending + test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: thirdNode.Name}), // ignored, no domain on node + test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels, Namespace: wrongNamespace}, NodeName: firstNode.Name}), // ignored, wrong namespace + test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels, DeletionTimestamp: &metav1.Time{Time: time.Now().Add(10 * time.Second)}}}), // ignored, terminating + test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name, Phase: v1.PodFailed}), // ignored, phase=Failed + test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name, Phase: v1.PodSucceeded}), // ignored, phase=Succeeded + test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name}), + test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name}), + test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: secondNode.Name}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + ) + nodes := v1.NodeList{} + Expect(env.Client.List(ctx, &nodes)).To(Succeed()) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2, 1)) + }) + It("should match all pods when labelSelector is not specified", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(), + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1)) + }) + It("should handle interdependent selectors", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + pods := MakePods(5, test.PodOptions{TopologySpreadConstraints: topology}) + ExpectProvisioned(ctx, env.Client, cluster, prov, + pods..., + ) + // This is weird, but the topology label selector is used for determining domain counts. The pod that + // owns the topology is what the spread actually applies to. In this test case, there are no pods matching + // the label selector, so the max skew is zero. This means we can pack all the pods onto the same node since + // it doesn't violate the topology spread constraint (i.e. adding new pods doesn't increase skew since the + // pods we are adding don't count toward skew). This behavior is called out at + // https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ , though it's not + // recommended for users. + nodeNames := sets.NewString() + for _, p := range pods { + nodeNames.Insert(p.Spec.NodeName) + } + Expect(nodeNames).To(HaveLen(1)) + }) + }) + + Context("Hostname", func() { + It("should balance pods across nodes", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1, 1, 1)) + }) + It("should balance pods on the same hostname up to maxskew", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 4, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(4)) + }) + It("balance multiple deployments with hostname topology spread", func() { + // Issue #1425 + spreadPod := func(appName string) test.PodOptions { + return test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": appName, + }, + }, + TopologySpreadConstraints: []v1.TopologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": appName}, + }, + }, + }, + } + } + + ExpectApplied(ctx, env.Client, provisioner) + pods := []*v1.Pod{ + test.UnschedulablePod(spreadPod("app1")), test.UnschedulablePod(spreadPod("app1")), + test.UnschedulablePod(spreadPod("app2")), test.UnschedulablePod(spreadPod("app2")), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + for _, p := range pods { + ExpectScheduled(ctx, env.Client, p) + } + nodes := v1.NodeList{} + Expect(env.Client.List(ctx, &nodes)).To(Succeed()) + // this wasn't part of #1425, but ensures that we launch the minimum number of nodes + Expect(nodes.Items).To(HaveLen(2)) + }) + It("balance multiple deployments with hostname topology spread & varying arch", func() { + // Issue #1425 + spreadPod := func(appName, arch string) test.PodOptions { + return test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": appName, + }, + }, + NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{arch}, + }, + }, + TopologySpreadConstraints: []v1.TopologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": appName}, + }, + }, + }, + } + } + + ExpectApplied(ctx, env.Client, provisioner) + pods := []*v1.Pod{ + test.UnschedulablePod(spreadPod("app1", v1alpha5.ArchitectureAmd64)), test.UnschedulablePod(spreadPod("app1", v1alpha5.ArchitectureAmd64)), + test.UnschedulablePod(spreadPod("app2", v1alpha5.ArchitectureArm64)), test.UnschedulablePod(spreadPod("app2", v1alpha5.ArchitectureArm64)), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + for _, p := range pods { + ExpectScheduled(ctx, env.Client, p) + } + nodes := v1.NodeList{} + Expect(env.Client.List(ctx, &nodes)).To(Succeed()) + // same test as the previous one, but now the architectures are different so we need four nodes in total + Expect(nodes.Items).To(HaveLen(4)) + }) + }) + + Context("CapacityType", func() { + It("should balance pods across capacity types", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1alpha5.LabelCapacityType, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2)) + }) + It("should respect provisioner capacity type constraints", func() { + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeSpot, v1alpha5.CapacityTypeOnDemand}}} + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1alpha5.LabelCapacityType, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2)) + }) + It("should not violate max-skew when unsat = do not schedule (capacity type)", func() { + // this test can pass in a flaky manner if we don't restrict our min domain selection to valid choices + // per the provisioner spec + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1alpha5.LabelCapacityType, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + rr := v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1.1"), + }, + } + // force this pod onto spot + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeSpot}}} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, + ResourceRequirements: rr, TopologySpreadConstraints: topology})) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1)) + + // now only allow scheduling pods on on-demand + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeOnDemand}}} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(5, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, + ResourceRequirements: rr, TopologySpreadConstraints: topology})..., + ) + + // max skew of 1, so on-demand will have 2 pods and the rest of the pods will fail to schedule + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 2)) + }) + It("should violate max-skew when unsat = schedule anyway (capacity type)", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1alpha5.LabelCapacityType, + WhenUnsatisfiable: v1.ScheduleAnyway, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + rr := v1.ResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1.1"), + }, + } + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeSpot}}} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, + ResourceRequirements: rr, TopologySpreadConstraints: topology})) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1)) + + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeOnDemand}}} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(5, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, + ResourceRequirements: rr, TopologySpreadConstraints: topology})..., + ) + + // max skew of 1, on-demand will end up with 5 pods even though spot has a single pod + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 5)) + }) + It("should only count running/scheduled pods with matching labels scheduled to nodes with a corresponding domain", func() { + wrongNamespace := test.RandomName() + firstNode := test.Node(test.NodeOptions{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{v1alpha5.LabelCapacityType: v1alpha5.CapacityTypeSpot}}}) + secondNode := test.Node(test.NodeOptions{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{v1alpha5.LabelCapacityType: v1alpha5.CapacityTypeOnDemand}}}) + thirdNode := test.Node(test.NodeOptions{}) // missing topology capacity type + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1alpha5.LabelCapacityType, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner, firstNode, secondNode, thirdNode, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: wrongNamespace}}) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(firstNode)) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(secondNode)) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(thirdNode)) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.Pod(test.PodOptions{NodeName: firstNode.Name}), // ignored, missing labels + test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}}), // ignored, pending + test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: thirdNode.Name}), // ignored, no domain on node + test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels, Namespace: wrongNamespace}, NodeName: firstNode.Name}), // ignored, wrong namespace + test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels, DeletionTimestamp: &metav1.Time{Time: time.Now().Add(10 * time.Second)}}}), // ignored, terminating + test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name, Phase: v1.PodFailed}), // ignored, phase=Failed + test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name, Phase: v1.PodSucceeded}), // ignored, phase=Succeeded + test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name}), + test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: firstNode.Name}), + test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, NodeName: secondNode.Name}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}), + ) + nodes := v1.NodeList{} + Expect(env.Client.List(ctx, &nodes)).To(Succeed()) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 3)) + }) + It("should match all pods when labelSelector is not specified", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1alpha5.LabelCapacityType, + WhenUnsatisfiable: v1.DoNotSchedule, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + test.UnschedulablePod(), + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1)) + }) + It("should handle interdependent selectors", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + pods := MakePods(5, test.PodOptions{TopologySpreadConstraints: topology}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + // This is weird, but the topology label selector is used for determining domain counts. The pod that + // owns the topology is what the spread actually applies to. In this test case, there are no pods matching + // the label selector, so the max skew is zero. This means we can pack all the pods onto the same node since + // it doesn't violate the topology spread constraint (i.e. adding new pods doesn't increase skew since the + // pods we are adding don't count toward skew). This behavior is called out at + // https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ , though it's not + // recommended for users. + nodeNames := sets.NewString() + for _, p := range pods { + nodeNames.Insert(p.Spec.NodeName) + } + Expect(nodeNames).To(HaveLen(1)) + }) + It("should balance pods across capacity-types (node required affinity constrained)", func() { + ExpectApplied(ctx, env.Client, provisioner) + pods := MakePods(1, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + NodeRequirements: []v1.NodeSelectorRequirement{ + // launch this on-demand pod in zone-1 + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}, + {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{"on-demand"}}, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + ExpectScheduled(ctx, env.Client, pods[0]) + + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1alpha5.LabelCapacityType, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + + // Try to run 5 pods, with a node selector restricted to test-zone-2, they should all schedule on the same + // spot node. This doesn't violate the max-skew of 1 as the node selector requirement here excludes the + // existing on-demand pod from counting within this topology. + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(5, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + // limit our provisioner to only creating spot nodes + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}, + {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{"spot"}}, + }, + TopologySpreadConstraints: topology, + })..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 5)) + }) + It("should balance pods across capacity-types (no constraints)", func() { + rr := v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, + } + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod(test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "single-pod-instance-type"}, + NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1alpha5.LabelCapacityType, + Operator: v1.NodeSelectorOpIn, + Values: []string{"on-demand"}, + }, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1alpha5.LabelCapacityType, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + + // limit our provisioner to only creating spot nodes + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{"spot"}}, + } + + // since there is no node selector on this pod, the topology can see the single on-demand node that already + // exists and that limits us to scheduling 2 more spot pods before we would violate max-skew + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(5, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + ResourceRequirements: rr, + TopologySpreadConstraints: topology, + })..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 2)) + }) + It("should balance pods across arch (no constraints)", func() { + rr := v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, + } + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod(test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + NodeSelector: map[string]string{v1.LabelInstanceTypeStable: "single-pod-instance-type"}, + NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"amd64"}, + }, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + + ExpectScheduled(ctx, env.Client, pod) + + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelArchStable, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + + // limit our provisioner to only creating arm64 nodes + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelArchStable, Operator: v1.NodeSelectorOpIn, Values: []string{"arm64"}}} + + // since there is no node selector on this pod, the topology can see the single arm64 node that already + // exists and that limits us to scheduling 2 more spot pods before we would violate max-skew + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(5, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + ResourceRequirements: rr, + TopologySpreadConstraints: topology, + })..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 2)) + }) + }) + + Context("Combined Hostname and Zonal Topology", func() { + It("should spread pods while respecting both constraints (hostname and zonal)", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }, { + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 3, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(2, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1)) + ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(3, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(2, 2, 1)) + ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(5, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(4, 3, 3)) + ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(11, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(7, 7, 7)) + ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) + }) + It("should balance pods across provisioner requirements", func() { + spotProv := test.Provisioner(test.ProvisionerOptions{ + Requirements: []v1.NodeSelectorRequirement{ + { + Key: v1alpha5.LabelCapacityType, + Operator: v1.NodeSelectorOpIn, + Values: []string{"spot"}, + }, + { + Key: "capacity.spread.4-1", + Operator: v1.NodeSelectorOpIn, + Values: []string{"2", "3", "4", "5"}, + }, + }, + }) + onDemandProv := test.Provisioner(test.ProvisionerOptions{ + Requirements: []v1.NodeSelectorRequirement{ + { + Key: v1alpha5.LabelCapacityType, + Operator: v1.NodeSelectorOpIn, + Values: []string{"on-demand"}, + }, + { + Key: "capacity.spread.4-1", + Operator: v1.NodeSelectorOpIn, + Values: []string{"1"}, + }, + }, + }) + + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: "capacity.spread.4-1", + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, spotProv, onDemandProv) + pods := MakePods(20, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: topology, + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + for _, p := range pods { + ExpectScheduled(ctx, env.Client, p) + } + + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(4, 4, 4, 4, 4)) + // due to the spread across provisioners, we've forced a 4:1 spot to on-demand spread + ExpectSkew(ctx, env.Client, "default", &v1.TopologySpreadConstraint{ + TopologyKey: v1alpha5.LabelCapacityType, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }).To(ConsistOf(4, 16)) + }) + + It("should spread pods while respecting both constraints", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }, { + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.ScheduleAnyway, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}} + + // create a second provisioner that can't provision at all + provisionerB := test.Provisioner() + provisionerB.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-3"}}} + provisionerB.Spec.Limits = &v1alpha5.Limits{ + Resources: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("0"), + }, + } + + ExpectApplied(ctx, env.Client, provisioner, provisionerB) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(10, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., + ) + + // should get one pod per zone, can't schedule to test-zone-3 since that provisioner is effectively disabled + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1)) + // and one pod per node + ExpectSkew(ctx, env.Client, "default", &topology[1]).To(ConsistOf(1, 1)) + }) + + It("should spread pods while respecting both constraints", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1alpha5.LabelCapacityType, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }, { + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 3, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(2, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(1, 1)) + ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(3, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 2)) + ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(5, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(5, 5)) + ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(11, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(11, 10)) + ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 3))) + }) + }) + + Context("Combined Zonal and Capacity Type Topology", func() { + It("should spread pods while respecting both constraints", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1alpha5.LabelCapacityType, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }, { + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(2, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).ToNot(ContainElements(BeNumerically(">", 1))) + ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 1))) + + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(3, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).ToNot(ContainElements(BeNumerically(">", 3))) + ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 2))) + + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(5, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).ToNot(ContainElements(BeNumerically(">", 5))) + ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 4))) + + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(11, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology})..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).ToNot(ContainElements(BeNumerically(">", 11))) + ExpectSkew(ctx, env.Client, "default", &topology[1]).ToNot(ContainElements(BeNumerically(">", 7))) + }) + }) + + Context("Combined Hostname, Zonal, and Capacity Type Topology", func() { + It("should spread pods while respecting all constraints", func() { + // ensure we've got an instance type for every zone/capacity-type pair + cloudProv.InstanceTypes = fake.InstanceTypesAssorted() + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1alpha5.LabelCapacityType, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }, { + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 2, + }, { + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 3, + }} + + // add varying numbers of pods, checking after each scheduling to ensure that our max required max skew + // has not been violated for each constraint + for i := 1; i < 15; i++ { + pods := MakePods(i, test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: labels}, TopologySpreadConstraints: topology}) + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + ExpectMaxSkew(ctx, env.Client, "default", &topology[0]).To(BeNumerically("<=", 1)) + ExpectMaxSkew(ctx, env.Client, "default", &topology[1]).To(BeNumerically("<=", 2)) + ExpectMaxSkew(ctx, env.Client, "default", &topology[2]).To(BeNumerically("<=", 3)) + for _, pod := range pods { + ExpectScheduled(ctx, env.Client, pod) + } + } + }) + }) + + // https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#interaction-with-node-affinity-and-node-selectors + Context("Combined Zonal Topology and Machine Affinity", func() { + It("should limit spread options by nodeSelector", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + append( + MakePods(5, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: topology, + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}, + }), + MakePods(10, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: topology, + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}, + })..., + )..., + ) + // we limit the zones of each pod via node selectors, which causes the topology spreads to only consider + // the single zone as the only valid domain for the topology spread allowing us to schedule multiple pods per domain + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(5, 10)) + }) + It("should limit spread options by node requirements", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(10, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: topology, + NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-1", "test-zone-2"}, + }, + }, + })...) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(5, 5)) + }) + It("should limit spread options by node affinity", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(6, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: topology, + NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{ + "test-zone-1", "test-zone-2", + }}}, + })...) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 3)) + + // open the provisioner back to up so it can see all zones again + provisioner.Spec.Requirements = []v1.NodeSelectorRequirement{ + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2", "test-zone-3"}}} + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, MakePods(1, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: topology, + NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{ + "test-zone-2", "test-zone-3", + }}}, + })...) + + // it will schedule on the currently empty zone-3 even though max-skew is violated as it improves max-skew + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 3, 1)) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(5, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: topology, + })..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(4, 4, 4)) + }) + }) + + // https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#interaction-with-node-affinity-and-node-selectors + Context("Combined Capacity Type Topology and Machine Affinity", func() { + It("should limit spread options by nodeSelector", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1alpha5.LabelCapacityType, + WhenUnsatisfiable: v1.ScheduleAnyway, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + append( + MakePods(5, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: topology, + NodeSelector: map[string]string{v1alpha5.LabelCapacityType: v1alpha5.CapacityTypeSpot}, + }), + MakePods(5, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: topology, + NodeSelector: map[string]string{v1alpha5.LabelCapacityType: v1alpha5.CapacityTypeOnDemand}, + })..., + )..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(5, 5)) + }) + It("should limit spread options by node affinity (capacity type)", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1alpha5.LabelCapacityType, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + + // need to limit the rules to spot or else it will know that on-demand has 0 pods and won't violate the max-skew + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(3, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: topology, + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeSpot}}, + }, + })...) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3)) + + // open the rules back to up so it can see all capacity types + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, MakePods(1, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: topology, + NodeRequirements: []v1.NodeSelectorRequirement{ + {Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeOnDemand, v1alpha5.CapacityTypeSpot}}, + }, + })...) + + // it will schedule on the currently empty on-demand even though max-skew is violated as it improves max-skew + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(3, 1)) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, + MakePods(5, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: topology, + })..., + ) + ExpectSkew(ctx, env.Client, "default", &topology[0]).To(ConsistOf(5, 4)) + }) + }) + + Context("Pod Affinity/Anti-Affinity", func() { + It("should schedule a pod with empty pod affinity and anti-affinity", func() { + ExpectApplied(ctx, env.Client) + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod(test.PodOptions{ + PodRequirements: []v1.PodAffinityTerm{}, + PodAntiRequirements: []v1.PodAffinityTerm{}, + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should respect pod affinity (hostname)", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + + affLabels := map[string]string{"security": "s2"} + + affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) + // affPod2 will try to get scheduled with affPod1 + affPod2 := test.UnschedulablePod(test.PodOptions{PodRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelHostname, + }}}) + + var pods []*v1.Pod + pods = append(pods, MakePods(10, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: topology, + })...) + pods = append(pods, affPod1) + pods = append(pods, affPod2) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + n1 := ExpectScheduled(ctx, env.Client, affPod1) + n2 := ExpectScheduled(ctx, env.Client, affPod2) + // should be scheduled on the same node + Expect(n1.Name).To(Equal(n2.Name)) + }) + It("should respect pod affinity (arch)", func() { + affLabels := map[string]string{"security": "s2"} + tsc := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: affLabels}, + MaxSkew: 1, + }} + + affPod1 := test.UnschedulablePod(test.PodOptions{ + TopologySpreadConstraints: tsc, + ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, + }, + NodeSelector: map[string]string{ + v1.LabelArchStable: "arm64", + }}) + // affPod2 will try to get scheduled with affPod1 + affPod2 := test.UnschedulablePod(test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, + TopologySpreadConstraints: tsc, + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + }, + PodRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelArchStable, + }}}) + + pods := []*v1.Pod{affPod1, affPod2} + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + n1 := ExpectScheduled(ctx, env.Client, affPod1) + n2 := ExpectScheduled(ctx, env.Client, affPod2) + // should be scheduled on a node with the same arch + Expect(n1.Labels[v1.LabelArchStable]).To(Equal(n2.Labels[v1.LabelArchStable])) + // but due to TSC, not on the same node + Expect(n1.Name).ToNot(Equal(n2.Name)) + }) + It("should respect self pod affinity (hostname)", func() { + affLabels := map[string]string{"security": "s2"} + + pods := MakePods(3, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: affLabels, + }, + PodRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelHostname, + }}, + }) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + nodeNames := map[string]struct{}{} + for _, p := range pods { + n := ExpectScheduled(ctx, env.Client, p) + nodeNames[n.Name] = struct{}{} + } + Expect(len(nodeNames)).To(Equal(1)) + }) + It("should respect self pod affinity for first empty topology domain only (hostname)", func() { + affLabels := map[string]string{"security": "s2"} + createPods := func() []*v1.Pod { + return MakePods(10, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: affLabels, + }, + PodRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelHostname, + }}, + }) + } + ExpectApplied(ctx, env.Client, provisioner) + pods := createPods() + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + nodeNames := map[string]struct{}{} + unscheduledCount := 0 + scheduledCount := 0 + for _, p := range pods { + p = ExpectPodExists(ctx, env.Client, p.Name, p.Namespace) + if p.Spec.NodeName == "" { + unscheduledCount++ + } else { + nodeNames[p.Spec.NodeName] = struct{}{} + scheduledCount++ + } + } + // the node can only hold 5 pods, so we should get a single node with 5 pods and 5 unschedulable pods from that batch + Expect(len(nodeNames)).To(Equal(1)) + Expect(scheduledCount).To(BeNumerically("==", 5)) + Expect(unscheduledCount).To(BeNumerically("==", 5)) + + // and pods in a different batch should not schedule as well even if the node is not ready yet + pods = createPods() + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + for _, p := range pods { + ExpectNotScheduled(ctx, env.Client, p) + } + }) + It("should respect self pod affinity for first empty topology domain only (hostname/constrained zones)", func() { + affLabels := map[string]string{"security": "s2"} + // put one pod in test-zone-1, this does affect pod affinity even though we have different node selectors. + // The node selector and required node affinity restrictions to topology counting only apply to topology spread. + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, test.UnschedulablePod(test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: affLabels, + }, + NodeSelector: map[string]string{ + v1.LabelTopologyZone: "test-zone-1", + }, + PodRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelHostname, + }}, + })) + + pods := MakePods(10, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: affLabels, + }, + NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-2", "test-zone-3"}, + }, + }, + PodRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelHostname, + }}, + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + for _, p := range pods { + // none of this should schedule + ExpectNotScheduled(ctx, env.Client, p) + } + }) + It("should respect self pod affinity (zone)", func() { + affLabels := map[string]string{"security": "s2"} + + pods := MakePods(3, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: affLabels, + }, + PodRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelTopologyZone, + }}, + }) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + nodeNames := map[string]struct{}{} + for _, p := range pods { + n := ExpectScheduled(ctx, env.Client, p) + nodeNames[n.Name] = struct{}{} + } + Expect(len(nodeNames)).To(Equal(1)) + }) + It("should respect self pod affinity (zone w/ constraint)", func() { + affLabels := map[string]string{"security": "s2"} + // the pod needs to provide it's own zonal affinity, but we further limit it to only being on test-zone-3 + pods := MakePods(3, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: affLabels, + }, + PodRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelTopologyZone, + }}, + NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-3"}, + }, + }, + }) + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + nodeNames := map[string]struct{}{} + for _, p := range pods { + n := ExpectScheduled(ctx, env.Client, p) + nodeNames[n.Name] = struct{}{} + Expect(n.Labels[v1.LabelTopologyZone]).To(Equal("test-zone-3")) + } + Expect(len(nodeNames)).To(Equal(1)) + }) + It("should allow violation of preferred pod affinity", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + + affPod2 := test.UnschedulablePod(test.PodOptions{PodPreferences: []v1.WeightedPodAffinityTerm{{ + Weight: 50, + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"security": "s2"}, + }, + TopologyKey: v1.LabelHostname, + }, + }}}) + + var pods []*v1.Pod + pods = append(pods, MakePods(10, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: topology, + })...) + + pods = append(pods, affPod2) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + // should be scheduled as the pod it has affinity to doesn't exist, but it's only a preference and not a + // hard constraints + ExpectScheduled(ctx, env.Client, affPod2) + + }) + It("should allow violation of preferred pod anti-affinity", func() { + affPods := MakePods(10, test.PodOptions{PodAntiPreferences: []v1.WeightedPodAffinityTerm{ + { + Weight: 50, + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + TopologyKey: v1.LabelTopologyZone, + }, + }, + }}) + + var pods []*v1.Pod + pods = append(pods, MakePods(3, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }}, + })...) + + pods = append(pods, affPods...) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + for _, aff := range affPods { + ExpectScheduled(ctx, env.Client, aff) + } + + }) + It("should separate nodes using simple pod anti-affinity on hostname", func() { + affLabels := map[string]string{"security": "s2"} + // pod affinity/anti-affinity are bidirectional, so run this a few times to ensure we handle it regardless + // of pod scheduling order + ExpectApplied(ctx, env.Client, provisioner) + for i := 0; i < 10; i++ { + affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) + // affPod2 will avoid affPod1 + affPod2 := test.UnschedulablePod(test.PodOptions{PodAntiRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelHostname, + }}}) + + ExpectProvisioned(ctx, env.Client, cluster, prov, affPod2, affPod1) + n1 := ExpectScheduled(ctx, env.Client, affPod1) + n2 := ExpectScheduled(ctx, env.Client, affPod2) + // should not be scheduled on the same node + Expect(n1.Name).ToNot(Equal(n2.Name)) + } + }) + It("should not violate pod anti-affinity on zone", func() { + affLabels := map[string]string{"security": "s2"} + zone1Pod := test.UnschedulablePod(test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, + }, + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}) + zone2Pod := test.UnschedulablePod(test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, + }, + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}) + zone3Pod := test.UnschedulablePod(test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, + }, + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}}) + + affPod := test.UnschedulablePod(test.PodOptions{ + PodAntiRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelTopologyZone, + }}}) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, zone1Pod, zone2Pod, zone3Pod, affPod) + // the three larger zone specific pods should get scheduled first due to first fit descending onto one + // node per zone. + ExpectScheduled(ctx, env.Client, zone1Pod) + ExpectScheduled(ctx, env.Client, zone2Pod) + ExpectScheduled(ctx, env.Client, zone3Pod) + // the pod with anti-affinity + ExpectNotScheduled(ctx, env.Client, affPod) + }) + It("should not violate pod anti-affinity on zone (other schedules first)", func() { + affLabels := map[string]string{"security": "s2"} + pod := test.UnschedulablePod(test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, + }}) + affPod := test.UnschedulablePod(test.PodOptions{ + PodAntiRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelTopologyZone, + }}}) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod, affPod) + // the pod we need to avoid schedules first, but we don't know where. + ExpectScheduled(ctx, env.Client, pod) + // the pod with anti-affinity + ExpectNotScheduled(ctx, env.Client, affPod) + }) + It("should not violate pod anti-affinity (arch)", func() { + affLabels := map[string]string{"security": "s2"} + tsc := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: affLabels}, + MaxSkew: 1, + }} + + affPod1 := test.UnschedulablePod(test.PodOptions{ + TopologySpreadConstraints: tsc, + ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, + }, + NodeSelector: map[string]string{ + v1.LabelArchStable: "arm64", + }}) + + // affPod2 will try to get scheduled on a node with a different archi from affPod1. Due to resource + // requests we try to schedule it last + affPod2 := test.UnschedulablePod(test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, + TopologySpreadConstraints: tsc, + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + }, + PodAntiRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelArchStable, + }}}) + + pods := []*v1.Pod{affPod1, affPod2} + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + n1 := ExpectScheduled(ctx, env.Client, affPod1) + n2 := ExpectScheduled(ctx, env.Client, affPod2) + // should not be scheduled on nodes with the same arch + Expect(n1.Labels[v1.LabelArchStable]).ToNot(Equal(n2.Labels[v1.LabelArchStable])) + }) + It("should violate preferred pod anti-affinity on zone (inverse)", func() { + affLabels := map[string]string{"security": "s2"} + anti := []v1.WeightedPodAffinityTerm{ + { + Weight: 10, + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelTopologyZone, + }, + }, + } + rr := v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, + } + zone1Pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: rr, + PodAntiPreferences: anti, + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}) + zone2Pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: rr, + PodAntiPreferences: anti, + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}) + zone3Pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: rr, + PodAntiPreferences: anti, + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}}) + + affPod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, zone1Pod, zone2Pod, zone3Pod, affPod) + // three pods with anti-affinity will schedule first due to first fit-descending + ExpectScheduled(ctx, env.Client, zone1Pod) + ExpectScheduled(ctx, env.Client, zone2Pod) + ExpectScheduled(ctx, env.Client, zone3Pod) + // the anti-affinity was a preference, so this can schedule + ExpectScheduled(ctx, env.Client, affPod) + }) + It("should not violate pod anti-affinity on zone (inverse)", func() { + affLabels := map[string]string{"security": "s2"} + anti := []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelTopologyZone, + }} + rr := v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, + } + zone1Pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: rr, + PodAntiRequirements: anti, + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}) + zone2Pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: rr, + PodAntiRequirements: anti, + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}) + zone3Pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: rr, + PodAntiRequirements: anti, + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}}) + + affPod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, zone1Pod, zone2Pod, zone3Pod, affPod) + // three pods with anti-affinity will schedule first due to first fit-descending + ExpectScheduled(ctx, env.Client, zone1Pod) + ExpectScheduled(ctx, env.Client, zone2Pod) + ExpectScheduled(ctx, env.Client, zone3Pod) + // this pod with no anti-affinity rules can't schedule. It has no anti-affinity rules, but every zone has a + // pod with anti-affinity rules that prevent it from scheduling + ExpectNotScheduled(ctx, env.Client, affPod) + }) + It("should not violate pod anti-affinity on zone (Schrödinger)", func() { + affLabels := map[string]string{"security": "s2"} + anti := []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelTopologyZone, + }} + zoneAnywherePod := test.UnschedulablePod(test.PodOptions{ + PodAntiRequirements: anti, + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, + }, + }) + + affPod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, zoneAnywherePod, affPod) + // the pod with anti-affinity will schedule first due to first fit-descending, but we don't know which zone it landed in + node1 := ExpectScheduled(ctx, env.Client, zoneAnywherePod) + + // this pod cannot schedule since the pod with anti-affinity could potentially be in any zone + affPod = ExpectNotScheduled(ctx, env.Client, affPod) + + // a second batching will now allow the pod to schedule as the zoneAnywherePod has been committed to a zone + // by the actual node creation + ExpectProvisioned(ctx, env.Client, cluster, prov, affPod) + node2 := ExpectScheduled(ctx, env.Client, affPod) + Expect(node1.Labels[v1.LabelTopologyZone]).ToNot(Equal(node2.Labels[v1.LabelTopologyZone])) + + }) + It("should not violate pod anti-affinity on zone (inverse w/existing nodes)", func() { + affLabels := map[string]string{"security": "s2"} + anti := []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelTopologyZone, + }} + rr := v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, + } + zone1Pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: rr, + PodAntiRequirements: anti, + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}) + zone2Pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: rr, + PodAntiRequirements: anti, + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}) + zone3Pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: rr, + PodAntiRequirements: anti, + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}}) + + affPod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) + + // provision these so we get three nodes that exist in the cluster with anti-affinity to a pod that we will + // then try to schedule + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, zone1Pod, zone2Pod, zone3Pod) + node1 := ExpectScheduled(ctx, env.Client, zone1Pod) + node2 := ExpectScheduled(ctx, env.Client, zone2Pod) + node3 := ExpectScheduled(ctx, env.Client, zone3Pod) + + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node2)) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node3)) + ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone1Pod)) + ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone2Pod)) + ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone3Pod)) + + ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone1Pod)) + ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone2Pod)) + ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone3Pod)) + + // this pod with no anti-affinity rules can't schedule. It has no anti-affinity rules, but every zone has an + // existing pod (not from this batch) with anti-affinity rules that prevent it from scheduling + ExpectProvisioned(ctx, env.Client, cluster, prov, affPod) + ExpectNotScheduled(ctx, env.Client, affPod) + }) + It("should violate preferred pod anti-affinity on zone (inverse w/existing nodes)", func() { + affLabels := map[string]string{"security": "s2"} + anti := []v1.WeightedPodAffinityTerm{ + { + Weight: 10, + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelTopologyZone, + }, + }, + } + rr := v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, + } + zone1Pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: rr, + PodAntiPreferences: anti, + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-1"}}) + zone2Pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: rr, + PodAntiPreferences: anti, + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-2"}}) + zone3Pod := test.UnschedulablePod(test.PodOptions{ + ResourceRequirements: rr, + PodAntiPreferences: anti, + NodeSelector: map[string]string{v1.LabelTopologyZone: "test-zone-3"}}) + + affPod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) + + // provision these so we get three nodes that exist in the cluster with anti-affinity to a pod that we will + // then try to schedule + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, zone1Pod, zone2Pod, zone3Pod) + node1 := ExpectScheduled(ctx, env.Client, zone1Pod) + node2 := ExpectScheduled(ctx, env.Client, zone2Pod) + node3 := ExpectScheduled(ctx, env.Client, zone3Pod) + + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node1)) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node2)) + ExpectReconcileSucceeded(ctx, nodeStateController, client.ObjectKeyFromObject(node3)) + ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone1Pod)) + ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone2Pod)) + ExpectReconcileSucceeded(ctx, podStateController, client.ObjectKeyFromObject(zone3Pod)) + + // this pod with no anti-affinity rules can schedule, though it couldn't if the anti-affinity were required + ExpectProvisioned(ctx, env.Client, cluster, prov, affPod) + ExpectScheduled(ctx, env.Client, affPod) + }) + It("should allow violation of a pod affinity preference with a conflicting required constraint", func() { + affLabels := map[string]string{"security": "s2"} + constraint := v1.TopologySpreadConstraint{ + MaxSkew: 1, + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + } + affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) + affPods := MakePods(3, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + // limit these pods to one per host + TopologySpreadConstraints: []v1.TopologySpreadConstraint{constraint}, + // with a preference to the other pod + PodPreferences: []v1.WeightedPodAffinityTerm{{ + Weight: 50, + PodAffinityTerm: v1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelHostname, + }, + }}}) + ExpectApplied(ctx, env.Client, provisioner) + pods := append(affPods, affPod1) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + // all pods should be scheduled since the affinity term is just a preference + for _, pod := range pods { + ExpectScheduled(ctx, env.Client, pod) + } + // and we'll get three nodes due to the topology spread + ExpectSkew(ctx, env.Client, "", &constraint).To(ConsistOf(1, 1, 1)) + }) + It("should support pod anti-affinity with a zone topology", func() { + affLabels := map[string]string{"security": "s2"} + + // affPods will avoid being scheduled in the same zone + createPods := func() []*v1.Pod { + return MakePods(3, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, + PodAntiRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelTopologyZone, + }}}) + } + + top := &v1.TopologySpreadConstraint{TopologyKey: v1.LabelTopologyZone} + + // One of the downsides of late committal is that absent other constraints, it takes multiple batches of + // scheduling for zonal anti-affinities to work themselves out. The first schedule, we know that the pod + // will land in test-zone-1, test-zone-2, or test-zone-3, but don't know which it collapses to until the + // node is actually created. + + // one pod pod will schedule + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, createPods()...) + ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(1)) + // delete all of the unscheduled ones as provisioning will only bind pods passed into the provisioning call + // the scheduler looks at all pods though, so it may assume a pod from this batch schedules and no others do + ExpectDeleteAllUnscheduledPods(ctx, env.Client) + + // second pod in a second zone + ExpectProvisioned(ctx, env.Client, cluster, prov, createPods()...) + ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(1, 1)) + ExpectDeleteAllUnscheduledPods(ctx, env.Client) + + // third pod in the last zone + ExpectProvisioned(ctx, env.Client, cluster, prov, createPods()...) + ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(1, 1, 1)) + ExpectDeleteAllUnscheduledPods(ctx, env.Client) + + // and nothing else can schedule + ExpectProvisioned(ctx, env.Client, cluster, prov, createPods()...) + ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(1, 1, 1)) + ExpectDeleteAllUnscheduledPods(ctx, env.Client) + }) + It("should not schedule pods with affinity to a non-existent pod", func() { + affLabels := map[string]string{"security": "s2"} + affPods := MakePods(10, test.PodOptions{ + PodRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelTopologyZone, + }}}) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, affPods...) + // the pod we have affinity to is not in the cluster, so all of these pods are unschedulable + for _, p := range affPods { + ExpectNotScheduled(ctx, env.Client, p) + } + }) + It("should support pod affinity with zone topology (unconstrained target)", func() { + affLabels := map[string]string{"security": "s2"} + + // the pod that the others have an affinity to + targetPod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}}) + + // affPods all want to schedule in the same zone as targetPod, but can't as it's zone is undetermined + affPods := MakePods(10, test.PodOptions{ + PodRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelTopologyZone, + }}}) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, append(affPods, targetPod)...) + top := &v1.TopologySpreadConstraint{TopologyKey: v1.LabelTopologyZone} + // these pods can't schedule as the pod they have affinity to isn't limited to any particular zone + for i := range affPods { + ExpectNotScheduled(ctx, env.Client, affPods[i]) + affPods[i] = ExpectPodExists(ctx, env.Client, affPods[i].Name, affPods[i].Namespace) + } + ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(1)) + + // now that targetPod has been scheduled to a node, it's zone is committed and the pods with affinity to it + // should schedule in the same zone + ExpectProvisioned(ctx, env.Client, cluster, prov, affPods...) + for _, pod := range affPods { + ExpectScheduled(ctx, env.Client, pod) + } + ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(11)) + }) + It("should support pod affinity with zone topology (constrained target)", func() { + affLabels := map[string]string{"security": "s2"} + + // the pod that the others have an affinity to + affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels}, + NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-zone-1"}, + }, + }}) + + // affPods will all be scheduled in the same zone as affPod1 + affPods := MakePods(10, test.PodOptions{ + PodRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelTopologyZone, + }}}) + + affPods = append(affPods, affPod1) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, affPods...) + top := &v1.TopologySpreadConstraint{TopologyKey: v1.LabelTopologyZone} + ExpectSkew(ctx, env.Client, "default", top).To(ConsistOf(11)) + }) + It("should handle multiple dependent affinities", func() { + dbLabels := map[string]string{"type": "db", "spread": "spread"} + webLabels := map[string]string{"type": "web", "spread": "spread"} + cacheLabels := map[string]string{"type": "cache", "spread": "spread"} + uiLabels := map[string]string{"type": "ui", "spread": "spread"} + for i := 0; i < 50; i++ { + ExpectApplied(ctx, env.Client, provisioner.DeepCopy()) + // we have to schedule DB -> Web -> Cache -> UI in that order or else there are pod affinity violations + pods := []*v1.Pod{ + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: dbLabels}}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: webLabels}, + PodRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{MatchLabels: dbLabels}, + TopologyKey: v1.LabelHostname}, + }}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: cacheLabels}, + PodRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{MatchLabels: webLabels}, + TopologyKey: v1.LabelHostname}, + }}), + test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: uiLabels}, + PodRequirements: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{MatchLabels: cacheLabels}, + TopologyKey: v1.LabelHostname}, + }}), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + for i := range pods { + ExpectScheduled(ctx, env.Client, pods[i]) + } + ExpectCleanedUp(ctx, env.Client) + cluster.Reset() + } + }) + It("should fail to schedule pods with unsatisfiable dependencies", func() { + dbLabels := map[string]string{"type": "db", "spread": "spread"} + webLabels := map[string]string{"type": "web", "spread": "spread"} + ExpectApplied(ctx, env.Client, provisioner) + // this pods wants to schedule with a non-existent pod, this test just ensures that the scheduling loop + // doesn't infinite loop + pod := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: dbLabels}, + PodRequirements: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{MatchLabels: webLabels}, + TopologyKey: v1.LabelHostname, + }, + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + }) + It("should filter pod affinity topologies by namespace, no matching pods", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + + ExpectApplied(ctx, env.Client, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "other-ns-no-match"}}) + affLabels := map[string]string{"security": "s2"} + + affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels, Namespace: "other-ns-no-match"}}) + // affPod2 will try to get scheduled with affPod1 + affPod2 := test.UnschedulablePod(test.PodOptions{PodRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + TopologyKey: v1.LabelHostname, + }}}) + + var pods []*v1.Pod + // creates 10 nodes due to topo spread + pods = append(pods, MakePods(10, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: topology, + })...) + pods = append(pods, affPod1) + pods = append(pods, affPod2) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + + // the target pod gets scheduled + ExpectScheduled(ctx, env.Client, affPod1) + // but the one with affinity does not since the target pod is not in the same namespace and doesn't + // match the namespace list or namespace selector + ExpectNotScheduled(ctx, env.Client, affPod2) + }) + It("should filter pod affinity topologies by namespace, matching pods namespace list", func() { + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + + ExpectApplied(ctx, env.Client, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "other-ns-list"}}) + affLabels := map[string]string{"security": "s2"} + + affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels, Namespace: "other-ns-list"}}) + // affPod2 will try to get scheduled with affPod1 + affPod2 := test.UnschedulablePod(test.PodOptions{PodRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + Namespaces: []string{"other-ns-list"}, + TopologyKey: v1.LabelHostname, + }}}) + + var pods []*v1.Pod + // create 10 nodes + pods = append(pods, MakePods(10, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: topology, + })...) + // put our target pod on one of them + pods = append(pods, affPod1) + // and our pod with affinity should schedule on the same node + pods = append(pods, affPod2) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + n1 := ExpectScheduled(ctx, env.Client, affPod1) + n2 := ExpectScheduled(ctx, env.Client, affPod2) + // should be scheduled on the same node + Expect(n1.Name).To(Equal(n2.Name)) + }) + It("should filter pod affinity topologies by namespace, empty namespace selector", func() { + if env.Version.Minor() < 21 { + Skip("namespace selector is only supported on K8s >= 1.21.x") + } + topology := []v1.TopologySpreadConstraint{{ + TopologyKey: v1.LabelHostname, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + MaxSkew: 1, + }} + + ExpectApplied(ctx, env.Client, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "empty-ns-selector", Labels: map[string]string{"foo": "bar"}}}) + affLabels := map[string]string{"security": "s2"} + + affPod1 := test.UnschedulablePod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Labels: affLabels, Namespace: "empty-ns-selector"}}) + // affPod2 will try to get scheduled with affPod1 + affPod2 := test.UnschedulablePod(test.PodOptions{PodRequirements: []v1.PodAffinityTerm{{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: affLabels, + }, + // select all pods in all namespaces since the selector is empty + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{}}, + TopologyKey: v1.LabelHostname, + }}}) + + var pods []*v1.Pod + // create 10 nodes + pods = append(pods, MakePods(10, test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: topology, + })...) + // put our target pod on one of them + pods = append(pods, affPod1) + // and our pod with affinity should schedule on the same node + pods = append(pods, affPod2) + + ExpectApplied(ctx, env.Client, provisioner) + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + n1 := ExpectScheduled(ctx, env.Client, affPod1) + n2 := ExpectScheduled(ctx, env.Client, affPod2) + // should be scheduled on the same node due to the empty namespace selector + Expect(n1.Name).To(Equal(n2.Name)) + }) + It("should count topology across multiple provisioners", func() { + ExpectApplied(ctx, env.Client, + test.Provisioner(test.ProvisionerOptions{ + Requirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}}, + }), + test.Provisioner(test.ProvisionerOptions{ + Requirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2", "test-zone-3"}}}, + }), + ) + labels := map[string]string{"foo": "bar"} + topology := v1.TopologySpreadConstraint{ + TopologyKey: v1.LabelTopologyZone, + MaxSkew: 1, + LabelSelector: &metav1.LabelSelector{MatchLabels: labels}, + WhenUnsatisfiable: v1.DoNotSchedule, + } + ExpectProvisioned(ctx, env.Client, cluster, prov, test.Pods(10, test.UnscheduleablePodOptions(test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + TopologySpreadConstraints: []v1.TopologySpreadConstraint{topology}, + }))...) + ExpectSkew(ctx, env.Client, "default", &topology).To(ConsistOf(3, 3, 4)) + }) + }) +}) func ExpectDeleteAllUnscheduledPods(ctx2 context.Context, c client.Client) { var pods v1.PodList @@ -2223,52 +2246,57 @@ func ExpectDeleteAllUnscheduledPods(ctx2 context.Context, c client.Client) { } } -//var _ = Describe("Taints", func() { -// It("should taint nodes with provisioner taints", func() { -// provisioner.Spec.Taints = []v1.Taint{{Key: "test", Value: "bar", Effect: v1.TaintEffectNoSchedule}} -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( -// test.PodOptions{Tolerations: []v1.Toleration{{Effect: v1.TaintEffectNoSchedule, Operator: v1.TolerationOpExists}}}, -// ))[0] -// node := ExpectScheduled(ctx, env.Client, pod) -// Expect(node.Spec.Taints).To(ContainElement(provisioner.Spec.Taints[0])) -// }) -// It("should schedule pods that tolerate provisioner constraints", func() { -// provisioner.Spec.Taints = []v1.Taint{{Key: "test-key", Value: "test-value", Effect: v1.TaintEffectNoSchedule}} -// ExpectApplied(ctx, env.Client, provisioner) -// for _, pod := range ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// // Tolerates with OpExists -// test.UnschedulablePod(test.PodOptions{Tolerations: []v1.Toleration{{Key: "test-key", Operator: v1.TolerationOpExists, Effect: v1.TaintEffectNoSchedule}}}), -// // Tolerates with OpEqual -// test.UnschedulablePod(test.PodOptions{Tolerations: []v1.Toleration{{Key: "test-key", Value: "test-value", Operator: v1.TolerationOpEqual, Effect: v1.TaintEffectNoSchedule}}}), -// ) { -// ExpectScheduled(ctx, env.Client, pod) -// } -// ExpectApplied(ctx, env.Client, provisioner) -// for _, pod := range ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// // Missing toleration -// test.UnschedulablePod(), -// // key mismatch with OpExists -// test.UnschedulablePod(test.PodOptions{Tolerations: []v1.Toleration{{Key: "invalid", Operator: v1.TolerationOpExists}}}), -// // value mismatch -// test.UnschedulablePod(test.PodOptions{Tolerations: []v1.Toleration{{Key: "test-key", Operator: v1.TolerationOpEqual, Effect: v1.TaintEffectNoSchedule}}}), -// ) { -// ExpectNotScheduled(ctx, env.Client, pod) -// } -// }) -// It("should provision nodes with taints and schedule pods if the taint is only a startup taint", func() { -// provisioner.Spec.StartupTaints = []v1.Taint{{Key: "ignore-me", Value: "nothing-to-see-here", Effect: v1.TaintEffectNoSchedule}} -// -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod())[0] -// ExpectScheduled(ctx, env.Client, pod) -// }) -// It("should not generate taints for OpExists", func() { -// ExpectApplied(ctx, env.Client, provisioner) -// pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, -// test.UnschedulablePod(test.PodOptions{Tolerations: []v1.Toleration{{Key: "test-key", Operator: v1.TolerationOpExists, Effect: v1.TaintEffectNoExecute}}}), -// )[0] -// node := ExpectScheduled(ctx, env.Client, pod) -// Expect(node.Spec.Taints).To(HaveLen(1)) // Expect no taints generated beyond the default -// }) -//}) +var _ = Describe("Taints", func() { + It("should taint nodes with provisioner taints", func() { + provisioner.Spec.Taints = []v1.Taint{{Key: "test", Value: "bar", Effect: v1.TaintEffectNoSchedule}} + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod( + test.PodOptions{Tolerations: []v1.Toleration{{Effect: v1.TaintEffectNoSchedule, Operator: v1.TolerationOpExists}}}, + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Spec.Taints).To(ContainElement(provisioner.Spec.Taints[0])) + }) + It("should schedule pods that tolerate provisioner constraints", func() { + provisioner.Spec.Taints = []v1.Taint{{Key: "test-key", Value: "test-value", Effect: v1.TaintEffectNoSchedule}} + ExpectApplied(ctx, env.Client, provisioner) + pods := []*v1.Pod{ + // Tolerates with OpExists + test.UnschedulablePod(test.PodOptions{Tolerations: []v1.Toleration{{Key: "test-key", Operator: v1.TolerationOpExists, Effect: v1.TaintEffectNoSchedule}}}), + // Tolerates with OpEqual + test.UnschedulablePod(test.PodOptions{Tolerations: []v1.Toleration{{Key: "test-key", Value: "test-value", Operator: v1.TolerationOpEqual, Effect: v1.TaintEffectNoSchedule}}}), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + for _, pod := range pods { + ExpectScheduled(ctx, env.Client, pod) + } + ExpectApplied(ctx, env.Client, provisioner) + otherPods := []*v1.Pod{ + // Missing toleration + test.UnschedulablePod(), + // key mismatch with OpExists + test.UnschedulablePod(test.PodOptions{Tolerations: []v1.Toleration{{Key: "invalid", Operator: v1.TolerationOpExists}}}), + // value mismatch + test.UnschedulablePod(test.PodOptions{Tolerations: []v1.Toleration{{Key: "test-key", Operator: v1.TolerationOpEqual, Effect: v1.TaintEffectNoSchedule}}}), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, otherPods...) + for _, pod := range otherPods { + ExpectNotScheduled(ctx, env.Client, pod) + } + }) + It("should provision nodes with taints and schedule pods if the taint is only a startup taint", func() { + provisioner.Spec.StartupTaints = []v1.Taint{{Key: "ignore-me", Value: "nothing-to-see-here", Effect: v1.TaintEffectNoSchedule}} + + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + }) + It("should not generate taints for OpExists", func() { + ExpectApplied(ctx, env.Client, provisioner) + pod := test.UnschedulablePod(test.PodOptions{Tolerations: []v1.Toleration{{Key: "test-key", Operator: v1.TolerationOpExists, Effect: v1.TaintEffectNoExecute}}}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Spec.Taints).To(HaveLen(1)) // Expect no taints generated beyond the default + }) +}) diff --git a/pkg/controllers/provisioning/suite_test.go b/pkg/controllers/provisioning/suite_test.go index 29860ad8bd..d13177ea44 100644 --- a/pkg/controllers/provisioning/suite_test.go +++ b/pkg/controllers/provisioning/suite_test.go @@ -21,15 +21,15 @@ import ( "time" "github.com/samber/lo" - "knative.dev/pkg/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" - v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/record" clock "k8s.io/utils/clock/testing" + "knative.dev/pkg/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aws/karpenter-core/pkg/apis" "github.com/aws/karpenter-core/pkg/apis/settings" @@ -39,6 +39,7 @@ import ( "github.com/aws/karpenter-core/pkg/controllers/provisioning" "github.com/aws/karpenter-core/pkg/controllers/state" "github.com/aws/karpenter-core/pkg/controllers/state/informer" + "github.com/aws/karpenter-core/pkg/events" "github.com/aws/karpenter-core/pkg/operator/controller" "github.com/aws/karpenter-core/pkg/operator/scheme" "github.com/aws/karpenter-core/pkg/test" @@ -59,7 +60,6 @@ var cloudProvider *fake.CloudProvider var prov *provisioning.Provisioner var provisioningController controller.Controller var env *test.Environment -var recorder *test.EventRecorder var instanceTypeMap map[string]*cloudprovider.InstanceType func TestAPIs(t *testing.T) { @@ -72,12 +72,11 @@ var _ = BeforeSuite(func() { env = test.NewEnvironment(scheme.Scheme, test.WithCRDs(apis.CRDs...)) ctx = settings.ToContext(ctx, test.Settings()) cloudProvider = fake.NewCloudProvider() - recorder = test.NewEventRecorder() fakeClock = clock.NewFakeClock(time.Now()) cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) nodeController = informer.NewNodeController(env.Client, cluster) - prov = provisioning.NewProvisioner(ctx, env.Client, corev1.NewForConfigOrDie(env.Config), recorder, cloudProvider, cluster) - provisioningController = provisioning.NewController(env.Client, prov, recorder) + prov = provisioning.NewProvisioner(ctx, env.Client, corev1.NewForConfigOrDie(env.Config), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) + provisioningController = provisioning.NewController(env.Client, prov, events.NewRecorder(&record.FakeRecorder{})) instanceTypes, _ := cloudProvider.GetInstanceTypes(context.Background(), nil) instanceTypeMap = map[string]*cloudprovider.InstanceType{} for _, it := range instanceTypes { @@ -96,32 +95,29 @@ var _ = AfterSuite(func() { var _ = AfterEach(func() { ExpectCleanedUp(ctx, env.Client) - recorder.Reset() cluster.Reset() }) var _ = Describe("Provisioning", func() { It("should provision nodes", func() { ExpectApplied(ctx, env.Client, test.Provisioner()) - pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) nodes := &v1.NodeList{} Expect(env.Client.List(ctx, nodes)).To(Succeed()) Expect(len(nodes.Items)).To(Equal(1)) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } + ExpectScheduled(ctx, env.Client, pod) }) It("should ignore provisioners that are deleting", func() { provisioner := test.Provisioner() ExpectApplied(ctx, env.Client, provisioner) ExpectDeletionTimestampSet(ctx, env.Client, provisioner) - pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) nodes := &v1.NodeList{} Expect(env.Client.List(ctx, nodes)).To(Succeed()) Expect(len(nodes.Items)).To(Equal(0)) - for _, pod := range pods { - ExpectNotScheduled(ctx, env.Client, pod) - } + ExpectNotScheduled(ctx, env.Client, pod) }) It("should provision nodes for pods with supported node selectors", func() { provisioner := test.Provisioner() @@ -154,23 +150,27 @@ var _ = Describe("Provisioning", func() { test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{"foo": "bar"}}), } ExpectApplied(ctx, env.Client, provisioner) - for _, pod := range ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, schedulable...) { + ExpectProvisioned(ctx, env.Client, cluster, prov, schedulable...) + for _, pod := range schedulable { ExpectScheduled(ctx, env.Client, pod) } - for _, pod := range ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, unschedulable...) { + ExpectProvisioned(ctx, env.Client, cluster, prov, unschedulable...) + for _, pod := range unschedulable { ExpectNotScheduled(ctx, env.Client, pod) } }) It("should provision nodes for accelerators", func() { ExpectApplied(ctx, env.Client, test.Provisioner()) - for _, pod := range ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, + pods := []*v1.Pod{ test.UnschedulablePod(test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{Limits: v1.ResourceList{fake.ResourceGPUVendorA: resource.MustParse("1")}}, }), test.UnschedulablePod(test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{Limits: v1.ResourceList{fake.ResourceGPUVendorB: resource.MustParse("1")}}, }), - ) { + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + for _, pod := range pods { ExpectScheduled(ctx, env.Client, pod) } }) @@ -187,7 +187,10 @@ var _ = Describe("Provisioning", func() { }, }, })) - pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(), test.UnschedulablePod(), test.UnschedulablePod()) + pods := []*v1.Pod{ + test.UnschedulablePod(), test.UnschedulablePod(), test.UnschedulablePod(), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) nodes := &v1.NodeList{} Expect(env.Client.List(ctx, nodes)).To(Succeed()) Expect(len(nodes.Items)).To(Equal(3)) @@ -224,15 +227,15 @@ var _ = Describe("Provisioning", func() { // Provision without a binding since some pods will already be bound // Should all schedule to the new node, ignoring the old node - ExpectProvisionedNoBinding(ctx, env.Client, provisioningController, prov, test.UnschedulablePod(), test.UnschedulablePod()) + bindings := ExpectProvisionedNoBinding(ctx, env.Client, prov, test.UnschedulablePod(), test.UnschedulablePod()) nodes := &v1.NodeList{} Expect(env.Client.List(ctx, nodes)).To(Succeed()) Expect(len(nodes.Items)).To(Equal(2)) // Scheduler should attempt to schedule all the pods to the new node - recorder.ForEachBinding(func(p *v1.Pod, n *v1.Node) { + for _, n := range bindings { Expect(n.Name).ToNot(Equal(node.Name)) - }) + } }) Context("Resource Limits", func() { It("should not schedule when limits are exceeded", func() { @@ -244,20 +247,22 @@ var _ = Describe("Provisioning", func() { }, }, })) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod())[0] + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) }) It("should schedule if limits would be met", func() { ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{ Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, })) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( + pod := test.UnschedulablePod( test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{ // requires a 2 CPU node, but leaves room for overhead v1.ResourceCPU: resource.MustParse("1.75"), }, - }}))[0] + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) // A 2 CPU node can be launched ExpectScheduled(ctx, env.Client, pod) }) @@ -285,10 +290,11 @@ var _ = Describe("Provisioning", func() { Requests: v1.ResourceList{ v1.ResourceCPU: resource.MustParse("1.5"), }}} - pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, + pods := []*v1.Pod{ test.UnschedulablePod(opts), test.UnschedulablePod(opts), - ) + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) scheduledPodCount := 0 unscheduledPodCount := 0 pod0 := ExpectPodExists(ctx, env.Client, pods[0].Name, pods[0].Namespace) @@ -310,24 +316,26 @@ var _ = Describe("Provisioning", func() { ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{ Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, })) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( + pod := test.UnschedulablePod( test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{ v1.ResourceCPU: resource.MustParse("2.1"), }, - }}))[0] + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) }) It("should not schedule if limits would be exceeded (GPU)", func() { ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{ Limits: v1.ResourceList{v1.ResourcePods: resource.MustParse("1")}, })) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( + pod := test.UnschedulablePod( test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Limits: v1.ResourceList{ fake.ResourceGPUVendorA: resource.MustParse("1"), }, - }}))[0] + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) // only available instance type has 2 GPUs which would exceed the limit ExpectNotScheduled(ctx, env.Client, pod) }) @@ -335,24 +343,26 @@ var _ = Describe("Provisioning", func() { ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{ Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")}, })) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( + pod := test.UnschedulablePod( test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{ // requires a 2 CPU node, but leaves room for overhead v1.ResourceCPU: resource.MustParse("1.75"), }, - }}))[0] + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) // A 2 CPU node can be launched ExpectScheduled(ctx, env.Client, pod) // This pod requests over the existing limit (would add to 3.5 CPUs) so this should fail - pod = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( + pod = test.UnschedulablePod( test.PodOptions{ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{ // requires a 2 CPU node, but leaves room for overhead v1.ResourceCPU: resource.MustParse("1.75"), }, - }}))[0] + }}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) }) }) @@ -363,11 +373,12 @@ var _ = Describe("Provisioning", func() { ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }}, )) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( + pod := test.UnschedulablePod( test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }, - ))[0] + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity @@ -384,11 +395,12 @@ var _ = Describe("Provisioning", func() { ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }}, )) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( + pod := test.UnschedulablePod( test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }, - ))[0] + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity @@ -401,7 +413,8 @@ var _ = Describe("Provisioning", func() { ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("10000"), v1.ResourceMemory: resource.MustParse("10000Gi")}}, }}, )) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{}))[0] + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) }) It("should not schedule if resource requests are not defined and limits (requests) are too large", func() { @@ -413,7 +426,8 @@ var _ = Describe("Provisioning", func() { }, }}, )) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{}))[0] + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) }) It("should schedule based on the max resource requests of containers and initContainers", func() { @@ -430,7 +444,8 @@ var _ = Describe("Provisioning", func() { }, }}, )) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{}))[0] + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) @@ -450,7 +465,8 @@ var _ = Describe("Provisioning", func() { }, }}, )) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{}))[0] + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) }) It("should not schedule if initContainer resources are too large", func() { @@ -462,14 +478,16 @@ var _ = Describe("Provisioning", func() { }, }}, )) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{}))[0] + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) }) It("should be able to schedule pods if resource requests and limits are not defined", func() { ExpectApplied(ctx, env.Client, test.Provisioner(), test.DaemonSet( test.DaemonSetOptions{PodOptions: test.PodOptions{}}, )) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{}))[0] + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectScheduled(ctx, env.Client, pod) }) It("should ignore daemonsets without matching tolerations", func() { @@ -480,12 +498,13 @@ var _ = Describe("Provisioning", func() { ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }}, )) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( + pod := test.UnschedulablePod( test.PodOptions{ Tolerations: []v1.Toleration{{Operator: v1.TolerationOperator(v1.NodeSelectorOpExists)}}, ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }, - ))[0] + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("2"))) @@ -498,11 +517,12 @@ var _ = Describe("Provisioning", func() { ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }}, )) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( + pod := test.UnschedulablePod( test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }, - ))[0] + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("2"))) @@ -515,12 +535,13 @@ var _ = Describe("Provisioning", func() { ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }}, )) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( + pod := test.UnschedulablePod( test.PodOptions{ NodeRequirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-2"}}}, ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}}, }, - ))[0] + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) allocatable := instanceTypeMap[node.Labels[v1.LabelInstanceTypeStable]].Capacity Expect(*allocatable.Cpu()).To(Equal(resource.MustParse("4"))) @@ -533,10 +554,10 @@ var _ = Describe("Provisioning", func() { Annotations: map[string]string{v1alpha5.DoNotConsolidateNodeAnnotationKey: "true"}, }) ExpectApplied(ctx, env.Client, provisioner) - for _, pod := range ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) { - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Annotations).To(HaveKeyWithValue(v1alpha5.DoNotConsolidateNodeAnnotationKey, "true")) - } + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Annotations).To(HaveKeyWithValue(v1alpha5.DoNotConsolidateNodeAnnotationKey, "true")) }) }) Context("Labels", func() { @@ -553,27 +574,28 @@ var _ = Describe("Provisioning", func() { }, }) ExpectApplied(ctx, env.Client, provisioner) - for _, pod := range ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) { - node := ExpectScheduled(ctx, env.Client, pod) - Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.ProvisionerNameLabelKey, provisioner.Name)) - Expect(node.Labels).To(HaveKeyWithValue("test-key-1", "test-value-1")) - Expect(node.Labels).To(HaveKeyWithValue("test-key-2", "test-value-2")) - Expect(node.Labels).To(And(HaveKey("test-key-3"), Not(HaveValue(Equal("test-value-3"))))) - Expect(node.Labels).To(And(HaveKey("test-key-4"), Not(HaveValue(Equal("test-value-4"))))) - Expect(node.Labels).To(And(HaveKey("test-key-5"), Not(HaveValue(Equal("test-value-5"))))) - Expect(node.Labels).To(HaveKey("test-key-6")) - Expect(node.Labels).ToNot(HaveKey("test-key-7")) - } + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.ProvisionerNameLabelKey, provisioner.Name)) + Expect(node.Labels).To(HaveKeyWithValue("test-key-1", "test-value-1")) + Expect(node.Labels).To(HaveKeyWithValue("test-key-2", "test-value-2")) + Expect(node.Labels).To(And(HaveKey("test-key-3"), Not(HaveValue(Equal("test-value-3"))))) + Expect(node.Labels).To(And(HaveKey("test-key-4"), Not(HaveValue(Equal("test-value-4"))))) + Expect(node.Labels).To(And(HaveKey("test-key-5"), Not(HaveValue(Equal("test-value-5"))))) + Expect(node.Labels).To(HaveKey("test-key-6")) + Expect(node.Labels).ToNot(HaveKey("test-key-7")) }) It("should label nodes with labels in the LabelDomainExceptions list", func() { for domain := range v1alpha5.LabelDomainExceptions { provisioner := test.Provisioner(test.ProvisionerOptions{Labels: map[string]string{domain + "/test": "test-value"}}) ExpectApplied(ctx, env.Client, provisioner) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( + pod := test.UnschedulablePod( test.PodOptions{ NodeRequirements: []v1.NodeSelectorRequirement{{Key: domain + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test-value"}}}, }, - ))[0] + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(domain+"/test", "test-value")) } @@ -584,7 +606,7 @@ var _ = Describe("Provisioning", func() { It("should schedule pods that tolerate taints", func() { provisioner := test.Provisioner(test.ProvisionerOptions{Taints: []v1.Taint{{Key: "nvidia.com/gpu", Value: "true", Effect: v1.TaintEffectNoSchedule}}}) ExpectApplied(ctx, env.Client, provisioner) - for _, pod := range ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, + pods := []*v1.Pod{ test.UnschedulablePod( test.PodOptions{Tolerations: []v1.Toleration{ { @@ -615,7 +637,9 @@ var _ = Describe("Provisioning", func() { Operator: v1.TolerationOpExists, }, }}), - ) { + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) + for _, pod := range pods { ExpectScheduled(ctx, env.Client, pod) } }) @@ -624,7 +648,8 @@ var _ = Describe("Provisioning", func() { It("should create a machine request with expected requirements", func() { provisioner := test.Provisioner() ExpectApplied(ctx, env.Client, provisioner) - pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) Expect(cloudProvider.CreateCalls).To(HaveLen(1)) ExpectMachineRequirements(cloudProvider.CreateCalls[0], @@ -639,9 +664,7 @@ var _ = Describe("Provisioning", func() { Values: []string{provisioner.Name}, }, ) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } + ExpectScheduled(ctx, env.Client, pod) }) It("should create a machine request with additional expected requirements", func() { provisioner := test.Provisioner(test.ProvisionerOptions{ @@ -659,7 +682,8 @@ var _ = Describe("Provisioning", func() { }, }) ExpectApplied(ctx, env.Client, provisioner) - pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) Expect(cloudProvider.CreateCalls).To(HaveLen(1)) ExpectMachineRequirements(cloudProvider.CreateCalls[0], @@ -684,9 +708,7 @@ var _ = Describe("Provisioning", func() { Values: []string{"value"}, }, ) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } + ExpectScheduled(ctx, env.Client, pod) }) It("should create a machine request restricting instance types on architecture", func() { provisioner := test.Provisioner(test.ProvisionerOptions{ @@ -699,7 +721,8 @@ var _ = Describe("Provisioning", func() { }, }) ExpectApplied(ctx, env.Client, provisioner) - pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) Expect(cloudProvider.CreateCalls).To(HaveLen(1)) @@ -716,9 +739,7 @@ var _ = Describe("Provisioning", func() { Values: []string{"arm-instance-type"}, }, ) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } + ExpectScheduled(ctx, env.Client, pod) }) It("should create a machine request restricting instance types on operating system", func() { provisioner := test.Provisioner(test.ProvisionerOptions{ @@ -731,7 +752,8 @@ var _ = Describe("Provisioning", func() { }, }) ExpectApplied(ctx, env.Client, provisioner) - pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) Expect(cloudProvider.CreateCalls).To(HaveLen(1)) @@ -748,14 +770,12 @@ var _ = Describe("Provisioning", func() { Values: []string{"arm-instance-type"}, }, ) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } + ExpectScheduled(ctx, env.Client, pod) }) It("should create a machine request restricting instance types based on pod resource requests", func() { provisioner := test.Provisioner() ExpectApplied(ctx, env.Client, provisioner) - pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{ + pod := test.UnschedulablePod(test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{ fake.ResourceGPUVendorA: resource.MustParse("1"), @@ -764,7 +784,8 @@ var _ = Describe("Provisioning", func() { fake.ResourceGPUVendorA: resource.MustParse("1"), }, }, - })) + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) Expect(cloudProvider.CreateCalls).To(HaveLen(1)) @@ -776,14 +797,13 @@ var _ = Describe("Provisioning", func() { Values: []string{"gpu-vendor-instance-type"}, }, ) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } + ExpectScheduled(ctx, env.Client, pod) }) It("should create a machine request with the correct owner reference", func() { provisioner := test.Provisioner(test.ProvisionerOptions{}) ExpectApplied(ctx, env.Client, provisioner) - pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) Expect(cloudProvider.CreateCalls).To(HaveLen(1)) Expect(cloudProvider.CreateCalls[0].OwnerReferences).To(ContainElement( @@ -794,9 +814,7 @@ var _ = Describe("Provisioning", func() { UID: provisioner.UID, }, )) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } + ExpectScheduled(ctx, env.Client, pod) }) It("should create a machine request propagating the provider reference", func() { ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{ @@ -806,7 +824,8 @@ var _ = Describe("Provisioning", func() { Name: "default", }, })) - pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) Expect(cloudProvider.CreateCalls).To(HaveLen(1)) Expect(cloudProvider.CreateCalls[0].Spec.MachineTemplateRef).To(Equal( @@ -816,9 +835,7 @@ var _ = Describe("Provisioning", func() { Name: "default", }, )) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } + ExpectScheduled(ctx, env.Client, pod) }) It("should create a machine request with the karpenter.sh/compatibility/provider annotation", func() { ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{ @@ -827,7 +844,8 @@ var _ = Describe("Provisioning", func() { "providerField2": "value", }, })) - pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod()) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) Expect(cloudProvider.CreateCalls).To(HaveLen(1)) Expect(cloudProvider.CreateCalls[0].Annotations).To(HaveKey(v1alpha5.ProviderCompatabilityAnnotationKey)) @@ -837,10 +855,7 @@ var _ = Describe("Provisioning", func() { Expect(json.Unmarshal([]byte(cloudProvider.CreateCalls[0].Annotations[v1alpha5.ProviderCompatabilityAnnotationKey]), &provider)).To(Succeed()) Expect(provider).To(HaveKeyWithValue("providerField1", "value")) Expect(provider).To(HaveKeyWithValue("providerField2", "value")) - - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } + ExpectScheduled(ctx, env.Client, pod) }) It("should create a machine with resource requests", func() { ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{ @@ -849,7 +864,7 @@ var _ = Describe("Provisioning", func() { "providerField2": "value", }, })) - pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( + pod := test.UnschedulablePod( test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{ @@ -861,8 +876,8 @@ var _ = Describe("Provisioning", func() { fake.ResourceGPUVendorA: resource.MustParse("1"), }, }, - }), - ) + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) Expect(cloudProvider.CreateCalls).To(HaveLen(1)) Expect(cloudProvider.CreateCalls[0].Spec.Resources.Requests).To(HaveLen(4)) ExpectMachineRequests(cloudProvider.CreateCalls[0], v1.ResourceList{ @@ -871,9 +886,7 @@ var _ = Describe("Provisioning", func() { fake.ResourceGPUVendorA: resource.MustParse("1"), v1.ResourcePods: resource.MustParse("1"), }) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } + ExpectScheduled(ctx, env.Client, pod) }) It("should create a machine with resource requests with daemon overhead", func() { ExpectApplied(ctx, env.Client, test.Provisioner(), test.DaemonSet( @@ -881,20 +894,19 @@ var _ = Describe("Provisioning", func() { ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Mi")}}, }}, )) - pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod( + pod := test.UnschedulablePod( test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Mi")}}, }, - )) + ) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) Expect(cloudProvider.CreateCalls).To(HaveLen(1)) ExpectMachineRequests(cloudProvider.CreateCalls[0], v1.ResourceList{ v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("2Mi"), v1.ResourcePods: resource.MustParse("2"), }) - for _, pod := range pods { - ExpectScheduled(ctx, env.Client, pod) - } + ExpectScheduled(ctx, env.Client, pod) }) }) }) @@ -906,26 +918,30 @@ var _ = Describe("Volume Topology Requirements", func() { }) It("should not schedule if invalid pvc", func() { ExpectApplied(ctx, env.Client, test.Provisioner()) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{ + pod := test.UnschedulablePod(test.PodOptions{ PersistentVolumeClaims: []string{"invalid"}, - }))[0] + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) }) It("should schedule with an empty storage class", func() { storageClass := "" persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{StorageClassName: &storageClass}) ExpectApplied(ctx, env.Client, test.Provisioner(), persistentVolumeClaim) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{ + pod := test.UnschedulablePod(test.PodOptions{ PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - }))[0] + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectScheduled(ctx, env.Client, pod) }) It("should schedule valid pods when a pod with an invalid pvc is encountered (pvc)", func() { ExpectApplied(ctx, env.Client, test.Provisioner()) - invalidPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{ + invalidPod := test.UnschedulablePod(test.PodOptions{ PersistentVolumeClaims: []string{"invalid"}, - }))[0] - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{}))[0] + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, invalidPod) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectNotScheduled(ctx, env.Client, invalidPod) ExpectScheduled(ctx, env.Client, pod) }) @@ -933,10 +949,12 @@ var _ = Describe("Volume Topology Requirements", func() { invalidStorageClass := "invalid-storage-class" persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{StorageClassName: &invalidStorageClass}) ExpectApplied(ctx, env.Client, test.Provisioner(), persistentVolumeClaim) - invalidPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{ + invalidPod := test.UnschedulablePod(test.PodOptions{ PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - }))[0] - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{}))[0] + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, invalidPod) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectNotScheduled(ctx, env.Client, invalidPod) ExpectScheduled(ctx, env.Client, pod) }) @@ -944,43 +962,48 @@ var _ = Describe("Volume Topology Requirements", func() { invalidVolumeName := "invalid-volume-name" persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{VolumeName: invalidVolumeName}) ExpectApplied(ctx, env.Client, test.Provisioner(), persistentVolumeClaim) - invalidPod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{ + invalidPod := test.UnschedulablePod(test.PodOptions{ PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - }))[0] - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{}))[0] + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, invalidPod) + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectNotScheduled(ctx, env.Client, invalidPod) ExpectScheduled(ctx, env.Client, pod) }) It("should schedule to storage class zones if volume does not exist", func() { persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{StorageClassName: &storageClass.Name}) ExpectApplied(ctx, env.Client, test.Provisioner(), storageClass, persistentVolumeClaim) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{ + pod := test.UnschedulablePod(test.PodOptions{ PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, NodeRequirements: []v1.NodeSelectorRequirement{{ Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-3"}, }}, - }))[0] + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) }) It("should not schedule if storage class zones are incompatible", func() { persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{StorageClassName: &storageClass.Name}) ExpectApplied(ctx, env.Client, test.Provisioner(), storageClass, persistentVolumeClaim) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{ + pod := test.UnschedulablePod(test.PodOptions{ PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, NodeRequirements: []v1.NodeSelectorRequirement{{ Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}, }}, - }))[0] + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) }) It("should schedule to volume zones if volume already bound", func() { persistentVolume := test.PersistentVolume(test.PersistentVolumeOptions{Zones: []string{"test-zone-3"}}) persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{VolumeName: persistentVolume.Name, StorageClassName: &storageClass.Name}) ExpectApplied(ctx, env.Client, test.Provisioner(), storageClass, persistentVolumeClaim, persistentVolume) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{ + pod := test.UnschedulablePod(test.PodOptions{ PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, - }))[0] + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) }) @@ -988,12 +1011,13 @@ var _ = Describe("Volume Topology Requirements", func() { persistentVolume := test.PersistentVolume(test.PersistentVolumeOptions{Zones: []string{"test-zone-3"}}) persistentVolumeClaim := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{VolumeName: persistentVolume.Name, StorageClassName: &storageClass.Name}) ExpectApplied(ctx, env.Client, test.Provisioner(), storageClass, persistentVolumeClaim, persistentVolume) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(test.PodOptions{ + pod := test.UnschedulablePod(test.PodOptions{ PersistentVolumeClaims: []string{persistentVolumeClaim.Name}, NodeRequirements: []v1.NodeSelectorRequirement{{ Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}, }}, - }))[0] + }) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) }) It("should not relax an added volume topology zone node-selector away", func() { @@ -1025,7 +1049,7 @@ var _ = Describe("Volume Topology Requirements", func() { }, }, }) - pod = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod)[0] + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-3")) }) @@ -1042,7 +1066,7 @@ var _ = Describe("Preferential Fallback", func() { }}}} // Don't relax ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{Requirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}}})) - pod = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod)[0] + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectNotScheduled(ctx, env.Client, pod) }) It("should relax multiple terms", func() { @@ -1063,7 +1087,7 @@ var _ = Describe("Preferential Fallback", func() { }}}} // Success ExpectApplied(ctx, env.Client, test.Provisioner()) - pod = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod)[0] + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-1")) }) @@ -1085,7 +1109,7 @@ var _ = Describe("Preferential Fallback", func() { }}} // Success ExpectApplied(ctx, env.Client, test.Provisioner()) - pod = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod)[0] + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) ExpectScheduled(ctx, env.Client, pod) }) It("should relax to use lighter weights", func() { @@ -1109,7 +1133,7 @@ var _ = Describe("Preferential Fallback", func() { }}} // Success ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{Requirements: []v1.NodeSelectorRequirement{{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1", "test-zone-2"}}}})) - pod = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod)[0] + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1.LabelTopologyZone, "test-zone-2")) }) @@ -1129,7 +1153,7 @@ var _ = Describe("Preferential Fallback", func() { }}} // Success ExpectApplied(ctx, env.Client, test.Provisioner(test.ProvisionerOptions{Taints: []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectPreferNoSchedule}}})) - pod = ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, pod)[0] + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Spec.Taints).To(ContainElement(v1.Taint{Key: "foo", Value: "bar", Effect: v1.TaintEffectPreferNoSchedule})) }) @@ -1140,25 +1164,24 @@ var _ = Describe("Multiple Provisioners", func() { It("should schedule to an explicitly selected provisioner", func() { provisioner := test.Provisioner() ExpectApplied(ctx, env.Client, provisioner, test.Provisioner()) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1alpha5.ProvisionerNameLabelKey: provisioner.Name}}), - )[0] + pod := test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1alpha5.ProvisionerNameLabelKey: provisioner.Name}}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).To(Equal(provisioner.Name)) }) It("should schedule to a provisioner by labels", func() { provisioner := test.Provisioner(test.ProvisionerOptions{Labels: map[string]string{"foo": "bar"}}) ExpectApplied(ctx, env.Client, provisioner, test.Provisioner()) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, - test.UnschedulablePod(test.PodOptions{NodeSelector: provisioner.Spec.Labels}), - )[0] + pod := test.UnschedulablePod(test.PodOptions{NodeSelector: provisioner.Spec.Labels}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).To(Equal(provisioner.Name)) }) It("should not match provisioner with PreferNoSchedule taint when other provisioner match", func() { provisioner := test.Provisioner(test.ProvisionerOptions{Taints: []v1.Taint{{Key: "foo", Value: "bar", Effect: v1.TaintEffectPreferNoSchedule}}}) ExpectApplied(ctx, env.Client, provisioner, test.Provisioner()) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod())[0] + pod := test.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).ToNot(Equal(provisioner.Name)) }) @@ -1170,7 +1193,10 @@ var _ = Describe("Multiple Provisioners", func() { test.Provisioner(test.ProvisionerOptions{Weight: ptr.Int32(100)}), } ExpectApplied(ctx, env.Client, provisioners...) - pods := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, test.UnschedulablePod(), test.UnschedulablePod(), test.UnschedulablePod()) + pods := []*v1.Pod{ + test.UnschedulablePod(), test.UnschedulablePod(), test.UnschedulablePod(), + } + ExpectProvisioned(ctx, env.Client, cluster, prov, pods...) for _, pod := range pods { node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).To(Equal(provisioners[2].GetName())) @@ -1184,9 +1210,8 @@ var _ = Describe("Multiple Provisioners", func() { test.Provisioner(test.ProvisionerOptions{Weight: ptr.Int32(100)}), } ExpectApplied(ctx, env.Client, provisioners...) - pod := ExpectProvisioned(ctx, env.Client, cluster, recorder, provisioningController, prov, - test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1alpha5.ProvisionerNameLabelKey: targetedProvisioner.Name}}), - )[0] + pod := test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1alpha5.ProvisionerNameLabelKey: targetedProvisioner.Name}}) + ExpectProvisioned(ctx, env.Client, cluster, prov, pod) node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).To(Equal(targetedProvisioner.Name)) }) diff --git a/pkg/controllers/state/node.go b/pkg/controllers/state/node.go index 780bf4e557..09cadbd029 100644 --- a/pkg/controllers/state/node.go +++ b/pkg/controllers/state/node.go @@ -281,10 +281,10 @@ func (in *Node) PodLimits() v1.ResourceList { } func (in *Node) MarkedForDeletion() bool { - // The Machine is marked for the Deletion if: - // 1. The Machine has explicitly MarkedForDeletion - // 2. The Machine has a Machine counterpart and is actively deleting - // 3. The Machine has no Machine counterpart and is actively deleting + // The Node is marked for the Deletion if: + // 1. The Node has explicitly MarkedForDeletion + // 2. The Node has a Machine counterpart and is actively deleting + // 3. The Node has no Machine counterpart and is actively deleting return in.markedForDeletion || (in.Machine != nil && !in.Machine.DeletionTimestamp.IsZero()) || (in.Node != nil && in.Machine == nil && !in.Node.DeletionTimestamp.IsZero()) diff --git a/pkg/controllers/termination/suite_test.go b/pkg/controllers/termination/suite_test.go index 0e050915f1..750b312785 100644 --- a/pkg/controllers/termination/suite_test.go +++ b/pkg/controllers/termination/suite_test.go @@ -21,6 +21,7 @@ import ( "testing" "time" + "k8s.io/client-go/tools/record" clock "k8s.io/utils/clock/testing" "github.com/samber/lo" @@ -31,6 +32,7 @@ import ( "github.com/aws/karpenter-core/pkg/cloudprovider/fake" "github.com/aws/karpenter-core/pkg/controllers/machine/terminator" "github.com/aws/karpenter-core/pkg/controllers/termination" + "github.com/aws/karpenter-core/pkg/events" "github.com/aws/karpenter-core/pkg/operator/controller" "github.com/aws/karpenter-core/pkg/operator/scheme" "github.com/aws/karpenter-core/pkg/test" @@ -64,9 +66,8 @@ var _ = BeforeSuite(func() { env = test.NewEnvironment(scheme.Scheme, test.WithCRDs(apis.CRDs...)) cloudProvider := fake.NewCloudProvider() - eventRecorder := test.NewEventRecorder() - evictionQueue = terminator.NewEvictionQueue(ctx, env.KubernetesInterface.CoreV1(), eventRecorder) - terminationController = termination.NewController(env.Client, terminator.NewTerminator(fakeClock, env.Client, cloudProvider, evictionQueue), eventRecorder) + evictionQueue = terminator.NewEvictionQueue(ctx, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{})) + terminationController = termination.NewController(env.Client, terminator.NewTerminator(fakeClock, env.Client, cloudProvider, evictionQueue), events.NewRecorder(&record.FakeRecorder{})) }) var _ = AfterSuite(func() { diff --git a/pkg/events/suite_test.go b/pkg/events/suite_test.go index 1f4ed96330..77535f92af 100644 --- a/pkg/events/suite_test.go +++ b/pkg/events/suite_test.go @@ -16,12 +16,14 @@ package events_test import ( "fmt" + "sync" "testing" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/client-go/util/flowcontrol" @@ -29,8 +31,39 @@ import ( "github.com/aws/karpenter-core/pkg/test" ) -var internalRecorder *test.InternalRecorder var eventRecorder events.Recorder +var internalRecorder *InternalRecorder + +type InternalRecorder struct { + mu sync.RWMutex + calls map[string]int +} + +func NewInternalRecorder() *InternalRecorder { + return &InternalRecorder{ + calls: map[string]int{}, + } +} + +func (i *InternalRecorder) Event(_ runtime.Object, _, reason, _ string) { + i.mu.Lock() + defer i.mu.Unlock() + i.calls[reason]++ +} + +func (i *InternalRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, _ ...interface{}) { + i.Event(object, eventtype, reason, messageFmt) +} + +func (i *InternalRecorder) AnnotatedEventf(object runtime.Object, _ map[string]string, eventtype, reason, messageFmt string, _ ...interface{}) { + i.Event(object, eventtype, reason, messageFmt) +} + +func (i *InternalRecorder) Calls(reason string) int { + i.mu.RLock() + defer i.mu.RUnlock() + return i.calls[reason] +} func TestRecorder(t *testing.T) { RegisterFailHandler(Fail) @@ -38,7 +71,7 @@ func TestRecorder(t *testing.T) { } var _ = BeforeEach(func() { - internalRecorder = test.NewInternalRecorder() + internalRecorder = NewInternalRecorder() eventRecorder = events.NewRecorder(internalRecorder) events.PodNominationRateLimiter = flowcontrol.NewTokenBucketRateLimiter(5, 10) }) diff --git a/pkg/test/expectations/expectations.go b/pkg/test/expectations/expectations.go index ece86d555e..0c1e182bd5 100644 --- a/pkg/test/expectations/expectations.go +++ b/pkg/test/expectations/expectations.go @@ -35,6 +35,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "knative.dev/pkg/apis" "knative.dev/pkg/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -53,12 +54,6 @@ const ( RequestInterval = 1 * time.Second ) -// Binding is a potential binding that was reported through event recording. -type Binding struct { - Pod *v1.Pod - Node *v1.Node -} - func ExpectExists[T client.Object](ctx context.Context, c client.Client, obj T) T { return ExpectExistsWithOffset(1, ctx, c, obj) } @@ -218,9 +213,13 @@ func ExpectFinalizersRemoved(ctx context.Context, c client.Client, objectLists . func ExpectProvisioned(ctx context.Context, c client.Client, cluster *state.Cluster, provisioner *provisioning.Provisioner, pods ...*v1.Pod) map[*v1.Pod]*v1.Node { bindings := ExpectProvisionedNoBindingWithOffset(1, ctx, c, provisioner, pods...) + podNames := sets.NewString(lo.Map(pods, func(p *v1.Pod, _ int) string { return p.Name })...) for pod, node := range bindings { - ExpectManualBindingWithOffset(1, ctx, c, pod, node) - ExpectWithOffset(1, cluster.UpdatePod(ctx, pod)).To(Succeed()) // track pod bindings + // Only bind the pods that are passed through + if podNames.Has(pod.Name) { + ExpectManualBindingWithOffset(1, ctx, c, pod, node) + ExpectWithOffset(1, cluster.UpdatePod(ctx, pod)).To(Succeed()) // track pod bindings + } } return bindings } @@ -235,15 +234,23 @@ func ExpectProvisionedNoBindingWithOffset(offset int, ctx context.Context, c cli ExpectAppliedWithOffset(offset+1, ctx, c, pod) } // TODO: Check the error on the provisioner scheduling round - machines, _, _ := provisioner.Schedule(ctx) + machines, nodes, _ := provisioner.Schedule(ctx) bindings := map[*v1.Pod]*v1.Node{} for _, m := range machines { + // TODO: Check the error on the provisioner launch name, err := provisioner.Launch(ctx, m) - ExpectWithOffset(offset+1, err).ToNot(HaveOccurred()) + if err != nil { + return bindings + } for _, pod := range m.Pods { bindings[pod] = ExpectNodeExistsWithOffset(offset+1, ctx, c, name) } } + for _, node := range nodes { + for _, pod := range node.Pods { + bindings[pod] = node.Node.Node + } + } return bindings } diff --git a/pkg/utils/pod/scheduling.go b/pkg/utils/pod/scheduling.go index 1ba9705c7c..5048cb71d2 100644 --- a/pkg/utils/pod/scheduling.go +++ b/pkg/utils/pod/scheduling.go @@ -64,7 +64,7 @@ func IsOwnedByDaemonSet(pod *v1.Pod) bool { // IsOwnedByNode returns true if the pod is a static pod owned by a specific node func IsOwnedByNode(pod *v1.Pod) bool { return IsOwnedBy(pod, []schema.GroupVersionKind{ - {Version: "v1", Kind: "Machine"}, + {Version: "v1", Kind: "Node"}, }) }