diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/BUILD b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/BUILD index edcc7f53bf68..000741247cba 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/BUILD +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/BUILD @@ -50,7 +50,7 @@ go_library( "//pkg/apis/core/v1/helper:go_default_library", "//pkg/apis/core/v1/helper/qos:go_default_library", "//pkg/kubelet/cadvisor:go_default_library", - "//pkg/kubelet/cm/cpumanager/containermap:go_default_library", + "//pkg/kubelet/cm/containermap:go_default_library", "//pkg/kubelet/cm/cpumanager/topology:go_default_library", "//pkg/kubelet/cm/devicemanager:go_default_library", "//pkg/kubelet/cm/util:go_default_library", @@ -101,7 +101,7 @@ go_library( "//pkg/apis/core/v1/helper:go_default_library", "//pkg/apis/core/v1/helper/qos:go_default_library", "//pkg/kubelet/cadvisor:go_default_library", - "//pkg/kubelet/cm/cpumanager/containermap:go_default_library", + "//pkg/kubelet/cm/containermap:go_default_library", "//pkg/kubelet/cm/cpumanager/topology:go_default_library", "//pkg/kubelet/cm/devicemanager:go_default_library", "//pkg/kubelet/cm/util:go_default_library", @@ -212,6 +212,7 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", + "//pkg/kubelet/cm/containermap:all-srcs", "//pkg/kubelet/cm/cpumanager:all-srcs", "//pkg/kubelet/cm/cpuset:all-srcs", "//pkg/kubelet/cm/devicemanager:all-srcs", diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/container_manager_linux.go b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/container_manager_linux.go index 13c7176bdc27..2d77f955cbf5 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/container_manager_linux.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/container_manager_linux.go @@ -49,8 +49,8 @@ import ( kubefeatures "k8s.io/kubernetes/pkg/features" podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cadvisor" + "k8s.io/kubernetes/pkg/kubelet/cm/containermap" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager" - "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/containermap" cputopology "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/containermap/BUILD b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/containermap/BUILD similarity index 88% rename from vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/containermap/BUILD rename to vendor/k8s.io/kubernetes/pkg/kubelet/cm/containermap/BUILD index a7f53443c775..5260916cf67c 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/containermap/BUILD +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/containermap/BUILD @@ -3,7 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = ["container_map.go"], - importpath = "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/containermap", + importpath = "k8s.io/kubernetes/pkg/kubelet/cm/containermap", visibility = ["//visibility:public"], ) diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/containermap/container_map.go b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/containermap/container_map.go similarity index 100% rename from vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/containermap/container_map.go rename to vendor/k8s.io/kubernetes/pkg/kubelet/cm/containermap/container_map.go diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/containermap/container_map_test.go b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/containermap/container_map_test.go similarity index 100% rename from vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/containermap/container_map_test.go rename to vendor/k8s.io/kubernetes/pkg/kubelet/cm/containermap/container_map_test.go diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/BUILD b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/BUILD index 45092994ef06..ac49d344125c 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/BUILD +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/BUILD @@ -14,7 +14,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/apis/core/v1/helper/qos:go_default_library", - "//pkg/kubelet/cm/cpumanager/containermap:go_default_library", + "//pkg/kubelet/cm/containermap:go_default_library", "//pkg/kubelet/cm/cpumanager/state:go_default_library", "//pkg/kubelet/cm/cpumanager/topology:go_default_library", "//pkg/kubelet/cm/cpuset:go_default_library", @@ -43,7 +43,7 @@ go_test( ], embed = [":go_default_library"], deps = [ - "//pkg/kubelet/cm/cpumanager/containermap:go_default_library", + "//pkg/kubelet/cm/containermap:go_default_library", "//pkg/kubelet/cm/cpumanager/state:go_default_library", "//pkg/kubelet/cm/cpumanager/topology:go_default_library", "//pkg/kubelet/cm/cpuset:go_default_library", @@ -69,7 +69,6 @@ filegroup( name = "all-srcs", srcs = [ ":package-srcs", - "//pkg/kubelet/cm/cpumanager/containermap:all-srcs", "//pkg/kubelet/cm/cpumanager/state:all-srcs", "//pkg/kubelet/cm/cpumanager/topology:all-srcs", ], diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/cpu_manager.go b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/cpu_manager.go index 616a620f8ce5..9d52a09b9e16 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/cpu_manager.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/cpu_manager.go @@ -28,7 +28,7 @@ import ( "k8s.io/klog" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" - "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/containermap" + "k8s.io/kubernetes/pkg/kubelet/cm/containermap" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/cpu_manager_test.go b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/cpu_manager_test.go index e9c7852c602e..86d2063e59b0 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/cpu_manager_test.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/cpu_manager_test.go @@ -33,7 +33,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" - "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/containermap" + "k8s.io/kubernetes/pkg/kubelet/cm/containermap" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" @@ -503,6 +503,8 @@ func TestCPUManagerAddWithInitContainers(t *testing.T) { testCase.expInitCSets, testCase.expCSets...) + cumCSet := cpuset.NewCPUSet() + for i := range containers { err := mgr.Allocate(testCase.pod, &containers[i]) if err != nil { @@ -525,6 +527,13 @@ func TestCPUManagerAddWithInitContainers(t *testing.T) { t.Errorf("StaticPolicy AddContainer() error (%v). expected cpuset %v for container %v but got %v", testCase.description, expCSets[i], containers[i].Name, cset) } + + cumCSet = cumCSet.Union(cset) + } + + if !testCase.stDefaultCPUSet.Difference(cumCSet).Equals(state.defaultCPUSet) { + t.Errorf("StaticPolicy error (%v). expected final state for defaultCPUSet %v but got %v", + testCase.description, testCase.stDefaultCPUSet.Difference(cumCSet), state.defaultCPUSet) } } } diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go index b5537f787b65..56e88b1e1593 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/fake_cpu_manager.go @@ -19,7 +19,7 @@ package cpumanager import ( "k8s.io/api/core/v1" "k8s.io/klog" - "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/containermap" + "k8s.io/kubernetes/pkg/kubelet/cm/containermap" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state" "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/config" diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/policy_static.go b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/policy_static.go index da68ed808bda..02b88e9b4c00 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/policy_static.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/policy_static.go @@ -77,6 +77,8 @@ type staticPolicy struct { reserved cpuset.CPUSet // topology manager reference to get container Topology affinity affinity topologymanager.Store + // set of CPUs to reuse across allocations in a pod + cpusToReuse map[string]cpuset.CPUSet } // Ensure staticPolicy implements Policy interface @@ -107,9 +109,10 @@ func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int, reserv klog.Infof("[cpumanager] reserved %d CPUs (\"%s\") not available for exclusive assignment", reserved.Size(), reserved) return &staticPolicy{ - topology: topology, - reserved: reserved, - affinity: affinity, + topology: topology, + reserved: reserved, + affinity: affinity, + cpusToReuse: make(map[string]cpuset.CPUSet), }, nil } @@ -188,12 +191,37 @@ func (p *staticPolicy) assignableCPUs(s state.State) cpuset.CPUSet { return s.GetDefaultCPUSet().Difference(p.reserved) } +func (p *staticPolicy) updateCPUsToReuse(pod *v1.Pod, container *v1.Container, cset cpuset.CPUSet) { + // If pod entries to m.cpusToReuse other than the current pod exist, delete them. + for podUID := range p.cpusToReuse { + if podUID != string(pod.UID) { + delete(p.cpusToReuse, podUID) + } + } + // If no cpuset exists for cpusToReuse by this pod yet, create one. + if _, ok := p.cpusToReuse[string(pod.UID)]; !ok { + p.cpusToReuse[string(pod.UID)] = cpuset.NewCPUSet() + } + // Check if the container is an init container. + // If so, add its cpuset to the cpuset of reusable CPUs for any new allocations. + for _, initContainer := range pod.Spec.InitContainers { + if container.Name == initContainer.Name { + p.cpusToReuse[string(pod.UID)] = p.cpusToReuse[string(pod.UID)].Union(cset) + return + } + } + // Otherwise it is an app container. + // Remove its cpuset from the cpuset of reusable CPUs for any new allocations. + p.cpusToReuse[string(pod.UID)] = p.cpusToReuse[string(pod.UID)].Difference(cset) +} + func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Container) error { if numCPUs := p.guaranteedCPUs(pod, container); numCPUs != 0 { klog.Infof("[cpumanager] static policy: Allocate (pod: %s, container: %s)", pod.Name, container.Name) // container belongs in an exclusively allocated pool - if _, ok := s.GetCPUSet(string(pod.UID), container.Name); ok { + if cpuset, ok := s.GetCPUSet(string(pod.UID), container.Name); ok { + p.updateCPUsToReuse(pod, container, cpuset) klog.Infof("[cpumanager] static policy: container already present in state, skipping (pod: %s, container: %s)", pod.Name, container.Name) return nil } @@ -203,23 +231,14 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai klog.Infof("[cpumanager] Pod %v, Container %v Topology Affinity is: %v", pod.UID, container.Name, hint) // Allocate CPUs according to the NUMA affinity contained in the hint. - cpuset, err := p.allocateCPUs(s, numCPUs, hint.NUMANodeAffinity) + cpuset, err := p.allocateCPUs(s, numCPUs, hint.NUMANodeAffinity, p.cpusToReuse[string(pod.UID)]) if err != nil { klog.Errorf("[cpumanager] unable to allocate %d CPUs (pod: %s, container: %s, error: %v)", numCPUs, pod.Name, container.Name, err) return err } s.SetCPUSet(string(pod.UID), container.Name, cpuset) + p.updateCPUsToReuse(pod, container, cpuset) - // Check if the container that has just been allocated resources is an init container. - // If so, release its CPUs back into the shared pool so they can be reallocated. - for _, initContainer := range pod.Spec.InitContainers { - if container.Name == initContainer.Name { - if toRelease, ok := s.GetCPUSet(string(pod.UID), container.Name); ok { - // Mutate the shared pool, adding released cpus. - s.SetDefaultCPUSet(s.GetDefaultCPUSet().Union(toRelease)) - } - } - } } // container belongs in the shared pool (nothing to do; use default cpuset) return nil @@ -235,15 +254,17 @@ func (p *staticPolicy) RemoveContainer(s state.State, podUID string, containerNa return nil } -func (p *staticPolicy) allocateCPUs(s state.State, numCPUs int, numaAffinity bitmask.BitMask) (cpuset.CPUSet, error) { +func (p *staticPolicy) allocateCPUs(s state.State, numCPUs int, numaAffinity bitmask.BitMask, reusableCPUs cpuset.CPUSet) (cpuset.CPUSet, error) { klog.Infof("[cpumanager] allocateCpus: (numCPUs: %d, socket: %v)", numCPUs, numaAffinity) + assignableCPUs := p.assignableCPUs(s).Union(reusableCPUs) + // If there are aligned CPUs in numaAffinity, attempt to take those first. result := cpuset.NewCPUSet() if numaAffinity != nil { alignedCPUs := cpuset.NewCPUSet() for _, numaNodeID := range numaAffinity.GetBits() { - alignedCPUs = alignedCPUs.Union(p.assignableCPUs(s).Intersection(p.topology.CPUDetails.CPUsInNUMANodes(numaNodeID))) + alignedCPUs = alignedCPUs.Union(assignableCPUs.Intersection(p.topology.CPUDetails.CPUsInNUMANodes(numaNodeID))) } numAlignedToAlloc := alignedCPUs.Size() @@ -260,7 +281,7 @@ func (p *staticPolicy) allocateCPUs(s state.State, numCPUs int, numaAffinity bit } // Get any remaining CPUs from what's leftover after attempting to grab aligned ones. - remainingCPUs, err := takeByTopology(p.topology, p.assignableCPUs(s).Difference(result), numCPUs-result.Size()) + remainingCPUs, err := takeByTopology(p.topology, assignableCPUs.Difference(result), numCPUs-result.Size()) if err != nil { return cpuset.NewCPUSet(), err } diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/policy_static_test.go b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/policy_static_test.go index ea2bcf11333b..b4b46c68c17b 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/policy_static_test.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/policy_static_test.go @@ -639,7 +639,7 @@ func TestTopologyAwareAllocateCPUs(t *testing.T) { continue } - cset, err := policy.allocateCPUs(st, tc.numRequested, tc.socketMask) + cset, err := policy.allocateCPUs(st, tc.numRequested, tc.socketMask, cpuset.NewCPUSet()) if err != nil { t.Errorf("StaticPolicy allocateCPUs() error (%v). expected CPUSet %v not error %v", tc.description, tc.expCSet, err) diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/BUILD b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/BUILD index 438737c9d181..4033b194a3c0 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/BUILD +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/BUILD @@ -15,7 +15,7 @@ go_library( "//pkg/kubelet/checkpointmanager:go_default_library", "//pkg/kubelet/checkpointmanager/checksum:go_default_library", "//pkg/kubelet/checkpointmanager/errors:go_default_library", - "//pkg/kubelet/cm/cpumanager/containermap:go_default_library", + "//pkg/kubelet/cm/containermap:go_default_library", "//pkg/kubelet/cm/cpuset:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", "//vendor/k8s.io/klog:go_default_library", @@ -32,7 +32,7 @@ go_test( embed = [":go_default_library"], deps = [ "//pkg/kubelet/checkpointmanager:go_default_library", - "//pkg/kubelet/cm/cpumanager/containermap:go_default_library", + "//pkg/kubelet/cm/containermap:go_default_library", "//pkg/kubelet/cm/cpumanager/state/testing:go_default_library", "//pkg/kubelet/cm/cpuset:go_default_library", "//vendor/github.com/stretchr/testify/require:go_default_library", diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/state_checkpoint.go b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/state_checkpoint.go index de82f69d5c15..a505f131f21e 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/state_checkpoint.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/state_checkpoint.go @@ -24,7 +24,7 @@ import ( "k8s.io/klog" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager/errors" - "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/containermap" + "k8s.io/kubernetes/pkg/kubelet/cm/containermap" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" ) diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/state_checkpoint_test.go b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/state_checkpoint_test.go index fc96da2931ea..789f9674c2b9 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/state_checkpoint_test.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/state_checkpoint_test.go @@ -22,7 +22,7 @@ import ( "testing" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" - "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/containermap" + "k8s.io/kubernetes/pkg/kubelet/cm/containermap" testutil "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/testing" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" ) diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/state_file.go b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/state_file.go index b61505a3de41..aa944ffa9f9d 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/state_file.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/state_file.go @@ -24,7 +24,7 @@ import ( "sync" "k8s.io/klog" - "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/containermap" + "k8s.io/kubernetes/pkg/kubelet/cm/containermap" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" ) diff --git a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/state_file_test.go b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/state_file_test.go index a0eae9f1071a..0f48450a104f 100644 --- a/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/state_file_test.go +++ b/vendor/k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state/state_file_test.go @@ -27,7 +27,7 @@ import ( "strings" "testing" - "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/containermap" + "k8s.io/kubernetes/pkg/kubelet/cm/containermap" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" )