From 104b8dc951ee88628f9fbf3222961bb2de8b0294 Mon Sep 17 00:00:00 2001 From: Heran Yang Date: Sat, 9 Sep 2023 15:40:39 +0800 Subject: [PATCH] libct/cg: add swapOnlyUsage in MemoryStats This field reports swap-only usage. For cgroupv1, `Usage` and `Failcnt` are set by subtracting memory usage from memory+swap usage. For cgroupv2, `Usage`, `Limit`, and `MaxUsage` are set. This commit also export `MaxUsage` of memory under cgroupv2 mode, using `memory.peak` introduced in kernel 5.19. Signed-off-by: Heran Yang --- CHANGELOG.md | 4 ++++ libcontainer/cgroups/fs/memory.go | 4 ++++ libcontainer/cgroups/fs/memory_test.go | 13 +++++++------ libcontainer/cgroups/fs2/memory.go | 18 ++++++++++++++++-- libcontainer/cgroups/fs2/memory_test.go | 16 ++++++++++++++++ libcontainer/cgroups/stats.go | 2 ++ 6 files changed, 49 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index adcf084d0b6..ffdc9ae2fb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 This aligns cgroupv2 root usage more closely with cgroupv1 reporting. Additionally, report root swap usage as sum of swap and memory usage, aligned with v1 and existing non-root v2 reporting. (#3933) + * Add `swapOnlyUsage` in `MemoryStats`. This field reports swap-only usage. + For cgroupv1, `Usage` and `Failcnt` are set by subtracting memory usage + from memory+swap usage. For cgroupv2, `Usage`, `Limit`, and `MaxUsage` + are set. (#4010) ### Fixed diff --git a/libcontainer/cgroups/fs/memory.go b/libcontainer/cgroups/fs/memory.go index a0e78074980..783566d68f0 100644 --- a/libcontainer/cgroups/fs/memory.go +++ b/libcontainer/cgroups/fs/memory.go @@ -170,6 +170,10 @@ func (s *MemoryGroup) GetStats(path string, stats *cgroups.Stats) error { return err } stats.MemoryStats.SwapUsage = swapUsage + stats.MemoryStats.SwapOnlyUsage = cgroups.MemoryData{ + Usage: swapUsage.Usage - memoryUsage.Usage, + Failcnt: swapUsage.Failcnt - memoryUsage.Failcnt, + } kernelUsage, err := getMemoryData(path, "kmem") if err != nil { return err diff --git a/libcontainer/cgroups/fs/memory_test.go b/libcontainer/cgroups/fs/memory_test.go index d305a62a393..95e9d3cbaa4 100644 --- a/libcontainer/cgroups/fs/memory_test.go +++ b/libcontainer/cgroups/fs/memory_test.go @@ -249,12 +249,13 @@ func TestMemoryStats(t *testing.T) { t.Fatal(err) } expectedStats := cgroups.MemoryStats{ - Cache: 512, - Usage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, - SwapUsage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, - KernelUsage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, - Stats: map[string]uint64{"cache": 512, "rss": 1024}, - UseHierarchy: true, + Cache: 512, + Usage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, + SwapUsage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, + SwapOnlyUsage: cgroups.MemoryData{Usage: 0, MaxUsage: 0, Failcnt: 0, Limit: 0}, + KernelUsage: cgroups.MemoryData{Usage: 2048, MaxUsage: 4096, Failcnt: 100, Limit: 8192}, + Stats: map[string]uint64{"cache": 512, "rss": 1024}, + UseHierarchy: true, PageUsageByNUMA: cgroups.PageUsageByNUMA{ PageUsageByNUMAInner: cgroups.PageUsageByNUMAInner{ Total: cgroups.PageStats{Total: 44611, Nodes: map[uint8]uint64{0: 32631, 1: 7501, 2: 1982, 3: 2497}}, diff --git a/libcontainer/cgroups/fs2/memory.go b/libcontainer/cgroups/fs2/memory.go index 85e96b1ce98..29656597423 100644 --- a/libcontainer/cgroups/fs2/memory.go +++ b/libcontainer/cgroups/fs2/memory.go @@ -105,7 +105,7 @@ func statMemory(dirPath string, stats *cgroups.Stats) error { memoryUsage, err := getMemoryDataV2(dirPath, "") if err != nil { if errors.Is(err, unix.ENOENT) && dirPath == UnifiedMountpoint { - // The root cgroup does not have memory.{current,max} + // The root cgroup does not have memory.{current,max,peak} // so emulate those using data from /proc/meminfo and // /sys/fs/cgroup/memory.stat return rootStatsFromMeminfo(stats) @@ -113,10 +113,12 @@ func statMemory(dirPath string, stats *cgroups.Stats) error { return err } stats.MemoryStats.Usage = memoryUsage - swapUsage, err := getMemoryDataV2(dirPath, "swap") + swapOnlyUsage, err := getMemoryDataV2(dirPath, "swap") if err != nil { return err } + stats.MemoryStats.SwapOnlyUsage = swapOnlyUsage + swapUsage := swapOnlyUsage // As cgroup v1 reports SwapUsage values as mem+swap combined, // while in cgroup v2 swap values do not include memory, // report combined mem+swap for v1 compatibility. @@ -124,6 +126,9 @@ func statMemory(dirPath string, stats *cgroups.Stats) error { if swapUsage.Limit != math.MaxUint64 { swapUsage.Limit += memoryUsage.Limit } + // The `MaxUsage` of mem+swap cannot simply combine mem with + // swap. So set it to 0 for v1 compatibility. + swapUsage.MaxUsage = 0 stats.MemoryStats.SwapUsage = swapUsage return nil @@ -138,6 +143,7 @@ func getMemoryDataV2(path, name string) (cgroups.MemoryData, error) { } usage := moduleName + ".current" limit := moduleName + ".max" + maxUsage := moduleName + ".peak" value, err := fscommon.GetCgroupParamUint(path, usage) if err != nil { @@ -157,6 +163,14 @@ func getMemoryDataV2(path, name string) (cgroups.MemoryData, error) { } memoryData.Limit = value + // `memory.peak` since kernel 5.19 + // `memory.swap.peak` since kernel 6.5 + value, err = fscommon.GetCgroupParamUint(path, maxUsage) + if err != nil && !os.IsNotExist(err) { + return cgroups.MemoryData{}, err + } + memoryData.MaxUsage = value + return memoryData, nil } diff --git a/libcontainer/cgroups/fs2/memory_test.go b/libcontainer/cgroups/fs2/memory_test.go index 2e2713c29ae..89c999d0cee 100644 --- a/libcontainer/cgroups/fs2/memory_test.go +++ b/libcontainer/cgroups/fs2/memory_test.go @@ -94,6 +94,10 @@ func TestStatMemoryPodCgroup(t *testing.T) { t.Fatal(err) } + if err := os.WriteFile(filepath.Join(fakeCgroupDir, "memory.peak"), []byte("987654321"), 0o644); err != nil { + t.Fatal(err) + } + gotStats := cgroups.NewStats() // use a fake root path to trigger the pod cgroup lookup. @@ -107,6 +111,18 @@ func TestStatMemoryPodCgroup(t *testing.T) { if gotStats.MemoryStats.Usage.Usage != expectedUsageBytes { t.Errorf("parsed cgroupv2 memory.stat doesn't match expected result: \ngot %#v\nexpected %#v\n", gotStats.MemoryStats.Usage.Usage, expectedUsageBytes) } + + // result should be "memory.max" + var expectedLimitBytes uint64 = 999999999 + if gotStats.MemoryStats.Usage.Limit != expectedLimitBytes { + t.Errorf("parsed cgroupv2 memory.stat doesn't match expected result: \ngot %#v\nexpected %#v\n", gotStats.MemoryStats.Usage.Limit, expectedLimitBytes) + } + + // result should be "memory.peak" + var expectedMaxUsageBytes uint64 = 987654321 + if gotStats.MemoryStats.Usage.MaxUsage != expectedMaxUsageBytes { + t.Errorf("parsed cgroupv2 memory.stat doesn't match expected result: \ngot %#v\nexpected %#v\n", gotStats.MemoryStats.Usage.MaxUsage, expectedMaxUsageBytes) + } } func TestRootStatsFromMeminfo(t *testing.T) { diff --git a/libcontainer/cgroups/stats.go b/libcontainer/cgroups/stats.go index 8ff1fbb52bb..363017f1876 100644 --- a/libcontainer/cgroups/stats.go +++ b/libcontainer/cgroups/stats.go @@ -91,6 +91,8 @@ type MemoryStats struct { Usage MemoryData `json:"usage,omitempty"` // usage of memory + swap SwapUsage MemoryData `json:"swap_usage,omitempty"` + // usage of swap only + SwapOnlyUsage MemoryData `json:"swap_only_usage,omitempty"` // usage of kernel memory KernelUsage MemoryData `json:"kernel_usage,omitempty"` // usage of kernel TCP memory