Add support for rdma cgroup introduced in Linux Kernal 4.11

Signed-off-by: flouthoc <[email protected]>

libcontainer/cgroups/fs/rdma.go

@@ -0,0 +1,115 @@
+// +build linux
+
+package fs
+
+import (
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
+	"github.com/opencontainers/runc/libcontainer/configs"
+	"math"
+	"strconv"
+	"strings"
+)
+
+type RdmaGroup struct {
+}
+
+func (s *RdmaGroup) Name() string {
+	return "rdma"
+}
+
+func (s *RdmaGroup) Apply(path string, d *cgroupData) error {
+	return join(path, d.pid)
+}
+
+func createCmdString(device string, limits configs.LinuxRdma) string {
+	cmdString := device
+	if limits.HcaHandles != nil {
+		cmdString += " hca_handle=" + strconv.FormatUint(uint64(*limits.HcaHandles), 10)
+	}
+	if limits.HcaObjects != nil {
+		cmdString += " hca_object=" + strconv.FormatUint(uint64(*limits.HcaObjects), 10)
+	}
+	return cmdString
+}
+
+func (s *RdmaGroup) Set(path string, cgroup *configs.Cgroup) error {
+	for device, limits := range cgroup.Resources.Rdma {
+		if err := fscommon.WriteFile(path, "rdma.max", createCmdString(device, limits)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *RdmaGroup) GetStats(path string, stats *cgroups.Stats) error {
+	if !cgroups.PathExists(path) {
+		return nil
+	}
+	currentData, err := fscommon.ReadFile(path, "rdma.current")
+	if err != nil {
+		return err
+	}
+	currentPerDevices := strings.Split(currentData, "\n")
+	maxData, err := fscommon.ReadFile(path, "rdma.max")
+	if err != nil {
+		return err
+	}
+	maxPerDevices := strings.Split(maxData, "\n")
+	// If device got removed between reading two files, ignore returning
+	// stats.
+	if len(currentPerDevices) != len(maxPerDevices) {
+		return nil
+	}
+	currentEntries := ConvertRdmaEntry(currentPerDevices)
+	maxEntries := ConvertRdmaEntry(maxPerDevices)
+
+	stats.RdmaStats = cgroups.RdmaStats{
+		RdmaLimit:   maxEntries,
+		RdmaCurrent: currentEntries,
+	}
+
+	return nil
+}
+
+// Following is intentionally public as it going to be used by Rdma controller in fs2
+// ParseRdmaKV: parse raw string to RdmaEntry.
+func ParseRdmaKV(raw string, entry *cgroups.RdmaEntry) {
+	var value uint64
+	var err error
+
+	parts := strings.SplitN(raw, "=", 3)
+	if len(parts) == 2 {
+		if parts[1] == "max" {
+			value = math.MaxUint32
+		} else {
+			value, err = strconv.ParseUint(parts[1], 10, 32)
+			if err != nil {
+				return
+			}
+		}
+		if parts[0] == "hca_handle" {
+			entry.HcaHandles = uint32(value)
+		} else if parts[0] == "hca_object" {
+			entry.HcaObjects = uint32(value)
+		}
+	}
+}
+
+// Following is intentionally public as it going to be used by Rdma controller in fs2
+// ConvertRdmaEntry: Converts array of rawstrings to RdmaEntries
+func ConvertRdmaEntry(strEntries []string) []cgroups.RdmaEntry {
+	rdmaEntries := make([]cgroups.RdmaEntry, len(strEntries))
+	for i := range strEntries {
+		parts := strings.Fields(strEntries[i])
+		if len(parts) == 3{
+			entry := new(cgroups.RdmaEntry)
+			entry.Device = parts[0]
+			ParseRdmaKV(parts[1], entry)
+			ParseRdmaKV(parts[2], entry)
+
+			rdmaEntries = append(rdmaEntries, *entry)
+		}
+	}
+	return rdmaEntries
+}

libcontainer/cgroups/fs2/fs2.go

@@ -141,6 +141,12 @@ func (m *manager) GetStats() (*cgroups.Stats, error) {
 			errs = append(errs, err)
 		}
 	}
+	// rdma (since kernel 4.11)
+	if _, ok := m.controllers["rdma"]; ok {
+		if err := statRdma(m.dirPath, st); err != nil {
+			errs = append(errs, err)
+		}
+	}
 	if len(errs) > 0 && !m.rootless {
 		return st, errors.Errorf("error while statting cgroup v2: %+v", errs)
 	}
@@ -202,6 +208,10 @@ func (m *manager) Set(container *configs.Config) error {
 	if err := setHugeTlb(m.dirPath, container.Cgroups); err != nil {
 		return err
 	}
+	// rdma (since kernel 4.11)
+	if err := setRdma(m.dirPath, container.Cgroups); err != nil {
+		return err
+	}
 	// freezer (since kernel 5.2, pseudo-controller)
 	if err := setFreezer(m.dirPath, container.Cgroups.Freezer); err != nil {
 		return err

libcontainer/cgroups/fs2/rdma.go

@@ -0,0 +1,69 @@
+// +build linux
+
+package fs2
+
+import (
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/cgroups/fs"
+	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
+	"github.com/opencontainers/runc/libcontainer/configs"
+	"strconv"
+	"strings"
+)
+
+func isRdmaSet(cgroup *configs.Cgroup) bool {
+	return len(cgroup.Resources.Rdma) > 0
+}
+
+func createCmdString(device string, limits configs.LinuxRdma) string {
+	cmdString := device
+	if limits.HcaHandles != nil {
+		cmdString += " hca_handle=" + strconv.FormatUint(uint64(*limits.HcaHandles), 10)
+	}
+	if limits.HcaObjects != nil {
+		cmdString += " hca_object=" + strconv.FormatUint(uint64(*limits.HcaObjects), 10)
+	}
+	return cmdString
+}
+
+func setRdma(path string, cgroup *configs.Cgroup) error {
+	if !isRdmaSet(cgroup) {
+		return nil
+	}
+	for device, limits := range cgroup.Resources.Rdma {
+		if err := fscommon.WriteFile(path, "rdma.max", createCmdString(device, limits)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func statRdma(path string, stats *cgroups.Stats) error {
+	if !cgroups.PathExists(path) {
+		return nil
+	}
+	currentData, err := fscommon.ReadFile(path, "rdma.current")
+	if err != nil {
+		return err
+	}
+	currentPerDevices := strings.Split(currentData, "\n")
+	maxData, err := fscommon.ReadFile(path, "rdma.max")
+	if err != nil {
+		return err
+	}
+	maxPerDevices := strings.Split(maxData, "\n")
+	// If device got removed between reading two files, ignore returning
+	// stats.
+	if len(currentPerDevices) != len(maxPerDevices) {
+		return nil
+	}
+	currentEntries := fs.ConvertRdmaEntry(currentPerDevices)
+	maxEntries := fs.ConvertRdmaEntry(maxPerDevices)
+
+	stats.RdmaStats = cgroups.RdmaStats{
+		RdmaLimit:   maxEntries,
+		RdmaCurrent: currentEntries,
+	}
+
+	return nil
+}

libcontainer/cgroups/stats.go

@@ -146,6 +146,17 @@ type HugetlbStats struct {
 	Failcnt uint64 `json:"failcnt"`
 }
 
+type RdmaEntry struct {
+	Device     string `json:"device,omitempty"`
+	HcaHandles uint32 `json:"hca_handles,omitempty"`
+	HcaObjects uint32 `json:"hca_objects,omitempty"`
+}
+
+type RdmaStats struct {
+	RdmaLimit   []RdmaEntry `json:"rdma_limit,omitempty"`
+	RdmaCurrent []RdmaEntry `json:"rdma_current,omitempty"`
+}
+
 type Stats struct {
 	CpuStats    CpuStats    `json:"cpu_stats,omitempty"`
 	CPUSetStats CPUSetStats `json:"cpuset_stats,omitempty"`
@@ -154,6 +165,7 @@ type Stats struct {
 	BlkioStats  BlkioStats  `json:"blkio_stats,omitempty"`
 	// the map is in the format "size of hugepage: stats of the hugepage"
 	HugetlbStats map[string]HugetlbStats `json:"hugetlb_stats,omitempty"`
+	RdmaStats    RdmaStats               `json:"rdma_stats,omitempty"`
 }
 
 func NewStats() *Stats {

libcontainer/cgroups/systemd/v1.go

@@ -55,6 +55,7 @@ var legacySubsystems = []subsystem{
 	&fs.NetPrioGroup{},
 	&fs.NetClsGroup{},
 	&fs.NameGroup{GroupName: "name=systemd"},
+	&fs.RdmaGroup{},
 }
 
 func genV1ResourcesProperties(c *configs.Cgroup, conn *systemdDbus.Conn) ([]systemdDbus.Property, error) {

libcontainer/configs/cgroup_linux.go

@@ -123,6 +123,9 @@ type Resources struct {
 	// Set class identifier for container's network packets
 	NetClsClassid uint32 `json:"net_cls_classid_u"`
 
+	// Rdma resource restriction configuration
+	Rdma map[string]LinuxRdma `json:"rdma"`
+
 	// Used on cgroups v2:
 
 	// CpuWeight sets a proportional bandwidth limit.

libcontainer/configs/rdma.go

@@ -0,0 +1,9 @@
+package configs
+
+// LinuxRdma for Linux cgroup 'rdma' resource management (Linux 4.11)
+type LinuxRdma struct {
+	// Maximum number of HCA handles that can be opened. Default is "no limit".
+	HcaHandles *uint32 `json:"hcaHandles,omitempty"`
+	// Maximum number of HCA objects that can be created. Default is "no limit".
+	HcaObjects *uint32 `json:"hcaObjects,omitempty"`
+}

libcontainer/specconv/spec_linux.go

@@ -607,6 +607,15 @@ func CreateCgroupConfig(opts *CreateOpts, defaultDevs []*devices.Device) (*confi
 					Limit:    l.Limit,
 				})
 			}
+			if len(r.Rdma) > 0 {
+				for k, v := range r.Rdma {
+					c.Resources.Rdma = make(map[string]configs.LinuxRdma, len(r.Rdma))
+					c.Resources.Rdma[k] = configs.LinuxRdma{
+						HcaHandles: v.HcaHandles,
+						HcaObjects: v.HcaObjects,
+					}
+				}
+			}
 			if r.Network != nil {
 				if r.Network.ClassID != nil {
 					c.Resources.NetClsClassid = *r.Network.ClassID