Support cgroup namespaces for cgroup v1

cgroups/src/test_manager.rs

@@ -11,12 +11,14 @@ use crate::{
 #[derive(Debug)]
 pub struct TestManager {
     add_task_args: RefCell<Vec<Pid>>,
+    pub apply_called: RefCell<bool>,
 }
 
 impl Default for TestManager {
     fn default() -> Self {
         Self {
             add_task_args: RefCell::new(vec![]),
+            apply_called: RefCell::new(false),
         }
     }
 }
@@ -29,6 +31,7 @@ impl CgroupManager for TestManager {
 
     // NOTE: The argument cannot be stored due to lifetime.
     fn apply(&self, _controller_opt: &ControllerOpt) -> Result<()> {
+        *self.apply_called.borrow_mut() = true;
         Ok(())
     }
 
@@ -53,4 +56,8 @@ impl TestManager {
     pub fn get_add_task_args(&self) -> Vec<Pid> {
         self.add_task_args.borrow_mut().clone()
     }
+
+    pub fn apply_called(&self) -> bool {
+        *self.apply_called.borrow_mut()
+    }
 }

cgroups/src/v1/controller_type.rs

@@ -37,6 +37,25 @@ impl Display for ControllerType {
     }
 }
 
+impl AsRef<str> for ControllerType {
+    fn as_ref(&self) -> &str {
+        match *self {
+            Self::Cpu => "cpu",
+            Self::CpuAcct => "cpuacct",
+            Self::CpuSet => "cpuset",
+            Self::Devices => "devices",
+            Self::HugeTlb => "hugetlb",
+            Self::Pids => "pids",
+            Self::PerfEvent => "perf_event",
+            Self::Memory => "memory",
+            Self::Blkio => "blkio",
+            Self::NetworkPriority => "net_prio",
+            Self::NetworkClassifier => "net_cls",
+            Self::Freezer => "freezer",
+        }
+    }
+}
+
 pub const CONTROLLERS: &[ControllerType] = &[
     ControllerType::Cpu,
     ControllerType::CpuAcct,

cgroups/src/v1/util.rs

@@ -1,11 +1,24 @@
 use std::{collections::HashMap, path::PathBuf};
 
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, Context, Result};
 use procfs::process::Process;
 
 use super::{controller_type::CONTROLLERS, ControllerType};
 
-pub fn list_subsystem_mount_points() -> Result<HashMap<ControllerType, PathBuf>> {
+/// List all cgroup v1 subsystem mount points on the system. This can include unsupported
+/// subsystems, comounted controllers and named hierarchies.
+pub fn list_subsystem_mount_points() -> Result<Vec<PathBuf>> {
+    Ok(Process::myself()?
+        .mountinfo()
+        .context("failed to get mountinfo")?
+        .into_iter()
+        .filter(|m| m.fs_type == "cgroup")
+        .map(|m| m.mount_point)
+        .collect())
+}
+
+/// List the mount points of all currently supported cgroup subsystems.
+pub fn list_supported_mount_points() -> Result<HashMap<ControllerType, PathBuf>> {
     let mut mount_paths = HashMap::with_capacity(CONTROLLERS.len());
 
     for controller in CONTROLLERS {
@@ -20,7 +33,8 @@ pub fn list_subsystem_mount_points() -> Result<HashMap<ControllerType, PathBuf>>
 pub fn get_subsystem_mount_point(subsystem: &ControllerType) -> Result<PathBuf> {
     let subsystem = subsystem.to_string();
     Process::myself()?
-        .mountinfo()?
+        .mountinfo()
+        .context("failed to get mountinfo")?
         .into_iter()
         .find(|m| {
             if m.fs_type == "cgroup" {

src/commands/info.rs

@@ -107,7 +107,7 @@ pub fn print_cgroups() {
     }
 
     println!("Cgroup mounts");
-    if let Ok(v1_mounts) = cgroups::v1::util::list_subsystem_mount_points() {
+    if let Ok(v1_mounts) = cgroups::v1::util::list_supported_mount_points() {
         let mut v1_mounts: Vec<String> = v1_mounts
             .iter()
             .map(|kv| format!("  {:<16}{}", kv.0.to_string(), kv.1.display()))

src/container/builder_impl.rs

@@ -7,9 +7,8 @@ use crate::{
     utils,
 };
 use anyhow::{bail, Context, Result};
-use cgroups::{self, common::CgroupManager};
 use nix::unistd::Pid;
-use oci_spec::runtime::{LinuxResources, Spec};
+use oci_spec::runtime::Spec;
 use std::{fs, io::Write, os::unix::prelude::RawFd, path::PathBuf};
 
 use super::{Container, ContainerStatus};
@@ -117,6 +116,7 @@ impl<'a> ContainerBuilderImpl<'a> {
             preserve_fds: self.preserve_fds,
             container: self.container.clone(),
             rootless: self.rootless.clone(),
+            cgroup_manager: cmanager,
         };
         let intermediate_pid = fork::container_fork(|| {
             // The fds in the pipe is duplicated during fork, so we first close
@@ -155,12 +155,6 @@ impl<'a> ContainerBuilderImpl<'a> {
         let init_pid = receiver_from_intermediate.wait_for_intermediate_ready()?;
         log::debug!("init pid is {:?}", init_pid);
 
-        if self.rootless.is_none() && linux.resources().is_some() && self.init {
-            if let Some(resources) = linux.resources() {
-                apply_cgroups(resources, init_pid, cmanager.as_ref())?;
-            }
-        }
-
         // if file to write the pid to is specified, write pid of the child
         if let Some(pid_file) = &self.pid_file {
             fs::write(&pid_file, format!("{}", init_pid)).context("Failed to write pid file")?;
@@ -223,31 +217,8 @@ fn setup_mapping(rootless: &Rootless, pid: Pid) -> Result<()> {
     Ok(())
 }
 
-fn apply_cgroups<C: CgroupManager + ?Sized>(
-    resources: &LinuxResources,
-    pid: Pid,
-    cmanager: &C,
-) -> Result<()> {
-    let controller_opt = cgroups::common::ControllerOpt {
-        resources,
-        freezer_state: None,
-        oom_score_adj: None,
-        disable_oom_killer: false,
-    };
-    cmanager
-        .add_task(pid)
-        .with_context(|| format!("failed to add task {} to cgroup manager", pid))?;
-
-    cmanager
-        .apply(&controller_opt)
-        .context("failed to apply resource limits to cgroup")?;
-
-    Ok(())
-}
-
 #[cfg(test)]
 mod tests {
-    use cgroups::test_manager::TestManager;
     use nix::{
         sched::{unshare, CloneFlags},
         unistd::{self, getgid, getuid},
@@ -353,14 +324,4 @@ mod tests {
         }
         Ok(())
     }
-
-    #[test]
-    fn apply_cgroup_successed() -> Result<()> {
-        let cmanager = TestManager::default();
-        let sample_pid = Pid::from_raw(1000);
-        let resources = LinuxResources::default();
-        apply_cgroups(&resources, sample_pid, &cmanager)?;
-        assert_eq!(cmanager.get_add_task_args(), vec![sample_pid]);
-        Ok(())
-    }
 }

src/process/args.rs

@@ -1,3 +1,4 @@
+use cgroups::common::CgroupManager;
 use oci_spec::runtime::Spec;
 use std::os::unix::prelude::RawFd;
 use std::path::PathBuf;
@@ -24,4 +25,6 @@ pub struct ContainerArgs<'a> {
     pub container: Option<Container>,
     /// Options for rootless containers
     pub rootless: Option<Rootless<'a>>,
+    /// Cgroup Manager
+    pub cgroup_manager: Box<dyn CgroupManager>,
 }

src/process/init.rs

@@ -187,11 +187,11 @@ pub fn container_init(
                 && ns_type != CloneFlags::CLONE_NEWPID
                 && ns_type != CloneFlags::CLONE_NEWNS
         })
-        .with_context(|| "Failed to apply namespaces")?;
+        .with_context(|| "failed to apply namespaces")?;
     if let Some(mount_namespace) = namespaces.get(LinuxNamespaceType::Mount) {
         namespaces
             .unshare_or_setns(mount_namespace)
-            .with_context(|| format!("Failed to enter mount namespace: {:?}", mount_namespace))?;
+            .with_context(|| format!("failed to enter mount namespace: {:?}", mount_namespace))?;
     }
 
     // Only set the host name if entering into a new uts namespace
@@ -216,8 +216,13 @@ pub fn container_init(
         }
 
         let bind_service = namespaces.get(LinuxNamespaceType::User).is_some();
-        rootfs::prepare_rootfs(spec, rootfs, bind_service)
-            .with_context(|| "Failed to prepare rootfs")?;
+        rootfs::prepare_rootfs(
+            spec,
+            rootfs,
+            bind_service,
+            namespaces.get(LinuxNamespaceType::Cgroup).is_some(),
+        )
+        .with_context(|| "Failed to prepare rootfs")?;
 
         // Entering into the rootfs jail. If mount namespace is specified, then
         // we use pivot_root, but if we are on the host mount namespace, we will

src/process/intermediate.rs

@@ -1,7 +1,10 @@
 use crate::{namespaces::Namespaces, process::channel, process::fork};
-use anyhow::{Context, Result};
-use nix::unistd::{Gid, Uid};
-use oci_spec::runtime::LinuxNamespaceType;
+use anyhow::{Context, Error, Result};
+use cgroups::common::CgroupManager;
+use nix::unistd::{Gid, Pid, Uid};
+use oci_spec::runtime::{LinuxNamespaceType, LinuxResources};
+use procfs::process::Process;
+use std::convert::From;
 
 use super::args::ContainerArgs;
 use super::init::container_init;
@@ -23,7 +26,7 @@ pub fn container_intermediate(
     if let Some(user_namespace) = namespaces.get(LinuxNamespaceType::User) {
         namespaces
             .unshare_or_setns(user_namespace)
-            .with_context(|| format!("Failed to enter pid namespace: {:?}", user_namespace))?;
+            .with_context(|| format!("Failed to enter user namespace: {:?}", user_namespace))?;
         if user_namespace.path().is_none() {
             log::debug!("creating new user namespace");
             // child needs to be dumpable, otherwise the non root parent is not
@@ -60,6 +63,17 @@ pub fn container_intermediate(
             .with_context(|| format!("Failed to enter pid namespace: {:?}", pid_namespace))?;
     }
 
+    // this needs to be done before we create the init process, so that the init
+    // process will already be captured by the cgroup
+    if args.rootless.is_none() {
+        apply_cgroups(
+            args.cgroup_manager.as_ref(),
+            linux.resources().as_ref(),
+            args.init,
+        )
+        .context("failed to apply cgroups")?
+    }
+
     // We only need for init process to send us the ChildReady.
     let (sender_to_intermediate, receiver_from_init) = &mut channel::init_to_intermediate()?;
 
@@ -90,3 +104,94 @@ pub fn container_intermediate(
 
     Ok(())
 }
+
+fn apply_cgroups<C: CgroupManager + ?Sized>(
+    cmanager: &C,
+    resources: Option<&LinuxResources>,
+    init: bool,
+) -> Result<(), Error> {
+    let pid = Pid::from_raw(Process::myself()?.pid());
+    cmanager
+        .add_task(pid)
+        .with_context(|| format!("failed to add task {} to cgroup manager", pid))?;
+
+    if let Some(resources) = resources {
+        if init {
+            let controller_opt = cgroups::common::ControllerOpt {
+                resources,
+                freezer_state: None,
+                oom_score_adj: None,
+                disable_oom_killer: false,
+            };
+
+            cmanager
+                .apply(&controller_opt)
+                .context("failed to apply resource limits to cgroup")?;
+        }
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::apply_cgroups;
+    use anyhow::Result;
+    use cgroups::test_manager::TestManager;
+    use nix::unistd::Pid;
+    use oci_spec::runtime::LinuxResources;
+    use procfs::process::Process;
+
+    #[test]
+    fn apply_cgroup_init() -> Result<()> {
+        // arrange
+        let cmanager = TestManager::default();
+        let resources = LinuxResources::default();
+
+        // act
+        apply_cgroups(&cmanager, Some(&resources), true)?;
+
+        // assert
+        assert!(cmanager.get_add_task_args().len() == 1);
+        assert_eq!(
+            cmanager.get_add_task_args()[0],
+            Pid::from_raw(Process::myself()?.pid())
+        );
+        assert!(cmanager.apply_called());
+        Ok(())
+    }
+
+    #[test]
+    fn apply_cgroup_tenant() -> Result<()> {
+        // arrange
+        let cmanager = TestManager::default();
+        let resources = LinuxResources::default();
+
+        // act
+        apply_cgroups(&cmanager, Some(&resources), false)?;
+
+        // assert
+        assert_eq!(
+            cmanager.get_add_task_args()[0],
+            Pid::from_raw(Process::myself()?.pid())
+        );
+        assert!(!cmanager.apply_called());
+        Ok(())
+    }
+
+    #[test]
+    fn apply_cgroup_no_resources() -> Result<()> {
+        // arrange
+        let cmanager = TestManager::default();
+
+        // act
+        apply_cgroups(&cmanager, None, true)?;
+        // assert
+        assert_eq!(
+            cmanager.get_add_task_args()[0],
+            Pid::from_raw(Process::myself()?.pid())
+        );
+        assert!(!cmanager.apply_called());
+        Ok(())
+    }
+}