Networkd containers

jobset.nix

@@ -0,0 +1,18 @@
+# FIXME remove before merging!
+
+{ nixpkgs }:
+let
+  release = import ./nixos/release.nix {
+    supportedSystems = [ "x86_64-linux" ];
+    inherit nixpkgs;
+  };
+in
+
+{
+  container-tests = {
+    general = release.tests.containers-next;
+    migration = release.tests.containers-migration;
+    activation = release.tests.containers-config-activation;
+    imperative = release.tests.containers-next-imperative;
+  };
+}

nixos/lib/test-driver/test_driver/machine.py

@@ -478,6 +478,15 @@ def systemctl(self, q: str, user: Optional[str] = None) -> Tuple[int, str]:
             )
         return self.execute("systemctl {}".format(q))
 
+    def wait_until_unit_stops(self, unit: str) -> None:
+        def wait_inactive(_: Any) -> bool:
+            info = self.get_unit_info(unit)
+            state = info["ActiveState"]
+            return state == "inactive"
+
+        with self.nested(f"waiting for unit '{unit}' to stop"):
+            retry(wait_inactive)
+
     def require_unit_state(self, unit: str, require_state: str = "active") -> None:
         with self.nested(
             "checking if unit ‘{}’ has reached state '{}'".format(unit, require_state)

nixos/modules/module-list.nix

@@ -1180,6 +1180,7 @@
   ./virtualisation/container-config.nix
   ./virtualisation/containerd.nix
   ./virtualisation/containers.nix
+  ./virtualisation/containers-next
   ./virtualisation/nixos-containers.nix
   ./virtualisation/oci-containers.nix
   ./virtualisation/cri-o.nix

nixos/modules/system/activation/switch-to-configuration.pl

@@ -8,6 +8,7 @@
 use Net::DBus;
 use Sys::Syslog qw(:standard :macros);
 use Cwd 'abs_path';
+use experimental 'smartmatch';
 
 my $out = "@out@";
 
@@ -128,6 +129,24 @@ sub parseKeyValues {
     }
 }
 
+sub parseNspawn {
+    my ($filename) = @_;
+    my $info = {};
+    parseKeyValuesArray($info, read_file($filename));
+    return $info;
+}
+
+sub parseKeyValuesArray {
+    my $info = shift;
+    foreach my $line (@_) {
+        $line =~ /^([^=]+)=(.*)$/ or next;
+        unless (exists $info->{$1}) {
+            $info->{$1} = ();
+        }
+        push @{$info->{$1}}, $2;
+    }
+}
+
 sub boolIsTrue {
     my ($s) = @_;
     return $s eq "yes" || $s eq "true";
@@ -209,11 +228,15 @@ sub handleModifiedUnit {
                 # We write this to a file to ensure that the
                 # service gets restarted if we're interrupted.
                 if (!$socketActivated) {
-                    $unitsToStart->{$unit} = 1;
-                    recordUnit($startListFile, $unit);
+                    if (index($unit, "systemd-nspawn@") == -1) {
+                        $unitsToStart->{$unit} = 1;
+                        recordUnit($startListFile, $unit);
+                    }
                 }
 
-                $unitsToStop->{$unit} = 1;
+                if (index($unit, "systemd-nspawn@") == -1) {
+                    $unitsToStop->{$unit} = 1;
+                }
             }
         }
     }
@@ -233,6 +256,86 @@ sub handleModifiedUnit {
 $unitsToReload{$_} = 1 foreach
     split('\n', read_file($reloadListFile, err_mode => 'quiet') // "");
 
+
+sub deepCmp {
+    my ($a, $b) = @_;
+    if (@$a != @$b) {
+        return 1;
+    }
+    foreach (my $i = 0; $i < @$a; $i++) {
+        if (@$a[$i] ne @$b[$i]) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
+sub compareNspawnUnits {
+    my ($old, $new) = @_;
+    my $contentsOld = parseNspawn($old);
+    my $contentsNew = parseNspawn($new);
+
+    foreach (keys %$contentsOld) {
+        my $oldKey = $_;
+        foreach (keys %$contentsNew) {
+            my $newKey = $_;
+            next if $newKey ne $oldKey
+                or $newKey eq 'Parameters'
+                or $newKey eq 'X-ActivationStrategy';
+
+            if (deepCmp($contentsOld->{$oldKey}, $contentsNew->{$newKey}) != 0) {
+                return (1, $contentsNew->{'X-ActivationStrategy'}[0] // "dynamic");
+            }
+        }
+    }
+
+    return (0, $contentsNew->{'X-ActivationStrategy'}[0] // "dynamic");
+}
+
+my $activeContainersOut = `machinectl list`;
+sub isContainerRunning {
+    my ($name) = @_;
+    if (index($activeContainersOut, $name) != -1) {
+        return 1;
+    }
+    return 0;
+}
+
+my @currentNspawnUnits = glob("/etc/systemd/nspawn/*.nspawn");
+my @upcomingNspawnUnits = glob("$out/etc/systemd/nspawn/*.nspawn");
+foreach (@upcomingNspawnUnits) {
+    my $unit = basename($_);
+    $unit =~ s/\.nspawn//;
+    my $unitName = "systemd-nspawn\@$unit.service";
+    my $orig = $_;
+    $orig =~ s/^$out//;
+    if ($orig ~~ @currentNspawnUnits) {
+        if (fingerprintUnit($_) ne fingerprintUnit($orig)) {
+            my ($eq, $strategy) = compareNspawnUnits($orig, $_);
+            if ($strategy ne "none") {
+                if ($strategy ne "restart" and ($eq == 0 or $strategy eq "reload")) {
+                    if (isContainerRunning($unit) == 1) {
+                        $unitsToReload{$unitName} = 1;
+                    }
+                } elsif ($eq == 1) {
+                    $unitsToRestart{$unitName} = 1;
+                }
+            }
+        }
+    } else {
+        $unitsToStart{$unitName} = 1;
+    }
+}
+
+foreach (@currentNspawnUnits) {
+    unless (-f "$out$_") {
+        my $unit = basename($_);
+        $unit =~ s/\.nspawn//;
+        my $unitName = "systemd-nspawn\@$unit.service";
+        $unitsToStop{$unitName} = 1;
+    }
+}
+
 my $activePrev = getActiveUnits;
 while (my ($unit, $state) = each %{$activePrev}) {
     my $baseUnit = $unit;
@@ -477,8 +580,25 @@ sub filterUnits {
 # Reload units that need it. This includes remounting changed mount
 # units.
 if (scalar(keys %unitsToReload) > 0) {
-    print STDERR "reloading the following units: ", join(", ", sort(keys %unitsToReload)), "\n";
-    system("@systemd@/bin/systemctl", "reload", "--", sort(keys %unitsToReload)) == 0 or $res = 4;
+    my @to_reload = sort(keys %unitsToReload);
+    my (@services, @containers);
+    print STDERR "reloading the following units: ", join(", ", @to_reload), "\n";
+    foreach my $s (@to_reload) {
+        if (index($s, "systemd-nspawn@") == 0) {
+            push @containers, $s;
+        } else {
+            push @services, $s;
+        }
+    }
+
+    # Reloading containers & dbus.service in the same transaction causes
+    # the system to stall for about 1 minute.
+    if (scalar(@services) > 0) {
+        system("@systemd@/bin/systemctl", "reload", "--", @services) == 0 or $res = 4;
+    }
+    if (scalar(@containers) > 0) {
+        system("@systemd@/bin/systemctl", "reload", "--", @containers) == 0 or $res = 4;
+    }
     unlink($reloadListFile);
     unlink($reloadByActivationFile);
 }

nixos/modules/system/boot/systemd-nspawn.nix

@@ -6,7 +6,6 @@ with lib;
 
 let
   cfg = config.systemd.nspawn;
-
   checkExec = checkUnitConfig "Exec" [
     (assertOnlyFields [
       "Boot" "ProcessTwo" "Parameters" "Environment" "User" "WorkingDirectory"
@@ -16,7 +15,7 @@ let
       "LimitNOFILE" "LimitAS" "LimitNPROC" "LimitMEMLOCK" "LimitLOCKS"
       "LimitSIGPENDING" "LimitMSGQUEUE" "LimitNICE" "LimitRTPRIO" "LimitRTTIME"
       "OOMScoreAdjust" "CPUAffinity" "Hostname" "ResolvConf" "Timezone"
-      "LinkJournal"
+      "LinkJournal" "Ephemeral" "X-ActivationStrategy"
     ])
     (assertValueOneOf "Boot" boolValues)
     (assertValueOneOf "ProcessTwo" boolValues)
@@ -79,24 +78,57 @@ let
           <manvolnum>5</manvolnum></citerefentry> for details.
         '';
       };
+
+      extraDrvConfig = mkOption {
+        type = types.nullOr types.package;
+        default = null;
+        description = ''
+          Extra config for an nspawn-unit that is generated via `nix-build`.
+          This is necessary since nspawn doesn't support overrides in
+          <literal>/etc/systemd/nspawn</literal> natively and sometimes a derivation
+          is needed for configs (e.g. to determine all needed store-paths to bind-mount
+          into a machine).
+        '';
+      };
     };
 
   };
 
+  makeUnit' = name: def:
+    if def.extraDrvConfig == null || !def.enable
+      then pkgs.runCommand "nspawn-inst" { } ("cat ${makeUnit name def}/${shellEscape name} > $out")
+    else pkgs.runCommand "nspawn-${mkPathSafeName name}-custom"
+      { preferLocalBuild = true;
+        allowSubstitutes = false;
+      } (let
+        name' = shellEscape name;
+      in ''
+        if [ ! -f "${def.extraDrvConfig}" ]; then
+          echo "systemd.nspawn.${name}.extraDrvConfig is not a file!"
+          exit 1
+        fi
+
+        touch $out
+        cat ${makeUnit name def}/${name'} > $out
+        cat ${def.extraDrvConfig} >> $out
+      '');
+
   instanceToUnit = name: def:
-    let base = {
-      text = ''
-        [Exec]
-        ${attrsToSection def.execConfig}
+    let
+      base = {
+        text = ''
+          [Exec]
+          ${attrsToSection def.execConfig}
 
-        [Files]
-        ${attrsToSection def.filesConfig}
+          [Files]
+          ${attrsToSection def.filesConfig}
 
-        [Network]
-        ${attrsToSection def.networkConfig}
-      '';
-    } // def;
-    in base // { unit = makeUnit name base; };
+          [Network]
+          ${attrsToSection def.networkConfig}
+        '';
+      } // (filterAttrs (n: const (elem n optWhitelist)) def);
+      optWhitelist = [ "extraDrvConfig" "enable" ];
+    in makeUnit' name base;
 
 in {
 
@@ -112,22 +144,17 @@ in {
 
   config =
     let
-      units = mapAttrs' (n: v: let nspawnFile = "${n}.nspawn"; in nameValuePair nspawnFile (instanceToUnit nspawnFile v)) cfg;
+      units = mapAttrs' (name: value: {
+        name = "systemd/nspawn/${name}.nspawn";
+        value.source = instanceToUnit "${name}.nspawn" value;
+      }) cfg;
     in
       mkMerge [
         (mkIf (cfg != {}) {
-          environment.etc."systemd/nspawn".source = mkIf (cfg != {}) (generateUnits' false "nspawn" units [] []);
+          environment.etc = units;
         })
         {
           systemd.targets.multi-user.wants = [ "machines.target" ];
-
-          # Workaround for https://github.com/NixOS/nixpkgs/pull/67232#issuecomment-531315437 and https://github.com/systemd/systemd/issues/13622
-          # Once systemd fixes this upstream, we can re-enable -U
-          systemd.services."systemd-nspawn@".serviceConfig.ExecStart = [
-            ""  # deliberately empty. signals systemd to override the ExecStart
-            # Only difference between upstream is that we do not pass the -U flag
-            "${config.systemd.package}/bin/systemd-nspawn --quiet --keep-unit --boot --link-journal=try-guest --network-veth --settings=override --machine=%i"
-          ];
         }
       ];
 }

nixos/modules/tasks/filesystems.nix

@@ -365,19 +365,19 @@ in
 
     # Sync mount options with systemd's src/core/mount-setup.c: mount_table.
     boot.specialFileSystems = {
-      "/proc" = { fsType = "proc"; options = [ "nosuid" "noexec" "nodev" ]; };
-      "/run" = { fsType = "tmpfs"; options = [ "nosuid" "nodev" "strictatime" "mode=755" "size=${config.boot.runSize}" ]; };
-      "/dev" = { fsType = "devtmpfs"; options = [ "nosuid" "strictatime" "mode=755" "size=${config.boot.devSize}" ]; };
-      "/dev/shm" = { fsType = "tmpfs"; options = [ "nosuid" "nodev" "strictatime" "mode=1777" "size=${config.boot.devShmSize}" ]; };
-      "/dev/pts" = { fsType = "devpts"; options = [ "nosuid" "noexec" "mode=620" "ptmxmode=0666" "gid=${toString config.ids.gids.tty}" ]; };
-
       # To hold secrets that shouldn't be written to disk
       "/run/keys" = { fsType = "ramfs"; options = [ "nosuid" "nodev" "mode=750" ]; };
     } // optionalAttrs (!config.boot.isContainer) {
       # systemd-nspawn populates /sys by itself, and remounting it causes all
       # kinds of weird issues (most noticeably, waiting for host disk device
       # nodes).
       "/sys" = { fsType = "sysfs"; options = [ "nosuid" "noexec" "nodev" ]; };
+
+      "/proc" = { fsType = "proc"; options = [ "nosuid" "noexec" "nodev" ]; };
+      "/run" = { fsType = "tmpfs"; options = [ "nosuid" "nodev" "strictatime" "mode=755" "size=${config.boot.runSize}" ]; };
+      "/dev" = { fsType = "devtmpfs"; options = [ "nosuid" "strictatime" "mode=755" "size=${config.boot.devSize}" ]; };
+      "/dev/shm" = { fsType = "tmpfs"; options = [ "nosuid" "nodev" "strictatime" "mode=1777" "size=${config.boot.devShmSize}" ]; };
+      "/dev/pts" = { fsType = "devpts"; options = [ "nosuid" "noexec" "mode=620" "ptmxmode=0666" "gid=${toString config.ids.gids.tty}" ]; };
     };
 
   };

nixos/modules/tasks/network-interfaces.nix

@@ -540,7 +540,7 @@ in
       type = types.bool;
       default = false;
       description = ''
-        In containers, whether to use the
+        <emphasis>Deprecated:</emphasis> in containers, whether to use the
         <filename>resolv.conf</filename> supplied by the host.
       '';
     };

nixos/modules/virtualisation/containers-next/container-profile.nix

@@ -0,0 +1,29 @@
+{ pkgs, lib, ... }: with lib; {
+  boot.isContainer = true;
+
+  # We need a static libsudoers if we bind-mount into a user-namespaced
+  # container since the bind-mounts are owned by `nouser:nogroup` then (including
+  # `/nix/store`) and this doesn't like sudo.
+  security.sudo.package = mkDefault pkgs.sudo-nspawn;
+
+  # Containers are supposed to use systemd-networkd to have a proper
+  # networking stack even during boot-up.
+  networking = {
+    useHostResolvConf = false;
+    useDHCP = false;
+    useNetworkd = true;
+  };
+  systemd.network.networks."20-host0" = {
+    matchConfig = {
+      Virtualization = "container";
+      Name = "host0";
+    };
+    dhcpConfig.UseTimezone = "yes";
+    networkConfig = {
+      DHCP = "yes";
+      LLDP = "yes";
+      EmitLLDP = "customer-bridge";
+      LinkLocalAddressing = mkDefault "ipv6";
+    };
+  };
+}