Backport changes of networkd-containers to test on my infrastructure

Applied the diff from NixOS#140669 at revision cd533c3.
Ma27 · Dec 26, 2023 · c95f805 · c95f805
1 parent ebae1ef
commit c95f805
Show file tree

Hide file tree

Showing 19 changed files with 2,572 additions and 44 deletions.
diff --git a/jobset.nix b/jobset.nix
@@ -0,0 +1,18 @@
+# FIXME remove before merging!
+
+{ nixpkgs }:
+let
+  release = import ./nixos/release.nix {
+    supportedSystems = [ "x86_64-linux" ];
+    inherit nixpkgs;
+  };
+in
+
+{
+  container-tests = {
+    general = release.tests.containers-next;
+    migration = release.tests.containers-migration;
+    activation = release.tests.containers-config-activation;
+    imperative = release.tests.containers-next-imperative;
+  };
+}
diff --git a/nixos/lib/test-driver/test_driver/machine.py b/nixos/lib/test-driver/test_driver/machine.py
@@ -485,6 +485,15 @@ def systemctl(self, q: str, user: Optional[str] = None) -> Tuple[int, str]:
             )
         return self.execute(f"systemctl {q}")
 
+    def wait_until_unit_stops(self, unit: str) -> None:
+        def wait_inactive(_: Any) -> bool:
+            info = self.get_unit_info(unit)
+            state = info["ActiveState"]
+            return state == "inactive"
+
+        with self.nested(f"waiting for unit '{unit}' to stop"):
+            retry(wait_inactive)
+
     def require_unit_state(self, unit: str, require_state: str = "active") -> None:
         with self.nested(
             f"checking if unit '{unit}' has reached state '{require_state}'"

diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
@@ -1421,6 +1421,9 @@
   ./virtualisation/container-config.nix
   ./virtualisation/containerd.nix
   ./virtualisation/containers.nix
+  ./virtualisation/containers-next
+  ./virtualisation/nixos-containers.nix
+  ./virtualisation/oci-containers.nix
   ./virtualisation/cri-o.nix
   ./virtualisation/docker-rootless.nix
   ./virtualisation/docker.nix

diff --git a/nixos/modules/system/activation/switch-to-configuration.pl b/nixos/modules/system/activation/switch-to-configuration.pl
@@ -21,6 +21,7 @@
 use JSON::PP;
 use IPC::Cmd;
 use Sys::Syslog qw(:standard :macros);
+use experimental 'smartmatch';
 use Cwd qw(abs_path);
 
 ## no critic(ControlStructures::ProhibitDeepNests)
@@ -243,6 +244,29 @@ sub parse_systemd_ini {
     return;
 }
 
+sub parseNspawn {
+    my ($filename) = @_;
+    my $info = {};
+    parseKeyValuesArray($info, read_file($filename));
+    return $info;
+}
+
+sub parseKeyValuesArray {
+    my $info = shift;
+    foreach my $line (@_) {
+        $line =~ /^([^=]+)=(.*)$/ or next;
+        unless (exists $info->{$1}) {
+            $info->{$1} = ();
+        }
+        push @{$info->{$1}}, $2;
+    }
+}
+
+sub boolIsTrue {
+    my ($s) = @_;
+    return $s eq "yes" || $s eq "true";
+}
+
 # This subroutine takes the path to a systemd configuration file (like a unit configuration),
 # parses it, and returns a hash that contains the contents. The contents of this hash are
 # explained in the `parse_systemd_ini` subroutine. Neither the sections nor the keys inside
@@ -294,7 +318,9 @@ sub record_unit {
 sub unrecord_unit {
     my ($fn, $unit) = @_;
     if ($action ne "dry-activate") {
-        edit_file(sub { s/^$unit\n//msx }, $fn);
+        if (index($unit, "systemd-nspawn@") == -1) {
+            edit_file(sub { s/^$unit\n//msx }, $fn);
+        }
     }
     return;
 }
@@ -500,19 +526,23 @@ sub handle_modified_unit { ## no critic(Subroutines::ProhibitManyArgs, Subroutin
                 # We write this to a file to ensure that the
                 # service gets restarted if we're interrupted.
                 if (!$socket_activated) {
-                    $units_to_start->{$unit} = 1;
-                    if ($units_to_start eq $units_to_restart) {
-                        record_unit($restart_list_file, $unit);
-                    } else {
-                        record_unit($start_list_file, $unit);
+                    if (index($unit, "systemd-nspawn@") == -1) {
+                        $units_to_start->{$unit} = 1;
+                        if ($units_to_start eq $units_to_restart) {
+                            record_unit($restart_list_file, $unit);
+                        } else {
+                            record_unit($start_list_file, $unit);
+                        }
                     }
                 }
 
-                $units_to_stop->{$unit} = 1;
-                # Remove from units to reload so we don't restart and reload
-                if ($units_to_reload->{$unit}) {
-                    delete $units_to_reload->{$unit};
-                    unrecord_unit($reload_list_file, $unit);
+                if (index($unit, "systemd-nspawn@") == -1) {
+                    $units_to_stop->{$unit} = 1;
+                    # Remove from units to reload so we don't restart and reload
+                    if ($units_to_reload->{$unit}) {
+                        delete $units_to_reload->{$unit};
+                        unrecord_unit($reload_list_file, $unit);
+                    }
                 }
             }
         }
@@ -534,6 +564,89 @@ sub handle_modified_unit { ## no critic(Subroutines::ProhibitManyArgs, Subroutin
 %units_to_reload = map { $_ => 1 }
     split(/\n/msx, read_file($reload_list_file, err_mode => "quiet") // "");
 
+sub deepCmp {
+    my ($a, $b) = @_;
+    if (@$a != @$b) {
+        return 1;
+    }
+    foreach (my $i = 0; $i < @$a; $i++) {
+        if (@$a[$i] ne @$b[$i]) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
+sub compareNspawnUnits {
+    my ($old, $new) = @_;
+    my $contentsOld = parseNspawn($old);
+    my $contentsNew = parseNspawn($new);
+
+    foreach (keys %$contentsOld) {
+        my $oldKey = $_;
+        foreach (keys %$contentsNew) {
+            my $newKey = $_;
+            next if $newKey ne $oldKey
+                or $newKey eq 'Parameters'
+                or $newKey eq 'X-ActivationStrategy';
+
+            if (deepCmp($contentsOld->{$oldKey}, $contentsNew->{$newKey}) != 0) {
+                return (1, $contentsNew->{'X-ActivationStrategy'}[0] // "dynamic");
+            }
+        }
+    }
+
+    return (0, $contentsNew->{'X-ActivationStrategy'}[0] // "dynamic");
+}
+
+my $activeContainersOut = `machinectl list`;
+sub isContainerRunning {
+    my ($name) = @_;
+    if (index($activeContainersOut, $name) != -1) {
+        return 1;
+    }
+    return 0;
+}
+
+my @currentNspawnUnits = glob("/etc/systemd/nspawn/*.nspawn");
+my @upcomingNspawnUnits = glob("$out/etc/systemd/nspawn/*.nspawn");
+sub fingerprintUnit {
+    my ($s) = @_;
+    return abs_path($s) . (-f "${s}.d/overrides.conf" ? " " . abs_path "${s}.d/overrides.conf" : "");
+}
+foreach (@upcomingNspawnUnits) {
+    my $unit = basename($_);
+    $unit =~ s/\.nspawn//;
+    my $unitName = "systemd-nspawn\@$unit.service";
+    my $orig = $_;
+    $orig =~ s/^$out//;
+    if ($orig ~~ @currentNspawnUnits) {
+        if (fingerprintUnit($_) ne fingerprintUnit($orig)) {
+            my ($eq, $strategy) = compareNspawnUnits($orig, $_);
+            if ($strategy ne "none") {
+                if ($strategy ne "restart" and ($eq == 0 or $strategy eq "reload")) {
+                    if (isContainerRunning($unit) == 1) {
+                        $units_to_reload{$unitName} = 1;
+                    }
+                } elsif ($eq == 1) {
+                    $units_to_restart{$unitName} = 1;
+                }
+            }
+        }
+    } else {
+        $units_to_start{$unitName} = 1;
+    }
+}
+
+foreach (@currentNspawnUnits) {
+    unless (-f "$out$_") {
+        my $unit = basename($_);
+        $unit =~ s/\.nspawn//;
+        my $unitName = "systemd-nspawn\@$unit.service";
+        $units_to_stop{$unitName} = 1;
+    }
+}
+
 my $active_cur = get_active_units();
 while (my ($unit, $state) = each(%{$active_cur})) {
     my $base_unit = $unit;
@@ -859,8 +972,10 @@ sub filter_units {
             # Figure out if we need to start the unit
             my %unit_info = parse_unit("$out/etc/systemd/system/$unit");
             if (!(parse_systemd_bool(\%unit_info, "Unit", "RefuseManualStart", 0) || parse_systemd_bool(\%unit_info, "Unit", "X-OnlyManualStart", 0))) {
-                $units_to_start{$unit} = 1;
-                record_unit($start_list_file, $unit);
+                if (index($unit, "systemd-nspawn@") == -1) {
+                    $units_to_start{$unit} = 1;
+                    record_unit($start_list_file, $unit);
+                }
             }
             # Don't reload the unit, reloading would fail
             delete %units_to_reload{$unit};
@@ -872,7 +987,24 @@ sub filter_units {
 # units.
 if (scalar(keys(%units_to_reload)) > 0) {
     print STDERR "reloading the following units: ", join(", ", sort(keys(%units_to_reload))), "\n";
-    system("$new_systemd/bin/systemctl", "reload", "--", sort(keys(%units_to_reload))) == 0 or $res = 4;
+    my @to_reload = sort(keys %units_to_reload);
+    my (@services, @containers);
+    foreach my $s (@to_reload) {
+        if (index($s, "systemd-nspawn@") == 0) {
+            push @containers, $s;
+        } else {
+            push @services, $s;
+        }
+    }
+
+    # Reloading containers & dbus.service in the same transaction causes
+    # the system to stall for about 1 minute.
+    if (scalar(@services) > 0) {
+        system("@systemd@/bin/systemctl", "reload", "--", @services) == 0 or $res = 4;
+    }
+    if (scalar(@containers) > 0) {
+        system("@systemd@/bin/systemctl", "reload", "--", @containers) == 0 or $res = 4;
+    }
     unlink($reload_list_file);
 }
 

diff --git a/nixos/modules/system/boot/systemd/nspawn.nix b/nixos/modules/system/boot/systemd/nspawn.nix
@@ -6,7 +6,6 @@ with lib;
 
 let
   cfg = config.systemd.nspawn;
-
   checkExec = checkUnitConfig "Exec" [
     (assertOnlyFields [
       "Boot" "ProcessTwo" "Parameters" "Environment" "User" "WorkingDirectory"
@@ -16,7 +15,7 @@ let
       "LimitNOFILE" "LimitAS" "LimitNPROC" "LimitMEMLOCK" "LimitLOCKS"
       "LimitSIGPENDING" "LimitMSGQUEUE" "LimitNICE" "LimitRTPRIO" "LimitRTTIME"
       "OOMScoreAdjust" "CPUAffinity" "Hostname" "ResolvConf" "Timezone"
-      "LinkJournal" "Ephemeral" "AmbientCapability"
+      "LinkJournal" "Ephemeral" "AmbientCapability" "X-ActivationStrategy"
     ])
     (assertValueOneOf "Boot" boolValues)
     (assertValueOneOf "ProcessTwo" boolValues)
@@ -80,24 +79,57 @@ let
           {manpage}`systemd.nspawn(5)` for details.
         '';
       };
+
+      extraDrvConfig = mkOption {
+        type = types.nullOr types.package;
+        default = null;
+        description = ''
+          Extra config for an nspawn-unit that is generated via `nix-build`.
+          This is necessary since nspawn doesn't support overrides in
+          <literal>/etc/systemd/nspawn</literal> natively and sometimes a derivation
+          is needed for configs (e.g. to determine all needed store-paths to bind-mount
+          into a machine).
+        '';
+      };
     };
 
   };
 
+  makeUnit' = name: def:
+    if def.extraDrvConfig == null || !def.enable
+      then pkgs.runCommand "nspawn-inst" { } ("cat ${makeUnit name def}/${shellEscape name} > $out")
+    else pkgs.runCommand "nspawn-${mkPathSafeName name}-custom"
+      { preferLocalBuild = true;
+        allowSubstitutes = false;
+      } (let
+        name' = shellEscape name;
+      in ''
+        if [ ! -f "${def.extraDrvConfig}" ]; then
+          echo "systemd.nspawn.${name}.extraDrvConfig is not a file!"
+          exit 1
+        fi
+
+        touch $out
+        cat ${makeUnit name def}/${name'} > $out
+        cat ${def.extraDrvConfig} >> $out
+      '');
+
   instanceToUnit = name: def:
-    let base = {
-      text = ''
-        [Exec]
-        ${attrsToSection def.execConfig}
+    let
+      base = {
+        text = ''
+          [Exec]
+          ${attrsToSection def.execConfig}
 
-        [Files]
-        ${attrsToSection def.filesConfig}
+          [Files]
+          ${attrsToSection def.filesConfig}
 
-        [Network]
-        ${attrsToSection def.networkConfig}
-      '';
-    } // def;
-    in base // { unit = makeUnit name base; };
+          [Network]
+          ${attrsToSection def.networkConfig}
+        '';
+      } // (filterAttrs (n: const (elem n optWhitelist)) def);
+      optWhitelist = [ "extraDrvConfig" "enable" ];
+    in makeUnit' name base;
 
 in {
 
@@ -113,17 +145,14 @@ in {
 
   config =
     let
-      units = mapAttrs' (n: v: let nspawnFile = "${n}.nspawn"; in nameValuePair nspawnFile (instanceToUnit nspawnFile v)) cfg;
+      units = mapAttrs' (name: value: {
+        name = "systemd/nspawn/${name}.nspawn";
+        value.source = instanceToUnit "${name}.nspawn" value;
+      }) cfg;
     in
       mkMerge [
         (mkIf (cfg != {}) {
-          environment.etc."systemd/nspawn".source = mkIf (cfg != {}) (generateUnits {
-            allowCollisions = false;
-            type = "nspawn";
-            inherit units;
-            upstreamUnits = [];
-            upstreamWants = [];
-          });
+          environment.etc = units;
         })
         {
           systemd.targets.multi-user.wants = [ "machines.target" ];

diff --git a/nixos/modules/tasks/filesystems.nix b/nixos/modules/tasks/filesystems.nix
@@ -440,19 +440,19 @@ in
 
     # Sync mount options with systemd's src/core/mount-setup.c: mount_table.
     boot.specialFileSystems = {
-      "/proc" = { fsType = "proc"; options = [ "nosuid" "noexec" "nodev" ]; };
-      "/run" = { fsType = "tmpfs"; options = [ "nosuid" "nodev" "strictatime" "mode=755" "size=${config.boot.runSize}" ]; };
-      "/dev" = { fsType = "devtmpfs"; options = [ "nosuid" "strictatime" "mode=755" "size=${config.boot.devSize}" ]; };
-      "/dev/shm" = { fsType = "tmpfs"; options = [ "nosuid" "nodev" "strictatime" "mode=1777" "size=${config.boot.devShmSize}" ]; };
-      "/dev/pts" = { fsType = "devpts"; options = [ "nosuid" "noexec" "mode=620" "ptmxmode=0666" "gid=${toString config.ids.gids.tty}" ]; };
-
       # To hold secrets that shouldn't be written to disk
       "/run/keys" = { fsType = "ramfs"; options = [ "nosuid" "nodev" "mode=750" ]; };
     } // optionalAttrs (!config.boot.isContainer) {
       # systemd-nspawn populates /sys by itself, and remounting it causes all
       # kinds of weird issues (most noticeably, waiting for host disk device
       # nodes).
       "/sys" = { fsType = "sysfs"; options = [ "nosuid" "noexec" "nodev" ]; };
+
+      "/proc" = { fsType = "proc"; options = [ "nosuid" "noexec" "nodev" ]; };
+      "/run" = { fsType = "tmpfs"; options = [ "nosuid" "nodev" "strictatime" "mode=755" "size=${config.boot.runSize}" ]; };
+      "/dev" = { fsType = "devtmpfs"; options = [ "nosuid" "strictatime" "mode=755" "size=${config.boot.devSize}" ]; };
+      "/dev/shm" = { fsType = "tmpfs"; options = [ "nosuid" "nodev" "strictatime" "mode=1777" "size=${config.boot.devShmSize}" ]; };
+      "/dev/pts" = { fsType = "devpts"; options = [ "nosuid" "noexec" "mode=620" "ptmxmode=0666" "gid=${toString config.ids.gids.tty}" ]; };
     };
 
   };