diff --git a/ci/eval/compare.jq b/ci/eval/compare.jq
deleted file mode 100644
index 9f3a032d138419..00000000000000
--- a/ci/eval/compare.jq
+++ /dev/null
@@ -1,164 +0,0 @@
-# Turns
-#
-#   {
-#     "hello.aarch64-linux": "a",
-#     "hello.x86_64-linux": "b",
-#     "hello.aarch64-darwin": "c",
-#     "hello.x86_64-darwin": "d"
-#   }
-#
-# into
-#
-#   {
-#     "hello": {
-#       "linux": {
-#         "aarch64": "a",
-#         "x86_64": "b"
-#       },
-#       "darwin": {
-#         "aarch64": "c",
-#         "x86_64": "d"
-#       }
-#     }
-#   }
-#
-# while filtering out any attribute paths that don't match this pattern
-def expand_system:
-  to_entries
-  | map(
-    .key |= split(".")
-    | select(.key | length > 1)
-    | .double = (.key[-1] | split("-"))
-    | select(.double | length == 2)
-  )
-  | group_by(.key[0:-1])
-  | map(
-    {
-      key: .[0].key[0:-1] | join("."),
-      value:
-        group_by(.double[1])
-        | map(
-          {
-            key: .[0].double[1],
-            value: map(.key = .double[0]) | from_entries
-          }
-        )
-        | from_entries
-    })
-  | from_entries
-  ;
-
-# Transposes
-#
-#   {
-#     "a": [ "x", "y" ],
-#     "b": [ "x" ],
-#   }
-#
-# into
-#
-#   {
-#     "x": [ "a", "b" ],
-#     "y": [ "a" ]
-#   }
-def transpose:
-  [
-    to_entries[]
-    | {
-      key: .key,
-      value: .value[]
-    }
-  ]
-  | group_by(.value)
-  | map({
-    key: .[0].value,
-    value: map(.key)
-  })
-  | from_entries
-  ;
-
-# Computes the key difference for two objects:
-# {
-#   added: [ <keys only in the second object> ],
-#   removed: [ <keys only in the first object> ],
-#   changed: [ <keys with different values between the two objects> ],
-# }
-#
-def diff($before; $after):
-  {
-    added: $after | delpaths($before | keys | map([.])) | keys,
-    removed: $before | delpaths($after | keys | map([.])) | keys,
-    changed:
-      $before
-      | to_entries
-      | map(
-        $after."\(.key)" as $after2
-        | select(
-          # Filter out attributes that don't exist anymore
-          ($after2 != null)
-          and
-          # Filter out attributes that are the same as the new value
-          (.value != $after2)
-        )
-        | .key
-      )
-  }
-  ;
-
-($before[0] | expand_system) as $before
-| ($after[0] | expand_system) as $after
-| .attrdiff = diff($before; $after)
-| .rebuildsByKernel = (
-  [
-    (
-      .attrdiff.changed[]
-      | {
-        key: .,
-        value: diff($before."\(.)"; $after."\(.)").changed
-      }
-    )
-    ,
-    (
-      .attrdiff.added[]
-      | {
-        key: .,
-        value: ($after."\(.)" | keys)
-      }
-    )
-  ]
-  | from_entries
-  | transpose
-)
-| .rebuildCountByKernel = (
-  .rebuildsByKernel
-  | with_entries(.value |= length)
-  | pick(.linux, .darwin)
-  | {
-    linux: (.linux // 0),
-    darwin: (.darwin // 0),
-  }
-)
-| .labels = (
-  .rebuildCountByKernel
-  | to_entries
-  | map(
-    "10.rebuild-\(.key): " +
-      if .value == 0 then
-        "0"
-      elif .value <= 10 then
-        "1-10"
-      elif .value <= 100 then
-        "11-100"
-      elif .value <= 500 then
-        "101-500"
-      elif .value <= 1000 then
-        "501-1000"
-      elif .value <= 2500 then
-        "1001-2500"
-      elif .value <= 5000 then
-        "2501-5000"
-      else
-        "5001+"
-      end
-  )
-)
diff --git a/ci/eval/compare/default.nix b/ci/eval/compare/default.nix
new file mode 100644
index 00000000000000..8b5e9059cd5b8d
--- /dev/null
+++ b/ci/eval/compare/default.nix
@@ -0,0 +1,53 @@
+{
+  lib,
+  jq,
+  runCommand,
+  writeText,
+  supportedSystems,
+  ...
+}:
+{ beforeResultDir, afterResultDir }:
+let
+  inherit (import ./utils.nix { inherit lib; })
+    diff
+    groupByKernel
+    extractPackageNames
+    getLabels
+    uniqueStrings
+    ;
+
+  getAttrs = dir: builtins.fromJSON (builtins.readFile "${dir}/outpaths.json");
+  beforeAttrs = getAttrs beforeResultDir;
+  afterAttrs = getAttrs afterResultDir;
+
+  diffAttrs = diff beforeAttrs afterAttrs;
+
+  changed-paths =
+    let
+      rebuilds = uniqueStrings (diffAttrs.added ++ diffAttrs.changed);
+
+      rebuildsByKernel = groupByKernel rebuilds;
+      rebuildCountByKernel = lib.mapAttrs (
+        kernel: kernelRebuilds: lib.length kernelRebuilds
+      ) rebuildsByKernel;
+    in
+    writeText "changed-paths.json" (
+      builtins.toJSON {
+        attrdiff = lib.mapAttrs (_: v: extractPackageNames v) diffAttrs;
+        inherit rebuildsByKernel rebuildCountByKernel;
+        labels = getLabels rebuildCountByKernel;
+      }
+    );
+in
+runCommand "compare"
+  {
+    nativeBuildInputs = [ jq ];
+  }
+  ''
+    mkdir $out
+
+    cp ${changed-paths} $out/changed-paths.json
+
+    jq -r -f ${./generate-step-summary.jq} < ${changed-paths} > $out/step-summary.md
+    # TODO: Compare eval stats
+  ''
diff --git a/ci/eval/generate-step-summary.jq b/ci/eval/compare/generate-step-summary.jq
similarity index 100%
rename from ci/eval/generate-step-summary.jq
rename to ci/eval/compare/generate-step-summary.jq
diff --git a/ci/eval/compare/utils.nix b/ci/eval/compare/utils.nix
new file mode 100644
index 00000000000000..22353499d69933
--- /dev/null
+++ b/ci/eval/compare/utils.nix
@@ -0,0 +1,130 @@
+{ lib, ... }:
+rec {
+  # Borrowed from https://github.com/NixOS/nixpkgs/pull/355616
+  uniqueStrings = list: builtins.attrNames (builtins.groupBy lib.id list);
+
+  _processSystemPath =
+    packageSystemPath:
+    let
+      # python312Packages.torch.aarch64-linux -> ["python312Packages" "torch" "aarch64-linux"]
+      # splittedPath = lib.splitString "." attrName;
+      splittedPath = lib.splitString "." packageSystemPath;
+
+      # ["python312Packages" "torch" "aarch64-linux"] -> ["python312Packages" "torch"]
+      packagePath = lib.sublist 0 (lib.length splittedPath - 1) splittedPath;
+    in
+    {
+      # "python312Packages.torch"
+      name = lib.concatStringsSep "." packagePath;
+
+      # "aarch64-linux"
+      system = lib.last splittedPath;
+    };
+
+  # Turns
+  # [
+  #   "hello.aarch64-linux"
+  #   "hello.x86_64-linux"
+  #   "hello.aarch64-darwin"
+  #   "hello.x86_64-darwin"
+  #   "bye.x86_64-darwin"
+  #   "bye.aarch64-darwin"
+  # ]
+  #
+  # into
+  #
+  # [
+  #   "hello"
+  #   "bye"
+  # ]
+  extractPackageNames =
+    packageSystemPaths:
+    builtins.attrNames (
+      builtins.removeAttrs (builtins.groupBy (
+        packageSystemPath: (_processSystemPath packageSystemPath).name
+      ) packageSystemPaths) [ "" ]
+    );
+
+  # Computes a diff between two attrs
+  # {
+  #   added: [ <keys only in the second object> ],
+  #   removed: [ <keys only in the first object> ],
+  #   changed: [ <keys with different values between the two objects> ],
+  # }
+  #
+  diff =
+    let
+      filterKeys = cond: attrs: lib.attrNames (lib.filterAttrs cond attrs);
+    in
+    old: new: {
+      added = filterKeys (n: _: !(old ? ${n})) new;
+      removed = filterKeys (n: _: !(new ? ${n})) old;
+      changed = filterKeys (
+        n: v:
+        # Filter out attributes that don't exist anymore
+        (new ? ${n})
+
+        # Filter out attributes that are the same as the new value
+        && (v != (new.${n}))
+      ) old;
+    };
+
+  # Turns
+  # [
+  #   "hello.aarch64-linux"
+  #   "hello.x86_64-linux"
+  #   "hello.aarch64-darwin"
+  #   "hello.x86_64-darwin"
+  #   "bye.x86_64-darwin"
+  #   "bye.aarch64-darwin"
+  # ]
+  #
+  # into
+  #
+  # {
+  #   linux = [
+  #     "hello"
+  #   ];
+  #   darwin = [
+  #     "hello"
+  #     "bye"
+  #   ];
+  # }
+  groupByKernel =
+    systemPaths:
+    let
+      systemPaths' = builtins.map _processSystemPath systemPaths;
+
+      filterKernel =
+        kernel:
+        builtins.attrNames (
+          builtins.groupBy (systemPath: systemPath.name) (
+            builtins.filter (systemPath: lib.hasSuffix kernel systemPath.system) systemPaths'
+          )
+        );
+    in
+    lib.genAttrs [ "linux" "darwin" ] filterKernel;
+
+  getLabels = lib.mapAttrs (
+    kernel: rebuildCount:
+    let
+      number =
+        if rebuildCount == 0 then
+          "0"
+        else if rebuildCount <= 10 then
+          "1-10"
+        else if rebuildCount <= 500 then
+          "101-500"
+        else if rebuildCount <= 1000 then
+          "501-1000"
+        else if rebuildCount <= 2500 then
+          "1001-2500"
+        else if rebuildCount <= 5000 then
+          "2501-5000"
+        else
+          "5001+";
+
+    in
+    "10.rebuild-${kernel}: ${number}"
+  );
+}
diff --git a/ci/eval/default.nix b/ci/eval/default.nix
index ef107d4ce517a4..e55093be57c52c 100644
--- a/ci/eval/default.nix
+++ b/ci/eval/default.nix
@@ -2,6 +2,7 @@
   lib,
   runCommand,
   writeShellScript,
+  writeText,
   linkFarm,
   time,
   procps,
@@ -246,24 +247,15 @@ let
           jq -s from_entries > $out/stats.json
       '';
 
-  compare =
-    { beforeResultDir, afterResultDir }:
-    runCommand "compare"
-      {
-        nativeBuildInputs = [
-          jq
-        ];
-      }
-      ''
-        mkdir $out
-        jq -n -f ${./compare.jq} \
-          --slurpfile before ${beforeResultDir}/outpaths.json \
-          --slurpfile after ${afterResultDir}/outpaths.json \
-          > $out/changed-paths.json
-
-        jq -r -f ${./generate-step-summary.jq} < $out/changed-paths.json > $out/step-summary.md
-        # TODO: Compare eval stats
-      '';
+  compare = import ./compare {
+    inherit
+      lib
+      jq
+      runCommand
+      writeText
+      supportedSystems
+      ;
+  };
 
   full =
     {