build: Rework editables for full PEP-660 support

The previously used mechanism of synthezising a package using hatchling sort of works, and was enough to prove the concept of editables with Nix.

This changes `mkDerivationEditable` to use PEP-660 and should support pretty much everything.

The big complication in this development was the need for source level patching since editables are a pretty loose concept and are implemented differently by different build systems.
Source patching uses https://github.com/Instagram/LibCST/.
I'd like to rewrite this bit purely in Rust later.

So far I've successfully used this support with a few build systems, most notably meson.

For meson this requires manually calling `patch_editable.py` to create the initial files.
After that point dynamic recompilation "just works".
This will be exposed as a separate package somehow, just not yet.

build/checks/build-systems.nix

@@ -198,6 +198,30 @@ let
             };
         };
 
+      editables =
+        {
+          stdenv,
+          python3Packages,
+          pyprojectHook,
+          resolveBuildSystem,
+        }:
+        stdenv.mkDerivation {
+          inherit (python3Packages.editables)
+            pname
+            version
+            src
+            meta
+            ;
+
+          nativeBuildInputs =
+            [
+              pyprojectHook
+            ]
+            ++ resolveBuildSystem {
+              flit-core = [ ];
+            };
+        };
+
       hatchling =
         {
           stdenv,
@@ -221,6 +245,7 @@ let
               pathspec = [ ];
               pluggy = [ ];
               trove-classifiers = [ ];
+              editables = [ ];
             }
             // lib.optionalAttrs (python.pythonOlder "3.11") {
               tomli = [ ];
@@ -604,6 +629,65 @@ let
               flit-core = [ ];
             };
         };
+
+      libcst =
+        {
+          stdenv,
+          python3Packages,
+          pyprojectHook,
+          resolveBuildSystem,
+          rustPlatform,
+          cargo,
+          rustc,
+        }:
+        stdenv.mkDerivation {
+          inherit (python3Packages.libcst)
+            name
+            pname
+            src
+            version
+            cargoDeps
+            cargoRoot
+            ;
+          passthru.dependencies = {
+            pyyaml = [ ];
+          };
+          nativeBuildInputs =
+            [
+              pyprojectHook
+              rustPlatform.cargoSetupHook
+              cargo
+              rustc
+            ]
+            ++ resolveBuildSystem {
+              setuptools = [ ];
+              setuptools-rust = [ ];
+            };
+        };
+
+      pyyaml =
+        {
+          stdenv,
+          python3Packages,
+          pyprojectHook,
+          resolveBuildSystem,
+        }:
+        stdenv.mkDerivation {
+          inherit (python3Packages.pyyaml)
+            pname
+            version
+            src
+            meta
+            ;
+
+          nativeBuildInputs =
+            [
+              pyprojectHook
+            ]
+            ++ resolveBuildSystem {
+              setuptools = [ ];
+            };
+        };
     };
 
   crossOverlay = lib.composeExtensions (_final: prev: {

build/checks/default.nix

@@ -156,25 +156,21 @@ let
             final: _prev: {
               myapp = final.callPackage (
                 {
-                  python,
                   stdenv,
-                  pyprojectHook,
+                  pyprojectEditableHook,
                   resolveBuildSystem,
-                  pythonPkgsBuildHost,
                 }:
                 stdenv.mkDerivation (
                   renderers.mkDerivationEditable
                     {
                       project = myapp;
                       environ = testEnviron;
-                      root = "$NIX_BUILD_TOP/src";
+                      root = "$NIX_BUILD_TOP";
                     }
                     {
                       inherit
-                        python
-                        pyprojectHook
+                        pyprojectEditableHook
                         resolveBuildSystem
-                        pythonPkgsBuildHost
                         ;
                     }
                 )

build/hooks/default.nix

@@ -34,6 +34,9 @@ let
     else
       pythonOnBuildForHost.interpreter;
 
+  # Builder commands for editable packages
+  editableHook' = callPackage ./editable_hook { };
+
 in
 {
   /*
@@ -178,17 +181,21 @@ in
           pyprojectOutputSetupHook,
           pyprojectCrossShebangHook,
           python,
+          extraHooks ? [ ],
         }:
         makeSetupHook {
           name = "pyproject-hook";
           passthru.python = python;
-          propagatedBuildInputs = [
-            python
-            pyprojectConfigureHook
-            pyprojectBuildHook
-            pyprojectInstallHook
-            pyprojectOutputSetupHook
-          ] ++ lib.optional (stdenv.buildPlatform != stdenv.hostPlatform) pyprojectCrossShebangHook;
+          propagatedBuildInputs =
+            [
+              python
+              pyprojectConfigureHook
+              pyprojectBuildHook
+              pyprojectInstallHook
+              pyprojectOutputSetupHook
+            ]
+            ++ lib.optional (stdenv.buildPlatform != stdenv.hostPlatform) pyprojectCrossShebangHook
+            ++ extraHooks;
         } ./meta-hook.sh
       )
       {
@@ -203,4 +210,44 @@ in
   pyprojectWheelHook = hooks.pyprojectHook.override {
     pyprojectBuildHook = hooks.pyprojectWheelDistHook;
   };
+
+  /*
+    Hook used to build editable packages.
+
+    Use instead of pyprojectHook.
+  */
+  pyprojectEditableHook = hooks.pyprojectHook.override {
+    pyprojectBuildHook = hooks.pyprojectBuildEditableHook;
+    extraHooks = [
+      hooks.pyprojectFixupEditableHook
+    ];
+  };
+
+  /*
+    Build an editable package
+    .
+  */
+  pyprojectBuildEditableHook = callPackage (
+    { python }:
+    makeSetupHook {
+      name = "pyproject-editable-hook";
+      substitutions = {
+        editableHook = editableHook';
+      };
+    } ./pyproject-build-editable-hook.sh
+  ) { };
+
+  /*
+    Fixup path references in an editable package
+    .
+  */
+  pyprojectFixupEditableHook = callPackage (
+    { python }:
+    makeSetupHook {
+      name = "pyproject-fixup-editable-hook";
+      substitutions = {
+        editableHook = editableHook';
+      };
+    } ./pyproject-fixup-editable-hook.sh
+  ) { };
 }

build/hooks/editable_hook/default.nix

@@ -0,0 +1,33 @@
+{
+  runCommand,
+  lib,
+  python,
+  mkVirtualEnv,
+}:
+
+let
+  env =
+    mkVirtualEnv "editable-hook-env" {
+      libcst = [ ];
+      pyproject-hooks = [ ];
+    }
+    // lib.optionalAttrs (python.pythonOlder "3.11") {
+      tomli = [ ];
+    };
+
+in
+runCommand "editable-hook" { } ''
+  mkdir -p $out/bin
+
+  cat > $out/bin/build-editable << EOF
+  #!${env}/bin/python
+  EOF
+  cat ${./editable_hook/build_editable.py} >> $out/bin/build-editable
+
+  cat > $out/bin/patch-editable << EOF
+  #!${env}/bin/python
+  EOF
+  cat ${./editable_hook/patch_editable.py} >> $out/bin/patch-editable
+
+  chmod +x $out/bin/*
+''

build/hooks/editable_hook/editable_hook/build_editable.py

@@ -0,0 +1,41 @@
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib  # pyright: ignore[reportMissingImports]
+import pyproject_hooks
+
+
+def main():
+    build_dir = Path(os.getcwd())
+    dist = build_dir.joinpath("dist")
+
+    with open(build_dir.joinpath("pyproject.toml"), "rb") as pyproject_file:
+        pyproject: dict[str, Any] = tomllib.load(pyproject_file)  # pyright: ignore[reportUnknownMemberType,reportExplicitAny]
+
+    # Get build backend with fallback behaviour
+    # https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/#fallback-behaviour
+    try:
+        build_backend: str = pyproject["build-system"]["build-backend"]
+    except KeyError:
+        build_backend = "setuptools.build_meta:__legacy__"
+
+    try:
+        dist.mkdir()
+    except FileExistsError:
+        pass
+
+    # Call editable build hooks using pyproject-hooks
+    hook_caller = pyproject_hooks.BuildBackendHookCaller(
+        source_dir=str(build_dir),
+        build_backend=build_backend,
+    )
+    hook_caller.build_editable(str(dist))
+
+
+if __name__ == "__main__":
+    main()