lib.extendMkDerivation: init (#234651)

It's not the longest-open PR (May 28, 2023 -> Jan 22, 2025) but it took a while. This PR introduces a unified approach to implementing build helpers that support fixed-point arguments and bring such support to existing build helpers.

doc/build-helpers.md

@@ -17,6 +17,7 @@ There is no uniform interface for build helpers.
 [Language- or framework-specific build helpers](#chap-language-support) usually follow the style of `stdenv.mkDerivation`, which accepts an attribute set or a fixed-point function taking an attribute set.
 
 ```{=include=} chapters
+build-helpers/fixed-point-arguments.chapter.md
 build-helpers/fetchers.chapter.md
 build-helpers/trivial-build-helpers.chapter.md
 build-helpers/testers.chapter.md

doc/build-helpers/fixed-point-arguments.chapter.md

@@ -0,0 +1,74 @@
+# Fixed-point arguments of build helpers {#chap-build-helpers-finalAttrs}
+
+As mentioned in the beginning of this part, `stdenv.mkDerivation` could alternatively accept a fixed-point function. The input of such function, typically named `finalAttrs`, is expected to be the final state of the attribute set.
+A build helper like this is said to accept **fixed-point arguments**.
+
+Build helpers don't always support fixed-point arguments yet, as support in [`stdenv.mkDerivation`](#mkderivation-recursive-attributes) was first included in Nixpkgs 22.05.
+
+## Defining a build helper with `lib.extendMkDerivation` {#sec-build-helper-extendMkDerivation}
+
+Developers can use the Nixpkgs library function [`lib.customisation.extendMkDerivation`](#function-library-lib.customisation.extendMkDerivation) to define a build helper supporting fixed-point arguments from an existing one with such support, with an attribute overlay similar to the one taken by [`<pkg>.overrideAttrs`](#sec-pkg-overrideAttrs).
+
+Beside overriding, `lib.extendMkDerivation` also supports `excludeDrvArgNames` to optionally exclude some arguments in the input fixed-point argumnts from passing down the base build helper (specified as `constructDrv`).
+
+:::{.example #ex-build-helpers-extendMkDerivation}
+
+# Example definition of `mkLocalDerivation` extended from `stdenv.mkDerivation` with `lib.extendMkDerivation`
+
+We want to define a build helper named `mkLocalDerivation` that builds locally without using substitutes by default.
+
+Instead of taking a plain attribute set,
+
+```nix
+{
+  preferLocalBuild ? true,
+  allowSubstitute ? false,
+  specialArg ? (_: false),
+  ...
+}@args:
+
+stdenv.mkDerivation (
+  removeAttrs [
+    # Don't pass specialArg into mkDerivation.
+    "specialArg"
+  ] args
+  // {
+    # Arguments to pass
+    inherit preferLocalBuild allowSubstitute;
+    # Some expressions involving specialArg
+    greeting = if specialArg "hi" then "hi" else "hello";
+  }
+)
+```
+
+we could define with `lib.extendMkDerivation` an attribute overlay to make the result build helper also accepts the the attribute set's fixed point passing to the underlying `stdenv.mkDerivation`, named `finalAttrs` here:
+
+```nix
+lib.extendMkDerivation {
+  constructDrv = stdenv.mkDerivation;
+  excludeDrvArgNames = [
+    # Don't pass specialArg into mkDerivation.
+    "specialArg"
+  ];
+  extendDrvArgs =
+    finalAttrs:
+    {
+      preferLocalBuild ? true,
+      allowSubstitute ? false,
+      specialArg ? (_: false),
+      ...
+    }@args:
+    {
+      # Arguments to pass
+      inherit
+        preferLocalBuild
+        allowSubstitute
+        ;
+      # Some expressions involving specialArg
+      greeting = if specialArg "hi" then "hi" else "hello";
+    };
+}
+```
+:::
+
+If one needs to apply extra changes to the result derivation, pass the derivation transformation function to `lib.extendMkDerivation` as `lib.customisation.extendMkDerivation { transformDrv = drv: ...; }`.

doc/redirects.json

@@ -1,7 +1,13 @@
 {
+  "chap-build-helpers-finalAttrs": [
+    "index.html#chap-build-helpers-finalAttrs"
+  ],
   "chap-release-notes": [
     "release-notes.html#chap-release-notes"
   ],
+  "ex-build-helpers-extendMkDerivation": [
+    "index.html#ex-build-helpers-extendMkDerivation"
+  ],
   "neovim": [
     "index.html#neovim"
   ],
@@ -41,6 +47,9 @@
   "sec-allow-insecure": [
     "index.html#sec-allow-insecure"
   ],
+  "sec-build-helper-extendMkDerivation": [
+    "index.html#sec-build-helper-extendMkDerivation"
+  ],
   "sec-modify-via-packageOverrides": [
     "index.html#sec-modify-via-packageOverrides"
   ],

lib/customisation.nix

@@ -31,6 +31,8 @@ let
     flatten
     deepSeq
     extends
+    toFunction
+    id
     ;
   inherit (lib.strings) levenshtein levenshteinAtMost;
 
@@ -730,4 +732,126 @@ rec {
     in
     self;
 
+  /**
+    Define a `mkDerivation`-like function based on another `mkDerivation`-like function.
+
+    [`stdenv.mkDerivation`](#part-stdenv) gives access to
+    its final set of derivation attributes when it is passed a function,
+    or when it is passed an overlay-style function in `overrideAttrs`.
+
+    Instead of composing new `stdenv.mkDerivation`-like build helpers
+    using normal function composition,
+    `extendMkDerivation` makes sure that the returned build helper
+    supports such first class recursion like `mkDerivation` does.
+
+    `extendMkDerivation` takes an extra attribute set to configure its behaviour.
+    One can optionally specify
+    `transformDrv` to specify a function to apply to the result derivation,
+    or `inheritFunctionArgs` to decide whether to inherit the `__functionArgs`
+    from the base build helper.
+
+    # Inputs
+
+    `extendMkDerivation`-specific configurations
+    : `constructDrv`: Base build helper, the `mkDerivation`-like build helper to extend.
+    : `excludeDrvArgNames`: Argument names not to pass from the input fixed-point arguments to `constructDrv`. Note: It doesn't apply to the updating arguments returned by `extendDrvArgs`.
+    : `extendDrvArgs` : An extension (overlay) of the argument set, like the one taken by [overrideAttrs](#sec-pkg-overrideAttrs) but applied before passing to `constructDrv`.
+    : `inheritFunctionArgs`: Whether to inherit `__functionArgs` from the base build helper (default to `true`).
+    : `transformDrv`: Function to apply to the result derivation (default to `lib.id`).
+
+    # Type
+
+    ```
+    extendMkDerivation ::
+      {
+        constructDrv :: ((FixedPointArgs | AttrSet) -> a)
+        excludeDrvArgNames :: [ String ],
+        extendDrvArgs :: (AttrSet -> AttrSet -> AttrSet)
+        inheritFunctionArgs :: Bool,
+        transformDrv :: a -> a,
+      }
+      -> (FixedPointArgs | AttrSet) -> a
+
+    FixedPointArgs = AttrSet -> AttrSet
+    a = Derivation when defining a build helper
+    ```
+
+    # Examples
+
+    :::{.example}
+    ## `lib.customisation.extendMkDerivation` usage example
+    ```nix-repl
+    mkLocalDerivation = lib.extendMkDerivation {
+      constructDrv = pkgs.stdenv.mkDerivation;
+      excludeDrvArgNames = [ "specialArg" ];
+      extendDrvArgs =
+        finalAttrs: args@{ preferLocalBuild ? true, allowSubstitute ? false, specialArg ? (_: false), ... }:
+        { inherit preferLocalBuild allowSubstitute; passthru = { inherit specialArg; } // args.passthru or { }; };
+    }
+
+    mkLocalDerivation.__functionArgs
+    => { allowSubstitute = true; preferLocalBuild = true; specialArg = true; }
+
+    mkLocalDerivation { inherit (pkgs.hello) pname version src; specialArg = _: false; }
+    => «derivation /nix/store/xirl67m60ahg6jmzicx43a81g635g8z8-hello-2.12.1.drv»
+
+    mkLocalDerivation (finalAttrs: { inherit (pkgs.hello) pname version src; specialArg = _: false; })
+    => «derivation /nix/store/xirl67m60ahg6jmzicx43a81g635g8z8-hello-2.12.1.drv»
+
+    (mkLocalDerivation (finalAttrs: { inherit (pkgs.hello) pname version src; passthru = { foo = "a"; bar = "${finalAttrs.passthru.foo}b"; }; })).bar
+    => "ab"
+    ```
+    :::
+
+    :::{.note}
+    If `transformDrv` is specified,
+    it should take care of existing attributes that perform overriding
+    (e.g., [`overrideAttrs`](#sec-pkg-overrideAttrs))
+    to ensure that the overriding functionality of the result derivation
+    work as expected.
+    Modifications that breaks the overriding include
+    direct [attribute set update](https://nixos.org/manual/nix/stable/language/operators#update)
+    and [`lib.extendDerivation`](#function-library-lib.customisation.extendDerivation).
+    :::
+  */
+  extendMkDerivation =
+    let
+      extendsWithExclusion =
+        excludedNames: g: f: final:
+        let
+          previous = f final;
+        in
+        removeAttrs previous excludedNames // g final previous;
+    in
+    {
+      constructDrv,
+      excludeDrvArgNames ? [ ],
+      extendDrvArgs,
+      inheritFunctionArgs ? true,
+      transformDrv ? id,
+    }:
+    setFunctionArgs
+      # Adds the fixed-point style support
+      (
+        fpargs:
+        transformDrv (
+          constructDrv (extendsWithExclusion excludeDrvArgNames extendDrvArgs (toFunction fpargs))
+        )
+      )
+      # Add __functionArgs
+      (
+        # Inherit the __functionArgs from the base build helper
+        optionalAttrs inheritFunctionArgs (removeAttrs (functionArgs constructDrv) excludeDrvArgNames)
+        # Recover the __functionArgs from the derived build helper
+        // functionArgs (extendDrvArgs { })
+      )
+    // {
+      inherit
+        # Expose to the result build helper.
+        constructDrv
+        excludeDrvArgNames
+        extendDrvArgs
+        transformDrv
+        ;
+    };
 }

lib/default.nix

@@ -121,7 +121,8 @@ let
       noDepEntry fullDepEntry packEntry stringAfter;
     inherit (self.customisation) overrideDerivation makeOverridable
       callPackageWith callPackagesWith extendDerivation hydraJob
-      makeScope makeScopeWithSplicing makeScopeWithSplicing';
+      makeScope makeScopeWithSplicing makeScopeWithSplicing'
+      extendMkDerivation;
     inherit (self.derivations) lazyDerivation optionalDrvAttr warnOnInstantiate;
     inherit (self.meta) addMetaAttrs dontDistribute setName updateName
       appendToName mapDerivationAttrset setPrio lowPrio lowPrioSet hiPrio

pkgs/test/overriding.nix

@@ -5,7 +5,7 @@
 }:
 
 let
-  tests = tests-stdenv // tests-go // tests-python;
+  tests = tests-stdenv // test-extendMkDerivation // tests-go // tests-python;
 
   tests-stdenv =
     let
@@ -64,6 +64,73 @@ let
       };
     };
 
+  test-extendMkDerivation =
+    let
+      mkLocalDerivation = lib.extendMkDerivation {
+        constructDrv = pkgs.stdenv.mkDerivation;
+        excludeDrvArgNames = [
+          "specialArg"
+        ];
+        extendDrvArgs =
+          finalAttrs:
+          {
+            preferLocalBuild ? true,
+            allowSubstitute ? false,
+            specialArg ? (_: false),
+            ...
+          }@args:
+          {
+            inherit preferLocalBuild allowSubstitute;
+            passthru = args.passthru or { } // {
+              greeting = if specialArg "Hi!" then "Hi!" else "Hello!";
+            };
+          };
+      };
+
+      helloLocalPlainAttrs = {
+        inherit (pkgs.hello) pname version src;
+        specialArg = throw "specialArg is broken.";
+      };
+
+      helloLocalPlain = mkLocalDerivation helloLocalPlainAttrs;
+
+      helloLocal = mkLocalDerivation (
+        finalAttrs:
+        helloLocalPlainAttrs
+        // {
+          passthru = pkgs.hello.passthru or { } // {
+            foo = "a";
+            bar = "${finalAttrs.passthru.foo}b";
+          };
+        }
+      );
+
+      hiLocal = mkLocalDerivation (
+        helloLocalPlainAttrs
+        // {
+          specialArg = s: lib.stringLength s == 3;
+        }
+      );
+    in
+    {
+      extendMkDerivation-helloLocal-imp-arguments = {
+        expr = helloLocal.preferLocalBuild;
+        expected = true;
+      };
+      extendMkDerivation-helloLocal-plain-equivalence = {
+        expr = helloLocal.drvPath == helloLocalPlain.drvPath;
+        expected = true;
+      };
+      extendMkDerivation-helloLocal-finalAttrs = {
+        expr = helloLocal.bar == "ab";
+        expected = true;
+      };
+      extendMkDerivation-helloLocal-specialArg = {
+        expr = hiLocal.greeting == "Hi!";
+        expected = true;
+      };
+    };
+
   tests-go =
     let
       pet_0_3_4 = pkgs.buildGoModule rec {
@@ -194,6 +261,7 @@ let
         expected = true;
       };
     };
+
 in
 
 stdenvNoCC.mkDerivation {