Build nix itself somewhat incrementally, optionally

Makefile

@@ -46,3 +46,10 @@ endif
 include mk/lib.mk
 
 GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src
+
+ifeq ($(missingMakefiles), ok)
+$(makefiles):
+	@echo "WARNING: Touching missing makefile $@"
+	@mkdir -p $(dir $@)
+	@touch $@
+endif

doc/manual/src/contributing/hacking.md

@@ -421,3 +421,12 @@ can build it yourself:
 
 Metrics about the change in line/function coverage over time are also
 [available](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts).
+
+## Coarse incremental cached build
+
+The flake provides an extra package, `.#quickBuild`, which builds Nix using
+individual derivations for the libraries and such. This allows cached versions
+of those intermediate build steps to be used, so that it builds a lot quicker.
+
+This package is useful for reviewing changes to `/doc`, assuming that the
+base commit is in the binary cache.

flake.nix

@@ -312,6 +312,172 @@
           # Forward from the previous stage as we don’t want it to pick the lowdown override
           nixUnstable = prev.nixUnstable;
 
+          # A build where build products are taken from component-specific derivations
+          # meant for hacking on Nix, not for production use, and will be removed
+          # when dynamic derivations (RFC 92) is implemented and available, or sooner.
+          nixQuick =
+            let
+              prepper = import ./prepper.nix { pkgs = final; };
+
+              inherit (prepper.prep {
+                  drv = drvSimple;
+                  inherit componentSpecs;
+                })
+                components
+                copyPrebuilt
+                toplevel;
+
+              # Prepper doesn't support multiple outputs, so we remove that.
+              # Also disable tests and filter the source a bit more.
+              drvSimple = final.nix.overrideAttrs (o: {
+                outputs = ["out"];
+                separateDebugInfo = false;
+                doCheck = false;
+                doInstallCheck = false;
+                preConfigure = ''
+                  ${o.preConfigure or ""}
+                  doc=$out
+                '';
+
+                makeFlags = o.makeFlags
+                  # It was setting rpath to /build/... for some reason
+                  + " SET_RPATH_TO_LIBS=0";
+
+                src =
+                  lib.cleanSourceWith {
+                    src = o.src;
+                    filter = path: type:
+                      (toString (dirOf path) == toString ./.
+                        -> ! lib.strings.hasSuffix ".nix" (baseNameOf path)
+                      )
+                      && ! lib.strings.hasPrefix (toString ./doc) path
+                      ;
+                  };
+              });
+
+              preConfigure = o: {
+                configurePhase =
+                ''
+                  runHook preConfigure
+                  echo "Using prebuilt configure outputs..."
+                  touch timestamper
+                  ${copyPrebuilt components.configure}
+                  rm timestamper
+                  runHook postConfigure
+                '';
+                nativeBuildInputs = lib.filter (p:
+                  p.pname or p.name or null != "autoreconf-hook"
+                  # TODO: remove once the rename comes through in our nixpkgs input
+                  && p.pname or p.name or null != "hook"
+                  ) o.nativeBuildInputs;
+              };
+
+              componentSpecs = {
+                "configure" = {
+                  dirs = [ "Makefile.config" "configure" "config.status" "config" "config.h" ];
+                  attrsOverride = o: {
+                    buildPhase = ":";
+                  };
+                };
+                "libutil" = {
+                  deps = [ "configure" ];
+                  dirs = [ "src/libutil" ];
+                  targets = "src/libutil/libnixutil.so";
+                  checkTargets = "libutil-tests_RUN";
+                  attrsOverride =
+                    preConfigure
+                    # The local .so produces a build sandbox reference in rpath, which
+                    # is not ok for an output.
+                    # This can be avoided by changing -rpath to -rpath-link, but I do not
+                    # know enough to assess whether that's the right thing to do, and I'd
+                    # like to stay in sync with the regular build.
+                    # composeOverrides (removeFile "src/libutil/libnixutil.so");
+                    ;
+                };
+                "libmain" = {
+                  dirs = [ "src/libmain" ];
+                  deps = [ "libutil" "libstore" ];
+                  targets = "src/libmain/libnixmain.so";
+                  checkTargets = "libmain-tests_RUN";
+                  attrsOverride = preConfigure;
+                };
+                "libstore" = {
+                  dirs = [ "src/libstore" ];
+                  deps = [ "libutil" ];
+                  targets = "src/libstore/libnixstore.so";
+                  checkTargets = "libstore-tests_RUN";
+                  attrsOverride = preConfigure;
+                };
+                "libfetchers" = {
+                  dirs = [ "src/libfetchers" ];
+                  deps = [ "libutil" "libstore" ];
+                  targets = "src/libfetchers/libnixfetchers.so";
+                  checkTargets = "libfetchers-tests_RUN";
+                  attrsOverride = preConfigure;
+                };
+                "libexpr" = {
+                  dirs = [ "src/libexpr" ];
+                  deps = [ "libutil" "libstore" "libfetchers" ];
+                  targets = "src/libexpr/libnixexpr.so";
+                  checkTargets = "libexpr-tests_RUN";
+                  attrsOverride = preConfigure;
+                };
+                "libcmd" = {
+                  dirs = [ "src/libcmd" ];
+                  deps = [ "libutil" "libstore" "libfetchers" "libmain" "libexpr" ];
+                  targets = "src/libcmd/libnixcmd.so";
+                  checkTargets = "libcmd-tests_RUN";
+                  attrsOverride = preConfigure;
+                };
+                "programs" = {
+                  dirs = [
+                    "src/build-remote"
+                    "src/nix"
+                    "src/nix-build"
+                    "src/nix-channel"
+                    "src/nix-collect-garbage"
+                    "src/nix-copy-closure"
+                    "src/nix-env"
+                    "src/nix-instantiate"
+                    "src/nix-store"
+                    "src/resolve-system-dependencies"
+                  ];
+                  deps = [ "libutil" "libstore" "libfetchers" "libmain" "libexpr" "libcmd" ];
+                  targets = lib.concatStringsSep " " ([
+                    "src/nix/nix"
+                  ] ++ lib.optionals final.stdenv.hostPlatform.isDarwin [
+                    "src/resolve-system-dependencies/resolve-system-dependencies"
+                  ]);
+                  checkTargets = "";
+                  attrsOverride = prepper.composeOverrides preConfigure (o: {
+                    src = lib.cleanSourceWith {
+                      src = ./.;
+                      filter = path: type:
+                        if lib.hasPrefix (toString ./doc) path
+                        then
+                          path == toString ./doc
+                          || path == toString ./doc/manual
+                          || path == toString ./doc/manual/local.mk
+                          || path == toString ./doc/manual/generate-manpage.nix
+                          || path == toString ./doc/manual/utils.nix
+                          || path == toString ./doc/manual/src
+                          || lib.hasPrefix (toString ./doc/manual/src/command-ref) path
+                        else
+                          o.src.filter path type;
+                    };
+                  });
+                };
+              };
+
+            in
+              toplevel.overrideAttrs
+                (o: preConfigure o // { # FIXME use composition combinator
+                  # undo the source override; we depend on everything anyway
+                  src = final.nix.src;
+                  buildFlags = "${o.buildFlags or ""} all";
+
+                });
+
           nix =
           with final;
           with commonDeps {
@@ -656,6 +822,7 @@
       packages = forAllSystems (system: rec {
         inherit (nixpkgsFor.${system}.native) nix;
         default = nix;
+        quickBuild = nixpkgsFor.${system}.native.nixQuick;
       } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) {
         nix-static = nixpkgsFor.${system}.static.nix;
         dockerImage =

mk/lib.mk

@@ -1,4 +1,4 @@
-default: all
+default: src/libutil/libnixutil.so all
 
 
 # Get rid of default suffixes. FIXME: is this a good idea?

mk/libraries.mk

@@ -95,7 +95,7 @@ define build-library
 	+$$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED)
 
     ifndef HOST_DARWIN
-      $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d))
+      $(1)_LDFLAGS_USE += -Wl,-rpath,'$$$$ORIGIN'
     endif
     $(1)_LDFLAGS_USE += -L$$(_d) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME)))
 
@@ -133,10 +133,10 @@ define build-library
 
     $(1)_INSTALL_PATH := $$(libdir)/$$($(1)_NAME).a
 
-    $(1)_LIB_CLOSURE += $$($(1)_LIBS)
-
   endif
 
+	$(1)_LIB_CLOSURE += $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LIB_CLOSURE))
+
   $(1)_LDFLAGS_USE += $$($(1)_LDFLAGS_PROPAGATED)
   $(1)_LDFLAGS_USE_INSTALLED += $$($(1)_LDFLAGS_PROPAGATED)
 

mk/programs.mk

@@ -36,7 +36,7 @@ define build-program
   $$(eval $$(call create-dir, $$(_d)))
 
   $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/
-	+$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE))
+	+$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$(foreach lib, $$($(1)_LIBS), $$(foreach lib2, $$($$(lib)_LIB_CLOSURE), -Wl,-rpath,$$($$(lib2)_PATH)))
 
   $(1)_INSTALL_DIR ?= $$(bindir)
 

mk/tracing.mk

@@ -5,7 +5,7 @@ ifeq ($(V), 0)
   trace-gen     = @echo "  GEN   " $@;
   trace-cc      = @echo "  CC    " $@;
   trace-cxx     = @echo "  CXX   " $@;
-  trace-ld      = @echo "  LD    " $@;
+  trace-ld      = @echo "  LD    " $@; set -x;
   trace-ar      = @echo "  AR    " $@;
   trace-install = @echo "  INST  " $@;
   trace-mkdir   = @echo "  MKDIR " $@;