diff --git a/Makefile b/Makefile index c6220482adb..8fc5302c51d 100644 --- a/Makefile +++ b/Makefile @@ -46,3 +46,10 @@ endif include mk/lib.mk GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src + +ifeq ($(missingMakefiles), ok) +$(makefiles): + @echo "WARNING: Touching missing makefile $@" + @mkdir -p $(dir $@) + @touch $@ +endif diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index b954a21673c..be6de9bca8e 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -421,3 +421,12 @@ can build it yourself: Metrics about the change in line/function coverage over time are also [available](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts). + +## Coarse incremental cached build + +The flake provides an extra package, `.#quickBuild`, which builds Nix using +individual derivations for the libraries and such. This allows cached versions +of those intermediate build steps to be used, so that it builds a lot quicker. + +This package is useful for reviewing changes to `/doc`, assuming that the +base commit is in the binary cache. diff --git a/flake.nix b/flake.nix index bdbf541693a..696e9801d1f 100644 --- a/flake.nix +++ b/flake.nix @@ -312,6 +312,172 @@ # Forward from the previous stage as we don’t want it to pick the lowdown override nixUnstable = prev.nixUnstable; + # A build where build products are taken from component-specific derivations + # meant for hacking on Nix, not for production use, and will be removed + # when dynamic derivations (RFC 92) is implemented and available, or sooner. + nixQuick = + let + prepper = import ./prepper.nix { pkgs = final; }; + + inherit (prepper.prep { + drv = drvSimple; + inherit componentSpecs; + }) + components + copyPrebuilt + toplevel; + + # Prepper doesn't support multiple outputs, so we remove that. + # Also disable tests and filter the source a bit more. + drvSimple = final.nix.overrideAttrs (o: { + outputs = ["out"]; + separateDebugInfo = false; + doCheck = false; + doInstallCheck = false; + preConfigure = '' + ${o.preConfigure or ""} + doc=$out + ''; + + makeFlags = o.makeFlags + # It was setting rpath to /build/... for some reason + + " SET_RPATH_TO_LIBS=0"; + + src = + lib.cleanSourceWith { + src = o.src; + filter = path: type: + (toString (dirOf path) == toString ./. + -> ! lib.strings.hasSuffix ".nix" (baseNameOf path) + ) + && ! lib.strings.hasPrefix (toString ./doc) path + ; + }; + }); + + preConfigure = o: { + configurePhase = + '' + runHook preConfigure + echo "Using prebuilt configure outputs..." + touch timestamper + ${copyPrebuilt components.configure} + rm timestamper + runHook postConfigure + ''; + nativeBuildInputs = lib.filter (p: + p.pname or p.name or null != "autoreconf-hook" + # TODO: remove once the rename comes through in our nixpkgs input + && p.pname or p.name or null != "hook" + ) o.nativeBuildInputs; + }; + + componentSpecs = { + "configure" = { + dirs = [ "Makefile.config" "configure" "config.status" "config" "config.h" ]; + attrsOverride = o: { + buildPhase = ":"; + }; + }; + "libutil" = { + deps = [ "configure" ]; + dirs = [ "src/libutil" ]; + targets = "src/libutil/libnixutil.so"; + checkTargets = "libutil-tests_RUN"; + attrsOverride = + preConfigure + # The local .so produces a build sandbox reference in rpath, which + # is not ok for an output. + # This can be avoided by changing -rpath to -rpath-link, but I do not + # know enough to assess whether that's the right thing to do, and I'd + # like to stay in sync with the regular build. + # composeOverrides (removeFile "src/libutil/libnixutil.so"); + ; + }; + "libmain" = { + dirs = [ "src/libmain" ]; + deps = [ "libutil" "libstore" ]; + targets = "src/libmain/libnixmain.so"; + checkTargets = "libmain-tests_RUN"; + attrsOverride = preConfigure; + }; + "libstore" = { + dirs = [ "src/libstore" ]; + deps = [ "libutil" ]; + targets = "src/libstore/libnixstore.so"; + checkTargets = "libstore-tests_RUN"; + attrsOverride = preConfigure; + }; + "libfetchers" = { + dirs = [ "src/libfetchers" ]; + deps = [ "libutil" "libstore" ]; + targets = "src/libfetchers/libnixfetchers.so"; + checkTargets = "libfetchers-tests_RUN"; + attrsOverride = preConfigure; + }; + "libexpr" = { + dirs = [ "src/libexpr" ]; + deps = [ "libutil" "libstore" "libfetchers" ]; + targets = "src/libexpr/libnixexpr.so"; + checkTargets = "libexpr-tests_RUN"; + attrsOverride = preConfigure; + }; + "libcmd" = { + dirs = [ "src/libcmd" ]; + deps = [ "libutil" "libstore" "libfetchers" "libmain" "libexpr" ]; + targets = "src/libcmd/libnixcmd.so"; + checkTargets = "libcmd-tests_RUN"; + attrsOverride = preConfigure; + }; + "programs" = { + dirs = [ + "src/build-remote" + "src/nix" + "src/nix-build" + "src/nix-channel" + "src/nix-collect-garbage" + "src/nix-copy-closure" + "src/nix-env" + "src/nix-instantiate" + "src/nix-store" + "src/resolve-system-dependencies" + ]; + deps = [ "libutil" "libstore" "libfetchers" "libmain" "libexpr" "libcmd" ]; + targets = lib.concatStringsSep " " ([ + "src/nix/nix" + ] ++ lib.optionals final.stdenv.hostPlatform.isDarwin [ + "src/resolve-system-dependencies/resolve-system-dependencies" + ]); + checkTargets = ""; + attrsOverride = prepper.composeOverrides preConfigure (o: { + src = lib.cleanSourceWith { + src = ./.; + filter = path: type: + if lib.hasPrefix (toString ./doc) path + then + path == toString ./doc + || path == toString ./doc/manual + || path == toString ./doc/manual/local.mk + || path == toString ./doc/manual/generate-manpage.nix + || path == toString ./doc/manual/utils.nix + || path == toString ./doc/manual/src + || lib.hasPrefix (toString ./doc/manual/src/command-ref) path + else + o.src.filter path type; + }; + }); + }; + }; + + in + toplevel.overrideAttrs + (o: preConfigure o // { # FIXME use composition combinator + # undo the source override; we depend on everything anyway + src = final.nix.src; + buildFlags = "${o.buildFlags or ""} all"; + + }); + nix = with final; with commonDeps { @@ -656,6 +822,7 @@ packages = forAllSystems (system: rec { inherit (nixpkgsFor.${system}.native) nix; default = nix; + quickBuild = nixpkgsFor.${system}.native.nixQuick; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) { nix-static = nixpkgsFor.${system}.static.nix; dockerImage = diff --git a/mk/lib.mk b/mk/lib.mk index 34fa624d8d5..c6ac09912b2 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -1,4 +1,4 @@ -default: all +default: src/libutil/libnixutil.so all # Get rid of default suffixes. FIXME: is this a good idea? diff --git a/mk/libraries.mk b/mk/libraries.mk index 1bc73d7f7a9..6cbf79ed36c 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -95,7 +95,7 @@ define build-library +$$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) ifndef HOST_DARWIN - $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d)) + $(1)_LDFLAGS_USE += -Wl,-rpath,'$$$$ORIGIN' endif $(1)_LDFLAGS_USE += -L$$(_d) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME))) @@ -133,10 +133,10 @@ define build-library $(1)_INSTALL_PATH := $$(libdir)/$$($(1)_NAME).a - $(1)_LIB_CLOSURE += $$($(1)_LIBS) - endif + $(1)_LIB_CLOSURE += $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LIB_CLOSURE)) + $(1)_LDFLAGS_USE += $$($(1)_LDFLAGS_PROPAGATED) $(1)_LDFLAGS_USE_INSTALLED += $$($(1)_LDFLAGS_PROPAGATED) diff --git a/mk/programs.mk b/mk/programs.mk index 1ee1d3fa5de..6f333dd6660 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -36,7 +36,7 @@ define build-program $$(eval $$(call create-dir, $$(_d))) $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ - +$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) + +$$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$(foreach lib, $$($(1)_LIBS), $$(foreach lib2, $$($$(lib)_LIB_CLOSURE), -Wl,-rpath,$$($$(lib2)_PATH))) $(1)_INSTALL_DIR ?= $$(bindir) diff --git a/mk/tracing.mk b/mk/tracing.mk index 1fc5573d7c1..9392b5db68f 100644 --- a/mk/tracing.mk +++ b/mk/tracing.mk @@ -5,7 +5,7 @@ ifeq ($(V), 0) trace-gen = @echo " GEN " $@; trace-cc = @echo " CC " $@; trace-cxx = @echo " CXX " $@; - trace-ld = @echo " LD " $@; + trace-ld = @echo " LD " $@; set -x; trace-ar = @echo " AR " $@; trace-install = @echo " INST " $@; trace-mkdir = @echo " MKDIR " $@; diff --git a/prepper.nix b/prepper.nix new file mode 100644 index 00000000000..cae2f103e3e --- /dev/null +++ b/prepper.nix @@ -0,0 +1,191 @@ +/* + prepper.nix - mediam grained incremental build while RFC 92 is in progress + + This splits the build into separate derivations that build each component, so + that changes to the higher-level components don't need a rebuild of the + components they depend on. + It achieves this by overriding the simpler original derivation that would + build everything in one go. + + It does so by saving the working directory in the output of the intermediate + steps. + + Flaws: + - The dependency management and source filtering are not super accurate + while any mistakes will be caught by the build, it still includes many + files that are not needed for the build. + - Files are included in every component build by default. This means that + changes to the build graph generally work, but may rebuild more than + necessary. + - Installed files from outputs are not accounted for. The docs build relies + on the installed nix binary. This means linking the nix binary in the docs + build. + - Object files could be built separately, so that their library dependency is + only on headers. This will save more rebuilds. + - A component might rebuild its dependencies. So this is fail safe in the + sense that you don't have to bother with the problem immediately, but it + does slow down the build and you'd have to check the logs to make sure it + doesn't happen. + + What we should do: + - RFC 92. That will let us replace this by a principled solution that is + more accurate, more efficient, and easier to maintain. + - Low hanging fruit: support installed files in the components. + - Short of that, we could make the libraries into individual packages. This + way we won't have to copy as many files around. +*/ + +{ pkgs }: + +let + inherit (pkgs) lib; + + composeOverrides = f: g: x: + let x2 = g x; + in x2 // f (x // x2); + + # removeFile = f: o: { + # buildPhase = o.buildPhase + '' + # rm ${f} + # ''; + # }; + + prep = { drv, componentSpecs }: + let + depsClosure = + let clList = lib.genericClosure { + startSet = # + lib.concatLists (lib.mapAttrsToList + (name: spec@{deps ? [], ...}: map (dep: { inherit name dep; key = "${name} ---> ${dep}"; }) deps) + componentSpecs); + operator = item: map (dep: { inherit (item) name; inherit dep; key = "${item.name} ---> ${dep}";}) componentSpecs.${item.name}.deps or []; + }; + in + # Ensure an empty attr exists for each component + lib.mapAttrs (name: _: []) componentSpecs + # Add the dependencies + // lib.zipAttrs ( + map + (item: { ${item.name} = item.dep; }) + clList + ); + + nonDeps = + lib.mapAttrs + (name: deps: lib.attrNames (builtins.removeAttrs componentSpecs ([name] ++ deps))) + depsClosure; + + components = + lib.mapAttrs + (name: spec@{ dirs, deps ? [], attrsOverride ? (o: {}), findFilter ? "", ... }: + let prebuiltDeps = lib.genAttrs deps (x: components.${x}); + in + ((addPrebuilt drv prebuiltDeps).overrideAttrs (o: { + + # Remove the reverse dependencies + src = lib.cleanSourceWith { + src = o.src; + filter = path: type: + lib.all + (dep: + let depSpec = componentSpecs.${dep}; + in + lib.all + (src: + toString path != toString (o.src.origSrc + ("/" + src))) + depSpec.dirs + ) + (nonDeps.${name}); + }; + preBuild = '' + ${o.preBuild or ""} + buildFlagsArray+=( missingMakefiles=ok ${spec.targets or "default"} ) + echo "Building component ${name}..." + ''; + # TODO hook into setup.sh like in preBuild above + # checkPhase = lib.optionalString (spec?checkTargets) '' + # make -j $NIX_BUILD_CORES missingMakefiles=ok $makeFlags ${spec.checkTargets} + # ''; + installPhase = '' + runHook preInstall + for f in $(find ${lib.escapeShellArgs dirs} -type f ${findFilter}); do + mkdir -p $(dirname $out/progress/$f) + install $f $out/progress/$f + done + runHook postInstall + ''; + # doInstallCheck can be whatever it needs to be so that it's as much as + # possible like a normal build. + installCheckPhase = ":"; + }) + ).overrideAttrs attrsOverride + ) + componentSpecs; + + copyPrebuilt = layer: '' + for f in $(cd ${layer}/progress; find . -type f); do + if [[ -e $f ]]; then continue; fi + echo Copying pre-built $f + mkdir -p $(dirname $f) + ${lib.getExe pkgs.buildPackages.perl} -pe "s|\Q${layer}\E|$out|g" \ + < ${layer}/progress/$f > $f + [[ -x ${layer}/progress/$f ]] && chmod +x $f + touch --reference=timestamper $f + done + ''; + + addPrebuilt = drv: deps: + drv.overrideAttrs (o: { + postConfigure = '' + ${o.postConfigure or ""} + touch timestamper + ${lib.concatMapStringsSep "\n" copyPrebuilt (lib.attrValues deps)} + rm timestamper + ''; + disallowedReferences = lib.attrValues deps; + passthru = o.passthru // { + prebuiltDeps = deps; + }; + }); + + toplevel = addPrebuilt drv components; + + in { + inherit components copyPrebuilt toplevel; + }; +in +{ + inherit + composeOverrides + prep + ; +} + + # Unused: + + # revDeps = + # lib.zipAttrs ( + # lib.concatLists ( + # lib.mapAttrsToList + # (name: spec@{ deps ? [], ... }: + # map + # (dep: { ${dep} = name; }) + # deps + # ) + # componentSpecs + # ) + # ); + + # revDepsClosure = + # let clList = lib.genericClosure { + # startSet = # + # lib.concatLists (lib.mapAttrsToList + # (name: rds: map (rd: { inherit name rd; key = "${name} <--- ${rd}"; }) rds) + # revDeps); + # operator = item: map (rd: { inherit (item) name; inherit rd; key = "${item.name} <--- ${rd}";}) revDeps.${item.rd} or []; + # }; + # in lib.zipAttrs ( + # map + # (item: { ${item.name} = item.rd; }) + # clList + # );