Support incremental builds with CMake

When CMake builds a project, it generates a file called `CMakeCache.txt` that contains - among others - a dump of environmental variables. This file is generated both for the deps-only- and the main- derivation, and if env-vars happen to differ between both¹, CMake will refuse to compile the latter derivation, saying: ``` CMake Error: The current CMakeCache.txt directory ... is different than the directory ... where CMakeCache.txt was created. ``` Solving this problem is easy - one just has to remove `CMakeCache.txt` to let CMake regenerate it. Note that the original bug report (linked below) suggests to use `sed`, but that doesn't work on Darwin which uses a different convention for temporary paths - compare error messages for the failing derivations: ... on Linux: ``` CMake Error: The current CMakeCache.txt directory /build/source/target/release/build/oqs-sys-a19e222d9ef825d3/out/build/CMakeCache.txt is different than the directory /build/dummy-src/target/release/build/oqs-sys-a19e222d9ef825d3/out/build where CMakeCache.txt was created. This may result in binaries being created in the wrong place. If you are not sure, reedit the CMakeCache.txt ``` ... and on Darwin: ``` CMake Error: The current CMakeCache.txt directory /tmp/nix-build-app-0.1.0.drv-5/source/target/release/build/oqs-sys-4b5e25b1671aa1e5/out/build/CMakeCache.txt is different than the directory /tmp/nix-build-app-deps-0.1.0.drv-0/source/target/release/build/oqs-sys-4b5e25b1671aa1e5/out/build where CMakeCache.txt was created. This may result in binaries being created in the wrong place. If you are not sure, reedit the CMakeCache.txt ``` As for the implementation, since two-step builds are a Naersk-specific construct, I think solving this transparently on Naersk's side instead of just adding a tip into README.md is the better approach here, since it keeps Naersk plug-and-play'able. Closes #285. ¹ which is not that difficult, e.g. that's how `pkgs.rustPlatform.bindgenHook` behaves
nix-community · May 6, 2023 · ebd830b · ebd830b
1 parent 88cd223
commit ebd830b
Show file tree

Hide file tree

Showing 13 changed files with 610 additions and 89 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -8,7 +8,7 @@ jobs:
   linux:
     strategy:
         matrix:
-            nixpkgs: [ nixpkgs, nixpkgs-21.05, nixpkgs-21.11 ]
+            nixpkgs: [ nixpkgs ]
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2

diff --git a/default.nix b/default.nix
@@ -61,13 +61,33 @@ let
       # the top-level build
       buildTopLevel =
         let
-          drv =
+          drv' =
             build
               {
                 pname = config.packageName;
                 inherit (config) userAttrs src;
                 builtDependencies = lib.optional (! config.isSingleStep) buildDeps;
               };
+
+          # If the project we're building uses CMake, let's proactively get rid
+          # of `CMakeCache.txt`.
+          #
+          # That's because this file contains a dump of environmental variables
+          # from the deps-only derivation and it could happen that the main
+          # derivation uses different env-vars - when this happens, CMake will
+          # fail, saying:
+          #
+          # ```
+          # CMake Error: The current CMakeCache.txt directory ... is different than the directory ... where CMakeCache.txt was created.
+          # ```
+          drv = drv'.overrideAttrs (attrs: attrs // {
+            preBuild = (attrs.preBuild or "") + ''
+              find \
+                  -name CMakeCache.txt \
+                  -exec rm {} \;
+            '';
+          });
+
         in
           drv.overrideAttrs config.overrideMain;
     in

diff --git a/nix/sources.json b/nix/sources.json
@@ -1,14 +1,26 @@
 {
     "agent-rs": {
-        "branch": "paulliu/add-cargo-lock",
+        "branch": "main",
         "description": "A collection of libraries and tools for building software around the Internet Computer, in Rust.",
         "homepage": "https://sdk.dfinity.org/",
-        "owner": "ninegua",
+        "owner": "dfinity",
         "repo": "agent-rs",
-        "rev": "4a22e590516bc79ec3c75a320f7941e7762ea098",
-        "sha256": "0sacddc34nlfgldqghlwchgzjki177h5dsgpmdv70cm8hfy0sg7l",
+        "rev": "71987142bb94a55ac0cb74a52396e01b6a142a16",
+        "sha256": "0hv3cg3f8jl0qpzm1qfrg1x7vbcqqv0mzjrgmh4g3w6f8xfj68is",
+        "type": "tarball",
+        "url": "https://github.com/dfinity/agent-rs/archive/71987142bb94a55ac0cb74a52396e01b6a142a16.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "fenix": {
+        "branch": "main",
+        "description": "Rust toolchains and rust-analyzer nightly for Nix [maintainer=@figsoda]",
+        "homepage": "",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "65fdcbdc0bf35510a013d8a0883b0fa7a4ecd2a8",
+        "sha256": "1kqhwbd0abxx1hbvvw4ss7whl67p1w780i9f86r453d56qf6nh5g",
         "type": "tarball",
-        "url": "https://github.com/ninegua/agent-rs/archive/4a22e590516bc79ec3c75a320f7941e7762ea098.tar.gz",
+        "url": "https://github.com/nix-community/fenix/archive/65fdcbdc0bf35510a013d8a0883b0fa7a4ecd2a8.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "lorri": {
@@ -53,10 +65,10 @@
         "homepage": null,
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "6c2318e451fc0eaf338fb461d9bfcc99869de758",
-        "sha256": "1djwkvjnn26v0xw1swwh7dgmi0yl0b1kb8kansrdwk0jhhqbl7zq",
+        "rev": "89abc99d5b6351b2984c6126ef3bda7929905cb9",
+        "sha256": "088n804fbf2p4gd22nymlgflx14fnknns0xavfc5w2c3vmmk7ql1",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/6c2318e451fc0eaf338fb461d9bfcc99869de758.tar.gz",
+        "url": "https://github.com/NixOS/nixpkgs/archive/89abc99d5b6351b2984c6126ef3bda7929905cb9.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "nixpkgs-21.05": {
@@ -83,28 +95,16 @@
         "url": "https://github.com/NixOS/nixpkgs/archive/432864f33c84af53192d4b23ee5871697e57f094.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
-    "nixpkgs-mozilla": {
-        "branch": "master",
-        "description": "Mozilla overlay for Nixpkgs.",
-        "homepage": "",
-        "owner": "mozilla",
-        "repo": "nixpkgs-mozilla",
-        "rev": "85eb0ba7d8e5d6d4b79e5b0180aadbdd25d76404",
-        "sha256": "15a7zd7nrnfgjzs8gq2cpkxg7l3c38jradkxxyaf136kkqhlc0k4",
-        "type": "tarball",
-        "url": "https://github.com/mozilla/nixpkgs-mozilla/archive/85eb0ba7d8e5d6d4b79e5b0180aadbdd25d76404.tar.gz",
-        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
-    },
     "nushell": {
         "branch": "main",
         "description": "A new type of shell",
         "homepage": "https://www.nushell.sh/",
         "owner": "nushell",
         "repo": "nushell",
-        "rev": "85bfdca578157072e51e6972d370cfe63b0fda77",
-        "sha256": "0virlm3wpzsavdlci725iw0f8amr6ijxg9p2xp2q41zngjgb3w98",
+        "rev": "6dc7ff2335c9547182094451cd40c19e66c0678b",
+        "sha256": "1hxrsh7bpi3x7fr8n6589iaak96svncd6ck2sq6d5smjlbjryn1c",
         "type": "tarball",
-        "url": "https://github.com/nushell/nushell/archive/85bfdca578157072e51e6972d370cfe63b0fda77.tar.gz",
+        "url": "https://github.com/nushell/nushell/archive/6dc7ff2335c9547182094451cd40c19e66c0678b.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "ripgrep-all": {

diff --git a/nix/sources.nix b/nix/sources.nix
@@ -10,29 +10,50 @@ let
     let
       name' = sanitizeName name + "-src";
     in
-      if spec.builtin or true then
-        builtins_fetchurl { inherit (spec) url sha256; name = name'; }
-      else
-        pkgs.fetchurl { inherit (spec) url sha256; name = name'; };
+    if spec.builtin or true then
+      builtins_fetchurl { inherit (spec) url sha256; name = name'; }
+    else
+      pkgs.fetchurl { inherit (spec) url sha256; name = name'; };
 
   fetch_tarball = pkgs: name: spec:
     let
       name' = sanitizeName name + "-src";
     in
-      if spec.builtin or true then
-        builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
-      else
-        pkgs.fetchzip { name = name'; inherit (spec) url sha256; };
+    if spec.builtin or true then
+      builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
+    else
+      pkgs.fetchzip { name = name'; inherit (spec) url sha256; };
 
   fetch_git = name: spec:
     let
       ref =
-        if spec ? ref then spec.ref else
+        spec.ref or (
           if spec ? branch then "refs/heads/${spec.branch}" else
-            if spec ? tag then "refs/tags/${spec.tag}" else
-              abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!";
+          if spec ? tag then "refs/tags/${spec.tag}" else
+          abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!"
+        );
+      submodules = spec.submodules or false;
+      submoduleArg =
+        let
+          nixSupportsSubmodules = builtins.compareVersions builtins.nixVersion "2.4" >= 0;
+          emptyArgWithWarning =
+            if submodules
+            then
+              builtins.trace
+                (
+                  "The niv input \"${name}\" uses submodules "
+                  + "but your nix's (${builtins.nixVersion}) builtins.fetchGit "
+                  + "does not support them"
+                )
+                { }
+            else { };
+        in
+        if nixSupportsSubmodules
+        then { inherit submodules; }
+        else emptyArgWithWarning;
     in
-      builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; };
+    builtins.fetchGit
+      ({ url = spec.repo; inherit (spec) rev; inherit ref; } // submoduleArg);
 
   fetch_local = spec: spec.path;
 
@@ -66,16 +87,16 @@ let
       hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
       hasThisAsNixpkgsPath = <nixpkgs> == ./.;
     in
-      if builtins.hasAttr "nixpkgs" sources
-      then sourcesNixpkgs
-      else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
-        import <nixpkgs> {}
-      else
-        abort
-          ''
-            Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
-            add a package called "nixpkgs" to your sources.json.
-          '';
+    if builtins.hasAttr "nixpkgs" sources
+    then sourcesNixpkgs
+    else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
+      import <nixpkgs> { }
+    else
+      abort
+        ''
+          Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
+          add a package called "nixpkgs" to your sources.json.
+        '';
 
   # The actual fetching function.
   fetch = pkgs: name: spec:
@@ -95,13 +116,13 @@ let
   # the path directly as opposed to the fetched source.
   replace = name: drv:
     let
-      saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name;
+      saneName = stringAsChars (c: if (builtins.match "[a-zA-Z0-9]" c) == null then "_" else c) name;
       ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}";
     in
-      if ersatz == "" then drv else
-        # this turns the string into an actual Nix path (for both absolute and
-        # relative paths)
-        if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}";
+    if ersatz == "" then drv else
+      # this turns the string into an actual Nix path (for both absolute and
+      # relative paths)
+    if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}";
 
   # Ports of functions for older nix versions
 
@@ -112,7 +133,7 @@ let
   );
 
   # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295
-  range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1);
+  range = first: last: if first > last then [ ] else builtins.genList (n: first + n) (last - first + 1);
 
   # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257
   stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1));
@@ -123,43 +144,46 @@ let
   concatStrings = builtins.concatStringsSep "";
 
   # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331
-  optionalAttrs = cond: as: if cond then as else {};
+  optionalAttrs = cond: as: if cond then as else { };
 
   # fetchTarball version that is compatible between all the versions of Nix
   builtins_fetchTarball = { url, name ? null, sha256 }@attrs:
     let
       inherit (builtins) lessThan nixVersion fetchTarball;
     in
-      if lessThan nixVersion "1.12" then
-        fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
-      else
-        fetchTarball attrs;
+    if lessThan nixVersion "1.12" then
+      fetchTarball ({ inherit url; } // (optionalAttrs (name != null) { inherit name; }))
+    else
+      fetchTarball attrs;
 
   # fetchurl version that is compatible between all the versions of Nix
   builtins_fetchurl = { url, name ? null, sha256 }@attrs:
     let
       inherit (builtins) lessThan nixVersion fetchurl;
     in
-      if lessThan nixVersion "1.12" then
-        fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
-      else
-        fetchurl attrs;
+    if lessThan nixVersion "1.12" then
+      fetchurl ({ inherit url; } // (optionalAttrs (name != null) { inherit name; }))
+    else
+      fetchurl attrs;
 
   # Create the final "sources" from the config
   mkSources = config:
-    mapAttrs (
-      name: spec:
-        if builtins.hasAttr "outPath" spec
-        then abort
-          "The values in sources.json should not have an 'outPath' attribute"
-        else
-          spec // { outPath = replace name (fetch config.pkgs name spec); }
-    ) config.sources;
+    mapAttrs
+      (
+        name: spec:
+          if builtins.hasAttr "outPath" spec
+          then
+            abort
+              "The values in sources.json should not have an 'outPath' attribute"
+          else
+            spec // { outPath = replace name (fetch config.pkgs name spec); }
+      )
+      config.sources;
 
   # The "config" used by the fetchers
   mkConfig =
     { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null
-    , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile)
+    , sources ? if sourcesFile == null then { } else builtins.fromJSON (builtins.readFile sourcesFile)
     , system ? builtins.currentSystem
     , pkgs ? mkPkgs sources system
     }: rec {
@@ -171,4 +195,4 @@ let
     };
 
 in
-mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }
+mkSources (mkConfig { }) // { __functor = _: settings: mkSources (mkConfig settings); }
diff --git a/test/default.nix b/test/default.nix
@@ -2,9 +2,36 @@
 let
   sources = import ../nix/sources.nix;
 
-  pkgs = import ../nix {
-    inherit system nixpkgs;
-  };
+  pkgs =
+    let
+      pkgs' = import ../nix {
+        inherit system nixpkgs;
+      };
+
+      older-pkgs = import ../nix {
+        inherit system;
+
+        nixpkgs = "nixpkgs-21.05";
+      };
+
+    in
+    pkgs' // {
+      # HACK Some of our tests here manually construct a Git repository which
+      #      newer Git versions have a problem with, saying:
+      #
+      # > fatal: detected dubious ownership in repository at '/nix/store/...'
+      # > To add an exception for this directory, call:
+      # >
+      # >     git config --global --add safe.directory /nix/store/...
+      #
+      # Unfortunately, adding an exception doesn't really seem to work and so
+      # for the testing purposes here we default to the older Git.
+      #
+      # Another approach would be to patch Git (commenting out that warning),
+      # but then `./script/test --nixpkgs ...` would stop working (since that
+      # patch wouldn't be applicable to the older Git versions).
+      git = older-pkgs.git;
+    };
 
   naersk = pkgs.callPackage ../default.nix {
     inherit (pkgs.rustPackages) cargo rustc;

diff --git a/test/slow/agent-rs/default.nix b/test/slow/agent-rs/default.nix
@@ -2,7 +2,6 @@
 
 naersk.buildPackage {
   src = sources.agent-rs;
-  doCheck = true;
 
   buildInputs = [
     pkgs.openssl

diff --git a/test/slow/default.nix b/test/slow/default.nix
@@ -1,8 +1,15 @@
 args: {
   agent-rs = import ./agent-rs args;
   lorri = import ./lorri args;
-  nushell = import ./nushell args;
   ripgrep-all = import ./ripgrep-all args;
   rustlings = import ./rustlings args;
   talent-plan = import ./talent-plan args;
+
+  # Make sure we can compile projects with a complex workspace setup
+  # (https://github.com/nix-community/naersk/issues/274)
+  nushell = import ./nushell args;
+
+  # Make sure we can compile projects that use CMake
+  # (https://github.com/nix-community/naersk/issues/285)
+  oqs = import ./oqs args;
 }