From 5987fb7459e42ee970b22a9e7d896fc415321743 Mon Sep 17 00:00:00 2001 From: squalus Date: Tue, 4 Oct 2022 00:47:43 -0700 Subject: [PATCH 001/284] Add fsync-store-paths option - Add recursiveSync function to flush a directory tree to disk - Add AutoCloseFD::startFsync to initiate an asynchronous fsync without waiting for the result - Initiate an asynchronous fsync while extracting NAR files - Implement the fsync-store-paths option in LocalStore --- src/libstore/globals.hh | 7 ++++++ src/libstore/local-store.cc | 23 +++++++++++++++---- src/libutil/archive.cc | 11 ++++++++-- src/libutil/archive.hh | 2 +- src/libutil/filesystem.cc | 44 +++++++++++++++++++++++++++++++++++++ src/libutil/util.cc | 14 +++++++++++- src/libutil/util.hh | 11 ++++++++-- 7 files changed, 102 insertions(+), 10 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 274a15dd72f..fd5cce7adb4 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -234,6 +234,13 @@ public: default is `true`. )"}; + Setting fsyncStorePaths{this, false, "fsync-store-paths", + R"( + "Whether to call `fsync()` on store paths before registering them, to + flush them to disk. This improves robustness in case of system crashes, + but reduces performance. The default is `false`. + )"}; + Setting useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal", "Whether SQLite should use WAL mode."}; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b67668e5255..4bbeebc3a65 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1299,7 +1299,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, TeeSource wrapperSource { source, hashSink }; - restorePath(realPath, wrapperSource); + restorePath(realPath, wrapperSource, settings.fsyncStorePaths); auto hashResult = hashSink.finish(); @@ -1342,6 +1342,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, optimisePath(realPath, repair); // FIXME: combine with hashPath() + if (settings.fsyncStorePaths) { + recursiveSync(realPath); + syncParent(realPath); + } + registerValidPath(info); } @@ -1402,7 +1407,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name tempPath = tempDir + "/x"; if (method == FileIngestionMethod::Recursive) - restorePath(tempPath, bothSource); + restorePath(tempPath, bothSource, settings.fsyncStorePaths); else writeFile(tempPath, bothSource); @@ -1434,7 +1439,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name StringSource dumpSource { dump }; /* Restore from the NAR in memory. */ if (method == FileIngestionMethod::Recursive) - restorePath(realPath, dumpSource); + restorePath(realPath, dumpSource, settings.fsyncStorePaths); else writeFile(realPath, dumpSource); } else { @@ -1459,6 +1464,12 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name info.narSize = narHash.second; info.references = references; info.ca = FixedOutputHash { .method = method, .hash = hash }; + + if (settings.fsyncStorePaths) { + recursiveSync(realPath); + syncParent(realPath); + } + registerValidPath(info); } @@ -1491,7 +1502,7 @@ StorePath LocalStore::addTextToStore( autoGC(); - writeFile(realPath, s); + writeFile(realPath, s, 0666, settings.fsyncStorePaths); canonicalisePathMetaData(realPath, {}); @@ -1505,6 +1516,10 @@ StorePath LocalStore::addTextToStore( info.narSize = sink.s.size(); info.references = references; info.ca = TextHash { .hash = hash }; + + if (settings.fsyncStorePaths) + syncParent(realPath); + registerValidPath(info); } diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 0e2b9d12c84..e85fe3d3f6d 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -306,6 +306,9 @@ struct RestoreSink : ParseSink { Path dstPath; AutoCloseFD fd; + bool startFsync; + + explicit RestoreSink(bool startFsync) : startFsync{startFsync} {} void createDirectory(const Path & path) override { @@ -323,6 +326,10 @@ struct RestoreSink : ParseSink void closeRegularFile() override { + /* Initiate an fsync operation without waiting for the result. The real fsync should be run before registering + a store path, but this is a performance optimization to allow the disk write to start early. */ + if (startFsync) + fd.startFsync(); /* Call close explicitly to make sure the error is checked */ fd.close(); } @@ -367,9 +374,9 @@ struct RestoreSink : ParseSink }; -void restorePath(const Path & path, Source & source) +void restorePath(const Path & path, Source & source, bool startFsync) { - RestoreSink sink; + RestoreSink sink { startFsync }; sink.dstPath = path; parseDump(sink, source); } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index e42dea54074..64b3501b67c 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -95,7 +95,7 @@ struct RetrieveRegularNARSink : ParseSink void parseDump(ParseSink & sink, Source & source); -void restorePath(const Path & path, Source & source); +void restorePath(const Path & path, Source & source, bool startFsync = false); /* Read a NAR from 'source' and write it to 'sink'. */ void copyNAR(Source & source, Sink & sink); diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 3a732cff8c1..5666fc809dc 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -1,6 +1,7 @@ #include #include #include +#include #include "finally.hh" #include "util.hh" @@ -170,4 +171,47 @@ void moveFile(const Path & oldName, const Path & newName) } } +void recursiveSync(const Path & path) +{ + /* If it's a file, just fsync and return */ + auto st = lstat(path); + if (S_ISREG(st.st_mode)) { + AutoCloseFD fd = open(path.c_str(), O_RDONLY, 0); + if (!fd) + throw SysError("opening file '%1%'", path); + fd.fsync(); + return; + } + + /* Otherwise, perform a depth-first traversal of the directory and fsync all the files */ + std::deque dirsToEnumerate; + dirsToEnumerate.push_back(path); + std::vector dirsToFsync; + while (!dirsToEnumerate.empty()) { + auto currentDir = dirsToEnumerate.back(); + dirsToEnumerate.pop_back(); + const auto dirEntries = readDirectory(currentDir); + for (const auto& dirEntry : dirEntries) { + auto entryPath = currentDir + "/" + dirEntry.name; + if (dirEntry.type == DT_DIR) { + dirsToEnumerate.emplace_back(std::move(entryPath)); + } else if (dirEntry.type == DT_REG) { + AutoCloseFD fd = open(entryPath.c_str(), O_RDONLY, 0); + if (!fd) + throw SysError("opening file '%1%'", entryPath); + fd.fsync(); + } + } + dirsToFsync.emplace_back(std::move(currentDir)); + } + + /* fsync all the directories */ + for (auto dir = dirsToFsync.rbegin(); dir != dirsToFsync.rend(); ++dir) { + AutoCloseFD fd = open(dir->c_str(), O_RDONLY, 0); + if (!fd) + throw SysError("opening directory '%1%'", *dir); + fd.fsync(); + } +} + } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 993dc1cb683..383288667ce 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -838,7 +839,7 @@ void AutoCloseFD::close() } } -void AutoCloseFD::fsync() +void AutoCloseFD::fsync() const { if (fd != -1) { int result; @@ -853,6 +854,17 @@ void AutoCloseFD::fsync() } +void AutoCloseFD::startFsync() const +{ +#if __linux__ + if (fd != -1) { + /* Ignore failure, since fsync must be run later anyway. This is just a performance optimization. */ + ::sync_file_range(fd, 0, 0, SYNC_FILE_RANGE_WRITE); + } +#endif +} + + AutoCloseFD::operator bool() const { return fd != -1; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 9b149de8073..ea83351a781 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -119,9 +119,12 @@ void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool s void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); -/* Flush a file's parent directory to disk */ +/* Flush a path's parent directory to disk */ void syncParent(const Path & path); +/* Flush a file or entire directory tree to disk */ +void recursiveSync(const Path & path); + /* Read a line from a file descriptor. */ std::string readLine(int fd); @@ -234,7 +237,11 @@ public: explicit operator bool() const; int release(); void close(); - void fsync(); + /* Perform a blocking fsync operation */ + void fsync() const; + /* Asynchronously flush to disk without blocking, if available on the platform. This is just a performance + * optimization, and fsync must be run later even if this is called. */ + void startFsync() const; }; From 2dc7598779178188fddf87e6da940636c9d67b2e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 14 Jun 2024 16:30:34 +0200 Subject: [PATCH 002/284] C API: Add nix_clear_err --- src/libutil-c/nix_api_util.cc | 6 ++++++ src/libutil-c/nix_api_util.h | 29 ++++++++++++++++++++++++++--- tests/unit/libutil/nix_api_util.cc | 3 +++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/libutil-c/nix_api_util.cc b/src/libutil-c/nix_api_util.cc index 0a9b493459c..c976b181577 100644 --- a/src/libutil-c/nix_api_util.cc +++ b/src/libutil-c/nix_api_util.cc @@ -57,6 +57,12 @@ nix_err nix_set_err_msg(nix_c_context * context, nix_err err, const char * msg) return err; } +void nix_clear_err(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; +} + const char * nix_version_get() { return PACKAGE_VERSION; diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index e0ca04e69ec..9a9f1d98d08 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -221,7 +221,9 @@ const char * nix_version_get(); * @param[out] n optional: a pointer to an unsigned int that is set to the * length of the error. * @return nullptr if no error message was ever set, - * a borrowed pointer to the error message otherwise. + * a borrowed pointer to the error message otherwise, which is valid + * until the next call to a Nix function, or until the context is + * destroyed. */ const char * nix_err_msg(nix_c_context * context, const nix_c_context * ctx, unsigned int * n); @@ -282,13 +284,34 @@ nix_err nix_err_code(const nix_c_context * read_context); * * All other use is internal to the API. * - * @param context context to write the error message to, or NULL + * @param context context to write the error message to, required unless C++ exceptions are supported * @param err The error code to set and return - * @param msg The error message to set. + * @param msg The error message to set. This string is copied. * @returns the error code set */ nix_err nix_set_err_msg(nix_c_context * context, nix_err err, const char * msg); +/** + * @brief Clear the error message from a nix context. + * + * This is performed implicitly by all functions that accept a context, so + * this won't be necessary in most cases. + * However, if you want to clear the error message without calling another + * function, you can use this. + * + * Example use case: a higher order function that helps with error handling, + * to make it more robust in the following scenario: + * + * 1. A previous call failed, and the error was caught and handled. + * 2. The context is reused with our error handling helper function. + * 3. The callback passed to the helper function doesn't actually make a call to + * a Nix function. + * 4. The handled error is raised again, from an unrelated call. + * + * This failure can be avoided by clearing the error message after handling it. + */ +void nix_clear_err(nix_c_context * context); + /** * @} */ diff --git a/tests/unit/libutil/nix_api_util.cc b/tests/unit/libutil/nix_api_util.cc index d2999f55b39..052d0c533fa 100644 --- a/tests/unit/libutil/nix_api_util.cc +++ b/tests/unit/libutil/nix_api_util.cc @@ -31,6 +31,9 @@ TEST_F(nix_api_util_context, nix_context_error) } ASSERT_EQ(ctx->last_err_code, NIX_ERR_UNKNOWN); ASSERT_EQ(*ctx->last_err, err_msg_ref); + + nix_clear_err(ctx); + ASSERT_EQ(ctx->last_err_code, NIX_OK); } TEST_F(nix_api_util_context, nix_set_err_msg) From 61381c9964c69afd720264272a0f42d298d6616c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 14 Jun 2024 16:36:23 +0200 Subject: [PATCH 003/284] C API: Make nix_err_msg treat NIX_OK as having no message The documentation "solved" this by specifying a precondition, but let's just make it more robust, and not leak irrelevant messages that might linger. We don't clear the message when clearing the status, in order to keep clearing fast; see last_err field doc. --- src/libutil-c/nix_api_util.cc | 2 +- src/libutil-c/nix_api_util_internal.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libutil-c/nix_api_util.cc b/src/libutil-c/nix_api_util.cc index c976b181577..cf468c8cd18 100644 --- a/src/libutil-c/nix_api_util.cc +++ b/src/libutil-c/nix_api_util.cc @@ -112,7 +112,7 @@ const char * nix_err_msg(nix_c_context * context, const nix_c_context * read_con { if (context) context->last_err_code = NIX_OK; - if (read_context->last_err) { + if (read_context->last_err && read_context->last_err_code != NIX_OK) { if (n) *n = read_context->last_err->size(); return read_context->last_err->c_str(); diff --git a/src/libutil-c/nix_api_util_internal.h b/src/libutil-c/nix_api_util_internal.h index aa829feaf57..7fa4252acfd 100644 --- a/src/libutil-c/nix_api_util_internal.h +++ b/src/libutil-c/nix_api_util_internal.h @@ -10,6 +10,7 @@ struct nix_c_context { nix_err last_err_code = NIX_OK; + /** The last error message. Always check last_err_code. This may not have been cleared, so that clearing is fast. */ std::optional last_err = {}; std::optional info = {}; std::string name = ""; From 36cc8d5f4b221ad27e1a172148363e9780329277 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Apr 2024 15:26:18 +0200 Subject: [PATCH 004/284] Run the flake-regressions test suite --- .github/workflows/ci.yml | 20 ++++++++++++++++++++ scripts/flake-regressions.sh | 27 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100755 scripts/flake-regressions.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1aa3b776ef2..a0235eb0f96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,3 +176,23 @@ jobs: - uses: DeterminateSystems/nix-installer-action@main - uses: DeterminateSystems/magic-nix-cache-action@main - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes + + flake_regressions: + needs: vm_tests + runs-on: ubuntu-22.04 + steps: + - name: Checkout nix + uses: actions/checkout@v4 + - name: Checkout flake-regressions + uses: actions/checkout@v4 + with: + repository: DeterminateSystems/flake-regressions + path: flake-regressions + - name: Checkout flake-regressions-data + uses: actions/checkout@v4 + with: + repository: DeterminateSystems/flake-regressions-data + path: flake-regressions/tests + - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main + - run: nix build --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH scripts/flake-regressions.sh diff --git a/scripts/flake-regressions.sh b/scripts/flake-regressions.sh new file mode 100755 index 00000000000..e6cfbfa24f9 --- /dev/null +++ b/scripts/flake-regressions.sh @@ -0,0 +1,27 @@ +#! /usr/bin/env bash + +set -e + +echo "Nix version:" +nix --version + +cd flake-regressions + +status=0 + +flakes=$(ls -d tests/*/*/* | head -n25) + +echo "Running flake tests..." + +for flake in $flakes; do + + if ! REGENERATE=0 ./eval-flake.sh $flake; then + status=1 + echo "❌ $flake" + else + echo "✅ $flake" + fi + +done + +exit "$status" From 0eec60968ad393281ae9e31f84f2cbc15fe0bc2f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 May 2024 15:58:37 +0200 Subject: [PATCH 005/284] flake-regressions.sh: Make the sort order deterministic --- scripts/flake-regressions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/flake-regressions.sh b/scripts/flake-regressions.sh index e6cfbfa24f9..5cc55bf4f77 100755 --- a/scripts/flake-regressions.sh +++ b/scripts/flake-regressions.sh @@ -9,7 +9,7 @@ cd flake-regressions status=0 -flakes=$(ls -d tests/*/*/* | head -n25) +flakes=$(ls -d tests/*/*/* | sort | head -n25) echo "Running flake tests..." From 6f3d2daee66e8de80faa45e1a30abbabcc08764a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 10 Jun 2024 15:16:41 +0200 Subject: [PATCH 006/284] Fix spellcheck --- scripts/flake-regressions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/flake-regressions.sh b/scripts/flake-regressions.sh index 5cc55bf4f77..d765311345c 100755 --- a/scripts/flake-regressions.sh +++ b/scripts/flake-regressions.sh @@ -9,13 +9,13 @@ cd flake-regressions status=0 -flakes=$(ls -d tests/*/*/* | sort | head -n25) +flakes=$(find tests -mindepth 3 -maxdepth 3 -type d -not -path '*/.*' | sort | head -n25) echo "Running flake tests..." for flake in $flakes; do - if ! REGENERATE=0 ./eval-flake.sh $flake; then + if ! REGENERATE=0 ./eval-flake.sh "$flake"; then status=1 echo "❌ $flake" else From d4a70b67a0d76328644ab975ef0f714b1f28c2cd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 21 Jun 2024 15:38:03 +0200 Subject: [PATCH 007/284] Move flake-regressions repos to the NixOS org --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0235eb0f96..6f0b63eaa44 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -186,12 +186,12 @@ jobs: - name: Checkout flake-regressions uses: actions/checkout@v4 with: - repository: DeterminateSystems/flake-regressions + repository: NixOS/flake-regressions path: flake-regressions - name: Checkout flake-regressions-data uses: actions/checkout@v4 with: - repository: DeterminateSystems/flake-regressions-data + repository: NixOS/flake-regressions-data path: flake-regressions/tests - uses: DeterminateSystems/nix-installer-action@main - uses: DeterminateSystems/magic-nix-cache-action@main From caabdb06d1a8b7d75928b9c35d796a4da5355314 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Tue, 2 Jul 2024 20:55:46 -0500 Subject: [PATCH 008/284] preserve early sequoia migration script --- scripts/sequoia-nixbld-user-migration.sh | 36 ++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 scripts/sequoia-nixbld-user-migration.sh diff --git a/scripts/sequoia-nixbld-user-migration.sh b/scripts/sequoia-nixbld-user-migration.sh new file mode 100644 index 00000000000..778a9568810 --- /dev/null +++ b/scripts/sequoia-nixbld-user-migration.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +((NEW_NIX_FIRST_BUILD_UID=331)) + +id_available(){ + dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null +} + +change_nixbld_names_and_ids(){ + local name uid next_id + ((next_id=NEW_NIX_FIRST_BUILD_UID)) + echo "Attempting to migrate _nixbld users." + echo "Each _nixbld# user should have its UID moved to $next_id+" + while read -r name uid; do + echo " Checking $name (uid: $uid)" + # iterate for a clean ID + while id_available "$next_id"; do + ((next_id++)) + if ((next_id >= 400)); then + echo "We've hit UID 400 without placing all of your users :(" + echo "You should use the commands in this script as a starting" + echo "point to review your UID-space and manually move the" + echo "remaining users (or delete them, if you don't need them)." + exit 1 + fi + done + + # first 2 are cleanup, it's OK if they aren't here + sudo dscl . delete "/Users/$name" dsAttrTypeNative:_writers_passwd &>/dev/null || true + sudo dscl . change "/Users/$name" NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true + sudo dscl . change "/Users/$name" UniqueID "$uid" "$next_id" + echo " $name migrated to uid: $next_id" + done < <(dscl . list /Users UniqueID | grep _nixbld | sort -n -k2) +} + +change_nixbld_names_and_ids From d63bd8295e05b31dd8dbf85c91a8a782e471a383 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 5 Jul 2024 16:43:48 +0200 Subject: [PATCH 009/284] assert: Report why values aren't equal --- src/libexpr/eval.cc | 232 +++++++++++++++++- src/libexpr/eval.hh | 9 + ...al-fail-assert-equal-attrs-names-2.err.exp | 8 + .../eval-fail-assert-equal-attrs-names-2.nix | 2 + ...eval-fail-assert-equal-attrs-names.err.exp | 8 + .../eval-fail-assert-equal-attrs-names.nix | 2 + ...ail-assert-equal-derivations-extra.err.exp | 26 ++ ...al-fail-assert-equal-derivations-extra.nix | 5 + ...eval-fail-assert-equal-derivations.err.exp | 26 ++ .../eval-fail-assert-equal-derivations.nix | 5 + .../eval-fail-assert-equal-floats.err.exp | 22 ++ .../lang/eval-fail-assert-equal-floats.nix | 2 + ...-fail-assert-equal-function-direct.err.exp | 9 + ...eval-fail-assert-equal-function-direct.nix | 7 + .../eval-fail-assert-equal-int-float.err.exp | 8 + .../lang/eval-fail-assert-equal-int-float.nix | 2 + .../lang/eval-fail-assert-equal-ints.err.exp | 22 ++ .../lang/eval-fail-assert-equal-ints.nix | 2 + ...eval-fail-assert-equal-list-length.err.exp | 8 + .../eval-fail-assert-equal-list-length.nix | 2 + .../lang/eval-fail-assert-equal-paths.err.exp | 8 + .../lang/eval-fail-assert-equal-paths.nix | 2 + ...eval-fail-assert-equal-type-nested.err.exp | 22 ++ .../eval-fail-assert-equal-type-nested.nix | 2 + .../lang/eval-fail-assert-equal-type.err.exp | 8 + .../lang/eval-fail-assert-equal-type.nix | 2 + .../lang/eval-fail-assert-nested-bool.err.exp | 74 ++++++ .../lang/eval-fail-assert-nested-bool.nix | 6 + .../functional/lang/eval-fail-assert.err.exp | 6 +- 29 files changed, 532 insertions(+), 5 deletions(-) create mode 100644 tests/functional/lang/eval-fail-assert-equal-attrs-names-2.err.exp create mode 100644 tests/functional/lang/eval-fail-assert-equal-attrs-names-2.nix create mode 100644 tests/functional/lang/eval-fail-assert-equal-attrs-names.err.exp create mode 100644 tests/functional/lang/eval-fail-assert-equal-attrs-names.nix create mode 100644 tests/functional/lang/eval-fail-assert-equal-derivations-extra.err.exp create mode 100644 tests/functional/lang/eval-fail-assert-equal-derivations-extra.nix create mode 100644 tests/functional/lang/eval-fail-assert-equal-derivations.err.exp create mode 100644 tests/functional/lang/eval-fail-assert-equal-derivations.nix create mode 100644 tests/functional/lang/eval-fail-assert-equal-floats.err.exp create mode 100644 tests/functional/lang/eval-fail-assert-equal-floats.nix create mode 100644 tests/functional/lang/eval-fail-assert-equal-function-direct.err.exp create mode 100644 tests/functional/lang/eval-fail-assert-equal-function-direct.nix create mode 100644 tests/functional/lang/eval-fail-assert-equal-int-float.err.exp create mode 100644 tests/functional/lang/eval-fail-assert-equal-int-float.nix create mode 100644 tests/functional/lang/eval-fail-assert-equal-ints.err.exp create mode 100644 tests/functional/lang/eval-fail-assert-equal-ints.nix create mode 100644 tests/functional/lang/eval-fail-assert-equal-list-length.err.exp create mode 100644 tests/functional/lang/eval-fail-assert-equal-list-length.nix create mode 100644 tests/functional/lang/eval-fail-assert-equal-paths.err.exp create mode 100644 tests/functional/lang/eval-fail-assert-equal-paths.nix create mode 100644 tests/functional/lang/eval-fail-assert-equal-type-nested.err.exp create mode 100644 tests/functional/lang/eval-fail-assert-equal-type-nested.nix create mode 100644 tests/functional/lang/eval-fail-assert-equal-type.err.exp create mode 100644 tests/functional/lang/eval-fail-assert-equal-type.nix create mode 100644 tests/functional/lang/eval-fail-assert-nested-bool.err.exp create mode 100644 tests/functional/lang/eval-fail-assert-nested-bool.nix diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d2be00e55f1..fb6050e5077 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1759,9 +1759,24 @@ void ExprIf::eval(EvalState & state, Env & env, Value & v) void ExprAssert::eval(EvalState & state, Env & env, Value & v) { if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) { - std::ostringstream out; - cond->show(state.symbols, out); - state.error("assertion '%1%' failed", out.str()).atPos(pos).withFrame(env, *this).debugThrow(); + auto exprStr = ({ + std::ostringstream out; + cond->show(state.symbols, out); + out.str(); + }); + + if (auto eq = dynamic_cast(cond)) { + try { + Value v1; eq->e1->eval(state, env, v1); + Value v2; eq->e2->eval(state, env, v2); + state.assertEqValues(v1, v2, eq->pos, "in an equality assertion"); + } catch (AssertionError & e) { + e.addTrace(state.positions[pos], "while evaluating the condition of the assertion '%s'", exprStr); + throw; + } + } + + state.error("assertion '%1%' failed", exprStr).atPos(pos).withFrame(env, *this).debugThrow(); } body->eval(state, env, v); } @@ -2418,6 +2433,216 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & } + +// NOTE: This implementation must match eqValues! +// We accept this burden because informative error messages for +// `assert a == b; x` are critical for our users' testing UX. +void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx) +{ + // This implementation must match eqValues. + forceValue(v1, pos); + forceValue(v2, pos); + + if (&v1 == &v2) + return; + + // Special case type-compatibility between float and int + if ((v1.type() == nInt || v1.type() == nFloat) && (v2.type() == nInt || v2.type() == nFloat)) { + if (eqValues(v1, v2, pos, errorCtx)) { + return; + } else { + error( + "%s with value '%s' is not equal to %s with value '%s'", + showType(v1), + ValuePrinter(*this, v1, errorPrintOptions), + showType(v2), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + } + + if (v1.type() != v2.type()) { + error( + "%s of value '%s' is not equal to %s of value '%s'", + showType(v1), + ValuePrinter(*this, v1, errorPrintOptions), + showType(v2), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + + switch (v1.type()) { + case nInt: + if (v1.integer() != v2.integer()) { + error("integer '%d' is not equal to integer '%d'", v1.integer(), v2.integer()).debugThrow(); + } + return; + + case nBool: + if (v1.boolean() != v2.boolean()) { + error( + "boolean '%s' is not equal to boolean '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + return; + + case nString: + if (strcmp(v1.c_str(), v2.c_str()) != 0) { + error( + "string '%s' is not equal to string '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + return; + + case nPath: + if (v1.payload.path.accessor != v2.payload.path.accessor) { + error( + "path '%s' is not equal to path '%s' because their accessors are different", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + if (strcmp(v1.payload.path.path, v2.payload.path.path) != 0) { + error( + "path '%s' is not equal to path '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + return; + + case nNull: + return; + + case nList: + if (v1.listSize() != v2.listSize()) { + error( + "list of size '%d' is not equal to list of size '%d', left hand side is '%s', right hand side is '%s'", + v1.listSize(), + v2.listSize(), + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + for (size_t n = 0; n < v1.listSize(); ++n) { + try { + assertEqValues(*v1.listElems()[n], *v2.listElems()[n], pos, errorCtx); + } catch (Error & e) { + e.addTrace(positions[pos], "while comparing list element %d", n); + throw; + } + } + return; + + case nAttrs: { + if (isDerivation(v1) && isDerivation(v2)) { + auto i = v1.attrs()->get(sOutPath); + auto j = v2.attrs()->get(sOutPath); + if (i && j) { + try { + assertEqValues(*i->value, *j->value, pos, errorCtx); + return; + } catch (Error & e) { + e.addTrace(positions[pos], "while comparing a derivation by its '%s' attribute", "outPath"); + throw; + } + assert(false); + } + } + + if (v1.attrs()->size() != v2.attrs()->size()) { + error( + "attribute names of attribute set '%s' differs from attribute set '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + + // Like normal comparison, we compare the attributes in non-deterministic Symbol index order. + // This function is called when eqValues has found a difference, so to reliably + // report about its result, we should follow in its literal footsteps and not + // try anything fancy that could lead to an error. + Bindings::const_iterator i, j; + for (i = v1.attrs()->begin(), j = v2.attrs()->begin(); i != v1.attrs()->end(); ++i, ++j) { + if (i->name != j->name) { + // A difference in a sorted list means that one attribute is not contained in the other, but we don't + // know which. Let's find out. Could use <, but this is more clear. + if (!v2.attrs()->get(i->name)) { + error( + "attribute name '%s' is contained in '%s', but not in '%s'", + symbols[i->name], + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + if (!v1.attrs()->get(j->name)) { + error( + "attribute name '%s' is missing in '%s', but is contained in '%s'", + symbols[j->name], + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + assert(false); + } + try { + assertEqValues(*i->value, *j->value, pos, errorCtx); + } catch (Error & e) { + // The order of traces is reversed, so this presents as + // where left hand side is + // at + // where right hand side is + // at + // while comparing attribute '' + if (j->pos != noPos) + e.addTrace(positions[j->pos], "where right hand side is"); + if (i->pos != noPos) + e.addTrace(positions[i->pos], "where left hand side is"); + e.addTrace(positions[pos], "while comparing attribute '%s'", symbols[i->name]); + throw; + } + } + return; + } + + case nFunction: + error("distinct functions and immediate comparisons of identical functions compare as unequal") + .debugThrow(); + + case nExternal: + if (!(*v1.external() == *v2.external())) { + error( + "external value '%s' is not equal to external value '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + return; + + case nFloat: + // !!! + if (!(v1.fpoint() == v2.fpoint())) { + error("float '%f' is not equal to float '%f'", v1.fpoint(), v2.fpoint()).debugThrow(); + } + return; + + case nThunk: // Must not be left by forceValue + default: + // This should never happen, because eqValues already throws an + // error for this, and this function should only be called when + // eqValues has found a difference, and it should match + // its behavior. + error( + "cannot compare %1% with %2%; is assertEqValues out of sync with eqValues?", showType(v1), showType(v2)) + .debugThrow(); + } +} + +// This implementation must match assertEqValues bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx) { forceValue(v1, pos); @@ -2491,6 +2716,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return *v1.external() == *v2.external(); case nFloat: + // !!! return v1.fpoint() == v2.fpoint(); case nThunk: // Must not be left by forceValue diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index b84bc9907c8..df44bed70d3 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -644,6 +644,15 @@ public: */ bool eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx); + /** + * Like `eqValues`, but throws an `AssertionError` if not equal. + * + * WARNING: + * Callers should call `eqValues` first and report if `assertEqValues` behaves + * incorrectly. (e.g. if it doesn't throw if eqValues returns false or vice versa) + */ + void assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx); + bool isFunctor(Value & fun); // FIXME: use std::span diff --git a/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.err.exp b/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.err.exp new file mode 100644 index 00000000000..4b68d97c20c --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '({ a = true; } == { a = true; b = true; })' + at /pwd/lang/eval-fail-assert-equal-attrs-names-2.nix:1:1: + 1| assert { a = true; } == { a = true; b = true; }; + | ^ + 2| throw "unreachable" + + error: attribute names of attribute set '{ a = true; }' differs from attribute set '{ a = true; b = true; }' diff --git a/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.nix b/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.nix new file mode 100644 index 00000000000..8e7ac9cf2be --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.nix @@ -0,0 +1,2 @@ +assert { a = true; } == { a = true; b = true; }; +throw "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-attrs-names.err.exp b/tests/functional/lang/eval-fail-assert-equal-attrs-names.err.exp new file mode 100644 index 00000000000..bc61ca63a27 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-attrs-names.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '({ a = true; b = true; } == { a = true; })' + at /pwd/lang/eval-fail-assert-equal-attrs-names.nix:1:1: + 1| assert { a = true; b = true; } == { a = true; }; + | ^ + 2| throw "unreachable" + + error: attribute names of attribute set '{ a = true; b = true; }' differs from attribute set '{ a = true; }' diff --git a/tests/functional/lang/eval-fail-assert-equal-attrs-names.nix b/tests/functional/lang/eval-fail-assert-equal-attrs-names.nix new file mode 100644 index 00000000000..e2f53a85ad6 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-attrs-names.nix @@ -0,0 +1,2 @@ +assert { a = true; b = true; } == { a = true; }; +throw "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-derivations-extra.err.exp b/tests/functional/lang/eval-fail-assert-equal-derivations-extra.err.exp new file mode 100644 index 00000000000..7f49240747c --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-derivations-extra.err.exp @@ -0,0 +1,26 @@ +error: + … while evaluating the condition of the assertion '({ foo = { outPath = "/nix/store/0"; type = "derivation"; }; } == { foo = { devious = true; outPath = "/nix/store/1"; type = "derivation"; }; })' + at /pwd/lang/eval-fail-assert-equal-derivations-extra.nix:1:1: + 1| assert + | ^ + 2| { foo = { type = "derivation"; outPath = "/nix/store/0"; }; } + + … while comparing attribute 'foo' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-derivations-extra.nix:2:5: + 1| assert + 2| { foo = { type = "derivation"; outPath = "/nix/store/0"; }; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-derivations-extra.nix:4:5: + 3| == + 4| { foo = { type = "derivation"; outPath = "/nix/store/1"; devious = true; }; }; + | ^ + 5| throw "unreachable" + + … while comparing a derivation by its 'outPath' attribute + + error: string '"/nix/store/0"' is not equal to string '"/nix/store/1"' diff --git a/tests/functional/lang/eval-fail-assert-equal-derivations-extra.nix b/tests/functional/lang/eval-fail-assert-equal-derivations-extra.nix new file mode 100644 index 00000000000..fd8bc3f26ca --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-derivations-extra.nix @@ -0,0 +1,5 @@ +assert + { foo = { type = "derivation"; outPath = "/nix/store/0"; }; } + == + { foo = { type = "derivation"; outPath = "/nix/store/1"; devious = true; }; }; +throw "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-derivations.err.exp b/tests/functional/lang/eval-fail-assert-equal-derivations.err.exp new file mode 100644 index 00000000000..d7f0face077 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-derivations.err.exp @@ -0,0 +1,26 @@ +error: + … while evaluating the condition of the assertion '({ foo = { ignored = (abort "not ignored"); outPath = "/nix/store/0"; type = "derivation"; }; } == { foo = { ignored = (abort "not ignored"); outPath = "/nix/store/1"; type = "derivation"; }; })' + at /pwd/lang/eval-fail-assert-equal-derivations.nix:1:1: + 1| assert + | ^ + 2| { foo = { type = "derivation"; outPath = "/nix/store/0"; ignored = abort "not ignored"; }; } + + … while comparing attribute 'foo' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-derivations.nix:2:5: + 1| assert + 2| { foo = { type = "derivation"; outPath = "/nix/store/0"; ignored = abort "not ignored"; }; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-derivations.nix:4:5: + 3| == + 4| { foo = { type = "derivation"; outPath = "/nix/store/1"; ignored = abort "not ignored"; }; }; + | ^ + 5| throw "unreachable" + + … while comparing a derivation by its 'outPath' attribute + + error: string '"/nix/store/0"' is not equal to string '"/nix/store/1"' diff --git a/tests/functional/lang/eval-fail-assert-equal-derivations.nix b/tests/functional/lang/eval-fail-assert-equal-derivations.nix new file mode 100644 index 00000000000..c648eae374b --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-derivations.nix @@ -0,0 +1,5 @@ +assert + { foo = { type = "derivation"; outPath = "/nix/store/0"; ignored = abort "not ignored"; }; } + == + { foo = { type = "derivation"; outPath = "/nix/store/1"; ignored = abort "not ignored"; }; }; +throw "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-floats.err.exp b/tests/functional/lang/eval-fail-assert-equal-floats.err.exp new file mode 100644 index 00000000000..d8545e2db9e --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-floats.err.exp @@ -0,0 +1,22 @@ +error: + … while evaluating the condition of the assertion '({ b = 1; } == { b = 1.01; })' + at /pwd/lang/eval-fail-assert-equal-floats.nix:1:1: + 1| assert { b = 1.0; } == { b = 1.01; }; + | ^ + 2| abort "unreachable" + + … while comparing attribute 'b' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-floats.nix:1:10: + 1| assert { b = 1.0; } == { b = 1.01; }; + | ^ + 2| abort "unreachable" + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-floats.nix:1:26: + 1| assert { b = 1.0; } == { b = 1.01; }; + | ^ + 2| abort "unreachable" + + error: a float with value '1' is not equal to a float with value '1.01' diff --git a/tests/functional/lang/eval-fail-assert-equal-floats.nix b/tests/functional/lang/eval-fail-assert-equal-floats.nix new file mode 100644 index 00000000000..438e85abfab --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-floats.nix @@ -0,0 +1,2 @@ +assert { b = 1.0; } == { b = 1.01; }; +abort "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-function-direct.err.exp b/tests/functional/lang/eval-fail-assert-equal-function-direct.err.exp new file mode 100644 index 00000000000..f06d796981b --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-function-direct.err.exp @@ -0,0 +1,9 @@ +error: + … while evaluating the condition of the assertion '((x: x) == (x: x))' + at /pwd/lang/eval-fail-assert-equal-function-direct.nix:3:1: + 2| # This only compares a direct comparison and makes no claims about functions in nested structures. + 3| assert + | ^ + 4| (x: x) + + error: distinct functions and immediate comparisons of identical functions compare as unequal diff --git a/tests/functional/lang/eval-fail-assert-equal-function-direct.nix b/tests/functional/lang/eval-fail-assert-equal-function-direct.nix new file mode 100644 index 00000000000..68e5e390823 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-function-direct.nix @@ -0,0 +1,7 @@ +# Note: functions in nested structures, e.g. attributes, may be optimized away by pointer identity optimization. +# This only compares a direct comparison and makes no claims about functions in nested structures. +assert + (x: x) + == + (x: x); +abort "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-int-float.err.exp b/tests/functional/lang/eval-fail-assert-equal-int-float.err.exp new file mode 100644 index 00000000000..c927e38d6d6 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-int-float.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '(1 == 1.1)' + at /pwd/lang/eval-fail-assert-equal-int-float.nix:1:1: + 1| assert 1 == 1.1; + | ^ + 2| throw "unreachable" + + error: an integer with value '1' is not equal to a float with value '1.1' diff --git a/tests/functional/lang/eval-fail-assert-equal-int-float.nix b/tests/functional/lang/eval-fail-assert-equal-int-float.nix new file mode 100644 index 00000000000..1dfdf2bdac6 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-int-float.nix @@ -0,0 +1,2 @@ +assert 1 == 1.1; +throw "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-ints.err.exp b/tests/functional/lang/eval-fail-assert-equal-ints.err.exp new file mode 100644 index 00000000000..d6219e200dd --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-ints.err.exp @@ -0,0 +1,22 @@ +error: + … while evaluating the condition of the assertion '({ b = 1; } == { b = 2; })' + at /pwd/lang/eval-fail-assert-equal-ints.nix:1:1: + 1| assert { b = 1; } == { b = 2; }; + | ^ + 2| abort "unreachable" + + … while comparing attribute 'b' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-ints.nix:1:10: + 1| assert { b = 1; } == { b = 2; }; + | ^ + 2| abort "unreachable" + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-ints.nix:1:24: + 1| assert { b = 1; } == { b = 2; }; + | ^ + 2| abort "unreachable" + + error: an integer with value '1' is not equal to an integer with value '2' diff --git a/tests/functional/lang/eval-fail-assert-equal-ints.nix b/tests/functional/lang/eval-fail-assert-equal-ints.nix new file mode 100644 index 00000000000..645258ea6d1 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-ints.nix @@ -0,0 +1,2 @@ +assert { b = 1; } == { b = 2; }; +abort "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-list-length.err.exp b/tests/functional/lang/eval-fail-assert-equal-list-length.err.exp new file mode 100644 index 00000000000..90108552cf0 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-list-length.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '([ (1) (0) ] == [ (10) ])' + at /pwd/lang/eval-fail-assert-equal-list-length.nix:1:1: + 1| assert [ 1 0 ] == [ 10 ]; + | ^ + 2| throw "unreachable" + + error: list of size '2' is not equal to list of size '1', left hand side is '[ 1 0 ]', right hand side is '[ 10 ]' diff --git a/tests/functional/lang/eval-fail-assert-equal-list-length.nix b/tests/functional/lang/eval-fail-assert-equal-list-length.nix new file mode 100644 index 00000000000..6d40f4d8e83 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-list-length.nix @@ -0,0 +1,2 @@ +assert [ 1 0 ] == [ 10 ]; +throw "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-paths.err.exp b/tests/functional/lang/eval-fail-assert-equal-paths.err.exp new file mode 100644 index 00000000000..66c34e97123 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-paths.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '(/pwd/lang/foo == /pwd/lang/bar)' + at /pwd/lang/eval-fail-assert-equal-paths.nix:1:1: + 1| assert ./foo == ./bar; + | ^ + 2| throw "unreachable" + + error: path '/pwd/lang/foo' is not equal to path '/pwd/lang/bar' diff --git a/tests/functional/lang/eval-fail-assert-equal-paths.nix b/tests/functional/lang/eval-fail-assert-equal-paths.nix new file mode 100644 index 00000000000..ef0b6702466 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-paths.nix @@ -0,0 +1,2 @@ +assert ./foo == ./bar; +throw "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-type-nested.err.exp b/tests/functional/lang/eval-fail-assert-equal-type-nested.err.exp new file mode 100644 index 00000000000..f78badd25f2 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-type-nested.err.exp @@ -0,0 +1,22 @@ +error: + … while evaluating the condition of the assertion '({ ding = false; } == { ding = null; })' + at /pwd/lang/eval-fail-assert-equal-type-nested.nix:1:1: + 1| assert { ding = false; } == { ding = null; }; + | ^ + 2| abort "unreachable" + + … while comparing attribute 'ding' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-type-nested.nix:1:10: + 1| assert { ding = false; } == { ding = null; }; + | ^ + 2| abort "unreachable" + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-type-nested.nix:1:31: + 1| assert { ding = false; } == { ding = null; }; + | ^ + 2| abort "unreachable" + + error: a Boolean of value 'false' is not equal to null of value 'null' diff --git a/tests/functional/lang/eval-fail-assert-equal-type-nested.nix b/tests/functional/lang/eval-fail-assert-equal-type-nested.nix new file mode 100644 index 00000000000..3fbd14ce6bb --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-type-nested.nix @@ -0,0 +1,2 @@ +assert { ding = false; } == { ding = null; }; +abort "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-type.err.exp b/tests/functional/lang/eval-fail-assert-equal-type.err.exp new file mode 100644 index 00000000000..4dc3f2ecec5 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-type.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '(false == null)' + at /pwd/lang/eval-fail-assert-equal-type.nix:1:1: + 1| assert false == null; + | ^ + 2| abort "unreachable" + + error: a Boolean of value 'false' is not equal to null of value 'null' diff --git a/tests/functional/lang/eval-fail-assert-equal-type.nix b/tests/functional/lang/eval-fail-assert-equal-type.nix new file mode 100644 index 00000000000..7023ea007e3 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-type.nix @@ -0,0 +1,2 @@ +assert false == null; +abort "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-nested-bool.err.exp b/tests/functional/lang/eval-fail-assert-nested-bool.err.exp new file mode 100644 index 00000000000..1debb668c98 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-nested-bool.err.exp @@ -0,0 +1,74 @@ +error: + … while evaluating the condition of the assertion '({ a = { b = [ ({ c = { d = true; }; }) ]; }; } == { a = { b = [ ({ c = { d = false; }; }) ]; }; })' + at /pwd/lang/eval-fail-assert-nested-bool.nix:1:1: + 1| assert + | ^ + 2| { a.b = [ { c.d = true; } ]; } + + … while comparing attribute 'a' + + … where left hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:2:5: + 1| assert + 2| { a.b = [ { c.d = true; } ]; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:4:5: + 3| == + 4| { a.b = [ { c.d = false; } ]; }; + | ^ + 5| + + … while comparing attribute 'b' + + … where left hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:2:5: + 1| assert + 2| { a.b = [ { c.d = true; } ]; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:4:5: + 3| == + 4| { a.b = [ { c.d = false; } ]; }; + | ^ + 5| + + … while comparing list element 0 + + … while comparing attribute 'c' + + … where left hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:2:15: + 1| assert + 2| { a.b = [ { c.d = true; } ]; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:4:15: + 3| == + 4| { a.b = [ { c.d = false; } ]; }; + | ^ + 5| + + … while comparing attribute 'd' + + … where left hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:2:15: + 1| assert + 2| { a.b = [ { c.d = true; } ]; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:4:15: + 3| == + 4| { a.b = [ { c.d = false; } ]; }; + | ^ + 5| + + error: boolean 'true' is not equal to boolean 'false' diff --git a/tests/functional/lang/eval-fail-assert-nested-bool.nix b/tests/functional/lang/eval-fail-assert-nested-bool.nix new file mode 100644 index 00000000000..2285769839e --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-nested-bool.nix @@ -0,0 +1,6 @@ +assert + { a.b = [ { c.d = true; } ]; } + == + { a.b = [ { c.d = false; } ]; }; + +abort "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert.err.exp b/tests/functional/lang/eval-fail-assert.err.exp index 0656ec81c1d..7be9e238797 100644 --- a/tests/functional/lang/eval-fail-assert.err.exp +++ b/tests/functional/lang/eval-fail-assert.err.exp @@ -20,9 +20,11 @@ error: | ^ 3| - error: assertion '(arg == "y")' failed - at /pwd/lang/eval-fail-assert.nix:2:12: + … while evaluating the condition of the assertion '(arg == "y")' + at /pwd/lang/eval-fail-assert.nix:2:12: 1| let { 2| x = arg: assert arg == "y"; 123; | ^ 3| + + error: string '"x"' is not equal to string '"y"' From 13522229a9efc83b4a3d90c66445355c6bc7c815 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 10 Jul 2024 16:08:10 +0200 Subject: [PATCH 010/284] assertEqValues: clarify potential bug error message --- src/libexpr/eval.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index fb6050e5077..6f1a7d6186b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2636,8 +2636,11 @@ void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::st // error for this, and this function should only be called when // eqValues has found a difference, and it should match // its behavior. + // Note that as of writing, we make the compiler require that all enum + // values are handled explicitly with `case`s, _despite_ having a + // `default:`. error( - "cannot compare %1% with %2%; is assertEqValues out of sync with eqValues?", showType(v1), showType(v2)) + "BUG: cannot compare %1% with %2%; did forceValue leave a thunk, or might assertEqValues be out of sync with eqValues?", showType(v1), showType(v2)) .debugThrow(); } } From 61577402ba331451a10051b11cf77bdc80f83fa8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 11 Jul 2024 11:35:58 +0200 Subject: [PATCH 011/284] Add EvalErrorBuilder::panic() An nicer alternative to printError + abort, or assert(false /* foo */) --- src/libexpr/eval-error.cc | 8 ++++++++ src/libexpr/eval-error.hh | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc index bd84e0428a3..cdb0b477242 100644 --- a/src/libexpr/eval-error.cc +++ b/src/libexpr/eval-error.cc @@ -92,6 +92,14 @@ void EvalErrorBuilder::debugThrow() throw error; } +template +void EvalErrorBuilder::panic() +{ + logError(error.info()); + printError("This is a bug! An unexpected condition occurred, causing the Nix evaluator to have to stop. If you could share a reproducible example or a core dump, please open an issue at https://github.com/NixOS/nix/issues"); + abort(); +} + template class EvalErrorBuilder; template class EvalErrorBuilder; template class EvalErrorBuilder; diff --git a/src/libexpr/eval-error.hh b/src/libexpr/eval-error.hh index fe48e054bf2..6409dc68a28 100644 --- a/src/libexpr/eval-error.hh +++ b/src/libexpr/eval-error.hh @@ -112,6 +112,12 @@ public: * Delete the `EvalErrorBuilder` and throw the underlying exception. */ [[gnu::noinline, gnu::noreturn]] void debugThrow(); + + /** + * A programming error or fatal condition occurred. Abort the process for core dump and debugging. + * This does not print a proper backtrace, because unwinding the stack is destructive. + */ + [[gnu::noinline, gnu::noreturn]] void panic(); }; } From 56bf39e9056ae7a15ec9c0347fd0043782e2b8cd Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 11 Jul 2024 11:37:45 +0200 Subject: [PATCH 012/284] eqValues/assertEqValues: Clean up assertions It's still paranoid, and probably a waste of words, but at least now it's consistent and readily identifyable from a log. --- src/libexpr/eval.cc | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 6f1a7d6186b..31d0c635a4b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2631,17 +2631,12 @@ void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::st return; case nThunk: // Must not be left by forceValue - default: - // This should never happen, because eqValues already throws an - // error for this, and this function should only be called when - // eqValues has found a difference, and it should match - // its behavior. - // Note that as of writing, we make the compiler require that all enum - // values are handled explicitly with `case`s, _despite_ having a - // `default:`. - error( - "BUG: cannot compare %1% with %2%; did forceValue leave a thunk, or might assertEqValues be out of sync with eqValues?", showType(v1), showType(v2)) - .debugThrow(); + assert(false); + default: // Note that we pass compiler flags that should make `default:` unreachable. + // Also note that this probably ran after `eqValues`, which implements + // the same logic more efficiently (without having to unwind stacks), + // so maybe `assertEqValues` and `eqValues` are out of sync. Check it for solutions. + error("assertEqValues: cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).panic(); } } @@ -2723,8 +2718,9 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return v1.fpoint() == v2.fpoint(); case nThunk: // Must not be left by forceValue - default: - error("cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).debugThrow(); + assert(false); + default: // Note that we pass compiler flags that should make `default:` unreachable. + error("eqValues: cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).panic(); } } From a4ce96e5f1e78537e650025870011f6fa2ba7e3c Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Sun, 14 Jul 2024 19:07:18 -0700 Subject: [PATCH 013/284] doc: Add comment for fetchurl for name & url fetchurl can be given a name and url aside from just the url. Giving a name can be useful if the url has invalid characters such as tilde for the store. --- src/libexpr/primops/fetchTree.cc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 6a7accad7f6..a9956ad88d8 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -529,9 +529,20 @@ static void prim_fetchurl(EvalState & state, const PosIdx pos, Value * * args, V static RegisterPrimOp primop_fetchurl({ .name = "__fetchurl", - .args = {"url"}, + .args = {"args"}, .doc = R"( - Download the specified URL and return the path of the downloaded file. + If args is a URL, return the path of the downloaded file. + Otherwise, it can be an attribute with the following attributes + (all except url are optional): + + - `url` + + The URL of the file to download. + + - `name` (default: `url without the protocol`) + + A name for the file in the store. This can be useful if the URL has any + characters that are invalid for the store. Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval). )", From 945fff5674e4dc8c9a2a365d555d4561c770ec20 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Mon, 15 Jul 2024 09:12:56 -0700 Subject: [PATCH 014/284] Apply suggestions from code review Add @edolstra suggestion fixes. Co-authored-by: Eelco Dolstra --- src/libexpr/primops/fetchTree.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index a9956ad88d8..333e486fd8e 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -529,17 +529,16 @@ static void prim_fetchurl(EvalState & state, const PosIdx pos, Value * * args, V static RegisterPrimOp primop_fetchurl({ .name = "__fetchurl", - .args = {"args"}, + .args = {"arg"}, .doc = R"( - If args is a URL, return the path of the downloaded file. - Otherwise, it can be an attribute with the following attributes - (all except url are optional): + Download the specified URL and return the path of the downloaded file. + `arg` can be either a string denoting the URL, or an attribute set with the following attributes: - `url` The URL of the file to download. - - `name` (default: `url without the protocol`) + - `name` (default: the last path component of the URL) A name for the file in the store. This can be useful if the URL has any characters that are invalid for the store. From 783a8341ee2dedb8fc0790e803a5f3d0362d67d9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 16 Jul 2024 01:28:28 +0200 Subject: [PATCH 015/284] tests/functional: Support negative codes in expect, expectStderr --- tests/functional/common/vars-and-functions.sh | 6 ++++-- tests/functional/test-infra.sh | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/vars-and-functions.sh index 4316a30d5ce..e5cc04bb358 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/vars-and-functions.sh @@ -236,7 +236,8 @@ expect() { expected="$1" shift "$@" && res=0 || res="$?" - if [[ $res -ne $expected ]]; then + # also match "negative" codes, which wrap around to >127 + if [[ $res -ne $expected && $res -ne $[256 + expected] ]]; then echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi @@ -250,7 +251,8 @@ expectStderr() { expected="$1" shift "$@" 2>&1 && res=0 || res="$?" - if [[ $res -ne $expected ]]; then + # also match "negative" codes, which wrap around to >127 + if [[ $res -ne $expected && $res -ne $[256 + expected] ]]; then echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi diff --git a/tests/functional/test-infra.sh b/tests/functional/test-infra.sh index 37322b356d4..f6f84eae99f 100755 --- a/tests/functional/test-infra.sh +++ b/tests/functional/test-infra.sh @@ -13,6 +13,25 @@ expect 1 false # `expect` will fail when we get it wrong expect 1 expect 0 false +function ret() { + return $1 +} + +# `expect` can call functions, not just executables +expect 0 ret 0 +expect 1 ret 1 + +# `expect` supports negative exit codes +expect -1 ret -1 + +# or high positive ones, equivalent to negative ones +expect 255 ret 255 +expect 255 ret -1 +expect -1 ret 255 + +# but it doesn't confuse negative exit codes with positive ones +expect 1 expect -10 ret 10 + noisyTrue () { echo YAY! >&2 true From f2df3f0c6c78cb742a87dbe2d2f9bcf5d5395795 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 16 Jul 2024 01:40:14 +0200 Subject: [PATCH 016/284] tests/vars-and-functions: Add callerPrefix helper --- tests/functional/common/vars-and-functions.sh | 39 +++++++++++++++++++ tests/functional/test-infra.sh | 4 ++ 2 files changed, 43 insertions(+) diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/vars-and-functions.sh index e5cc04bb358..062f3d9f93f 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/vars-and-functions.sh @@ -297,6 +297,45 @@ onError() { done } +# Prints an error message prefix referring to the last call into this file. +# Ignores `expect` and `expectStderr` calls. +# Set a special exit code when test suite functions are misused, so that +# functions like expectStderr won't mistake them for expected Nix CLI errors. +# Suggestion: -101 (negative to indicate very abnormal, and beyond the normal +# range of signals) +# Example (showns as string): 'repl.sh:123: in call to grepQuiet: ' +# This function is inefficient, so it should only be used in error messages. +callerPrefix() { + # Find the closes caller that's not from this file + local i file line fn savedFn + # Use `caller` + for i in $(seq 0 100); do + caller $i > /dev/null || { + if [[ -n "${file:-}" ]]; then + echo "$file:$line: ${savedFn+in call to $savedFn: }" + fi + break + } + line="$(caller $i | cut -d' ' -f1)" + fn="$(caller $i | cut -d' ' -f2)" + file="$(caller $i | cut -d' ' -f3)" + if [[ $file != "${BASH_SOURCE[0]}" ]]; then + echo "$file:$line: ${savedFn+in call to $savedFn: }" + return + fi + case "$fn" in + # Ignore higher order functions that don't report any misuse of themselves + # This way a misuse of a foo in `expectStderr 1 foo` will be reported as + # calling foo, not expectStderr. + expect|expectStderr|callerPrefix) + ;; + *) + savedFn="$fn" + ;; + esac + done +} + # `grep -v` doesn't work well for exit codes. We want `!(exist line l. l # matches)`. It gives us `exist line l. !(l matches)`. # diff --git a/tests/functional/test-infra.sh b/tests/functional/test-infra.sh index f6f84eae99f..93e0bd64b2b 100755 --- a/tests/functional/test-infra.sh +++ b/tests/functional/test-infra.sh @@ -88,6 +88,10 @@ funBang () { expect 1 funBang unset funBang +# callerPrefix can be used by the test framework to improve error messages +# it reports about our call site here +echo "<[$(callerPrefix)]>" | grepQuiet -F "<[test-infra.sh:$LINENO: ]>" + # `grep -v -q` is not what we want for exit codes, but `grepInverse` is # Avoid `grep -v -q`. The following line proves the point, and if it fails, # we'll know that `grep` had a breaking change or `-v -q` may not be portable. From 644b97ce2574fe22a3fe14daeb6a3d0711d75731 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 16 Jul 2024 01:41:22 +0200 Subject: [PATCH 017/284] tests/functional: Make our grep* helpers reject newlines in the query Newlines behave like *OR*; not "and then". --- tests/functional/common/vars-and-functions.sh | 19 ++++++++++++++++--- tests/functional/test-infra.sh | 5 +++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/vars-and-functions.sh index 062f3d9f93f..6cce08fbcd0 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/vars-and-functions.sh @@ -336,13 +336,24 @@ callerPrefix() { done } +checkGrepArgs() { + local arg + for arg in "$@"; do + if [[ "$arg" != "${arg//$'\n'/_}" ]]; then + echo "$(callerPrefix)newline not allowed in arguments; grep would try each line individually as if connected by an OR operator" >&2 + return -101 + fi + done +} + # `grep -v` doesn't work well for exit codes. We want `!(exist line l. l # matches)`. It gives us `exist line l. !(l matches)`. # # `!` normally doesn't work well with `set -e`, but when we wrap in a # function it *does*. grepInverse() { - ! grep "$@" + checkGrepArgs "$@" && \ + ! grep "$@" } # A shorthand, `> /dev/null` is a bit noisy. @@ -357,12 +368,14 @@ grepInverse() { # the producer into the pipe. But rest assured we've seen it happen in # CI reliably. grepQuiet() { - grep "$@" > /dev/null + checkGrepArgs "$@" && \ + grep "$@" > /dev/null } # The previous two, combined grepQuietInverse() { - ! grep "$@" > /dev/null + checkGrepArgs "$@" && \ + ! grep "$@" > /dev/null } # Return the number of arguments diff --git a/tests/functional/test-infra.sh b/tests/functional/test-infra.sh index 93e0bd64b2b..983b4e8604d 100755 --- a/tests/functional/test-infra.sh +++ b/tests/functional/test-infra.sh @@ -108,3 +108,8 @@ unset res res=$(set -eu -o pipefail; echo foo | expect 1 grepQuietInverse foo | wc -c) (( res == 0 )) unset res + +# `grepQuiet` does not allow newlines in its arguments, because grep quietly +# treats them as multiple queries. +( echo foo; echo bar; ) | expectStderr -101 grepQuiet $'foo\nbar' \ + | grepQuiet -E 'test-infra\.sh:[0-9]+: in call to grepQuiet: newline not allowed in arguments; grep would try each line individually as if connected by an OR operator' From 41a03738d63b94366d96dfc3e5cfb052c0ad5e2a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 16 Jul 2024 01:54:12 +0200 Subject: [PATCH 018/284] tests/functional: Also keep plain grep calls safe from newlines --- tests/functional/common/vars-and-functions.sh | 20 ++++++++++++++++--- tests/functional/test-infra.sh | 4 ++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/vars-and-functions.sh index 6cce08fbcd0..7a399f6d494 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/vars-and-functions.sh @@ -351,9 +351,12 @@ checkGrepArgs() { # # `!` normally doesn't work well with `set -e`, but when we wrap in a # function it *does*. +# +# `command grep` lets us avoid re-checking the args by going directly to the +# executable. grepInverse() { checkGrepArgs "$@" && \ - ! grep "$@" + ! command grep "$@" } # A shorthand, `> /dev/null` is a bit noisy. @@ -367,15 +370,26 @@ grepInverse() { # the closing of the pipe, the buffering of the pipe, and the speed of # the producer into the pipe. But rest assured we've seen it happen in # CI reliably. +# +# `command grep` lets us avoid re-checking the args by going directly to the +# executable. grepQuiet() { checkGrepArgs "$@" && \ - grep "$@" > /dev/null + command grep "$@" > /dev/null } # The previous two, combined grepQuietInverse() { checkGrepArgs "$@" && \ - ! grep "$@" > /dev/null + ! command grep "$@" > /dev/null +} + +# Wrap grep to remove its newline footgun; see checkGrepArgs. +# Note that we keep the checkGrepArgs calls in the other helpers, because some +# of them are negated and that would defeat this check. +grep() { + checkGrepArgs "$@" && \ + command grep "$@" } # Return the number of arguments diff --git a/tests/functional/test-infra.sh b/tests/functional/test-infra.sh index 983b4e8604d..1dab069fb2f 100755 --- a/tests/functional/test-infra.sh +++ b/tests/functional/test-infra.sh @@ -113,3 +113,7 @@ unset res # treats them as multiple queries. ( echo foo; echo bar; ) | expectStderr -101 grepQuiet $'foo\nbar' \ | grepQuiet -E 'test-infra\.sh:[0-9]+: in call to grepQuiet: newline not allowed in arguments; grep would try each line individually as if connected by an OR operator' + +# We took the blue pill and woke up in a world where `grep` is moderately safe. +expectStderr -101 grep $'foo\nbar' \ + | grepQuiet -E 'test-infra\.sh:[0-9]+: in call to grep: newline not allowed in arguments; grep would try each line individually as if connected by an OR operator' From 74698d54c82fa302dc0c82f68c796eda673090d9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 10 Jul 2024 13:34:42 +0200 Subject: [PATCH 019/284] Document builtins.derivation --- src/libexpr/primops/derivation.nix | 29 +++++++++++++++++-- .../lang/eval-fail-derivation-name.err.exp | 24 +++++++-------- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/src/libexpr/primops/derivation.nix b/src/libexpr/primops/derivation.nix index c0fbe8082cd..f329ff71e32 100644 --- a/src/libexpr/primops/derivation.nix +++ b/src/libexpr/primops/derivation.nix @@ -1,6 +1,31 @@ -/* This is the implementation of the ‘derivation’ builtin function. - It's actually a wrapper around the ‘derivationStrict’ primop. */ +# This is the implementation of the ‘derivation’ builtin function. +# It's actually a wrapper around the ‘derivationStrict’ primop. +# Note that the following comment will be shown in :doc in the repl, but not in the manual. +/** + Create a derivation. + + # Inputs + + The single argument is an attribute set that describes what to build and how to build it. + See https://nix.dev/manual/nix/2.23/language/derivations + + # Output + + The result is an attribute set that describes the derivation. + Notably it contains the outputs, which in the context of the Nix language are special strings that refer to the output paths, which may not yet exist. + The realisation of these outputs only occurs when needed; for example + + * When `nix-build` or a similar command is run, it realises the outputs that were requested on its command line. + See https://nix.dev/manual/nix/2.23/command-ref/nix-build + + * When `import`, `readFile`, `readDir` or some other functions are called, they have to realise the outputs they depend on. + This is referred to as "import from derivation". + See https://nix.dev/manual/nix/2.23/language/import-from-derivation + + Note that `derivation` is very bare-bones, and provides almost no commands during the build. + Most likely, you'll want to use functions like `stdenv.mkDerivation` in Nixpkgs to set up a basic environment. +*/ drvAttrs @ { outputs ? [ "out" ], ... }: let diff --git a/tests/functional/lang/eval-fail-derivation-name.err.exp b/tests/functional/lang/eval-fail-derivation-name.err.exp index eb2206df16d..ae7b47712b9 100644 --- a/tests/functional/lang/eval-fail-derivation-name.err.exp +++ b/tests/functional/lang/eval-fail-derivation-name.err.exp @@ -1,24 +1,24 @@ error: … while evaluating the attribute 'outPath' - at :19:9: - 18| value = commonAttrs // { - 19| outPath = builtins.getAttr outputName strict; + at :44:9: + 43| value = commonAttrs // { + 44| outPath = builtins.getAttr outputName strict; | ^ - 20| drvPath = strict.drvPath; + 45| drvPath = strict.drvPath; … while calling the 'getAttr' builtin - at :19:19: - 18| value = commonAttrs // { - 19| outPath = builtins.getAttr outputName strict; + at :44:19: + 43| value = commonAttrs // { + 44| outPath = builtins.getAttr outputName strict; | ^ - 20| drvPath = strict.drvPath; + 45| drvPath = strict.drvPath; … while calling the 'derivationStrict' builtin - at :9:12: - 8| - 9| strict = derivationStrict drvAttrs; + at :34:12: + 33| + 34| strict = derivationStrict drvAttrs; | ^ - 10| + 35| … while evaluating derivation '~jiggle~' whose name attribute is located at /pwd/lang/eval-fail-derivation-name.nix:2:3 From f9a1d6b0188a2d0e9ad84f1e735e560320380501 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 16 Jul 2024 17:36:30 +0200 Subject: [PATCH 020/284] tests/functional/lang: Add post processing and remove certain line numbers --- tests/functional/lang.sh | 15 +++++++++++ .../lang/eval-fail-derivation-name.err.exp | 26 +++++++++---------- .../eval-fail-derivation-name.postprocess | 9 +++++++ 3 files changed, 37 insertions(+), 13 deletions(-) create mode 100755 tests/functional/lang/eval-fail-derivation-name.postprocess diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index 8cb8e98fb7e..5a8cefd133d 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -50,11 +50,22 @@ set +x badDiff=0 badExitCode=0 +# Extra post-processing that's specific to each test case +postprocess() { + if [[ -e "lang/$1.postprocess" ]]; then + ( + set -x; + "lang/$1.postprocess" "lang/$1" + ) + fi +} + for i in lang/parse-fail-*.nix; do echo "parsing $i (should fail)"; i=$(basename "$i" .nix) if expectStderr 1 nix-instantiate --parse - < "lang/$i.nix" > "lang/$i.err" then + postprocess "$i" diffAndAccept "$i" err err.exp else echo "FAIL: $i shouldn't parse" @@ -71,6 +82,7 @@ for i in lang/parse-okay-*.nix; do 2> "lang/$i.err" then sed "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + postprocess "$i" diffAndAccept "$i" out exp diffAndAccept "$i" err err.exp else @@ -94,6 +106,7 @@ for i in lang/eval-fail-*.nix; do expectStderr 1 nix-instantiate $flags "lang/$i.nix" \ | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" then + postprocess "$i" diffAndAccept "$i" err err.exp else echo "FAIL: $i shouldn't evaluate" @@ -109,6 +122,7 @@ for i in lang/eval-okay-*.nix; do if expect 0 nix-instantiate --eval --xml --no-location --strict \ "lang/$i.nix" > "lang/$i.out.xml" then + postprocess "$i" diffAndAccept "$i" out.xml exp.xml else echo "FAIL: $i should evaluate" @@ -129,6 +143,7 @@ for i in lang/eval-okay-*.nix; do 2> "lang/$i.err" then sed -i "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + postprocess "$i" diffAndAccept "$i" out exp diffAndAccept "$i" err err.exp else diff --git a/tests/functional/lang/eval-fail-derivation-name.err.exp b/tests/functional/lang/eval-fail-derivation-name.err.exp index ae7b47712b9..0ef98674d81 100644 --- a/tests/functional/lang/eval-fail-derivation-name.err.exp +++ b/tests/functional/lang/eval-fail-derivation-name.err.exp @@ -1,26 +1,26 @@ error: … while evaluating the attribute 'outPath' - at :44:9: - 43| value = commonAttrs // { - 44| outPath = builtins.getAttr outputName strict; + at ::: + | value = commonAttrs // { + | outPath = builtins.getAttr outputName strict; | ^ - 45| drvPath = strict.drvPath; + | drvPath = strict.drvPath; … while calling the 'getAttr' builtin - at :44:19: - 43| value = commonAttrs // { - 44| outPath = builtins.getAttr outputName strict; + at ::: + | value = commonAttrs // { + | outPath = builtins.getAttr outputName strict; | ^ - 45| drvPath = strict.drvPath; + | drvPath = strict.drvPath; … while calling the 'derivationStrict' builtin - at :34:12: - 33| - 34| strict = derivationStrict drvAttrs; + at ::: + | + | strict = derivationStrict drvAttrs; | ^ - 35| + | … while evaluating derivation '~jiggle~' - whose name attribute is located at /pwd/lang/eval-fail-derivation-name.nix:2:3 + whose name attribute is located at /pwd/lang/eval-fail-derivation-name.nix:: error: invalid derivation name: name '~jiggle~' contains illegal character '~'. Please pass a different 'name'. diff --git a/tests/functional/lang/eval-fail-derivation-name.postprocess b/tests/functional/lang/eval-fail-derivation-name.postprocess new file mode 100755 index 00000000000..ab9fa5b5d8d --- /dev/null +++ b/tests/functional/lang/eval-fail-derivation-name.postprocess @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +testcaseBasename=$1 + +# Line numbers change when derivation.nix docs are updated. +sed -i "$testcaseBasename.err" \ + -e 's/[0-9 ][0-9 ][0-9 ][0-9 ][0-9 ][0-9 ][0-9 ][0-9]\([^0-9]\)/\1/g' \ + -e 's/[0-9][0-9]*//g' \ + ; From 7dce07463429bb4a42a3b47a869f5eb488206c26 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 16 Jul 2024 22:43:04 +0200 Subject: [PATCH 021/284] tests/functional/lang: Avoid /usr/bin/env for sandbox --- tests/functional/lang.sh | 5 ++++- tests/functional/lang/eval-fail-derivation-name.postprocess | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) mode change 100755 => 100644 tests/functional/lang/eval-fail-derivation-name.postprocess diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index 5a8cefd133d..46cf3f1fe9d 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -54,8 +54,11 @@ badExitCode=0 postprocess() { if [[ -e "lang/$1.postprocess" ]]; then ( + # We could allow arbitrary interpreters in .postprocess, but that + # just exposes us to the complexity of not having /usr/bin/env in + # the sandbox. So let's just hardcode bash for now. set -x; - "lang/$1.postprocess" "lang/$1" + bash "lang/$1.postprocess" "lang/$1" ) fi } diff --git a/tests/functional/lang/eval-fail-derivation-name.postprocess b/tests/functional/lang/eval-fail-derivation-name.postprocess old mode 100755 new mode 100644 index ab9fa5b5d8d..ffbc2b5d444 --- a/tests/functional/lang/eval-fail-derivation-name.postprocess +++ b/tests/functional/lang/eval-fail-derivation-name.postprocess @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +# shellcheck shell=bash set -euo pipefail testcaseBasename=$1 From 9fae50ed4be6c7f8bd16ece9626709d78bb4b01c Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 17 Jul 2024 02:42:18 +0200 Subject: [PATCH 022/284] Add parser test for indented strings So that in the next commit we can see what changes about this test --- .../functional/lang/parse-okay-ind-string.exp | 1 + .../functional/lang/parse-okay-ind-string.nix | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 tests/functional/lang/parse-okay-ind-string.exp create mode 100644 tests/functional/lang/parse-okay-ind-string.nix diff --git a/tests/functional/lang/parse-okay-ind-string.exp b/tests/functional/lang/parse-okay-ind-string.exp new file mode 100644 index 00000000000..5f8d4868830 --- /dev/null +++ b/tests/functional/lang/parse-okay-ind-string.exp @@ -0,0 +1 @@ +(let string = "str"; in [ (/some/path) ((/some/path)) (("" + /some/path)) ((/some/path + "\n end")) (string) ((string)) (("" + string)) ((string + "\n end")) ("") ("") ("end") ]) diff --git a/tests/functional/lang/parse-okay-ind-string.nix b/tests/functional/lang/parse-okay-ind-string.nix new file mode 100644 index 00000000000..97c9de3cd1d --- /dev/null +++ b/tests/functional/lang/parse-okay-ind-string.nix @@ -0,0 +1,31 @@ +let + string = "str"; +in [ + /some/path + + ''${/some/path}'' + + '' + ${/some/path}'' + + ''${/some/path} + end'' + + string + + ''${string}'' + + '' + ${string}'' + + ''${string} + end'' + + '''' + + '' + '' + + '' + end'' +] From 83d585b423737591f865d905882122211995c308 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 17 Jul 2024 15:19:17 +0200 Subject: [PATCH 023/284] C API: Make nix_err an enum This generally gives a better experience with bindings generators, possibly other tooling. A possible risk is that some generators may not represent unknown codes correctly. Rust bindgen by default generates suitable code: * a type alias nix_err = c_int * individual constants for the known enum values It does _not_ generate a closed type that can only hold the values that were known at code generation time. If this proves to be a problem, we could instead split the type: `typedef int nix_err;` for return values `enum nix_known_err` for code generation. This would complicate the interface, so let's not do it unless it is shown to be needed. --- src/libutil-c/nix_api_util.h | 76 +++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index e0ca04e69ec..ad6f32859bd 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -56,47 +56,51 @@ extern "C" { * - NIX_ERR_KEY: A key error occurred (-3) * - NIX_ERR_NIX_ERROR: A generic Nix error occurred (-4) */ -typedef int nix_err; +enum nix_err { -/** - * @brief No error occurred. - * - * This error code is returned when no error has occurred during the function - * execution. - */ -#define NIX_OK 0 + /** + * @brief No error occurred. + * + * This error code is returned when no error has occurred during the function + * execution. + */ + NIX_OK = 0, -/** - * @brief An unknown error occurred. - * - * This error code is returned when an unknown error occurred during the - * function execution. - */ -#define NIX_ERR_UNKNOWN -1 + /** + * @brief An unknown error occurred. + * + * This error code is returned when an unknown error occurred during the + * function execution. + */ + NIX_ERR_UNKNOWN = -1, -/** - * @brief An overflow error occurred. - * - * This error code is returned when an overflow error occurred during the - * function execution. - */ -#define NIX_ERR_OVERFLOW -2 + /** + * @brief An overflow error occurred. + * + * This error code is returned when an overflow error occurred during the + * function execution. + */ + NIX_ERR_OVERFLOW = -2, -/** - * @brief A key error occurred. - * - * This error code is returned when a key error occurred during the function - * execution. - */ -#define NIX_ERR_KEY -3 + /** + * @brief A key error occurred. + * + * This error code is returned when a key error occurred during the function + * execution. + */ + NIX_ERR_KEY = -3, -/** - * @brief A generic Nix error occurred. - * - * This error code is returned when a generic Nix error occurred during the - * function execution. - */ -#define NIX_ERR_NIX_ERROR -4 + /** + * @brief A generic Nix error occurred. + * + * This error code is returned when a generic Nix error occurred during the + * function execution. + */ + NIX_ERR_NIX_ERROR = -4, + +}; + +typedef enum nix_err nix_err; /** * @brief This object stores error state. From c1d5cf6f349a459c8b5dee9dad271a529cee6679 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Jul 2024 15:27:15 +0200 Subject: [PATCH 024/284] Factor out commonality between WorkerProto::Basic{Client,Server}Connection This also renames clientVersion and daemonVersion to the more correct protoVersion (since it's the version agreed to by both sides). --- src/libstore/daemon.cc | 29 +++-- src/libstore/daemon.hh | 4 +- src/libstore/remote-store.cc | 38 +++--- .../unix/build/local-derivation-goal.cc | 7 +- src/libstore/worker-protocol-connection.cc | 20 ++-- src/libstore/worker-protocol-connection.hh | 112 ++++++------------ src/libstore/worker-protocol.hh | 1 + src/nix/unix/daemon.cc | 17 ++- 8 files changed, 99 insertions(+), 129 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 5c5080f8aa9..fc14eda3c96 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1020,8 +1020,8 @@ static void performOp(TunnelLogger * logger, ref store, void processConnection( ref store, - FdSource & from, - FdSink & to, + FdSource && from, + FdSink && to, TrustedFlag trusted, RecursiveFlag recursive) { @@ -1037,7 +1037,12 @@ void processConnection( if (clientVersion < 0x10a) throw Error("the Nix client version is too old"); - auto tunnelLogger = new TunnelLogger(to, clientVersion); + WorkerProto::BasicServerConnection conn; + conn.to = std::move(to); + conn.from = std::move(from); + conn.protoVersion = clientVersion; + + auto tunnelLogger = new TunnelLogger(conn.to, clientVersion); auto prevLogger = nix::logger; // FIXME if (!recursive) @@ -1050,12 +1055,6 @@ void processConnection( printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount); }); - WorkerProto::BasicServerConnection conn { - .to = to, - .from = from, - .clientVersion = clientVersion, - }; - conn.postHandshake(*store, { .daemonNixVersion = nixVersion, // We and the underlying store both need to trust the client for @@ -1071,13 +1070,13 @@ void processConnection( try { tunnelLogger->stopWork(); - to.flush(); + conn.to.flush(); /* Process client requests. */ while (true) { WorkerProto::Op op; try { - op = (enum WorkerProto::Op) readInt(from); + op = (enum WorkerProto::Op) readInt(conn.from); } catch (Interrupted & e) { break; } catch (EndOfFile & e) { @@ -1091,7 +1090,7 @@ void processConnection( debug("performing daemon worker op: %d", op); try { - performOp(tunnelLogger, store, trusted, recursive, clientVersion, from, to, op); + performOp(tunnelLogger, store, trusted, recursive, clientVersion, conn.from, conn.to, op); } catch (Error & e) { /* If we're not in a state where we can send replies, then something went wrong processing the input of the @@ -1107,19 +1106,19 @@ void processConnection( throw; } - to.flush(); + conn.to.flush(); assert(!tunnelLogger->state_.lock()->canSendStderr); }; } catch (Error & e) { tunnelLogger->stopWork(&e); - to.flush(); + conn.to.flush(); return; } catch (std::exception & e) { auto ex = Error(e.what()); tunnelLogger->stopWork(&ex); - to.flush(); + conn.to.flush(); return; } } diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 1964c0d997c..a8ce32d8deb 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -10,8 +10,8 @@ enum RecursiveFlag : bool { NotRecursive = false, Recursive = true }; void processConnection( ref store, - FdSource & from, - FdSink & to, + FdSource && from, + FdSink && to, TrustedFlag trusted, RecursiveFlag recursive); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 6e8931ca2e4..ebb0864c555 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -73,7 +73,7 @@ void RemoteStore::initConnection(Connection & conn) StringSink saved; TeeSource tee(conn.from, saved); try { - conn.daemonVersion = WorkerProto::BasicClientConnection::handshake( + conn.protoVersion = WorkerProto::BasicClientConnection::handshake( conn.to, tee, PROTOCOL_VERSION); } catch (SerialisationError & e) { /* In case the other side is waiting for our input, close @@ -115,7 +115,7 @@ void RemoteStore::setOptions(Connection & conn) << settings.buildCores << settings.useSubstitutes; - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 12) { std::map overrides; settings.getSettings(overrides, true); // libstore settings fileTransferSettings.getSettings(overrides, true); @@ -175,7 +175,7 @@ bool RemoteStore::isValidPathUncached(const StorePath & path) StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) { StorePathSet res; for (auto & i : paths) if (isValidPath(i)) res.insert(i); @@ -198,7 +198,7 @@ StorePathSet RemoteStore::queryAllValidPaths() StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) { StorePathSet res; for (auto & i : paths) { conn->to << WorkerProto::Op::HasSubstitutes << printStorePath(i); @@ -221,7 +221,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) { for (auto & i : pathsMap) { SubstitutablePathInfo info; @@ -241,7 +241,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S } else { conn->to << WorkerProto::Op::QuerySubstitutablePathInfos; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 22) { StorePathSet paths; for (auto & path : pathsMap) paths.insert(path.first); @@ -368,7 +368,7 @@ ref RemoteStore::addCAToStore( std::optional conn_(getConnection()); auto & conn = *conn_; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 25) { conn->to << WorkerProto::Op::AddToStore @@ -485,7 +485,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 18) { auto source2 = sinkToSource([&](Sink & sink) { sink << 1 // == path follows ; @@ -513,11 +513,11 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 23) { conn.withFramedSink([&](Sink & sink) { copyNAR(source, sink); }); - } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { + } else if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 21) { conn.processStderr(0, &source); } else { copyNAR(source, conn->to); @@ -554,7 +554,7 @@ void RemoteStore::addMultipleToStore( RepairFlag repair, CheckSigsFlag checkSigs) { - if (GET_PROTOCOL_MINOR(getConnection()->daemonVersion) >= 32) { + if (GET_PROTOCOL_MINOR(getConnection()->protoVersion) >= 32) { auto conn(getConnection()); conn->to << WorkerProto::Op::AddMultipleToStore @@ -572,7 +572,7 @@ void RemoteStore::registerDrvOutput(const Realisation & info) { auto conn(getConnection()); conn->to << WorkerProto::Op::RegisterDrvOutput; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 31) { conn->to << info.id.to_string(); conn->to << std::string(info.outPath.to_string()); } else { @@ -587,7 +587,7 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, try { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 27) { warn("the daemon is too old to support content-addressed derivations, please upgrade it to 2.4"); return callback(nullptr); } @@ -597,7 +597,7 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, conn.processStderr(); auto real = [&]() -> std::shared_ptr { - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 31) { auto outPaths = WorkerProto::Serialise>::read( *this, *conn); if (outPaths.empty()) @@ -644,9 +644,9 @@ void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMod auto conn(getConnection()); conn->to << WorkerProto::Op::BuildPaths; - assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); + assert(GET_PROTOCOL_MINOR(conn->protoVersion) >= 13); WorkerProto::write(*this, *conn, drvPaths); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 15) conn->to << buildMode; else /* Old daemons did not take a 'buildMode' parameter, so we @@ -667,7 +667,7 @@ std::vector RemoteStore::buildPathsWithResults( std::optional conn_(getConnection()); auto & conn = *conn_; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 34) { conn->to << WorkerProto::Op::BuildPathsWithResults; WorkerProto::write(*this, *conn, paths); conn->to << buildMode; @@ -841,7 +841,7 @@ void RemoteStore::queryMissing(const std::vector & targets, { { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 19) // Don't hold the connection handle in the fallback case // to prevent a deadlock. goto fallback; @@ -889,7 +889,7 @@ void RemoteStore::connect() unsigned int RemoteStore::getProtocol() { auto conn(connections->get()); - return conn->daemonVersion; + return conn->protoVersion; } std::optional RemoteStore::isTrustedClient() diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index f968bbc5b7f..0dd102200a5 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -1526,10 +1526,11 @@ void LocalDerivationGoal::startDaemon() debug("received daemon connection"); auto workerThread = std::thread([store, remote{std::move(remote)}]() { - FdSource from(remote.get()); - FdSink to(remote.get()); try { - daemon::processConnection(store, from, to, + daemon::processConnection( + store, + FdSource(remote.get()), + FdSink(remote.get()), NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SystemError &) { diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc index 3a640051ebc..93d13d48e55 100644 --- a/src/libstore/worker-protocol-connection.cc +++ b/src/libstore/worker-protocol-connection.cc @@ -58,7 +58,7 @@ std::exception_ptr WorkerProto::BasicClientConnection::processStderrReturn(Sink } else if (msg == STDERR_ERROR) { - if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) { + if (GET_PROTOCOL_MINOR(protoVersion) >= 26) { ex = std::make_exception_ptr(readError(from)); } else { auto error = readString(from); @@ -114,7 +114,7 @@ std::exception_ptr WorkerProto::BasicClientConnection::processStderrReturn(Sink // explain to users what's going on when their daemon is // older than #4628 (2023). if (experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations) - && GET_PROTOCOL_MINOR(daemonVersion) <= 35) { + && GET_PROTOCOL_MINOR(protoVersion) <= 35) { auto m = e.msg(); if (m.find("parsing derivation") != std::string::npos && m.find("expected string") != std::string::npos && m.find("Derive([") != std::string::npos) @@ -172,15 +172,15 @@ WorkerProto::ClientHandshakeInfo WorkerProto::BasicClientConnection::postHandsha { WorkerProto::ClientHandshakeInfo res; - if (GET_PROTOCOL_MINOR(daemonVersion) >= 14) { + if (GET_PROTOCOL_MINOR(protoVersion) >= 14) { // Obsolete CPU affinity. to << 0; } - if (GET_PROTOCOL_MINOR(daemonVersion) >= 11) + if (GET_PROTOCOL_MINOR(protoVersion) >= 11) to << false; // obsolete reserveSpace - if (GET_PROTOCOL_MINOR(daemonVersion) >= 33) + if (GET_PROTOCOL_MINOR(protoVersion) >= 33) to.flush(); return WorkerProto::Serialise::read(store, *this); @@ -188,12 +188,12 @@ WorkerProto::ClientHandshakeInfo WorkerProto::BasicClientConnection::postHandsha void WorkerProto::BasicServerConnection::postHandshake(const StoreDirConfig & store, const ClientHandshakeInfo & info) { - if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) { + if (GET_PROTOCOL_MINOR(protoVersion) >= 14 && readInt(from)) { // Obsolete CPU affinity. readInt(from); } - if (GET_PROTOCOL_MINOR(clientVersion) >= 11) + if (GET_PROTOCOL_MINOR(protoVersion) >= 11) readInt(from); // obsolete reserveSpace WorkerProto::write(store, *this, info); @@ -211,7 +211,7 @@ UnkeyedValidPathInfo WorkerProto::BasicClientConnection::queryPathInfo( throw InvalidPath(std::move(e.info())); throw; } - if (GET_PROTOCOL_MINOR(daemonVersion) >= 17) { + if (GET_PROTOCOL_MINOR(protoVersion) >= 17) { bool valid; from >> valid; if (!valid) @@ -223,10 +223,10 @@ UnkeyedValidPathInfo WorkerProto::BasicClientConnection::queryPathInfo( StorePathSet WorkerProto::BasicClientConnection::queryValidPaths( const StoreDirConfig & store, bool * daemonException, const StorePathSet & paths, SubstituteFlag maybeSubstitute) { - assert(GET_PROTOCOL_MINOR(daemonVersion) >= 12); + assert(GET_PROTOCOL_MINOR(protoVersion) >= 12); to << WorkerProto::Op::QueryValidPaths; WorkerProto::write(store, *this, paths); - if (GET_PROTOCOL_MINOR(daemonVersion) >= 27) { + if (GET_PROTOCOL_MINOR(protoVersion) >= 27) { to << maybeSubstitute; } processStderr(daemonException); diff --git a/src/libstore/worker-protocol-connection.hh b/src/libstore/worker-protocol-connection.hh index 9dd723fd089..38287d08e31 100644 --- a/src/libstore/worker-protocol-connection.hh +++ b/src/libstore/worker-protocol-connection.hh @@ -6,7 +6,7 @@ namespace nix { -struct WorkerProto::BasicClientConnection +struct WorkerProto::BasicConnection { /** * Send with this. @@ -19,14 +19,45 @@ struct WorkerProto::BasicClientConnection FdSource from; /** - * Worker protocol version used for the connection. + * The protocol version agreed by both sides. + */ + WorkerProto::Version protoVersion; + + /** + * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the + * factored out serve protocol serializers with a + * `LegacySSHStore::Connection`. + * + * The serve protocol connection types are unidirectional, unlike + * this type. + */ + operator WorkerProto::ReadConn() + { + return WorkerProto::ReadConn{ + .from = from, + .version = protoVersion, + }; + } + + /** + * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the + * factored out serve protocol serializers with a + * `LegacySSHStore::Connection`. * - * Despite its name, it is actually the maximum version both - * sides support. (If the maximum doesn't exist, we would fail to - * establish a connection and produce a value of this type.) + * The serve protocol connection types are unidirectional, unlike + * this type. */ - WorkerProto::Version daemonVersion; + operator WorkerProto::WriteConn() + { + return WorkerProto::WriteConn{ + .to = to, + .version = protoVersion, + }; + } +}; +struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection +{ /** * Flush to direction */ @@ -60,38 +91,6 @@ struct WorkerProto::BasicClientConnection */ ClientHandshakeInfo postHandshake(const StoreDirConfig & store); - /** - * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the - * factored out serve protocol serializers with a - * `LegacySSHStore::Connection`. - * - * The serve protocol connection types are unidirectional, unlike - * this type. - */ - operator WorkerProto::ReadConn() - { - return WorkerProto::ReadConn{ - .from = from, - .version = daemonVersion, - }; - } - - /** - * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the - * factored out serve protocol serializers with a - * `LegacySSHStore::Connection`. - * - * The serve protocol connection types are unidirectional, unlike - * this type. - */ - operator WorkerProto::WriteConn() - { - return WorkerProto::WriteConn{ - .to = to, - .version = daemonVersion, - }; - } - void addTempRoot(const StoreDirConfig & remoteStore, bool * daemonException, const StorePath & path); StorePathSet queryValidPaths( @@ -124,43 +123,8 @@ struct WorkerProto::BasicClientConnection void importPaths(const StoreDirConfig & store, bool * daemonException, Source & source); }; -struct WorkerProto::BasicServerConnection +struct WorkerProto::BasicServerConnection : WorkerProto::BasicConnection { - /** - * Send with this. - */ - FdSink & to; - - /** - * Receive with this. - */ - FdSource & from; - - /** - * Worker protocol version used for the connection. - * - * Despite its name, it is actually the maximum version both - * sides support. (If the maximum doesn't exist, we would fail to - * establish a connection and produce a value of this type.) - */ - WorkerProto::Version clientVersion; - - operator WorkerProto::ReadConn() - { - return WorkerProto::ReadConn{ - .from = from, - .version = clientVersion, - }; - } - - operator WorkerProto::WriteConn() - { - return WorkerProto::WriteConn{ - .to = to, - .version = clientVersion, - }; - } - /** * Establishes connection, negotiating version. * diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 62a12d18201..f4023f7ea94 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -82,6 +82,7 @@ struct WorkerProto * * @todo remove once Hydra uses Store abstraction consistently. */ + struct BasicConnection; struct BasicClientConnection; struct BasicServerConnection; diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 4a7997b1fb0..66d8dbcf092 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -370,9 +370,12 @@ static void daemonLoop(std::optional forceTrustClientOpt) } // Handle the connection. - FdSource from(remote.get()); - FdSink to(remote.get()); - processConnection(openUncachedStore(), from, to, trusted, NotRecursive); + processConnection( + openUncachedStore(), + FdSource(remote.get()), + FdSink(remote.get()), + trusted, + NotRecursive); exit(0); }, options); @@ -437,9 +440,11 @@ static void forwardStdioConnection(RemoteStore & store) { */ static void processStdioConnection(ref store, TrustedFlag trustClient) { - FdSource from(STDIN_FILENO); - FdSink to(STDOUT_FILENO); - processConnection(store, from, to, trustClient, NotRecursive); + processConnection( + store, + FdSource(STDIN_FILENO), + FdSink(STDOUT_FILENO), + trustClient, NotRecursive); } /** From d231d802f501a1992ebb5138b58883ced56234fe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Jul 2024 16:30:24 +0200 Subject: [PATCH 025/284] Typo --- src/libstore/worker-protocol.hh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index f4023f7ea94..9fc16d0153e 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -184,8 +184,7 @@ enum struct WorkerProto::Op : uint64_t struct WorkerProto::ClientHandshakeInfo { /** - * The version of the Nix daemon that is processing our requests -. + * The version of the Nix daemon that is processing our requests. * * Do note, it may or may not communicating with another daemon, * rather than being an "end" `LocalStore` or similar. From fa7aa0389a2c13f5253eb4bf529aafab5d4a1d5f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Jul 2024 20:37:22 +0200 Subject: [PATCH 026/284] FdSource: Fix operator = This wasn't moving the underlying buffer, so if the buffer was non-empty, it could lose data. --- src/libutil/serialise.hh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 18f4a79c398..8254a5f899d 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -159,13 +159,7 @@ struct FdSource : BufferedSource FdSource(Descriptor fd) : fd(fd) { } FdSource(FdSource &&) = default; - FdSource & operator=(FdSource && s) - { - fd = s.fd; - s.fd = INVALID_DESCRIPTOR; - read = s.read; - return *this; - } + FdSource & operator=(FdSource && s) = default; bool good() override; protected: From 6e5d7fc5d88b5fab7788c84b8dd85e4d9de5419a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 15:23:24 -0400 Subject: [PATCH 027/284] Convert ssh:// to new system --- src/libstore/legacy-ssh-store.cc | 58 +++++++++++++++++++++------ src/libstore/legacy-ssh-store.hh | 51 +++++++++++++---------- src/libstore/s3-binary-cache-store.hh | 11 ++--- 3 files changed, 80 insertions(+), 40 deletions(-) diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index eac360a4f7a..c27c7bc563e 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -12,18 +12,55 @@ #include "ssh.hh" #include "derivations.hh" #include "callback.hh" +#include "config-parse-impl.hh" +#include "store-registration.hh" namespace nix { -LegacySSHStoreConfig::LegacySSHStoreConfig( +LegacySSHStore::Config::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , CommonSSHStoreConfig::Descriptions{CommonSSHStoreConfig::descriptions} + , LegacySSHStoreConfigT{ + .remoteProgram{ + .name = "remote-program", + .description = "Path to the `nix-store` executable on the remote machine.", + }, + .maxConnections{ + .name = "max-connections", + .description = "Maximum number of concurrent SSH connections.", + }, + } +{} + + +const LegacySSHStore::Config::Descriptions LegacySSHStore::Config::descriptions{}; + + +decltype(LegacySSHStore::Config::defaults) LegacySSHStore::Config::defaults = { + .remoteProgram = {{"nix-store"}}, + .maxConnections = {1}, +}; + + +LegacySSHStore::Config::LegacySSHStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params) - : StoreConfig(params) + const StoreReference::Params & params) + : Store::Config(params) , CommonSSHStoreConfig(scheme, authority, params) + , LegacySSHStoreConfigT{ + CONFIG_ROW(remoteProgram), + CONFIG_ROW(maxConnections), + } { +#ifndef _WIN32 + if (auto * p = get(params, "log-fd")) { + logFD = p->get(); + } +#endif } + std::string LegacySSHStoreConfig::doc() { return @@ -38,14 +75,11 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection bool good = true; }; -LegacySSHStore::LegacySSHStore( - std::string_view scheme, - std::string_view host, - const Params & params) - : StoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , LegacySSHStoreConfig(scheme, host, params) - , Store(params) +LegacySSHStore::LegacySSHStore(const Config & config) + : Store::Config(config) + , CommonSSHStoreConfig(config) + , LegacySSHStore::Config(config) + , Store(static_cast(*this)) , connections(make_ref>( std::max(1, (int) maxConnections), [this]() { return openConnection(); }, @@ -317,6 +351,6 @@ std::optional isTrustedClient() } -static RegisterStoreImplementation regLegacySSHStore; +static RegisterStoreImplementation regLegacySSHStore; } diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index b541455b4e5..0a58c803126 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -9,38 +9,50 @@ namespace nix { -struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig +template class F> +struct LegacySSHStoreConfigT { - using CommonSSHStoreConfig::CommonSSHStoreConfig; + const F remoteProgram; + const F maxConnections; +}; + +struct LegacySSHStoreConfig : + virtual CommonSSHStoreConfig, + LegacySSHStoreConfigT +{ + struct Descriptions : + virtual CommonSSHStoreConfig::Descriptions, + LegacySSHStoreConfigT + { + Descriptions(); + }; + + static const Descriptions descriptions; + + static LegacySSHStoreConfigT defaults; + /** + * Hack for getting remote build log output. Intentionally not a + * documented user-visible setting. + */ + Descriptor logFD = INVALID_DESCRIPTOR; LegacySSHStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params); - - const Setting remoteProgram{this, {"nix-store"}, "remote-program", - "Path to the `nix-store` executable on the remote machine."}; - - const Setting maxConnections{this, 1, "max-connections", - "Maximum number of concurrent SSH connections."}; + const StoreReference::Params & params); const std::string name() override { return "SSH Store"; } static std::set uriSchemes() { return {"ssh"}; } std::string doc() override; + + ref openStore() const override; }; struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store { -#ifndef _WIN32 - // Hack for getting remote build log output. - // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in - // the documentation - const Setting logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"}; -#else - Descriptor logFD = INVALID_DESCRIPTOR; -#endif + using Config = LegacySSHStoreConfig; struct Connection; @@ -48,10 +60,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor SSHMaster master; - LegacySSHStore( - std::string_view scheme, - std::string_view host, - const Params & params); + LegacySSHStore(const Config &); ref openConnection(); diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index bff26f6b3d0..df7197113ae 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -21,14 +21,11 @@ struct S3BinaryCacheStoreConfigT const F bufferSize; }; -struct S3BinaryCacheStoreConfig : - virtual BinaryCacheStoreConfig, - S3BinaryCacheStoreConfigT +struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig, S3BinaryCacheStoreConfigT { - struct Descriptions : - virtual Store::Config::Descriptions, - virtual BinaryCacheStore::Config::Descriptions, - S3BinaryCacheStoreConfigT + struct Descriptions : virtual Store::Config::Descriptions, + virtual BinaryCacheStore::Config::Descriptions, + S3BinaryCacheStoreConfigT { Descriptions(); }; From 74620b97d9687806b6daae80f08883c586d8caf5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 15:54:42 -0400 Subject: [PATCH 028/284] WIP --- src/libstore/legacy-ssh-store.cc | 12 ++-- src/libstore/legacy-ssh-store.hh | 1 + src/libstore/remote-store.cc | 53 +++++++++----- src/libstore/remote-store.hh | 3 +- src/libstore/ssh-store.cc | 118 ++++++++++++++++++------------- src/libstore/ssh-store.hh | 39 +++++++--- 6 files changed, 139 insertions(+), 87 deletions(-) diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index c27c7bc563e..10301b79e66 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -46,8 +46,8 @@ LegacySSHStore::Config::LegacySSHStoreConfig( std::string_view scheme, std::string_view authority, const StoreReference::Params & params) - : Store::Config(params) - , CommonSSHStoreConfig(scheme, authority, params) + : Store::Config{params} + , CommonSSHStoreConfig{scheme, authority, params} , LegacySSHStoreConfigT{ CONFIG_ROW(remoteProgram), CONFIG_ROW(maxConnections), @@ -76,10 +76,10 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection }; LegacySSHStore::LegacySSHStore(const Config & config) - : Store::Config(config) - , CommonSSHStoreConfig(config) - , LegacySSHStore::Config(config) - , Store(static_cast(*this)) + : Store::Config{config} + , CommonSSHStoreConfig{config} + , LegacySSHStore::Config{config} + , Store{static_cast(*this)} , connections(make_ref>( std::max(1, (int) maxConnections), [this]() { return openConnection(); }, diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index 0a58c803126..9d2234faaf7 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -30,6 +30,7 @@ struct LegacySSHStoreConfig : static const Descriptions descriptions; static LegacySSHStoreConfigT defaults; + /** * Hack for getting remote build log output. Intentionally not a * documented user-visible setting. diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 30d1523b550..4caabdf9f3c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -18,33 +18,50 @@ #include "callback.hh" #include "filetransfer.hh" #include "signals.hh" +#include "config-parse-impl.hh" #include namespace nix { -#if 0 - const Setting maxConnections{this, 1, "max-connections", - "Maximum number of concurrent connections to the Nix daemon."}; +RemoteStore::Config::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , RemoteStoreConfigT{ + .maxConnections{ + .name = "max-connections", + .description = "Maximum number of concurrent connections to the Nix daemon.", + }, + .maxConnectionAge{ + .name = "max-connection-age", + .description = "Maximum age of a connection before it is closed.", + }, + } +{} + + +const RemoteStore::Config::Descriptions RemoteStore::Config::descriptions{}; + - const Setting maxConnectionAge{this, - std::numeric_limits::max(), - "max-connection-age", - "Maximum age of a connection before it is closed."}; +decltype(RemoteStore::Config::defaults) RemoteStore::Config::defaults = { + .maxConnections = {1}, + .maxConnectionAge = {std::numeric_limits::max()}, +}; - const Setting maxConnections{this, 1, "max-connections", - "Maximum number of concurrent connections to the Nix daemon."}; - const Setting maxConnectionAge{this, - std::numeric_limits::max(), - "max-connection-age", - "Maximum age of a connection before it is closed."}; -#endif +RemoteStore::Config::RemoteStoreConfig(const StoreReference::Params & params) + : Store::Config(params) + , RemoteStoreConfigT{ + CONFIG_ROW(maxConnections), + CONFIG_ROW(maxConnectionAge), + } +{ +} + /* TODO: Separate these store types into different files, give them better names */ -RemoteStore::RemoteStore(const Params & params) - : RemoteStoreConfig(params) - , Store(params) +RemoteStore::RemoteStore(const Config & config) + : RemoteStore::Config(config) + , Store(static_cast(*this)) , connections(make_ref>( std::max(1, (int) maxConnections), [this]() { @@ -134,7 +151,7 @@ void RemoteStore::setOptions(Connection & conn) << settings.useSubstitutes; if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { - std::map overrides; + std::map overrides; settings.getSettings(overrides, true); // libstore settings fileTransferSettings.getSettings(overrides, true); overrides.erase(settings.keepFailed.name); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 437c4aee734..126f060a02b 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -22,7 +22,6 @@ template class F> struct RemoteStoreConfigT { const F maxConnections; - const F maxConnectionAge; }; @@ -43,6 +42,8 @@ struct RemoteStoreConfig : * The other defaults depend on the choice of `storeDir` and `rootDir` */ static RemoteStoreConfigT defaults; + + RemoteStoreConfig(const StoreReference::Params &); }; /** diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 954a9746774..eb97360160f 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -7,19 +7,45 @@ #include "worker-protocol-impl.hh" #include "pool.hh" #include "ssh.hh" +#include "config-parse-impl.hh" +#include "store-registration.hh" namespace nix { +SSHStoreConfig::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , CommonSSHStoreConfig::Descriptions{CommonSSHStoreConfig::descriptions} + , SSHStoreConfigT{ + .remoteProgram{ + .name = "remote-program", + .description = "Path to the `nix-daemon` executable on the remote machine.", + }, + } +{} + + +const SSHStoreConfig::Descriptions SSHStoreConfig::descriptions{}; + + +decltype(SSHStoreConfig::defaults) SSHStoreConfig::defaults = { + .remoteProgram = {{"nix-daemon"}}, +}; + + SSHStoreConfig::SSHStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, authority, params) + const StoreReference::Params & params) + : Store::Config{params} + , RemoteStore::Config{params} + , CommonSSHStoreConfig{scheme, authority, params} + , SSHStoreConfigT{ + CONFIG_ROW(remoteProgram), + } { } + std::string SSHStoreConfig::doc() { return @@ -27,20 +53,19 @@ std::string SSHStoreConfig::doc() ; } -class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore +struct SSHStore : + public virtual SSHStoreConfig, + public virtual RemoteStore { -public: - - SSHStore( - std::string_view scheme, - std::string_view host, - const Params & params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(scheme, host, params) - , Store(params) - , RemoteStore(params) + using Config = SSHStoreConfig; + + SSHStore(const Config & config) + : Store::Config{config} + , RemoteStore::Config{config} + , CommonSSHStoreConfig{config} + , SSHStore::Config{config} + , Store{static_cast(*this)} + , RemoteStore{static_cast(*this)} , master(createSSHMaster( // Use SSH master only if using more than 1 connection. connections->capacity() > 1)) @@ -88,21 +113,15 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore }; -MountedSSHStoreConfig::MountedSSHStoreConfig(StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) -{ -} - -MountedSSHStoreConfig::MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) +MountedSSHStoreConfig::MountedSSHStoreConfig( + std::string_view scheme, + std::string_view host, + const StoreReference::Params & params) + : Store::Config{params} + , RemoteStore::Config{params} + , CommonSSHStoreConfig{scheme, host, params} + , SSHStoreConfig{scheme, host, params} + , LocalFSStoreConfig{params} { } @@ -128,24 +147,21 @@ std::string MountedSSHStoreConfig::doc() * The difference lies in how they manage GC roots. See addPermRoot * below for details. */ -class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSHStore, public virtual LocalFSStore +struct MountedSSHStore : virtual MountedSSHStoreConfig, virtual SSHStore, virtual LocalFSStore { -public: - - MountedSSHStore( - std::string_view scheme, - std::string_view host, - const Params & params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) - , MountedSSHStoreConfig(params) - , Store(params) - , RemoteStore(params) - , SSHStore(scheme, host, params) - , LocalFSStore(params) + using Config = MountedSSHStoreConfig; + + MountedSSHStore(const Config & config) + : Store::Config{config} + , RemoteStore::Config{config} + , CommonSSHStoreConfig{config} + , SSHStore::Config{config} + , LocalFSStore::Config{config} + , MountedSSHStore::Config{config} + , Store{static_cast(*this)} + , RemoteStore{static_cast(*this)} + , SSHStore{static_cast(*this)} + , LocalFSStore{static_cast(*this)} { extraRemoteProgramArgs = { "--process-ops", @@ -215,7 +231,7 @@ ref SSHStore::openConnection() return conn; } -static RegisterStoreImplementation regSSHStore; -static RegisterStoreImplementation regMountedSSHStore; +static RegisterStoreImplementation regSSHStore; +static RegisterStoreImplementation regMountedSSHStore; } diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh index 29a2a8b2c2d..c7ed62d7577 100644 --- a/src/libstore/ssh-store.hh +++ b/src/libstore/ssh-store.hh @@ -8,15 +8,26 @@ namespace nix { -struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig +template class F> +struct SSHStoreConfigT { - using CommonSSHStoreConfig::CommonSSHStoreConfig; - using RemoteStoreConfig::RemoteStoreConfig; + const F remoteProgram; +}; + +struct SSHStoreConfig : virtual RemoteStoreConfig, + virtual CommonSSHStoreConfig, + SSHStoreConfigT +{ + struct Descriptions : virtual CommonSSHStoreConfig::Descriptions, SSHStoreConfigT + { + Descriptions(); + }; - SSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); + static const Descriptions descriptions; - const Setting remoteProgram{ - this, {"nix-daemon"}, "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; + static SSHStoreConfigT defaults; + + SSHStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params); const std::string name() override { @@ -29,16 +40,20 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig } std::string doc() override; + + ref openStore() const override; }; -struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig +struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStore::Config { - using LocalFSStoreConfig::LocalFSStoreConfig; - using SSHStoreConfig::SSHStoreConfig; + struct Descriptions : virtual SSHStoreConfig::Descriptions, virtual LocalFSStore::Config::Descriptions + { + Descriptions(); + }; - MountedSSHStoreConfig(StringMap params); + static const Descriptions descriptions; - MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params); + MountedSSHStoreConfig(std::string_view scheme, std::string_view host, const StoreReference::Params & params); const std::string name() override { @@ -56,6 +71,8 @@ struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfi { return ExperimentalFeature::MountedSSHStore; } + + ref openStore() const override; }; } From 469ac00e91d3dc8ba07fdc989dba3c8a50bb309e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 16:06:12 -0400 Subject: [PATCH 029/284] Convert `UDSRemoteStore` --- src/libstore/uds-remote-store.cc | 34 ++++++++++++-------------------- src/libstore/uds-remote-store.hh | 30 ++++++++++++++++------------ 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 3c445eb1318..6f6dd70f6e2 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,6 +1,7 @@ #include "uds-remote-store.hh" #include "unix-domain-socket.hh" #include "worker-protocol.hh" +#include "store-registration.hh" #include #include @@ -20,13 +21,13 @@ namespace nix { UDSRemoteStoreConfig::UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params) + const StoreReference::Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) , RemoteStoreConfig(params) , path{authority.empty() ? settings.nixDaemonSocketFile : authority} { - if (scheme != UDSRemoteStoreConfig::scheme) { + if (uriSchemes().count(std::string{scheme}) == 0) { throw UsageError("Scheme must be 'unix'"); } } @@ -40,23 +41,14 @@ std::string UDSRemoteStoreConfig::doc() } -// A bit gross that we now pass empty string but this is knowing that -// empty string will later default to the same nixDaemonSocketFile. Why -// don't we just wire it all through? I believe there are cases where it -// will live reload so we want to continue to account for that. -UDSRemoteStore::UDSRemoteStore(const Params & params) - : UDSRemoteStore(scheme, "", params) -{} - - -UDSRemoteStore::UDSRemoteStore(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RemoteStoreConfig(params) - , UDSRemoteStoreConfig(scheme, authority, params) - , Store(params) - , LocalFSStore(params) - , RemoteStore(params) +UDSRemoteStore::UDSRemoteStore(const Config & config) + : Store::Config{config} + , LocalFSStore::Config{config} + , RemoteStore::Config{config} + , UDSRemoteStore::Config{config} + , Store(static_cast(*this)) + , LocalFSStore(static_cast(*this)) + , RemoteStore(static_cast(*this)) { } @@ -69,7 +61,7 @@ std::string UDSRemoteStore::getUri() // // unix:// with no path also works. Change what we return? "daemon" - : std::string(scheme) + "://" + path; + : std::string(*uriSchemes().begin()) + "://" + path; } @@ -106,6 +98,6 @@ void UDSRemoteStore::addIndirectRoot(const Path & path) } -static RegisterStoreImplementation regUDSRemoteStore; +static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 8d19ad5f5bc..69fbf6f60a1 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -8,13 +8,17 @@ namespace nix { struct UDSRemoteStoreConfig : - virtual LocalFSStoreConfig, - virtual RemoteStoreConfig + virtual LocalFSStore::Config, + virtual RemoteStore::Config { - // TODO(fzakaria): Delete this constructor once moved over to the factory pattern - // outlined in https://github.com/NixOS/nix/issues/10766 - using LocalFSStoreConfig::LocalFSStoreConfig; - using RemoteStoreConfig::RemoteStoreConfig; + struct Descriptions : + virtual LocalFSStore::Config::Descriptions, + virtual RemoteStore::Config::Descriptions + { + Descriptions(); + }; + + static const Descriptions descriptions; /** * @param authority is the socket path. @@ -22,7 +26,7 @@ struct UDSRemoteStoreConfig : UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params); + const StoreReference::Params & params); const std::string name() override { return "Local Daemon Store"; } @@ -36,12 +40,10 @@ struct UDSRemoteStoreConfig : */ Path path; -private: - static constexpr char const * scheme = "unix"; - -public: static std::set uriSchemes() - { return {scheme}; } + { return {"unix"}; } + + ref openStore() const override; }; struct UDSRemoteStore : @@ -49,7 +51,9 @@ struct UDSRemoteStore : virtual IndirectRootStore, virtual RemoteStore { - UDSRemoteStore(const UDSRemoteStoreConfig &); + using Config = UDSRemoteStoreConfig; + + UDSRemoteStore(const Config &); std::string getUri() override; From 2fcb196c8e34b00ec18abe8306db05990b374c1f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 16:38:31 -0400 Subject: [PATCH 030/284] Fix `StoreReference` --- src/libstore/store-reference.cc | 37 +++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc index 5e5d30fc117..40a2867ed31 100644 --- a/src/libstore/store-reference.cc +++ b/src/libstore/store-reference.cc @@ -35,20 +35,49 @@ std::string StoreReference::render() const }, variant); + StringMap params2; + for (auto & [k, v] : params) { + auto * p = v.get_ptr(); + // if it is a JSON string, just use that + + // FIXME: Ensure the literal string isn't itself valid JSON. If + // it is, we still need to dump to escape it. + params2.insert_or_assign(k, p ? *p : v.dump()); + } + if (!params.empty()) { res += "?"; - res += encodeQuery(params); + res += encodeQuery(params2); } return res; } +static StoreReference::Params decodeParamsJson(StringMap paramsRaw) +{ + StoreReference::Params params; + for (auto && [k, v] : std::move(paramsRaw)) { + nlohmann::json j; + try { + j = nlohmann::json::parse(v); + } catch (nlohmann::json::exception &) { + // if its not valid JSON, keep the literal string + j = std::move(v); + } + params.insert_or_assign(std::move(k), std::move(j)); + } + return params; +} + StoreReference StoreReference::parse(const std::string & uri, const StoreReference::Params & extraParams) { auto params = extraParams; try { auto parsedUri = parseURL(uri); - params.insert(parsedUri.query.begin(), parsedUri.query.end()); + { + auto params2 = decodeParamsJson(std::move(parsedUri.query)); + params.insert(params2.begin(), params2.end()); + } auto baseURI = parsedUri.authority.value_or("") + parsedUri.path; @@ -106,13 +135,13 @@ StoreReference StoreReference::parse(const std::string & uri, const StoreReferen std::pair splitUriAndParams(const std::string & uri_) { auto uri(uri_); - StoreReference::Params params; + StringMap params; auto q = uri.find('?'); if (q != std::string::npos) { params = decodeQuery(uri.substr(q + 1)); uri = uri_.substr(0, q); } - return {uri, params}; + return {uri, decodeParamsJson(std::move(params))}; } } From 103c90b96714955a66732b3bbeb55feeac91e517 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 17:14:26 -0400 Subject: [PATCH 031/284] More progress --- src/libstore/local-overlay-store.cc | 73 ++++++++++++++++++---- src/libstore/local-overlay-store.hh | 96 +++++++++++------------------ 2 files changed, 98 insertions(+), 71 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index ec2c5f4e936..19eaa04c126 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -1,12 +1,63 @@ +#include + #include "local-overlay-store.hh" #include "callback.hh" #include "realisation.hh" #include "processes.hh" #include "url.hh" -#include +#include "store-open.hh" +#include "store-registration.hh" namespace nix { +#if 0 + const Setting lowerStoreUri{(StoreConfig*) this, "", "lower-store", + R"( + [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). + + Must be a store with a store dir on the file system. + Must be used as OverlayFS lower layer for this store's store dir. + )"}; + + const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer", + R"( + Directory containing the OverlayFS upper layer for this store's store dir. + )"}; + + Setting checkMount{(StoreConfig*) this, true, "check-mount", + R"( + Check that the overlay filesystem is correctly mounted. + + Nix does not manage the overlayfs mount point itself, but the correct + functioning of the overlay store does depend on this mount point being set up + correctly. Rather than just assume this is the case, check that the lowerdir + and upperdir options are what we expect them to be. This check is on by + default, but can be disabled if needed. + )"}; + + const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook", + R"( + Script or other executable to run when overlay filesystem needs remounting. + + This is occasionally necessary when deleting a store path that exists in both upper and lower layers. + In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly + is the only way to perform the deletion without creating a "whiteout". + However this causes the OverlayFS kernel data structures to get out-of-sync, + and can lead to 'stale file handle' errors; remounting solves the problem. + + The store directory is passed as an argument to the invoked executable. + )"}; + + : Store::Config{config} + , LocalFSStore::Configcpath, params) + , LocalStoreConfig(params) + , LocalOverlayStoreConfig(scheme, path, params) + , Store(params) + , LocalFSStore(params) + , LocalStore(params) +#endif + std::string LocalOverlayStoreConfig::doc() { return @@ -18,15 +69,15 @@ Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { return upperLayer + "/" + path.to_string(); } -LocalOverlayStore::LocalOverlayStore(std::string_view scheme, PathView path, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(path, params) - , LocalStoreConfig(params) - , LocalOverlayStoreConfig(scheme, path, params) - , Store(params) - , LocalFSStore(params) - , LocalStore(params) - , lowerStore(openStore(percentDecode(lowerStoreUri.get())).dynamic_pointer_cast()) +LocalOverlayStore::LocalOverlayStore(const Config & config) + : Store::Config{config} + , LocalFSStore::Config{config} + , LocalStore::Config{config} + , LocalOverlayStore::Config{config} + , Store{static_cast(*this)} + , LocalFSStore{static_cast(*this)} + , LocalStore{static_cast(*this)} + , lowerStore(nix::openStore(lowerStoreUri.get()).dynamic_pointer_cast()) { if (checkMount.get()) { std::smatch match; @@ -287,6 +338,6 @@ void LocalOverlayStore::remountIfNecessary() } -static RegisterStoreImplementation regLocalOverlayStore; +static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 63628abed50..927d006027f 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -2,59 +2,39 @@ namespace nix { + +template class F> +struct LocalOverlayStoreConfigT +{ + const F lowerStoreUri; + const F upperLayer; + const F checkMount; + const F remountHook; +}; + /** * Configuration for `LocalOverlayStore`. */ -struct LocalOverlayStoreConfig : virtual LocalStoreConfig +struct LocalOverlayStoreConfig : + virtual LocalStoreConfig, + LocalOverlayStoreConfigT { - LocalOverlayStoreConfig(const StringMap & params) - : LocalOverlayStoreConfig("local-overlay", "", params) - { } - - LocalOverlayStoreConfig(std::string_view scheme, PathView path, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(path, params) - , LocalStoreConfig(scheme, path, params) + struct Descriptions : + virtual Store::Config::Descriptions, + virtual LocalStore::Config::Descriptions, + LocalOverlayStoreConfigT { - } - - const Setting lowerStoreUri{(StoreConfig*) this, "", "lower-store", - R"( - [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) - for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). - - Must be a store with a store dir on the file system. - Must be used as OverlayFS lower layer for this store's store dir. - )"}; + Descriptions(); + }; - const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer", - R"( - Directory containing the OverlayFS upper layer for this store's store dir. - )"}; + static const Descriptions descriptions; - Setting checkMount{(StoreConfig*) this, true, "check-mount", - R"( - Check that the overlay filesystem is correctly mounted. + static LocalStoreConfigT defaults; - Nix does not manage the overlayfs mount point itself, but the correct - functioning of the overlay store does depend on this mount point being set up - correctly. Rather than just assume this is the case, check that the lowerdir - and upperdir options are what we expect them to be. This check is on by - default, but can be disabled if needed. - )"}; - - const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook", - R"( - Script or other executable to run when overlay filesystem needs remounting. - - This is occasionally necessary when deleting a store path that exists in both upper and lower layers. - In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly - is the only way to perform the deletion without creating a "whiteout". - However this causes the OverlayFS kernel data structures to get out-of-sync, - and can lead to 'stale file handle' errors; remounting solves the problem. - - The store directory is passed as an argument to the invoked executable. - )"}; + LocalOverlayStoreConfig( + std::string_view scheme, + PathView path, + const StoreReference::Params & params); const std::string name() override { return "Experimental Local Overlay Store"; } @@ -88,8 +68,18 @@ protected: * Documentation on overridden methods states how they differ from their * `LocalStore` counterparts. */ -class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual LocalStore +struct LocalOverlayStore : virtual LocalOverlayStoreConfig, virtual LocalStore { + using Config = LocalOverlayStoreConfig; + + LocalOverlayStore(const Config &); + + std::string getUri() override + { + return "local-overlay://"; + } + +private: /** * The store beneath us. * @@ -99,20 +89,6 @@ class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual */ ref lowerStore; -public: - LocalOverlayStore(const Params & params) - : LocalOverlayStore("local-overlay", "", params) - { - } - - LocalOverlayStore(std::string_view scheme, PathView path, const Params & params); - - std::string getUri() override - { - return "local-overlay://"; - } - -private: /** * First copy up any lower store realisation with the same key, so we * merge rather than mask it. From 3a149b521dee082a085d3afc077590bb88143d37 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 17:32:31 -0400 Subject: [PATCH 032/284] Getting close! --- src/libstore/store-open.hh | 9 ++++- src/libstore/store-registration.cc | 40 ++++++++++++------- src/libstore/store-registration.hh | 6 +-- .../functional/test-libstoreconsumer/main.cc | 2 +- tests/unit/libstore/ssh-store.cc | 4 +- 5 files changed, 38 insertions(+), 23 deletions(-) diff --git a/src/libstore/store-open.hh b/src/libstore/store-open.hh index ab6f3cb7739..c074b846bcc 100644 --- a/src/libstore/store-open.hh +++ b/src/libstore/store-open.hh @@ -15,6 +15,11 @@ namespace nix { +/** + * @return The store config denoted `uri` (slight misnomer...). + */ +ref resolveStoreConfig(StoreReference && storeURI); + /** * @return a Store object to access the Nix store denoted by * ‘uri’ (slight misnomer...). @@ -22,8 +27,8 @@ namespace nix { ref openStore(StoreReference && storeURI); /** - * Opens the store at `uri`, where `uri` is in the format expected by `StoreReference::parse` - + * Opens the store at `uri`, where `uri` is in the format expected by + * `StoreReference::parse` */ ref openStore( const std::string & uri = settings.storeUri.get(), diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc index 85ece9623cf..3b23debe9b8 100644 --- a/src/libstore/store-registration.cc +++ b/src/libstore/store-registration.cc @@ -2,26 +2,40 @@ #include "store-open.hh" #include "local-store.hh" #include "uds-remote-store.hh" +#include "json-utils.hh" namespace nix { -ref openStore(const std::string & uri, const Store::Params & extraParams) +ref openStore(const std::string & uri, const StoreReference::Params & extraParams) { return openStore(StoreReference::parse(uri, extraParams)); } ref openStore(StoreReference && storeURI) +{ + auto store = resolveStoreConfig(std::move(storeURI))->openStore(); + + experimentalFeatureSettings.require(store->experimentalFeature()); +#if 0 // FIXME + store->warnUnknownSettings(); + store->init(); +#endif + + return store; +} + +ref resolveStoreConfig(StoreReference && storeURI) { auto & params = storeURI.params; - auto store = std::visit( + auto storeConfig = std::visit( overloaded{ - [&](const StoreReference::Auto &) -> std::shared_ptr { - auto stateDir = getOr(params, "state", settings.nixStateDir); + [&](const StoreReference::Auto &) -> ref { + auto stateDir = getString(getOr(params, "state", settings.nixStateDir)); if (access(stateDir.c_str(), R_OK | W_OK) == 0) - return std::make_shared(params); + return make_ref(params); else if (pathExists(settings.nixDaemonSocketFile)) - return std::make_shared(params); + return make_ref("unix", "", params); #if __linux__ else if ( !pathExists(stateDir) && params.empty() && !isRootUser() && !getEnv("NIX_STORE_DIR").has_value() @@ -34,32 +48,28 @@ ref openStore(StoreReference && storeURI) try { createDirs(chrootStore); } catch (SystemError & e) { - return std::make_shared(params); + return make_ref(params); } warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); } else debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); - return std::make_shared("local", chrootStore, params); + return make_ref("local", chrootStore, params); } #endif else - return std::make_shared(params); + return make_ref(params); }, [&](const StoreReference::Specified & g) { for (auto implem : *Implementations::registered) if (implem.uriSchemes.count(g.scheme)) - return implem.create(g.scheme, g.authority, params); + return implem.parseConfig(g.scheme, g.authority, params); throw Error("don't know how to open Nix store with scheme '%s'", g.scheme); }, }, storeURI.variant); - experimentalFeatureSettings.require(store->experimentalFeature()); - store->warnUnknownSettings(); - store->init(); - - return ref{store}; + return storeConfig; } std::vector * Implementations::registered = 0; diff --git a/src/libstore/store-registration.hh b/src/libstore/store-registration.hh index 8d57f4a587a..dd61a45f1ae 100644 --- a/src/libstore/store-registration.hh +++ b/src/libstore/store-registration.hh @@ -18,7 +18,7 @@ struct StoreFactory * The `authorityPath` parameter is `/`, or really * whatever comes after `://` and before `?`. */ - std::function( + std::function( std::string_view scheme, std::string_view authorityPath, const StoreReference::Params & params)> parseConfig; const Store::Config::Descriptions & configDescriptions; @@ -35,8 +35,8 @@ struct Implementations registered = new std::vector(); StoreFactory factory{ .uriSchemes = T::Config::uriSchemes(), - .parseConfig = ([](auto scheme, auto uri, auto & params) -> std::shared_ptr { - return std::make_shared(scheme, uri, params); + .parseConfig = ([](auto scheme, auto uri, auto & params) -> ref { + return make_ref(scheme, uri, params); }), .configDescriptions = T::Config::descriptions, }; diff --git a/tests/functional/test-libstoreconsumer/main.cc b/tests/functional/test-libstoreconsumer/main.cc index c61489af69a..00763038e9f 100644 --- a/tests/functional/test-libstoreconsumer/main.cc +++ b/tests/functional/test-libstoreconsumer/main.cc @@ -1,5 +1,5 @@ #include "globals.hh" -#include "store-api.hh" +#include "store-open.hh" #include "build-result.hh" #include diff --git a/tests/unit/libstore/ssh-store.cc b/tests/unit/libstore/ssh-store.cc index a1fcef6b863..dad20025873 100644 --- a/tests/unit/libstore/ssh-store.cc +++ b/tests/unit/libstore/ssh-store.cc @@ -9,7 +9,7 @@ TEST(SSHStore, constructConfig) SSHStoreConfig config{ "ssh", "localhost", - StoreConfig::Params{ + StoreReference::Params{ { "remote-program", // TODO #11106, no more split on space @@ -31,7 +31,7 @@ TEST(MountedSSHStore, constructConfig) MountedSSHStoreConfig config{ "mounted-ssh", "localhost", - StoreConfig::Params{ + StoreReference::Params{ { "remote-program", // TODO #11106, no more split on space From 31e151386b1ce086de6a6ec4a6f0688b1b5dced6 Mon Sep 17 00:00:00 2001 From: Emily Date: Thu, 18 Jul 2024 17:56:12 +0100 Subject: [PATCH 033/284] libmain: add missing header include --- src/libmain/plugin.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libmain/plugin.cc b/src/libmain/plugin.cc index 52b5b60e415..ccfd7f9003a 100644 --- a/src/libmain/plugin.cc +++ b/src/libmain/plugin.cc @@ -2,6 +2,8 @@ # include #endif +#include + #include "config-global.hh" #include "signals.hh" From a6dccae2235ce4bb6a1f1552770eb59ff96f7e05 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 17 Jul 2024 02:05:38 +0200 Subject: [PATCH 034/284] Fix non-deterministic parser printing In _very_ rare cases (I had about 7 cases out of 32200 files!), the order of how inherit-from bindings are printed when using `nix-instantiate --parse` gets messed up. The cause of this seems to be because the std::map the bindings are placed in is keyed on a _pointer_, which then uses an [implementation-defined strict total order](https://en.cppreference.com/w/cpp/language/operator_comparison#Pointer_total_order). The fix here is to key the bindings on their displacement instead, which maintains the same order as they appear in the file. Unfortunately I wasn't able to make a reproducible test for this in the source, there's something about the local environment that makes it unreproducible for me. However I was able to make a reproducible test in a Nix build on a Nix version from a very recent master: nix build github:infinisil/non-det-nix-parsing-repro Co-authored-by: Robert Hensing --- src/libexpr/nixexpr.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 93c8bdef6b6..6c6769cfd44 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -82,7 +82,9 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co return sa < sb; }); std::vector inherits; - std::map> inheritsFrom; + // We can use the displacement as a proxy for the order in which the symbols were parsed. + // The assignment of displacements should be deterministic, so that showBindings is deterministic. + std::map> inheritsFrom; for (auto & i : sorted) { switch (i->second.kind) { case AttrDef::Kind::Plain: @@ -93,7 +95,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co case AttrDef::Kind::InheritedFrom: { auto & select = dynamic_cast(*i->second.e); auto & from = dynamic_cast(*select.e); - inheritsFrom[&from].push_back(i->first); + inheritsFrom[from.displ].push_back(i->first); break; } } @@ -105,7 +107,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co } for (const auto & [from, syms] : inheritsFrom) { str << "inherit ("; - (*inheritFromExprs)[from->displ]->show(symbols, str); + (*inheritFromExprs)[from]->show(symbols, str); str << ")"; for (auto sym : syms) str << " " << symbols[sym]; str << "; "; From 0c91bb97e5579169fdac94ebfbb352ee1cc4a5ae Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Wed, 17 Jul 2024 02:43:42 +0200 Subject: [PATCH 035/284] parser: Remove empty multiline string parts earlier Makes parsing more consistent and is a super minor optimisation Co-authored-by: Robert Hensing --- src/libexpr/parser-state.hh | 13 ++++++++++++- tests/functional/lang/parse-okay-ind-string.exp | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 983a17a2e98..d8287db9b27 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -291,12 +291,23 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos, s2 = std::string(s2, 0, p + 1); } - es2->emplace_back(i->first, new ExprString(std::move(s2))); + // Ignore empty strings for a minor optimisation and AST simplification + if (s2 != "") { + es2->emplace_back(i->first, new ExprString(std::move(s2))); + } }; for (; i != es.end(); ++i, --n) { std::visit(overloaded { trimExpr, trimString }, i->second); } + // If there is nothing at all, return the empty string directly. + // This also ensures that equivalent empty strings result in the same ast, which is helpful when testing formatters. + if (es2->size() == 0) { + auto *const result = new ExprString(""); + delete es2; + return result; + } + /* If this is a single string, then don't do a concatenation. */ if (es2->size() == 1 && dynamic_cast((*es2)[0].second)) { auto *const result = (*es2)[0].second; diff --git a/tests/functional/lang/parse-okay-ind-string.exp b/tests/functional/lang/parse-okay-ind-string.exp index 5f8d4868830..82e9940a2e9 100644 --- a/tests/functional/lang/parse-okay-ind-string.exp +++ b/tests/functional/lang/parse-okay-ind-string.exp @@ -1 +1 @@ -(let string = "str"; in [ (/some/path) ((/some/path)) (("" + /some/path)) ((/some/path + "\n end")) (string) ((string)) (("" + string)) ((string + "\n end")) ("") ("") ("end") ]) +(let string = "str"; in [ (/some/path) ((/some/path)) ((/some/path)) ((/some/path + "\n end")) (string) ((string)) ((string)) ((string + "\n end")) ("") ("") ("end") ]) From 68ae4dc3fd9c2fa4b2e8f766e9b9b178599b703c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 18:57:41 -0400 Subject: [PATCH 036/284] It builds, debugging --- src/libstore/binary-cache-store.cc | 8 +- src/libstore/http-binary-cache-store.cc | 6 +- src/libstore/legacy-ssh-store.cc | 7 +- src/libstore/legacy-ssh-store.hh | 5 +- src/libstore/local-binary-cache-store.cc | 4 + src/libstore/local-overlay-store.cc | 124 ++++++++++++++--------- src/libstore/local-overlay-store.hh | 2 +- src/libstore/local-store.hh | 4 +- src/libstore/s3-binary-cache-store.cc | 7 +- src/libstore/ssh-store.cc | 21 ++++ src/libstore/store-api.cc | 2 + src/libstore/store-api.hh | 16 ++- src/libstore/store-reference.cc | 10 +- src/libstore/store-registration.cc | 2 +- src/libstore/uds-remote-store.cc | 15 +++ src/libstore/uds-remote-store.hh | 4 + tests/unit/libstore/legacy-ssh-store.cc | 2 +- 17 files changed, 174 insertions(+), 65 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4adc599007e..08367595842 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -101,6 +101,8 @@ BinaryCacheStore::BinaryCacheStore(const Config & config) StringSink sink; sink << narVersionMagic1; narMagic = sink.s; + + init(); } void BinaryCacheStore::init() @@ -121,9 +123,11 @@ void BinaryCacheStore::init() throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'", getUri(), value, storeDir); } else if (name == "WantMassQuery") { - wantMassQuery.setDefault(value == "1"); + if (defaultWantMassQuery) + wantMassQuery.value = value == "1"; } else if (name == "Priority") { - priority.setDefault(std::stoi(value)); + if (defaultPriority) + priority.value = std::stoi(value); } } } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index accd84899f1..8d9a9d96335 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -71,8 +71,10 @@ class HttpBinaryCacheStore : { // FIXME: do this lazily? if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) { - wantMassQuery.setDefault(cacheInfo->wantMassQuery); - priority.setDefault(cacheInfo->priority); + if (defaultWantMassQuery) + wantMassQuery.value = cacheInfo->wantMassQuery; + if (defaultPriority) + priority.value = cacheInfo->priority; } else { try { BinaryCacheStore::init(); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 10301b79e66..d95a155f5c4 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -345,12 +345,17 @@ unsigned int LegacySSHStore::getProtocol() * The legacy ssh protocol doesn't support checking for trusted-user. * Try using ssh-ng:// instead if you want to know. */ -std::optional isTrustedClient() +std::optional LegacySSHStore::isTrustedClient() { return std::nullopt; } +ref LegacySSHStore::Config::openStore() const { + return make_ref(*this); +} + + static RegisterStoreImplementation regLegacySSHStore; } diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index 9d2234faaf7..779798390d8 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -137,10 +137,7 @@ public: * The legacy ssh protocol doesn't support checking for trusted-user. * Try using ssh-ng:// instead if you want to know. */ - std::optional isTrustedClient() override - { - return std::nullopt; - } + std::optional isTrustedClient() override; void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept override diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 8fb163e3064..c00a34d2bce 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -125,6 +125,10 @@ std::set LocalBinaryCacheStoreConfig::uriSchemes() return {"file"}; } +ref LocalBinaryCacheStoreConfig::openStore() const { + return make_ref(*this); +} + static RegisterStoreImplementation regLocalBinaryCacheStore; } diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 19eaa04c126..ade81b20a9f 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -7,56 +7,86 @@ #include "url.hh" #include "store-open.hh" #include "store-registration.hh" +#include "config-parse-impl.hh" namespace nix { -#if 0 - const Setting lowerStoreUri{(StoreConfig*) this, "", "lower-store", - R"( - [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) - for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). - - Must be a store with a store dir on the file system. - Must be used as OverlayFS lower layer for this store's store dir. - )"}; - - const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer", - R"( - Directory containing the OverlayFS upper layer for this store's store dir. - )"}; - - Setting checkMount{(StoreConfig*) this, true, "check-mount", - R"( - Check that the overlay filesystem is correctly mounted. - - Nix does not manage the overlayfs mount point itself, but the correct - functioning of the overlay store does depend on this mount point being set up - correctly. Rather than just assume this is the case, check that the lowerdir - and upperdir options are what we expect them to be. This check is on by - default, but can be disabled if needed. - )"}; - - const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook", - R"( - Script or other executable to run when overlay filesystem needs remounting. - - This is occasionally necessary when deleting a store path that exists in both upper and lower layers. - In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly - is the only way to perform the deletion without creating a "whiteout". - However this causes the OverlayFS kernel data structures to get out-of-sync, - and can lead to 'stale file handle' errors; remounting solves the problem. - - The store directory is passed as an argument to the invoked executable. - )"}; - - : Store::Config{config} - , LocalFSStore::Configcpath, params) - , LocalStoreConfig(params) - , LocalOverlayStoreConfig(scheme, path, params) - , Store(params) - , LocalFSStore(params) - , LocalStore(params) -#endif +LocalOverlayStore::Config::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , LocalFSStore::Config::Descriptions{LocalFSStore::Config::descriptions} + , LocalStore::Config::Descriptions{LocalStore::Config::descriptions} + , LocalOverlayStoreConfigT{ + .lowerStoreUri{ + .name = "lower-store", + .description = R"( + [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). + + Must be a store with a store dir on the file system. + Must be used as OverlayFS lower layer for this store's store dir. + )", + }, + .upperLayer{ + .name = "upper-layer", + .description = R"( + Directory containing the OverlayFS upper layer for this store's store dir. + )", + }, + .checkMount{ + .name = "check-mount", + .description = R"( + Check that the overlay filesystem is correctly mounted. + + Nix does not manage the overlayfs mount point itself, but the correct + functioning of the overlay store does depend on this mount point being set up + correctly. Rather than just assume this is the case, check that the lowerdir + and upperdir options are what we expect them to be. This check is on by + default, but can be disabled if needed. + )", + }, + .remountHook{ + .name = "remount-hook", + .description = R"( + Script or other executable to run when overlay filesystem needs remounting. + + This is occasionally necessary when deleting a store path that exists in both upper and lower layers. + In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly + is the only way to perform the deletion without creating a "whiteout". + However this causes the OverlayFS kernel data structures to get out-of-sync, + and can lead to 'stale file handle' errors; remounting solves the problem. + + The store directory is passed as an argument to the invoked executable. + )", + }, + } +{} + + +const LocalOverlayStore::Config::Descriptions LocalOverlayStore::Config::descriptions{}; + + +decltype(LocalOverlayStore::Config::defaults) LocalOverlayStore::Config::defaults = { + .lowerStoreUri{""}, + .upperLayer{""}, + .checkMount{true}, + .remountHook{""}, +}; + +LocalOverlayStore::Config::LocalOverlayStoreConfig( + std::string_view scheme, + std::string_view authority, + const StoreReference::Params & params) + : Store::Config(params) + , LocalFSStore::Config(authority, params) + , LocalStore::Config(scheme, authority, params) + , LocalOverlayStoreConfigT{ + CONFIG_ROW(lowerStoreUri), + CONFIG_ROW(upperLayer), + CONFIG_ROW(checkMount), + CONFIG_ROW(remountHook), + } +{ +} std::string LocalOverlayStoreConfig::doc() { diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 927d006027f..4cf898b8fc7 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -29,7 +29,7 @@ struct LocalOverlayStoreConfig : static const Descriptions descriptions; - static LocalStoreConfigT defaults; + static LocalOverlayStoreConfigT defaults; LocalOverlayStoreConfig( std::string_view scheme, diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 3d5aa1d3dbd..ca483df1584 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -58,7 +58,9 @@ struct LocalStoreConfig : static LocalStoreConfigT defaults; - LocalStoreConfig(const StoreReference::Params &); + LocalStoreConfig(const StoreReference::Params & params) + : LocalStoreConfig{"local", "", params} + {} LocalStoreConfig( std::string_view scheme, diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 6b9a809af96..7f8ce3a7bdd 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -290,6 +290,7 @@ S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig( CONFIG_ROW(multipartUpload), CONFIG_ROW(bufferSize), } + , bucketName{authority} { if (bucketName.empty()) throw UsageError("`%s` store requires a bucket name in its Store URI", scheme); @@ -334,8 +335,10 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual void init() override { if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) { - wantMassQuery.setDefault(cacheInfo->wantMassQuery); - priority.setDefault(cacheInfo->priority); + if (defaultWantMassQuery) + wantMassQuery.value = cacheInfo->wantMassQuery; + if (defaultPriority) + priority.value = cacheInfo->priority; } else { BinaryCacheStore::init(); diskCache->createCache(getUri(), storeDir, wantMassQuery, priority); diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index eb97360160f..0e1af43a131 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -113,6 +113,21 @@ struct SSHStore : }; +ref SSHStore::Config::openStore() const { + return make_ref(*this); +} + + +MountedSSHStoreConfig::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , CommonSSHStoreConfig::Descriptions{CommonSSHStoreConfig::descriptions} + , SSHStore::Config::Descriptions{SSHStoreConfig::descriptions} +{} + + +const MountedSSHStoreConfig::Descriptions MountedSSHStoreConfig::descriptions{}; + + MountedSSHStoreConfig::MountedSSHStoreConfig( std::string_view scheme, std::string_view host, @@ -214,6 +229,12 @@ struct MountedSSHStore : virtual MountedSSHStoreConfig, virtual SSHStore, virtua } }; + +ref MountedSSHStore::Config::openStore() const { + return make_ref(*this); +} + + ref SSHStore::openConnection() { auto conn = make_ref(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b67faf38a5e..f51e7c27cf9 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -256,6 +256,8 @@ Store::Config::StoreConfig(const StoreReference::Params & params) CONFIG_ROW(wantMassQuery), CONFIG_ROW(systemFeatures), } + , defaultPriority{params.count(descriptions.priority.name) == 0} + , defaultWantMassQuery{params.count(descriptions.wantMassQuery.name) == 0} { } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 56fef0f1a7b..e2151ef5b42 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -103,7 +103,7 @@ struct StoreConfigT const F isTrusted; F priority; F wantMassQuery; - F systemFeatures; + const F systemFeatures; }; StoreConfigT parseStoreConfig(const StoreReference::Params &); @@ -155,6 +155,20 @@ struct StoreConfig : * type. */ virtual ref openStore() const = 0; + +protected: + + /** + * So caches themselves (not our config) can update default + * settings, but aren't allowed to update settings specied by the + * client (i.e. us). + */ + bool defaultPriority; + + /** + * @see defaultPriority + */ + bool defaultWantMassQuery; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc index 40a2867ed31..7be513f4fcc 100644 --- a/src/libstore/store-reference.cc +++ b/src/libstore/store-reference.cc @@ -61,8 +61,14 @@ static StoreReference::Params decodeParamsJson(StringMap paramsRaw) try { j = nlohmann::json::parse(v); } catch (nlohmann::json::exception &) { - // if its not valid JSON, keep the literal string - j = std::move(v); + // if its not valid JSON... + if (k == "remoteProgram" || k == "systemFeatures") { + // Back compat hack! Split and take that array + j = tokenizeString>(v); + } else { + // ...keep the literal string. + j = std::move(v); + } } params.insert_or_assign(std::move(k), std::move(j)); } diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc index 3b23debe9b8..1f3edfc71c1 100644 --- a/src/libstore/store-registration.cc +++ b/src/libstore/store-registration.cc @@ -35,7 +35,7 @@ ref resolveStoreConfig(StoreReference && storeURI) if (access(stateDir.c_str(), R_OK | W_OK) == 0) return make_ref(params); else if (pathExists(settings.nixDaemonSocketFile)) - return make_ref("unix", "", params); + return make_ref(params); #if __linux__ else if ( !pathExists(stateDir) && params.empty() && !isRootUser() && !getEnv("NIX_STORE_DIR").has_value() diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 6f6dd70f6e2..8604961fbe5 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -18,6 +18,16 @@ namespace nix { +UDSRemoteStoreConfig::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , LocalFSStore::Config::Descriptions{LocalFSStore::Config::descriptions} + , RemoteStore::Config::Descriptions{RemoteStore::Config::descriptions} +{} + + +const UDSRemoteStoreConfig::Descriptions UDSRemoteStoreConfig::descriptions{}; + + UDSRemoteStoreConfig::UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, @@ -98,6 +108,11 @@ void UDSRemoteStore::addIndirectRoot(const Path & path) } +ref UDSRemoteStore::Config::openStore() const { + return make_ref(*this); +} + + static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 69fbf6f60a1..4af4eac624e 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -20,6 +20,10 @@ struct UDSRemoteStoreConfig : static const Descriptions descriptions; + UDSRemoteStoreConfig(const StoreReference::Params & params) + : UDSRemoteStoreConfig{"unix", "", params} + {} + /** * @param authority is the socket path. */ diff --git a/tests/unit/libstore/legacy-ssh-store.cc b/tests/unit/libstore/legacy-ssh-store.cc index eb31a240804..4d0510cc656 100644 --- a/tests/unit/libstore/legacy-ssh-store.cc +++ b/tests/unit/libstore/legacy-ssh-store.cc @@ -9,7 +9,7 @@ TEST(LegacySSHStore, constructConfig) LegacySSHStoreConfig config{ "ssh", "localhost", - StoreConfig::Params{ + StoreReference::Params{ { "remote-program", // TODO #11106, no more split on space From 387767b9e944653c8c1439b4c25c78b02202842e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 19:07:45 -0400 Subject: [PATCH 037/284] Fix some bugs --- src/libstore/local-fs-store.cc | 2 +- src/libstore/ssh-store.hh | 4 +--- tests/unit/libstore/legacy-ssh-store.cc | 6 ++++-- tests/unit/libstore/ssh-store.cc | 12 ++++++++---- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 12035c1efe7..f30abe88d22 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -38,7 +38,7 @@ LocalFSStoreConfigT LocalFSStore::Config::defaults( const std::optional rootDir) { return { - .rootDir = {std::nullopt}, + .rootDir = {rootDir}, .stateDir = {rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir}, .logDir = {rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir}, .realStoreDir = {rootDir ? *rootDir + "/nix/store" : storeConfig.storeDir}, diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh index c7ed62d7577..6aa9c9cdf09 100644 --- a/src/libstore/ssh-store.hh +++ b/src/libstore/ssh-store.hh @@ -14,9 +14,7 @@ struct SSHStoreConfigT const F remoteProgram; }; -struct SSHStoreConfig : virtual RemoteStoreConfig, - virtual CommonSSHStoreConfig, - SSHStoreConfigT +struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig, SSHStoreConfigT { struct Descriptions : virtual CommonSSHStoreConfig::Descriptions, SSHStoreConfigT { diff --git a/tests/unit/libstore/legacy-ssh-store.cc b/tests/unit/libstore/legacy-ssh-store.cc index 4d0510cc656..ba877d10318 100644 --- a/tests/unit/libstore/legacy-ssh-store.cc +++ b/tests/unit/libstore/legacy-ssh-store.cc @@ -12,8 +12,10 @@ TEST(LegacySSHStore, constructConfig) StoreReference::Params{ { "remote-program", - // TODO #11106, no more split on space - "foo bar", + { + "foo", + "bar", + }, }, }}; EXPECT_EQ( diff --git a/tests/unit/libstore/ssh-store.cc b/tests/unit/libstore/ssh-store.cc index dad20025873..0e2651f2e9b 100644 --- a/tests/unit/libstore/ssh-store.cc +++ b/tests/unit/libstore/ssh-store.cc @@ -12,8 +12,10 @@ TEST(SSHStore, constructConfig) StoreReference::Params{ { "remote-program", - // TODO #11106, no more split on space - "foo bar", + { + "foo", + "bar", + }, }, }, }; @@ -34,8 +36,10 @@ TEST(MountedSSHStore, constructConfig) StoreReference::Params{ { "remote-program", - // TODO #11106, no more split on space - "foo bar", + { + "foo", + "bar", + }, }, }, }; From b53a4a0188b338c809f8f95b50f408657b607f29 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 21:57:35 -0400 Subject: [PATCH 038/284] More tests --- src/libstore/dummy-store.cc | 48 +++++++++--------------------- tests/unit/libstore/dummy-store.cc | 20 +++++++++++++ 2 files changed, 34 insertions(+), 34 deletions(-) create mode 100644 tests/unit/libstore/dummy-store.cc diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 3d2931cd9ba..e8934ec9a19 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -1,43 +1,23 @@ -#include "store-api.hh" +#include "dummy-store.hh" #include "store-registration.hh" #include "callback.hh" namespace nix { -struct DummyStoreConfig : virtual StoreConfig { - using StoreConfig::StoreConfig; - - struct Descriptions : virtual Store::Config::Descriptions - { - Descriptions() - : StoreConfig::Descriptions{Store::Config::descriptions} - {} - }; - - static const Descriptions descriptions; - - DummyStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params) - : StoreConfig{params} - { - if (!authority.empty()) - throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); - } - - const std::string name() override { return "Dummy Store"; } - - std::string doc() override - { - return - #include "dummy-store.md" - ; - } - - static std::set uriSchemes() { - return {"dummy"}; - } +DummyStoreConfig::DummyStoreConfig( + std::string_view scheme, std::string_view authority, const StoreReference::Params & params) + : StoreConfig{params} +{ + if (!authority.empty()) + throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); +} - ref openStore() const override; -}; +std::string DummyStoreConfig::doc() +{ + return + #include "dummy-store.md" + ; +} const DummyStoreConfig::Descriptions DummyStoreConfig::descriptions{}; diff --git a/tests/unit/libstore/dummy-store.cc b/tests/unit/libstore/dummy-store.cc new file mode 100644 index 00000000000..df005da61ca --- /dev/null +++ b/tests/unit/libstore/dummy-store.cc @@ -0,0 +1,20 @@ +#include + +#include "dummy-store.hh" +#include "globals.hh" + +namespace nix { + +TEST(DummyStore, constructConfig) +{ + DummyStoreConfig config{"dummy", "", {}}; + + EXPECT_EQ(config.storeDir, settings.nixStore); +} + +TEST(DummyStore, constructConfigNoAuthority) +{ + EXPECT_THROW(DummyStoreConfig("dummy", "not-allowed", {}), UsageError); +} + +} // namespace nix From 34f9b13ec2282e134b8aa1286b934c83a8631a86 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 22:50:36 -0400 Subject: [PATCH 039/284] Inline defaults This helps avoid reading globals before they are initialized --- src/libstore/binary-cache-store.cc | 22 ++++++------------ src/libstore/binary-cache-store.hh | 2 -- src/libstore/common-ssh-store-config.cc | 15 ++++--------- src/libstore/common-ssh-store-config.hh | 2 -- src/libstore/config-parse-impl.hh | 6 ++--- src/libstore/config-parse.hh | 2 +- src/libstore/legacy-ssh-store.cc | 10 ++------- src/libstore/legacy-ssh-store.hh | 2 -- src/libstore/local-fs-store.cc | 22 ++++-------------- src/libstore/local-fs-store.hh | 7 ------ src/libstore/local-overlay-store.cc | 18 ++++++--------- src/libstore/local-overlay-store.hh | 2 -- src/libstore/local-store.cc | 9 ++------ src/libstore/local-store.hh | 2 -- src/libstore/remote-store.cc | 10 ++------- src/libstore/remote-store.hh | 5 ----- src/libstore/s3-binary-cache-store.cc | 30 ++++++++----------------- src/libstore/s3-binary-cache-store.hh | 2 -- src/libstore/ssh-store.cc | 7 +----- src/libstore/ssh-store.hh | 2 -- src/libstore/store-api.cc | 19 +++++----------- src/libstore/store-api.hh | 2 -- src/libstore/store-dir-config.cc | 6 +---- src/libstore/store-dir-config.hh | 2 -- 24 files changed, 48 insertions(+), 158 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 08367595842..1bd9363a272 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -68,24 +68,16 @@ BinaryCacheStore::Config::Descriptions::Descriptions() const BinaryCacheStore::Config::Descriptions BinaryCacheStore::Config::descriptions{}; -decltype(BinaryCacheStore::Config::defaults) BinaryCacheStore::Config::defaults = { - .compression = {"xz"}, - .writeNARListing{false}, - .writeDebugInfo{false}, - .secretKeyFile{""}, - .localNarCache{""}, - .parallelCompression{false}, - .compressionLevel{-1}, -}; - BinaryCacheStore::Config::BinaryCacheStoreConfig(const StoreReference::Params & params) : StoreConfig{params} , BinaryCacheStoreConfigT{ - CONFIG_ROW(compression), - CONFIG_ROW(secretKeyFile), - CONFIG_ROW(localNarCache), - CONFIG_ROW(parallelCompression), - CONFIG_ROW(compressionLevel), + CONFIG_ROW(compression, "xz"), + CONFIG_ROW(writeNARListing, false), + CONFIG_ROW(writeDebugInfo, false), + CONFIG_ROW(secretKeyFile, ""), + CONFIG_ROW(localNarCache, ""), + CONFIG_ROW(parallelCompression, false), + CONFIG_ROW(compressionLevel, -1), } { } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 27d7a845ab7..877a5c3343c 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -38,8 +38,6 @@ struct BinaryCacheStoreConfig : static const Descriptions descriptions; - static BinaryCacheStoreConfigT defaults; - BinaryCacheStoreConfig(const StoreReference::Params &); }; diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index a2d7e2ebad6..113ba6b8b4e 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -35,13 +35,6 @@ CommonSSHStoreConfig::Descriptions::Descriptions() const CommonSSHStoreConfig::Descriptions CommonSSHStoreConfig::descriptions{}; -decltype(CommonSSHStoreConfig::defaults) CommonSSHStoreConfig::defaults = { - .sshKey{""}, - .sshPublicHostKey{""}, - .compress{false}, - .remoteStore{""}, -}; - static std::string extractConnStr(std::string_view scheme, std::string_view _connStr) { if (_connStr.empty()) @@ -65,10 +58,10 @@ CommonSSHStoreConfig::CommonSSHStoreConfig( const StoreReference::Params & params) : Store::Config(params) , CommonSSHStoreConfigT{ - CONFIG_ROW(sshKey), - CONFIG_ROW(sshPublicHostKey), - CONFIG_ROW(compress), - CONFIG_ROW(remoteStore), + CONFIG_ROW(sshKey, ""), + CONFIG_ROW(sshPublicHostKey, ""), + CONFIG_ROW(compress, false), + CONFIG_ROW(remoteStore, ""), } , host(extractConnStr(scheme, host)) { diff --git a/src/libstore/common-ssh-store-config.hh b/src/libstore/common-ssh-store-config.hh index 4def51da0e5..e7bb16367ff 100644 --- a/src/libstore/common-ssh-store-config.hh +++ b/src/libstore/common-ssh-store-config.hh @@ -29,8 +29,6 @@ struct CommonSSHStoreConfig : static const Descriptions descriptions; - static CommonSSHStoreConfigT defaults; - /** * @param scheme Note this isn't stored by this mix-in class, but * just used for better error messages. diff --git a/src/libstore/config-parse-impl.hh b/src/libstore/config-parse-impl.hh index ebb65305ae3..ed640019dbe 100644 --- a/src/libstore/config-parse-impl.hh +++ b/src/libstore/config-parse-impl.hh @@ -9,16 +9,16 @@ namespace nix::config { template -std::optional> SettingInfo::parseConfig(const nlohmann::json::object_t & map) const +std::optional SettingInfo::parseConfig(const nlohmann::json::object_t & map) const { auto * p = get(map, name); - return p ? (JustValue{.value = p->get()}) : (std::optional>{}); + return p ? p->get() : (std::optional{}); } /** * Look up the setting's name in a map, falling back on the default if * it does not exist. */ -#define CONFIG_ROW(FIELD) .FIELD = descriptions.FIELD.parseConfig(params).value_or(defaults.FIELD) +#define CONFIG_ROW(FIELD, DEFAULT) .FIELD = {.value = descriptions.FIELD.parseConfig(params).value_or(DEFAULT)} } diff --git a/src/libstore/config-parse.hh b/src/libstore/config-parse.hh index 5d17f9f8d6a..e016caf15d9 100644 --- a/src/libstore/config-parse.hh +++ b/src/libstore/config-parse.hh @@ -15,7 +15,7 @@ struct SettingInfo std::string description; bool documentDefault = true; - std::optional> parseConfig(const nlohmann::json::object_t & map) const; + std::optional parseConfig(const nlohmann::json::object_t & map) const; }; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index d95a155f5c4..0fc5181ff6d 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -36,12 +36,6 @@ LegacySSHStore::Config::Descriptions::Descriptions() const LegacySSHStore::Config::Descriptions LegacySSHStore::Config::descriptions{}; -decltype(LegacySSHStore::Config::defaults) LegacySSHStore::Config::defaults = { - .remoteProgram = {{"nix-store"}}, - .maxConnections = {1}, -}; - - LegacySSHStore::Config::LegacySSHStoreConfig( std::string_view scheme, std::string_view authority, @@ -49,8 +43,8 @@ LegacySSHStore::Config::LegacySSHStoreConfig( : Store::Config{params} , CommonSSHStoreConfig{scheme, authority, params} , LegacySSHStoreConfigT{ - CONFIG_ROW(remoteProgram), - CONFIG_ROW(maxConnections), + CONFIG_ROW(remoteProgram, Strings{"nix-store"}), + CONFIG_ROW(maxConnections, 1), } { #ifndef _WIN32 diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index 779798390d8..12ca9f12514 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -29,8 +29,6 @@ struct LegacySSHStoreConfig : static const Descriptions descriptions; - static LegacySSHStoreConfigT defaults; - /** * Hack for getting remote build log output. Intentionally not a * documented user-visible setting. diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index f30abe88d22..bedc0aca94e 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -33,18 +33,6 @@ LocalFSStore::Config::Descriptions::Descriptions() const LocalFSStore::Config::Descriptions LocalFSStore::Config::descriptions{}; -LocalFSStoreConfigT LocalFSStore::Config::defaults( - const Store::Config & storeConfig, - const std::optional rootDir) -{ - return { - .rootDir = {rootDir}, - .stateDir = {rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir}, - .logDir = {rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir}, - .realStoreDir = {rootDir ? *rootDir + "/nix/store" : storeConfig.storeDir}, - }; -} - /** * @param rootDir Fallback if not in `params` */ @@ -58,13 +46,11 @@ auto localFSStoreConfig( auto rootDir = descriptions.rootDir.parseConfig(params) .value_or(config::JustValue{.value = std::move(_rootDir)}); - auto defaults = LocalFSStore::Config::defaults(storeConfig, rootDir.value); - return LocalFSStoreConfigT{ - CONFIG_ROW(rootDir), - CONFIG_ROW(stateDir), - CONFIG_ROW(logDir), - CONFIG_ROW(realStoreDir), + .rootDir = std::move(rootDir), + CONFIG_ROW(stateDir, rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir), + CONFIG_ROW(logDir, rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir), + CONFIG_ROW(realStoreDir, rootDir ? *rootDir + "/nix/store" : storeConfig.storeDir), }; } diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 051e4494012..678e93ec0a4 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -29,13 +29,6 @@ struct LocalFSStoreConfig : static const Descriptions descriptions; - /** - * The other defaults depend on the choice of `storeDir` and `rootDir` - */ - static LocalFSStoreConfigT defaults( - const Store::Config &, - const std::optional rootDir); - LocalFSStoreConfig(const StoreReference::Params &); /** diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index ade81b20a9f..7a483304841 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -65,13 +65,6 @@ LocalOverlayStore::Config::Descriptions::Descriptions() const LocalOverlayStore::Config::Descriptions LocalOverlayStore::Config::descriptions{}; -decltype(LocalOverlayStore::Config::defaults) LocalOverlayStore::Config::defaults = { - .lowerStoreUri{""}, - .upperLayer{""}, - .checkMount{true}, - .remountHook{""}, -}; - LocalOverlayStore::Config::LocalOverlayStoreConfig( std::string_view scheme, std::string_view authority, @@ -80,14 +73,15 @@ LocalOverlayStore::Config::LocalOverlayStoreConfig( , LocalFSStore::Config(authority, params) , LocalStore::Config(scheme, authority, params) , LocalOverlayStoreConfigT{ - CONFIG_ROW(lowerStoreUri), - CONFIG_ROW(upperLayer), - CONFIG_ROW(checkMount), - CONFIG_ROW(remountHook), + CONFIG_ROW(lowerStoreUri, ""), + CONFIG_ROW(upperLayer, ""), + CONFIG_ROW(checkMount, true), + CONFIG_ROW(remountHook, ""), } { } + std::string LocalOverlayStoreConfig::doc() { return @@ -95,10 +89,12 @@ std::string LocalOverlayStoreConfig::doc() ; } + Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { return upperLayer + "/" + path.to_string(); } + LocalOverlayStore::LocalOverlayStore(const Config & config) : Store::Config{config} , LocalFSStore::Config{config} diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 4cf898b8fc7..3d16c0c4695 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -29,8 +29,6 @@ struct LocalOverlayStoreConfig : static const Descriptions descriptions; - static LocalOverlayStoreConfigT defaults; - LocalOverlayStoreConfig( std::string_view scheme, PathView path, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2a930051d4f..9100879c923 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -88,11 +88,6 @@ LocalStore::Config::Descriptions::Descriptions() const LocalStore::Config::Descriptions LocalStore::Config::descriptions{}; -decltype(LocalStore::Config::defaults) LocalStore::Config::defaults = { - .requireSigs = {settings.requireSigs}, - .readOnly = {false}, -}; - LocalStore::Config::LocalStoreConfig( std::string_view scheme, std::string_view authority, @@ -100,8 +95,8 @@ LocalStore::Config::LocalStoreConfig( : Store::Config(params) , LocalFSStore::Config(authority, params) , LocalStoreConfigT{ - CONFIG_ROW(requireSigs), - CONFIG_ROW(readOnly), + CONFIG_ROW(requireSigs, settings.requireSigs), + CONFIG_ROW(readOnly, false), } { } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ca483df1584..55a2e504ff9 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -56,8 +56,6 @@ struct LocalStoreConfig : static const Descriptions descriptions; - static LocalStoreConfigT defaults; - LocalStoreConfig(const StoreReference::Params & params) : LocalStoreConfig{"local", "", params} {} diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 4caabdf9f3c..b2bb6f7d6a6 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -42,17 +42,11 @@ RemoteStore::Config::Descriptions::Descriptions() const RemoteStore::Config::Descriptions RemoteStore::Config::descriptions{}; -decltype(RemoteStore::Config::defaults) RemoteStore::Config::defaults = { - .maxConnections = {1}, - .maxConnectionAge = {std::numeric_limits::max()}, -}; - - RemoteStore::Config::RemoteStoreConfig(const StoreReference::Params & params) : Store::Config(params) , RemoteStoreConfigT{ - CONFIG_ROW(maxConnections), - CONFIG_ROW(maxConnectionAge), + CONFIG_ROW(maxConnections, 1), + CONFIG_ROW(maxConnectionAge, std::numeric_limits::max()), } { } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 126f060a02b..d4a63de1f13 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -38,11 +38,6 @@ struct RemoteStoreConfig : static const Descriptions descriptions; - /** - * The other defaults depend on the choice of `storeDir` and `rootDir` - */ - static RemoteStoreConfigT defaults; - RemoteStoreConfig(const StoreReference::Params &); }; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 7f8ce3a7bdd..7f0fcc671d1 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -261,18 +261,6 @@ S3BinaryCacheStore::Config::Descriptions::Descriptions() const S3BinaryCacheStore::Config::Descriptions S3BinaryCacheStore::Config::descriptions{}; -decltype(S3BinaryCacheStore::Config::defaults) S3BinaryCacheStore::Config::defaults = { - .profile{""}, - .region{Aws::Region::US_EAST_1}, - .scheme{""}, - .endpoint{""}, - .narinfoCompression{""}, - .lsCompression{""}, - .logCompression{""}, - .multipartUpload{false}, - .bufferSize{5 * 1024 * 1024}, -}; - S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig( std::string_view scheme, std::string_view authority, @@ -280,15 +268,15 @@ S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig( : Store::Config(params) , BinaryCacheStore::Config(params) , S3BinaryCacheStoreConfigT{ - CONFIG_ROW(profile), - CONFIG_ROW(region), - CONFIG_ROW(scheme), - CONFIG_ROW(endpoint), - CONFIG_ROW(narinfoCompression), - CONFIG_ROW(lsCompression), - CONFIG_ROW(logCompression), - CONFIG_ROW(multipartUpload), - CONFIG_ROW(bufferSize), + CONFIG_ROW(profile, ""), + CONFIG_ROW(region, Aws::Region::US_EAST_1), + CONFIG_ROW(scheme, ""), + CONFIG_ROW(endpoint, ""), + CONFIG_ROW(narinfoCompression, ""), + CONFIG_ROW(lsCompression, ""), + CONFIG_ROW(logCompression, ""), + CONFIG_ROW(multipartUpload, false), + CONFIG_ROW(bufferSize, 5 * 1024 * 1024), } , bucketName{authority} { diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index df7197113ae..2097bb5b427 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -32,8 +32,6 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig, S3BinaryCacheS static const Descriptions descriptions; - static S3BinaryCacheStoreConfigT defaults; - S3BinaryCacheStoreConfig( std::string_view uriScheme, std::string_view bucketName, const StoreReference::Params & params); diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 0e1af43a131..3b04fec8227 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -27,11 +27,6 @@ SSHStoreConfig::Descriptions::Descriptions() const SSHStoreConfig::Descriptions SSHStoreConfig::descriptions{}; -decltype(SSHStoreConfig::defaults) SSHStoreConfig::defaults = { - .remoteProgram = {{"nix-daemon"}}, -}; - - SSHStoreConfig::SSHStoreConfig( std::string_view scheme, std::string_view authority, @@ -40,7 +35,7 @@ SSHStoreConfig::SSHStoreConfig( , RemoteStore::Config{params} , CommonSSHStoreConfig{scheme, authority, params} , SSHStoreConfigT{ - CONFIG_ROW(remoteProgram), + CONFIG_ROW(remoteProgram, Strings{"nix-daemon"}), } { } diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh index 6aa9c9cdf09..7f664fd2d6e 100644 --- a/src/libstore/ssh-store.hh +++ b/src/libstore/ssh-store.hh @@ -23,8 +23,6 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig, static const Descriptions descriptions; - static SSHStoreConfigT defaults; - SSHStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params); const std::string name() override diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index f51e7c27cf9..78931a5d097 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -238,23 +238,14 @@ Store::Config::Descriptions::Descriptions() const Store::Config::Descriptions Store::Config::descriptions{}; -decltype(Store::Config::defaults) Store::Config::defaults = { - .pathInfoCacheSize = {65536}, - .isTrusted = {false}, - .priority = {0}, - .wantMassQuery = {false}, - .systemFeatures = {Store::Config::getDefaultSystemFeatures()}, -}; - - Store::Config::StoreConfig(const StoreReference::Params & params) : StoreDirConfig{params} , StoreConfigT{ - CONFIG_ROW(pathInfoCacheSize), - CONFIG_ROW(isTrusted), - CONFIG_ROW(priority), - CONFIG_ROW(wantMassQuery), - CONFIG_ROW(systemFeatures), + CONFIG_ROW(pathInfoCacheSize, 65536), + CONFIG_ROW(isTrusted, false), + CONFIG_ROW(priority, 0), + CONFIG_ROW(wantMassQuery, false), + CONFIG_ROW(systemFeatures, getDefaultSystemFeatures()), } , defaultPriority{params.count(descriptions.priority.name) == 0} , defaultWantMassQuery{params.count(descriptions.wantMassQuery.name) == 0} diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e2151ef5b42..f744c441cdb 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -118,8 +118,6 @@ struct StoreConfig : Descriptions(); }; - static const StoreConfigT defaults; - static const Descriptions descriptions; StoreConfig(const StoreReference::Params &); diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc index 026a6cd9837..2f4873ec3c4 100644 --- a/src/libstore/store-dir-config.cc +++ b/src/libstore/store-dir-config.cc @@ -16,13 +16,9 @@ const StoreDirConfigT StoreDirConfig::descriptions = { }, }; -const StoreDirConfigT StoreDirConfig::defaults = { - ._storeDir = {settings.nixStore}, -}; - StoreDirConfig::StoreDirConfig(const StoreReference::Params & params) : StoreDirConfigT{ - CONFIG_ROW(_storeDir), + CONFIG_ROW(_storeDir, settings.nixStore), } { } diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index 19391322907..d4a4288f58e 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -27,8 +27,6 @@ struct StoreDirConfigT struct StoreDirConfig : StoreDirConfigT { - static const StoreDirConfigT defaults; - using Descriptions = StoreDirConfigT; static const Descriptions descriptions; From ae9377617c0b7fecd12926e9a44b590f4df0ed20 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 23:29:30 -0400 Subject: [PATCH 040/284] Avoid potentially problematic move --- src/libstore/local-fs-store.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index bedc0aca94e..47f24f27c1b 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -43,11 +43,10 @@ auto localFSStoreConfig( { const auto & descriptions = LocalFSStore::Config::descriptions; - auto rootDir = descriptions.rootDir.parseConfig(params) - .value_or(config::JustValue{.value = std::move(_rootDir)}); + auto rootDir = descriptions.rootDir.parseConfig(params).value_or(std::move(_rootDir)); return LocalFSStoreConfigT{ - .rootDir = std::move(rootDir), + .rootDir = {.value = rootDir}, CONFIG_ROW(stateDir, rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir), CONFIG_ROW(logDir, rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir), CONFIG_ROW(realStoreDir, rootDir ? *rootDir + "/nix/store" : storeConfig.storeDir), From 40955fa5bf37cf97d5a7b8c73b628cc0aa1edf42 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 23:32:00 -0400 Subject: [PATCH 041/284] WIP hack so it builds --- src/nix/main.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/main.cc b/src/nix/main.cc index 0d839425afb..524c4626350 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -223,12 +223,14 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs auto stores = nlohmann::json::object(); for (auto & implem : *Implementations::registered) { +#if 0 auto storeConfig = implem.getConfig(); auto storeName = storeConfig->name(); auto & j = stores[storeName]; j["doc"] = storeConfig->doc(); j["settings"] = storeConfig->toJSON(); j["experimentalFeature"] = storeConfig->experimentalFeature(); +#endif } res["stores"] = std::move(stores); res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo(); From 58a79b6943c7b102d876a2fc0af19ba45082f175 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Jul 2024 13:35:46 +0200 Subject: [PATCH 042/284] performOp(): Take a WorkerProto::BasicServerConnection --- src/libstore/daemon.cc | 255 ++++++++++++++++++++--------------------- 1 file changed, 125 insertions(+), 130 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index fc14eda3c96..6533b2f58c4 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -269,26 +269,21 @@ struct ClientSettings }; static void performOp(TunnelLogger * logger, ref store, - TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::Version clientVersion, - Source & from, BufferedSink & to, WorkerProto::Op op) + TrustedFlag trusted, RecursiveFlag recursive, + WorkerProto::BasicServerConnection & conn, + WorkerProto::Op op) { - WorkerProto::ReadConn rconn { - .from = from, - .version = clientVersion, - }; - WorkerProto::WriteConn wconn { - .to = to, - .version = clientVersion, - }; + WorkerProto::ReadConn rconn(conn); + WorkerProto::WriteConn wconn(conn); switch (op) { case WorkerProto::Op::IsValidPath: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); bool result = store->isValidPath(path); logger->stopWork(); - to << result; + conn.to << result; break; } @@ -296,8 +291,8 @@ static void performOp(TunnelLogger * logger, ref store, auto paths = WorkerProto::Serialise::read(*store, rconn); SubstituteFlag substitute = NoSubstitute; - if (GET_PROTOCOL_MINOR(clientVersion) >= 27) { - substitute = readInt(from) ? Substitute : NoSubstitute; + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 27) { + substitute = readInt(conn.from) ? Substitute : NoSubstitute; } logger->startWork(); @@ -311,13 +306,13 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::HasSubstitutes: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); StorePathSet paths; // FIXME paths.insert(path); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - to << (res.count(path) != 0); + conn.to << (res.count(path) != 0); break; } @@ -331,11 +326,11 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryPathHash: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(HashFormat::Base16, false); + conn.to << hash.to_string(HashFormat::Base16, false); break; } @@ -343,7 +338,7 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::QueryReferrers: case WorkerProto::Op::QueryValidDerivers: case WorkerProto::Op::QueryDerivationOutputs: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); StorePathSet paths; if (op == WorkerProto::Op::QueryReferences) @@ -360,16 +355,16 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryDerivationOutputNames: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto names = store->readDerivation(path).outputNames(); logger->stopWork(); - to << names; + conn.to << names; break; } case WorkerProto::Op::QueryDerivationOutputMap: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto outputs = store->queryPartialDerivationOutputMap(path); logger->stopWork(); @@ -378,37 +373,37 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryDeriver: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto info = store->queryPathInfo(path); logger->stopWork(); - to << (info->deriver ? store->printStorePath(*info->deriver) : ""); + conn.to << (info->deriver ? store->printStorePath(*info->deriver) : ""); break; } case WorkerProto::Op::QueryPathFromHashPart: { - auto hashPart = readString(from); + auto hashPart = readString(conn.from); logger->startWork(); auto path = store->queryPathFromHashPart(hashPart); logger->stopWork(); - to << (path ? store->printStorePath(*path) : ""); + conn.to << (path ? store->printStorePath(*path) : ""); break; } case WorkerProto::Op::AddToStore: { - if (GET_PROTOCOL_MINOR(clientVersion) >= 25) { - auto name = readString(from); - auto camStr = readString(from); + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 25) { + auto name = readString(conn.from); + auto camStr = readString(conn.from); auto refs = WorkerProto::Serialise::read(*store, rconn); bool repairBool; - from >> repairBool; + conn.from >> repairBool; auto repair = RepairFlag{repairBool}; logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); auto [contentAddressMethod, hashAlgo] = ContentAddressMethod::parseWithAlgo(camStr); - FramedSource source(from); + FramedSource source(conn.from); FileSerialisationMethod dumpMethod; switch (contentAddressMethod.getFileIngestionMethod()) { case FileIngestionMethod::Flat: @@ -439,7 +434,7 @@ static void performOp(TunnelLogger * logger, ref store, bool fixed; uint8_t recursive; std::string hashAlgoRaw; - from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; + conn.from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; if (recursive > true) throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); method = recursive @@ -459,11 +454,11 @@ static void performOp(TunnelLogger * logger, ref store, so why all this extra work? We still parse the NAR so that we aren't sending arbitrary data to `saved` unwittingly`, and we know when the NAR ends so we don't - consume the rest of `from` and can't parse another + consume the rest of `conn.from` and can't parse another command. (We don't trust `addToStoreFromDump` to not eagerly consume the entire stream it's given, past the length of the Nar. */ - TeeSource savedNARSource(from, saved); + TeeSource savedNARSource(conn.from, saved); NullFileSystemObjectSink sink; /* just parse the NAR */ parseDump(sink, savedNARSource); }); @@ -472,20 +467,20 @@ static void performOp(TunnelLogger * logger, ref store, *dumpSource, baseName, FileSerialisationMethod::NixArchive, method, hashAlgo); logger->stopWork(); - to << store->printStorePath(path); + conn.to << store->printStorePath(path); } break; } case WorkerProto::Op::AddMultipleToStore: { bool repair, dontCheckSigs; - from >> repair >> dontCheckSigs; + conn.from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; logger->startWork(); { - FramedSource source(from); + FramedSource source(conn.from); store->addMultipleToStore(source, RepairFlag{repair}, dontCheckSigs ? NoCheckSigs : CheckSigs); @@ -495,8 +490,8 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::AddTextToStore: { - std::string suffix = readString(from); - std::string s = readString(from); + std::string suffix = readString(conn.from); + std::string s = readString(conn.from); auto refs = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); auto path = ({ @@ -504,37 +499,37 @@ static void performOp(TunnelLogger * logger, ref store, store->addToStoreFromDump(source, suffix, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, NoRepair); }); logger->stopWork(); - to << store->printStorePath(path); + conn.to << store->printStorePath(path); break; } case WorkerProto::Op::ExportPath: { - auto path = store->parseStorePath(readString(from)); - readInt(from); // obsolete + auto path = store->parseStorePath(readString(conn.from)); + readInt(conn.from); // obsolete logger->startWork(); - TunnelSink sink(to); + TunnelSink sink(conn.to); store->exportPath(path, sink); logger->stopWork(); - to << 1; + conn.to << 1; break; } case WorkerProto::Op::ImportPaths: { logger->startWork(); - TunnelSource source(from, to); + TunnelSource source(conn.from, conn.to); auto paths = store->importPaths(source, trusted ? NoCheckSigs : CheckSigs); logger->stopWork(); Strings paths2; for (auto & i : paths) paths2.push_back(store->printStorePath(i)); - to << paths2; + conn.to << paths2; break; } case WorkerProto::Op::BuildPaths: { auto drvs = WorkerProto::Serialise::read(*store, rconn); BuildMode mode = bmNormal; - if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 15) { mode = WorkerProto::Serialise::read(*store, rconn); /* Repairing is not atomic, so disallowed for "untrusted" @@ -552,7 +547,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); store->buildPaths(drvs, mode); logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -578,7 +573,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::BuildDerivation: { - auto drvPath = store->parseStorePath(readString(from)); + auto drvPath = store->parseStorePath(readString(conn.from)); BasicDerivation drv; /* * Note: unlike wopEnsurePath, this operation reads a @@ -589,7 +584,7 @@ static void performOp(TunnelLogger * logger, ref store, * it cannot be trusted that its outPath was calculated * correctly. */ - readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath)); + readDerivation(conn.from, *store, drv, Derivation::nameFromPath(drvPath)); auto buildMode = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); @@ -655,20 +650,20 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::EnsurePath: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); store->ensurePath(path); logger->stopWork(); - to << 1; + conn.to << 1; break; } case WorkerProto::Op::AddTempRoot: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); store->addTempRoot(path); logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -678,24 +673,24 @@ static void performOp(TunnelLogger * logger, ref store, "you are not privileged to create perm roots\n\n" "hint: you can just do this client-side without special privileges, and probably want to do that instead."); auto storePath = WorkerProto::Serialise::read(*store, rconn); - Path gcRoot = absPath(readString(from)); + Path gcRoot = absPath(readString(conn.from)); logger->startWork(); auto & localFSStore = require(*store); localFSStore.addPermRoot(storePath, gcRoot); logger->stopWork(); - to << gcRoot; + conn.to << gcRoot; break; } case WorkerProto::Op::AddIndirectRoot: { - Path path = absPath(readString(from)); + Path path = absPath(readString(conn.from)); logger->startWork(); auto & indirectRootStore = require(*store); indirectRootStore.addIndirectRoot(path); logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -703,7 +698,7 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::SyncWithGC: { logger->startWork(); logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -717,24 +712,24 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : roots) size += i.second.size(); - to << size; + conn.to << size; for (auto & [target, links] : roots) for (auto & link : links) - to << link << store->printStorePath(target); + conn.to << link << store->printStorePath(target); break; } case WorkerProto::Op::CollectGarbage: { GCOptions options; - options.action = (GCOptions::GCAction) readInt(from); + options.action = (GCOptions::GCAction) readInt(conn.from); options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); - from >> options.ignoreLiveness >> options.maxFreed; + conn.from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields - readInt(from); - readInt(from); - readInt(from); + readInt(conn.from); + readInt(conn.from); + readInt(conn.from); GCResults results; @@ -745,7 +740,7 @@ static void performOp(TunnelLogger * logger, ref store, gcStore.collectGarbage(options, results); logger->stopWork(); - to << results.paths << results.bytesFreed << 0 /* obsolete */; + conn.to << results.paths << results.bytesFreed << 0 /* obsolete */; break; } @@ -754,24 +749,24 @@ static void performOp(TunnelLogger * logger, ref store, ClientSettings clientSettings; - clientSettings.keepFailed = readInt(from); - clientSettings.keepGoing = readInt(from); - clientSettings.tryFallback = readInt(from); - clientSettings.verbosity = (Verbosity) readInt(from); - clientSettings.maxBuildJobs = readInt(from); - clientSettings.maxSilentTime = readInt(from); - readInt(from); // obsolete useBuildHook - clientSettings.verboseBuild = lvlError == (Verbosity) readInt(from); - readInt(from); // obsolete logType - readInt(from); // obsolete printBuildTrace - clientSettings.buildCores = readInt(from); - clientSettings.useSubstitutes = readInt(from); - - if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { - unsigned int n = readInt(from); + clientSettings.keepFailed = readInt(conn.from); + clientSettings.keepGoing = readInt(conn.from); + clientSettings.tryFallback = readInt(conn.from); + clientSettings.verbosity = (Verbosity) readInt(conn.from); + clientSettings.maxBuildJobs = readInt(conn.from); + clientSettings.maxSilentTime = readInt(conn.from); + readInt(conn.from); // obsolete useBuildHook + clientSettings.verboseBuild = lvlError == (Verbosity) readInt(conn.from); + readInt(conn.from); // obsolete logType + readInt(conn.from); // obsolete printBuildTrace + clientSettings.buildCores = readInt(conn.from); + clientSettings.useSubstitutes = readInt(conn.from); + + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 12) { + unsigned int n = readInt(conn.from); for (unsigned int i = 0; i < n; i++) { - auto name = readString(from); - auto value = readString(from); + auto name = readString(conn.from); + auto value = readString(conn.from); clientSettings.overrides.emplace(name, value); } } @@ -788,20 +783,20 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QuerySubstitutablePathInfo: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); SubstitutablePathInfos infos; store->querySubstitutablePathInfos({{path, std::nullopt}}, infos); logger->stopWork(); auto i = infos.find(path); if (i == infos.end()) - to << 0; + conn.to << 0; else { - to << 1 + conn.to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); WorkerProto::write(*store, wconn, i->second.references); - to << i->second.downloadSize - << i->second.narSize; + conn.to << i->second.downloadSize + << i->second.narSize; } break; } @@ -809,7 +804,7 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::QuerySubstitutablePathInfos: { SubstitutablePathInfos infos; StorePathCAMap pathsMap = {}; - if (GET_PROTOCOL_MINOR(clientVersion) < 22) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 22) { auto paths = WorkerProto::Serialise::read(*store, rconn); for (auto & path : paths) pathsMap.emplace(path, std::nullopt); @@ -818,12 +813,12 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); - to << infos.size(); + conn.to << infos.size(); for (auto & i : infos) { - to << store->printStorePath(i.first) - << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); + conn.to << store->printStorePath(i.first) + << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); WorkerProto::write(*store, wconn, i.second.references); - to << i.second.downloadSize << i.second.narSize; + conn.to << i.second.downloadSize << i.second.narSize; } break; } @@ -837,22 +832,22 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryPathInfo: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); std::shared_ptr info; logger->startWork(); try { info = store->queryPathInfo(path); } catch (InvalidPath &) { - if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw; + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 17) throw; } logger->stopWork(); if (info) { - if (GET_PROTOCOL_MINOR(clientVersion) >= 17) - to << 1; + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 17) + conn.to << 1; WorkerProto::write(*store, wconn, static_cast(*info)); } else { - assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); - to << 0; + assert(GET_PROTOCOL_MINOR(conn.protoVersion) >= 17); + conn.to << 0; } break; } @@ -861,61 +856,61 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); store->optimiseStore(); logger->stopWork(); - to << 1; + conn.to << 1; break; case WorkerProto::Op::VerifyStore: { bool checkContents, repair; - from >> checkContents >> repair; + conn.from >> checkContents >> repair; logger->startWork(); if (repair && !trusted) throw Error("you are not privileged to repair paths"); bool errors = store->verifyStore(checkContents, (RepairFlag) repair); logger->stopWork(); - to << errors; + conn.to << errors; break; } case WorkerProto::Op::AddSignatures: { - auto path = store->parseStorePath(readString(from)); - StringSet sigs = readStrings(from); + auto path = store->parseStorePath(readString(conn.from)); + StringSet sigs = readStrings(conn.from); logger->startWork(); store->addSignatures(path, sigs); logger->stopWork(); - to << 1; + conn.to << 1; break; } case WorkerProto::Op::NarFromPath: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); logger->stopWork(); - dumpPath(store->toRealPath(path), to); + dumpPath(store->toRealPath(path), conn.to); break; } case WorkerProto::Op::AddToStoreNar: { bool repair, dontCheckSigs; - auto path = store->parseStorePath(readString(from)); - auto deriver = readString(from); - auto narHash = Hash::parseAny(readString(from), HashAlgorithm::SHA256); + auto path = store->parseStorePath(readString(conn.from)); + auto deriver = readString(conn.from); + auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256); ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); info.references = WorkerProto::Serialise::read(*store, rconn); - from >> info.registrationTime >> info.narSize >> info.ultimate; - info.sigs = readStrings(from); - info.ca = ContentAddress::parseOpt(readString(from)); - from >> repair >> dontCheckSigs; + conn.from >> info.registrationTime >> info.narSize >> info.ultimate; + info.sigs = readStrings(conn.from); + info.ca = ContentAddress::parseOpt(readString(conn.from)); + conn.from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; if (!trusted) info.ultimate = false; - if (GET_PROTOCOL_MINOR(clientVersion) >= 23) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 23) { logger->startWork(); { - FramedSource source(from); + FramedSource source(conn.from); store->addToStore(info, source, (RepairFlag) repair, dontCheckSigs ? NoCheckSigs : CheckSigs); } @@ -925,10 +920,10 @@ static void performOp(TunnelLogger * logger, ref store, else { std::unique_ptr source; StringSink saved; - if (GET_PROTOCOL_MINOR(clientVersion) >= 21) - source = std::make_unique(from, to); + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 21) + source = std::make_unique(conn.from, conn.to); else { - TeeSource tee { from, saved }; + TeeSource tee { conn.from, saved }; NullFileSystemObjectSink ether; parseDump(ether, tee); source = std::make_unique(saved.s); @@ -956,15 +951,15 @@ static void performOp(TunnelLogger * logger, ref store, WorkerProto::write(*store, wconn, willBuild); WorkerProto::write(*store, wconn, willSubstitute); WorkerProto::write(*store, wconn, unknown); - to << downloadSize << narSize; + conn.to << downloadSize << narSize; break; } case WorkerProto::Op::RegisterDrvOutput: { logger->startWork(); - if (GET_PROTOCOL_MINOR(clientVersion) < 31) { - auto outputId = DrvOutput::parse(readString(from)); - auto outputPath = StorePath(readString(from)); + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { + auto outputId = DrvOutput::parse(readString(conn.from)); + auto outputPath = StorePath(readString(conn.from)); store->registerDrvOutput(Realisation{ .id = outputId, .outPath = outputPath}); } else { @@ -977,10 +972,10 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::QueryRealisation: { logger->startWork(); - auto outputId = DrvOutput::parse(readString(from)); + auto outputId = DrvOutput::parse(readString(conn.from)); auto info = store->queryRealisation(outputId); logger->stopWork(); - if (GET_PROTOCOL_MINOR(clientVersion) < 31) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { std::set outPaths; if (info) outPaths.insert(info->outPath); WorkerProto::write(*store, wconn, outPaths); @@ -993,19 +988,19 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::AddBuildLog: { - StorePath path{readString(from)}; + StorePath path{readString(conn.from)}; logger->startWork(); if (!trusted) throw Error("you are not privileged to add logs"); auto & logStore = require(*store); { - FramedSource source(from); + FramedSource source(conn.from); StringSink sink; source.drainInto(sink); logStore.addBuildLog(path, sink.s); } logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -1090,7 +1085,7 @@ void processConnection( debug("performing daemon worker op: %d", op); try { - performOp(tunnelLogger, store, trusted, recursive, clientVersion, conn.from, conn.to, op); + performOp(tunnelLogger, store, trusted, recursive, conn, op); } catch (Error & e) { /* If we're not in a state where we can send replies, then something went wrong processing the input of the From 1c5fac5534cd560da65034ef43299d2a204d4847 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 19 Jul 2024 10:37:11 -0400 Subject: [PATCH 043/284] Adjust registration infra, fix Meson build --- src/libstore/dummy-store.cc | 2 +- src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-binary-cache-store.cc | 2 +- src/libstore/local-overlay-store.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/meson.build | 4 ++++ src/libstore/s3-binary-cache-store.cc | 2 +- src/libstore/ssh-store.cc | 4 ++-- src/libstore/store-registration.hh | 12 ++++++------ src/libstore/uds-remote-store.cc | 2 +- src/perl/lib/Nix/Store.xs | 2 +- tests/unit/libstore/meson.build | 1 + 13 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index e8934ec9a19..2f3269b6830 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -85,6 +85,6 @@ ref DummyStore::Config::openStore() const return make_ref(*this); } -static RegisterStoreImplementation regDummyStore; +static RegisterStoreImplementation regDummyStore; } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 8d9a9d96335..3cdd7e8874d 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -214,6 +214,6 @@ ref HttpBinaryCacheStore::Config::openStore() const return make_ref(*this); } -static RegisterStoreImplementation regHttpBinaryCacheStore; +static RegisterStoreImplementation regHttpBinaryCacheStore; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 0fc5181ff6d..ea49b6c8ac6 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -350,6 +350,6 @@ ref LegacySSHStore::Config::openStore() const { } -static RegisterStoreImplementation regLegacySSHStore; +static RegisterStoreImplementation regLegacySSHStore; } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index c00a34d2bce..0997cfd77c1 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -129,6 +129,6 @@ ref LocalBinaryCacheStoreConfig::openStore() const { return make_ref(*this); } -static RegisterStoreImplementation regLocalBinaryCacheStore; +static RegisterStoreImplementation regLocalBinaryCacheStore; } diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 7a483304841..a1bf594cdb4 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -364,6 +364,6 @@ void LocalOverlayStore::remountIfNecessary() } -static RegisterStoreImplementation regLocalOverlayStore; +static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9100879c923..66bdc51c756 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1834,6 +1834,6 @@ std::optional LocalStore::getVersion() return nixVersion; } -static RegisterStoreImplementation regLocalStore; +static RegisterStoreImplementation regLocalStore; } // namespace nix diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 29ee95b75bc..a9c92545a4d 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -210,6 +210,8 @@ sources = files( 'ssh-store.cc', 'ssh.cc', 'store-api.cc', + 'store-dir-config.cc', + 'store-registration.cc', 'store-reference.cc', 'uds-remote-store.cc', 'worker-protocol-connection.cc', @@ -282,6 +284,8 @@ headers = [config_h] + files( 'sqlite.hh', 'ssh.hh', 'store-api.hh', + 'store-open.hh', + 'store-registration.hh', 'store-cast.hh', 'store-dir-config.hh', 'store-reference.hh', diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 7f0fcc671d1..19f2a69c419 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -558,7 +558,7 @@ ref S3BinaryCacheStoreImpl::Config::openStore() const return make_ref(*this); } -static RegisterStoreImplementation regS3BinaryCacheStore; +static RegisterStoreImplementation regS3BinaryCacheStore; } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 3b04fec8227..a83a11b4fdd 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -247,7 +247,7 @@ ref SSHStore::openConnection() return conn; } -static RegisterStoreImplementation regSSHStore; -static RegisterStoreImplementation regMountedSSHStore; +static RegisterStoreImplementation regSSHStore; +static RegisterStoreImplementation regMountedSSHStore; } diff --git a/src/libstore/store-registration.hh b/src/libstore/store-registration.hh index dd61a45f1ae..20ca0c93e48 100644 --- a/src/libstore/store-registration.hh +++ b/src/libstore/store-registration.hh @@ -28,28 +28,28 @@ struct Implementations { static std::vector * registered; - template + template static void add() { if (!registered) registered = new std::vector(); StoreFactory factory{ - .uriSchemes = T::Config::uriSchemes(), + .uriSchemes = TConfig::uriSchemes(), .parseConfig = ([](auto scheme, auto uri, auto & params) -> ref { - return make_ref(scheme, uri, params); + return make_ref(scheme, uri, params); }), - .configDescriptions = T::Config::descriptions, + .configDescriptions = TConfig::descriptions, }; registered->push_back(factory); } }; -template +template struct RegisterStoreImplementation { RegisterStoreImplementation() { - Implementations::add(); + Implementations::add(); } }; diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 8604961fbe5..0cc9140edb7 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -113,6 +113,6 @@ ref UDSRemoteStore::Config::openStore() const { } -static RegisterStoreImplementation regUDSRemoteStore; +static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index f951437c899..d88e2648ebc 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -12,7 +12,7 @@ #include "derivations.hh" #include "realisation.hh" #include "globals.hh" -#include "store-api.hh" +#include "store-open.hh" #include "posix-source-accessor.hh" #include diff --git a/tests/unit/libstore/meson.build b/tests/unit/libstore/meson.build index 8534ba8c52d..5d5382a6da0 100644 --- a/tests/unit/libstore/meson.build +++ b/tests/unit/libstore/meson.build @@ -57,6 +57,7 @@ sources = files( 'derivation-advanced-attrs.cc', 'derivation.cc', 'derived-path.cc', + 'dummy-store.cc', 'downstream-placeholder.cc', 'http-binary-cache-store.cc', 'legacy-ssh-store.cc', From 74dccef004baed15c957715a8f3c93f10854d105 Mon Sep 17 00:00:00 2001 From: detroyejr Date: Thu, 18 Jul 2024 18:30:31 -0400 Subject: [PATCH 044/284] addFlag: use aliases --- src/libstore/globals.cc | 3 +++ src/libutil/config-impl.hh | 2 ++ src/libutil/config.cc | 2 ++ 3 files changed, 7 insertions(+) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index fa4c0ba7fad..60b3ee34e78 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -304,18 +304,21 @@ template<> void BaseSetting::convertToArg(Args & args, const std::s { args.addFlag({ .longName = name, + .aliases = aliases, .description = "Enable sandboxing.", .category = category, .handler = {[this]() { override(smEnabled); }} }); args.addFlag({ .longName = "no-" + name, + .aliases = aliases, .description = "Disable sandboxing.", .category = category, .handler = {[this]() { override(smDisabled); }} }); args.addFlag({ .longName = "relaxed-" + name, + .aliases = aliases, .description = "Enable sandboxing, but allow builds to disable it.", .category = category, .handler = {[this]() { override(smRelaxed); }} diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index 1d349fab5db..c3aa61ddb95 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -81,6 +81,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) { args.addFlag({ .longName = name, + .aliases = aliases, .description = fmt("Set the `%s` setting.", name), .category = category, .labels = {"value"}, @@ -91,6 +92,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) if (isAppendable()) args.addFlag({ .longName = "extra-" + name, + .aliases = aliases, .description = fmt("Append to the `%s` setting.", name), .category = category, .labels = {"value"}, diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 726e5091e54..b3994826129 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -292,6 +292,7 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & { args.addFlag({ .longName = name, + .aliases = aliases, .description = fmt("Enable the `%s` setting.", name), .category = category, .handler = {[this] { override(true); }}, @@ -299,6 +300,7 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & }); args.addFlag({ .longName = "no-" + name, + .aliases = aliases, .description = fmt("Disable the `%s` setting.", name), .category = category, .handler = {[this] { override(false); }}, From d54dfbf879d8c4e2f98aca54221e757fd1035b10 Mon Sep 17 00:00:00 2001 From: detroyejr Date: Fri, 19 Jul 2024 15:49:56 -0400 Subject: [PATCH 045/284] addFlag: test that alias flags are allowed --- tests/functional/build.sh | 12 ++++++++++++ tests/functional/eval.sh | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/tests/functional/build.sh b/tests/functional/build.sh index 9de953d8cbf..5396a465fe3 100755 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -140,6 +140,18 @@ nix build --impure -f multiple-outputs.nix --json e --no-link | jq --exit-status (.outputs | keys == ["a_a", "b"])) ' +# Make sure that the 3 types of aliases work +# BaseSettings, BaseSettings, and BaseSettings. +nix build --impure -f multiple-outputs.nix --json e --no-link \ + --build-max-jobs 3 \ + --gc-keep-outputs \ + --build-use-sandbox | \ + jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-e.drv")) and + (.outputs | keys == ["a_a", "b"])) +' + # Make sure that `--stdin` works and does not apply any defaults printf "" | nix build --no-link --stdin --json | jq --exit-status '. == []' printf "%s\n" "$drv^*" | nix build --no-link --stdin --json | jq --exit-status '.[0]|has("drvPath")' diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh index 27cdce478e3..22d2d02a28d 100755 --- a/tests/functional/eval.sh +++ b/tests/functional/eval.sh @@ -58,3 +58,7 @@ fi # Test that unknown settings are warned about out="$(expectStderr 0 nix eval --option foobar baz --expr '""' --raw)" [[ "$(echo "$out" | grep foobar | wc -l)" = 1 ]] + +# Test flag alias +out="$(nix eval --expr '{}' --build-cores 1)" +[[ "$(echo "$out" | wc -l)" = 1 ]] From a6c6852afc6ee5f1b929f5e1a7737edf5f51f546 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 20 Jul 2024 14:50:28 -0400 Subject: [PATCH 046/284] WIP Store doesn't inherit from config --- src/libstore/common-ssh-store-config.cc | 40 +++--- src/libstore/common-ssh-store-config.hh | 2 - src/libstore/config-parse-impl.hh | 43 +++++- src/libstore/config-parse.hh | 82 ++++++++++- src/libstore/legacy-ssh-store.hh | 2 + src/libstore/local-fs-store.cc | 129 +++++++++++------- src/libstore/local-fs-store.hh | 35 +++-- src/libstore/local-store.cc | 149 ++++++++++---------- src/libstore/local-store.hh | 24 ++-- src/libstore/meson.build | 1 + src/libstore/path.cc | 12 +- src/libstore/ssh-store.hh | 4 +- src/libstore/store-api.cc | 173 +++++++++++++++--------- src/libstore/store-api.hh | 70 ++++++---- src/libstore/store-dir-config.cc | 37 +++-- src/libstore/store-dir-config.hh | 49 +++++-- src/libstore/store-registration.hh | 4 +- src/libutil/config-abstract.hh | 8 ++ src/libutil/config.hh | 3 +- 19 files changed, 561 insertions(+), 306 deletions(-) diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index 113ba6b8b4e..ab328cdd3e2 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -7,29 +7,28 @@ namespace nix { CommonSSHStoreConfig::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , CommonSSHStoreConfigT{ - .sshKey{ - .name = "ssh-key", - .description = "Path to the SSH private key used to authenticate to the remote machine.", - }, - .sshPublicHostKey{ - .name = "base64-ssh-public-host-key", - .description = "The public host key of the remote machine.", - }, - .compress{ - .name = "compress", - .description = "Whether to enable SSH compression.", - }, - .remoteStore{ - .name = "remote-store", - .description = R"( + : CommonSSHStoreConfigT{ + .sshKey{ + .name = "ssh-key", + .description = "Path to the SSH private key used to authenticate to the remote machine.", + }, + .sshPublicHostKey{ + .name = "base64-ssh-public-host-key", + .description = "The public host key of the remote machine.", + }, + .compress{ + .name = "compress", + .description = "Whether to enable SSH compression.", + }, + .remoteStore{ + .name = "remote-store", + .description = R"( [Store URL](@docroot@/store/types/index.md#store-url-format) to be used on the remote machine. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). )", - }, - } + }, + } { } @@ -56,8 +55,7 @@ CommonSSHStoreConfig::CommonSSHStoreConfig( std::string_view scheme, std::string_view host, const StoreReference::Params & params) - : Store::Config(params) - , CommonSSHStoreConfigT{ + : CommonSSHStoreConfigT{ CONFIG_ROW(sshKey, ""), CONFIG_ROW(sshPublicHostKey, ""), CONFIG_ROW(compress, false), diff --git a/src/libstore/common-ssh-store-config.hh b/src/libstore/common-ssh-store-config.hh index e7bb16367ff..d9361e259a3 100644 --- a/src/libstore/common-ssh-store-config.hh +++ b/src/libstore/common-ssh-store-config.hh @@ -17,11 +17,9 @@ struct CommonSSHStoreConfigT }; struct CommonSSHStoreConfig : - virtual Store::Config, CommonSSHStoreConfigT { struct Descriptions : - virtual Store::Config::Descriptions, CommonSSHStoreConfigT { Descriptions(); diff --git a/src/libstore/config-parse-impl.hh b/src/libstore/config-parse-impl.hh index ed640019dbe..b73801793e8 100644 --- a/src/libstore/config-parse-impl.hh +++ b/src/libstore/config-parse-impl.hh @@ -9,16 +9,51 @@ namespace nix::config { template -std::optional SettingInfo::parseConfig(const nlohmann::json::object_t & map) const +OptValue SettingInfo::parseConfig(const nlohmann::json::object_t & map) const { - auto * p = get(map, name); - return p ? p->get() : (std::optional{}); + const nlohmann::json * p = get(map, name); + return {.optValue = p ? (std::optional{p->get()}) : std::nullopt}; +} + +template +std::pair SettingInfo::describe(const JustValue & def) const +{ + return { + name, + SettingDescription{ + .description = description, + .defaultValue = + documentDefault ? (std::optional{}) : (std::optional{nlohmann::json{def.value}}), + }, + }; } /** * Look up the setting's name in a map, falling back on the default if * it does not exist. */ -#define CONFIG_ROW(FIELD, DEFAULT) .FIELD = {.value = descriptions.FIELD.parseConfig(params).value_or(DEFAULT)} +#define CONFIG_ROW(FIELD) .FIELD = descriptions.FIELD.parseConfig(params) + +#define APPLY_ROW(FIELD) .FIELD = {.value = parsed.FIELD.optValue.value_or(std::move(defaults.FIELD))} + +#define DESC_ROW(FIELD) \ + { \ + descriptions.FIELD.describe(defaults.FIELD), \ + } + +#define MAKE_PARSE(CAPITAL, LOWER, FIELDS) \ + static CAPITAL##T LOWER##Parse(const StoreReference::Params & params) \ + { \ + constexpr auto & descriptions = LOWER##Descriptions; \ + return {FIELDS(CONFIG_ROW)}; \ + } + +#define MAKE_APPLY_PARSE(CAPITAL, LOWER, FIELDS) \ + static CAPITAL##T LOWER##ApplyParse(const StoreReference::Params & params) \ + { \ + auto defaults = LOWER##Defaults(); \ + auto parsed = LOWER##Parse(params); \ + return {FIELDS(APPLY_ROW)}; \ + } } diff --git a/src/libstore/config-parse.hh b/src/libstore/config-parse.hh index e016caf15d9..4b44e8197c8 100644 --- a/src/libstore/config-parse.hh +++ b/src/libstore/config-parse.hh @@ -4,18 +4,96 @@ #include #include "config-abstract.hh" -#include "util.hh" +#include "json-impls.hh" namespace nix::config { +struct SettingDescription; + +/** + * Typed version used as source of truth, and for operations like + * defaulting configurations. + */ template struct SettingInfo { + /** + * Name of the setting, used when parsing configuration maps + */ std::string name; + + /** + * Description of the setting. It is used just for documentation. + */ std::string description; + +#if 0 + /** + * Other names of the setting also used when parsing configuration + * maps. This is useful for back-compat, etc. + */ + std::set aliases; + + /** + * `ExperimentalFeature` that must be enabled if the setting is + * allowed to be used + */ + std::optional experimentalFeature; +#endif + + /** + * Whether to document the default value. (Some defaults are + * system-specific and should not be documented.) + */ bool documentDefault = true; - std::optional parseConfig(const nlohmann::json::object_t & map) const; + /** + * Describe the setting as a key-value pair (name -> other info). + * The default value will be rendered to JSON if it is to be + * documented. + */ + std::pair describe(const JustValue & def) const; + + OptValue parseConfig(const nlohmann::json::object_t & map) const; }; +/** + * Untyped version used for rendering docs. This is not the source of + * truth, it is generated from the typed one. + * + * @note No `name` field because this is intended to be used as the value type + * of a map + */ +struct SettingDescription +{ + /** + * @see SettingInfo::description + */ + std::string description; + +#if 0 + /** + * @see SettingInfo::aliases + */ + std::set aliases; + + /** + * @see SettingInfo::experimentalFeature + */ + std::optional experimentalFeature; +#endif + + /** + * Optional, for the `SettingInfo::documentDefault = false` case. + */ + std::optional defaultValue; +}; + +/** + * Map of setting names to descriptions of those settings. + */ +using SettingDescriptionMap = std::map; + } + +JSON_IMPL(config::SettingDescription) diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index 12ca9f12514..4da2d02dfa6 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -17,10 +17,12 @@ struct LegacySSHStoreConfigT }; struct LegacySSHStoreConfig : + virtual Store::Config, virtual CommonSSHStoreConfig, LegacySSHStoreConfigT { struct Descriptions : + virtual Store::Config::Descriptions, virtual CommonSSHStoreConfig::Descriptions, LegacySSHStoreConfigT { diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 47f24f27c1b..ce383e6cc6e 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -9,69 +9,104 @@ namespace nix { -LocalFSStore::Config::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , LocalFSStoreConfigT{ - .rootDir = { - .name = "root", - .description = "Directory prefixed to all other paths.", - }, - .stateDir = { - .name = "state", - .description = "Directory where Nix will store state.", - }, - .logDir = { - .name = "log", - .description = "directory where Nix will store log files.", - }, - .realStoreDir{ - .name = "real", - .description = "Physical path of the Nix store.", - }, - } -{} +static const LocalFSStoreConfigT localFSStoreConfigDescriptions = { + .rootDir = { + .name = "root", + .description = "Directory prefixed to all other paths.", + }, + .stateDir = { + .name = "state", + .description = "Directory where Nix will store state.", + }, + .logDir = { + .name = "log", + .description = "directory where Nix will store log files.", + }, + .realStoreDir{ + .name = "real", + .description = "Physical path of the Nix store.", + }, +}; + +#define LOCAL_FS_STORE_CONFIG_FIELDS(X) \ + X(rootDir), \ + X(stateDir), \ + X(logDir), \ + X(realStoreDir), -const LocalFSStore::Config::Descriptions LocalFSStore::Config::descriptions{}; +MAKE_PARSE(LocalFSStoreConfig, localFSStoreConfig, LOCAL_FS_STORE_CONFIG_FIELDS) /** * @param rootDir Fallback if not in `params` */ -auto localFSStoreConfig( +static LocalFSStoreConfigT localFSStoreConfigDefaults( + const Path & storeDir, + const std::optional & rootDir) +{ + return { + .rootDir = {std::nullopt}, + .stateDir = {rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir}, + .logDir = {rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir}, + .realStoreDir = {rootDir ? *rootDir + "/nix/store" : storeDir}, + }; +} + +static LocalFSStoreConfigT localFSStoreConfigApplyParse( + const Path & storeDir, + LocalFSStoreConfigT parsed) +{ + auto defaults = localFSStoreConfigDefaults( + storeDir, + parsed.rootDir.optValue.value_or(std::nullopt)); + return {LOCAL_FS_STORE_CONFIG_FIELDS(APPLY_ROW)}; +} + +LocalFSStore::Config::LocalFSStoreConfig( const Store::Config & storeConfig, - const std::optional _rootDir, const StoreReference::Params & params) + : LocalFSStoreConfigT{ + localFSStoreConfigApplyParse( + storeConfig.storeDir, + localFSStoreConfigParse(params))} + , storeConfig{storeConfig} { - const auto & descriptions = LocalFSStore::Config::descriptions; - - auto rootDir = descriptions.rootDir.parseConfig(params).value_or(std::move(_rootDir)); +} - return LocalFSStoreConfigT{ - .rootDir = {.value = rootDir}, - CONFIG_ROW(stateDir, rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir), - CONFIG_ROW(logDir, rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir), - CONFIG_ROW(realStoreDir, rootDir ? *rootDir + "/nix/store" : storeConfig.storeDir), - }; +static LocalFSStoreConfigT applyAuthority( + LocalFSStoreConfigT parsed, + PathView rootDir) +{ + if (!rootDir.empty()) + parsed.rootDir = {.optValue = {Path{rootDir}}}; + return parsed; } -LocalFSStore::Config::LocalFSStoreConfig(const StoreReference::Params & params) - : StoreConfig{params} - , LocalFSStoreConfigT{localFSStoreConfig(*this, std::nullopt, params)} +LocalFSStore::Config::LocalFSStoreConfig( + const Store::Config & storeConfig, + PathView rootDir, + const StoreReference::Params & params) + : LocalFSStoreConfigT{ + localFSStoreConfigApplyParse( + storeConfig.storeDir, + applyAuthority( + localFSStoreConfigParse(params), + rootDir))} + , storeConfig{storeConfig} { } -LocalFSStore::Config::LocalFSStoreConfig(PathView rootDir, const StoreReference::Params & params) - : StoreConfig(params) - , LocalFSStoreConfigT{localFSStoreConfig( - *this, - // Default `?root` from `rootDir` if non set - !rootDir.empty() ? (std::optional{rootDir}) : std::nullopt, - params)} +config::SettingDescriptionMap LocalFSStoreConfig::descriptions() { + constexpr auto & descriptions = localFSStoreConfigDescriptions; + auto defaults = localFSStoreConfigDefaults(settings.nixStore, std::nullopt); + return { + LOCAL_FS_STORE_CONFIG_FIELDS(DESC_ROW) + }; } LocalFSStore::LocalFSStore(const Config & config) - : LocalFSStore::Config{config} - , Store{static_cast(*this)} + : Store{static_cast(*this)} + , config{config} { } @@ -145,8 +180,8 @@ std::optional LocalFSStore::getBuildLogExact(const StorePath & path Path logPath = j == 0 - ? fmt("%s/%s/%s/%s", logDir.get(), drvsLogDir, baseName.substr(0, 2), baseName.substr(2)) - : fmt("%s/%s/%s", logDir.get(), drvsLogDir, baseName); + ? fmt("%s/%s/%s/%s", config.logDir.get(), drvsLogDir, baseName.substr(0, 2), baseName.substr(2)) + : fmt("%s/%s/%s", config.logDir.get(), drvsLogDir, baseName); Path logBz2Path = logPath + ".bz2"; if (pathExists(logPath)) diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 678e93ec0a4..27ef32ee94c 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -10,26 +10,21 @@ namespace nix { template class F> struct LocalFSStoreConfigT { - const F> rootDir; - const F stateDir; - const F logDir; - const F realStoreDir; + F> rootDir; + F stateDir; + F logDir; + F realStoreDir; }; -struct LocalFSStoreConfig : - virtual Store::Config, - LocalFSStoreConfigT +struct LocalFSStoreConfig : LocalFSStoreConfigT { - struct Descriptions : - virtual Store::Config::Descriptions, - LocalFSStoreConfigT - { - Descriptions(); - }; + const Store::Config & storeConfig; - static const Descriptions descriptions; + config::SettingDescriptionMap descriptions(); - LocalFSStoreConfig(const StoreReference::Params &); + LocalFSStoreConfig( + const Store::Config & storeConfig, + const StoreReference::Params &); /** * Used to override the `root` settings. Can't be done via modifying @@ -38,17 +33,21 @@ struct LocalFSStoreConfig : * * @todo Make this less error-prone with new store settings system. */ - LocalFSStoreConfig(PathView path, const StoreReference::Params & params); + LocalFSStoreConfig( + const Store::Config & storeConfig, + PathView path, + const StoreReference::Params & params); }; struct LocalFSStore : - virtual LocalFSStoreConfig, virtual Store, virtual GcStore, virtual LogStore { using Config = LocalFSStoreConfig; + const Config & config; + inline static std::string operationName = "Local Filesystem Store"; const static std::string drvsLogDir; @@ -74,7 +73,7 @@ struct LocalFSStore : */ virtual Path addPermRoot(const StorePath & storePath, const Path & gcRoot) = 0; - virtual Path getRealStoreDir() { return realStoreDir; } + virtual Path getRealStoreDir() { return config.realStoreDir; } Path toRealPath(const Path & storePath) override { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 66bdc51c756..b16d56ebb3d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -59,45 +59,52 @@ namespace nix { -LocalStore::Config::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , LocalFSStore::Config::Descriptions{LocalFSStore::Config::descriptions} - , LocalStoreConfigT{ - .requireSigs = { - .name = "require-sigs", - .description = "Whether store paths copied into this store should have a trusted signature.", - }, - .readOnly = { - .name = "read-only", - .description = R"( - Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. - - Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. - - Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. - - > **Warning** - > Do not use this unless the filesystem is read-only. - > - > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. - > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. - )", - }, - } -{} +static const LocalStoreConfigT localStoreConfigDescriptions = { + .requireSigs = { + .name = "require-sigs", + .description = "Whether store paths copied into this store should have a trusted signature.", + }, + .readOnly = { + .name = "read-only", + .description = R"( + Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. + + Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. + + Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. + + > **Warning** + > Do not use this unless the filesystem is read-only. + > + > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. + > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. + )", + }, +}; + +#define LOCAL_STORE_CONFIG_FIELDS(X) \ + X(requireSigs), \ + X(readOnly), + +MAKE_PARSE(LocalStoreConfig, localStoreConfig, LOCAL_STORE_CONFIG_FIELDS) + +static LocalStoreConfigT localStoreConfigDefaults() +{ + return { + .requireSigs = {settings.requireSigs}, + .readOnly = {false}, + }; +} -const LocalStore::Config::Descriptions LocalStore::Config::descriptions{}; +MAKE_APPLY_PARSE(LocalStoreConfig, localStoreConfig, LOCAL_STORE_CONFIG_FIELDS) LocalStore::Config::LocalStoreConfig( std::string_view scheme, std::string_view authority, const StoreReference::Params & params) : Store::Config(params) - , LocalFSStore::Config(authority, params) - , LocalStoreConfigT{ - CONFIG_ROW(requireSigs, settings.requireSigs), - CONFIG_ROW(readOnly, false), - } + , LocalFSStore::Config(*this, authority, params) + , LocalStoreConfigT{localStoreConfigApplyParse(params)} { } @@ -110,7 +117,7 @@ std::string LocalStoreConfig::doc() ref LocalStore::Config::openStore() const { - return make_ref(*this); + return make_ref(ref{shared_from_this()}); } struct LocalStore::State::Stmts { @@ -233,17 +240,15 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) } } -LocalStore::LocalStore(const Config & config) - : Store::Config{config} - , LocalFSStore::Config{config} - , LocalStore::Config{config} - , Store{static_cast(*this)} - , LocalFSStore{static_cast(*this)} - , dbDir(stateDir + "/db") - , linksDir(realStoreDir + "/.links") +LocalStore::LocalStore(ref config) + : Store{*config} + , LocalFSStore{*config} + , config{config} + , dbDir(config->stateDir + "/db") + , linksDir(config->realStoreDir + "/.links") , reservedPath(dbDir + "/reserved") , schemaPath(dbDir + "/schema") - , tempRootsDir(stateDir + "/temproots") + , tempRootsDir(config->stateDir + "/temproots") , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) , locksHeld(tokenizeString(getEnv("NIX_HELD_LOCKS").value_or(""))) { @@ -251,18 +256,18 @@ LocalStore::LocalStore(const Config & config) state->stmts = std::make_unique(); /* Create missing state directories if they don't already exist. */ - createDirs(realStoreDir); - if (readOnly) { + createDirs(config->realStoreDir); + if (config->readOnly) { experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore); } else { makeStoreWritable(); } createDirs(linksDir); - Path profilesDir = stateDir + "/profiles"; + Path profilesDir = config->stateDir + "/profiles"; createDirs(profilesDir); createDirs(tempRootsDir); createDirs(dbDir); - Path gcRootsDir = stateDir + "/gcroots"; + Path gcRootsDir = config->stateDir + "/gcroots"; if (!pathExists(gcRootsDir)) { createDirs(gcRootsDir); createSymlink(profilesDir, gcRootsDir + "/profiles"); @@ -270,7 +275,7 @@ LocalStore::LocalStore(const Config & config) for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { createDirs(perUserDir); - if (!readOnly) { + if (!config->readOnly) { if (chmod(perUserDir.c_str(), 0755) == -1) throw SysError("could not set permissions on '%s' to 755", perUserDir); } @@ -285,16 +290,16 @@ LocalStore::LocalStore(const Config & config) struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup); - else if (!readOnly) { + else if (!config->readOnly) { struct stat st; - if (stat(realStoreDir.get().c_str(), &st)) - throw SysError("getting attributes of path '%1%'", realStoreDir); + if (stat(config->realStoreDir.get().c_str(), &st)) + throw SysError("getting attributes of path '%1%'", config->realStoreDir); if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { - if (chown(realStoreDir.get().c_str(), 0, gr->gr_gid) == -1) - throw SysError("changing ownership of path '%1%'", realStoreDir); - if (chmod(realStoreDir.get().c_str(), perm) == -1) - throw SysError("changing permissions on path '%1%'", realStoreDir); + if (chown(config->realStoreDir.get().c_str(), 0, gr->gr_gid) == -1) + throw SysError("changing ownership of path '%1%'", config->realStoreDir); + if (chmod(config->realStoreDir.get().c_str(), perm) == -1) + throw SysError("changing permissions on path '%1%'", config->realStoreDir); } } } @@ -302,7 +307,7 @@ LocalStore::LocalStore(const Config & config) /* Ensure that the store and its parents are not symlinks. */ if (!settings.allowSymlinkedStore) { - Path path = realStoreDir; + Path path = config->realStoreDir; struct stat st; while (path != "/") { st = lstat(path); @@ -350,12 +355,12 @@ LocalStore::LocalStore(const Config & config) /* Acquire the big fat lock in shared mode to make sure that no schema upgrade is in progress. */ - if (!readOnly) { + if (!config->readOnly) { Path globalLockPath = dbDir + "/big-lock"; globalLock = openLockFile(globalLockPath.c_str(), true); } - if (!readOnly && !lockFile(globalLock.get(), ltRead, false)) { + if (!config->readOnly && !lockFile(globalLock.get(), ltRead, false)) { printInfo("waiting for the big Nix store lock..."); lockFile(globalLock.get(), ltRead, true); } @@ -363,7 +368,7 @@ LocalStore::LocalStore(const Config & config) /* Check the current database schema and if necessary do an upgrade. */ int curSchema = getSchema(); - if (readOnly && curSchema < nixSchemaVersion) { + if (config->readOnly && curSchema < nixSchemaVersion) { debug("current schema version: %d", curSchema); debug("supported schema version: %d", nixSchemaVersion); throw Error(curSchema == 0 ? @@ -435,7 +440,7 @@ LocalStore::LocalStore(const Config & config) else openDB(*state, false); if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - if (!readOnly) { + if (!config->readOnly) { migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); } else { throw Error("need to migrate to content-addressed schema, but this cannot be done in read-only mode"); @@ -517,7 +522,7 @@ LocalStore::LocalStore(const Config & config) AutoCloseFD LocalStore::openGCLock() { - Path fnGCLock = stateDir + "/gc.lock"; + Path fnGCLock = config->stateDir + "/gc.lock"; auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT #ifndef _WIN32 | O_CLOEXEC @@ -573,17 +578,17 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { - if (create && readOnly) { + if (create && config->readOnly) { throw Error("cannot create database while in read-only mode"); } - if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK))) + if (access(dbDir.c_str(), R_OK | (config->readOnly ? 0 : W_OK))) throw SysError("Nix database directory '%1%' is not writable", dbDir); /* Open the Nix database. */ std::string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - auto openMode = readOnly ? SQLiteOpenMode::Immutable + auto openMode = config->readOnly ? SQLiteOpenMode::Immutable : create ? SQLiteOpenMode::Normal : SQLiteOpenMode::NoCreate; state.db = SQLite(dbPath, openMode); @@ -660,12 +665,12 @@ void LocalStore::makeStoreWritable() if (!isRootUser()) return; /* Check if /nix/store is on a read-only mount. */ struct statvfs stat; - if (statvfs(realStoreDir.get().c_str(), &stat) != 0) + if (statvfs(config->realStoreDir.get().c_str(), &stat) != 0) throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { - if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) - throw SysError("remounting %1% writable", realStoreDir); + if (mount(0, config->realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) + throw SysError("remounting %1% writable", config->realStoreDir); } #endif } @@ -1005,7 +1010,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) for (auto & sub : getDefaultSubstituters()) { if (remaining.empty()) break; if (sub->storeDir != storeDir) continue; - if (!sub->wantMassQuery) continue; + if (!sub->resolvedSubstConfig.wantMassQuery) continue; auto valid = sub->queryValidPaths(remaining); @@ -1117,12 +1122,12 @@ const PublicKeys & LocalStore::getPublicKeys() bool LocalStore::pathInfoIsUntrusted(const ValidPathInfo & info) { - return requireSigs && !info.checkSignatures(*this, getPublicKeys()); + return config->requireSigs && !info.checkSignatures(*this, getPublicKeys()); } bool LocalStore::realisationIsUntrusted(const Realisation & realisation) { - return requireSigs && !realisation.checkSignatures(getPublicKeys()); + return config->requireSigs && !realisation.checkSignatures(getPublicKeys()); } void LocalStore::addToStore(const ValidPathInfo & info, Source & source, @@ -1403,7 +1408,7 @@ std::pair LocalStore::createTempDirInStore() /* There is a slight possibility that `tmpDir' gets deleted by the GC between createTempDir() and when we acquire a lock on it. We'll repeat until 'tmpDir' exists and we've locked it. */ - tmpDirFn = createTempDir(realStoreDir, "tmp"); + tmpDirFn = createTempDir(config->realStoreDir, "tmp"); tmpDirFd = openDirectory(tmpDirFn); if (!tmpDirFd) { continue; @@ -1544,7 +1549,7 @@ LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair database and the filesystem) in the loop below, in order to catch invalid states. */ - for (auto & i : std::filesystem::directory_iterator{realStoreDir.get()}) { + for (auto & i : std::filesystem::directory_iterator{config->realStoreDir.get()}) { checkInterrupt(); try { storePathsInStoreDir.insert({i.path().filename().string()}); @@ -1816,7 +1821,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) auto baseName = drvPath.to_string(); - auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)); + auto logPath = fmt("%s/%s/%s/%s.bz2", config->logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)); if (pathExists(logPath)) return; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 55a2e504ff9..e05d3cc3b85 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -37,24 +37,17 @@ struct OptimiseStats template class F> struct LocalStoreConfigT { - const F requireSigs; - const F readOnly; + F requireSigs; + F readOnly; }; struct LocalStoreConfig : - virtual Store::Config, - virtual LocalFSStore::Config, + std::enable_shared_from_this, + Store::Config, + LocalFSStore::Config, LocalStoreConfigT { - struct Descriptions : - virtual Store::Config::Descriptions, - virtual LocalFSStore::Config::Descriptions, - LocalStoreConfigT - { - Descriptions(); - }; - - static const Descriptions descriptions; + static config::SettingDescriptionMap descriptions(); LocalStoreConfig(const StoreReference::Params & params) : LocalStoreConfig{"local", "", params} @@ -76,7 +69,6 @@ struct LocalStoreConfig : }; class LocalStore : - public virtual LocalStoreConfig, public virtual IndirectRootStore, public virtual GcStore { @@ -84,6 +76,8 @@ public: using Config = LocalStoreConfig; + ref config; + private: /** @@ -151,7 +145,7 @@ public: * Initialise the local store, upgrading the schema if * necessary. */ - LocalStore(const Config & params); + LocalStore(ref params); ~LocalStore(); diff --git a/src/libstore/meson.build b/src/libstore/meson.build index a9c92545a4d..ef01704de9a 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -163,6 +163,7 @@ sources = files( 'builtins/unpack-channel.cc', 'common-protocol.cc', 'common-ssh-store-config.cc', + 'config-parse.cc', 'content-address.cc', 'daemon.cc', 'derivations.cc', diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 3e9d054778c..b0ec8db52c0 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -75,7 +75,7 @@ StorePath StorePath::random(std::string_view name) return StorePath(Hash::random(HashAlgorithm::SHA1), name); } -StorePath StoreDirConfig::parseStorePath(std::string_view path) const +StorePath MixStoreDirMethods::parseStorePath(std::string_view path) const { // On Windows, `/nix/store` is not a canonical path. More broadly it // is unclear whether this function should be using the native @@ -94,7 +94,7 @@ StorePath StoreDirConfig::parseStorePath(std::string_view path) const return StorePath(baseNameOf(p)); } -std::optional StoreDirConfig::maybeParseStorePath(std::string_view path) const +std::optional MixStoreDirMethods::maybeParseStorePath(std::string_view path) const { try { return parseStorePath(path); @@ -103,24 +103,24 @@ std::optional StoreDirConfig::maybeParseStorePath(std::string_view pa } } -bool StoreDirConfig::isStorePath(std::string_view path) const +bool MixStoreDirMethods::isStorePath(std::string_view path) const { return (bool) maybeParseStorePath(path); } -StorePathSet StoreDirConfig::parseStorePathSet(const PathSet & paths) const +StorePathSet MixStoreDirMethods::parseStorePathSet(const PathSet & paths) const { StorePathSet res; for (auto & i : paths) res.insert(parseStorePath(i)); return res; } -std::string StoreDirConfig::printStorePath(const StorePath & path) const +std::string MixStoreDirMethods::printStorePath(const StorePath & path) const { return (storeDir + "/").append(path.to_string()); } -PathSet StoreDirConfig::printStorePathSet(const StorePathSet & paths) const +PathSet MixStoreDirMethods::printStorePathSet(const StorePathSet & paths) const { PathSet res; for (auto & i : paths) res.insert(printStorePath(i)); diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh index 7f664fd2d6e..e934876c393 100644 --- a/src/libstore/ssh-store.hh +++ b/src/libstore/ssh-store.hh @@ -16,7 +16,9 @@ struct SSHStoreConfigT struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig, SSHStoreConfigT { - struct Descriptions : virtual CommonSSHStoreConfig::Descriptions, SSHStoreConfigT + struct Descriptions : virtual RemoteStoreConfig::Descriptions, + virtual CommonSSHStoreConfig::Descriptions, + SSHStoreConfigT { Descriptions(); }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 78931a5d097..675f09a1231 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -30,13 +30,13 @@ using json = nlohmann::json; namespace nix { -bool StoreDirConfig::isInStore(PathView path) const +bool MixStoreDirMethods::isInStore(PathView path) const { return isInDir(path, storeDir); } -std::pair StoreDirConfig::toStorePath(PathView path) const +std::pair MixStoreDirMethods::toStorePath(PathView path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); @@ -78,7 +78,7 @@ to match. */ -StorePath StoreDirConfig::makeStorePath(std::string_view type, +StorePath MixStoreDirMethods::makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ @@ -89,14 +89,14 @@ StorePath StoreDirConfig::makeStorePath(std::string_view type, } -StorePath StoreDirConfig::makeStorePath(std::string_view type, +StorePath MixStoreDirMethods::makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const { return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name); } -StorePath StoreDirConfig::makeOutputPath(std::string_view id, +StorePath MixStoreDirMethods::makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const { return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id)); @@ -107,7 +107,7 @@ StorePath StoreDirConfig::makeOutputPath(std::string_view id, hacky, but we can't put them in, say, (per the grammar above) since that would be ambiguous. */ static std::string makeType( - const StoreDirConfig & store, + const MixStoreDirMethods & store, std::string && type, const StoreReferences & references) { @@ -120,7 +120,7 @@ static std::string makeType( } -StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const +StorePath MixStoreDirMethods::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { if (info.method == FileIngestionMethod::Git && info.hash.algo != HashAlgorithm::SHA1) throw Error("Git file ingestion must use SHA-1 hash"); @@ -142,7 +142,7 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed } -StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const +StorePath MixStoreDirMethods::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const { // New template return std::visit(overloaded { @@ -163,7 +163,7 @@ StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const } -std::pair StoreDirConfig::computeStorePath( +std::pair MixStoreDirMethods::computeStorePath( std::string_view name, const SourcePath & path, ContentAddressMethod method, @@ -189,67 +189,111 @@ std::pair StoreDirConfig::computeStorePath( } -Store::Config::Descriptions::Descriptions() - : StoreDirConfig::Descriptions{StoreDirConfig::descriptions} - , StoreConfigT{ - .pathInfoCacheSize = { - .name = "path-info-cache-size", - .description = "Size of the in-memory store path metadata cache.", - }, - .isTrusted = { - .name = "trusted", - .description = R"( - Whether paths from this store can be used as substitutes - even if they are not signed by a key listed in the - [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) - setting. - )", - }, - .priority = { - .name = "priority", - .description = R"( - Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). - A lower value means a higher priority. - )", - }, - .wantMassQuery = { - .name = "want-mass-query", - .description = R"( - Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). - )", - }, - - .systemFeatures = { - .name = "system-features", - .description = R"( - Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. +static const StoreConfigT storeConfigDescriptions = { + .pathInfoCacheSize{ + .name = "path-info-cache-size", + .description = "Size of the in-memory store path metadata cache.", + }, + .isTrusted{ + .name = "trusted", + .description = R"( + Whether paths from this store can be used as substitutes + even if they are not signed by a key listed in the + [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) + setting. + )", + }, + .systemFeatures{ + .name = "system-features", + .description = R"( + Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. + + Example: `"kvm"` + )", + // The default value is CPU- and OS-specific, and thus + // unsuitable to be rendered in the documentation. + .documentDefault = false, + }, +}; + +static const SubstituterConfigT substituterConfigDescriptions = { + .priority{ + .name = "priority", + .description = R"( + Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + A lower value means a higher priority. + )", + }, + .wantMassQuery{ + .name = "want-mass-query", + .description = R"( + Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + )", + }, +}; + + +#define STORE_CONFIG_FIELDS(X) \ + X(pathInfoCacheSize), \ + X(isTrusted), \ + X(systemFeatures), + +#define SUBSTITUTER_CONFIG_FIELDS(X) \ + X(priority), \ + X(wantMassQuery), + + +MAKE_PARSE(StoreConfig, storeConfig, STORE_CONFIG_FIELDS) +MAKE_PARSE(SubstituterConfig, substituterConfig, SUBSTITUTER_CONFIG_FIELDS) + + +static StoreConfigT storeConfigDefaults() +{ + return { + .pathInfoCacheSize = {65536}, + .isTrusted = {false}, + .systemFeatures = {StoreConfig::getDefaultSystemFeatures()}, + }; +}; - Example: `"kvm"` - )", - // The default value is CPU- and OS-specific, and thus - // unsuitable to be rendered in the documentation. - .documentDefault = false, - }, - } +static SubstituterConfigT substituterConfigDefaults() { -} + return { + .priority = {0}, + .wantMassQuery = {false}, + }; +}; -const Store::Config::Descriptions Store::Config::descriptions{}; +MAKE_APPLY_PARSE(StoreConfig, storeConfig, STORE_CONFIG_FIELDS) Store::Config::StoreConfig(const StoreReference::Params & params) : StoreDirConfig{params} - , StoreConfigT{ - CONFIG_ROW(pathInfoCacheSize, 65536), - CONFIG_ROW(isTrusted, false), - CONFIG_ROW(priority, 0), - CONFIG_ROW(wantMassQuery, false), - CONFIG_ROW(systemFeatures, getDefaultSystemFeatures()), - } - , defaultPriority{params.count(descriptions.priority.name) == 0} - , defaultWantMassQuery{params.count(descriptions.wantMassQuery.name) == 0} + , StoreConfigT{storeConfigApplyParse(params)} + , SubstituterConfigT{substituterConfigParse(params)} +{ +} + + +config::SettingDescriptionMap StoreConfig::descriptions() { + auto ret = StoreDirConfig::descriptions(); + { + constexpr auto & descriptions = storeConfigDescriptions; + auto defaults = storeConfigDefaults(); + ret.merge(config::SettingDescriptionMap { + STORE_CONFIG_FIELDS(DESC_ROW) + }); + } + { + constexpr auto & descriptions = substituterConfigDescriptions; + auto defaults = substituterConfigDefaults(); + ret.merge(config::SettingDescriptionMap { + SUBSTITUTER_CONFIG_FIELDS(DESC_ROW) + }); + }; + return ret; } @@ -498,8 +542,9 @@ StringSet Store::Config::getDefaultSystemFeatures() } Store::Store(const Store::Config & config) - : Store::Config(config) - , state({(size_t) pathInfoCacheSize}) + : MixStoreDirMethods{config} + , config{config} + , state({(size_t) config.pathInfoCacheSize}) { assertLibStoreInitialized(); } @@ -1262,7 +1307,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre } -std::string StoreDirConfig::showPaths(const StorePathSet & paths) +std::string MixStoreDirMethods::showPaths(const StorePathSet & paths) { std::string s; for (auto & i : paths) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f744c441cdb..206a35373db 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -99,26 +99,30 @@ typedef std::map> StorePathCAMap; template class F> struct StoreConfigT { - const F pathInfoCacheSize; - const F isTrusted; + F pathInfoCacheSize; + F isTrusted; + F systemFeatures; +}; + +template class F> +struct SubstituterConfigT +{ F priority; F wantMassQuery; - const F systemFeatures; }; -StoreConfigT parseStoreConfig(const StoreReference::Params &); +/** + * @note `config::OptValue` rather than `config::JustValue` is applied to + * `SubstitutorConfigT` because these are overrides. Caches themselves (not our + * config) can update default settings, but aren't allowed to update settings + * specified by the client (i.e. us). + */ struct StoreConfig : StoreDirConfig, - StoreConfigT + StoreConfigT, + SubstituterConfigT { - struct Descriptions : - StoreDirConfig::Descriptions, - StoreConfigT - { - Descriptions(); - }; - - static const Descriptions descriptions; + static config::SettingDescriptionMap descriptions(); StoreConfig(const StoreReference::Params &); @@ -153,27 +157,39 @@ struct StoreConfig : * type. */ virtual ref openStore() const = 0; +}; -protected: +/** + * A Store (client) + * + * This is an interface type allowing for create and read operations on + * a collection of store objects, and also building new store objects + * from `Derivation`s. See the manual for further details. + * + * "client" used is because this is just one view/actor onto an + * underlying resource, which could be an external process (daemon + * server), file system state, etc. + */ +class Store : public std::enable_shared_from_this, public MixStoreDirMethods +{ +public: + + using Config = StoreConfig; + + const Config & config; /** - * So caches themselves (not our config) can update default - * settings, but aren't allowed to update settings specied by the - * client (i.e. us). + * @note Avoid churn, since we used to inherit from `Config`. */ - bool defaultPriority; + operator const Config &() const { return config; } /** - * @see defaultPriority + * Resolved substituter configuration. This is intentionally mutable + * as store clients may do IO to ask the underlying store for their + * default setting values if the client config did not statically + * override them. */ - bool defaultWantMassQuery; -}; - -class Store : public std::enable_shared_from_this, public virtual StoreConfig -{ -public: - - using Config = StoreConfig; + SubstituterConfigT resolvedSubstConfig; protected: diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc index 2f4873ec3c4..ae22ba159f1 100644 --- a/src/libstore/store-dir-config.cc +++ b/src/libstore/store-dir-config.cc @@ -1,26 +1,45 @@ #include "store-dir-config.hh" #include "config-parse-impl.hh" #include "util.hh" +#include "globals.hh" namespace nix { -const StoreDirConfigT StoreDirConfig::descriptions = { - ._storeDir = - { - .name = "store", - .description = R"( +static const StoreDirConfigT storeDirConfigDescriptions = { + ._storeDir{ + .name = "store", + .description = R"( Logical location of the Nix store, usually `/nix/store`. Note that you can only copy store paths between stores if they have the same `store` setting. )", - }, + }, }; +#define STORE_DIR_CONFIG_FIELDS(X) X(_storeDir), + +MAKE_PARSE(StoreDirConfig, storeDirConfig, STORE_DIR_CONFIG_FIELDS) + +static StoreDirConfigT storeDirConfigDefaults() +{ + return { + ._storeDir = {settings.nixStore}, + }; +} + +MAKE_APPLY_PARSE(StoreDirConfig, storeDirConfig, STORE_DIR_CONFIG_FIELDS) + StoreDirConfig::StoreDirConfig(const StoreReference::Params & params) - : StoreDirConfigT{ - CONFIG_ROW(_storeDir, settings.nixStore), - } + : StoreDirConfigT{storeDirConfigApplyParse(params)} + , MixStoreDirMethods{_storeDir.value} +{ +} + +config::SettingDescriptionMap StoreDirConfig::descriptions() { + constexpr auto & descriptions = storeDirConfigDescriptions; + auto defaults = storeDirConfigDefaults(); + return {STORE_DIR_CONFIG_FIELDS(DESC_ROW)}; } } diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index d4a4288f58e..f247f917a50 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -5,7 +5,6 @@ #include "content-address.hh" #include "store-reference.hh" #include "config-parse.hh" -#include "globals.hh" #include #include @@ -19,25 +18,32 @@ struct SourcePath; MakeError(BadStorePath, Error); MakeError(BadStorePathName, BadStorePath); +/** + * Underlying store directory configuration type. + * + * Don't worry to much about the `F` parameter, it just some abstract + * nonsense for the "higher-kinded data" pattern. It is used in each + * settings record in order to ensure don't forgot to parse or document + * settings field. + */ template class F> struct StoreDirConfigT { - const F _storeDir; + F _storeDir; }; -struct StoreDirConfig : StoreDirConfigT +/** + * @todo This should just be part of `StoreDirConfig`. However, it would + * be a huge amount of churn if `Store` didn't have these methods + * anymore, forcing a bunch of code to go from `store.method(...)` to + * `store.config.method(...)`. + * + * So we instead pull out the methods into their own mix-in, so can put + * them directly on the Store too. + */ +struct MixStoreDirMethods { - using Descriptions = StoreDirConfigT; - - static const Descriptions descriptions; - - StoreDirConfig(const StoreReference::Params & params); - - virtual ~StoreDirConfig() = default; - - const Path & storeDir = _storeDir.value; - - // pure methods + const Path & storeDir; StorePath parseStorePath(std::string_view path) const; @@ -106,4 +112,19 @@ struct StoreDirConfig : StoreDirConfigT PathFilter & filter = defaultPathFilter) const; }; +/** + * Store directory configuration type. + * + * Combines the underlying `*T` type (with plain values for the fields) + * and the methods. + */ +struct StoreDirConfig : StoreDirConfigT, MixStoreDirMethods +{ + static config::SettingDescriptionMap descriptions(); + + StoreDirConfig(const StoreReference::Params & params); + + virtual ~StoreDirConfig() = default; +}; + } diff --git a/src/libstore/store-registration.hh b/src/libstore/store-registration.hh index 20ca0c93e48..289bdf341ff 100644 --- a/src/libstore/store-registration.hh +++ b/src/libstore/store-registration.hh @@ -21,7 +21,7 @@ struct StoreFactory std::function( std::string_view scheme, std::string_view authorityPath, const StoreReference::Params & params)> parseConfig; - const Store::Config::Descriptions & configDescriptions; + config::SettingDescriptionMap configDescriptions; }; struct Implementations @@ -38,7 +38,7 @@ struct Implementations .parseConfig = ([](auto scheme, auto uri, auto & params) -> ref { return make_ref(scheme, uri, params); }), - .configDescriptions = TConfig::descriptions, + .configDescriptions = TConfig::descriptions(), }; registered->push_back(factory); } diff --git a/src/libutil/config-abstract.hh b/src/libutil/config-abstract.hh index fcde0696b98..2c34ff50bdd 100644 --- a/src/libutil/config-abstract.hh +++ b/src/libutil/config-abstract.hh @@ -1,6 +1,8 @@ #pragma once ///@type +#include + namespace nix::config { template @@ -40,4 +42,10 @@ auto && operator<<(auto && str, const JustValue & opt) return str << opt.get(); } +template +struct OptValue +{ + std::optional optValue; +}; + } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 1952ba1b8d7..595538e9574 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -178,13 +178,12 @@ public: const std::string name; const std::string description; const std::set aliases; + std::optional experimentalFeature; int created = 123; bool overridden = false; - std::optional experimentalFeature; - protected: AbstractSetting( From 48aa3bb98e9dd513df8ee3e1e4e2dbd5ebb18ac8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 20 Jul 2024 15:51:21 -0400 Subject: [PATCH 047/284] WIP --- src/libstore/local-fs-store.hh | 2 +- src/libstore/local-overlay-store.cc | 172 ++++++++++++++-------------- src/libstore/local-overlay-store.hh | 28 ++--- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 4 +- src/libstore/remote-store.hh | 20 +--- src/libstore/store-api.cc | 2 +- src/libstore/store-api.hh | 7 +- src/libstore/store-dir-config.hh | 2 +- src/libstore/store-registration.cc | 58 +++++++++- src/libstore/uds-remote-store.hh | 22 ++-- 11 files changed, 181 insertions(+), 138 deletions(-) diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 27ef32ee94c..8fbd98db99d 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -20,7 +20,7 @@ struct LocalFSStoreConfig : LocalFSStoreConfigT { const Store::Config & storeConfig; - config::SettingDescriptionMap descriptions(); + static config::SettingDescriptionMap descriptions(); LocalFSStoreConfig( const Store::Config & storeConfig, diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index a1bf594cdb4..5f3d9281814 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -5,84 +5,88 @@ #include "realisation.hh" #include "processes.hh" #include "url.hh" -#include "store-open.hh" +#include "store-api.hh" #include "store-registration.hh" #include "config-parse-impl.hh" namespace nix { -LocalOverlayStore::Config::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , LocalFSStore::Config::Descriptions{LocalFSStore::Config::descriptions} - , LocalStore::Config::Descriptions{LocalStore::Config::descriptions} - , LocalOverlayStoreConfigT{ - .lowerStoreUri{ - .name = "lower-store", - .description = R"( - [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) - for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). - - Must be a store with a store dir on the file system. - Must be used as OverlayFS lower layer for this store's store dir. - )", - }, - .upperLayer{ - .name = "upper-layer", - .description = R"( - Directory containing the OverlayFS upper layer for this store's store dir. - )", - }, - .checkMount{ - .name = "check-mount", - .description = R"( - Check that the overlay filesystem is correctly mounted. - - Nix does not manage the overlayfs mount point itself, but the correct - functioning of the overlay store does depend on this mount point being set up - correctly. Rather than just assume this is the case, check that the lowerdir - and upperdir options are what we expect them to be. This check is on by - default, but can be disabled if needed. - )", - }, - .remountHook{ - .name = "remount-hook", - .description = R"( - Script or other executable to run when overlay filesystem needs remounting. - - This is occasionally necessary when deleting a store path that exists in both upper and lower layers. - In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly - is the only way to perform the deletion without creating a "whiteout". - However this causes the OverlayFS kernel data structures to get out-of-sync, - and can lead to 'stale file handle' errors; remounting solves the problem. - - The store directory is passed as an argument to the invoked executable. - )", - }, - } -{} - - -const LocalOverlayStore::Config::Descriptions LocalOverlayStore::Config::descriptions{}; +static LocalOverlayStoreConfigT localOverlayStoreConfigDescriptions = { + .lowerStoreConfig{ + .name = "lower-store", + .description = R"( + [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). + + Must be a store with a store dir on the file system. + Must be used as OverlayFS lower layer for this store's store dir. + )", + }, + .upperLayer{ + .name = "upper-layer", + .description = R"( + Directory containing the OverlayFS upper layer for this store's store dir. + )", + }, + .checkMount{ + .name = "check-mount", + .description = R"( + Check that the overlay filesystem is correctly mounted. + + Nix does not manage the overlayfs mount point itself, but the correct + functioning of the overlay store does depend on this mount point being set up + correctly. Rather than just assume this is the case, check that the lowerdir + and upperdir options are what we expect them to be. This check is on by + default, but can be disabled if needed. + )", + }, + .remountHook{ + .name = "remount-hook", + .description = R"( + Script or other executable to run when overlay filesystem needs remounting. + + This is occasionally necessary when deleting a store path that exists in both upper and lower layers. + In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly + is the only way to perform the deletion without creating a "whiteout". + However this causes the OverlayFS kernel data structures to get out-of-sync, + and can lead to 'stale file handle' errors; remounting solves the problem. + + The store directory is passed as an argument to the invoked executable. + )", + }, +}; + +#define LOCAL_OVERLAY_STORE_CONFIG_FIELDS(X) \ + X(lowerStoreConfig), \ + X(upperLayer), \ + X(checkMount), \ + X(remountHook), + +MAKE_PARSE(LocalOverlayStoreConfig, localOverlayStoreConfig, LOCAL_OVERLAY_STORE_CONFIG_FIELDS) + +static LocalOverlayStoreConfigT localOverlayStoreConfigDefaults() +{ + return { + .lowerStoreConfig = {make_ref(StoreReference::Params{})}, + .upperLayer = {""}, + .checkMount = {true}, + .remountHook = {""}, + }; +} +MAKE_APPLY_PARSE(LocalOverlayStoreConfig, localOverlayStoreConfig, LOCAL_OVERLAY_STORE_CONFIG_FIELDS) LocalOverlayStore::Config::LocalOverlayStoreConfig( std::string_view scheme, std::string_view authority, const StoreReference::Params & params) - : Store::Config(params) - , LocalFSStore::Config(authority, params) - , LocalStore::Config(scheme, authority, params) - , LocalOverlayStoreConfigT{ - CONFIG_ROW(lowerStoreUri, ""), - CONFIG_ROW(upperLayer, ""), - CONFIG_ROW(checkMount, true), - CONFIG_ROW(remountHook, ""), - } + : LocalStore::Config(scheme, authority, params) + , LocalOverlayStoreConfigT{localOverlayStoreConfigApplyParse(params)} { } -std::string LocalOverlayStoreConfig::doc() +std::string LocalOverlayStoreConfig::doc() const { return #include "local-overlay-store.md" @@ -90,26 +94,24 @@ std::string LocalOverlayStoreConfig::doc() } -Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { +Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) const +{ return upperLayer + "/" + path.to_string(); } -LocalOverlayStore::LocalOverlayStore(const Config & config) - : Store::Config{config} - , LocalFSStore::Config{config} - , LocalStore::Config{config} - , LocalOverlayStore::Config{config} - , Store{static_cast(*this)} - , LocalFSStore{static_cast(*this)} - , LocalStore{static_cast(*this)} - , lowerStore(nix::openStore(lowerStoreUri.get()).dynamic_pointer_cast()) +LocalOverlayStore::LocalOverlayStore(ref config) + : Store{*config} + , LocalFSStore{*config} + , LocalStore{static_cast>(config)} + , config{config} + , lowerStore(config->lowerStoreConfig.value->openStore().dynamic_pointer_cast()) { - if (checkMount.get()) { + if (config->checkMount.get()) { std::smatch match; std::string mountInfo; auto mounts = readFile("/proc/self/mounts"); - auto regex = std::regex(R"((^|\n)overlay )" + realStoreDir.get() + R"( .*(\n|$))"); + auto regex = std::regex(R"((^|\n)overlay )" + config->realStoreDir.get() + R"( .*(\n|$))"); // Mount points can be stacked, so there might be multiple matching entries. // Loop until the last match, which will be the current state of the mount point. @@ -122,13 +124,13 @@ LocalOverlayStore::LocalOverlayStore(const Config & config) return std::regex_search(mountInfo, std::regex("\\b" + option + "=" + value + "( |,)")); }; - auto expectedLowerDir = lowerStore->realStoreDir.get(); - if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", upperLayer)) { + auto expectedLowerDir = lowerStore->config.realStoreDir.get(); + if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", config->upperLayer)) { debug("expected lowerdir: %s", expectedLowerDir); - debug("expected upperdir: %s", upperLayer); + debug("expected upperdir: %s", config->upperLayer); debug("actual mount: %s", mountInfo); throw Error("overlay filesystem '%s' mounted incorrectly", - realStoreDir.get()); + config->realStoreDir.get()); } } } @@ -278,14 +280,14 @@ void LocalOverlayStore::collectGarbage(const GCOptions & options, GCResults & re void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed) { - auto mergedDir = realStoreDir.get() + "/"; + auto mergedDir = config->realStoreDir.get() + "/"; if (path.substr(0, mergedDir.length()) != mergedDir) { warn("local-overlay: unexpected gc path '%s' ", path); return; } StorePath storePath = {path.substr(mergedDir.length())}; - auto upperPath = toUpperPath(storePath); + auto upperPath = config->toUpperPath(storePath); if (pathExists(upperPath)) { debug("upper exists: %s", path); @@ -334,7 +336,7 @@ LocalStore::VerificationResult LocalOverlayStore::verifyAllValidPaths(RepairFlag StorePathSet done; auto existsInStoreDir = [&](const StorePath & storePath) { - return pathExists(realStoreDir.get() + "/" + storePath.to_string()); + return pathExists(config->realStoreDir.get() + "/" + storePath.to_string()); }; bool errors = false; @@ -354,10 +356,10 @@ void LocalOverlayStore::remountIfNecessary() { if (!_remountRequired) return; - if (remountHook.get().empty()) { - warn("'%s' needs remounting, set remount-hook to do this automatically", realStoreDir.get()); + if (config->remountHook.get().empty()) { + warn("'%s' needs remounting, set remount-hook to do this automatically", config->realStoreDir.get()); } else { - runProgram(remountHook, false, {realStoreDir}); + runProgram(config->remountHook, false, {config->realStoreDir}); } _remountRequired = false; diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 3d16c0c4695..112abd6be6b 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -6,7 +6,7 @@ namespace nix { template class F> struct LocalOverlayStoreConfigT { - const F lowerStoreUri; + const F> lowerStoreConfig; const F upperLayer; const F checkMount; const F remountHook; @@ -16,25 +16,17 @@ struct LocalOverlayStoreConfigT * Configuration for `LocalOverlayStore`. */ struct LocalOverlayStoreConfig : - virtual LocalStoreConfig, + LocalStoreConfig, LocalOverlayStoreConfigT { - struct Descriptions : - virtual Store::Config::Descriptions, - virtual LocalStore::Config::Descriptions, - LocalOverlayStoreConfigT - { - Descriptions(); - }; - - static const Descriptions descriptions; + static config::SettingDescriptionMap descriptions(); LocalOverlayStoreConfig( std::string_view scheme, PathView path, const StoreReference::Params & params); - const std::string name() override { return "Experimental Local Overlay Store"; } + const std::string name() const override { return "Experimental Local Overlay Store"; } std::optional experimentalFeature() const override { @@ -46,7 +38,7 @@ struct LocalOverlayStoreConfig : return { "local-overlay" }; } - std::string doc() override; + std::string doc() const override; protected: /** @@ -57,7 +49,9 @@ protected: * at that file path. It might be stored in the lower layer instead, * or it might not be part of this store at all. */ - Path toUpperPath(const StorePath & path); + Path toUpperPath(const StorePath & path) const; + + friend struct LocalOverlayStore; }; /** @@ -66,11 +60,13 @@ protected: * Documentation on overridden methods states how they differ from their * `LocalStore` counterparts. */ -struct LocalOverlayStore : virtual LocalOverlayStoreConfig, virtual LocalStore +struct LocalOverlayStore : virtual LocalStore { using Config = LocalOverlayStoreConfig; - LocalOverlayStore(const Config &); + ref config; + + LocalOverlayStore(ref); std::string getUri() override { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b16d56ebb3d..2572ed9e75a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -108,7 +108,7 @@ LocalStore::Config::LocalStoreConfig( { } -std::string LocalStoreConfig::doc() +std::string LocalStoreConfig::doc() const { return #include "local-store.md" diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index e05d3cc3b85..fd72249abe0 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -58,12 +58,12 @@ struct LocalStoreConfig : std::string_view authority, const StoreReference::Params & params); - const std::string name() override { return "Local Store"; } + const std::string name() const override { return "Local Store"; } static std::set uriSchemes() { return {"local"}; } - std::string doc() override; + std::string doc() const override; ref openStore() const override; }; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index d4a63de1f13..b79f68b3c53 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -21,22 +21,13 @@ template class Pool; template class F> struct RemoteStoreConfigT { - const F maxConnections; - const F maxConnectionAge; + F maxConnections; + F maxConnectionAge; }; -struct RemoteStoreConfig : - virtual Store::Config, - RemoteStoreConfigT +struct RemoteStoreConfig : RemoteStoreConfigT { - struct Descriptions : - virtual Store::Config::Descriptions, - RemoteStoreConfigT - { - Descriptions(); - }; - - static const Descriptions descriptions; + static config::SettingDescriptionMap descriptions(); RemoteStoreConfig(const StoreReference::Params &); }; @@ -46,13 +37,14 @@ struct RemoteStoreConfig : * DaemonStore. */ struct RemoteStore : - public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore, public virtual LogStore { using Config = RemoteStoreConfig; + const Config & config; + RemoteStore(const Config & config); /* Implementations of abstract store API methods. */ diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 675f09a1231..1629135b751 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1307,7 +1307,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre } -std::string MixStoreDirMethods::showPaths(const StorePathSet & paths) +std::string MixStoreDirMethods::showPaths(const StorePathSet & paths) const { std::string s; for (auto & i : paths) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 206a35373db..87b8335430a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -133,12 +133,12 @@ struct StoreConfig : /** * The name of this type of store. */ - virtual const std::string name() = 0; + virtual const std::string name() const = 0; /** * Documentation for this type of store. */ - virtual std::string doc() + virtual std::string doc() const { return ""; } @@ -897,3 +897,6 @@ std::map drvOutputReferences( Store * evalStore = nullptr); } + +// Parses a Store URL, uses global state not pure so think about this +JSON_IMPL(ref) diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index f247f917a50..dae372147c9 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -64,7 +64,7 @@ struct MixStoreDirMethods * Display a set of paths in human-readable form (i.e., between quotes * and separated by commas). */ - std::string showPaths(const StorePathSet & paths); + std::string showPaths(const StorePathSet & paths) const; /** * @return true if ‘path’ is in the Nix store (but not the Nix diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc index 1f3edfc71c1..306416b2681 100644 --- a/src/libstore/store-registration.cc +++ b/src/libstore/store-registration.cc @@ -15,7 +15,6 @@ ref openStore(StoreReference && storeURI) { auto store = resolveStoreConfig(std::move(storeURI))->openStore(); - experimentalFeatureSettings.require(store->experimentalFeature()); #if 0 // FIXME store->warnUnknownSettings(); store->init(); @@ -69,6 +68,8 @@ ref resolveStoreConfig(StoreReference && storeURI) }, storeURI.variant); + experimentalFeatureSettings.require(storeConfig->experimentalFeature()); + return storeConfig; } @@ -94,7 +95,9 @@ std::list> getDefaultSubstituters() for (auto uri : settings.substituters.get()) addStore(uri); - stores.sort([](ref & a, ref & b) { return a->priority < b->priority; }); + stores.sort([](ref & a, ref & b) { + return a->resolvedSubstConfig.priority < b->resolvedSubstConfig.priority; + }); return stores; }()); @@ -103,3 +106,54 @@ std::list> getDefaultSubstituters() } } + +namespace nlohmann { + +using namespace nix::config; + +ref adl_serializer>::from_json(const json & json) +{ + StoreReference ref; + switch (json.type()) { + + case json::value_t::string: { + ref = StoreReference::parse(json.get_ref()); + break; + } + + case json::value_t::object: { + auto & obj = json.get_ref(); + ref = StoreReference { + .variant = StoreReference::Specified{ + .scheme = getString(valueAt(obj, "scheme")), + .authority = getString(valueAt(obj, "authority")), + }, + .params = obj, + }; + break; + } + + case json::value_t::null: + case json::value_t::number_unsigned: + case json::value_t::number_integer: + case json::value_t::number_float: + case json::value_t::boolean: + case json::value_t::array: + case json::value_t::binary: + case json::value_t::discarded: + default: + throw UsageError( + "Invalid JSON for Store configuration: is type '%s' but must be string or object", + json.type_name()); + }; + + return resolveStoreConfig(std::move(ref)); +} + +void adl_serializer>::to_json(json & obj, ref s) +{ + // TODO, for tests maybe + assert(false); +} + +} diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 4af4eac624e..a584bb90385 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -8,17 +8,12 @@ namespace nix { struct UDSRemoteStoreConfig : - virtual LocalFSStore::Config, - virtual RemoteStore::Config + std::enable_shared_from_this, + Store::Config, + LocalFSStore::Config, + RemoteStore::Config { - struct Descriptions : - virtual LocalFSStore::Config::Descriptions, - virtual RemoteStore::Config::Descriptions - { - Descriptions(); - }; - - static const Descriptions descriptions; + static config::SettingDescriptionMap descriptions(); UDSRemoteStoreConfig(const StoreReference::Params & params) : UDSRemoteStoreConfig{"unix", "", params} @@ -32,9 +27,9 @@ struct UDSRemoteStoreConfig : std::string_view authority, const StoreReference::Params & params); - const std::string name() override { return "Local Daemon Store"; } + const std::string name() const override { return "Local Daemon Store"; } - std::string doc() override; + std::string doc() const override; /** * The path to the unix domain socket. @@ -51,12 +46,13 @@ struct UDSRemoteStoreConfig : }; struct UDSRemoteStore : - virtual UDSRemoteStoreConfig, virtual IndirectRootStore, virtual RemoteStore { using Config = UDSRemoteStoreConfig; + ref config; + UDSRemoteStore(const Config &); std::string getUri() override; From 1344bcd0b2b8364447779ffb256ec1724fc47cd6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 20 Jul 2024 22:07:48 -0400 Subject: [PATCH 048/284] WIP --- src/libstore/dummy-store.cc | 20 +++++++++----------- src/libstore/store-registration.cc | 14 +++++++------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 2f3269b6830..503ec346d13 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -12,7 +12,7 @@ DummyStoreConfig::DummyStoreConfig( throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); } -std::string DummyStoreConfig::doc() +std::string DummyStoreConfig::doc() const { return #include "dummy-store.md" @@ -20,22 +20,20 @@ std::string DummyStoreConfig::doc() } -const DummyStoreConfig::Descriptions DummyStoreConfig::descriptions{}; - - -struct DummyStore : public virtual DummyStoreConfig, public virtual Store +struct DummyStore : virtual Store { using Config = DummyStoreConfig; - DummyStore(const Config & config) - : StoreConfig(config) - , DummyStoreConfig(config) - , Store{static_cast(*this)} + ref config; + + DummyStore(ref config) + : Store{*config} + , config(config) { } std::string getUri() override { - return *uriSchemes().begin(); + return *Config::uriSchemes().begin() + "://"; } void queryPathInfoUncached(const StorePath & path, @@ -82,7 +80,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store ref DummyStore::Config::openStore() const { - return make_ref(*this); + return make_ref(ref{shared_from_this()}); } static RegisterStoreImplementation regDummyStore; diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc index 306416b2681..00bd522d600 100644 --- a/src/libstore/store-registration.cc +++ b/src/libstore/store-registration.cc @@ -123,11 +123,12 @@ ref adl_serializer>::from_json(const j case json::value_t::object: { auto & obj = json.get_ref(); - ref = StoreReference { - .variant = StoreReference::Specified{ - .scheme = getString(valueAt(obj, "scheme")), - .authority = getString(valueAt(obj, "authority")), - }, + ref = StoreReference{ + .variant = + StoreReference::Specified{ + .scheme = getString(valueAt(obj, "scheme")), + .authority = getString(valueAt(obj, "authority")), + }, .params = obj, }; break; @@ -143,8 +144,7 @@ ref adl_serializer>::from_json(const j case json::value_t::discarded: default: throw UsageError( - "Invalid JSON for Store configuration: is type '%s' but must be string or object", - json.type_name()); + "Invalid JSON for Store configuration: is type '%s' but must be string or object", json.type_name()); }; return resolveStoreConfig(std::move(ref)); From 5a1b10969dfbde502c6f6f906e9515a09446052e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 20 Jul 2024 22:14:05 -0400 Subject: [PATCH 049/284] WIP --- src/libstore/uds-remote-store.cc | 35 ++++++++++---------------------- src/libstore/uds-remote-store.hh | 2 +- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 0cc9140edb7..5b3518c0d8f 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -18,22 +18,12 @@ namespace nix { -UDSRemoteStoreConfig::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , LocalFSStore::Config::Descriptions{LocalFSStore::Config::descriptions} - , RemoteStore::Config::Descriptions{RemoteStore::Config::descriptions} -{} - - -const UDSRemoteStoreConfig::Descriptions UDSRemoteStoreConfig::descriptions{}; - - UDSRemoteStoreConfig::UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, const StoreReference::Params & params) : StoreConfig(params) - , LocalFSStoreConfig(params) + , LocalFSStoreConfig(*this, params) , RemoteStoreConfig(params) , path{authority.empty() ? settings.nixDaemonSocketFile : authority} { @@ -43,7 +33,7 @@ UDSRemoteStoreConfig::UDSRemoteStoreConfig( } -std::string UDSRemoteStoreConfig::doc() +std::string UDSRemoteStoreConfig::doc() const { return #include "uds-remote-store.md" @@ -51,27 +41,24 @@ std::string UDSRemoteStoreConfig::doc() } -UDSRemoteStore::UDSRemoteStore(const Config & config) - : Store::Config{config} - , LocalFSStore::Config{config} - , RemoteStore::Config{config} - , UDSRemoteStore::Config{config} - , Store(static_cast(*this)) - , LocalFSStore(static_cast(*this)) - , RemoteStore(static_cast(*this)) +UDSRemoteStore::UDSRemoteStore(ref config) + : Store{*config} + , LocalFSStore{*config} + , RemoteStore{*config} + , config{config} { } std::string UDSRemoteStore::getUri() { - return path == settings.nixDaemonSocketFile + return config->path == settings.nixDaemonSocketFile ? // FIXME: Not clear why we return daemon here and not default // to settings.nixDaemonSocketFile // // unix:// with no path also works. Change what we return? "daemon" - : std::string(*uriSchemes().begin()) + "://" + path; + : std::string(*Config::uriSchemes().begin()) + "://" + config->path; } @@ -88,7 +75,7 @@ ref UDSRemoteStore::openConnection() /* Connect to a daemon that does the privileged work for us. */ conn->fd = createUnixDomainSocket(); - nix::connect(toSocket(conn->fd.get()), path); + nix::connect(toSocket(conn->fd.get()), config->path); conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); @@ -109,7 +96,7 @@ void UDSRemoteStore::addIndirectRoot(const Path & path) ref UDSRemoteStore::Config::openStore() const { - return make_ref(*this); + return make_ref(ref{shared_from_this()}); } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index a584bb90385..75203cbd20c 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -53,7 +53,7 @@ struct UDSRemoteStore : ref config; - UDSRemoteStore(const Config &); + UDSRemoteStore(ref); std::string getUri() override; From 0c8330cae3cefa688abae2a8e65c11314dc075c4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 20 Jul 2024 22:17:13 -0400 Subject: [PATCH 050/284] WIP --- src/libstore/uds-remote-store.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 5b3518c0d8f..e8ec4e8c31d 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -18,6 +18,16 @@ namespace nix { +config::SettingDescriptionMap UDSRemoteStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(LocalFSStoreConfig::descriptions()); + ret.merge(RemoteStoreConfig::descriptions()); + return ret; +} + + UDSRemoteStoreConfig::UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, From f5df5e7c46f3c320017317c1bf5cac097c85253b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 20 Jul 2024 22:26:40 -0400 Subject: [PATCH 051/284] WIP --- src/libstore/common-ssh-store-config.cc | 79 ++++++++++++++----------- src/libstore/common-ssh-store-config.hh | 19 ++---- 2 files changed, 51 insertions(+), 47 deletions(-) diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index ab328cdd3e2..72b44cc41ff 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -6,33 +6,51 @@ namespace nix { -CommonSSHStoreConfig::Descriptions::Descriptions() - : CommonSSHStoreConfigT{ - .sshKey{ - .name = "ssh-key", - .description = "Path to the SSH private key used to authenticate to the remote machine.", - }, - .sshPublicHostKey{ - .name = "base64-ssh-public-host-key", - .description = "The public host key of the remote machine.", - }, - .compress{ - .name = "compress", - .description = "Whether to enable SSH compression.", - }, - .remoteStore{ - .name = "remote-store", - .description = R"( - [Store URL](@docroot@/store/types/index.md#store-url-format) - to be used on the remote machine. The default is `auto` - (i.e. use the Nix daemon or `/nix/store` directly). - )", - }, - } +static const CommonSSHStoreConfigT commonSSHStoreConfigDescriptions = { + .sshKey{ + .name = "ssh-key", + .description = "Path to the SSH private key used to authenticate to the remote machine.", + }, + .sshPublicHostKey{ + .name = "base64-ssh-public-host-key", + .description = "The public host key of the remote machine.", + }, + .compress{ + .name = "compress", + .description = "Whether to enable SSH compression.", + }, + .remoteStore{ + .name = "remote-store", + .description = R"( + [Store URL](@docroot@/store/types/index.md#store-url-format) + to be used on the remote machine. The default is `auto` + (i.e. use the Nix daemon or `/nix/store` directly). + )", + }, +}; + +#define COMMON_SSH_STORE_CONFIG_FIELDS(X) X(sshKey), X(sshPublicHostKey), X(compress), X(remoteStore), + +MAKE_PARSE(CommonSSHStoreConfig, commonSSHStoreConfig, COMMON_SSH_STORE_CONFIG_FIELDS) + +static CommonSSHStoreConfigT commonSSHStoreConfigDefaults() { + return { + .sshKey = {""}, + .sshPublicHostKey = {""}, + .compress = {false}, + .remoteStore = {""}, + }; } -const CommonSSHStoreConfig::Descriptions CommonSSHStoreConfig::descriptions{}; +MAKE_APPLY_PARSE(CommonSSHStoreConfig, commonSSHStoreConfig, COMMON_SSH_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap CommonSSHStoreConfig::descriptions() +{ + constexpr auto & descriptions = commonSSHStoreConfigDescriptions; + auto defaults = commonSSHStoreConfigDefaults(); + return {COMMON_SSH_STORE_CONFIG_FIELDS(DESC_ROW)}; +} static std::string extractConnStr(std::string_view scheme, std::string_view _connStr) { @@ -52,16 +70,9 @@ static std::string extractConnStr(std::string_view scheme, std::string_view _con } CommonSSHStoreConfig::CommonSSHStoreConfig( - std::string_view scheme, - std::string_view host, - const StoreReference::Params & params) - : CommonSSHStoreConfigT{ - CONFIG_ROW(sshKey, ""), - CONFIG_ROW(sshPublicHostKey, ""), - CONFIG_ROW(compress, false), - CONFIG_ROW(remoteStore, ""), - } - , host(extractConnStr(scheme, host)) + std::string_view scheme, std::string_view host, const StoreReference::Params & params) + : CommonSSHStoreConfigT{commonSSHStoreConfigApplyParse(params)} + , host{extractConnStr(scheme, host)} { } diff --git a/src/libstore/common-ssh-store-config.hh b/src/libstore/common-ssh-store-config.hh index d9361e259a3..357c3ced5cb 100644 --- a/src/libstore/common-ssh-store-config.hh +++ b/src/libstore/common-ssh-store-config.hh @@ -10,22 +10,15 @@ class SSHMaster; template class F> struct CommonSSHStoreConfigT { - const F sshKey; - const F sshPublicHostKey; - const F compress; - const F remoteStore; + F sshKey; + F sshPublicHostKey; + F compress; + F remoteStore; }; -struct CommonSSHStoreConfig : - CommonSSHStoreConfigT +struct CommonSSHStoreConfig : CommonSSHStoreConfigT { - struct Descriptions : - CommonSSHStoreConfigT - { - Descriptions(); - }; - - static const Descriptions descriptions; + static config::SettingDescriptionMap descriptions(); /** * @param scheme Note this isn't stored by this mix-in class, but From 6ab4817f8802ff0c49f4c3c4f695b4e331f96ddc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 20 Jul 2024 22:48:06 -0400 Subject: [PATCH 052/284] WIP --- src/libstore/common-ssh-store-config.cc | 2 +- src/libstore/common-ssh-store-config.hh | 2 +- src/libstore/legacy-ssh-store.cc | 81 +++++++++++++------------ src/libstore/legacy-ssh-store.hh | 29 ++++----- 4 files changed, 56 insertions(+), 58 deletions(-) diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index 72b44cc41ff..96ec172998f 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -76,7 +76,7 @@ CommonSSHStoreConfig::CommonSSHStoreConfig( { } -SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) +SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) const { return { host, diff --git a/src/libstore/common-ssh-store-config.hh b/src/libstore/common-ssh-store-config.hh index 357c3ced5cb..d998e16a2b3 100644 --- a/src/libstore/common-ssh-store-config.hh +++ b/src/libstore/common-ssh-store-config.hh @@ -56,7 +56,7 @@ struct CommonSSHStoreConfig : CommonSSHStoreConfigT */ SSHMaster createSSHMaster( bool useMaster, - Descriptor logFD = INVALID_DESCRIPTOR); + Descriptor logFD = INVALID_DESCRIPTOR) const; }; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index ea49b6c8ac6..1b5aded4448 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -17,24 +17,32 @@ namespace nix { -LegacySSHStore::Config::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , CommonSSHStoreConfig::Descriptions{CommonSSHStoreConfig::descriptions} - , LegacySSHStoreConfigT{ - .remoteProgram{ - .name = "remote-program", - .description = "Path to the `nix-store` executable on the remote machine.", - }, - .maxConnections{ - .name = "max-connections", - .description = "Maximum number of concurrent SSH connections.", - }, - } -{} +static const LegacySSHStoreConfigT legacySSHStoreConfigDescriptions = { + .remoteProgram{ + .name = "remote-program", + .description = "Path to the `nix-store` executable on the remote machine.", + }, + .maxConnections{ + .name = "max-connections", + .description = "Maximum number of concurrent SSH connections.", + }, +}; +#define LEGACY_SSH_STORE_CONFIG_FIELDS(X) \ + X(remoteProgram), \ + X(maxConnections) -const LegacySSHStore::Config::Descriptions LegacySSHStore::Config::descriptions{}; +MAKE_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS) + +static LegacySSHStoreConfigT legacySSHStoreConfigDefaults() +{ + return { + .remoteProgram = {{"nix-store"}}, + .maxConnections = {1}, + }; +} +MAKE_APPLY_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS) LegacySSHStore::Config::LegacySSHStoreConfig( std::string_view scheme, @@ -42,10 +50,7 @@ LegacySSHStore::Config::LegacySSHStoreConfig( const StoreReference::Params & params) : Store::Config{params} , CommonSSHStoreConfig{scheme, authority, params} - , LegacySSHStoreConfigT{ - CONFIG_ROW(remoteProgram, Strings{"nix-store"}), - CONFIG_ROW(maxConnections, 1), - } + , LegacySSHStoreConfigT{legacySSHStoreConfigApplyParse(params)} { #ifndef _WIN32 if (auto * p = get(params, "log-fd")) { @@ -55,7 +60,7 @@ LegacySSHStore::Config::LegacySSHStoreConfig( } -std::string LegacySSHStoreConfig::doc() +std::string LegacySSHStoreConfig::doc() const { return #include "legacy-ssh-store.md" @@ -69,20 +74,18 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection bool good = true; }; -LegacySSHStore::LegacySSHStore(const Config & config) - : Store::Config{config} - , CommonSSHStoreConfig{config} - , LegacySSHStore::Config{config} - , Store{static_cast(*this)} +LegacySSHStore::LegacySSHStore(ref config) + : Store{*config} + , config{config} , connections(make_ref>( - std::max(1, (int) maxConnections), + std::max(1, (int) config->maxConnections), [this]() { return openConnection(); }, [](const ref & r) { return r->good; } )) - , master(createSSHMaster( + , master(config->createSSHMaster( // Use SSH master only if using more than 1 connection. connections->capacity() > 1, - logFD)) + config->logFD)) { } @@ -90,12 +93,12 @@ LegacySSHStore::LegacySSHStore(const Config & config) ref LegacySSHStore::openConnection() { auto conn = make_ref(); - Strings command = remoteProgram.get(); + Strings command = config->remoteProgram.get(); command.push_back("--serve"); command.push_back("--write"); - if (remoteStore.get() != "") { + if (config->remoteStore.get() != "") { command.push_back("--store"); - command.push_back(remoteStore.get()); + command.push_back(config->remoteStore.get()); } conn->sshConn = master.startCommand(std::move(command)); conn->to = FdSink(conn->sshConn->in.get()); @@ -105,7 +108,7 @@ ref LegacySSHStore::openConnection() TeeSource tee(conn->from, saved); try { conn->remoteVersion = ServeProto::BasicClientConnection::handshake( - conn->to, tee, SERVE_PROTOCOL_VERSION, host); + conn->to, tee, SERVE_PROTOCOL_VERSION, config->host); } catch (SerialisationError & e) { // in.close(): Don't let the remote block on us not writing. conn->sshConn->in.close(); @@ -114,9 +117,9 @@ ref LegacySSHStore::openConnection() tee.drainInto(nullSink); } throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'", - host, chomp(saved.s)); + config->host, chomp(saved.s)); } catch (EndOfFile & e) { - throw Error("cannot connect to '%1%'", host); + throw Error("cannot connect to '%1%'", config->host); } return conn; @@ -125,7 +128,7 @@ ref LegacySSHStore::openConnection() std::string LegacySSHStore::getUri() { - return *uriSchemes().begin() + "://" + host; + return *Config::uriSchemes().begin() + "://" + config->host; } @@ -138,7 +141,7 @@ void LegacySSHStore::queryPathInfoUncached(const StorePath & path, /* No longer support missing NAR hash */ assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); - debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); + debug("querying remote host '%s' for info on '%s'", config->host, printStorePath(path)); auto infos = conn->queryPathInfos(*this, {path}); @@ -167,7 +170,7 @@ void LegacySSHStore::queryPathInfoUncached(const StorePath & path, void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { - debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); + debug("adding path '%s' to remote host '%s'", printStorePath(info.path), config->host); auto conn(connections->get()); @@ -194,7 +197,7 @@ void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source, conn->to.flush(); if (readInt(conn->from) != 1) - throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); + throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), config->host); } else { @@ -346,7 +349,7 @@ std::optional LegacySSHStore::isTrustedClient() ref LegacySSHStore::Config::openStore() const { - return make_ref(*this); + return make_ref(ref{shared_from_this()}); } diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index 4da2d02dfa6..9dc744bfd49 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -12,24 +12,17 @@ namespace nix { template class F> struct LegacySSHStoreConfigT { - const F remoteProgram; - const F maxConnections; + F remoteProgram; + F maxConnections; }; struct LegacySSHStoreConfig : - virtual Store::Config, - virtual CommonSSHStoreConfig, + std::enable_shared_from_this, + Store::Config, + CommonSSHStoreConfig, LegacySSHStoreConfigT { - struct Descriptions : - virtual Store::Config::Descriptions, - virtual CommonSSHStoreConfig::Descriptions, - LegacySSHStoreConfigT - { - Descriptions(); - }; - - static const Descriptions descriptions; + static config::SettingDescriptionMap descriptions(); /** * Hack for getting remote build log output. Intentionally not a @@ -42,26 +35,28 @@ struct LegacySSHStoreConfig : std::string_view authority, const StoreReference::Params & params); - const std::string name() override { return "SSH Store"; } + const std::string name() const override { return "SSH Store"; } static std::set uriSchemes() { return {"ssh"}; } - std::string doc() override; + std::string doc() const override; ref openStore() const override; }; -struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store +struct LegacySSHStore : public virtual Store { using Config = LegacySSHStoreConfig; + ref config; + struct Connection; ref> connections; SSHMaster master; - LegacySSHStore(const Config &); + LegacySSHStore(ref); ref openConnection(); From cc1ce6a4c66ff2c2d875d7c6ccb1ae4fb8aec842 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 20 Jul 2024 23:24:25 -0400 Subject: [PATCH 053/284] WIP --- src/libstore/legacy-ssh-store.cc | 23 ++++ src/libstore/mounted-ssh-store.md | 18 --- src/libstore/ssh-store.cc | 178 ++++++++++++++++-------------- src/libstore/ssh-store.hh | 52 ++------- src/libstore/ssh-store.md | 14 +++ src/libstore/uds-remote-store.cc | 6 +- 6 files changed, 145 insertions(+), 146 deletions(-) delete mode 100644 src/libstore/mounted-ssh-store.md diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 1b5aded4448..110a832e05d 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -28,12 +28,15 @@ static const LegacySSHStoreConfigT legacySSHStoreConfigDesc }, }; + #define LEGACY_SSH_STORE_CONFIG_FIELDS(X) \ X(remoteProgram), \ X(maxConnections) + MAKE_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS) + static LegacySSHStoreConfigT legacySSHStoreConfigDefaults() { return { @@ -42,8 +45,27 @@ static LegacySSHStoreConfigT legacySSHStoreConfigDefaults() }; } + MAKE_APPLY_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap LegacySSHStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(CommonSSHStoreConfig::descriptions()); + ret.merge(RemoteStoreConfig::descriptions()); + { + constexpr auto & descriptions = legacySSHStoreConfigDescriptions; + auto defaults = legacySSHStoreConfigDefaults(); + ret.merge(decltype(ret){ + LEGACY_SSH_STORE_CONFIG_FIELDS(DESC_ROW) + }); + } + return ret; +} + + LegacySSHStore::Config::LegacySSHStoreConfig( std::string_view scheme, std::string_view authority, @@ -74,6 +96,7 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection bool good = true; }; + LegacySSHStore::LegacySSHStore(ref config) : Store{*config} , config{config} diff --git a/src/libstore/mounted-ssh-store.md b/src/libstore/mounted-ssh-store.md deleted file mode 100644 index 1ebfe3081dc..00000000000 --- a/src/libstore/mounted-ssh-store.md +++ /dev/null @@ -1,18 +0,0 @@ -R"( - -**Store URL format**: `mounted-ssh-ng://[username@]hostname` - -Experimental store type that allows full access to a Nix store on a remote machine, -and additionally requires that store be mounted in the local file system. - -The mounting of that store is not managed by Nix, and must by managed manually. -It could be accomplished with SSHFS or NFS, for example. - -The local file system is used to optimize certain operations. -For example, rather than serializing Nix archives and sending over the Nix channel, -we can directly access the file system data via the mount-point. - -The local file system is also used to make certain operations possible that wouldn't otherwise be. -For example, persistent GC roots can be created if they reside on the same file system as the remote store: -the remote side will create the symlinks necessary to avoid race conditions. -)" diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index a83a11b4fdd..25d9f7818ea 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -12,19 +12,61 @@ namespace nix { -SSHStoreConfig::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , CommonSSHStoreConfig::Descriptions{CommonSSHStoreConfig::descriptions} - , SSHStoreConfigT{ - .remoteProgram{ - .name = "remote-program", - .description = "Path to the `nix-daemon` executable on the remote machine.", - }, +static const SSHStoreConfigT sshStoreConfigDescriptions = { + .remoteProgram{ + .name = "remote-program", + .description = "Path to the `nix-daemon` executable on the remote machine.", + }, +}; + + +#define SSH_STORE_CONFIG_FIELDS(X) \ + X(remoteProgram) + + +MAKE_PARSE(SSHStoreConfig, sshStoreConfig, SSH_STORE_CONFIG_FIELDS) + + +static SSHStoreConfigT sshStoreConfigDefaults() +{ + return { + .remoteProgram = {{"nix-daemon"}}, + }; +} + + +MAKE_APPLY_PARSE(SSHStoreConfig, sshStoreConfig, SSH_STORE_CONFIG_FIELDS) + + +config::SettingDescriptionMap SSHStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(CommonSSHStoreConfig::descriptions()); + ret.merge(RemoteStoreConfig::descriptions()); + { + constexpr auto & descriptions = sshStoreConfigDescriptions; + auto defaults = sshStoreConfigDefaults(); + ret.merge(decltype(ret){ + SSH_STORE_CONFIG_FIELDS(DESC_ROW) + }); } -{} + //ret.merge(LocalFSStoreConfig::descriptions()); + return ret; +} -const SSHStoreConfig::Descriptions SSHStoreConfig::descriptions{}; +static std::optional getMounted( + const Store::Config & storeConfig, + const StoreReference::Params & params) +{ + auto mountedParamsOpt = optionalValueAt(params, "mounted"); + if (!mountedParamsOpt) return {}; + auto * mountedParamsP = getNullable(*mountedParamsOpt); + if (!mountedParamsP) return {}; + auto & mountedParams = getObject(*mountedParamsP); + return {{storeConfig, mountedParams}}; +} SSHStoreConfig::SSHStoreConfig( @@ -34,34 +76,40 @@ SSHStoreConfig::SSHStoreConfig( : Store::Config{params} , RemoteStore::Config{params} , CommonSSHStoreConfig{scheme, authority, params} - , SSHStoreConfigT{ - CONFIG_ROW(remoteProgram, Strings{"nix-daemon"}), - } + , SSHStoreConfigT{sshStoreConfigApplyParse(params)} + , mounted{getMounted(*this, params)} { } -std::string SSHStoreConfig::doc() +const std::string SSHStoreConfig::name() const +{ + std::string ret = "Experimental SSH Store"; + if (mounted) + ret += " with filesystem mounted"; + return ret; +} + + +std::string SSHStoreConfig::doc() const { return #include "ssh-store.md" ; } -struct SSHStore : - public virtual SSHStoreConfig, - public virtual RemoteStore + +struct SSHStore : virtual RemoteStore { using Config = SSHStoreConfig; - SSHStore(const Config & config) - : Store::Config{config} - , RemoteStore::Config{config} - , CommonSSHStoreConfig{config} - , SSHStore::Config{config} - , Store{static_cast(*this)} - , RemoteStore{static_cast(*this)} - , master(createSSHMaster( + ref config; + + SSHStore(ref config) + : Store{*config} + , RemoteStore{*config} + , config{config} + , master(config->createSSHMaster( // Use SSH master only if using more than 1 connection. connections->capacity() > 1)) { @@ -69,7 +117,7 @@ struct SSHStore : std::string getUri() override { - return *uriSchemes().begin() + "://" + host; + return *Config::uriSchemes().begin() + "://" + host; } // FIXME extend daemon protocol, move implementation to RemoteStore @@ -108,41 +156,6 @@ struct SSHStore : }; -ref SSHStore::Config::openStore() const { - return make_ref(*this); -} - - -MountedSSHStoreConfig::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , CommonSSHStoreConfig::Descriptions{CommonSSHStoreConfig::descriptions} - , SSHStore::Config::Descriptions{SSHStoreConfig::descriptions} -{} - - -const MountedSSHStoreConfig::Descriptions MountedSSHStoreConfig::descriptions{}; - - -MountedSSHStoreConfig::MountedSSHStoreConfig( - std::string_view scheme, - std::string_view host, - const StoreReference::Params & params) - : Store::Config{params} - , RemoteStore::Config{params} - , CommonSSHStoreConfig{scheme, host, params} - , SSHStoreConfig{scheme, host, params} - , LocalFSStoreConfig{params} -{ -} - -std::string MountedSSHStoreConfig::doc() -{ - return - #include "mounted-ssh-store.md" - ; -} - - /** * The mounted ssh store assumes that filesystems on the remote host are * shared with the local host. This means that the remote nix store is @@ -157,32 +170,24 @@ std::string MountedSSHStoreConfig::doc() * The difference lies in how they manage GC roots. See addPermRoot * below for details. */ -struct MountedSSHStore : virtual MountedSSHStoreConfig, virtual SSHStore, virtual LocalFSStore +struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore { - using Config = MountedSSHStoreConfig; - - MountedSSHStore(const Config & config) - : Store::Config{config} - , RemoteStore::Config{config} - , CommonSSHStoreConfig{config} - , SSHStore::Config{config} - , LocalFSStore::Config{config} - , MountedSSHStore::Config{config} - , Store{static_cast(*this)} - , RemoteStore{static_cast(*this)} - , SSHStore{static_cast(*this)} - , LocalFSStore{static_cast(*this)} + using Config = SSHStore::Config; + + const LocalFSStore::Config & mountedConfig; + + MountedSSHStore(ref config, const LocalFSStore::Config & mountedConfig) + : Store{*config} + , RemoteStore{*config} + , SSHStore{config} + , LocalFSStore{mountedConfig} + , mountedConfig{mountedConfig} { extraRemoteProgramArgs = { "--process-ops", }; } - std::string getUri() override - { - return *uriSchemes().begin() + "://" + host; - } - void narFromPath(const StorePath & path, Sink & sink) override { return LocalFSStore::narFromPath(path, sink); @@ -226,18 +231,23 @@ struct MountedSSHStore : virtual MountedSSHStoreConfig, virtual SSHStore, virtua ref MountedSSHStore::Config::openStore() const { - return make_ref(*this); + ref config {shared_from_this()}; + + if (config->mounted) + return make_ref(config, *config->mounted); + else + return make_ref(config); } ref SSHStore::openConnection() { auto conn = make_ref(); - Strings command = remoteProgram.get(); + Strings command = config->remoteProgram.get(); command.push_back("--stdio"); - if (remoteStore.get() != "") { + if (config->remoteStore.get() != "") { command.push_back("--store"); - command.push_back(remoteStore.get()); + command.push_back(config->remoteStore.get()); } command.insert(command.end(), extraRemoteProgramArgs.begin(), extraRemoteProgramArgs.end()); diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh index e934876c393..4f5796728a2 100644 --- a/src/libstore/ssh-store.hh +++ b/src/libstore/ssh-store.hh @@ -11,63 +11,33 @@ namespace nix { template class F> struct SSHStoreConfigT { - const F remoteProgram; + F remoteProgram; }; -struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig, SSHStoreConfigT +struct SSHStoreConfig : std::enable_shared_from_this, + Store::Config, + RemoteStore::Config, + CommonSSHStoreConfig, + SSHStoreConfigT { - struct Descriptions : virtual RemoteStoreConfig::Descriptions, - virtual CommonSSHStoreConfig::Descriptions, - SSHStoreConfigT - { - Descriptions(); - }; + static config::SettingDescriptionMap descriptions(); - static const Descriptions descriptions; + std::optional mounted; SSHStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params); - const std::string name() override - { - return "Experimental SSH Store"; - } + const std::string name() const override; static std::set uriSchemes() { return {"ssh-ng"}; } - std::string doc() override; - - ref openStore() const override; -}; - -struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStore::Config -{ - struct Descriptions : virtual SSHStoreConfig::Descriptions, virtual LocalFSStore::Config::Descriptions - { - Descriptions(); - }; - - static const Descriptions descriptions; - - MountedSSHStoreConfig(std::string_view scheme, std::string_view host, const StoreReference::Params & params); - - const std::string name() override - { - return "Experimental SSH Store with filesystem mounted"; - } - - static std::set uriSchemes() - { - return {"mounted-ssh-ng"}; - } - - std::string doc() override; + std::string doc() const override; std::optional experimentalFeature() const override { - return ExperimentalFeature::MountedSSHStore; + return mounted ? std::optional{ExperimentalFeature::MountedSSHStore} : std::nullopt; } ref openStore() const override; diff --git a/src/libstore/ssh-store.md b/src/libstore/ssh-store.md index 881537e7114..bdd15ef99ed 100644 --- a/src/libstore/ssh-store.md +++ b/src/libstore/ssh-store.md @@ -5,4 +5,18 @@ R"( Experimental store type that allows full access to a Nix store on a remote machine. +### The `mounted` configuration subject + +This additionally requires that store be mounted in the local file system. + +The mounting of that store is not managed by Nix, and must by managed manually. +It could be accomplished with SSHFS or NFS, for example. + +The local file system is used to optimize certain operations. +For example, rather than serializing Nix archives and sending over the Nix channel, +we can directly access the file system data via the mount-point. + +The local file system is also used to make certain operations possible that wouldn't otherwise be. +For example, persistent GC roots can be created if they reside on the same file system as the remote store: +the remote side will create the symlinks necessary to avoid race conditions. )" diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index e8ec4e8c31d..eff0743a933 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -32,9 +32,9 @@ UDSRemoteStoreConfig::UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, const StoreReference::Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(*this, params) - , RemoteStoreConfig(params) + : Store::Config{params} + , LocalFSStore::Config{*this, params} + , RemoteStore::Config{params} , path{authority.empty() ? settings.nixDaemonSocketFile : authority} { if (uriSchemes().count(std::string{scheme}) == 0) { From 53f692ba602e3d1b3ba7a4cdbba221dc78740dff Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 20 Jul 2024 23:39:05 -0400 Subject: [PATCH 054/284] WIP --- src/libstore/store-registration.hh | 20 +++++++++++++++----- src/nix/main.cc | 13 +++++-------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/libstore/store-registration.hh b/src/libstore/store-registration.hh index 289bdf341ff..7c254d61b3f 100644 --- a/src/libstore/store-registration.hh +++ b/src/libstore/store-registration.hh @@ -13,7 +13,10 @@ namespace nix { struct StoreFactory { + std::string doc; std::set uriSchemes; + config::SettingDescriptionMap configDescriptions; + std::optional experimentalFeature; /** * The `authorityPath` parameter is `/`, or really * whatever comes after `://` and before `?`. @@ -21,26 +24,33 @@ struct StoreFactory std::function( std::string_view scheme, std::string_view authorityPath, const StoreReference::Params & params)> parseConfig; - config::SettingDescriptionMap configDescriptions; }; struct Implementations { - static std::vector * registered; +private: + + using V = std::vector>; + +public: + + static V * registered; template static void add() { if (!registered) - registered = new std::vector(); + registered = new V{}; StoreFactory factory{ + .doc = TConfig::doc(), .uriSchemes = TConfig::uriSchemes(), + .configDescriptions = TConfig::descriptions(), + .experimentalFeature = TConfig::experimentalFeature(), .parseConfig = ([](auto scheme, auto uri, auto & params) -> ref { return make_ref(scheme, uri, params); }), - .configDescriptions = TConfig::descriptions(), }; - registered->push_back(factory); + registered->push_back({TConfig::name(), std::move(factory)}); } }; diff --git a/src/nix/main.cc b/src/nix/main.cc index 524c4626350..158f758e589 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -222,15 +222,12 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs res["args"] = toJSON(); auto stores = nlohmann::json::object(); - for (auto & implem : *Implementations::registered) { -#if 0 - auto storeConfig = implem.getConfig(); - auto storeName = storeConfig->name(); + for (auto & [storeName, implem] : *Implementations::registered) { auto & j = stores[storeName]; - j["doc"] = storeConfig->doc(); - j["settings"] = storeConfig->toJSON(); - j["experimentalFeature"] = storeConfig->experimentalFeature(); -#endif + j["doc"] = implem.doc; + j["uri-schemes"] = implem.uriSchemes; + j["settings"] = implem.configDescriptions; + j["experimentalFeature"] = implem.experimentalFeature; } res["stores"] = std::move(stores); res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo(); From e6212087f04a869943f59a3368508031dfbf2f13 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 21 Jul 2024 10:05:50 -0400 Subject: [PATCH 055/284] WIP --- src/libstore/binary-cache-store.cc | 118 ++++++++++-------- src/libstore/binary-cache-store.hh | 25 ++-- src/libstore/config-parse.cc | 32 +++++ src/libstore/dummy-store.cc | 2 +- src/libstore/dummy-store.hh | 20 +++ src/libstore/http-binary-cache-store.cc | 8 ++ src/libstore/http-binary-cache-store.hh | 18 ++- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/legacy-ssh-store.hh | 4 +- src/libstore/local-binary-cache-store.hh | 9 +- src/libstore/local-overlay-store.cc | 2 +- src/libstore/local-overlay-store.hh | 6 +- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 4 +- src/libstore/s3-binary-cache-store.hh | 41 +++--- src/libstore/ssh-store.cc | 12 +- src/libstore/ssh-store.hh | 12 +- src/libstore/store-api.hh | 9 +- src/libstore/store-registration.cc | 4 +- src/libstore/store-registration.hh | 17 +++ src/libstore/uds-remote-store.cc | 2 +- src/libstore/uds-remote-store.hh | 4 +- .../unix/build/local-derivation-goal.cc | 2 - 23 files changed, 208 insertions(+), 147 deletions(-) create mode 100644 src/libstore/config-parse.cc create mode 100644 src/libstore/dummy-store.hh diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 1bd9363a272..23bd633296b 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -25,66 +25,78 @@ namespace nix { -BinaryCacheStore::Config::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , BinaryCacheStoreConfigT{ - .compression = { - .name = "compression", - .description = "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`).", - }, - .writeNARListing = { - .name = "write-nar-listing", - .description = "Whether to write a JSON file that lists the files in each NAR.", - }, - .writeDebugInfo = { - .name = "index-debug-info", - .description = R"( - Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to - fetch debug info on demand - )", - }, - .secretKeyFile{ - .name = "secret-key", - .description = "Path to the secret key used to sign the binary cache.", - }, - .localNarCache{ - .name = "local-nar-cache", - .description = "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`.", - }, - .parallelCompression{ - .name = "parallel-compression", - .description = "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`.", - }, - .compressionLevel{ - .name = "compression-level", - .description = R"( - The *preset level* to be used when compressing NARs. - The meaning and accepted values depend on the compression method selected. - `-1` specifies that the default compression level should be used. - )", - }, - } -{} +static const BinaryCacheStoreConfigT binaryCacheStoreConfigDescriptions = { + .compression = { + .name = "compression", + .description = "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`).", + }, + .writeNARListing = { + .name = "write-nar-listing", + .description = "Whether to write a JSON file that lists the files in each NAR.", + }, + .writeDebugInfo = { + .name = "index-debug-info", + .description = R"( + Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to + fetch debug info on demand + )", + }, + .secretKeyFile{ + .name = "secret-key", + .description = "Path to the secret key used to sign the binary cache.", + }, + .localNarCache{ + .name = "local-nar-cache", + .description = "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`.", + }, + .parallelCompression{ + .name = "parallel-compression", + .description = "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`.", + }, + .compressionLevel{ + .name = "compression-level", + .description = R"( + The *preset level* to be used when compressing NARs. + The meaning and accepted values depend on the compression method selected. + `-1` specifies that the default compression level should be used. + )", + }, +}; + +#define BINARY_CACHE_STORE_CONFIG_FIELDS(X) \ + X(compression), \ + X(writeNARListing), \ + X(writeDebugInfo), \ + X(secretKeyFile), \ + X(localNarCache), \ + X(parallelCompression), \ + X(compressionLevel), + +MAKE_PARSE(BinaryCacheStoreConfig, binaryCacheStoreConfig, BINARY_CACHE_STORE_CONFIG_FIELDS) + +static BinaryCacheStoreConfigT binaryCacheStoreConfigDefaults() +{ + return { + .compression = {"xz"}, + .writeNARListing = {false}, + .writeDebugInfo = {false}, + .secretKeyFile = {""}, + .localNarCache = {""}, + .parallelCompression = {false}, + .compressionLevel = {-1}, + }; +} -const BinaryCacheStore::Config::Descriptions BinaryCacheStore::Config::descriptions{}; +MAKE_APPLY_PARSE(BinaryCacheStoreConfig, binaryCacheStoreConfig, BINARY_CACHE_STORE_CONFIG_FIELDS) BinaryCacheStore::Config::BinaryCacheStoreConfig(const StoreReference::Params & params) - : StoreConfig{params} - , BinaryCacheStoreConfigT{ - CONFIG_ROW(compression, "xz"), - CONFIG_ROW(writeNARListing, false), - CONFIG_ROW(writeDebugInfo, false), - CONFIG_ROW(secretKeyFile, ""), - CONFIG_ROW(localNarCache, ""), - CONFIG_ROW(parallelCompression, false), - CONFIG_ROW(compressionLevel, -1), - } + : BinaryCacheStoreConfigT{binaryCacheStoreConfigApplyParse(params)} { } BinaryCacheStore::BinaryCacheStore(const Config & config) - : Config{config} - , Store{static_cast(*this)} + : Store{config} + , Config{config} { if (secretKeyFile != "") signer = std::make_unique( diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 877a5c3343c..c1c8a06b3f2 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -16,27 +16,20 @@ struct NarInfo; template class F> struct BinaryCacheStoreConfigT { - const F compression; - const F writeNARListing; - const F writeDebugInfo; - const F secretKeyFile; - const F localNarCache; - const F parallelCompression; - const F compressionLevel; + F compression; + F writeNARListing; + F writeDebugInfo; + F secretKeyFile; + F localNarCache; + F parallelCompression; + F compressionLevel; }; struct BinaryCacheStoreConfig : - virtual Store::Config, BinaryCacheStoreConfigT { - struct Descriptions : - virtual Store::Config::Descriptions, - BinaryCacheStoreConfigT - { - Descriptions(); - }; - - static const Descriptions descriptions; + virtual Store::Config, + static config::SettingDescriptionMap descriptions(); BinaryCacheStoreConfig(const StoreReference::Params &); }; diff --git a/src/libstore/config-parse.cc b/src/libstore/config-parse.cc new file mode 100644 index 00000000000..727bc391917 --- /dev/null +++ b/src/libstore/config-parse.cc @@ -0,0 +1,32 @@ +#include + +#include "config-parse.hh" + +namespace nix::config { + + +}; + +namespace nlohmann { + +using namespace nix::config; + +SettingDescription adl_serializer::from_json(const json & json) +{ + // TODO implement if we ever need (testing?) + assert(false); +} + +void adl_serializer::to_json(json & obj, SettingDescription s) +{ + obj.emplace("description", s.description); + //obj.emplace("aliases", s.aliases); + //obj.emplace("experimentalFeature", s.experimentalFeature); + + // Cannot use `null` because the default value might itself be + // `null`. + if (s.defaultValue) + obj.emplace("defaultValue", *s.defaultValue); +} + +} diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 503ec346d13..628b6bd3d07 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -12,7 +12,7 @@ DummyStoreConfig::DummyStoreConfig( throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); } -std::string DummyStoreConfig::doc() const +std::string DummyStoreConfig::doc() { return #include "dummy-store.md" diff --git a/src/libstore/dummy-store.hh b/src/libstore/dummy-store.hh new file mode 100644 index 00000000000..41683800b4a --- /dev/null +++ b/src/libstore/dummy-store.hh @@ -0,0 +1,20 @@ +#include "store-api.hh" + +namespace nix { + +struct DummyStoreConfig : std::enable_shared_from_this, StoreConfig +{ + DummyStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params); + + static const std::string name() { return "Dummy Store"; } + + static std::string doc(); + + static std::set uriSchemes() { + return {"dummy"}; + } + + ref openStore() const override; +}; + +} diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 3cdd7e8874d..6aee70a9c4a 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -9,6 +9,14 @@ namespace nix { MakeError(UploadToHTTP, Error); +std::set HttpBinaryCacheStoreConfig::uriSchemes() +{ + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; + auto ret = std::set({"http", "https"}); + if (forceHttp) + ret.insert("file"); + return ret; +} HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( std::string_view scheme, diff --git a/src/libstore/http-binary-cache-store.hh b/src/libstore/http-binary-cache-store.hh index 397bd0592a6..c7cc1bccf03 100644 --- a/src/libstore/http-binary-cache-store.hh +++ b/src/libstore/http-binary-cache-store.hh @@ -2,28 +2,24 @@ namespace nix { -struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +struct HttpBinaryCacheStoreConfig : + std::enable_shared_from_this, + Store::Config, + BinaryCacheStoreConfig { HttpBinaryCacheStoreConfig( std::string_view scheme, std::string_view cacheUri, const StoreReference::Params & params); Path cacheUri; - const std::string name() override + static const std::string name() { return "HTTP Binary Cache Store"; } - static std::set uriSchemes() - { - static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; - auto ret = std::set({"http", "https"}); - if (forceHttp) - ret.insert("file"); - return ret; - } + static std::set uriSchemes(); - std::string doc() override; + static std::string doc(); ref openStore() const override; }; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 110a832e05d..85dba07c4a5 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -82,7 +82,7 @@ LegacySSHStore::Config::LegacySSHStoreConfig( } -std::string LegacySSHStoreConfig::doc() const +std::string LegacySSHStoreConfig::doc() { return #include "legacy-ssh-store.md" diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index 9dc744bfd49..d87b19aa118 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -35,11 +35,11 @@ struct LegacySSHStoreConfig : std::string_view authority, const StoreReference::Params & params); - const std::string name() const override { return "SSH Store"; } + static const std::string name() { return "SSH Store"; } static std::set uriSchemes() { return {"ssh"}; } - std::string doc() const override; + static std::string doc(); ref openStore() const override; }; diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh index 647cd83d099..b29dfc3c89f 100644 --- a/src/libstore/local-binary-cache-store.hh +++ b/src/libstore/local-binary-cache-store.hh @@ -2,7 +2,10 @@ namespace nix { -struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +struct LocalBinaryCacheStoreConfig : + std::enable_shared_from_this, + Store::Config, + BinaryCacheStoreConfig { /** * @param binaryCacheDir `file://` is a short-hand for `file:///` @@ -13,14 +16,14 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig Path binaryCacheDir; - const std::string name() override + static const std::string name() { return "Local Binary Cache Store"; } static std::set uriSchemes(); - std::string doc() override; + static std::string doc(); ref openStore() const override; }; diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 5f3d9281814..d00584263aa 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -86,7 +86,7 @@ LocalOverlayStore::Config::LocalOverlayStoreConfig( } -std::string LocalOverlayStoreConfig::doc() const +std::string LocalOverlayStoreConfig::doc() { return #include "local-overlay-store.md" diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 112abd6be6b..dd449c4fc9c 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -26,9 +26,9 @@ struct LocalOverlayStoreConfig : PathView path, const StoreReference::Params & params); - const std::string name() const override { return "Experimental Local Overlay Store"; } + static const std::string name() { return "Experimental Local Overlay Store"; } - std::optional experimentalFeature() const override + static std::optional experimentalFeature() { return ExperimentalFeature::LocalOverlayStore; } @@ -38,7 +38,7 @@ struct LocalOverlayStoreConfig : return { "local-overlay" }; } - std::string doc() const override; + static std::string doc(); protected: /** diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2572ed9e75a..b16d56ebb3d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -108,7 +108,7 @@ LocalStore::Config::LocalStoreConfig( { } -std::string LocalStoreConfig::doc() const +std::string LocalStoreConfig::doc() { return #include "local-store.md" diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index fd72249abe0..a37301ebfd3 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -58,12 +58,12 @@ struct LocalStoreConfig : std::string_view authority, const StoreReference::Params & params); - const std::string name() const override { return "Local Store"; } + static const std::string name() { return "Local Store"; } static std::set uriSchemes() { return {"local"}; } - std::string doc() const override; + static std::string doc(); ref openStore() const override; }; diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index 2097bb5b427..e67f682d773 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -10,34 +10,31 @@ namespace nix { template class F> struct S3BinaryCacheStoreConfigT { - const F profile; - const F region; - const F scheme; - const F endpoint; - const F narinfoCompression; - const F lsCompression; - const F logCompression; - const F multipartUpload; - const F bufferSize; + F profile; + F region; + F scheme; + F endpoint; + F narinfoCompression; + F lsCompression; + F logCompression; + F multipartUpload; + F bufferSize; }; -struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig, S3BinaryCacheStoreConfigT +struct S3BinaryCacheStoreConfig : + std::enable_shared_from_this, + Store::Config, + BinaryCacheStoreConfig, + S3BinaryCacheStoreConfigT { - struct Descriptions : virtual Store::Config::Descriptions, - virtual BinaryCacheStore::Config::Descriptions, - S3BinaryCacheStoreConfigT - { - Descriptions(); - }; - - static const Descriptions descriptions; + static config::SettingDescriptionMap configDescriptions(); S3BinaryCacheStoreConfig( std::string_view uriScheme, std::string_view bucketName, const StoreReference::Params & params); std::string bucketName; - const std::string name() override + static std::string name() { return "S3 Binary Cache Store"; } @@ -47,7 +44,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig, S3BinaryCacheS return {"s3"}; } - std::string doc() override; + static std::string doc(); ref openStore() const override; }; @@ -56,9 +53,11 @@ struct S3BinaryCacheStore : virtual BinaryCacheStore { using Config = S3BinaryCacheStoreConfig; + ref config; + protected: - S3BinaryCacheStore(const Config &); + S3BinaryCacheStore(ref); public: diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 25d9f7818ea..53140955a46 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -82,18 +82,8 @@ SSHStoreConfig::SSHStoreConfig( } -const std::string SSHStoreConfig::name() const +std::string SSHStoreConfig::doc() { - std::string ret = "Experimental SSH Store"; - if (mounted) - ret += " with filesystem mounted"; - return ret; -} - - -std::string SSHStoreConfig::doc() const -{ - return #include "ssh-store.md" ; } diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh index 4f5796728a2..7f71c275a0b 100644 --- a/src/libstore/ssh-store.hh +++ b/src/libstore/ssh-store.hh @@ -26,19 +26,17 @@ struct SSHStoreConfig : std::enable_shared_from_this, SSHStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params); - const std::string name() const override; + static const std::string name() + { + return "Experimental SSH Store"; + } static std::set uriSchemes() { return {"ssh-ng"}; } - std::string doc() const override; - - std::optional experimentalFeature() const override - { - return mounted ? std::optional{ExperimentalFeature::MountedSSHStore} : std::nullopt; - } + static std::string doc(); ref openStore() const override; }; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 87b8335430a..099f2aee724 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -130,15 +130,10 @@ struct StoreConfig : static StringSet getDefaultSystemFeatures(); - /** - * The name of this type of store. - */ - virtual const std::string name() const = 0; - /** * Documentation for this type of store. */ - virtual std::string doc() const + static std::string doc() { return ""; } @@ -147,7 +142,7 @@ struct StoreConfig : * An experimental feature this type store is gated, if it is to be * experimental. */ - virtual std::optional experimentalFeature() const + static std::optional experimentalFeature() { return std::nullopt; } diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc index 00bd522d600..83f6255331b 100644 --- a/src/libstore/store-registration.cc +++ b/src/libstore/store-registration.cc @@ -59,7 +59,7 @@ ref resolveStoreConfig(StoreReference && storeURI) return make_ref(params); }, [&](const StoreReference::Specified & g) { - for (auto implem : *Implementations::registered) + for (auto & [name, implem] : *Implementations::registered) if (implem.uriSchemes.count(g.scheme)) return implem.parseConfig(g.scheme, g.authority, params); @@ -73,7 +73,7 @@ ref resolveStoreConfig(StoreReference && storeURI) return storeConfig; } -std::vector * Implementations::registered = 0; +Implementations::V * Implementations::registered = 0; std::list> getDefaultSubstituters() { diff --git a/src/libstore/store-registration.hh b/src/libstore/store-registration.hh index 7c254d61b3f..e479966250e 100644 --- a/src/libstore/store-registration.hh +++ b/src/libstore/store-registration.hh @@ -13,10 +13,24 @@ namespace nix { struct StoreFactory { + /** + * Documentation for this type of store. + */ std::string doc; + + /** + * URIs with these schemes should be handled by this factory + */ std::set uriSchemes; + config::SettingDescriptionMap configDescriptions; + + /** + * An experimental feature this type store is gated, if it is to be + * experimental. + */ std::optional experimentalFeature; + /** * The `authorityPath` parameter is `/`, or really * whatever comes after `://` and before `?`. @@ -30,6 +44,9 @@ struct Implementations { private: + /** + * The name of this type of store, and a factory for it. + */ using V = std::vector>; public: diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index eff0743a933..b6a108d2d9e 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -43,7 +43,7 @@ UDSRemoteStoreConfig::UDSRemoteStoreConfig( } -std::string UDSRemoteStoreConfig::doc() const +std::string UDSRemoteStoreConfig::doc() { return #include "uds-remote-store.md" diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 75203cbd20c..ce9fcbfeec8 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -27,9 +27,9 @@ struct UDSRemoteStoreConfig : std::string_view authority, const StoreReference::Params & params); - const std::string name() const override { return "Local Daemon Store"; } + static const std::string name() { return "Local Daemon Store"; } - std::string doc() const override; + static std::string doc(); /** * The path to the unix domain socket. diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 1d242f5927a..de30718481b 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -1256,8 +1256,6 @@ bool LocalDerivationGoal::isAllowed(const DerivedPath & req) struct RestrictedStoreConfig : virtual LocalFSStoreConfig { - const std::string name() override { return "Restricted Store"; } - ref openStore() const override; ref next; From 05861c4008951450ca5ba944c9f83e9d217edd0e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 21 Jul 2024 11:16:09 -0400 Subject: [PATCH 056/284] WIP --- src/build-remote/build-remote.cc | 4 +- src/libstore/binary-cache-store.cc | 47 +++++----- src/libstore/binary-cache-store.hh | 8 +- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/build/substitution-goal.cc | 4 +- src/libstore/config-parse.cc | 5 +- src/libstore/dummy-store.hh | 8 +- src/libstore/gc.cc | 24 +++--- src/libstore/http-binary-cache-store.cc | 39 +++++---- src/libstore/http-binary-cache-store.hh | 9 +- src/libstore/local-binary-cache-store.cc | 39 +++++---- src/libstore/local-binary-cache-store.hh | 9 +- src/libstore/local-store.hh | 5 ++ src/libstore/meson.build | 1 + src/libstore/optimise-store.cc | 8 +- src/libstore/parsed-derivations.cc | 2 +- src/libstore/remote-store.cc | 61 +++++++------ src/libstore/remote-store.hh | 4 +- src/libstore/s3-binary-cache-store.hh | 11 ++- src/libstore/ssh-store.cc | 2 +- src/libstore/uds-remote-store.cc | 2 +- .../unix/build/local-derivation-goal.cc | 85 ++++++++----------- tests/unit/libstore/ssh-store.cc | 14 ++- 23 files changed, 207 insertions(+), 186 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 1824e20024e..929394dfd72 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -44,7 +44,7 @@ static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { for (auto & feature : requiredFeatures) - if (!store.systemFeatures.get().count(feature)) return false; + if (!store.config.systemFeatures.get().count(feature)) return false; return true; } @@ -85,7 +85,7 @@ static int main_build_remote(int argc, char * * argv) that gets cleared on reboot, but it wouldn't work on macOS. */ auto currentLoadName = "/current-load"; if (auto localStore = store.dynamic_pointer_cast()) - currentLoad = std::string { localStore->stateDir } + currentLoadName; + currentLoad = std::string { localStore->config.stateDir } + currentLoadName; else currentLoad = settings.nixStateDir + currentLoadName; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 23bd633296b..1246845130b 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -89,18 +89,21 @@ static BinaryCacheStoreConfigT binaryCacheStoreConfigDefaults MAKE_APPLY_PARSE(BinaryCacheStoreConfig, binaryCacheStoreConfig, BINARY_CACHE_STORE_CONFIG_FIELDS) -BinaryCacheStore::Config::BinaryCacheStoreConfig(const StoreReference::Params & params) +BinaryCacheStore::Config::BinaryCacheStoreConfig( + const Store::Config & storeConfig, + const StoreReference::Params & params) : BinaryCacheStoreConfigT{binaryCacheStoreConfigApplyParse(params)} + , storeConfig{storeConfig} { } BinaryCacheStore::BinaryCacheStore(const Config & config) - : Store{config} - , Config{config} + : Store{config.storeConfig} + , config{config} { - if (secretKeyFile != "") + if (config.secretKeyFile != "") signer = std::make_unique( - SecretKey { readFile(secretKeyFile) }); + SecretKey { readFile(config.secretKeyFile) }); StringSink sink; sink << narVersionMagic1; @@ -127,11 +130,11 @@ void BinaryCacheStore::init() throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'", getUri(), value, storeDir); } else if (name == "WantMassQuery") { - if (defaultWantMassQuery) - wantMassQuery.value = value == "1"; + resolvedSubstConfig.wantMassQuery.value = + config.storeConfig.wantMassQuery.optValue.value_or(value == "1"); } else if (name == "Priority") { - if (defaultPriority) - priority.value = std::stoi(value); + resolvedSubstConfig.priority.value = + config.storeConfig.priority.optValue.value_or(std::stoi(value)); } } } @@ -218,7 +221,11 @@ ref BinaryCacheStore::addToStoreCommon( { FdSink fileSink(fdTemp.get()); TeeSink teeSinkCompressed { fileSink, fileHashSink }; - auto compressionSink = makeCompressionSink(compression, teeSinkCompressed, parallelCompression, compressionLevel); + auto compressionSink = makeCompressionSink( + config.compression, + teeSinkCompressed, + config.parallelCompression, + config.compressionLevel); TeeSink teeSinkUncompressed { *compressionSink, narHashSink }; TeeSource teeSource { narSource, teeSinkUncompressed }; narAccessor = makeNarAccessor(teeSource); @@ -230,17 +237,17 @@ ref BinaryCacheStore::addToStoreCommon( auto info = mkInfo(narHashSink.finish()); auto narInfo = make_ref(info); - narInfo->compression = compression; + narInfo->compression = config.compression; auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar" - + (compression == "xz" ? ".xz" : - compression == "bzip2" ? ".bz2" : - compression == "zstd" ? ".zst" : - compression == "lzip" ? ".lzip" : - compression == "lz4" ? ".lz4" : - compression == "br" ? ".br" : + + (config.compression == "xz" ? ".xz" : + config.compression == "bzip2" ? ".bz2" : + config.compression == "zstd" ? ".zst" : + config.compression == "lzip" ? ".lzip" : + config.compression == "lz4" ? ".lz4" : + config.compression == "br" ? ".br" : ""); auto duration = std::chrono::duration_cast(now2 - now1).count(); @@ -262,7 +269,7 @@ ref BinaryCacheStore::addToStoreCommon( /* Optionally write a JSON file containing a listing of the contents of the NAR. */ - if (writeNARListing) { + if (config.writeNARListing) { nlohmann::json j = { {"version", 1}, {"root", listNar(ref(narAccessor), CanonPath::root, true)}, @@ -274,7 +281,7 @@ ref BinaryCacheStore::addToStoreCommon( /* Optionally maintain an index of DWARF debug info files consisting of JSON files named 'debuginfo/' that specify the NAR file and member containing the debug info. */ - if (writeDebugInfo) { + if (config.writeDebugInfo) { CanonPath buildIdDir("lib/debug/.build-id"); @@ -586,7 +593,7 @@ void BinaryCacheStore::registerDrvOutput(const Realisation& info) { ref BinaryCacheStore::getFSAccessor(bool requireValidPath) { - return make_ref(ref(shared_from_this()), requireValidPath, localNarCache); + return make_ref(ref(shared_from_this()), requireValidPath, config.localNarCache); } void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index c1c8a06b3f2..af649aaab86 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -28,10 +28,11 @@ struct BinaryCacheStoreConfigT struct BinaryCacheStoreConfig : BinaryCacheStoreConfigT { - virtual Store::Config, static config::SettingDescriptionMap descriptions(); - BinaryCacheStoreConfig(const StoreReference::Params &); + const Store::Config & storeConfig; + + BinaryCacheStoreConfig(const Store::Config &, const StoreReference::Params &); }; /** @@ -39,12 +40,13 @@ struct BinaryCacheStoreConfig : * virtual getFile() methods. */ struct BinaryCacheStore : - virtual BinaryCacheStoreConfig, virtual Store, virtual LogStore { using Config = BinaryCacheStoreConfig; + const Config & config; + private: std::unique_ptr signer; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index b809e3ffe3f..606151cf557 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1261,7 +1261,7 @@ Path DerivationGoal::openLogFile() /* Create a log file. */ Path logDir; if (auto localStore = dynamic_cast(&worker.store)) - logDir = localStore->logDir; + logDir = localStore->config->logDir; else logDir = settings.nixLogDir; Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, baseName.substr(0, 2)); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 32189ca7ae5..6d8cfe2d2cf 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -122,7 +122,7 @@ Goal::Co PathSubstitutionGoal::init() /* Bail out early if this substituter lacks a valid signature. LocalStore::addToStore() also checks for this, but only after we've downloaded the path. */ - if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) + if (!sub->config.isTrusted && worker.store.pathInfoIsUntrusted(*info)) { warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", worker.store.printStorePath(storePath), sub->getUri()); @@ -209,7 +209,7 @@ Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, PushActivity pact(act.id); copyStorePath(*sub, worker.store, - subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + subPath, repair, sub->config.isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { diff --git a/src/libstore/config-parse.cc b/src/libstore/config-parse.cc index 727bc391917..5d3a29574f6 100644 --- a/src/libstore/config-parse.cc +++ b/src/libstore/config-parse.cc @@ -4,7 +4,6 @@ namespace nix::config { - }; namespace nlohmann { @@ -20,8 +19,8 @@ SettingDescription adl_serializer::from_json(const json & js void adl_serializer::to_json(json & obj, SettingDescription s) { obj.emplace("description", s.description); - //obj.emplace("aliases", s.aliases); - //obj.emplace("experimentalFeature", s.experimentalFeature); + // obj.emplace("aliases", s.aliases); + // obj.emplace("experimentalFeature", s.experimentalFeature); // Cannot use `null` because the default value might itself be // `null`. diff --git a/src/libstore/dummy-store.hh b/src/libstore/dummy-store.hh index 41683800b4a..80f19a67a2a 100644 --- a/src/libstore/dummy-store.hh +++ b/src/libstore/dummy-store.hh @@ -6,11 +6,15 @@ struct DummyStoreConfig : std::enable_shared_from_this, StoreC { DummyStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params); - static const std::string name() { return "Dummy Store"; } + static const std::string name() + { + return "Dummy Store"; + } static std::string doc(); - static std::set uriSchemes() { + static std::set uriSchemes() + { return {"dummy"}; } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 54e09cb1639..ab8130ee221 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -40,7 +40,7 @@ static std::string gcRootsDir = "gcroots"; void LocalStore::addIndirectRoot(const Path & path) { std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Nix32, false); - Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); + Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", config->stateDir, gcRootsDir, hash)); makeSymlink(realRoot, path); } @@ -79,7 +79,7 @@ void LocalStore::createTempRootsFile() void LocalStore::addTempRoot(const StorePath & path) { - if (readOnly) { + if (config->readOnly) { debug("Read-only store doesn't support creating lock files for temp roots, but nothing can be deleted anyways."); return; } @@ -106,7 +106,7 @@ void LocalStore::addTempRoot(const StorePath & path) auto fdRootsSocket(_fdRootsSocket.lock()); if (!*fdRootsSocket) { - auto socketPath = stateDir.get() + gcSocketPath; + auto socketPath = config->stateDir.get() + gcSocketPath; debug("connecting to '%s'", socketPath); *fdRootsSocket = createUnixDomainSocket(); try { @@ -244,7 +244,7 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R else { target = absPath(target, dirOf(path)); if (!pathExists(target)) { - if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) { + if (isInDir(path, config->stateDir + "/" + gcRootsDir + "/auto")) { printInfo("removing stale link from '%1%' to '%2%'", path, target); unlink(path.c_str()); } @@ -285,8 +285,8 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R void LocalStore::findRootsNoTemp(Roots & roots, bool censor) { /* Process direct roots in {gcroots,profiles}. */ - findRoots(stateDir + "/" + gcRootsDir, std::filesystem::file_type::unknown, roots); - findRoots(stateDir + "/profiles", std::filesystem::file_type::unknown, roots); + findRoots(config->stateDir + "/" + gcRootsDir, std::filesystem::file_type::unknown, roots); + findRoots(config->stateDir + "/profiles", std::filesystem::file_type::unknown, roots); /* Add additional roots returned by different platforms-specific heuristics. This is typically used to add running programs to @@ -495,7 +495,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) readFile(*p); /* Start the server for receiving new roots. */ - auto socketPath = stateDir.get() + gcSocketPath; + auto socketPath = config->stateDir.get() + gcSocketPath; createDirs(dirOf(socketPath)); auto fdServer = createUnixDomainSocket(socketPath, 0666); @@ -632,7 +632,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) auto deleteFromStore = [&](std::string_view baseName) { Path path = storeDir + "/" + std::string(baseName); - Path realPath = realStoreDir + "/" + std::string(baseName); + Path realPath = config->realStoreDir + "/" + std::string(baseName); /* There may be temp directories in the store that are still in use by another process. We need to be sure that we can acquire an @@ -796,8 +796,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) printInfo("determining live/dead paths..."); try { - AutoCloseDir dir(opendir(realStoreDir.get().c_str())); - if (!dir) throw SysError("opening directory '%1%'", realStoreDir); + AutoCloseDir dir(opendir(config->realStoreDir.get().c_str())); + if (!dir) throw SysError("opening directory '%1%'", config->realStoreDir); /* Read the store and delete all paths that are invalid or unreachable. We don't use readDirectory() here so that @@ -899,8 +899,8 @@ void LocalStore::autoGC(bool sync) return std::stoll(readFile(*fakeFreeSpaceFile)); struct statvfs st; - if (statvfs(realStoreDir.get().c_str(), &st)) - throw SysError("getting filesystem info about '%s'", realStoreDir); + if (statvfs(config->realStoreDir.get().c_str(), &st)) + throw SysError("getting filesystem info about '%s'", config->realStoreDir); return (uint64_t) st.f_bavail * st.f_frsize; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 6aee70a9c4a..7a54218f579 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -22,8 +22,8 @@ HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( std::string_view scheme, std::string_view _cacheUri, const StoreReference::Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) + : Store::Config{params} + , BinaryCacheStoreConfig{*this, params} , cacheUri( std::string { scheme } + "://" @@ -45,7 +45,6 @@ std::string HttpBinaryCacheStoreConfig::doc() class HttpBinaryCacheStore : - public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore { struct State @@ -60,36 +59,36 @@ class HttpBinaryCacheStore : using Config = HttpBinaryCacheStoreConfig; - HttpBinaryCacheStore(const Config & config) - : Store::Config{config} - , BinaryCacheStore::Config{config} - , HttpBinaryCacheStore::Config{config} - , Store{static_cast(*this)} - , BinaryCacheStore{static_cast(*this)} + ref config; + + HttpBinaryCacheStore(ref config) + : Store{*config} + , BinaryCacheStore{*config} + , config{config} { diskCache = getNarInfoDiskCache(); } std::string getUri() override { - return cacheUri; + return config->cacheUri; } void init() override { // FIXME: do this lazily? - if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) { - if (defaultWantMassQuery) - wantMassQuery.value = cacheInfo->wantMassQuery; - if (defaultPriority) - priority.value = cacheInfo->priority; + if (auto cacheInfo = diskCache->upToDateCacheExists(config->cacheUri)) { + resolvedSubstConfig.wantMassQuery.value = + config->storeConfig.wantMassQuery.optValue.value_or(cacheInfo->wantMassQuery); + resolvedSubstConfig.priority.value = + config->storeConfig.priority.optValue.value_or(cacheInfo->priority); } else { try { BinaryCacheStore::init(); } catch (UploadToHTTP &) { - throw Error("'%s' does not appear to be a binary cache", cacheUri); + throw Error("'%s' does not appear to be a binary cache", config->cacheUri); } - diskCache->createCache(cacheUri, storeDir, wantMassQuery, priority); + diskCache->createCache(config->cacheUri, storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority); } } @@ -147,7 +146,7 @@ class HttpBinaryCacheStore : try { getFileTransfer()->upload(req); } catch (FileTransferError & e) { - throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); + throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", config->cacheUri, e.msg()); } } @@ -156,7 +155,7 @@ class HttpBinaryCacheStore : return FileTransferRequest( hasPrefix(path, "https://") || hasPrefix(path, "http://") || hasPrefix(path, "file://") ? path - : cacheUri + "/" + path); + : config->cacheUri + "/" + path); } @@ -219,7 +218,7 @@ class HttpBinaryCacheStore : ref HttpBinaryCacheStore::Config::openStore() const { - return make_ref(*this); + return make_ref(ref{shared_from_this()}); } static RegisterStoreImplementation regHttpBinaryCacheStore; diff --git a/src/libstore/http-binary-cache-store.hh b/src/libstore/http-binary-cache-store.hh index c7cc1bccf03..0767e7dc8da 100644 --- a/src/libstore/http-binary-cache-store.hh +++ b/src/libstore/http-binary-cache-store.hh @@ -2,11 +2,12 @@ namespace nix { -struct HttpBinaryCacheStoreConfig : - std::enable_shared_from_this, - Store::Config, - BinaryCacheStoreConfig +struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this, + Store::Config, + BinaryCacheStoreConfig { + static config::SettingDescriptionMap descriptions(); + HttpBinaryCacheStoreConfig( std::string_view scheme, std::string_view cacheUri, const StoreReference::Params & params); diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 0997cfd77c1..143ff3c5445 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -12,8 +12,8 @@ LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig( std::string_view scheme, PathView binaryCacheDir, const StoreReference::Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) + : Store::Config{params} + , BinaryCacheStoreConfig{*this, params} , binaryCacheDir(binaryCacheDir) { } @@ -28,17 +28,16 @@ std::string LocalBinaryCacheStoreConfig::doc() struct LocalBinaryCacheStore : - virtual LocalBinaryCacheStoreConfig, virtual BinaryCacheStore { using Config = LocalBinaryCacheStoreConfig; - LocalBinaryCacheStore(const Config & config) - : Store::Config{config} - , BinaryCacheStore::Config{config} - , LocalBinaryCacheStore::Config{config} - , Store{static_cast(*this)} - , BinaryCacheStore{static_cast(*this)} + ref config; + + LocalBinaryCacheStore(ref config) + : Store{*config} + , BinaryCacheStore{*config} + , config{config} { } @@ -46,7 +45,7 @@ struct LocalBinaryCacheStore : std::string getUri() override { - return "file://" + binaryCacheDir; + return "file://" + config->binaryCacheDir; } protected: @@ -57,7 +56,7 @@ struct LocalBinaryCacheStore : std::shared_ptr> istream, const std::string & mimeType) override { - auto path2 = binaryCacheDir + "/" + path; + auto path2 = config->binaryCacheDir + "/" + path; static std::atomic counter{0}; Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter); AutoDelete del(tmp, false); @@ -70,7 +69,7 @@ struct LocalBinaryCacheStore : void getFile(const std::string & path, Sink & sink) override { try { - readFile(binaryCacheDir + "/" + path, sink); + readFile(config->binaryCacheDir + "/" + path, sink); } catch (SysError & e) { if (e.errNo == ENOENT) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); @@ -82,7 +81,7 @@ struct LocalBinaryCacheStore : { StorePathSet paths; - for (auto & entry : std::filesystem::directory_iterator{binaryCacheDir}) { + for (auto & entry : std::filesystem::directory_iterator{config->binaryCacheDir}) { checkInterrupt(); auto name = entry.path().filename().string(); if (name.size() != 40 || @@ -104,17 +103,17 @@ struct LocalBinaryCacheStore : void LocalBinaryCacheStore::init() { - createDirs(binaryCacheDir + "/nar"); - createDirs(binaryCacheDir + "/" + realisationsPrefix); - if (writeDebugInfo) - createDirs(binaryCacheDir + "/debuginfo"); - createDirs(binaryCacheDir + "/log"); + createDirs(config->binaryCacheDir + "/nar"); + createDirs(config->binaryCacheDir + "/" + realisationsPrefix); + if (config->writeDebugInfo) + createDirs(config->binaryCacheDir + "/debuginfo"); + createDirs(config->binaryCacheDir + "/log"); BinaryCacheStore::init(); } bool LocalBinaryCacheStore::fileExists(const std::string & path) { - return pathExists(binaryCacheDir + "/" + path); + return pathExists(config->binaryCacheDir + "/" + path); } std::set LocalBinaryCacheStoreConfig::uriSchemes() @@ -126,7 +125,7 @@ std::set LocalBinaryCacheStoreConfig::uriSchemes() } ref LocalBinaryCacheStoreConfig::openStore() const { - return make_ref(*this); + return make_ref(ref{shared_from_this()}); } static RegisterStoreImplementation regLocalBinaryCacheStore; diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh index b29dfc3c89f..9c12b4221c6 100644 --- a/src/libstore/local-binary-cache-store.hh +++ b/src/libstore/local-binary-cache-store.hh @@ -2,11 +2,12 @@ namespace nix { -struct LocalBinaryCacheStoreConfig : - std::enable_shared_from_this, - Store::Config, - BinaryCacheStoreConfig +struct LocalBinaryCacheStoreConfig : std::enable_shared_from_this, + Store::Config, + BinaryCacheStoreConfig { + static config::SettingDescriptionMap descriptions(); + /** * @param binaryCacheDir `file://` is a short-hand for `file:///` * for now. diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index a37301ebfd3..2bd4a5ea76e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -58,6 +58,11 @@ struct LocalStoreConfig : std::string_view authority, const StoreReference::Params & params); + /** + * For `RestrictedStore` + */ + LocalStoreConfig(const LocalStoreConfig &); + static const std::string name() { return "Local Store"; } static std::set uriSchemes() diff --git a/src/libstore/meson.build b/src/libstore/meson.build index ef01704de9a..d389655635f 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -55,6 +55,7 @@ configdata.set('CAN_LINK_SYMLINK', can_link_symlink.to_int()) check_funcs = [ # Optionally used for canonicalising files from the build 'lchown', + 'statvfs', ] foreach funcspec : check_funcs define_name = 'HAVE_' + funcspec.underscorify().to_upper() diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 9d903f21869..6ebd5b947d7 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -100,7 +100,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* HFS/macOS has some undocumented security feature disabling hardlinking for special files within .app dirs. Known affected paths include *.app/Contents/{PkgInfo,Resources/\*.lproj,_CodeSignature} and .DS_Store. - See https://github.com/NixOS/nix/issues/1443 and + See https://github.com/NixOS/nix/issues/1443 and https://github.com/NixOS/nix/pull/2230 for more discussion. */ if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) @@ -215,14 +215,14 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, the store itself (we don't want or need to mess with its permissions). */ const Path dirOfPath(dirOf(path)); - bool mustToggle = dirOfPath != realStoreDir.get(); + bool mustToggle = dirOfPath != config->realStoreDir.get(); if (mustToggle) makeWritable(dirOfPath); /* When we're done, make the directory read-only again and reset its timestamp back to 0. */ MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : ""); - std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", realStoreDir, getpid(), rand()); + std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", config->realStoreDir, getpid(), rand()); try { std::filesystem::create_hard_link(linkPath, tempLink); @@ -284,7 +284,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) if (!isValidPath(i)) continue; /* path was GC'ed, probably */ { Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i))); - optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash, NoRepair); + optimisePath_(&act, stats, config->realStoreDir + "/" + std::string(i.to_string()), inodeHash, NoRepair); } done++; act.progress(done, paths.size()); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index d8459d4d71c..c95d348d05c 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -110,7 +110,7 @@ bool ParsedDerivation::canBuildLocally(Store & localStore) const return false; for (auto & feature : getRequiredSystemFeatures()) - if (!localStore.systemFeatures.get().count(feature)) return false; + if (!localStore.config.systemFeatures.get().count(feature)) return false; return true; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index cee331dde88..ba7e1a5d87b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -24,40 +24,51 @@ namespace nix { -RemoteStore::Config::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , RemoteStoreConfigT{ - .maxConnections{ - .name = "max-connections", - .description = "Maximum number of concurrent connections to the Nix daemon.", - }, - .maxConnectionAge{ - .name = "max-connection-age", - .description = "Maximum age of a connection before it is closed.", - }, - } -{} +static const RemoteStoreConfigT remoteStoreConfigDescriptions = { + .maxConnections{ + .name = "max-connections", + .description = "Maximum number of concurrent connections to the Nix daemon.", + }, + .maxConnectionAge{ + .name = "max-connection-age", + .description = "Maximum age of a connection before it is closed.", + }, +}; -const RemoteStore::Config::Descriptions RemoteStore::Config::descriptions{}; +#define REMOTE_STORE_CONFIG_FIELDS(X) \ + X(maxConnections), \ + X(maxConnectionAge), -RemoteStore::Config::RemoteStoreConfig(const StoreReference::Params & params) - : Store::Config(params) - , RemoteStoreConfigT{ - CONFIG_ROW(maxConnections, 1), - CONFIG_ROW(maxConnectionAge, std::numeric_limits::max()), - } +MAKE_PARSE(RemoteStoreConfig, remoteStoreConfig, REMOTE_STORE_CONFIG_FIELDS) + + +static RemoteStoreConfigT remoteStoreConfigDefaults() +{ + return { + .maxConnections = {1}, + .maxConnectionAge = {std::numeric_limits::max()}, + }; +} + + +MAKE_APPLY_PARSE(RemoteStoreConfig, remoteStoreConfig, REMOTE_STORE_CONFIG_FIELDS) + + +RemoteStore::Config::RemoteStoreConfig(const Store::Config & storeConfig, const StoreReference::Params & params) + : RemoteStoreConfigT{remoteStoreConfigApplyParse(params)} + , storeConfig{storeConfig} { } /* TODO: Separate these store types into different files, give them better names */ RemoteStore::RemoteStore(const Config & config) - : RemoteStore::Config(config) - , Store(static_cast(*this)) + : Store{config.storeConfig} + , config{config} , connections(make_ref>( - std::max(1, (int) maxConnections), + std::max(1, (int) config.maxConnections), [this]() { auto conn = openConnectionWrapper(); try { @@ -68,12 +79,12 @@ RemoteStore::RemoteStore(const Config & config) } return conn; }, - [this](const ref & r) { + [config](const ref & r) { return r->to.good() && r->from.good() && std::chrono::duration_cast( - std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge; + std::chrono::steady_clock::now() - r->startTime).count() < config.maxConnectionAge; } )) { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index b79f68b3c53..41dd673e31c 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -29,7 +29,9 @@ struct RemoteStoreConfig : RemoteStoreConfigT { static config::SettingDescriptionMap descriptions(); - RemoteStoreConfig(const StoreReference::Params &); + const Store::Config & storeConfig; + + RemoteStoreConfig(const Store::Config &, const StoreReference::Params &); }; /** diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index e67f682d773..18ea4cd1592 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -21,13 +21,12 @@ struct S3BinaryCacheStoreConfigT F bufferSize; }; -struct S3BinaryCacheStoreConfig : - std::enable_shared_from_this, - Store::Config, - BinaryCacheStoreConfig, - S3BinaryCacheStoreConfigT +struct S3BinaryCacheStoreConfig : std::enable_shared_from_this, + Store::Config, + BinaryCacheStoreConfig, + S3BinaryCacheStoreConfigT { - static config::SettingDescriptionMap configDescriptions(); + static config::SettingDescriptionMap descriptions(); S3BinaryCacheStoreConfig( std::string_view uriScheme, std::string_view bucketName, const StoreReference::Params & params); diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 53140955a46..070da11666e 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -74,7 +74,7 @@ SSHStoreConfig::SSHStoreConfig( std::string_view authority, const StoreReference::Params & params) : Store::Config{params} - , RemoteStore::Config{params} + , RemoteStore::Config{*this, params} , CommonSSHStoreConfig{scheme, authority, params} , SSHStoreConfigT{sshStoreConfigApplyParse(params)} , mounted{getMounted(*this, params)} diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index b6a108d2d9e..5e8b40fe61d 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -34,7 +34,7 @@ UDSRemoteStoreConfig::UDSRemoteStoreConfig( const StoreReference::Params & params) : Store::Config{params} , LocalFSStore::Config{*this, params} - , RemoteStore::Config{params} + , RemoteStore::Config{*this, params} , path{authority.empty() ? settings.nixDaemonSocketFile : authority} { if (uriSchemes().count(std::string{scheme}) == 0) { diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index de30718481b..7c3c73cea1d 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -32,7 +32,7 @@ #include #if HAVE_STATVFS -#include +# include #endif /* Includes required for chroot support. */ @@ -56,8 +56,8 @@ #endif #if __APPLE__ -#include -#include +# include +# include #endif #include @@ -214,7 +214,7 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() } auto & localStore = getLocalStore(); - if (localStore.storeDir != localStore.realStoreDir.get()) { + if (localStore.storeDir != localStore.config->realStoreDir.get()) { #if __linux__ useChroot = true; #else @@ -354,7 +354,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() auto & localStore = getLocalStore(); uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable struct statvfs st; - if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 && + if (statvfs(localStore.config->realStoreDir.get().c_str(), &st) == 0 && (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; if (statvfs(tmpDir.c_str(), &st) == 0 && @@ -500,7 +500,7 @@ void LocalDerivationGoal::startBuilder() concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), worker.store.printStorePath(drvPath), settings.thisSystem, - concatStringsSep(", ", worker.store.systemFeatures)); + concatStringsSep(", ", worker.store.config.systemFeatures)); /* Create a temporary directory where the build will take place. */ @@ -1254,48 +1254,46 @@ bool LocalDerivationGoal::isAllowed(const DerivedPath & req) return this->isAllowed(pathPartOfReq(req)); } -struct RestrictedStoreConfig : virtual LocalFSStoreConfig +/** + * A wrapper around LocalStore that only allows building/querying of + * paths that are in the input closures of the build or were added via + * recursive Nix calls. + */ +struct RestrictedStore : + virtual IndirectRootStore, + virtual GcStore { - ref openStore() const override; + ref config; ref next; LocalDerivationGoal & goal; - RestrictedStoreConfig( - const StoreReference::Params & params, + RestrictedStore( + ref config, ref next, LocalDerivationGoal & goal) - : Store::Config{params} - , LocalFSStore::Config{params} + : Store{*config} + , LocalFSStore{*config} + , config{config} , next{next} , goal{goal} { } -}; -/** - * A wrapper around LocalStore that only allows building/querying of - * paths that are in the input closures of the build or were added via - * recursive Nix calls. - */ -struct RestrictedStore : - virtual RestrictedStoreConfig, - virtual IndirectRootStore, - virtual GcStore -{ - using Config = RestrictedStoreConfig; - - RestrictedStore(const RestrictedStoreConfig & config) - : Store::Config(config) - , LocalFSStore::Config(config) - , RestrictedStore::Config(config) - , Store(static_cast(*this)) - , LocalFSStore(static_cast(*this)) - { } + static ref make( + ref next, + LocalDerivationGoal & goal) + { + ref config = make_ref(*next->config); + config->pathInfoCacheSize.value = 0; + config->stateDir.value = "/no-such-path"; + config->logDir.value = "/no-such-path"; + return make_ref(std::move(config), std::move(next), goal); + } Path getRealStoreDir() override - { return next->realStoreDir; } + { return next->config->realStoreDir; } std::string getUri() override { return next->getUri(); } @@ -1495,28 +1493,13 @@ struct RestrictedStore : }; -ref RestrictedStore::Config::openStore() const -{ - return make_ref(*this); -} - - void LocalDerivationGoal::startDaemon() { experimentalFeatureSettings.require(Xp::RecursiveNix); - StoreReference::Params params; - params["path-info-cache-size"] = "0"; - params["store"] = worker.store.storeDir; - if (auto & optRoot = getLocalStore().rootDir.get()) - params["root"] = *optRoot; - params["state"] = "/no-such-path"; - params["log"] = "/no-such-path"; - auto store = RestrictedStore::Config{ - params, + auto store = RestrictedStore::make( ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), - *this, - }.openStore(); + *this); addedPaths.clear(); @@ -1844,7 +1827,7 @@ void LocalDerivationGoal::runChild() createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); - if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) + if (worker.store.config.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) ss.push_back("/dev/kvm"); ss.push_back("/dev/null"); ss.push_back("/dev/random"); diff --git a/tests/unit/libstore/ssh-store.cc b/tests/unit/libstore/ssh-store.cc index 0e2651f2e9b..8852273dcf9 100644 --- a/tests/unit/libstore/ssh-store.cc +++ b/tests/unit/libstore/ssh-store.cc @@ -7,7 +7,7 @@ namespace nix { TEST(SSHStore, constructConfig) { SSHStoreConfig config{ - "ssh", + "ssh-ng", "localhost", StoreReference::Params{ { @@ -30,8 +30,8 @@ TEST(SSHStore, constructConfig) TEST(MountedSSHStore, constructConfig) { - MountedSSHStoreConfig config{ - "mounted-ssh", + SSHStoreConfig config{ + "ssh-ng", "localhost", StoreReference::Params{ { @@ -41,6 +41,10 @@ TEST(MountedSSHStore, constructConfig) "bar", }, }, + { + "mounted", + nlohmann::json::object_t{}, + }, }, }; @@ -50,6 +54,10 @@ TEST(MountedSSHStore, constructConfig) "foo", "bar", })); + + ASSERT_TRUE(config.mounted); + + EXPECT_EQ(config.mounted->realStoreDir, "/nix/store"); } } From b0c6f187811d52e37d32014c087b72d04e2cc299 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 21 Jul 2024 13:50:43 -0400 Subject: [PATCH 057/284] Build `S3BinaryCacheStore` with Meson --- src/libstore/meson.build | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/libstore/meson.build b/src/libstore/meson.build index d389655635f..1eb413c1913 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -100,6 +100,20 @@ deps_public += nlohmann_json sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') deps_private += sqlite +# AWS C++ SDK has bad pkg-config +aws_s3 = dependency('aws-cpp-sdk-s3', required : false) +configdata.set('ENABLE_S3', aws_s3.found().to_int()) +if aws_s3.found() + aws_s3 = declare_dependency( + include_directories: include_directories(aws_s3.get_variable('includedir')), + link_args: [ + '-L' + aws_s3.get_variable('libdir'), + '-laws-cpp-sdk-s3', + ], + ) +endif +deps_other += aws_s3 + subdir('build-utils-meson/generate-header') generated_headers = [] From 40ebe0f5a76b52b5c1ba46bc9b3fccc1f34c8134 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 21 Jul 2024 17:34:48 -0400 Subject: [PATCH 058/284] S3 binary cache store --- configure.ac | 7 - src/libstore/http-binary-cache-store.cc | 3 +- src/libstore/s3-binary-cache-store.cc | 247 +++++++++++++----------- src/libstore/s3-binary-cache-store.hh | 4 - 4 files changed, 134 insertions(+), 127 deletions(-) diff --git a/configure.ac b/configure.ac index 4f66a3efcf6..5c22ed17636 100644 --- a/configure.ac +++ b/configure.ac @@ -340,13 +340,6 @@ AC_CHECK_HEADERS([aws/s3/S3Client.h], AC_SUBST(ENABLE_S3, [$enable_s3]) AC_LANG_POP(C++) -if test -n "$enable_s3"; then - declare -a aws_version_tokens=($(printf '#include \nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' ')) - AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.]) - AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.]) - AC_DEFINE_UNQUOTED([AWS_VERSION_PATCH], ${aws_version_tokens@<:@2@:>@}, [Patch version of aws-sdk-cpp.]) -fi - # Whether to use the Boehm garbage collector. AC_ARG_ENABLE(gc, AS_HELP_STRING([--enable-gc],[enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]), diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 7a54218f579..201cd8bba51 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -88,7 +88,8 @@ class HttpBinaryCacheStore : } catch (UploadToHTTP &) { throw Error("'%s' does not appear to be a binary cache", config->cacheUri); } - diskCache->createCache(config->cacheUri, storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority); + diskCache->createCache( + config->cacheUri, storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority); } } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 19f2a69c419..efd3a9bbd4d 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -60,7 +60,7 @@ class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem debug("AWS: %s", chomp(statement)); } -#if !(AWS_VERSION_MAJOR <= 1 && AWS_VERSION_MINOR <= 7 && AWS_VERSION_PATCH <= 115) +#if !(AWS_SDK_VERSION_MAJOR <= 1 && AWS_SDK_VERSION_MINOR <= 7 && AWS_SDK_VERSION_PATCH <= 115) void Flush() override {} #endif }; @@ -103,7 +103,7 @@ S3Helper::S3Helper( std::make_shared(profile.c_str())), *config, // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759 -#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3 +#if AWS_SDK_VERSION_MAJOR == 1 && AWS_SDK_VERSION_MINOR < 3 false, #else Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, @@ -188,96 +188,110 @@ S3Helper::FileTransferResult S3Helper::getObject( } -S3BinaryCacheStore::Config::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , BinaryCacheStore::Config::Descriptions{BinaryCacheStore::Config::descriptions} - , S3BinaryCacheStoreConfigT{ - .profile{ - .name = "profile", - .description = R"( - The name of the AWS configuration profile to use. By default - Nix will use the `default` profile. - )", - }, - .region{ - .name = "region", - .description = R"( - The region of the S3 bucket. If your bucket is not in - `us–east-1`, you should always explicitly specify the region - parameter. - )", - }, - .scheme{ - .name = "scheme", - .description = R"( - The scheme used for S3 requests, `https` (default) or `http`. This - option allows you to disable HTTPS for binary caches which don't - support it. - - > **Note** - > - > HTTPS should be used if the cache might contain sensitive - > information. - )", - }, - .endpoint{ - .name = "endpoint", - .description = R"( - The URL of the endpoint of an S3-compatible service such as MinIO. - Do not specify this setting if you're using Amazon S3. - - > **Note** - > - > This endpoint must support HTTPS and will use path-based - > addressing instead of virtual host based addressing. - )", - }, - .narinfoCompression{ - .name = "narinfo-compression", - .description = "Compression method for `.narinfo` files.", - }, - .lsCompression{ - .name = "ls-compression", - .description = "Compression method for `.ls` files.", - }, - .logCompression{ - .name = "log-compression", - .description = R"( - Compression method for `log/*` files. It is recommended to - use a compression method supported by most web browsers - (e.g. `brotli`). - )", - }, - .multipartUpload{ - .name = "multipart-upload", - .description = "Whether to use multi-part uploads.", - }, - .bufferSize{ - .name = "buffer-size", - .description = "Size (in bytes) of each part in multi-part uploads.", - }, - } -{} +static const S3BinaryCacheStoreConfigT s3BinaryCacheStoreConfigDescriptions = { + .profile{ + .name = "profile", + .description = R"( + The name of the AWS configuration profile to use. By default + Nix will use the `default` profile. + )", + }, + .region{ + .name = "region", + .description = R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )", + }, + .scheme{ + .name = "scheme", + .description = R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )", + }, + .endpoint{ + .name = "endpoint", + .description = R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and will use path-based + > addressing instead of virtual host based addressing. + )", + }, + .narinfoCompression{ + .name = "narinfo-compression", + .description = "Compression method for `.narinfo` files.", + }, + .lsCompression{ + .name = "ls-compression", + .description = "Compression method for `.ls` files.", + }, + .logCompression{ + .name = "log-compression", + .description = R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )", + }, + .multipartUpload{ + .name = "multipart-upload", + .description = "Whether to use multi-part uploads.", + }, + .bufferSize{ + .name = "buffer-size", + .description = "Size (in bytes) of each part in multi-part uploads.", + }, +}; + +#define S3_BINARY_CACHE_STORE_CONFIG_FIELDS(X) \ + X(profile), \ + X(region), \ + X(scheme), \ + X(endpoint), \ + X(narinfoCompression), \ + X(lsCompression), \ + X(logCompression), \ + X(multipartUpload), \ + X(bufferSize), + +MAKE_PARSE(S3BinaryCacheStoreConfig, s3BinaryCacheStoreConfig, S3_BINARY_CACHE_STORE_CONFIG_FIELDS) + +static S3BinaryCacheStoreConfigT s3BinaryCacheStoreConfigDefaults() +{ + return { + .profile = {""}, + .region = {Aws::Region::US_EAST_1}, + .scheme = {""}, + .endpoint = {""}, + .narinfoCompression = {""}, + .lsCompression = {""}, + .logCompression = {""}, + .multipartUpload = {false}, + .bufferSize = {5 * 1024 * 1024}, + }; +} -const S3BinaryCacheStore::Config::Descriptions S3BinaryCacheStore::Config::descriptions{}; +MAKE_APPLY_PARSE(S3BinaryCacheStoreConfig, s3BinaryCacheStoreConfig, S3_BINARY_CACHE_STORE_CONFIG_FIELDS) S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig( std::string_view scheme, std::string_view authority, const StoreReference::Params & params) - : Store::Config(params) - , BinaryCacheStore::Config(params) - , S3BinaryCacheStoreConfigT{ - CONFIG_ROW(profile, ""), - CONFIG_ROW(region, Aws::Region::US_EAST_1), - CONFIG_ROW(scheme, ""), - CONFIG_ROW(endpoint, ""), - CONFIG_ROW(narinfoCompression, ""), - CONFIG_ROW(lsCompression, ""), - CONFIG_ROW(logCompression, ""), - CONFIG_ROW(multipartUpload, false), - CONFIG_ROW(bufferSize, 5 * 1024 * 1024), - } + : Store::Config{params} + , BinaryCacheStore::Config{*this, params} + , S3BinaryCacheStoreConfigT{s3BinaryCacheStoreConfigApplyParse(params)} , bucketName{authority} { if (bucketName.empty()) @@ -285,8 +299,9 @@ S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig( } -S3BinaryCacheStore::S3BinaryCacheStore(const Config & config) - : BinaryCacheStore(config) +S3BinaryCacheStore::S3BinaryCacheStore(ref config) + : BinaryCacheStore(*config) + , config{config} { } std::string S3BinaryCacheStoreConfig::doc() @@ -297,39 +312,37 @@ std::string S3BinaryCacheStoreConfig::doc() } -struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore +struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore { Stats stats; S3Helper s3Helper; - S3BinaryCacheStoreImpl(const Config & config) - : Store::Config{config} - , BinaryCacheStore::Config{config} - , S3BinaryCacheStore::Config{config} - , Store{static_cast(*this)} - , BinaryCacheStore{static_cast(*this)} - , S3BinaryCacheStore{static_cast(*this)} - , s3Helper(profile, region, scheme, endpoint) + S3BinaryCacheStoreImpl(ref config) + : Store{*config} + , BinaryCacheStore{*config} + , S3BinaryCacheStore{config} + , s3Helper(config->profile, config->region, config->scheme, config->endpoint) { diskCache = getNarInfoDiskCache(); } std::string getUri() override { - return "s3://" + bucketName; + return "s3://" + config->bucketName; } void init() override { if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) { - if (defaultWantMassQuery) - wantMassQuery.value = cacheInfo->wantMassQuery; - if (defaultPriority) - priority.value = cacheInfo->priority; + resolvedSubstConfig.wantMassQuery.value = + config->storeConfig.wantMassQuery.optValue.value_or(cacheInfo->wantMassQuery); + resolvedSubstConfig.priority.value = + config->storeConfig.priority.optValue.value_or(cacheInfo->priority); } else { BinaryCacheStore::init(); - diskCache->createCache(getUri(), storeDir, wantMassQuery, priority); + diskCache->createCache( + getUri(), config->storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority); } } @@ -358,7 +371,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual auto res = s3Helper.client->HeadObject( Aws::S3::Model::HeadObjectRequest() - .WithBucket(bucketName) + .WithBucket(config->bucketName) .WithKey(path)); if (!res.IsSuccess()) { @@ -393,11 +406,11 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual std::call_once(transferManagerCreated, [&]() { - if (multipartUpload) { + if (config->multipartUpload) { TransferManagerConfiguration transferConfig(executor.get()); transferConfig.s3Client = s3Helper.client; - transferConfig.bufferSize = bufferSize; + transferConfig.bufferSize = config->bufferSize; transferConfig.uploadProgressCallback = [](const TransferManager *transferManager, @@ -418,6 +431,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual auto now1 = std::chrono::steady_clock::now(); + auto & bucketName = config->bucketName; + if (transferManager) { if (contentEncoding != "") @@ -481,12 +496,12 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual return std::make_shared(std::move(compressed)); }; - if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) - uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression); - else if (lsCompression != "" && hasSuffix(path, ".ls")) - uploadFile(path, compress(lsCompression), mimeType, lsCompression); - else if (logCompression != "" && hasPrefix(path, "log/")) - uploadFile(path, compress(logCompression), mimeType, logCompression); + if (config->narinfoCompression != "" && hasSuffix(path, ".narinfo")) + uploadFile(path, compress(config->narinfoCompression), mimeType, config->narinfoCompression); + else if (config->lsCompression != "" && hasSuffix(path, ".ls")) + uploadFile(path, compress(config->lsCompression), mimeType, config->lsCompression); + else if (config->logCompression != "" && hasPrefix(path, "log/")) + uploadFile(path, compress(config->logCompression), mimeType, config->logCompression); else uploadFile(path, istream, mimeType, ""); } @@ -496,14 +511,14 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual stats.get++; // FIXME: stream output to sink. - auto res = s3Helper.getObject(bucketName, path); + auto res = s3Helper.getObject(config->bucketName, path); stats.getBytes += res.data ? res.data->size() : 0; stats.getTimeMs += res.durationMs; if (res.data) { printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", - bucketName, path, res.data->size(), res.durationMs); + config->bucketName, path, res.data->size(), res.durationMs); sink(*res.data); } else @@ -515,6 +530,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual StorePathSet paths; std::string marker; + auto & bucketName = config->bucketName; + do { debug("listing bucket 's3://%s' from key '%s'...", bucketName, marker); @@ -555,7 +572,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual ref S3BinaryCacheStoreImpl::Config::openStore() const { - return make_ref(*this); + return make_ref(ref{shared_from_this()}); } static RegisterStoreImplementation regS3BinaryCacheStore; diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index 18ea4cd1592..aa3b92bc99c 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -54,12 +54,8 @@ struct S3BinaryCacheStore : virtual BinaryCacheStore ref config; -protected: - S3BinaryCacheStore(ref); -public: - struct Stats { std::atomic put{0}; From 0ec5e3a1bccb17b79a1927fd4b9e99195062db7f Mon Sep 17 00:00:00 2001 From: poweredbypie <67135060+poweredbypie@users.noreply.github.com> Date: Sun, 21 Jul 2024 15:03:04 -0700 Subject: [PATCH 059/284] Progress on Wine CI support, MinGW dev shell with Meson (#10975) * Only build perl subproject on Linux * Fix various Windows regressions * Don't put the emulator hook in test builds We run the tests in a separate derivation. Only need it for the dev shell. * Fix native dev shells * Fix cross dev shells we don't know how to emulate Co-authored-by: PoweredByPie Co-authored-by: Joachim Schiele Co-authored-by: John Ericson --- flake.nix | 20 ++++++++++++----- meson.build | 4 +++- src/libexpr-c/nix_api_expr.cc | 12 +++++----- src/libexpr-c/nix_api_external.cc | 4 ++-- src/libexpr-c/nix_api_value.cc | 4 ++-- .../build/drv-output-substitution-goal.cc | 2 +- src/libutil/file-system.cc | 6 ++++- src/libutil/fs-sink.cc | 4 ++-- src/nix/meson.build | 22 +++++++++++-------- 9 files changed, 49 insertions(+), 29 deletions(-) diff --git a/flake.nix b/flake.nix index 51dbc309108..ff2c8ecfaed 100644 --- a/flake.nix +++ b/flake.nix @@ -278,6 +278,7 @@ in "-D${prefix}:${rest}"; havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; + ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; in { pname = "shell-for-" + attrs.pname; @@ -309,10 +310,12 @@ }; mesonFlags = - map (transformFlag "libutil") pkgs.nixComponents.nix-util.mesonFlags - ++ map (transformFlag "libstore") pkgs.nixComponents.nix-store.mesonFlags - ++ map (transformFlag "libfetchers") pkgs.nixComponents.nix-fetchers.mesonFlags - ++ lib.optionals havePerl (map (transformFlag "perl") pkgs.nixComponents.nix-perl-bindings.mesonFlags) + map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags) + ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags) + ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags) + ++ lib.optionals havePerl (map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags)) + ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags) + ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags) ; nativeBuildInputs = attrs.nativeBuildInputs or [] @@ -322,9 +325,16 @@ ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs + ++ lib.optional + (!stdenv.buildPlatform.canExecute stdenv.hostPlatform + # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 + && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) + && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages + && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages)) + pkgs.buildPackages.mesonEmulatorHook ++ [ pkgs.buildPackages.cmake - pkgs.shellcheck + pkgs.buildPackages.shellcheck modular.pre-commit.settings.package (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) diff --git a/meson.build b/meson.build index 1c46c5c2881..1554244ab66 100644 --- a/meson.build +++ b/meson.build @@ -29,7 +29,9 @@ subproject('libexpr-c') subproject('libmain-c') # Language Bindings -subproject('perl') +if not meson.is_cross_build() + subproject('perl') +endif # Testing subproject('nix-util-test-support') diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 547453f8f93..8f21d702221 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -14,10 +14,10 @@ #include "nix_api_util.h" #include "nix_api_util_internal.h" -#ifdef HAVE_BOEHMGC -#include -#define GC_INCLUDE_NEW 1 -#include "gc_cpp.h" +#if HAVE_BOEHMGC +# include +# define GC_INCLUDE_NEW 1 +# include "gc_cpp.h" #endif nix_err nix_libexpr_init(nix_c_context * context) @@ -131,7 +131,7 @@ void nix_state_free(EvalState * state) delete state; } -#ifdef HAVE_BOEHMGC +#if HAVE_BOEHMGC std::unordered_map< const void *, unsigned int, @@ -207,7 +207,7 @@ nix_err nix_value_decref(nix_c_context * context, nix_value *x) void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd)) { -#ifdef HAVE_BOEHMGC +#if HAVE_BOEHMGC GC_REGISTER_FINALIZER(obj, finalizer, cd, 0, 0); #endif } diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc index 3565092a4a6..fa78eb5df39 100644 --- a/src/libexpr-c/nix_api_external.cc +++ b/src/libexpr-c/nix_api_external.cc @@ -14,7 +14,7 @@ #include -#ifdef HAVE_BOEHMGC +#if HAVE_BOEHMGC # include "gc/gc.h" # define GC_INCLUDE_NEW 1 # include "gc_cpp.h" @@ -174,7 +174,7 @@ ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalV context->last_err_code = NIX_OK; try { auto ret = new -#ifdef HAVE_BOEHMGC +#if HAVE_BOEHMGC (GC) #endif NixCExternalValue(*desc, v); diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index cb5d9ee8928..845e879352f 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -14,7 +14,7 @@ #include "nix_api_value.h" #include "value/context.hh" -#ifdef HAVE_BOEHMGC +#if HAVE_BOEHMGC # include "gc/gc.h" # define GC_INCLUDE_NEW 1 # include "gc_cpp.h" @@ -131,7 +131,7 @@ PrimOp * nix_alloc_primop( try { using namespace std::placeholders; auto p = new -#ifdef HAVE_BOEHMGC +#if HAVE_BOEHMGC (GC) #endif nix::PrimOp{ diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 02284d93c1a..dedcad2b100 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -62,7 +62,7 @@ Goal::Co DrvOutputSubstitutionGoal::init() #ifndef _WIN32 outPipe->readSide.get() #else - &outPipe + &*outPipe #endif }, true, false); diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 9042e3a5e67..060a806fbc5 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -416,7 +416,11 @@ void deletePath(const fs::path & path) void createDir(const Path & path, mode_t mode) { - if (mkdir(path.c_str(), mode) == -1) + if (mkdir(path.c_str() +#ifndef _WIN32 + , mode +#endif + ) == -1) throw SysError("creating directory '%1%'", path); } diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 194e86fdd6b..3246e0902fb 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -97,7 +97,7 @@ void RestoreSink::createRegularFile(const CanonPath & path, std::function Date: Mon, 22 Jul 2024 12:05:50 +0200 Subject: [PATCH 060/284] maintainers/README: Update Monday meeting time (#11147) --- maintainers/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintainers/README.md b/maintainers/README.md index 2a718e283a2..b92833497ab 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -50,7 +50,7 @@ The team meets twice a week (times are denoted in the [Europe/Amsterdam](https:/ - mark it as draft if it is blocked on the contributor - escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again. -- Work meeting: [Mondays 13:00-15:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Work meeting: [Mondays 14:00-16:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. From b16861d82eab5627f79d4573221f5f97048711f5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 22 Jul 2024 12:56:01 +0200 Subject: [PATCH 061/284] libexpr: Track and show GC time and cycle number --- src/libexpr/eval-gc.cc | 12 ++++++++++++ src/libexpr/eval-gc.hh | 7 +++++++ src/libexpr/eval.cc | 13 +++++++++++++ 3 files changed, 32 insertions(+) diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index baf9df3324a..c8e2adb944e 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -155,6 +155,10 @@ static inline void initGCReal() there. */ GC_set_no_dls(1); + /* Enable perf measurements. This is just a setting; not much of a + start of something. */ + GC_start_performance_measurement(); + GC_INIT(); GC_set_oom_fn(oomHandler); @@ -205,6 +209,7 @@ static inline void initGCReal() #endif static bool gcInitialised = false; +static GC_word gcCyclesAfterInit = 0; void initGC() { @@ -216,6 +221,7 @@ void initGC() #endif gcInitialised = true; + gcCyclesAfterInit = GC_get_gc_no(); } void assertGCInitialized() @@ -223,4 +229,10 @@ void assertGCInitialized() assert(gcInitialised); } +size_t getGCCycles() +{ + assertGCInitialized(); + return GC_get_gc_no() - gcCyclesAfterInit; } + +} // namespace nix diff --git a/src/libexpr/eval-gc.hh b/src/libexpr/eval-gc.hh index cd4ea914d9f..76f01972950 100644 --- a/src/libexpr/eval-gc.hh +++ b/src/libexpr/eval-gc.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + namespace nix { /** @@ -13,4 +15,9 @@ void initGC(); */ void assertGCInitialized(); +/** + * The number of GC cycles since initGC(). + */ +size_t getGCCycles(); + } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 4f01d0a62d3..11c3282ad01 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2610,6 +2610,11 @@ void EvalState::printStatistics() #if HAVE_BOEHMGC GC_word heapSize, totalBytes; GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes); + double gcFullOnlyTime = ({ + auto ms = GC_get_full_gc_total_time(); + ms * 0.001; + }); + auto gcCycles = getGCCycles(); #endif auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-"); @@ -2620,6 +2625,13 @@ void EvalState::printStatistics() #ifndef _WIN32 // TODO implement topObj["cpuTime"] = cpuTime; #endif + topObj["time"] = { + {"cpu", cpuTime}, +#ifdef HAVE_BOEHMGC + {GC_is_incremental_mode() ? "gcNonIncremental" : "gc", gcFullOnlyTime}, + {GC_is_incremental_mode() ? "gcNonIncrementalFraction" : "gcFraction", gcFullOnlyTime / cpuTime}, +#endif + }; topObj["envs"] = { {"number", nrEnvs}, {"elements", nrValuesInEnvs}, @@ -2661,6 +2673,7 @@ void EvalState::printStatistics() topObj["gc"] = { {"heapSize", heapSize}, {"totalBytes", totalBytes}, + {"cycles", gcCycles}, }; #endif From 380becf0dbbf700c32aaa2e574cc8e05b6411056 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 22 Jul 2024 14:48:18 +0200 Subject: [PATCH 062/284] Fix #11141 broken sp corrector --- src/libexpr/eval-gc.cc | 2 +- tests/functional/lang-gc.sh | 34 ++++++++++ .../lang-gc/issue-11141-gc-coroutine-test.nix | 65 +++++++++++++++++++ tests/functional/local.mk | 1 + 4 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 tests/functional/lang-gc.sh create mode 100644 tests/functional/lang-gc/issue-11141-gc-coroutine-test.nix diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index baf9df3324a..914d48da874 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -121,7 +121,7 @@ void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) // NOTE: We assume the stack grows down, as it does on all architectures we support. // Architectures that grow the stack up are rare. if (sp >= osStackBase || sp < osStackLow) { // lo is outside the os stack - sp = osStackBase; + sp = osStackLow; } } diff --git a/tests/functional/lang-gc.sh b/tests/functional/lang-gc.sh new file mode 100644 index 00000000000..8e2383854b2 --- /dev/null +++ b/tests/functional/lang-gc.sh @@ -0,0 +1,34 @@ +# shellcheck shell=bash + +# Regression tests for the evaluator +# These are not in lang.sh because they generally only need to run in CI, +# whereas lang.sh is often run locally during development + + +source common.sh + +set -o pipefail + +# Regression test for #11141. The stack pointer corrector assigned the base +# instead of the top (which resides at the low end of the stack). Sounds confusing? +# Stacks grow downwards, so that's why this mistake happened. +# My manual testing did not uncover this, because it didn't rely on the stack enough. +# https://github.com/NixOS/nix/issues/11141 +test_issue_11141() { + mkdir -p "$TEST_ROOT/issue-11141/src" + cp lang-gc/issue-11141-gc-coroutine-test.nix "$TEST_ROOT/issue-11141/" + ( + set +x; + n=10 + echo "populating $TEST_ROOT/issue-11141/src with $((n*100)) files..." + for i in $(seq 0 $n); do + touch "$TEST_ROOT/issue-11141/src/file-$i"{0,1,2,3,4,5,6,7,8,9}{0,1,2,3,4,5,6,7,8,9} + done + ) + + GC_INITIAL_HEAP_SIZE=$((1024 * 1024)) \ + NIX_SHOW_STATS=1 \ + nix eval -vvv\ + -f "$TEST_ROOT/issue-11141/issue-11141-gc-coroutine-test.nix" +} +test_issue_11141 diff --git a/tests/functional/lang-gc/issue-11141-gc-coroutine-test.nix b/tests/functional/lang-gc/issue-11141-gc-coroutine-test.nix new file mode 100644 index 00000000000..4f311af75d7 --- /dev/null +++ b/tests/functional/lang-gc/issue-11141-gc-coroutine-test.nix @@ -0,0 +1,65 @@ + +# Run: +# GC_INITIAL_HEAP_SIZE=$[1024 * 1024] NIX_SHOW_STATS=1 nix eval -f gc-coroutine-test.nix -vvvv + +let + inherit (builtins) + foldl' + isList + ; + + # Generate a tree of numbers, n deep, such that the numbers add up to (1 + salt) * 10^n. + # The salting makes the numbers all different, increasing the likelihood of catching + # any memory corruptions that might be caused by the GC or otherwise. + garbage = salt: n: + if n == 0 + then [(1 + salt)] + else [ + (garbage (10 * salt + 1) (n - 1)) + (garbage (10 * salt - 1) (n - 1)) + (garbage (10 * salt + 2) (n - 1)) + (garbage (10 * salt - 2) (n - 1)) + (garbage (10 * salt + 3) (n - 1)) + (garbage (10 * salt - 3) (n - 1)) + (garbage (10 * salt + 4) (n - 1)) + (garbage (10 * salt - 4) (n - 1)) + (garbage (10 * salt + 5) (n - 1)) + (garbage (10 * salt - 5) (n - 1)) + ]; + + pow = base: n: + if n == 0 + then 1 + else base * (pow base (n - 1)); + + sumNestedLists = l: + if isList l + then foldl' (a: b: a + sumNestedLists b) 0 l + else l; + +in + assert sumNestedLists (garbage 0 3) == pow 10 3; + assert sumNestedLists (garbage 0 6) == pow 10 6; + builtins.foldl' + (a: b: + assert + "${ + builtins.path { + path = ./src; + filter = path: type: + # We're not doing common subexpression elimination, so this reallocates + # the fairly big tree over and over, producing a lot of garbage during + # source filtering, whose filter runs in a coroutine. + assert sumNestedLists (garbage 0 3) == pow 10 3; + true; + } + }" + == "${./src}"; + + # These asserts don't seem necessary, as the lambda value get corrupted first + assert a.okay; + assert b.okay; + { okay = true; } + ) + { okay = true; } + [ { okay = true; } { okay = true; } { okay = true; } ] diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 49ee31284bc..797002e92a0 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -23,6 +23,7 @@ nix_tests = \ remote-store.sh \ legacy-ssh-store.sh \ lang.sh \ + lang-gc.sh \ characterisation-test-infra.sh \ experimental-features.sh \ fetchMercurial.sh \ From 112373c03cade928e2417a53623a2a1fd4773609 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 21 Jul 2024 12:16:47 -0400 Subject: [PATCH 063/284] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/88269ab3044128b7c2f4c7d68448b2fb50456870' (2024-06-03) → 'github:NixOS/nixpkgs/be3ca229c85e978880babdeda9748b14e6aa008f' (2024-07-21) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index f64e3ea3712..1d59439dac7 100644 --- a/flake.lock +++ b/flake.lock @@ -69,11 +69,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1717432640, - "narHash": "sha256-+f9c4/ZX5MWDOuB1rKoWj+lBNm0z0rs4CK47HBLxy1o=", + "lastModified": 1721560568, + "narHash": "sha256-L61BXz7n/yNzOeZ3FqlnUmxj4145JOVeq9fvQTQzbNM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "88269ab3044128b7c2f4c7d68448b2fb50456870", + "rev": "be3ca229c85e978880babdeda9748b14e6aa008f", "type": "github" }, "original": { From dc6dbbc1a5226fe6e25035bf5eb284df3c5c4607 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 21 Jul 2024 12:24:21 -0400 Subject: [PATCH 064/284] Test exes do not need dev outputs --- tests/unit/libexpr/package.nix | 2 -- tests/unit/libfetchers/package.nix | 2 -- tests/unit/libflake/package.nix | 2 -- tests/unit/libstore/package.nix | 2 -- tests/unit/libutil/package.nix | 2 -- 5 files changed, 10 deletions(-) diff --git a/tests/unit/libexpr/package.nix b/tests/unit/libexpr/package.nix index 6b7e12c4a0d..5f52170ba79 100644 --- a/tests/unit/libexpr/package.nix +++ b/tests/unit/libexpr/package.nix @@ -41,8 +41,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - outputs = [ "out" "dev" ]; - nativeBuildInputs = [ meson ninja diff --git a/tests/unit/libfetchers/package.nix b/tests/unit/libfetchers/package.nix index 9522f9639eb..88ccaca7f7c 100644 --- a/tests/unit/libfetchers/package.nix +++ b/tests/unit/libfetchers/package.nix @@ -40,8 +40,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - outputs = [ "out" "dev" ]; - nativeBuildInputs = [ meson ninja diff --git a/tests/unit/libflake/package.nix b/tests/unit/libflake/package.nix index 859bc49d02e..3c99efbba99 100644 --- a/tests/unit/libflake/package.nix +++ b/tests/unit/libflake/package.nix @@ -40,8 +40,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - outputs = [ "out" "dev" ]; - nativeBuildInputs = [ meson ninja diff --git a/tests/unit/libstore/package.nix b/tests/unit/libstore/package.nix index efffd00631b..e68cc3a62a6 100644 --- a/tests/unit/libstore/package.nix +++ b/tests/unit/libstore/package.nix @@ -42,8 +42,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - outputs = [ "out" "dev" ]; - nativeBuildInputs = [ meson ninja diff --git a/tests/unit/libutil/package.nix b/tests/unit/libutil/package.nix index ad5ff7d1e83..b635aa81101 100644 --- a/tests/unit/libutil/package.nix +++ b/tests/unit/libutil/package.nix @@ -40,8 +40,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - outputs = [ "out" "dev" ]; - nativeBuildInputs = [ meson ninja From eea63d5f993f0ce27ecee41d0c80f3e521f4f247 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 21 Jul 2024 12:57:49 -0400 Subject: [PATCH 065/284] Run unit tests with wine Fix #10547 The tests do not all run successfully, but that is a separate issue. --- tests/unit/libexpr/package.nix | 13 +++++++++---- tests/unit/libfetchers/package.nix | 13 +++++++++---- tests/unit/libflake/package.nix | 13 +++++++++---- tests/unit/libstore/package.nix | 14 ++++++++++---- tests/unit/libutil/package.nix | 13 +++++++++---- 5 files changed, 46 insertions(+), 20 deletions(-) diff --git a/tests/unit/libexpr/package.nix b/tests/unit/libexpr/package.nix index 5f52170ba79..015e3fbc6b4 100644 --- a/tests/unit/libexpr/package.nix +++ b/tests/unit/libexpr/package.nix @@ -1,4 +1,5 @@ { lib +, buildPackages , stdenv , mkMesonDerivation , releaseTools @@ -81,17 +82,21 @@ mkMesonDerivation (finalAttrs: { passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { - } '' - PATH="${lib.makeBinPath [ finalAttrs.finalPackage ]}:$PATH" + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' export _NIX_TEST_UNIT_DATA=${resolvePath ./data} - nix-expr-tests + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out - ''; + ''); }; }; meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; }; }) diff --git a/tests/unit/libfetchers/package.nix b/tests/unit/libfetchers/package.nix index 88ccaca7f7c..cf75f68e50f 100644 --- a/tests/unit/libfetchers/package.nix +++ b/tests/unit/libfetchers/package.nix @@ -1,4 +1,5 @@ { lib +, buildPackages , stdenv , mkMesonDerivation , releaseTools @@ -79,17 +80,21 @@ mkMesonDerivation (finalAttrs: { passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { - } '' - PATH="${lib.makeBinPath [ finalAttrs.finalPackage ]}:$PATH" + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' export _NIX_TEST_UNIT_DATA=${resolvePath ./data} - nix-fetchers-tests + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out - ''; + ''); }; }; meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; }; }) diff --git a/tests/unit/libflake/package.nix b/tests/unit/libflake/package.nix index 3c99efbba99..d2c9fdb89a4 100644 --- a/tests/unit/libflake/package.nix +++ b/tests/unit/libflake/package.nix @@ -1,4 +1,5 @@ { lib +, buildPackages , stdenv , mkMesonDerivation , releaseTools @@ -79,17 +80,21 @@ mkMesonDerivation (finalAttrs: { passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { - } '' - PATH="${lib.makeBinPath [ finalAttrs.finalPackage ]}:$PATH" + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' export _NIX_TEST_UNIT_DATA=${resolvePath ./data} - nix-flake-tests + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out - ''; + ''); }; }; meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; }; }) diff --git a/tests/unit/libstore/package.nix b/tests/unit/libstore/package.nix index e68cc3a62a6..39bf7758587 100644 --- a/tests/unit/libstore/package.nix +++ b/tests/unit/libstore/package.nix @@ -1,4 +1,5 @@ { lib +, buildPackages , stdenv , mkMesonDerivation , releaseTools @@ -92,17 +93,22 @@ mkMesonDerivation (finalAttrs: { ../../functional/derivation ]; }; - in runCommand "${finalAttrs.pname}-run" {} '' - PATH="${lib.makeBinPath [ finalAttrs.finalPackage ]}:$PATH" + in runCommand "${finalAttrs.pname}-run" { + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' export _NIX_TEST_UNIT_DATA=${data + "/unit/libstore/data"} - nix-store-tests + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out - ''; + ''); }; }; meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; }; }) diff --git a/tests/unit/libutil/package.nix b/tests/unit/libutil/package.nix index b635aa81101..c7827e74ff9 100644 --- a/tests/unit/libutil/package.nix +++ b/tests/unit/libutil/package.nix @@ -1,4 +1,5 @@ { lib +, buildPackages , stdenv , mkMesonDerivation , releaseTools @@ -80,17 +81,21 @@ mkMesonDerivation (finalAttrs: { passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { - } '' - PATH="${lib.makeBinPath [ finalAttrs.finalPackage ]}:$PATH" + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' export _NIX_TEST_UNIT_DATA=${./data} - nix-util-tests + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out - ''; + ''); }; }; meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; }; }) From 4878c3181575287b65c67aa0efa766b123cce5db Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Jul 2024 16:39:02 +0200 Subject: [PATCH 066/284] nix ping-store: Redirect to 'nix store info' This avoids the double warning warning: 'ping-store' is a deprecated alias for 'store ping' warning: 'nix store ping' is a deprecated alias for 'nix store info' --- src/nix/main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/main.cc b/src/nix/main.cc index 00ad6fe2c97..9d7d617ccda 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -164,7 +164,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs {"ls-store", { AliasStatus::Deprecated, {"store", "ls"}}}, {"make-content-addressable", { AliasStatus::Deprecated, {"store", "make-content-addressed"}}}, {"optimise-store", { AliasStatus::Deprecated, {"store", "optimise"}}}, - {"ping-store", { AliasStatus::Deprecated, {"store", "ping"}}}, + {"ping-store", { AliasStatus::Deprecated, {"store", "info"}}}, {"sign-paths", { AliasStatus::Deprecated, {"store", "sign"}}}, {"shell", { AliasStatus::AcceptedShorthand, {"env", "shell"}}}, {"show-derivation", { AliasStatus::Deprecated, {"derivation", "show"}}}, From 823baa25f35fd9047c715de675e2fef6e62d3583 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 22 Jul 2024 10:41:19 -0400 Subject: [PATCH 067/284] Meson build: libstore check for `statvfs` --- src/libstore/meson.build | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 29ee95b75bc..3b7f7a57a10 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -55,6 +55,7 @@ configdata.set('CAN_LINK_SYMLINK', can_link_symlink.to_int()) check_funcs = [ # Optionally used for canonicalising files from the build 'lchown', + 'statvfs', ] foreach funcspec : check_funcs define_name = 'HAVE_' + funcspec.underscorify().to_upper() From d7024ac9b75032f2cfaabc53c27dbd4584bfd6db Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 22 Jul 2024 10:40:21 -0400 Subject: [PATCH 068/284] Add S3 opt dep to Meson, and simplify build Numeric version macros are now defined upstream, so we don't need roll our own. --- configure.ac | 7 ------- src/libstore/meson.build | 17 +++++++++++++++++ src/libstore/s3-binary-cache-store.cc | 4 ++-- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/configure.ac b/configure.ac index 4f66a3efcf6..5c22ed17636 100644 --- a/configure.ac +++ b/configure.ac @@ -340,13 +340,6 @@ AC_CHECK_HEADERS([aws/s3/S3Client.h], AC_SUBST(ENABLE_S3, [$enable_s3]) AC_LANG_POP(C++) -if test -n "$enable_s3"; then - declare -a aws_version_tokens=($(printf '#include \nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' ')) - AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.]) - AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.]) - AC_DEFINE_UNQUOTED([AWS_VERSION_PATCH], ${aws_version_tokens@<:@2@:>@}, [Patch version of aws-sdk-cpp.]) -fi - # Whether to use the Boehm garbage collector. AC_ARG_ENABLE(gc, AS_HELP_STRING([--enable-gc],[enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]), diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 3b7f7a57a10..cb8110f3fa7 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -100,6 +100,23 @@ deps_public += nlohmann_json sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') deps_private += sqlite +# AWS C++ SDK has bad pkg-config +aws_s3 = dependency('aws-cpp-sdk-s3', required : false) +configdata.set('ENABLE_S3', aws_s3.found().to_int()) +if aws_s3.found() + aws_s3 = declare_dependency( + include_directories: include_directories(aws_s3.get_variable('includedir')), + link_args: [ + '-L' + aws_s3.get_variable('libdir'), + '-laws-cpp-sdk-transfer', + '-laws-cpp-sdk-s3', + '-laws-cpp-sdk-core', + '-laws-crt-cpp', + ], + ) +endif +deps_other += aws_s3 + subdir('build-utils-meson/generate-header') generated_headers = [] diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 1a0ec11112e..92ab47cd66d 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -60,7 +60,7 @@ class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem debug("AWS: %s", chomp(statement)); } -#if !(AWS_VERSION_MAJOR <= 1 && AWS_VERSION_MINOR <= 7 && AWS_VERSION_PATCH <= 115) +#if !(AWS_SDK_VERSION_MAJOR <= 1 && AWS_SDK_VERSION_MINOR <= 7 && AWS_SDK_VERSION_PATCH <= 115) void Flush() override {} #endif }; @@ -103,7 +103,7 @@ S3Helper::S3Helper( std::make_shared(profile.c_str())), *config, // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759 -#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3 +#if AWS_SDK_VERSION_MAJOR == 1 && AWS_SDK_VERSION_MINOR < 3 false, #else Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, From f2e0cecf34f6e3b23c2499bb65478a620c4340aa Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 22 Jul 2024 17:45:19 +0200 Subject: [PATCH 069/284] tests/functional/lang-gc: Disable for now --- tests/functional/lang-gc.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/functional/lang-gc.sh b/tests/functional/lang-gc.sh index 8e2383854b2..1746fa4c151 100644 --- a/tests/functional/lang-gc.sh +++ b/tests/functional/lang-gc.sh @@ -9,6 +9,8 @@ source common.sh set -o pipefail +skipTest "Too memory instensive for CI. Attempt to reduce memory usage was unsuccessful, because it made detection of the bug unreliable." + # Regression test for #11141. The stack pointer corrector assigned the base # instead of the top (which resides at the low end of the stack). Sounds confusing? # Stacks grow downwards, so that's why this mistake happened. From d3cee8160cecf4902f7f26b1dbfa0110a6a195bd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 22 Jul 2024 11:33:37 -0400 Subject: [PATCH 070/284] Add missing threads deps --- src/libcmd/meson.build | 2 ++ src/libexpr-c/meson.build | 2 ++ src/libfetchers/meson.build | 2 ++ src/libflake/meson.build | 2 ++ src/libmain-c/meson.build | 2 ++ src/libmain/meson.build | 1 + src/libstore-c/meson.build | 2 ++ src/libutil-c/meson.build | 2 ++ src/nix/meson.build | 2 ++ tests/unit/libexpr-support/meson.build | 2 ++ tests/unit/libexpr/meson.build | 2 ++ tests/unit/libfetchers/meson.build | 2 ++ tests/unit/libflake/meson.build | 2 ++ tests/unit/libstore-support/meson.build | 2 ++ tests/unit/libstore/meson.build | 2 ++ tests/unit/libutil-support/meson.build | 2 ++ tests/unit/libutil/meson.build | 2 ++ 17 files changed, 33 insertions(+) diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 687b37aacbd..c484cf998aa 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -30,6 +30,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index 2a2669b3e5e..6db5b83b89a 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -29,6 +29,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index c39fe99f316..d4f202796e4 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -26,6 +26,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json diff --git a/src/libflake/meson.build b/src/libflake/meson.build index 38d70c678a5..d2bb179dfd6 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -26,6 +26,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build index 1d6b2f959ff..3453827120c 100644 --- a/src/libmain-c/meson.build +++ b/src/libmain-c/meson.build @@ -29,6 +29,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) diff --git a/src/libmain/meson.build b/src/libmain/meson.build index fe6133596a9..7fcadf06ddc 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -26,6 +26,7 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') pubsetbuf_test = ''' #include diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build index 917de4cda51..4bfd944c63c 100644 --- a/src/libstore-c/meson.build +++ b/src/libstore-c/meson.build @@ -27,6 +27,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build index 3f0d9628201..b5ed19631d1 100644 --- a/src/libutil-c/meson.build +++ b/src/libutil-c/meson.build @@ -25,6 +25,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) diff --git a/src/nix/meson.build b/src/nix/meson.build index b42f6169bc1..53bb083a9c3 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -28,6 +28,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + subdir('build-utils-meson/export-all-symbols') add_project_arguments( diff --git a/tests/unit/libexpr-support/meson.build b/tests/unit/libexpr-support/meson.build index 7056722049e..4f50478aadb 100644 --- a/tests/unit/libexpr-support/meson.build +++ b/tests/unit/libexpr-support/meson.build @@ -27,6 +27,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + rapidcheck = dependency('rapidcheck') deps_public += rapidcheck diff --git a/tests/unit/libexpr/meson.build b/tests/unit/libexpr/meson.build index ee35258cf27..21c32133402 100644 --- a/tests/unit/libexpr/meson.build +++ b/tests/unit/libexpr/meson.build @@ -25,6 +25,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + subdir('build-utils-meson/export-all-symbols') rapidcheck = dependency('rapidcheck') diff --git a/tests/unit/libfetchers/meson.build b/tests/unit/libfetchers/meson.build index d2de9382933..dc9818e2744 100644 --- a/tests/unit/libfetchers/meson.build +++ b/tests/unit/libfetchers/meson.build @@ -24,6 +24,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + subdir('build-utils-meson/export-all-symbols') rapidcheck = dependency('rapidcheck') diff --git a/tests/unit/libflake/meson.build b/tests/unit/libflake/meson.build index 2d6bbca0f17..c022d7f4118 100644 --- a/tests/unit/libflake/meson.build +++ b/tests/unit/libflake/meson.build @@ -24,6 +24,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + subdir('build-utils-meson/export-all-symbols') rapidcheck = dependency('rapidcheck') diff --git a/tests/unit/libstore-support/meson.build b/tests/unit/libstore-support/meson.build index ddb067c1b52..f09d26a31dc 100644 --- a/tests/unit/libstore-support/meson.build +++ b/tests/unit/libstore-support/meson.build @@ -25,6 +25,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + rapidcheck = dependency('rapidcheck') deps_public += rapidcheck diff --git a/tests/unit/libstore/meson.build b/tests/unit/libstore/meson.build index 8534ba8c52d..3b36cd62f8c 100644 --- a/tests/unit/libstore/meson.build +++ b/tests/unit/libstore/meson.build @@ -25,6 +25,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + subdir('build-utils-meson/export-all-symbols') sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') diff --git a/tests/unit/libutil-support/meson.build b/tests/unit/libutil-support/meson.build index 7d0e9c2fc30..6be4972c6bd 100644 --- a/tests/unit/libutil-support/meson.build +++ b/tests/unit/libutil-support/meson.build @@ -23,6 +23,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + rapidcheck = dependency('rapidcheck') deps_public += rapidcheck diff --git a/tests/unit/libutil/meson.build b/tests/unit/libutil/meson.build index 4f055cabda0..7f024e6f25b 100644 --- a/tests/unit/libutil/meson.build +++ b/tests/unit/libutil/meson.build @@ -25,6 +25,8 @@ deps_public_maybe_subproject = [ ] subdir('build-utils-meson/subprojects') +subdir('build-utils-meson/threads') + subdir('build-utils-meson/export-all-symbols') rapidcheck = dependency('rapidcheck') From d39bbcabb99415db916e8d9abf8e82eccc77e0e0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 22 Jul 2024 11:44:53 -0400 Subject: [PATCH 071/284] Fix some BSD builds missing pthread functions In addition to adding the missing thread deps in the last commit, we also appear to need to skip `-Wl,--as-needed` flags that Meson wants to use, but doesn't work with our *BSD toolchains. See https://github.com/mesonbuild/meson/issues/3593 --- packaging/dependencies.nix | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 73ba9cd586f..f09ca5d1835 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -61,6 +61,12 @@ let workDir = null; }; + # Work around weird `--as-needed` linker behavior with BSD, see + # https://github.com/mesonbuild/meson/issues/3593 + bsdNoLinkAsNeeded = finalAttrs: prevAttrs: lib.optionalAttrs stdenv.hostPlatform.isBSD { + mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or []; + }; + in scope: { inherit stdenv versionSuffix; @@ -130,5 +136,8 @@ scope: { inherit resolvePath filesetToSource; - mkMesonDerivation = f: stdenv.mkDerivation (lib.extends localSourceLayer f); + mkMesonDerivation = f: stdenv.mkDerivation + (lib.extends + (lib.composeExtensions bsdNoLinkAsNeeded localSourceLayer) + f); } From 4457cebe05c444facc68db281c58f90a87f785ce Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Jul 2024 10:24:18 +0200 Subject: [PATCH 072/284] Update comment in tests//vars-and-functions.sh Co-authored-by: tomberek --- tests/functional/common/vars-and-functions.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/vars-and-functions.sh index 7a399f6d494..954435593d5 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/vars-and-functions.sh @@ -306,7 +306,8 @@ onError() { # Example (showns as string): 'repl.sh:123: in call to grepQuiet: ' # This function is inefficient, so it should only be used in error messages. callerPrefix() { - # Find the closes caller that's not from this file + # Find the closest caller that's not from this file + # using the bash `caller` builtin. local i file line fn savedFn # Use `caller` for i in $(seq 0 100); do From 498eed0a25ee1707677d56e0ea3628584991453b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Jul 2024 10:56:29 +0200 Subject: [PATCH 073/284] ~FramedSource(): Don't throw an exception if the remote has disconnected This would cause the daemon to crash with a call to terminate(). --- src/libutil/serialise.hh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 8254a5f899d..8137db5f40e 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -483,13 +483,17 @@ struct FramedSource : Source ~FramedSource() { - if (!eof) { - while (true) { - auto n = readInt(from); - if (!n) break; - std::vector data(n); - from(data.data(), n); + try { + if (!eof) { + while (true) { + auto n = readInt(from); + if (!n) break; + std::vector data(n); + from(data.data(), n); + } } + } catch (...) { + ignoreException(); } } From eb89e50cbb4584bb42a425089640b2931b723cca Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Jul 2024 16:21:43 +0200 Subject: [PATCH 074/284] Rejiggle getGCCycles() for buildNoGc --- src/libexpr/eval-gc.cc | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index 97b24e3fb97..73ab809e5bd 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -206,10 +206,17 @@ static inline void initGCReal() } } +static size_t gcCyclesAfterInit = 0; + +size_t getGCCycles() +{ + assertGCInitialized(); + return static_cast(GC_get_gc_no()) - gcCyclesAfterInit; +} + #endif static bool gcInitialised = false; -static GC_word gcCyclesAfterInit = 0; void initGC() { @@ -218,10 +225,11 @@ void initGC() #if HAVE_BOEHMGC initGCReal(); + + gcCyclesAfterInit = GC_get_gc_no(); #endif gcInitialised = true; - gcCyclesAfterInit = GC_get_gc_no(); } void assertGCInitialized() @@ -229,10 +237,4 @@ void assertGCInitialized() assert(gcInitialised); } -size_t getGCCycles() -{ - assertGCInitialized(); - return GC_get_gc_no() - gcCyclesAfterInit; -} - } // namespace nix From 5d6bc484beab6ff822815b5b26745710d8c86305 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Jul 2024 16:24:43 +0200 Subject: [PATCH 075/284] Hide getGCCycles when we have no GC Alternatively, we could make it return 0, but we don't need it in the first place because the caller exists conditionally too. --- src/libexpr/eval-gc.hh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval-gc.hh b/src/libexpr/eval-gc.hh index 76f01972950..005175eb78d 100644 --- a/src/libexpr/eval-gc.hh +++ b/src/libexpr/eval-gc.hh @@ -15,9 +15,11 @@ void initGC(); */ void assertGCInitialized(); +#ifdef HAVE_BOEHMGC /** * The number of GC cycles since initGC(). */ size_t getGCCycles(); +#endif -} +} // namespace nix From 2b4e3f04a433d96e779fb91fcd9fa01329413d02 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Jul 2024 16:57:45 +0200 Subject: [PATCH 076/284] markdown.cc: Format Slightly custom because the automated formatting messes up the braced initializer with named fields. --- src/libcmd/markdown.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 88c3f640bad..882f95faa11 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -4,8 +4,8 @@ #include "terminal.hh" #if HAVE_LOWDOWN -# include -# include +# include +# include #endif namespace nix { @@ -15,7 +15,8 @@ std::string renderMarkdownToTerminal(std::string_view markdown) #if HAVE_LOWDOWN int windowWidth = getWindowSize().second; - struct lowdown_opts opts { + struct lowdown_opts opts + { .type = LOWDOWN_TERM, .maxdepth = 20, .cols = (size_t) std::max(windowWidth - 5, 60), From 97b0114ab84144e3574c74c7fb0e0e38681f3b6a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Jul 2024 16:54:04 +0200 Subject: [PATCH 077/284] renderMarkdownToTerminal: Add _NIX_TEST_RAW_MARKDOWN env var For testing only. --- src/libcmd/markdown.cc | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 882f95faa11..6a0d05d9fba 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,5 +1,6 @@ #include "markdown.hh" -#include "util.hh" +#include "environment-variables.hh" +#include "error.hh" #include "finally.hh" #include "terminal.hh" @@ -10,9 +11,9 @@ namespace nix { -std::string renderMarkdownToTerminal(std::string_view markdown) -{ #if HAVE_LOWDOWN +static std::string doRenderMarkdownToTerminal(std::string_view markdown) +{ int windowWidth = getWindowSize().second; struct lowdown_opts opts @@ -52,9 +53,21 @@ std::string renderMarkdownToTerminal(std::string_view markdown) throw Error("allocation error while rendering Markdown"); return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY()); +} + +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + if (auto e = getEnv("_NIX_TEST_RAW_MARKDOWN"); e && *e == "1") + return std::string(markdown); + else + return doRenderMarkdownToTerminal(markdown); +} + #else +std::string renderMarkdownToTerminal(std::string_view markdown) +{ return std::string(markdown); -#endif } +#endif -} +} // namespace nix From 712ce2feacdc0cde7249ae6da5f8063858e80ae2 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Jul 2024 17:50:51 +0200 Subject: [PATCH 078/284] ReadlineLikeInteracter::getLine: Add _NIX_TEST_REPL_ECHO env var ... for testing --- src/libcmd/repl-interacter.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index b285c8a9ae5..a12e7d8c799 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -19,6 +19,7 @@ extern "C" { #include "repl-interacter.hh" #include "file-system.hh" #include "repl.hh" +#include "environment-variables.hh" namespace nix { @@ -179,6 +180,19 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT return false; input += s; input += '\n'; + +#ifndef USE_READLINE + // editline doesn't echo the input to the output when non-interactive, unlike readline + // this results in a different behavior when running tests. The echoing is + // quite useful for reading the test output, so we add it here. + if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") + { + // This is probably not right for multi-line input, but we don't use that + // in the characterisation tests, so it's fine. + std::cout << "nix-repl> " << s << std::endl; + } +#endif + return true; } From ca2cc26e12d7264fb141206e0bc35ade2378b8f8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Jul 2024 17:52:28 +0200 Subject: [PATCH 079/284] tests/functional/repl: Improve precision and readability ... as well as match buildReadlineNoMarkdown. Unfortunately it doesn't support long inputs or multiline inputs for now. This needs to make better use of the interacter interface. --- src/libcmd/repl-interacter.cc | 16 ++--- tests/functional/repl.sh | 11 ++++ .../repl/doc-comment-curried-args.expected | 26 ++++---- .../repl/doc-comment-curried-args.in | 2 +- .../repl/doc-comment-formals.expected | 13 ++-- .../repl/doc-comment-function.expected | 7 ++- tests/functional/repl/doc-compact.expected | 12 ++-- tests/functional/repl/doc-constant.expected | 60 +++++++++++-------- tests/functional/repl/doc-floatedIn.expected | 12 ++-- .../repl/doc-lambda-flavors.expected | 36 +++++------ .../functional/repl/doc-measurement.expected | 12 ++-- tests/functional/repl/doc-multiply.expected | 18 +++--- .../functional/repl/doc-unambiguous.expected | 12 ++-- .../repl/pretty-print-idempotent.expected | 12 +++- 14 files changed, 152 insertions(+), 97 deletions(-) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index a12e7d8c799..76fe38780b6 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -176,22 +176,22 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT return true; } - if (!s) - return false; - input += s; - input += '\n'; - -#ifndef USE_READLINE // editline doesn't echo the input to the output when non-interactive, unlike readline // this results in a different behavior when running tests. The echoing is // quite useful for reading the test output, so we add it here. if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") { +#ifndef USE_READLINE // This is probably not right for multi-line input, but we don't use that // in the characterisation tests, so it's fine. - std::cout << "nix-repl> " << s << std::endl; - } + std::cout << promptForType(promptType) << s << std::endl; #endif + } + + if (!s) + return false; + input += s; + input += '\n'; return true; } diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 4f5bb36aae9..a6c075abd84 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -262,6 +262,14 @@ badExitCode=0 nixVersion="$(nix eval --impure --raw --expr 'builtins.nixVersion' --extra-experimental-features nix-command)" +# I couldn't get readline and editline to agree on the newline before the prompt, +# so let's just force it to be one empty line. Ideally we get the two to agree +# or use a simpler interacter for testing. +stripEmptyLinesBeforePrompt() { + # --null-data: treat input as NUL-terminated instead of newline-terminated + sed --null-data 's/\n\n*nix-repl>/\n\nnix-repl>/g' +} + runRepl () { # That is right, we are also filtering out the testdir _without underscores_. @@ -273,8 +281,11 @@ runRepl () { testDirNoUnderscores="${testDir//_/}" # TODO: pass arguments to nix repl; see lang.sh + _NIX_TEST_RAW_MARKDOWN=1 \ + _NIX_TEST_REPL_ECHO=1 \ nix repl 2>&1 \ | stripColors \ + | stripEmptyLinesBeforePrompt \ | sed \ -e "s@$testDir@/path/to/tests/functional@g" \ -e "s@$testDirNoUnderscores@/path/to/tests/functional@g" \ diff --git a/tests/functional/repl/doc-comment-curried-args.expected b/tests/functional/repl/doc-comment-curried-args.expected index c10c171e1e8..f97f593c667 100644 --- a/tests/functional/repl/doc-comment-curried-args.expected +++ b/tests/functional/repl/doc-comment-curried-args.expected @@ -1,24 +1,30 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function curriedArgs - … defined at - /path/to/tests/functional/repl/doc-comments.nix:48:5 +nix-repl> :doc curriedArgs +Function `curriedArgs`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:48:5 - A documented function. +A documented function. +nix-repl> x = curriedArgs 1 +nix-repl> "Note that users may not expect this to behave as it currently does" "Note that users may not expect this to behave as it currently does" -Function curriedArgs - … defined at - /path/to/tests/functional/repl/doc-comments.nix:50:5 +nix-repl> :doc x +Function `curriedArgs`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:50:5 - The function returned by applying once +The function returned by applying once -"This won't produce documentation, because we can't actually document arbitrary values" +nix-repl> "This won't produce docs; no support for arbitrary values" +"This won't produce docs; no support for arbitrary values" +nix-repl> :doc x 2 error: value does not have documentation - +nix-repl> diff --git a/tests/functional/repl/doc-comment-curried-args.in b/tests/functional/repl/doc-comment-curried-args.in index 8dbbfc37052..06ba21dcca9 100644 --- a/tests/functional/repl/doc-comment-curried-args.in +++ b/tests/functional/repl/doc-comment-curried-args.in @@ -3,5 +3,5 @@ x = curriedArgs 1 "Note that users may not expect this to behave as it currently does" :doc x -"This won't produce documentation, because we can't actually document arbitrary values" +"This won't produce docs; no support for arbitrary values" :doc x 2 diff --git a/tests/functional/repl/doc-comment-formals.expected b/tests/functional/repl/doc-comment-formals.expected index 704c0050bd8..4c9757f89ef 100644 --- a/tests/functional/repl/doc-comment-formals.expected +++ b/tests/functional/repl/doc-comment-formals.expected @@ -1,13 +1,16 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. +nix-repl> "Note that this is not yet complete" "Note that this is not yet complete" -Function documentedFormals - … defined at - /path/to/tests/functional/repl/doc-comments.nix:57:5 - - Finds x +nix-repl> :doc documentedFormals +Function `documentedFormals`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:57:5 +Finds x +nix-repl> diff --git a/tests/functional/repl/doc-comment-function.expected b/tests/functional/repl/doc-comment-function.expected index 5ec465a96c9..9718ccc66a8 100644 --- a/tests/functional/repl/doc-comment-function.expected +++ b/tests/functional/repl/doc-comment-function.expected @@ -1,8 +1,9 @@ Nix Type :? for help. -Function defined at - /path/to/tests/functional/repl/doc-comment-function.nix:2:1 - A doc comment for a file that only contains a function +nix-repl> :doc import ./doc-comment-function.nix +Function defined at /path/to/tests/functional/repl/doc-comment-function.nix:2:1 +A doc comment for a file that only contains a function +nix-repl> diff --git a/tests/functional/repl/doc-compact.expected b/tests/functional/repl/doc-compact.expected index 4b05b653c45..17603d81369 100644 --- a/tests/functional/repl/doc-compact.expected +++ b/tests/functional/repl/doc-compact.expected @@ -1,11 +1,13 @@ Nix Type :? for help. -Added variables. -Function compact - … defined at - /path/to/tests/functional/repl/doc-comments.nix:18:20 +nix-repl> :l doc-comments.nix +Added variables. - boom +nix-repl> :doc compact +Function `compact`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:18:20 +boom +nix-repl> diff --git a/tests/functional/repl/doc-constant.expected b/tests/functional/repl/doc-constant.expected index c6655833376..e0092453e41 100644 --- a/tests/functional/repl/doc-constant.expected +++ b/tests/functional/repl/doc-constant.expected @@ -1,23 +1,27 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. +nix-repl> :doc constant error: value does not have documentation -Attribute version +nix-repl> :doc lib.version +Attribute `version` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:30:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:30:3 - Immovably fixed. +Immovably fixed. -Attribute empty +nix-repl> :doc lib.attr.empty +Attribute `empty` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:33:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:33:3 - Unchangeably constant. +Unchangeably constant. +nix-repl> :doc lib.attr.undocument error: … while evaluating the attribute 'attr.undocument' at /path/to/tests/functional/repl/doc-comments.nix:33:3: @@ -32,59 +36,65 @@ error: | ^ Did you mean undocumented? -Attribute constant +nix-repl> :doc (import ./doc-comments.nix).constant +Attribute `constant` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:27:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:27:3 - Firmly rigid. +Firmly rigid. -Attribute version +nix-repl> :doc (import ./doc-comments.nix).lib.version +Attribute `version` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:30:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:30:3 - Immovably fixed. +Immovably fixed. -Attribute empty +nix-repl> :doc (import ./doc-comments.nix).lib.attr.empty +Attribute `empty` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:33:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:33:3 - Unchangeably constant. +Unchangeably constant. -Attribute undocumented +nix-repl> :doc (import ./doc-comments.nix).lib.attr.undocumented +Attribute `undocumented` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:35:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:35:3 - No documentation found. +No documentation found. +nix-repl> :doc missing error: undefined variable 'missing' at «string»:1:1: 1| missing | ^ +nix-repl> :doc constanz error: undefined variable 'constanz' at «string»:1:1: 1| constanz | ^ +nix-repl> :doc missing.attr error: undefined variable 'missing' at «string»:1:1: 1| missing.attr | ^ +nix-repl> :doc lib.missing error: attribute 'missing' missing at «string»:1:1: 1| lib.missing | ^ +nix-repl> :doc lib.missing.attr error: attribute 'missing' missing at «string»:1:1: 1| lib.missing.attr | ^ +nix-repl> :doc lib.attr.undocumental error: … while evaluating the attribute 'attr.undocumental' at /path/to/tests/functional/repl/doc-comments.nix:33:3: @@ -99,4 +109,4 @@ error: | ^ Did you mean undocumented? - +nix-repl> diff --git a/tests/functional/repl/doc-floatedIn.expected b/tests/functional/repl/doc-floatedIn.expected index 30f13572586..d3f1c3f65a5 100644 --- a/tests/functional/repl/doc-floatedIn.expected +++ b/tests/functional/repl/doc-floatedIn.expected @@ -1,11 +1,13 @@ Nix Type :? for help. -Added variables. -Function floatedIn - … defined at - /path/to/tests/functional/repl/doc-comments.nix:16:5 +nix-repl> :l doc-comments.nix +Added variables. - This also works. +nix-repl> :doc floatedIn +Function `floatedIn`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:16:5 +This also works. +nix-repl> diff --git a/tests/functional/repl/doc-lambda-flavors.expected b/tests/functional/repl/doc-lambda-flavors.expected index 43f483ce943..268cac05e9e 100644 --- a/tests/functional/repl/doc-lambda-flavors.expected +++ b/tests/functional/repl/doc-lambda-flavors.expected @@ -1,29 +1,31 @@ Nix Type :? for help. -Added variables. -Function nonStrict - … defined at - /path/to/tests/functional/repl/doc-comments.nix:37:70 +nix-repl> :l doc-comments.nix +Added variables. - My syntax is not strict, but I'm strict anyway. +nix-repl> :doc nonStrict +Function `nonStrict`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:37:70 -Function strict - … defined at - /path/to/tests/functional/repl/doc-comments.nix:38:63 +My syntax is not strict, but I'm strict anyway. - I don't have to be strict, but I am anyway. +nix-repl> :doc strict +Function `strict`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:38:63 -Function strictPre - … defined at - /path/to/tests/functional/repl/doc-comments.nix:40:48 +I don't have to be strict, but I am anyway. - Here's one way to do this +nix-repl> :doc strictPre +Function `strictPre`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:40:48 -Function strictPost - … defined at - /path/to/tests/functional/repl/doc-comments.nix:41:53 +Here's one way to do this - Here's another way to do this +nix-repl> :doc strictPost +Function `strictPost`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:41:53 +Here's another way to do this +nix-repl> diff --git a/tests/functional/repl/doc-measurement.expected b/tests/functional/repl/doc-measurement.expected index 8598aaedbc7..14ae11e800f 100644 --- a/tests/functional/repl/doc-measurement.expected +++ b/tests/functional/repl/doc-measurement.expected @@ -1,11 +1,13 @@ Nix Type :? for help. -Added variables. -Function measurement - … defined at - /path/to/tests/functional/repl/doc-comments.nix:13:17 +nix-repl> :l doc-comments.nix +Added variables. - 👈 precisely this wide 👉 +nix-repl> :doc measurement +Function `measurement`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:13:17 +👈 precisely this wide 👉 +nix-repl> diff --git a/tests/functional/repl/doc-multiply.expected b/tests/functional/repl/doc-multiply.expected index db82af91fc6..5cefda51650 100644 --- a/tests/functional/repl/doc-multiply.expected +++ b/tests/functional/repl/doc-multiply.expected @@ -1,15 +1,19 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function multiply - … defined at - /path/to/tests/functional/repl/doc-comments.nix:10:14 +nix-repl> :doc multiply +Function `multiply`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:10:14 - Perform arithmetic multiplication. It's kind of like - repeated addition, very neat. - | multiply 2 3 - | => 6 +Perform *arithmetic* multiplication. It's kind of like repeated **addition**, very neat. +```nix +multiply 2 3 +=> 6 +``` +nix-repl> diff --git a/tests/functional/repl/doc-unambiguous.expected b/tests/functional/repl/doc-unambiguous.expected index 825aa1ee170..0872750a16a 100644 --- a/tests/functional/repl/doc-unambiguous.expected +++ b/tests/functional/repl/doc-unambiguous.expected @@ -1,11 +1,13 @@ Nix Type :? for help. -Added variables. -Function unambiguous - … defined at - /path/to/tests/functional/repl/doc-comments.nix:24:5 +nix-repl> :l doc-comments.nix +Added variables. - Very close +nix-repl> :doc unambiguous +Function `unambiguous`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:24:5 +Very close +nix-repl> diff --git a/tests/functional/repl/pretty-print-idempotent.expected b/tests/functional/repl/pretty-print-idempotent.expected index f38b9b56969..f09aec88d53 100644 --- a/tests/functional/repl/pretty-print-idempotent.expected +++ b/tests/functional/repl/pretty-print-idempotent.expected @@ -1,29 +1,39 @@ Nix Type :? for help. + +nix-repl> :l pretty-print-idempotent.nix Added variables. +nix-repl> oneDeep { homepage = "https://example.com"; } +nix-repl> oneDeep { homepage = "https://example.com"; } +nix-repl> twoDeep { layerOne = { ... }; } +nix-repl> twoDeep { layerOne = { ... }; } +nix-repl> oneDeepList [ "https://example.com" ] +nix-repl> oneDeepList [ "https://example.com" ] +nix-repl> twoDeepList [ [ ... ] ] +nix-repl> twoDeepList [ [ ... ] ] - +nix-repl> From c4ae9bb45b72c852ecff8bf5d5153951116e2039 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 01:01:00 +0200 Subject: [PATCH 080/284] tests/functional/repl: Normalize final prompt --- tests/functional/repl.sh | 15 +++++++++++++-- .../repl/doc-comment-curried-args.expected | 2 -- .../functional/repl/doc-comment-formals.expected | 2 -- .../functional/repl/doc-comment-function.expected | 2 -- tests/functional/repl/doc-compact.expected | 2 -- tests/functional/repl/doc-constant.expected | 2 -- tests/functional/repl/doc-floatedIn.expected | 2 -- tests/functional/repl/doc-lambda-flavors.expected | 2 -- tests/functional/repl/doc-measurement.expected | 2 -- tests/functional/repl/doc-multiply.expected | 2 -- tests/functional/repl/doc-unambiguous.expected | 2 -- .../repl/pretty-print-idempotent.expected | 2 -- 12 files changed, 13 insertions(+), 24 deletions(-) diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index a6c075abd84..a149916ac9c 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -262,14 +262,23 @@ badExitCode=0 nixVersion="$(nix eval --impure --raw --expr 'builtins.nixVersion' --extra-experimental-features nix-command)" +# TODO: write a repl interacter for testing. Papering over the differences between readline / editline and between platforms is a pain. + # I couldn't get readline and editline to agree on the newline before the prompt, -# so let's just force it to be one empty line. Ideally we get the two to agree -# or use a simpler interacter for testing. +# so let's just force it to be one empty line. stripEmptyLinesBeforePrompt() { # --null-data: treat input as NUL-terminated instead of newline-terminated sed --null-data 's/\n\n*nix-repl>/\n\nnix-repl>/g' } +# We don't get a final prompt on darwin, so we strip this as well. +stripFinalPrompt() { + # Strip the final prompt and/or any trailing spaces + sed --null-data \ + -e 's/\(.*[^\n]\)\n\n*nix-repl>[ \n]*$/\1/' \ + -e 's/[ \n]*$/\n/' +} + runRepl () { # That is right, we are also filtering out the testdir _without underscores_. @@ -285,7 +294,9 @@ runRepl () { _NIX_TEST_REPL_ECHO=1 \ nix repl 2>&1 \ | stripColors \ + | tr -d '\0' \ | stripEmptyLinesBeforePrompt \ + | stripFinalPrompt \ | sed \ -e "s@$testDir@/path/to/tests/functional@g" \ -e "s@$testDirNoUnderscores@/path/to/tests/functional@g" \ diff --git a/tests/functional/repl/doc-comment-curried-args.expected b/tests/functional/repl/doc-comment-curried-args.expected index f97f593c667..56607e911e8 100644 --- a/tests/functional/repl/doc-comment-curried-args.expected +++ b/tests/functional/repl/doc-comment-curried-args.expected @@ -26,5 +26,3 @@ nix-repl> "This won't produce docs; no support for arbitrary values" nix-repl> :doc x 2 error: value does not have documentation - -nix-repl> diff --git a/tests/functional/repl/doc-comment-formals.expected b/tests/functional/repl/doc-comment-formals.expected index 4c9757f89ef..1024919f4b9 100644 --- a/tests/functional/repl/doc-comment-formals.expected +++ b/tests/functional/repl/doc-comment-formals.expected @@ -12,5 +12,3 @@ Function `documentedFormals`\ … defined at /path/to/tests/functional/repl/doc-comments.nix:57:5 Finds x - -nix-repl> diff --git a/tests/functional/repl/doc-comment-function.expected b/tests/functional/repl/doc-comment-function.expected index 9718ccc66a8..3889c4f7860 100644 --- a/tests/functional/repl/doc-comment-function.expected +++ b/tests/functional/repl/doc-comment-function.expected @@ -5,5 +5,3 @@ nix-repl> :doc import ./doc-comment-function.nix Function defined at /path/to/tests/functional/repl/doc-comment-function.nix:2:1 A doc comment for a file that only contains a function - -nix-repl> diff --git a/tests/functional/repl/doc-compact.expected b/tests/functional/repl/doc-compact.expected index 17603d81369..79f1fd44f59 100644 --- a/tests/functional/repl/doc-compact.expected +++ b/tests/functional/repl/doc-compact.expected @@ -9,5 +9,3 @@ Function `compact`\ … defined at /path/to/tests/functional/repl/doc-comments.nix:18:20 boom - -nix-repl> diff --git a/tests/functional/repl/doc-constant.expected b/tests/functional/repl/doc-constant.expected index e0092453e41..5787e04dc19 100644 --- a/tests/functional/repl/doc-constant.expected +++ b/tests/functional/repl/doc-constant.expected @@ -108,5 +108,3 @@ error: 1| lib.attr.undocumental | ^ Did you mean undocumented? - -nix-repl> diff --git a/tests/functional/repl/doc-floatedIn.expected b/tests/functional/repl/doc-floatedIn.expected index d3f1c3f65a5..82bb80b9501 100644 --- a/tests/functional/repl/doc-floatedIn.expected +++ b/tests/functional/repl/doc-floatedIn.expected @@ -9,5 +9,3 @@ Function `floatedIn`\ … defined at /path/to/tests/functional/repl/doc-comments.nix:16:5 This also works. - -nix-repl> diff --git a/tests/functional/repl/doc-lambda-flavors.expected b/tests/functional/repl/doc-lambda-flavors.expected index 268cac05e9e..ab5c956390f 100644 --- a/tests/functional/repl/doc-lambda-flavors.expected +++ b/tests/functional/repl/doc-lambda-flavors.expected @@ -27,5 +27,3 @@ Function `strictPost`\ … defined at /path/to/tests/functional/repl/doc-comments.nix:41:53 Here's another way to do this - -nix-repl> diff --git a/tests/functional/repl/doc-measurement.expected b/tests/functional/repl/doc-measurement.expected index 14ae11e800f..555cac9a2a0 100644 --- a/tests/functional/repl/doc-measurement.expected +++ b/tests/functional/repl/doc-measurement.expected @@ -9,5 +9,3 @@ Function `measurement`\ … defined at /path/to/tests/functional/repl/doc-comments.nix:13:17 👈 precisely this wide 👉 - -nix-repl> diff --git a/tests/functional/repl/doc-multiply.expected b/tests/functional/repl/doc-multiply.expected index 5cefda51650..21523e24c81 100644 --- a/tests/functional/repl/doc-multiply.expected +++ b/tests/functional/repl/doc-multiply.expected @@ -15,5 +15,3 @@ Perform *arithmetic* multiplication. It's kind of like repeated **addition**, ve multiply 2 3 => 6 ``` - -nix-repl> diff --git a/tests/functional/repl/doc-unambiguous.expected b/tests/functional/repl/doc-unambiguous.expected index 0872750a16a..0db5505d781 100644 --- a/tests/functional/repl/doc-unambiguous.expected +++ b/tests/functional/repl/doc-unambiguous.expected @@ -9,5 +9,3 @@ Function `unambiguous`\ … defined at /path/to/tests/functional/repl/doc-comments.nix:24:5 Very close - -nix-repl> diff --git a/tests/functional/repl/pretty-print-idempotent.expected b/tests/functional/repl/pretty-print-idempotent.expected index f09aec88d53..311855dae36 100644 --- a/tests/functional/repl/pretty-print-idempotent.expected +++ b/tests/functional/repl/pretty-print-idempotent.expected @@ -35,5 +35,3 @@ nix-repl> twoDeepList [ [ ... ] ] - -nix-repl> From 6e680a664421927cd7d99f956a1e8c4b5bdc7db6 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 01:01:14 +0200 Subject: [PATCH 081/284] tests/functional/repl: Improve failure reporting --- tests/functional/repl.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index a149916ac9c..706e0f5dba9 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -311,7 +311,10 @@ for test in $(cd "$testDir/repl"; echo *.in); do in="$testDir/repl/$test.in" actual="$testDir/repl/$test.actual" expected="$testDir/repl/$test.expected" - (cd "$testDir/repl"; set +x; runRepl 2>&1) < "$in" > "$actual" + (cd "$testDir/repl"; set +x; runRepl 2>&1) < "$in" > "$actual" || { + echo "FAIL: $test (exit code $?)" >&2 + badExitCode=1 + } diffAndAcceptInner "$test" "$actual" "$expected" done From 7d4d34a27d8d95e8a63d3b3940bd0ee7b623ff82 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 01:02:21 +0200 Subject: [PATCH 082/284] eval-gc.cc: Fix warning --- src/libexpr/eval-gc.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index 73ab809e5bd..1bc8cd28f50 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -84,7 +84,9 @@ void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) { void *& sp = *sp_ptr; auto pthread_id = reinterpret_cast(_pthread_id); +# ifndef __APPLE__ pthread_attr_t pattr; +# endif size_t osStackSize; void * osStackLow; void * osStackBase; From 0bd2d363750db35926460cbc4c3e62dcb1d09bf5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 12:53:37 +0200 Subject: [PATCH 083/284] Document renderMarkdownToTerminal --- src/libcmd/markdown.hh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index a04d32a4fe4..164f99b9881 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -5,6 +5,13 @@ namespace nix { +/** + * Render the given Markdown text to the terminal. + * + * If Nix is compiled without Markdown support, this function will return the input text as-is. + * + * The renderer takes into account the terminal width, and wraps text accordingly. + */ std::string renderMarkdownToTerminal(std::string_view markdown); } From e48e0cbab0eed9b9174c27d3f3ddfa1afff5186b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 12:54:40 +0200 Subject: [PATCH 084/284] markdown.hh: Improve includes --- src/libcmd/markdown.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index 164f99b9881..66db1736c65 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "types.hh" +#include namespace nix { From 907b0a371abe92b3314612faea0b3d68e048c156 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Jul 2024 16:02:40 +0200 Subject: [PATCH 085/284] Rename osStackLow -> osStackLimit This is in accordance with ARM's naming convention. "Low" is confusing, because it could refer to either the cold end of the stack as an abstract data type, or a low address. These are different places, because the stack grows down through the address space. --- src/libexpr/eval-gc.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index 73ab809e5bd..75368c5ac6a 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -86,12 +86,13 @@ void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) auto pthread_id = reinterpret_cast(_pthread_id); pthread_attr_t pattr; size_t osStackSize; - void * osStackLow; + // The low address of the stack, which grows down. + void * osStackLimit; void * osStackBase; # ifdef __APPLE__ osStackSize = pthread_get_stacksize_np(pthread_id); - osStackLow = pthread_get_stackaddr_np(pthread_id); + osStackLimit = pthread_get_stackaddr_np(pthread_id); # else if (pthread_attr_init(&pattr)) { throw Error("fixupBoehmStackPointer: pthread_attr_init failed"); @@ -110,18 +111,18 @@ void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) # else # error "Need one of `pthread_attr_get_np` or `pthread_getattr_np`" # endif - if (pthread_attr_getstack(&pattr, &osStackLow, &osStackSize)) { + if (pthread_attr_getstack(&pattr, &osStackLimit, &osStackSize)) { throw Error("fixupBoehmStackPointer: pthread_attr_getstack failed"); } if (pthread_attr_destroy(&pattr)) { throw Error("fixupBoehmStackPointer: pthread_attr_destroy failed"); } # endif - osStackBase = (char *) osStackLow + osStackSize; + osStackBase = (char *) osStackLimit + osStackSize; // NOTE: We assume the stack grows down, as it does on all architectures we support. // Architectures that grow the stack up are rare. - if (sp >= osStackBase || sp < osStackLow) { // lo is outside the os stack - sp = osStackLow; + if (sp >= osStackBase || sp < osStackLimit) { // lo is outside the os stack + sp = osStackLimit; } } From 68693276f9c5829489132c1f5a142717da84f44a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Jul 2024 16:27:26 +0200 Subject: [PATCH 086/284] Update fixupBoehmStackPointer doc lo might have made sense in the bdwgc code, maybe?, but not here. --- src/libexpr/eval-gc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index 75368c5ac6a..ba19cd74ede 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -121,7 +121,7 @@ void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) osStackBase = (char *) osStackLimit + osStackSize; // NOTE: We assume the stack grows down, as it does on all architectures we support. // Architectures that grow the stack up are rare. - if (sp >= osStackBase || sp < osStackLimit) { // lo is outside the os stack + if (sp >= osStackBase || sp < osStackLimit) { // sp is outside the os stack sp = osStackLimit; } } From a16df88252a34160fabe7c2710ba11ca51d9fee1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 15:18:50 +0200 Subject: [PATCH 087/284] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-compat': 'github:edolstra/flake-compat/35bb57c0c8d8b62bbfd284272c928ceb64ddbde9?narHash=sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm%2B504Ch3sNKLd8%3D' (2023-01-17) → 'github:edolstra/flake-compat/0f9255e01c2351cc7d116c072cb317785dd33b33?narHash=sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U%3D' (2023-10-04) • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/9126214d0a59633752a136528f5f3b9aa8565b7d?narHash=sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm%2BGpZNw%3D' (2024-04-01) → 'github:hercules-ci/flake-parts/9227223f6d922fee3c7b190b2cc238a99527bbb7?narHash=sha256-pQMhCCHyQGRzdfAkdJ4cIWiw%2BJNuWsTX7f0ZYSyz0VY%3D' (2024-07-03) • Updated input 'libgit2': 'github:libgit2/libgit2/45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5?narHash=sha256-oX4Z3S9WtJlwvj0uH9HlYcWv%2Bx1hqp8mhXl7HsLu2f0%3D' (2023-10-18) → 'github:libgit2/libgit2/503b66cf00ad7dca940148529f60b1a409ccc462?narHash=sha256-tDUQi%2Bs8sxJ30SmUH7Ln9WmDz5jGatlgKumjwi7KnCo%3D' (2024-07-17) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/be3ca229c85e978880babdeda9748b14e6aa008f?narHash=sha256-L61BXz7n/yNzOeZ3FqlnUmxj4145JOVeq9fvQTQzbNM%3D' (2024-07-21) → 'github:NixOS/nixpkgs/d0907b75146a0ccc1ec0d6c3db287ec287588ef6?narHash=sha256-PhmkdTJs2SfqKzSyDB74rDKp1MH4mGk0pG/%2BWqrnGEw%3D' (2024-07-24) • Updated input 'pre-commit-hooks': 'github:cachix/pre-commit-hooks.nix/40e6053ecb65fcbf12863338a6dcefb3f55f1bf8?narHash=sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y%3D' (2024-04-12) → 'github:cachix/pre-commit-hooks.nix/f451c19376071a90d8c58ab1a953c6e9840527fd?narHash=sha256-6FPUl7HVtvRHCCBQne7Ylp4p%2BdpP3P/OYuzjztZ4s70%3D' (2024-07-15) • Removed input 'pre-commit-hooks/flake-utils' --- flake.lock | 46 +++++++++++++++------------------------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/flake.lock b/flake.lock index 1d59439dac7..932c5fcf51d 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1712014858, - "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=", + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", "type": "github" }, "original": { @@ -36,29 +36,14 @@ "type": "github" } }, - "flake-utils": { - "locked": { - "lastModified": 1667395993, - "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, "libgit2": { "flake": false, "locked": { - "lastModified": 1697646580, - "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "lastModified": 1721209236, + "narHash": "sha256-tDUQi+s8sxJ30SmUH7Ln9WmDz5jGatlgKumjwi7KnCo=", "owner": "libgit2", "repo": "libgit2", - "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "rev": "503b66cf00ad7dca940148529f60b1a409ccc462", "type": "github" }, "original": { @@ -69,11 +54,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1721560568, - "narHash": "sha256-L61BXz7n/yNzOeZ3FqlnUmxj4145JOVeq9fvQTQzbNM=", + "lastModified": 1721821769, + "narHash": "sha256-PhmkdTJs2SfqKzSyDB74rDKp1MH4mGk0pG/+WqrnGEw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "be3ca229c85e978880babdeda9748b14e6aa008f", + "rev": "d0907b75146a0ccc1ec0d6c3db287ec287588ef6", "type": "github" }, "original": { @@ -118,7 +103,6 @@ "pre-commit-hooks": { "inputs": { "flake-compat": [], - "flake-utils": "flake-utils", "gitignore": [], "nixpkgs": [ "nixpkgs" @@ -128,11 +112,11 @@ ] }, "locked": { - "lastModified": 1712897695, - "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=", + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", "owner": "cachix", "repo": "pre-commit-hooks.nix", - "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", "type": "github" }, "original": { From 650f1894020c586360f30fa4e728fe9eea76d114 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 15:24:10 +0200 Subject: [PATCH 088/284] flake.nix: Pin libgit2 to a release --- flake.lock | 7 ++++--- flake.nix | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 932c5fcf51d..96c35b088d3 100644 --- a/flake.lock +++ b/flake.lock @@ -39,15 +39,16 @@ "libgit2": { "flake": false, "locked": { - "lastModified": 1721209236, - "narHash": "sha256-tDUQi+s8sxJ30SmUH7Ln9WmDz5jGatlgKumjwi7KnCo=", + "lastModified": 1715853528, + "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=", "owner": "libgit2", "repo": "libgit2", - "rev": "503b66cf00ad7dca940148529f60b1a409ccc462", + "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96", "type": "github" }, "original": { "owner": "libgit2", + "ref": "v1.8.1", "repo": "libgit2", "type": "github" } diff --git a/flake.nix b/flake.nix index ff2c8ecfaed..45c493c6f2d 100644 --- a/flake.nix +++ b/flake.nix @@ -7,7 +7,7 @@ inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; - inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; + inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; }; # dev tooling inputs.flake-parts.url = "github:hercules-ci/flake-parts"; From f9a23c8d2180e644afe930c07680eba9e360d9c5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 15:30:28 +0200 Subject: [PATCH 089/284] flake.nix: Remove outdated comment --- flake.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/flake.nix b/flake.nix index 45c493c6f2d..bfa4c409aa9 100644 --- a/flake.nix +++ b/flake.nix @@ -1,8 +1,6 @@ { description = "The purely functional package manager"; - # TODO switch to nixos-23.11-small - # https://nixpk.gs/pr-tracker.html?pr=291954 inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-24.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; From c316f1557d9c71a0a6864fa6774a913a52748023 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 15:30:56 +0200 Subject: [PATCH 090/284] flake: Switch to nixos-24.05 channel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/d0907b75146a0ccc1ec0d6c3db287ec287588ef6?narHash=sha256-PhmkdTJs2SfqKzSyDB74rDKp1MH4mGk0pG/%2BWqrnGEw%3D' (2024-07-24) → 'github:NixOS/nixpkgs/63d37ccd2d178d54e7fb691d7ec76000740ea24a?narHash=sha256-7cCC8%2BTdq1%2B3OPyc3%2BgVo9dzUNkNIQfwSDJ2HSi2u3o%3D' (2024-07-21) --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 96c35b088d3..d1b54eb878e 100644 --- a/flake.lock +++ b/flake.lock @@ -55,16 +55,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1721821769, - "narHash": "sha256-PhmkdTJs2SfqKzSyDB74rDKp1MH4mGk0pG/+WqrnGEw=", + "lastModified": 1721548954, + "narHash": "sha256-7cCC8+Tdq1+3OPyc3+gVo9dzUNkNIQfwSDJ2HSi2u3o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d0907b75146a0ccc1ec0d6c3db287ec287588ef6", + "rev": "63d37ccd2d178d54e7fb691d7ec76000740ea24a", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-24.05", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index bfa4c409aa9..e6af8772352 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-24.05"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; From 171ef75218ec1bc0cdfe7b1b49796452d1d6bea6 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 15:55:57 +0200 Subject: [PATCH 091/284] Rename pre-commit-hooks -> git-hooks-nix Following the upstream rename --- flake.lock | 54 ++++++++++++++++++------------------ flake.nix | 10 +++---- maintainers/flake-module.nix | 2 +- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/flake.lock b/flake.lock index d1b54eb878e..2ac413a6986 100644 --- a/flake.lock +++ b/flake.lock @@ -36,6 +36,31 @@ "type": "github" } }, + "git-hooks-nix": { + "inputs": { + "flake-compat": [], + "gitignore": [], + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-stable": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, "libgit2": { "flake": false, "locked": { @@ -101,40 +126,15 @@ "type": "github" } }, - "pre-commit-hooks": { - "inputs": { - "flake-compat": [], - "gitignore": [], - "nixpkgs": [ - "nixpkgs" - ], - "nixpkgs-stable": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1721042469, - "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "type": "github" - } - }, "root": { "inputs": { "flake-compat": "flake-compat", "flake-parts": "flake-parts", + "git-hooks-nix": "git-hooks-nix", "libgit2": "libgit2", "nixpkgs": "nixpkgs", "nixpkgs-23-11": "nixpkgs-23-11", - "nixpkgs-regression": "nixpkgs-regression", - "pre-commit-hooks": "pre-commit-hooks" + "nixpkgs-regression": "nixpkgs-regression" } } }, diff --git a/flake.nix b/flake.nix index e6af8772352..2384c29741b 100644 --- a/flake.nix +++ b/flake.nix @@ -9,14 +9,14 @@ # dev tooling inputs.flake-parts.url = "github:hercules-ci/flake-parts"; - inputs.pre-commit-hooks.url = "github:cachix/pre-commit-hooks.nix"; + inputs.git-hooks-nix.url = "github:cachix/git-hooks.nix"; # work around https://github.com/NixOS/nix/issues/7730 inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; - inputs.pre-commit-hooks.inputs.nixpkgs.follows = "nixpkgs"; - inputs.pre-commit-hooks.inputs.nixpkgs-stable.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; # work around 7730 and https://github.com/NixOS/nix/issues/7807 - inputs.pre-commit-hooks.inputs.flake-compat.follows = ""; - inputs.pre-commit-hooks.inputs.gitignore.follows = ""; + inputs.git-hooks-nix.inputs.flake-compat.follows = ""; + inputs.git-hooks-nix.inputs.gitignore.follows = ""; outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }: diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index b5c7bfd530b..be91df536ad 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -2,7 +2,7 @@ { imports = [ - inputs.pre-commit-hooks.flakeModule + inputs.git-hooks-nix.flakeModule ]; perSystem = { config, pkgs, ... }: { From 3be7c0037eb4728e201b49572230d563cd2ac096 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Jul 2024 15:48:19 +0200 Subject: [PATCH 092/284] WorkerProto: Support fine-grained protocol feature negotiation Currently, the worker protocol has a version number that we increment whenever we change something in the protocol. However, this can cause a collision between Nix PRs / forks that make protocol changes (e.g. PR #9857 increments the version, which could collide with another PR). So instead, the client and daemon now exchange a set of protocol features (such as `auth-forwarding`). They will use the intersection of the sets of features, i.e. the features they both support. Note that protocol features are completely distinct from `ExperimentalFeature`s. --- src/libstore/daemon.cc | 11 ++--- src/libstore/remote-store.cc | 10 ++++- src/libstore/worker-protocol-connection.cc | 51 +++++++++++++++++++--- src/libstore/worker-protocol-connection.hh | 27 ++++++++++-- src/libstore/worker-protocol.hh | 8 +++- tests/unit/libstore/worker-protocol.cc | 49 ++++++++++++++++----- 6 files changed, 127 insertions(+), 29 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 6533b2f58c4..94f00cfb6ba 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1025,19 +1025,20 @@ void processConnection( #endif /* Exchange the greeting. */ - WorkerProto::Version clientVersion = + auto [protoVersion, features] = WorkerProto::BasicServerConnection::handshake( - to, from, PROTOCOL_VERSION); + to, from, PROTOCOL_VERSION, WorkerProto::allFeatures); - if (clientVersion < 0x10a) + if (protoVersion < 0x10a) throw Error("the Nix client version is too old"); WorkerProto::BasicServerConnection conn; conn.to = std::move(to); conn.from = std::move(from); - conn.protoVersion = clientVersion; + conn.protoVersion = protoVersion; + conn.features = features; - auto tunnelLogger = new TunnelLogger(conn.to, clientVersion); + auto tunnelLogger = new TunnelLogger(conn.to, protoVersion); auto prevLogger = nix::logger; // FIXME if (!recursive) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ebb0864c555..555936c186f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -73,8 +73,11 @@ void RemoteStore::initConnection(Connection & conn) StringSink saved; TeeSource tee(conn.from, saved); try { - conn.protoVersion = WorkerProto::BasicClientConnection::handshake( - conn.to, tee, PROTOCOL_VERSION); + auto [protoVersion, features] = WorkerProto::BasicClientConnection::handshake( + conn.to, tee, PROTOCOL_VERSION, + WorkerProto::allFeatures); + conn.protoVersion = protoVersion; + conn.features = features; } catch (SerialisationError & e) { /* In case the other side is waiting for our input, close it. */ @@ -88,6 +91,9 @@ void RemoteStore::initConnection(Connection & conn) static_cast(conn) = conn.postHandshake(*this); + for (auto & feature : conn.features) + debug("negotiated feature '%s'", feature); + auto ex = conn.processStderrReturn(); if (ex) std::rethrow_exception(ex); } diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc index 93d13d48e55..a47dbb689d8 100644 --- a/src/libstore/worker-protocol-connection.cc +++ b/src/libstore/worker-protocol-connection.cc @@ -5,6 +5,8 @@ namespace nix { +const std::set WorkerProto::allFeatures{}; + WorkerProto::BasicClientConnection::~BasicClientConnection() { try { @@ -137,8 +139,21 @@ void WorkerProto::BasicClientConnection::processStderr(bool * daemonException, S } } -WorkerProto::Version -WorkerProto::BasicClientConnection::handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion) +static std::set +intersectFeatures(const std::set & a, const std::set & b) +{ + std::set res; + for (auto & x : a) + if (b.contains(x)) + res.insert(x); + return res; +} + +std::tuple> WorkerProto::BasicClientConnection::handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures) { to << WORKER_MAGIC_1 << localVersion; to.flush(); @@ -153,11 +168,24 @@ WorkerProto::BasicClientConnection::handshake(BufferedSink & to, Source & from, if (GET_PROTOCOL_MINOR(daemonVersion) < 10) throw Error("the Nix daemon version is too old"); - return std::min(daemonVersion, localVersion); + auto protoVersion = std::min(daemonVersion, localVersion); + + /* Exchange features. */ + std::set daemonFeatures; + if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { + to << supportedFeatures; + to.flush(); + daemonFeatures = readStrings>(from); + } + + return {protoVersion, intersectFeatures(daemonFeatures, supportedFeatures)}; } -WorkerProto::Version -WorkerProto::BasicServerConnection::handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion) +std::tuple> WorkerProto::BasicServerConnection::handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures) { unsigned int magic = readInt(from); if (magic != WORKER_MAGIC_1) @@ -165,7 +193,18 @@ WorkerProto::BasicServerConnection::handshake(BufferedSink & to, Source & from, to << WORKER_MAGIC_2 << localVersion; to.flush(); auto clientVersion = readInt(from); - return std::min(clientVersion, localVersion); + + auto protoVersion = std::min(clientVersion, localVersion); + + /* Exchange features. */ + std::set clientFeatures; + if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { + clientFeatures = readStrings>(from); + to << supportedFeatures; + to.flush(); + } + + return {protoVersion, intersectFeatures(clientFeatures, supportedFeatures)}; } WorkerProto::ClientHandshakeInfo WorkerProto::BasicClientConnection::postHandshake(const StoreDirConfig & store) diff --git a/src/libstore/worker-protocol-connection.hh b/src/libstore/worker-protocol-connection.hh index 38287d08e31..9c96195b5f4 100644 --- a/src/libstore/worker-protocol-connection.hh +++ b/src/libstore/worker-protocol-connection.hh @@ -23,6 +23,11 @@ struct WorkerProto::BasicConnection */ WorkerProto::Version protoVersion; + /** + * The set of features that both sides support. + */ + std::set features; + /** * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the * factored out serve protocol serializers with a @@ -72,8 +77,8 @@ struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection /** * Establishes connection, negotiating version. * - * @return the version provided by the other side of the - * connection. + * @return the minimum version supported by both sides and the set + * of protocol features supported by both sides. * * @param to Taken by reference to allow for various error handling * mechanisms. @@ -82,8 +87,15 @@ struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection * handling mechanisms. * * @param localVersion Our version which is sent over + * + * @param features The protocol features that we support */ - static Version handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion); + // FIXME: this should probably be a constructor. + static std::tuple> handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures); /** * After calling handshake, must call this to exchange some basic @@ -138,8 +150,15 @@ struct WorkerProto::BasicServerConnection : WorkerProto::BasicConnection * handling mechanisms. * * @param localVersion Our version which is sent over + * + * @param features The protocol features that we support */ - static WorkerProto::Version handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion); + // FIXME: this should probably be a constructor. + static std::tuple> handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures); /** * After calling handshake, must call this to exchange some basic diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 9fc16d0153e..c356fa1bf37 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -11,7 +11,9 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 37) +/* Note: you generally shouldn't change the protocol version. Define a + new `WorkerProto::Feature` instead. */ +#define PROTOCOL_VERSION (1 << 8 | 38) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -131,6 +133,10 @@ struct WorkerProto { WorkerProto::Serialise::write(store, conn, t); } + + using Feature = std::string; + + static const std::set allFeatures; }; enum struct WorkerProto::Op : uint64_t diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc index c1512001093..bbea9ed7513 100644 --- a/tests/unit/libstore/worker-protocol.cc +++ b/tests/unit/libstore/worker-protocol.cc @@ -658,15 +658,15 @@ TEST_F(WorkerProtoTest, handshake_log) FdSink out { toServer.writeSide.get() }; FdSource in0 { toClient.readSide.get() }; TeeSource in { in0, toClientLog }; - clientResult = WorkerProto::BasicClientConnection::handshake( - out, in, defaultVersion); + clientResult = std::get<0>(WorkerProto::BasicClientConnection::handshake( + out, in, defaultVersion, {})); }); { FdSink out { toClient.writeSide.get() }; FdSource in { toServer.readSide.get() }; WorkerProto::BasicServerConnection::handshake( - out, in, defaultVersion); + out, in, defaultVersion, {}); }; thread.join(); @@ -675,6 +675,33 @@ TEST_F(WorkerProtoTest, handshake_log) }); } +TEST_F(WorkerProtoTest, handshake_features) +{ + Pipe toClient, toServer; + toClient.create(); + toServer.create(); + + std::tuple> clientResult; + + auto clientThread = std::thread([&]() { + FdSink out { toServer.writeSide.get() }; + FdSource in { toClient.readSide.get() }; + clientResult = WorkerProto::BasicClientConnection::handshake( + out, in, 123, {"bar", "aap", "mies", "xyzzy"}); + }); + + FdSink out { toClient.writeSide.get() }; + FdSource in { toServer.readSide.get() }; + auto daemonResult = WorkerProto::BasicServerConnection::handshake( + out, in, 456, {"foo", "bar", "xyzzy"}); + + clientThread.join(); + + EXPECT_EQ(clientResult, daemonResult); + EXPECT_EQ(std::get<0>(clientResult), 123); + EXPECT_EQ(std::get<1>(clientResult), std::set({"bar", "xyzzy"})); +} + /// Has to be a `BufferedSink` for handshake. struct NullBufferedSink : BufferedSink { void writeUnbuffered(std::string_view data) override { } @@ -686,8 +713,8 @@ TEST_F(WorkerProtoTest, handshake_client_replay) NullBufferedSink nullSink; StringSource in { toClientLog }; - auto clientResult = WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion); + auto clientResult = std::get<0>(WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {})); EXPECT_EQ(clientResult, defaultVersion); }); @@ -705,13 +732,13 @@ TEST_F(WorkerProtoTest, handshake_client_truncated_replay_throws) if (len < 8) { EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), EndOfFile); } else { // Not sure why cannot keep on checking for `EndOfFile`. EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), Error); } } @@ -734,17 +761,17 @@ TEST_F(WorkerProtoTest, handshake_client_corrupted_throws) // magic bytes don't match EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), Error); } else if (idx < 8 || idx >= 12) { // Number out of bounds EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), SerialisationError); } else { - auto ver = WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion); + auto ver = std::get<0>(WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {})); // `std::min` of this and the other version saves us EXPECT_EQ(ver, defaultVersion); } From 0194f815870ed875b791d60335be920de249a510 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Jul 2024 15:10:47 +0200 Subject: [PATCH 093/284] addToStore(): Do evaluation on the main stack This hopefully avoids the need for all our Boehm GC coroutine workarounds, since the GC roots will be on the main stack of the thread. Fixes #11141. --- src/libstore/local-store.cc | 1 + src/libstore/store-api.cc | 16 +++++++++------- src/libutil/serialise.cc | 8 ++++---- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 819cee34532..956a4453d4a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1245,6 +1245,7 @@ StorePath LocalStore::addToStoreFromDump( }); try { got = source.read(dumpBuffer.get() + oldSize, want); + if (!got) break; } catch (EndOfFile &) { inMemory = true; break; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2c4dee518b4..5d6109bae3e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -210,14 +210,16 @@ StorePath Store::addToStore( fsm = FileSerialisationMethod::NixArchive; break; } - auto source = sinkToSource([&](Sink & sink) { - dumpPath(path, sink, fsm, filter); + std::optional storePath; + auto sink = sourceToSink([&](Source & source) { + LengthSource lengthSource(source); + storePath = addToStoreFromDump(lengthSource, name, fsm, method, hashAlgo, references, repair); + if (lengthSource.total >= settings.warnLargePathThreshold) + warn("copied large path '%s' to the store (%s)", path, renderSize(lengthSource.total)); }); - LengthSource lengthSource(*source); - auto storePath = addToStoreFromDump(lengthSource, name, fsm, method, hashAlgo, references, repair); - if (lengthSource.total >= settings.warnLargePathThreshold) - warn("copied large path '%s' to the store (%s)", path, renderSize(lengthSource.total)); - return storePath; + dumpPath(path, *sink, fsm, filter); + sink->finish(); + return storePath.value(); } void Store::addMultipleToStore( diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 36b99905aab..0f2febaed67 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -243,11 +243,11 @@ std::unique_ptr sourceToSink(std::function fun) if (!coro) { CoroutineContext ctx; coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) { - LambdaSource source([&](char *out, size_t out_len) { + LambdaSource source([&](char * out, size_t out_len) { if (cur.empty()) { yield(); if (yield.get()) { - return (size_t)0; + return (size_t) 0; } } @@ -271,12 +271,12 @@ std::unique_ptr sourceToSink(std::function fun) void finish() override { if (!coro) return; - if (!*coro) abort(); + //if (!*coro) abort(); { CoroutineContext ctx; (*coro)(true); } - if (*coro) abort(); + //if (*coro) abort(); } }; From 609df83c011ad74c20cd806e00d59f6ed4a580b5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Jul 2024 15:18:58 +0200 Subject: [PATCH 094/284] sourceToSink(): Throw EndOfFile --- src/libstore/local-store.cc | 1 - src/libutil/serialise.cc | 9 ++++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 956a4453d4a..819cee34532 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1245,7 +1245,6 @@ StorePath LocalStore::addToStoreFromDump( }); try { got = source.read(dumpBuffer.get() + oldSize, want); - if (!got) break; } catch (EndOfFile &) { inMemory = true; break; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 0f2febaed67..d203a01bf46 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -246,9 +246,8 @@ std::unique_ptr sourceToSink(std::function fun) LambdaSource source([&](char * out, size_t out_len) { if (cur.empty()) { yield(); - if (yield.get()) { - return (size_t) 0; - } + if (yield.get()) + throw EndOfFile("coroutine has finished"); } size_t n = std::min(cur.size(), out_len); @@ -271,12 +270,12 @@ std::unique_ptr sourceToSink(std::function fun) void finish() override { if (!coro) return; - //if (!*coro) abort(); + if (!*coro) abort(); { CoroutineContext ctx; (*coro)(true); } - //if (*coro) abort(); + if (*coro) abort(); } }; From ca0f7db843d2c58bb5ca7b45f2b2aa57273e5f7e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Jul 2024 15:28:21 +0200 Subject: [PATCH 095/284] Remove hacks to support evaluation from coroutines Since we're not doing this anymore. --- .../src/installation/prerequisites-source.md | 2 - src/libexpr/eval-gc.cc | 132 ------------------ src/libutil/serialise.cc | 58 +------- src/libutil/serialise.hh | 23 --- 4 files changed, 2 insertions(+), 213 deletions(-) diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index 4aafa6d27ab..c346a0a4b27 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -39,8 +39,6 @@ `pkgconfig` and the Boehm garbage collector, and pass the flag `--enable-gc` to `configure`. - For `bdw-gc` <= 8.2.4 Nix needs a [small patch](https://github.com/NixOS/nix/blob/ac4d2e7b857acdfeac35ac8a592bdecee2d29838/boehmgc-traceable_allocator-public.diff) to be applied. - - The `boost` library of version 1.66.0 or higher. It can be obtained from the official web site . diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index ba19cd74ede..4bf48194e12 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -30,120 +30,6 @@ static void * oomHandler(size_t requested) throw std::bad_alloc(); } -class BoehmGCStackAllocator : public StackAllocator -{ - boost::coroutines2::protected_fixedsize_stack stack{ - // We allocate 8 MB, the default max stack size on NixOS. - // A smaller stack might be quicker to allocate but reduces the stack - // depth available for source filter expressions etc. - std::max(boost::context::stack_traits::default_size(), static_cast(8 * 1024 * 1024))}; - - // This is specific to boost::coroutines2::protected_fixedsize_stack. - // The stack protection page is included in sctx.size, so we have to - // subtract one page size from the stack size. - std::size_t pfss_usable_stack_size(boost::context::stack_context & sctx) - { - return sctx.size - boost::context::stack_traits::page_size(); - } - -public: - boost::context::stack_context allocate() override - { - auto sctx = stack.allocate(); - - // Stacks generally start at a high address and grow to lower addresses. - // Architectures that do the opposite are rare; in fact so rare that - // boost_routine does not implement it. - // So we subtract the stack size. - GC_add_roots(static_cast(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); - return sctx; - } - - void deallocate(boost::context::stack_context sctx) override - { - GC_remove_roots(static_cast(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); - stack.deallocate(sctx); - } -}; - -static BoehmGCStackAllocator boehmGCStackAllocator; - -/** - * When a thread goes into a coroutine, we lose its original sp until - * control flow returns to the thread. - * While in the coroutine, the sp points outside the thread stack, - * so we can detect this and push the entire thread stack instead, - * as an approximation. - * The coroutine's stack is covered by `BoehmGCStackAllocator`. - * This is not an optimal solution, because the garbage is scanned when a - * coroutine is active, for both the coroutine and the original thread stack. - * However, the implementation is quite lean, and usually we don't have active - * coroutines during evaluation, so this is acceptable. - */ -void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) -{ - void *& sp = *sp_ptr; - auto pthread_id = reinterpret_cast(_pthread_id); - pthread_attr_t pattr; - size_t osStackSize; - // The low address of the stack, which grows down. - void * osStackLimit; - void * osStackBase; - -# ifdef __APPLE__ - osStackSize = pthread_get_stacksize_np(pthread_id); - osStackLimit = pthread_get_stackaddr_np(pthread_id); -# else - if (pthread_attr_init(&pattr)) { - throw Error("fixupBoehmStackPointer: pthread_attr_init failed"); - } -# ifdef HAVE_PTHREAD_GETATTR_NP - if (pthread_getattr_np(pthread_id, &pattr)) { - throw Error("fixupBoehmStackPointer: pthread_getattr_np failed"); - } -# elif HAVE_PTHREAD_ATTR_GET_NP - if (!pthread_attr_init(&pattr)) { - throw Error("fixupBoehmStackPointer: pthread_attr_init failed"); - } - if (!pthread_attr_get_np(pthread_id, &pattr)) { - throw Error("fixupBoehmStackPointer: pthread_attr_get_np failed"); - } -# else -# error "Need one of `pthread_attr_get_np` or `pthread_getattr_np`" -# endif - if (pthread_attr_getstack(&pattr, &osStackLimit, &osStackSize)) { - throw Error("fixupBoehmStackPointer: pthread_attr_getstack failed"); - } - if (pthread_attr_destroy(&pattr)) { - throw Error("fixupBoehmStackPointer: pthread_attr_destroy failed"); - } -# endif - osStackBase = (char *) osStackLimit + osStackSize; - // NOTE: We assume the stack grows down, as it does on all architectures we support. - // Architectures that grow the stack up are rare. - if (sp >= osStackBase || sp < osStackLimit) { // sp is outside the os stack - sp = osStackLimit; - } -} - -/* Disable GC while this object lives. Used by CoroutineContext. - * - * Boehm keeps a count of GC_disable() and GC_enable() calls, - * and only enables GC when the count matches. - */ -class BoehmDisableGC -{ -public: - BoehmDisableGC() - { - GC_disable(); - }; - ~BoehmDisableGC() - { - GC_enable(); - }; -}; - static inline void initGCReal() { /* Initialise the Boehm garbage collector. */ @@ -164,24 +50,6 @@ static inline void initGCReal() GC_set_oom_fn(oomHandler); - StackAllocator::defaultAllocator = &boehmGCStackAllocator; - -// TODO: Remove __APPLE__ condition. -// Comment suggests an implementation that works on darwin and windows -// https://github.com/ivmai/bdwgc/issues/362#issuecomment-1936672196 -# if GC_VERSION_MAJOR >= 8 && GC_VERSION_MINOR >= 2 && GC_VERSION_MICRO >= 4 && !defined(__APPLE__) - GC_set_sp_corrector(&fixupBoehmStackPointer); - - if (!GC_get_sp_corrector()) { - printTalkative("BoehmGC on this platform does not support sp_corrector; will disable GC inside coroutines"); - /* Used to disable GC when entering coroutines on macOS */ - create_coro_gc_hook = []() -> std::shared_ptr { return std::make_shared(); }; - } -# else -# warning \ - "BoehmGC version does not support GC while coroutine exists. GC will be disabled inside coroutines. Consider updating bdw-gc to 8.2.4 or later." -# endif - /* Set the initial heap size to something fairly big (25% of physical RAM, up to a maximum of 384 MiB) so that in most cases we don't need to garbage collect at all. (Collection has a diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index d203a01bf46..cd52fbfb472 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -171,55 +171,6 @@ size_t StringSource::read(char * data, size_t len) #error Coroutines are broken in this version of Boost! #endif -/* A concrete datatype allow virtual dispatch of stack allocation methods. */ -struct VirtualStackAllocator { - StackAllocator *allocator = StackAllocator::defaultAllocator; - - boost::context::stack_context allocate() { - return allocator->allocate(); - } - - void deallocate(boost::context::stack_context sctx) { - allocator->deallocate(sctx); - } -}; - - -/* This class reifies the default boost coroutine stack allocation strategy with - a virtual interface. */ -class DefaultStackAllocator : public StackAllocator { - boost::coroutines2::default_stack stack; - - boost::context::stack_context allocate() { - return stack.allocate(); - } - - void deallocate(boost::context::stack_context sctx) { - stack.deallocate(sctx); - } -}; - -static DefaultStackAllocator defaultAllocatorSingleton; - -StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton; - - -std::shared_ptr (*create_coro_gc_hook)() = []() -> std::shared_ptr { - return {}; -}; - -/* This class is used for entry and exit hooks on coroutines */ -class CoroutineContext { - /* Disable GC when entering the coroutine without the boehm patch, - * since it doesn't find the main thread stack in this case. - * std::shared_ptr performs type-erasure, so it will call the right - * deleter. */ - const std::shared_ptr coro_gc_hook = create_coro_gc_hook(); -public: - CoroutineContext() {}; - ~CoroutineContext() {}; -}; - std::unique_ptr sourceToSink(std::function fun) { struct SourceToSink : FinishSink @@ -241,8 +192,7 @@ std::unique_ptr sourceToSink(std::function fun) cur = in; if (!coro) { - CoroutineContext ctx; - coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) { + coro = coro_t::push_type([&](coro_t::pull_type & yield) { LambdaSource source([&](char * out, size_t out_len) { if (cur.empty()) { yield(); @@ -262,7 +212,6 @@ std::unique_ptr sourceToSink(std::function fun) if (!*coro) { abort(); } if (!cur.empty()) { - CoroutineContext ctx; (*coro)(false); } } @@ -272,7 +221,6 @@ std::unique_ptr sourceToSink(std::function fun) if (!coro) return; if (!*coro) abort(); { - CoroutineContext ctx; (*coro)(true); } if (*coro) abort(); @@ -306,8 +254,7 @@ std::unique_ptr sinkToSource( size_t read(char * data, size_t len) override { if (!coro) { - CoroutineContext ctx; - coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { + coro = coro_t::pull_type([&](coro_t::push_type & yield) { LambdaSink sink([&](std::string_view data) { if (!data.empty()) yield(std::string(data)); }); @@ -319,7 +266,6 @@ std::unique_ptr sinkToSource( if (pos == cur.size()) { if (!cur.empty()) { - CoroutineContext ctx; (*coro)(); } cur = coro->get(); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 8137db5f40e..a795bb08821 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -557,27 +557,4 @@ struct FramedSink : nix::BufferedSink }; }; -/** - * Stack allocation strategy for sinkToSource. - * Mutable to avoid a boehm gc dependency in libutil. - * - * boost::context doesn't provide a virtual class, so we define our own. - */ -struct StackAllocator { - virtual boost::context::stack_context allocate() = 0; - virtual void deallocate(boost::context::stack_context sctx) = 0; - - /** - * The stack allocator to use in sinkToSource and potentially elsewhere. - * It is reassigned by the initGC() method in libexpr. - */ - static StackAllocator *defaultAllocator; -}; - -/* Disabling GC when entering a coroutine (without the boehm patch). - mutable to avoid boehm gc dependency in libutil. - */ -extern std::shared_ptr (*create_coro_gc_hook)(); - - } From b8684eb44723ad023e74a50c446f77ef56031f18 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Jul 2024 20:03:29 +0200 Subject: [PATCH 096/284] sourceToSink(): Fix abort --- src/libutil/serialise.cc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index cd52fbfb472..8fbfe248caf 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -218,12 +218,8 @@ std::unique_ptr sourceToSink(std::function fun) void finish() override { - if (!coro) return; - if (!*coro) abort(); - { + if (coro && *coro) (*coro)(true); - } - if (*coro) abort(); } }; From 3172e88af544ca53a54930ccf3f580b15141f01c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 20 Jul 2024 22:46:09 +0200 Subject: [PATCH 097/284] Make abort() call sites log first --- src/libcmd/repl.cc | 2 +- src/libexpr/attr-path.cc | 2 +- src/libexpr/eval.cc | 8 +++---- src/libexpr/nixexpr.cc | 4 ++-- src/libexpr/primops.cc | 2 +- src/libexpr/print-ambiguous.cc | 2 +- src/libexpr/print.cc | 4 ++-- src/libexpr/symbol-table.hh | 3 ++- src/libexpr/value.hh | 2 +- src/libfetchers/attrs.cc | 4 ++-- src/libmain/loggers.cc | 2 +- .../build/drv-output-substitution-goal.hh | 2 +- src/libstore/build/goal.hh | 4 ++-- src/libstore/build/substitution-goal.hh | 2 +- src/libstore/build/worker.cc | 4 ++-- src/libstore/daemon.cc | 2 +- src/libstore/gc.cc | 2 +- src/libstore/globals.cc | 2 +- src/libstore/nar-info-disk-cache.cc | 4 ++-- .../unix/build/local-derivation-goal.cc | 2 +- src/libstore/unix/pathlocks.cc | 2 +- src/libutil/chunked-vector.hh | 4 +++- src/libutil/error.cc | 12 ++++++++++ src/libutil/error.hh | 24 +++++++++++++++++++ src/libutil/file-content-address.cc | 2 +- src/libutil/fs-sink.cc | 2 +- src/libutil/git.cc | 2 +- src/libutil/hash.cc | 2 +- src/libutil/logging.cc | 2 +- src/libutil/serialise.cc | 8 +++---- src/libutil/sync.hh | 4 +++- src/libutil/unix/monitor-fd.hh | 4 +++- src/libutil/unix/processes.cc | 2 +- src/nix-env/nix-env.cc | 2 +- src/nix-store/nix-store.cc | 2 +- 35 files changed, 88 insertions(+), 45 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index b5d0816dd2c..bf0d820c270 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -217,7 +217,7 @@ ReplExitStatus NixRepl::mainLoop() case ProcessLineResult::PromptAgain: break; default: - abort(); + unreachable(); } } catch (ParseError & e) { if (e.msg().find("unexpected end of file") != std::string::npos) { diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index d61d9363070..2f67260c532 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -134,7 +134,7 @@ std::pair findPackageFilename(EvalState & state, Value & v return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno}; } catch (std::invalid_argument & e) { fail(); - abort(); + unreachable(); } } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b192f9b4b18..32bc68e6ded 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -149,7 +149,7 @@ std::string_view showType(ValueType type, bool withArticle) case nFloat: return WA("a", "float"); case nThunk: return WA("a", "thunk"); } - abort(); + unreachable(); } @@ -771,7 +771,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & case ReplExitStatus::Continue: break; default: - abort(); + unreachable(); } } } @@ -1140,7 +1140,7 @@ inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const PosIdx po void Expr::eval(EvalState & state, Env & env, Value & v) { - abort(); + unreachable(); } @@ -1573,7 +1573,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & .withFrame(*fun.payload.lambda.env, lambda) .debugThrow(); } - abort(); // can't happen + unreachable(); } } diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 6c6769cfd44..dbc74faf9a7 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -25,7 +25,7 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) void Expr::show(const SymbolTable & symbols, std::ostream & str) const { - abort(); + unreachable(); } void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const @@ -271,7 +271,7 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) void Expr::bindVars(EvalState & es, const std::shared_ptr & env) { - abort(); + unreachable(); } void ExprInt::bindVars(EvalState & es, const std::shared_ptr & env) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5a373a43bda..0b3b19b5764 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -426,7 +426,7 @@ static void prim_typeOf(EvalState & state, const PosIdx pos, Value * * args, Val t = args[0]->external()->typeOf(); break; case nFloat: t = "float"; break; - case nThunk: abort(); + case nThunk: unreachable(); } v.mkString(t); } diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc index 5d55b45da3b..a40c98643e3 100644 --- a/src/libexpr/print-ambiguous.cc +++ b/src/libexpr/print-ambiguous.cc @@ -94,7 +94,7 @@ void printAmbiguous( break; default: printError("Nix evaluator internal error: printAmbiguous: invalid value type"); - abort(); + unreachable(); } } diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index bc17d6bfe26..4d1a6868c67 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -475,7 +475,7 @@ class Printer else output << "primop"; } else { - abort(); + unreachable(); } output << "»"; @@ -504,7 +504,7 @@ class Printer if (options.ansiColors) output << ANSI_NORMAL; } else { - abort(); + unreachable(); } } diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index c7a3563b040..dee7369e80b 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -7,6 +7,7 @@ #include "types.hh" #include "chunked-vector.hh" +#include "error.hh" namespace nix { @@ -113,7 +114,7 @@ public: SymbolStr operator[](Symbol s) const { if (s.id == 0 || s.id > store.size()) - abort(); + unreachable(); return SymbolStr(store[s.id - 1]); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 1f4d72d3926..fdc6c84c4d3 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -285,7 +285,7 @@ public: if (invalidIsThunk) return nThunk; else - abort(); + unreachable(); } inline void finishValue(InternalType newType, Payload newPayload) diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index b788c5948e0..25d04cdc950 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -33,7 +33,7 @@ nlohmann::json attrsToJSON(const Attrs & attrs) json[attr.first] = *v; } else if (auto v = std::get_if>(&attr.second)) { json[attr.first] = v->t; - } else abort(); + } else unreachable(); } return json; } @@ -99,7 +99,7 @@ std::map attrsToQuery(const Attrs & attrs) query.insert_or_assign(attr.first, *v); } else if (auto v = std::get_if>(&attr.second)) { query.insert_or_assign(attr.first, v->t ? "1" : "0"); - } else abort(); + } else unreachable(); } return query; } diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index 9829859de32..a4e0530c8f9 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -36,7 +36,7 @@ Logger * makeDefaultLogger() { return logger; } default: - abort(); + unreachable(); } } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 80705492662..8c60d01987a 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -36,7 +36,7 @@ public: Co init() override; Co realisationFetched(std::shared_ptr outputInfo, nix::ref sub); - void timedOut(Error && ex) override { abort(); }; + void timedOut(Error && ex) override { unreachable(); }; std::string key() override; diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 162c392d066..9c6a40c845c 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -400,12 +400,12 @@ public: virtual void handleChildOutput(Descriptor fd, std::string_view data) { - abort(); + unreachable(); } virtual void handleEOF(Descriptor fd) { - abort(); + unreachable(); } void trace(std::string_view s); diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 86e4f542382..c1de45379f1 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -50,7 +50,7 @@ public: PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~PathSubstitutionGoal(); - void timedOut(Error && ex) override { abort(); }; + void timedOut(Error && ex) override { unreachable(); }; /** * We prepend "a$" to the key name to ensure substitution goals diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 7fc41b1214a..ab0ba67b521 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -216,7 +216,7 @@ void Worker::childStarted(GoalPtr goal, const std::set std::string BaseSetting::to_string() const if (value == smEnabled) return "true"; else if (value == smRelaxed) return "relaxed"; else if (value == smDisabled) return "false"; - else abort(); + else unreachable(); } template<> void BaseSetting::convertToArg(Args & args, const std::string & category) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 288f618d5d7..83e63794e0d 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -164,7 +164,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache Cache & getCache(State & state, const std::string & uri) { auto i = state.caches.find(uri); - if (i == state.caches.end()) abort(); + if (i == state.caches.end()) unreachable(); return i->second; } @@ -211,7 +211,7 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache { auto r(state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority)); - if (!r.next()) { abort(); } + if (!r.next()) { unreachable(); } ret.id = (int) r.getInt(0); } diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 0dd102200a5..ac5b4dd0b7e 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -165,7 +165,7 @@ void LocalDerivationGoal::killSandbox(bool getStats) buildResult.cpuSystem = stats.cpuSystem; } #else - abort(); + unreachable(); #endif } diff --git a/src/libstore/unix/pathlocks.cc b/src/libstore/unix/pathlocks.cc index af21319a758..1ec4579ec96 100644 --- a/src/libstore/unix/pathlocks.cc +++ b/src/libstore/unix/pathlocks.cc @@ -45,7 +45,7 @@ bool lockFile(Descriptor desc, LockType lockType, bool wait) if (lockType == ltRead) type = LOCK_SH; else if (lockType == ltWrite) type = LOCK_EX; else if (lockType == ltNone) type = LOCK_UN; - else abort(); + else unreachable(); if (wait) { while (flock(desc, type) != 0) { diff --git a/src/libutil/chunked-vector.hh b/src/libutil/chunked-vector.hh index d914e2542f0..4709679a62a 100644 --- a/src/libutil/chunked-vector.hh +++ b/src/libutil/chunked-vector.hh @@ -6,6 +6,8 @@ #include #include +#include "error.hh" + namespace nix { /** @@ -30,7 +32,7 @@ private: auto & addChunk() { if (size_ >= std::numeric_limits::max() - ChunkSize) - abort(); + unreachable(); chunks.emplace_back(); chunks.back().reserve(ChunkSize); return chunks.back(); diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 33c391963f3..b1858911aa5 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -430,4 +430,16 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s return out; } +void panic(std::string_view msg) +{ + printError(msg); + printError("This was a fatal error, aborting."); + abort(); +} + +void panic(const char * file, int line, const char * func) +{ + panic(std::string("Unexpected condition in ") + func + " at " + file + ":" + std::to_string(line)); +} + } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index d7fe902d6c0..572a1baf729 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -273,4 +273,28 @@ using NativeSysError = */ void throwExceptionSelfCheck(); +/** + * Print a message and abort(). + * + * @note: This assumes that the logger is operational + */ +[[noreturn]] +void panic(std::string_view msg); + +/** + * Print a basic error message with source position and abort(). + * Use the unreachable macro to call this. + * + * @note: This assumes that the logger is operational + */ +[[noreturn]] +void panic(const char * file, int line, const char * func); + +/** + * Print a basic error message with source position and abort(). + * + * @note: This assumes that the logger is operational + */ +#define unreachable() (panic(__FILE__, __LINE__, __func__)) + } diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc index 438dac7daa6..86378dd673d 100644 --- a/src/libutil/file-content-address.cc +++ b/src/libutil/file-content-address.cc @@ -63,7 +63,7 @@ std::string_view renderFileIngestionMethod(FileIngestionMethod method) case FileIngestionMethod::Git: return "git"; default: - abort(); + unreachable(); } } diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 3246e0902fb..f15324d0a9f 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -53,7 +53,7 @@ void copyRecursive( throw Error("file '%1%' has an unsupported type", from); default: - abort(); + unreachable(); } } diff --git a/src/libutil/git.cc b/src/libutil/git.cc index a6968a43eed..af91fa64399 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -201,7 +201,7 @@ std::optional convertMode(SourceAccessor::Type type) case SourceAccessor::tRegular: return Mode::Regular; case SourceAccessor::tDirectory: return Mode::Directory; case SourceAccessor::tMisc: return std::nullopt; - default: abort(); + default: unreachable(); } } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 35b913e4249..ab2a8695dd4 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -25,7 +25,7 @@ static size_t regularHashSize(HashAlgorithm type) { case HashAlgorithm::SHA256: return sha256HashSize; case HashAlgorithm::SHA512: return sha512HashSize; } - abort(); + unreachable(); } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 55751b4cf0d..29427f2f636 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -189,7 +189,7 @@ struct JSONLogger : Logger { else if (f.type == Logger::Field::tString) arr.push_back(f.s); else - abort(); + unreachable(); } void write(const nlohmann::json & json) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 36b99905aab..ee435154573 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -260,7 +260,7 @@ std::unique_ptr sourceToSink(std::function fun) }); } - if (!*coro) { abort(); } + if (!*coro) { unreachable(); } if (!cur.empty()) { CoroutineContext ctx; @@ -271,12 +271,12 @@ std::unique_ptr sourceToSink(std::function fun) void finish() override { if (!coro) return; - if (!*coro) abort(); + if (!*coro) unreachable(); { CoroutineContext ctx; (*coro)(true); } - if (*coro) abort(); + if (*coro) unreachable(); } }; @@ -316,7 +316,7 @@ std::unique_ptr sinkToSource( }); } - if (!*coro) { eof(); abort(); } + if (!*coro) { eof(); unreachable(); } if (pos == cur.size()) { if (!cur.empty()) { diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh index 20dd6ee52bc..c1b699ffc30 100644 --- a/src/libutil/sync.hh +++ b/src/libutil/sync.hh @@ -7,6 +7,8 @@ #include #include +#include "error.hh" + namespace nix { /** @@ -47,7 +49,7 @@ public: friend SyncBase; Lock(SyncBase * s) : s(s), lk(s->mutex) { } public: - Lock(Lock && l) : s(l.s) { abort(); } + Lock(Lock && l) : s(l.s) { unreachable(); } Lock(const Lock & l) = delete; ~Lock() { } diff --git a/src/libutil/unix/monitor-fd.hh b/src/libutil/unix/monitor-fd.hh index 103894de968..b6610feff98 100644 --- a/src/libutil/unix/monitor-fd.hh +++ b/src/libutil/unix/monitor-fd.hh @@ -40,7 +40,9 @@ public: #endif ; auto count = poll(fds, 1, -1); - if (count == -1) abort(); // can't happen + if (count == -1) + unreachable(); + /* This shouldn't happen, but can on macOS due to a bug. See rdar://37550628. diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 1af559a21b9..c5ce74acc33 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -182,7 +182,7 @@ static pid_t doFork(bool allowVfork, ChildWrapperFunction & fun) #endif if (pid != 0) return pid; fun(); - abort(); + unreachable(); } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 5e170c99d37..40c20054268 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1159,7 +1159,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) case cvEqual: ch = '='; break; case cvGreater: ch = '<'; break; case cvUnavail: ch = '-'; break; - default: abort(); + default: unreachable(); } if (xmlOutput) { diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index f073074e851..b4de42ba17b 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -480,7 +480,7 @@ static void opQuery(Strings opFlags, Strings opArgs) } default: - abort(); + unreachable(); } } From e086d5d899aaefdce8cc7a509cb2971be9732b2b Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Wed, 24 Jul 2024 13:17:28 -0400 Subject: [PATCH 098/284] libexpr: experimental pipe operators --- doc/manual/rl-next/pipe-operators.md | 28 ++++++++++ doc/manual/src/language/operators.md | 37 ++++++++++++- src/libexpr/lexer.l | 14 +++++ src/libexpr/parser.y | 30 ++++++++--- src/libutil/experimental-features.cc | 10 +++- src/libutil/experimental-features.hh | 1 + .../lang/eval-fail-pipe-operators.err.exp | 5 ++ .../lang/eval-fail-pipe-operators.nix | 1 + tests/unit/libexpr/main.cc | 3 ++ tests/unit/libexpr/trivial.cc | 54 +++++++++++++++++++ 10 files changed, 174 insertions(+), 9 deletions(-) create mode 100644 doc/manual/rl-next/pipe-operators.md create mode 100644 tests/functional/lang/eval-fail-pipe-operators.err.exp create mode 100644 tests/functional/lang/eval-fail-pipe-operators.nix diff --git a/doc/manual/rl-next/pipe-operators.md b/doc/manual/rl-next/pipe-operators.md new file mode 100644 index 00000000000..b4cbe30e354 --- /dev/null +++ b/doc/manual/rl-next/pipe-operators.md @@ -0,0 +1,28 @@ +--- +synopsis: "Add `pipe-operators` experimental feature" +prs: +- 11131 +--- + +This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). + +The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. +*a* `|>` *b* is equivalent to the function application *b* *a*, and +*a* `<|` *b* is equivalent to the function application *a* *b*. + +For example: + +``` +nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 +9 + +nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 +7 +``` + +`<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. +These properties may change in future releases. + +See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. + +[pipe operators]: @docroot@/language/operators.md#pipe-operators diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 9660a764d92..a1e28349bcd 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -26,13 +26,17 @@ | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | | [Logical implication] | *bool* `->` *bool* | right | 14 | +| [Pipe operator] (experimental) | *expr* `\|>` *func* | left | 15 | +| [Pipe operator] (experimental) | *func* `<\|` *expr* | right | 15 | [string]: ./types.md#type-string [path]: ./types.md#type-path -[number]: ./types.md#type-float +[number]: ./types.md#type-float [list]: ./types.md#list [attribute set]: ./types.md#attribute-set + + ## Attribute selection > **Syntax** @@ -182,3 +186,34 @@ Equivalent to `!`*b1* `||` *b2*. [Logical implication]: #logical-implication +## Pipe operators + +- *a* `|>` *b* is equivalent to *b* *a* +- *a* `<|` *b* is equivalent to *a* *b* + +> **Example** +> +> ``` +> nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 +> 9 +> +> nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 +> 7 +> ``` + +> **Warning** +> +> This syntax is part of an +> [experimental feature](@docroot@/contributing/experimental-features.md) +> and may change in future releases. +> +> To use this syntax, make sure the +> [`pipe-operators` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-pipe-operators) +> is enabled. +> For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): +> +> ``` +> extra-experimental-features = pipe-operators +> ``` + +[Pipe operator]: #pipe-operators diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 58401be8e71..eb1825b7c3f 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -67,6 +67,14 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) return {result, size_t(t - result)}; } +static void requireExperimentalFeature(const ExperimentalFeature & feature, const Pos & pos) +{ + if (!experimentalFeatureSettings.isEnabled(feature)) + throw ParseError(ErrorInfo{ + .msg = HintFmt("experimental Nix feature '%1%' is disabled; add '--extra-experimental-features %1%' to enable it", showExperimentalFeature(feature)), + .pos = pos, + }); +} } @@ -119,6 +127,12 @@ or { return OR_KW; } \-\> { return IMPL; } \/\/ { return UPDATE; } \+\+ { return CONCAT; } +\<\| { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]); + return PIPE_FROM; + } +\|\> { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]); + return PIPE_INTO; + } {ID} { yylval->id = {yytext, (size_t) yyleng}; return ID; } {INT} { errno = 0; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 8ea176b2461..9ad41c148dd 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -99,6 +99,14 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P } } +static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) { + if (auto e2 = dynamic_cast(fn)) { + e2->args.push_back(arg); + return fn; + } + return new ExprCall(pos, fn, {arg}); +} + %} @@ -123,6 +131,7 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P %type start expr expr_function expr_if expr_op %type expr_select expr_simple expr_app +%type expr_pipe_from expr_pipe_into %type expr_list %type binds %type formals @@ -140,6 +149,7 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P %token PATH HPATH SPATH PATH_END %token URI %token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW +%token PIPE_FROM PIPE_INTO /* <| and |> */ %token DOLLAR_CURLY /* == ${ */ %token IND_STRING_OPEN IND_STRING_CLOSE %token ELLIPSIS @@ -206,9 +216,21 @@ expr_function expr_if : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); } + | expr_pipe_from + | expr_pipe_into | expr_op ; +expr_pipe_from + : expr_op PIPE_FROM expr_pipe_from { $$ = makeCall(state->at(@2), $1, $3); } + | expr_op PIPE_FROM expr_op { $$ = makeCall(state->at(@2), $1, $3); } + ; + +expr_pipe_into + : expr_pipe_into PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); } + | expr_op PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); } + ; + expr_op : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); } | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->s.sub), {new ExprInt(0), $2}); } @@ -233,13 +255,7 @@ expr_op ; expr_app - : expr_app expr_select { - if (auto e2 = dynamic_cast($1)) { - e2->args.push_back($2); - $$ = $1; - } else - $$ = new ExprCall(CUR_POS, $1, {$2}); - } + : expr_app expr_select { $$ = makeCall(CUR_POS, $1, $2); } | expr_select ; diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 1c080e372f6..a0c955816e9 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -24,7 +24,7 @@ struct ExperimentalFeatureDetails * feature, we either have no issue at all if few features are not added * at the end of the list, or a proper merge conflict if they are. */ -constexpr size_t numXpFeatures = 1 + static_cast(Xp::VerifiedFetches); +constexpr size_t numXpFeatures = 1 + static_cast(Xp::PipeOperators); constexpr std::array xpFeatureDetails = {{ { @@ -294,6 +294,14 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "https://github.com/NixOS/nix/milestone/48", }, + { + .tag = Xp::PipeOperators, + .name = "pipe-operators", + .description = R"( + Add `|>` and `<|` operators to the Nix language. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/55", + }, }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 6ffbc0c1028..e65e51280a4 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -35,6 +35,7 @@ enum struct ExperimentalFeature ConfigurableImpureEnv, MountedSSHStore, VerifiedFetches, + PipeOperators, }; /** diff --git a/tests/functional/lang/eval-fail-pipe-operators.err.exp b/tests/functional/lang/eval-fail-pipe-operators.err.exp new file mode 100644 index 00000000000..49f3fa8ad4e --- /dev/null +++ b/tests/functional/lang/eval-fail-pipe-operators.err.exp @@ -0,0 +1,5 @@ +error: experimental Nix feature 'pipe-operators' is disabled; add '--extra-experimental-features pipe-operators' to enable it + at /pwd/lang/eval-fail-pipe-operators.nix:1:3: + 1| 1 |> 2 + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-pipe-operators.nix b/tests/functional/lang/eval-fail-pipe-operators.nix new file mode 100644 index 00000000000..433e0fd7f5a --- /dev/null +++ b/tests/functional/lang/eval-fail-pipe-operators.nix @@ -0,0 +1 @@ +1 |> 2 diff --git a/tests/unit/libexpr/main.cc b/tests/unit/libexpr/main.cc index cf7fcf5a30c..e3412d9ef9a 100644 --- a/tests/unit/libexpr/main.cc +++ b/tests/unit/libexpr/main.cc @@ -34,6 +34,9 @@ int main (int argc, char **argv) { setEnv("_NIX_TEST_NO_SANDBOX", "1"); #endif + // For pipe operator tests in trivial.cc + experimentalFeatureSettings.set("experimental-features", "pipe-operators"); + ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/tests/unit/libexpr/trivial.cc b/tests/unit/libexpr/trivial.cc index 61ea71a0f3c..e455a571b18 100644 --- a/tests/unit/libexpr/trivial.cc +++ b/tests/unit/libexpr/trivial.cc @@ -182,6 +182,60 @@ namespace nix { ASSERT_THAT(v, IsIntEq(15)); } + TEST_F(TrivialExpressionTest, forwardPipe) { + auto v = eval("1 |> builtins.add 2 |> builtins.mul 3"); + ASSERT_THAT(v, IsIntEq(9)); + } + + TEST_F(TrivialExpressionTest, backwardPipe) { + auto v = eval("builtins.add 1 <| builtins.mul 2 <| 3"); + ASSERT_THAT(v, IsIntEq(7)); + } + + TEST_F(TrivialExpressionTest, forwardPipeEvaluationOrder) { + auto v = eval("1 |> null |> (x: 2)"); + ASSERT_THAT(v, IsIntEq(2)); + } + + TEST_F(TrivialExpressionTest, backwardPipeEvaluationOrder) { + auto v = eval("(x: 1) <| null <| 2"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, differentPipeOperatorsDoNotAssociate) { + ASSERT_THROW(eval("(x: 1) <| 2 |> (x: 3)"), ParseError); + } + + TEST_F(TrivialExpressionTest, differentPipeOperatorsParensLeft) { + auto v = eval("((x: 1) <| 2) |> (x: 3)"); + ASSERT_THAT(v, IsIntEq(3)); + } + + TEST_F(TrivialExpressionTest, differentPipeOperatorsParensRight) { + auto v = eval("(x: 1) <| (2 |> (x: 3))"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, forwardPipeLowestPrecedence) { + auto v = eval("false -> true |> (x: !x)"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(TrivialExpressionTest, backwardPipeLowestPrecedence) { + auto v = eval("(x: !x) <| false -> true"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(TrivialExpressionTest, forwardPipeStrongerThanElse) { + auto v = eval("if true then 1 else 2 |> 3"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, backwardPipeStrongerThanElse) { + auto v = eval("if true then 1 else 2 <| 3"); + ASSERT_THAT(v, IsIntEq(1)); + } + TEST_F(TrivialExpressionTest, bindOr) { auto v = eval("{ or = 1; }"); ASSERT_THAT(v, IsAttrsOfSize(1)); From 459ee005633eb8094c82b6b6d9bff1df2732b893 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 19:22:53 +0200 Subject: [PATCH 099/284] Render the release notes when building the manual from dev shell --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 2384c29741b..fc6d1616917 100644 --- a/flake.nix +++ b/flake.nix @@ -333,6 +333,7 @@ ++ [ pkgs.buildPackages.cmake pkgs.buildPackages.shellcheck + pkgs.buildPackages.changelog-d modular.pre-commit.settings.package (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) From 4bfc96f376ff0e0cd5fba4b36d7afcd4abcee020 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 19:21:34 +0200 Subject: [PATCH 100/284] Fix and update release notes --- doc/manual/rl-next/shebang-relative.md | 2 +- doc/manual/rl-next/zzz-other.md | 50 ++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 doc/manual/rl-next/zzz-other.md diff --git a/doc/manual/rl-next/shebang-relative.md b/doc/manual/rl-next/shebang-relative.md index c887a598a05..d12c0f8dce2 100644 --- a/doc/manual/rl-next/shebang-relative.md +++ b/doc/manual/rl-next/shebang-relative.md @@ -8,7 +8,7 @@ issues: --- -Relative [path](@docroot@/language/values.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary?highlight=base%20directory#gloss-base-directory). +Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). Previously they were resolved relative to the current working directory. For example, consider the following script in `~/myproject/say-hi`: diff --git a/doc/manual/rl-next/zzz-other.md b/doc/manual/rl-next/zzz-other.md new file mode 100644 index 00000000000..f3721bd383e --- /dev/null +++ b/doc/manual/rl-next/zzz-other.md @@ -0,0 +1,50 @@ +--- +synopsis: Other changes +--- + +- [#9063](https://github.com/NixOS/nix/pull/9063): introduce `libnixflake` and move flakes-specific code there (Nix is internally composed of a set of libraries, and this better reflects the architecture wrt flakes) +- [#10852](https://github.com/NixOS/nix/pull/10852): make Nix commands respond better to interruption - Ctrl+C in the terminal +- [#10853](https://github.com/NixOS/nix/pull/10853): fix docs of `builtins.importNative` +- [#10858](https://github.com/NixOS/nix/pull/10858): `flake check`: Recognize well known `homeModule`/`homeModules` attribute +- [#10865](https://github.com/NixOS/nix/pull/10865): Update dependencies to Nixpkgs 24.05 (when using the `nix` flake), and support bdwgc 8.2.6 ([#11141](https://github.com/NixOS/nix/issues/11141), [#10880](https://github.com/NixOS/nix/pull/10880)) +- [#10883](https://github.com/NixOS/nix/pull/10883): Use `TMP` instead of `XDG_RUNTIME_DIR` +- [#10994](https://github.com/NixOS/nix/pull/10994): fix minor bug in elided item counts when printing values lazily +- [#10988](https://github.com/NixOS/nix/pull/10988): restore `commit-lockfile-summary` alias +- [#10941](https://github.com/NixOS/nix/pull/10941): invalid derivation name now causes an actionable error message +- Changes to the interaction between Nix's I/O coroutines and the garbage collector: +- [#10878](https://github.com/NixOS/nix/pull/10878): allow `ipc-sysv*` in the Darwin build sandbox +- [#11031](https://github.com/NixOS/nix/pull/11031): fix Darwin build sandbox +- [#10907](https://github.com/NixOS/nix/pull/10907): use opaque struct instead of `void *` in the C API +- [#10947](https://github.com/NixOS/nix/issues/10947): fix evaluation cache accidentally persisting disallowed IFD errors +- [#11020](https://github.com/NixOS/nix/pull/11020): enable fetch and eval caching for tarballs +- [#11041](https://github.com/NixOS/nix/pull/11041): add discovered attribute paths to the `--show-trace` error trace in `nix-build`, `nix-env`, OfBorg, and other callers of `getDerivations` +- [#11056](https://github.com/NixOS/nix/pull/11056): `s3` store now uses system defined proxy settings +- [#11077](https://github.com/NixOS/nix/pull/11077): support hardlinks in tarballs +- [#11100](https://github.com/NixOS/nix/pull/11100): pretty print values consistently regardless of prior thunk state +- [#11086](https://github.com/NixOS/nix/pull/11086): fix loss of evaluation cache additions in `nix env run`, `nix shell`, `nix develop`, and `nix fmt` +- [#11149](https://github.com/NixOS/nix/pull/11149): report GC time and number of GC cycles in `NIX_SHOW_STATS=1` report +- [#11142](https://github.com/NixOS/nix/pull/11142): aliased options can now also be passed as flags, just like their "normal" counterparts, e.g. `--build-max-jobs` now works +- [#11043](https://github.com/NixOS/nix/pull/11043): `assert a == b; e` now reports some detail about why `a` and `b` are different when they are +- [#11159](https://github.com/NixOS/nix/pull/11159): don't crash a nix-daemon worker process when the client disconnects +- Stability improvements and fixes [#10861](https://github.com/NixOS/nix/pull/10861), [#10865](https://github.com/NixOS/nix/pull/10865), [#10918](https://github.com/NixOS/nix/pull/10918), [#10916](https://github.com/NixOS/nix/pull/10916), [#10884](https://github.com/NixOS/nix/pull/10884), [#10943](https://github.com/NixOS/nix/pull/10943), [#11019](https://github.com/NixOS/nix/pull/11019), [#11122](https://github.com/NixOS/nix/pull/11122), [#11117](https://github.com/NixOS/nix/pull/11117) +- User documentation improvements [#10888](https://github.com/NixOS/nix/pull/10888), [#10966](https://github.com/NixOS/nix/pull/10966), [#10974](https://github.com/NixOS/nix/pull/10974), [#10997](https://github.com/NixOS/nix/pull/10997), [#11013](https://github.com/NixOS/nix/pull/11013), [#11059](https://github.com/NixOS/nix/pull/11059), [#11119](https://github.com/NixOS/nix/pull/11119), [#11116](https://github.com/NixOS/nix/pull/11116), [#11061](https://github.com/NixOS/nix/pull/11061), [#11102](https://github.com/NixOS/nix/pull/11102) +- BSD support: [#10896](https://github.com/NixOS/nix/pull/10896) [#11022](https://github.com/NixOS/nix/pull/11022) [#11156](https://github.com/NixOS/nix/pull/11156) +- Windows support: [#10769](https://github.com/NixOS/nix/pull/10769), [#10975](https://github.com/NixOS/nix/pull/10975) [#11153](https://github.com/NixOS/nix/pull/11153) +- Portability: [#7048](https://github.com/NixOS/nix/pull/7048) [#11090](https://github.com/NixOS/nix/pull/11090) +- Installer improvements [#10902](https://github.com/NixOS/nix/pull/10902) +- Performance improvements [#10853](https://github.com/NixOS/nix/pull/10853), [#10854](https://github.com/NixOS/nix/pull/10854), [#11082](https://github.com/NixOS/nix/pull/11082), [#11092](https://github.com/NixOS/nix/pull/11092), [#11113](https://github.com/NixOS/nix/pull/11113) + +Contributor experience improvements: + +Use Meson to build Nix (nearing completion) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10967](https://github.com/NixOS/nix/pull/10967) [#10963](https://github.com/NixOS/nix/pull/10963) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11064](https://github.com/NixOS/nix/pull/11064) [#11060](https://github.com/NixOS/nix/pull/11060) [#11155](https://github.com/NixOS/nix/pull/11155) +- Testing improvements [#10864](https://github.com/NixOS/nix/pull/10864), [#10903](https://github.com/NixOS/nix/pull/10903), [#10874](https://github.com/NixOS/nix/pull/10874), [#10922](https://github.com/NixOS/nix/pull/10922), [#11006](https://github.com/NixOS/nix/pull/11006), [#11110](https://github.com/NixOS/nix/pull/11110), [#10931](https://github.com/NixOS/nix/pull/10931), [#11123](https://github.com/NixOS/nix/pull/11123) + - [#10603](https://github.com/NixOS/nix/pull/10603): We now evaluate a set of flakes in CI + - [#10922](https://github.com/NixOS/nix/pull/10922): The functional test suite is now run in both in the build sandbox and in a NixOS environment +- CI improvements [#10929](https://github.com/NixOS/nix/pull/10929) [#10999](https://github.com/NixOS/nix/pull/10999) [#11009](https://github.com/NixOS/nix/pull/11009) [#11065](https://github.com/NixOS/nix/pull/11065) [#11071](https://github.com/NixOS/nix/pull/11071) +- Contributor documentation improvements [#10869](https://github.com/NixOS/nix/pull/10869), [#9871](https://github.com/NixOS/nix/pull/9871), [#10960](https://github.com/NixOS/nix/pull/10960), [#11147](https://github.com/NixOS/nix/pull/11147) +- Error message improvements: [#11050](https://github.com/NixOS/nix/pull/11050) [#11154](https://github.com/NixOS/nix/pull/11154) +- Cleaning up the Settings system (`nix.conf` and related architectural cleanups): [#10913](https://github.com/NixOS/nix/pull/10913), [#10951](https://github.com/NixOS/nix/pull/10951), [#11007](https://github.com/NixOS/nix/pull/11007), [#11108](https://github.com/NixOS/nix/pull/11108), [#11014](https://github.com/NixOS/nix/pull/11014), [#11109](https://github.com/NixOS/nix/pull/11109), [#11112](https://github.com/NixOS/nix/pull/11112) +- Other cleanups and refactors [#10857](https://github.com/NixOS/nix/pull/10857) [#10935](https://github.com/NixOS/nix/pull/10935) [#10873](https://github.com/NixOS/nix/pull/10873) [#10745](https://github.com/NixOS/nix/pull/10745) [#10961](https://github.com/NixOS/nix/pull/10961) [#10962](https://github.com/NixOS/nix/pull/10962) [#10972](https://github.com/NixOS/nix/pull/10972) [#11018](https://github.com/NixOS/nix/pull/11018) [#11035](https://github.com/NixOS/nix/pull/11035) [#11037](https://github.com/NixOS/nix/pull/11037) [#11081](https://github.com/NixOS/nix/pull/11081) [#11089](https://github.com/NixOS/nix/pull/11089) [#11093](https://github.com/NixOS/nix/pull/11093) [#11114](https://github.com/NixOS/nix/pull/11114) [#11103](https://github.com/NixOS/nix/pull/11103) [#11126](https://github.com/NixOS/nix/pull/11126) [#11125](https://github.com/NixOS/nix/pull/11125) [#11120](https://github.com/NixOS/nix/pull/11120) +- Scheduler/builder refactoring [#11005](https://github.com/NixOS/nix/pull/11005) +- [#11011](https://github.com/NixOS/nix/pull/11011): enable `-Werror=unused-result` + From caf4e98f0c0a0a7178538ee8ef7b199d2a655aac Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Jul 2024 20:10:41 +0200 Subject: [PATCH 101/284] Log download durations --- src/libstore/filetransfer.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index cbbb0fe7a34..e7dae333be2 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -71,7 +71,10 @@ struct curlFileTransfer : public FileTransfer curl_off_t writtenToSink = 0; + std::chrono::steady_clock::time_point startTime = std::chrono::steady_clock::now(); + inline static const std::set successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */}; + /* Get the HTTP status code, or 0 for other protocols. */ long getHTTPStatus() { @@ -373,10 +376,14 @@ struct curlFileTransfer : public FileTransfer void finish(CURLcode code) { + auto finishTime = std::chrono::steady_clock::now(); + auto httpStatus = getHTTPStatus(); - debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", - request.verb(), request.uri, code, httpStatus, result.bodySize); + debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s", + request.verb(), request.uri, code, httpStatus, result.bodySize, + std::chrono::duration_cast(finishTime - startTime).count() / 1000.0f + ); appendCurrentUrl(); From 8ffea0a018874e60584eabeb620ec3495873c30d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Jul 2024 20:10:45 +0200 Subject: [PATCH 102/284] Add 'download-buffer-size' setting We are piping curl downloads into `unpackTarfileToSink()`, but the latter is typically slower than the former if you're on a fast connection. So the download could appear unnecessarily slow. (There is even a risk that if the Git import is *really* slow for whatever reason, the TCP connection could time out.) So let's make the download buffer bigger by default - 64 MiB is big enough for the Nixpkgs tarball. Perhaps in the future, we could have an unlimited buffer that spills data to disk beyond a certain threshold, but that's probably overkill. --- src/libstore/filetransfer.cc | 2 +- src/libstore/filetransfer.hh | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index e7dae333be2..f48c2e22d7b 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -858,7 +858,7 @@ void FileTransfer::download( buffer). We don't wait forever to prevent stalling the download thread. (Hopefully sleeping will throttle the sender.) */ - if (state->data.size() > 1024 * 1024) { + if (state->data.size() > fileTransferSettings.downloadBufferSize) { debug("download buffer is full; going to sleep"); state.wait_for(state->request, std::chrono::seconds(10)); } diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 1f5b4ab934c..d836ab2c4a7 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -47,6 +47,12 @@ struct FileTransferSettings : Config Setting tries{this, 5, "download-attempts", "How often Nix will attempt to download a file before giving up."}; + + Setting downloadBufferSize{this, 64 * 1024 * 1024, "download-buffer-size", + R"( + The size of Nix's internal download buffer during `curl` transfers. If data is + not processed quickly enough to exceed the size of this buffer, downloads may stall. + )"}; }; extern FileTransferSettings fileTransferSettings; From f6a9a71b38b25d6c9fb0b9a7fbf0eccf99fa5520 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Jul 2024 20:14:31 +0200 Subject: [PATCH 103/284] Warn if the download buffer is full --- src/libstore/filetransfer.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index f48c2e22d7b..5ea8b6f962c 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -860,6 +860,8 @@ void FileTransfer::download( sender.) */ if (state->data.size() > fileTransferSettings.downloadBufferSize) { debug("download buffer is full; going to sleep"); + static bool haveWarned = false; + warnOnce(haveWarned, "download buffer is full; consider increasing the 'download-buffer-size' setting"); state.wait_for(state->request, std::chrono::seconds(10)); } From 01839b525c5e80d7f67f2808f7f7fb478ddf1ba0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Jul 2024 20:22:26 +0200 Subject: [PATCH 104/284] Show when we're unpacking an archive into the Git cache This happens in parallel with the download (which starts later), so you only see this message when the download has finished but the import hasn't. --- src/libfetchers/github.cc | 5 +++++ src/libfetchers/tarball.cc | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 2968d2df2a4..5710b94d523 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -254,10 +254,15 @@ struct GitArchiveInputScheme : InputScheme getFileTransfer()->download(std::move(req), sink); }); + auto act = std::make_unique(*logger, lvlInfo, actUnknown, + fmt("unpacking '%s' into the Git cache", input.to_string())); + TarArchive archive { *source }; auto parseSink = getTarballCache()->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); + act.reset(); + TarballInfo tarballInfo { .treeHash = parseSink->sync(), .lastModified = lastModified diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 55db3eafbc1..5837cf8a49c 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -143,6 +143,9 @@ DownloadTarballResult downloadTarball( // TODO: fall back to cached value if download fails. + auto act = std::make_unique(*logger, lvlInfo, actUnknown, + fmt("unpacking '%s' into the Git cache", url)); + AutoDelete cleanupTemp; /* Note: if the download is cached, `importTarball()` will receive @@ -167,6 +170,8 @@ DownloadTarballResult downloadTarball( auto parseSink = getTarballCache()->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); + act.reset(); + auto res(_res->lock()); Attrs infoAttrs; From e0620213146b1a581533302a2fab54af21d49685 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 24 Jul 2024 23:17:15 +0200 Subject: [PATCH 105/284] fix `NIX_PATH` for real (#11079) * fix NIX_PATH overriding - test restricted evaluation - test precedence for setting the search path Co-authored-by: Robert Hensing Co-authored-by: John Ericson --- src/libexpr/eval-gc.cc | 8 ++++ src/libexpr/eval-settings.cc | 7 +--- src/libexpr/eval-settings.hh | 4 +- src/libexpr/eval.cc | 17 +++++++-- src/libutil/config.cc | 14 ++++++- tests/functional/nix_path.sh | 70 ++++++++++++++++++++++++++++++++++ tests/functional/restricted.sh | 3 ++ 7 files changed, 111 insertions(+), 12 deletions(-) diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index ba19cd74ede..d82ed153444 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -1,5 +1,7 @@ #include "error.hh" #include "environment-variables.hh" +#include "eval-settings.hh" +#include "config-global.hh" #include "serialise.hh" #include "eval-gc.hh" @@ -230,6 +232,12 @@ void initGC() gcCyclesAfterInit = GC_get_gc_no(); #endif + // NIX_PATH must override the regular setting + // See the comment in applyConfig + if (auto nixPathEnv = getEnv("NIX_PATH")) { + globalConfig.set("nix-path", concatStringsSep(" ", EvalSettings::parseNixPath(nixPathEnv.value()))); + } + gcInitialised = true; } diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index eb5761638a7..2846eccbce1 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -8,7 +8,7 @@ namespace nix { /* Very hacky way to parse $NIX_PATH, which is colon-separated, but can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ -static Strings parseNixPath(const std::string & s) +Strings EvalSettings::parseNixPath(const std::string & s) { Strings res; @@ -48,10 +48,7 @@ EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lo : readOnlyMode{readOnlyMode} , lookupPathHooks{lookupPathHooks} { - auto var = getEnv("NIX_PATH"); - if (var) nixPath = parseNixPath(*var); - - var = getEnv("NIX_ABORT_ON_WARN"); + auto var = getEnv("NIX_ABORT_ON_WARN"); if (var && (var == "1" || var == "yes" || var == "true")) builtinsAbortOnWarn = true; } diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 89a42caba5d..8f48b53a566 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -47,6 +47,8 @@ struct EvalSettings : Config static bool isPseudoUrl(std::string_view s); + static Strings parseNixPath(const std::string & s); + static std::string resolvePseudoUrl(std::string_view url); LookupPathHooks lookupPathHooks; @@ -71,7 +73,7 @@ struct EvalSettings : Config )"}; Setting nixPath{ - this, getDefaultNixPath(), "nix-path", + this, {}, "nix-path", R"( List of search paths to use for [lookup path](@docroot@/language/constructs/lookup-path.md) resolution. This setting determines the value of diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b192f9b4b18..9a6b6c9e8e7 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -215,7 +215,7 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) static constexpr size_t BASE_ENV_SIZE = 128; EvalState::EvalState( - const LookupPath & _lookupPath, + const LookupPath & lookupPathFromArguments, ref store, const fetchers::Settings & fetchSettings, const EvalSettings & settings, @@ -331,12 +331,21 @@ EvalState::EvalState( vStringSymlink.mkString("symlink"); vStringUnknown.mkString("unknown"); - /* Initialise the Nix expression search path. */ + /* Construct the Nix expression search path. */ + assert(lookupPath.elements.empty()); if (!settings.pureEval) { - for (auto & i : _lookupPath.elements) + for (auto & i : lookupPathFromArguments.elements) { lookupPath.elements.emplace_back(LookupPath::Elem {i}); - for (auto & i : settings.nixPath.get()) + } + /* $NIX_PATH overriding regular settings is implemented as a hack in `initGC()` */ + for (auto & i : settings.nixPath.get()) { lookupPath.elements.emplace_back(LookupPath::Elem::parse(i)); + } + if (!settings.restrictEval) { + for (auto & i : EvalSettings::getDefaultNixPath()) { + lookupPath.elements.emplace_back(LookupPath::Elem::parse(i)); + } + } } /* Allow access to all paths in the search path. */ diff --git a/src/libutil/config.cc b/src/libutil/config.cc index b3994826129..ca8480304d2 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -1,6 +1,7 @@ #include "config.hh" #include "args.hh" #include "abstract-setting-to-json.hh" +#include "environment-variables.hh" #include "experimental-features.hh" #include "util.hh" #include "file-system.hh" @@ -170,9 +171,18 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string set(name, value); // Then apply other settings - for (const auto & [name, value] : parsedContents) - if (name != "experimental-features" && name != "extra-experimental-features") + // XXX: NIX_PATH must override the regular setting! This is done in `initGC()` + // Environment variables overriding settings should probably be part of the Config mechanism, + // but at the time of writing it's not worth building that for just one thing + for (const auto & [name, value] : parsedContents) { + if (name != "experimental-features" && name != "extra-experimental-features") { + if ((name == "nix-path" || name == "extra-nix-path") + && getEnv("NIX_PATH").has_value()) { + continue; + } set(name, value); + } + } } void Config::resetOverridden() diff --git a/tests/functional/nix_path.sh b/tests/functional/nix_path.sh index e6a2193f397..7e6a0458d10 100755 --- a/tests/functional/nix_path.sh +++ b/tests/functional/nix_path.sh @@ -14,3 +14,73 @@ nix-instantiate --eval -E '' --restrict-eval [[ $(nix-instantiate --find-file by-absolute-path/simple.nix) = $PWD/simple.nix ]] [[ $(nix-instantiate --find-file by-relative-path/simple.nix) = $PWD/simple.nix ]] + +# this is the human-readable specification for the following test cases of interactions between various ways of specifying NIX_PATH. +# TODO: the actual tests are incomplete and too manual. +# there should be 43 of them, since the table has 9 rows and columns, and 2 interactions are meaningless +# ideally they would work off the table programmatically. +# +# | precedence | hard-coded | nix-path in file | extra-nix-path in file | nix-path in env | extra-nix-path in env | NIX_PATH | nix-path | extra-nix-path | -I | +# |------------------------|------------|------------------|------------------------|-----------------|-----------------------|-----------|-----------|-----------------|-----------------| +# | hard-coded | x | ^override | ^append | ^override | ^append | ^override | ^override | ^append | ^append | +# | nix-path in file | | last wins | ^append | ^override | ^append | ^override | ^override | ^append | ^append | +# | extra-nix-path in file | | | append in order | ^override | ^append | ^override | ^override | ^append | ^append | +# | nix-path in env | | | | last wins | ^append | ^override | ^override | ^append | ^append | +# | extra-nix-path in env | | | | | append in order | ^override | ^override | ^append | ^append | +# | NIX_PATH | | | | | | x | ^override | ^append | ^append | +# | nix-path | | | | | | | last wins | ^append | ^append | +# | extra-nix-path | | | | | | | | append in order | append in order | +# | -I | | | | | | | | | append in order | + +unset NIX_PATH + +mkdir -p $TEST_ROOT/{from-nix-path-file,from-NIX_PATH,from-nix-path,from-extra-nix-path,from-I} +for i in from-nix-path-file from-NIX_PATH from-nix-path from-extra-nix-path from-I; do + touch $TEST_ROOT/$i/only-$i.nix +done + +# finding something that's not in any of the default paths fails +( ! $(nix-instantiate --find-file test) ) + +echo "nix-path = test=$TEST_ROOT/from-nix-path-file" >> "$test_nix_conf" + +# Use nix.conf in absence of NIX_PATH +[[ $(nix-instantiate --find-file test) = $TEST_ROOT/from-nix-path-file ]] + +# NIX_PATH overrides nix.conf +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --find-file test) = $TEST_ROOT/from-NIX_PATH ]] +# if NIX_PATH does not have the desired entry, it fails +(! NIX_PATH=test=$TEST_ROOT nix-instantiate --find-file test/only-from-nix-path-file.nix) + +# -I extends nix.conf +[[ $(nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +# if -I does not have the desired entry, the value from nix.conf is used +[[ $(nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-nix-path-file.nix) = $TEST_ROOT/from-nix-path-file/only-from-nix-path-file.nix ]] + +# -I extends NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +# if -I does not have the desired entry, the value from NIX_PATH is used +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] + +# --extra-nix-path extends NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-extra-nix-path.nix) = $TEST_ROOT/from-extra-nix-path/only-from-extra-nix-path.nix ]] +# if --extra-nix-path does not have the desired entry, the value from NIX_PATH is used +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] + +# --nix-path overrides NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test) = $TEST_ROOT/from-nix-path ]] +# if --nix-path does not have the desired entry, it fails +(! NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test/only-from-NIX_PATH.nix) + +# --nix-path overrides nix.conf +[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test) = $TEST_ROOT/from-nix-path ]] +(! nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test/only-from-nix-path-file.nix) + +# --extra-nix-path extends nix.conf +[[ $(nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-extra-nix-path.nix) = $TEST_ROOT/from-extra-nix-path/only-from-extra-nix-path.nix ]] +# if --extra-nix-path does not have the desired entry, it is taken from nix.conf +[[ $(nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test) = $TEST_ROOT/from-nix-path-file ]] + +# -I extends --nix-path +[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path -I test=$TEST_ROOT/from-I --find-file test/only-from-nix-path.nix) = $TEST_ROOT/from-nix-path/only-from-nix-path.nix ]] diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index 915d973b0dd..591367e9f2d 100755 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -10,6 +10,9 @@ nix-instantiate --restrict-eval --eval -E '1 + 2' nix-instantiate --restrict-eval ./simple.nix -I src=. nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh +# no default NIX_PATH +(unset NIX_PATH; ! nix-instantiate --restrict-eval --find-file .) + (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix') nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=../.. From dba1142c01111f1f98cc89aa850d5df0c012a907 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 25 Jul 2024 03:45:34 +0200 Subject: [PATCH 106/284] docs: add identifiers (#11174) * docs: add identifiers * clarify attribute set notation and add examples * add definition of names Co-authored-by: Ryan Hendrickson --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/language/identifiers.md | 50 +++++++++++++++++ doc/manual/src/language/operators.md | 6 -- doc/manual/src/language/syntax.md | 77 +++++++++++++++++++------- 4 files changed, 109 insertions(+), 25 deletions(-) create mode 100644 doc/manual/src/language/identifiers.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index a6a2101e9af..b4dd277e306 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -28,6 +28,7 @@ - [Data Types](language/types.md) - [String context](language/string-context.md) - [Syntax and semantics](language/syntax.md) + - [Identifiers](language/identifiers.md) - [Scoping rules](language/scope.md) - [String interpolation](language/string-interpolation.md) - [Lookup path](language/constructs/lookup-path.md) diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md new file mode 100644 index 00000000000..c9e981da626 --- /dev/null +++ b/doc/manual/src/language/identifiers.md @@ -0,0 +1,50 @@ +# Identifiers + +An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character sequence that: +- Starts with a letter (`a-z`, `A-Z`) or underscore (`_`) +- Can contain any number of: + - Letters (`a-z`, `A-Z`) + - Digits (`0-9`) + - Underscores (`_`) + - Apostrophes (`'`) + - Hyphens (`-`) +- Is not one of the [keywords](#keywords) + +> **Syntax** +> +> *identifier* ~ `[A-Za-z_][A-Za-z0-9_'-]*` + +# Names + +A name can be an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). + +> **Syntax** +> +> *name* → *identifier* | *string* + +Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting attributes). + +# Keywords + +These keywords are reserved and cannot be used as [identifiers](#identifiers): + +- [`assert`](./syntax.md#assertions) +- [`else`][if] +- [`if`][if] +- [`in`][let] +- [`inherit`](./syntax.md#inheriting-attributes) +- [`let`][let] +- [`or`](./operators.md#attribute-selection) (see note) +- [`rec`](./syntax.md#recursive-sets) +- [`then`][if] +- [`with`](./syntax.md#with-expressions) + +[if]: ./syntax.md#conditionals +[let]: ./syntax.md#let-expressions + +> **Note** +> +> The Nix language evaluator currently allows `or` to be used as a name in some contexts, for backwards compatibility reasons. +> Users are advised not to rely on this. +> +> There are long-standing issues with how `or` is parsed as a name, which can't be resolved without making a breaking change to the language. diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 9660a764d92..d2476c41343 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -42,12 +42,6 @@ Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation. -An attribute path is a dot-separated list of [attribute names](./types.md#attribute-set). - -> **Syntax** -> -> *attrpath* = *name* [ `.` *name* ]... - [Attribute selection]: #attribute-selection ## Has attribute diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md index b0779ea95b4..6108bacd676 100644 --- a/doc/manual/src/language/syntax.md +++ b/doc/manual/src/language/syntax.md @@ -247,37 +247,76 @@ Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#built ## Attribute Set {#attrs-literal} -An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). +An attribute set is a collection of name-value-pairs called *attributes*. -An attribute name can be an identifier or a [string](#string). -An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). +Attribute sets are written enclosed in curly brackets (`{ }`). +Attribute names and attribute values are separated by an equal sign (`=`). +Each value can be an arbitrary expression, terminated by a semicolon (`;`) + +An attribute name is a string without context, and is denoted by a [name] (an [identifier](./identifiers.md#identifiers) or [string literal](#string-literal)). + +[name]: ./identifiers.md#names > **Syntax** > -> *name* = *identifier* | *string* \ -> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` +> *attrset* → `{` { *name* `=` *expr* `;` } `}` + +Attributes can appear in any order. +An attribute name may only occur once in each attribute set. + +> **Example** +> +> This defines an attribute set with attributes named: +> - `x` with the value `123`, an integer +> - `text` with the value `"Hello"`, a string +> - `y` where the value is the result of applying the function `f` to the attribute set `{ bla = 456; }` +> +> ```nix +> { +> x = 123; +> text = "Hello"; +> y = f { bla = 456; }; +> } +> ``` -Names and values are separated by an equal sign (`=`). -Each value is an arbitrary expression terminated by a semicolon (`;`). +Attributes in nested attribute sets can be written using *attribute paths*. > **Syntax** > -> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` +> *attrset* → `{` { *attrpath* `=` *expr* `;` } `}` -Attributes can appear in any order. -An attribute name may only occur once. +An attribute path is a dot-separated list of [names][name]. -Example: +> **Syntax** +> +> *attrpath* = *name* { `.` *name* } -```nix -{ - x = 123; - text = "Hello"; - y = f { bla = 456; }; -} -``` + -This defines a set with attributes named `x`, `text`, `y`. +> **Example** +> +> ```nix +> { a.b.c = 1; a.b.d = 2; } +> ``` +> +> { +> a = { +> b = { +> c = 1; +> d = 2; +> }; +> }; +> } + +Attribute names can also be set implicitly by using the [`inherit` keyword](#inheriting-attributes). + +> **Example** +> +> ```nix +> { inherit (builtins) true; } +> ``` +> +> { true = true; } Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). From f4915af71c9a72a0519c90088abb3a36c265ca95 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jul 2024 22:25:08 -0400 Subject: [PATCH 107/284] Put flake functional tests in their own group This is a nice thing to have, and it made it easier to work on the Meson-ifcation of these functional tests too. --- Makefile | 1 + tests/functional/flakes/local.mk | 24 ++++++++++++++++++++++++ tests/functional/local.mk | 21 --------------------- 3 files changed, 25 insertions(+), 21 deletions(-) create mode 100644 tests/functional/flakes/local.mk diff --git a/Makefile b/Makefile index bb64a104e72..d3ea8513d2f 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,7 @@ ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes) ifdef HOST_UNIX makefiles += \ tests/functional/local.mk \ + tests/functional/flakes/local.mk \ tests/functional/ca/local.mk \ tests/functional/git-hashing/local.mk \ tests/functional/dyn-drv/local.mk \ diff --git a/tests/functional/flakes/local.mk b/tests/functional/flakes/local.mk new file mode 100644 index 00000000000..71e50ad073f --- /dev/null +++ b/tests/functional/flakes/local.mk @@ -0,0 +1,24 @@ +flake-tests := \ + $(d)/flakes.sh \ + $(d)/develop.sh \ + $(d)/edit.sh \ + $(d)/run.sh \ + $(d)/mercurial.sh \ + $(d)/circular.sh \ + $(d)/init.sh \ + $(d)/inputs.sh \ + $(d)/follow-paths.sh \ + $(d)/bundle.sh \ + $(d)/check.sh \ + $(d)/unlocked-override.sh \ + $(d)/absolute-paths.sh \ + $(d)/absolute-attr-paths.sh \ + $(d)/build-paths.sh \ + $(d)/flake-in-submodule.sh \ + $(d)/prefetch.sh \ + $(d)/eval-cache.sh \ + $(d)/search-root.sh \ + $(d)/config.sh \ + $(d)/show.sh + +install-tests-groups += flake diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 797002e92a0..8b4945cac3e 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -1,23 +1,5 @@ nix_tests = \ test-infra.sh \ - flakes/flakes.sh \ - flakes/develop.sh \ - flakes/edit.sh \ - flakes/run.sh \ - flakes/mercurial.sh \ - flakes/circular.sh \ - flakes/init.sh \ - flakes/inputs.sh \ - flakes/follow-paths.sh \ - flakes/bundle.sh \ - flakes/check.sh \ - flakes/unlocked-override.sh \ - flakes/absolute-paths.sh \ - flakes/absolute-attr-paths.sh \ - flakes/build-paths.sh \ - flakes/flake-in-submodule.sh \ - flakes/prefetch.sh \ - flakes/eval-cache.sh \ gc.sh \ nix-collect-garbage-d.sh \ remote-store.sh \ @@ -61,7 +43,6 @@ nix_tests = \ restricted.sh \ fetchGitSubmodules.sh \ fetchGitVerification.sh \ - flakes/search-root.sh \ readfile-context.sh \ nix-channel.sh \ recursive.sh \ @@ -102,7 +83,6 @@ nix_tests = \ nix-copy-ssh-ng.sh \ post-hook.sh \ function-trace.sh \ - flakes/config.sh \ fmt.sh \ eval-store.sh \ why-depends.sh \ @@ -125,7 +105,6 @@ nix_tests = \ store-info.sh \ fetchClosure.sh \ completions.sh \ - flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ path-info.sh \ From dcbe2453f536464a4747c20a057df3ea93c0c400 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jul 2024 22:36:43 -0400 Subject: [PATCH 108/284] Change skipped test error code from 99 to 77 Meson uses a venerable GNU convention described in https://www.gnu.org/software/automake/manual/html_node/Scripts_002dbased-Testsuites.html in which: > When no test protocol is in use, an exit status of 0 from a test > script will denote a success, an exit status of 77 a skipped test, an > exit status of 99 a hard error, and any other exit status will denote > a failure. 77 is thus what we want, not 99. --- mk/run-test.sh | 2 +- tests/functional/common/vars-and-functions.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mk/run-test.sh b/mk/run-test.sh index 543c845e122..7f9f1d5f837 100755 --- a/mk/run-test.sh +++ b/mk/run-test.sh @@ -28,7 +28,7 @@ run_test if [[ "$status" = 0 ]]; then echo "$post_run_msg [${green}PASS$normal]" -elif [[ "$status" = 99 ]]; then +elif [[ "$status" = 77 ]]; then echo "$post_run_msg [${yellow}SKIP$normal]" else echo "$post_run_msg [${red}FAIL$normal]" diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/vars-and-functions.sh index 4316a30d5ce..6a0988f1212 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/vars-and-functions.sh @@ -190,7 +190,7 @@ isDaemonNewer () { skipTest () { echo "$1, skipping this test..." >&2 - exit 99 + exit 77 } TODO_NixOS() { From a2fed6db9e6ec0ce8441d16bec0752b420dd49e9 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 25 Jul 2024 04:53:06 +0200 Subject: [PATCH 109/284] manual: Contributing -> Development, Hacking -> Building (#9014) * manual: Contributing -> Development, Hacking -> Building what's currently called "hacking" are really instructions for setting up a development environment and compiling from source. we have a contribution guide in the repo (which rightly focuses on GitHub workflows), and the material in the manual is more about working on the code itself. since we'd otherwise have three headings that amount to "Building Nix", this change also moves the "classic Nix" instructions to the top. we may want to reorganise this in the future, and bring contributor-oriented information closer to the code, but for now let's stick to more accurate names to ease navigation. --- CONTRIBUTING.md | 8 +- README.md | 7 +- doc/manual/generate-builtins.nix | 2 +- doc/manual/generate-manpage.nix | 2 +- doc/manual/generate-settings.nix | 4 +- doc/manual/generate-store-info.nix | 4 +- doc/manual/generate-xp-features-shortlist.nix | 2 +- doc/manual/local.mk | 8 +- doc/manual/redirects.js | 11 +- doc/manual/src/SUMMARY.md.in | 17 +- doc/manual/src/_redirects | 10 +- doc/manual/src/c-api.md | 2 +- .../src/command-ref/experimental-commands.md | 2 +- .../src/command-ref/nix-store/realise.md | 2 +- .../hacking.md => development/building.md} | 151 +++++------------- .../cli-guideline.md | 0 doc/manual/src/development/contributing.md | 79 +++++++++ .../src/{contributing => development}/cxx.md | 0 .../documentation.md | 2 +- .../experimental-features.md | 0 .../{contributing => development}/index.md | 2 +- .../json-guideline.md | 0 .../{contributing => development}/testing.md | 0 doc/manual/src/glossary.md | 4 +- .../src/installation/installing-binary.md | 2 +- .../src/language/advanced-attributes.md | 10 +- doc/manual/src/protocols/derivation-aterm.md | 4 +- doc/manual/src/protocols/json/derivation.md | 2 +- .../src/protocols/json/store-object-info.md | 2 +- doc/manual/src/release-notes/rl-2.18.md | 4 +- doc/manual/src/release-notes/rl-2.19.md | 6 +- doc/manual/src/release-notes/rl-2.23.md | 4 +- doc/manual/src/release-notes/rl-2.4.md | 2 +- .../file-system-object/content-address.md | 2 +- .../src/store/store-object/content-address.md | 2 +- src/libexpr/eval-settings.hh | 2 +- src/libexpr/primops/fetchTree.cc | 10 +- src/libstore/globals.hh | 6 +- src/libutil/config.hh | 2 +- src/nix/nix.md | 6 +- 40 files changed, 199 insertions(+), 186 deletions(-) rename doc/manual/src/{contributing/hacking.md => development/building.md} (74%) rename doc/manual/src/{contributing => development}/cli-guideline.md (100%) create mode 100644 doc/manual/src/development/contributing.md rename doc/manual/src/{contributing => development}/cxx.md (100%) rename doc/manual/src/{contributing => development}/documentation.md (99%) rename doc/manual/src/{contributing => development}/experimental-features.md (100%) rename doc/manual/src/{contributing => development}/index.md (77%) rename doc/manual/src/{contributing => development}/json-guideline.md (100%) rename doc/manual/src/{contributing => development}/testing.md (100%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 38f5d43b710..12423366a74 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,9 +41,9 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). There are many open pull requests that might already do what you intend to work on. You can use [labels](https://github.com/NixOS/nix/labels) to filter for relevant topics. -3. Check the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html) for information on building Nix and running its tests. +3. Check the [Nix reference manual](https://nix.dev/manual/nix/development/development/building.html) for information on building Nix and running its tests. - For contributions to the command line interface, please check the [CLI guidelines](https://nixos.org/manual/nix/unstable/contributing/cli-guideline.html). + For contributions to the command line interface, please check the [CLI guidelines](https://nix.dev/manual/nix/development/development/cli-guideline.html). 4. Make your change! @@ -69,7 +69,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). - [ ] API documentation in header files - [ ] Code and comments are self-explanatory - [ ] Commit message explains **why** the change was made - - [ ] New feature or incompatible change: [add a release note](https://nixos.org/manual/nix/stable/contributing/hacking#add-a-release-note) + - [ ] New feature or incompatible change: [add a release note](https://nix.dev/manual/nix/development/development/contributing.html#add-a-release-note) 7. If you need additional feedback or help to getting pull request into shape, ask other contributors using [@mentions](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#mentioning-people-and-teams). @@ -78,7 +78,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). The Nix reference manual is hosted on https://nixos.org/manual/nix. The underlying source files are located in [`doc/manual/src`](./doc/manual/src). For small changes you can [use GitHub to edit these files](https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files) -For larger changes see the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html). +For larger changes see the [Nix reference manual](https://nix.dev/manual/nix/development/development/contributing.html). ## Getting help diff --git a/README.md b/README.md index 931a60bba2c..ab647e53b1f 100644 --- a/README.md +++ b/README.md @@ -4,19 +4,18 @@ [![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions) Nix is a powerful package manager for Linux and other Unix systems that makes package -management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual) +management reliable and reproducible. Please refer to the [Nix manual](https://nix.dev/reference/nix-manual) for more details. ## Installation and first steps Visit [nix.dev](https://nix.dev) for [installation instructions](https://nix.dev/tutorials/install-nix) and [beginner tutorials](https://nix.dev/tutorials/first-steps). -Full reference documentation can be found in the [Nix manual](https://nixos.org/nix/manual). +Full reference documentation can be found in the [Nix manual](https://nix.dev/reference/nix-manual). ## Building and developing -See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacking.html) in our manual for instruction on how to - set up a development environment and build Nix from source. +Follow instructions in the Nix reference manual to [set up a development environment and build Nix from source](https://nix.dev/manual/nix/development/development/building.html). ## Contributing diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 13de6c3972c..37ed12a4330 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -12,7 +12,7 @@ let experimentalNotice = optionalString (experimental-feature != null) '' > **Note** > - > This function is only available if the [`${experimental-feature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) is enabled. + > This function is only available if the [`${experimental-feature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimental-feature}) is enabled. > > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 90eaa1a73fe..791bfd2c756 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -38,7 +38,7 @@ let result = '' > **Warning** \ > This program is - > [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and its interface is subject to change. # Name diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 504cda3621e..93a8e093e48 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -33,10 +33,10 @@ let > **Warning** > > This setting is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). + > [experimental feature](@docroot@/development/experimental-features.md). > > To change this setting, make sure the - > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > [`${experimentalFeature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimentalFeature}) > is enabled. > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index c311c3c3934..cc370412414 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -32,10 +32,10 @@ let > **Warning** > > This store is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). + > [experimental feature](@docroot@/development/experimental-features.md). > > To use this store, make sure the - > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > [`${experimentalFeature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimentalFeature}) > is enabled. > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix index ec09f4b75f7..eb735ba5f7a 100644 --- a/doc/manual/generate-xp-features-shortlist.nix +++ b/doc/manual/generate-xp-features-shortlist.nix @@ -4,6 +4,6 @@ with import ; let showExperimentalFeature = name: doc: '' - - [`${name}`](@docroot@/contributing/experimental-features.md#xp-feature-${name}) + - [`${name}`](@docroot@/development/experimental-features.md#xp-feature-${name}) ''; in xps: indent " " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 0cec5288504..fcc50f4605c 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -95,7 +95,7 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/development/experimental-feature-descriptions.md @cp $< $@ @$(call process-includes,$@,$@) @@ -124,7 +124,7 @@ $(d)/conf-file.json: $(doc_nix) $(trace-gen) $(dummy-env) $(doc_nix) config show --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) +$(d)/src/development/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ @@ -207,11 +207,11 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -# the `! -name 'contributing.md'` filter excludes the one place where +# the `! -name 'documentation.md'` filter excludes the one place where # `@docroot@` is to be preserved for documenting the mechanism # FIXME: maybe contributing guides should live right next to the code # instead of in the manual -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/development/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 0f9f91b037f..beef6ef4ab5 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -143,7 +143,7 @@ const redirects = { "opt-timeout": "command-ref/opt-common.html#opt-timeout", "sec-common-options": "command-ref/opt-common.html", "ch-utilities": "command-ref/utilities.html", - "chap-hacking": "contributing/hacking.html", + "chap-hacking": "development/building.html", "adv-attr-allowSubstitutes": "language/advanced-attributes.html#adv-attr-allowSubstitutes", "adv-attr-allowedReferences": "language/advanced-attributes.html#adv-attr-allowedReferences", "adv-attr-allowedRequisites": "language/advanced-attributes.html#adv-attr-allowedRequisites", @@ -350,7 +350,7 @@ const redirects = { "macos": "uninstall.html#macos", "uninstalling": "uninstall.html", }, - "contributing/hacking.html": { + "development/building.html": { "nix-with-flakes": "#building-nix-with-flakes", "classic-nix": "#building-nix", "running-tests": "testing.html#running-tests", @@ -361,7 +361,12 @@ const redirects = { "installer-tests": "testing.html#installer-tests", "one-time-setup": "testing.html#one-time-setup", "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", - "characterization-testing": "#characterisation-testing-unit", + "characterization-testing": "testing.html#characterisation-testing-unit", + "add-a-release-note": "contributing.html#add-a-release-note", + "add-an-entry": "contributing.html#add-an-entry", + "build-process": "contributing.html#build-process", + "reverting": "contributing.html#reverting", + "branches": "contributing.html#branches", }, "glossary.html": { "gloss-local-store": "store/types/local-store.html", diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index b4dd277e306..3918faeb2dd 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -116,14 +116,15 @@ - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [C API](c-api.md) - [Glossary](glossary.md) -- [Contributing](contributing/index.md) - - [Hacking](contributing/hacking.md) - - [Testing](contributing/testing.md) - - [Documentation](contributing/documentation.md) - - [Experimental Features](contributing/experimental-features.md) - - [CLI guideline](contributing/cli-guideline.md) - - [JSON guideline](contributing/json-guideline.md) - - [C++ style guide](contributing/cxx.md) +- [Development](development/index.md) + - [Building](development/building.md) + - [Testing](development/testing.md) + - [Documentation](development/documentation.md) + - [CLI guideline](development/cli-guideline.md) + - [JSON guideline](development/json-guideline.md) + - [C++ style guide](development/cxx.md) + - [Experimental Features](development/experimental-features.md) + - [Contributing](development/contributing.md) - [Releases](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} - [Release 2.23 (2024-06-03)](release-notes/rl-2.23.md) diff --git a/doc/manual/src/_redirects b/doc/manual/src/_redirects index 578c48f06f1..07b3130f9ce 100644 --- a/doc/manual/src/_redirects +++ b/doc/manual/src/_redirects @@ -20,7 +20,15 @@ /command-ref/command-ref /command-ref 301! -/contributing/contributing /contributing 301! +/contributing/contributing /development 301! +/contributing /development 301! +/contributing/hacking /development/building 301! +/contributing/testing /development/testing 301! +/contributing/documentation /development/documentation 301! +/contributing/experimental-features /development/experimental-features 301! +/contributing/cli-guideline /development/cli-guideline 301! +/contributing/json-guideline /development/json-guideline 301! +/contributing/cxx /development/cxx 301! /expressions/expression-language /language/ 301! /expressions/language-constructs /language/constructs 301! diff --git a/doc/manual/src/c-api.md b/doc/manual/src/c-api.md index 29df0b64419..0cdd8383231 100644 --- a/doc/manual/src/c-api.md +++ b/doc/manual/src/c-api.md @@ -10,7 +10,7 @@ See: - [Matrix Room *Nix Bindings*](https://matrix.to/#/#nix-bindings:nixos.org) for discussion and questions. - [Stabilisation Milestone](https://github.com/NixOS/nix/milestone/52) - [Other C API PRs and issues](https://github.com/NixOS/nix/labels/c%20api) -- [Contributing C API Documentation](contributing/documentation.md#c-api-documentation), including how to build it locally. +- [Contributing C API Documentation](development/documentation.md#c-api-documentation), including how to build it locally. [Getting Started]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs [Index]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs/globals.html diff --git a/doc/manual/src/command-ref/experimental-commands.md b/doc/manual/src/command-ref/experimental-commands.md index 286ddc6d634..1190729a230 100644 --- a/doc/manual/src/command-ref/experimental-commands.md +++ b/doc/manual/src/command-ref/experimental-commands.md @@ -1,6 +1,6 @@ # Experimental Commands -This section lists [experimental commands](@docroot@/contributing/experimental-features.md#xp-feature-nix-command). +This section lists [experimental commands](@docroot@/development/experimental-features.md#xp-feature-nix-command). > **Warning** > diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index e30b351a416..a899758dfab 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -32,7 +32,7 @@ If no substitutes are available and no store derivation is given, realisation fa [store objects]: @docroot@/store/store-object.md [closure]: @docroot@/glossary.md#gloss-closure [substituters]: @docroot@/command-ref/conf-file.md#conf-substituters -[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations +[content-addressed derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations [Nix database]: @docroot@/glossary.md#gloss-nix-database The resulting paths are printed on standard output. diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/development/building.md similarity index 74% rename from doc/manual/src/contributing/hacking.md rename to doc/manual/src/development/building.md index c128515e9ba..5a5fb33687f 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/development/building.md @@ -1,112 +1,111 @@ -# Hacking +# Building Nix -This section provides some notes on how to hack on Nix. To get the -latest version of Nix from GitHub: +This section provides some notes on how to start hacking on Nix. +To get the latest version of Nix from GitHub: ```console $ git clone https://github.com/NixOS/nix.git $ cd nix ``` -The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. - -[installation instructions]: ../installation/index.md - -## Building Nix with flakes - -This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. -See the [Building Nix](#building-nix) section for equivalent instructions using stable Nix interfaces. +> **Note** +> +> The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. +> If you don't have it installed, follow the [installation instructions](../installation/index.md). -[`flakes`]: @docroot@/contributing/experimental-features.md#xp-feature-flakes -[`nix-command`]: @docroot@/contributing/experimental-features.md#xp-nix-command To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: ```console -$ nix develop +$ nix-shell ``` -This shell also adds `./outputs/bin/nix` to your `$PATH` so you can run `nix` immediately after building it. - To get a shell with one of the other [supported compilation environments](#compilation-environments): ```console -$ nix develop .#native-clangStdenvPackages +$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages ``` > **Note** > -> Use `ccacheStdenv` to drastically improve rebuild time. +> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. > By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. To build Nix itself in this shell: ```console [nix-shell]$ autoreconfPhase -[nix-shell]$ configurePhase -[nix-shell]$ make -j $NIX_BUILD_CORES OPTIMIZE=0 +[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ make -j $NIX_BUILD_CORES ``` To install it in `$(pwd)/outputs` and test it: ```console -[nix-shell]$ make install OPTIMIZE=0 -[nix-shell]$ make installcheck check -j $NIX_BUILD_CORES -[nix-shell]$ nix --version +[nix-shell]$ make install +[nix-shell]$ make installcheck -j $NIX_BUILD_CORES +[nix-shell]$ ./outputs/out/bin/nix --version nix (Nix) 2.12 ``` -For more information on running and filtering tests, see -[`testing.md`](./testing.md). - To build a release version of Nix for the current operating system and CPU architecture: ```console -$ nix build +$ nix-build ``` You can also build Nix for one of the [supported platforms](#platforms). -## Building Nix +## Building Nix with flakes + +This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. + +[`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes +[`nix-command`]: @docroot@/development/experimental-features.md#xp-nix-command To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: ```console -$ nix-shell +$ nix develop ``` +This shell also adds `./outputs/bin/nix` to your `$PATH` so you can run `nix` immediately after building it. + To get a shell with one of the other [supported compilation environments](#compilation-environments): ```console -$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages +$ nix develop .#native-clangStdenvPackages ``` > **Note** > -> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. +> Use `ccacheStdenv` to drastically improve rebuild time. > By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. To build Nix itself in this shell: ```console [nix-shell]$ autoreconfPhase -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out -[nix-shell]$ make -j $NIX_BUILD_CORES +[nix-shell]$ configurePhase +[nix-shell]$ make -j $NIX_BUILD_CORES OPTIMIZE=0 ``` To install it in `$(pwd)/outputs` and test it: ```console -[nix-shell]$ make install -[nix-shell]$ make installcheck -j $NIX_BUILD_CORES -[nix-shell]$ ./outputs/out/bin/nix --version +[nix-shell]$ make install OPTIMIZE=0 +[nix-shell]$ make installcheck check -j $NIX_BUILD_CORES +[nix-shell]$ nix --version nix (Nix) 2.12 ``` +For more information on running and filtering tests, see +[`testing.md`](./testing.md). + To build a release version of Nix for the current operating system and CPU architecture: ```console -$ nix-build +$ nix build ``` You can also build Nix for one of the [supported platforms](#platforms). @@ -294,81 +293,3 @@ If it fails, run `git add --patch` to approve the suggestions _and commit again_ To refresh pre-commit hook's config file, do the following: 1. Exit the development shell and start it again by running `nix develop`. 2. If you also use the pre-commit hook, also run `pre-commit-hooks-install` again. - -## Add a release note - -`doc/manual/rl-next` contains release notes entries for all unreleased changes. - -User-visible changes should come with a release note. - -### Add an entry - -Here's what a complete entry looks like. The file name is not incorporated in the document. - -``` ---- -synopsis: Basically a title -issues: 1234 -prs: 1238 ---- - -Here's one or more paragraphs that describe the change. - -- It's markdown -- Add references to the manual using @docroot@ -``` - -Significant changes should add the following header, which moves them to the top. - -``` -significance: significant -``` - - -See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). - -### Build process - -Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. - -## Branches - -- [`master`](https://github.com/NixOS/nix/commits/master) - - The main development branch. All changes are approved and merged here. - When developing a change, create a branch based on the latest `master`. - - Maintainers try to [keep it in a release-worthy state](#reverting). - -- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) - - These branches are the subject of backports only, and are - also [kept](#reverting) in a release-worthy state. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) - - The latest patch release of the latest minor version. - - See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) - -- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) - - Generally branches created by the backport action. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [_other_](https://github.com/NixOS/nix/branches/all) - - Branches that do not conform to the above patterns should be feature branches. - -## Reverting - -If a change turns out to be merged by mistake, or contain a regression, it may be reverted. -A revert is not a rejection of the contribution, but merely part of an effective development process. -It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. -If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. -By embracing reverts as a good part of the development process, everyone wins. - -However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/development/cli-guideline.md similarity index 100% rename from doc/manual/src/contributing/cli-guideline.md rename to doc/manual/src/development/cli-guideline.md diff --git a/doc/manual/src/development/contributing.md b/doc/manual/src/development/contributing.md new file mode 100644 index 00000000000..7de7489dcb7 --- /dev/null +++ b/doc/manual/src/development/contributing.md @@ -0,0 +1,79 @@ +# Contributing + +## Add a release note + +`doc/manual/rl-next` contains release notes entries for all unreleased changes. + +User-visible changes should come with a release note. + +### Add an entry + +Here's what a complete entry looks like. The file name is not incorporated in the document. + +``` +--- +synopsis: Basically a title +issues: 1234 +prs: 1238 +--- + +Here's one or more paragraphs that describe the change. + +- It's markdown +- Add references to the manual using @docroot@ +``` + +Significant changes should add the following header, which moves them to the top. + +``` +significance: significant +``` + + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). + +### Build process + +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. + +## Branches + +- [`master`](https://github.com/NixOS/nix/commits/master) + + The main development branch. All changes are approved and merged here. + When developing a change, create a branch based on the latest `master`. + + Maintainers try to [keep it in a release-worthy state](#reverting). + +- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) + + These branches are the subject of backports only, and are + also [kept](#reverting) in a release-worthy state. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) + + The latest patch release of the latest minor version. + + See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) + +- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) + + Generally branches created by the backport action. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [_other_](https://github.com/NixOS/nix/branches/all) + + Branches that do not conform to the above patterns should be feature branches. + +## Reverting + +If a change turns out to be merged by mistake, or contain a regression, it may be reverted. +A revert is not a rejection of the contribution, but merely part of an effective development process. +It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. +If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. +By embracing reverts as a good part of the development process, everyone wins. + +However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/cxx.md b/doc/manual/src/development/cxx.md similarity index 100% rename from doc/manual/src/contributing/cxx.md rename to doc/manual/src/development/cxx.md diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/development/documentation.md similarity index 99% rename from doc/manual/src/contributing/documentation.md rename to doc/manual/src/development/documentation.md index a14ecedd6e6..63f574ab7aa 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/development/documentation.md @@ -24,7 +24,7 @@ nix build .#^doc and open `./result-doc/share/doc/nix/manual/index.html`. -To build the manual incrementally, [enter the development shell](./hacking.md) and run: +To build the manual incrementally, [enter the development shell](./building.md) and run: ```console make manual-html-open -j $NIX_BUILD_CORES diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/development/experimental-features.md similarity index 100% rename from doc/manual/src/contributing/experimental-features.md rename to doc/manual/src/development/experimental-features.md diff --git a/doc/manual/src/contributing/index.md b/doc/manual/src/development/index.md similarity index 77% rename from doc/manual/src/contributing/index.md rename to doc/manual/src/development/index.md index 4d55c17a46f..6403c3e66a4 100644 --- a/doc/manual/src/contributing/index.md +++ b/doc/manual/src/development/index.md @@ -5,4 +5,4 @@ Check the [contributing guide](https://github.com/NixOS/nix/blob/master/CONTRIBU This chapter is a collection of guides for making changes to the code and documentation. -If you're not sure where to start, try to [compile Nix from source](./hacking.md) and consider [making improvements to documentation](./documentation.md). +If you're not sure where to start, try to [compile Nix from source](./building.md) and consider [making improvements to documentation](./documentation.md). diff --git a/doc/manual/src/contributing/json-guideline.md b/doc/manual/src/development/json-guideline.md similarity index 100% rename from doc/manual/src/contributing/json-guideline.md rename to doc/manual/src/development/json-guideline.md diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/development/testing.md similarity index 100% rename from doc/manual/src/contributing/testing.md rename to doc/manual/src/development/testing.md diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index f65ada63a5c..877c4668bfe 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -168,7 +168,7 @@ - [impure derivation]{#gloss-impure-derivation} - [An experimental feature](#@docroot@/contributing/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + [An experimental feature](#@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. - [Nix database]{#gloss-nix-database} @@ -353,7 +353,7 @@ Not yet stabilized functionality guarded by named experimental feature flags. These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). [Nix language]: ./language/index.md diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 385008d8c16..6a168ff3dfd 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -77,7 +77,7 @@ $ su root # Installing from a binary tarball You can also download a binary tarball that contains Nix and all its dependencies: -- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../contributing/hacking.md#platforms) +- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../development/building.md#platforms) - Download and unpack the tarball - Run the installer diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index e916c736040..51b83fc8acc 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -113,7 +113,7 @@ Derivations can declare some infrequently used optional attributes. > `nix-build`. If the [`configurable-impure-env` experimental - feature](@docroot@/contributing/experimental-features.md#xp-feature-configurable-impure-env) + feature](@docroot@/development/experimental-features.md#xp-feature-configurable-impure-env) is enabled, these environment variables can also be controlled through the [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) @@ -226,7 +226,7 @@ Derivations can declare some infrequently used optional attributes. - [`__contentAddressed`]{#adv-attr-__contentAddressed} > **Warning** - > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > This attribute is part of an [experimental feature](@docroot@/development/experimental-features.md). > > To use this attribute, you must enable the > [`ca-derivations`][xp-feature-ca-derivations] experimental feature. @@ -370,6 +370,6 @@ Derivations can declare some infrequently used optional attributes. ensures that the derivation can only be built on a machine with the `kvm` feature. -[xp-feature-ca-derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations -[xp-feature-dynamic-derivations]: @docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations -[xp-feature-git-hashing]: @docroot@/contributing/experimental-features.md#xp-feature-git-hashing +[xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +[xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md index e58b602a349..1ba757ae024 100644 --- a/doc/manual/src/protocols/derivation-aterm.md +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -14,6 +14,6 @@ Derivations are serialised in one of the following formats: DrvWithVersion(, ...) ``` - The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md): + The only `version-string`s that are in use today are for [experimental features](@docroot@/development/experimental-features.md): - - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. diff --git a/doc/manual/src/protocols/json/derivation.md b/doc/manual/src/protocols/json/derivation.md index f881dd70381..2f85340d6c5 100644 --- a/doc/manual/src/protocols/json/derivation.md +++ b/doc/manual/src/protocols/json/derivation.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. The JSON serialization of a diff --git a/doc/manual/src/protocols/json/store-object-info.md b/doc/manual/src/protocols/json/store-object-info.md index 9f647a96c24..6b4f4843711 100644 --- a/doc/manual/src/protocols/json/store-object-info.md +++ b/doc/manual/src/protocols/json/store-object-info.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. Info about a [store object]. diff --git a/doc/manual/src/release-notes/rl-2.18.md b/doc/manual/src/release-notes/rl-2.18.md index 4bbc52b508f..eb26fc9e721 100644 --- a/doc/manual/src/release-notes/rl-2.18.md +++ b/doc/manual/src/release-notes/rl-2.18.md @@ -13,7 +13,7 @@ - The `discard-references` feature has been stabilized. This means that the - [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + [unsafeDiscardReferences](@docroot@/development/experimental-features.md#xp-feature-discard-references) attribute is no longer guarded by an experimental flag and can be used freely. @@ -21,7 +21,7 @@ This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. - A new builtin [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) has been added. - It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + It is part of the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. - Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md index ba6eb9c64a4..e2e2f85cc3c 100644 --- a/doc/manual/src/release-notes/rl-2.19.md +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -17,8 +17,8 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-fetch-tree). - The interface for creating and updating lock files has been overhauled: @@ -33,7 +33,7 @@ - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. They are superceded by `nix flake update`. -- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) (experimental) now returns a JSON map rather than JSON list. diff --git a/doc/manual/src/release-notes/rl-2.23.md b/doc/manual/src/release-notes/rl-2.23.md index 3c59b85838f..ac842fdc058 100644 --- a/doc/manual/src/release-notes/rl-2.23.md +++ b/doc/manual/src/release-notes/rl-2.23.md @@ -14,7 +14,7 @@ - Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) - The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/contributing/cli-guideline.md#returning-future-proof-json). + The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json). In particular, the hash algorithm and content addressing method of content-addresed derivation outputs are now separated into two fields `hashAlgo` and `method`, rather than one field with an arcane `:`-separated format. @@ -89,7 +89,7 @@ This makes records of this sort more self-describing, and easier to consume programmatically. We will follow this design principle going forward; - the [JSON guidelines](@docroot@/contributing/json-guideline.md) in the contributing section have been updated accordingly. + the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. - Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 8b566fc7b93..1201e53b6ae 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -23,7 +23,7 @@ more than 2800 commits from 195 contributors since release 2.3. * The **`nix` command** has seen a lot of work and is now almost at feature parity with the old command-line interface (the `nix-*` commands). It aims to be [more modern, consistent and pleasant to - use](../contributing/cli-guideline.md) than the old CLI. It is still + use](../development/cli-guideline.md) than the old CLI. It is still marked as experimental but its interface should not change much anymore in future releases. diff --git a/doc/manual/src/store/file-system-object/content-address.md b/doc/manual/src/store/file-system-object/content-address.md index 1c63c52ebda..410d7fb7c7c 100644 --- a/doc/manual/src/store/file-system-object/content-address.md +++ b/doc/manual/src/store/file-system-object/content-address.md @@ -82,4 +82,4 @@ In the future, we may support a Git-like hash for such file system objects, or w [file system object]: ../file-system-object.md [store object]: ../store-object.md -[xp-feature-git-hashing]: @docroot@/contributing/experimental-features.md#xp-feature-git-hashing +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-object/content-address.md b/doc/manual/src/store/store-object/content-address.md index f6f982035ef..02dce283650 100644 --- a/doc/manual/src/store/store-object/content-address.md +++ b/doc/manual/src/store/store-object/content-address.md @@ -92,4 +92,4 @@ becomes more widespread, this restriction will be revisited. [fso-ca]: ../file-system-object/content-address.md [sp-spec]: @docroot@/protocols/store-path.md -[xp-feature-git-hashing]: @docroot@/contributing/experimental-features.md#xp-feature-git-hashing +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 8f48b53a566..30a8c5c588a 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -65,7 +65,7 @@ struct EvalSettings : Config extern "C" typedef void (*ValueInitialiser) (EvalState & state, Value & v); ``` - The [Nix C++ API documentation](@docroot@/contributing/documentation.md#api-documentation) has more details on evaluator internals. + The [Nix C++ API documentation](@docroot@/development/documentation.md#api-documentation) has more details on evaluator internals. - `builtins.exec` *arguments* diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 333e486fd8e..e59d7fe6799 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -383,7 +383,7 @@ static RegisterPrimOp primop_fetchTree({ - `"mercurial"` *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references). - The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. + The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled. > **Example** > @@ -670,12 +670,12 @@ static RegisterPrimOp primop_fetchGit({ Whether to check `rev` for a signature matching `publicKey` or `publicKeys`. If `verifyCommit` is enabled, then `fetchGit` cannot use a local repository with uncommitted changes. - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - `publicKey` The public key against which `rev` is verified if `verifyCommit` is enabled. - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - `keytype` (default: `"ssh-ed25519"`) @@ -687,7 +687,7 @@ static RegisterPrimOp primop_fetchGit({ - `"ssh-ed25519"` - `"ssh-ed25519-sk"` - `"ssh-rsa"` - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - `publicKeys` @@ -701,7 +701,7 @@ static RegisterPrimOp primop_fetchGit({ } ``` - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). Here are some examples of how to use `fetchGit`. diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 30d7537bd52..8760c9d145b 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -286,7 +286,7 @@ public: For backward compatibility, `ssh://` may be omitted. The hostname may be an alias defined in `~/.ssh/config`. - 2. A comma-separated list of [Nix system types](@docroot@/contributing/hacking.md#system-type). + 2. A comma-separated list of [Nix system types](@docroot@/development/building.md#system-type). If omitted, this defaults to the local platform type. > **Example** @@ -866,13 +866,13 @@ public: - `ca-derivations` - Included by default if the [`ca-derivations` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) is enabled. + Included by default if the [`ca-derivations` experimental feature](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) is enabled. This system feature is implicitly required by derivations with the [`__contentAddressed` attribute](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed). - `recursive-nix` - Included by default if the [`recursive-nix` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-recursive-nix) is enabled. + Included by default if the [`recursive-nix` experimental feature](@docroot@/development/experimental-features.md#xp-feature-recursive-nix) is enabled. - `uid-range` diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 1952ba1b8d7..c0c59ac68f5 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -393,7 +393,7 @@ struct ExperimentalFeatureSettings : Config { {{#include experimental-features-shortlist.md}} - Experimental features are [further documented in the manual](@docroot@/contributing/experimental-features.md). + Experimental features are [further documented in the manual](@docroot@/development/experimental-features.md). )"}; /** diff --git a/src/nix/nix.md b/src/nix/nix.md index 4464bef370c..f958ce09acc 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -50,7 +50,7 @@ manual](https://nixos.org/manual/nix/stable/). > **Warning** \ > Installables are part of the unstable -> [`nix-command` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-nix-command), +> [`nix-command` experimental feature](@docroot@/development/experimental-features.md#xp-feature-nix-command), > and subject to change without notice. Many `nix` subcommands operate on one or more *installables*. @@ -70,9 +70,9 @@ That is, Nix will operate on the default flake output attribute of the flake in > **Warning** \ > Flake output attribute installables depend on both the -> [`flakes`](@docroot@/contributing/experimental-features.md#xp-feature-flakes) +> [`flakes`](@docroot@/development/experimental-features.md#xp-feature-flakes) > and -> [`nix-command`](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [`nix-command`](@docroot@/development/experimental-features.md#xp-feature-nix-command) > experimental features, and subject to change without notice. Example: `nixpkgs#hello` From 3b49f7a1439bc75cefddb0fe0031824f3708da83 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jul 2024 23:12:39 -0400 Subject: [PATCH 110/284] Deduplicate our many `package.nix` a bit (#11175) - They should all be built in parallel - They should all use strict deps by default --- packaging/dependencies.nix | 27 ++++++++++++++++++------- src/external-api-docs/package.nix | 4 ---- src/internal-api-docs/package.nix | 4 ---- src/libcmd/package.nix | 6 ------ src/libexpr-c/package.nix | 4 ---- src/libexpr/package.nix | 4 ---- src/libfetchers/package.nix | 6 ------ src/libflake/package.nix | 6 ------ src/libmain-c/package.nix | 4 ---- src/libmain/package.nix | 6 ------ src/libstore-c/package.nix | 4 ---- src/libstore/package.nix | 4 ---- src/libutil-c/package.nix | 4 ---- src/libutil/package.nix | 4 ---- src/nix/package.nix | 6 ------ src/perl/package.nix | 2 +- tests/unit/libexpr-support/package.nix | 4 ---- tests/unit/libexpr/package.nix | 4 ---- tests/unit/libfetchers/package.nix | 4 ---- tests/unit/libflake/package.nix | 4 ---- tests/unit/libstore-support/package.nix | 4 ---- tests/unit/libstore/package.nix | 4 ---- tests/unit/libutil-support/package.nix | 4 ---- tests/unit/libutil/package.nix | 4 ---- 24 files changed, 21 insertions(+), 106 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index f09ca5d1835..b77e1d14f38 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -63,9 +63,16 @@ let # Work around weird `--as-needed` linker behavior with BSD, see # https://github.com/mesonbuild/meson/issues/3593 - bsdNoLinkAsNeeded = finalAttrs: prevAttrs: lib.optionalAttrs stdenv.hostPlatform.isBSD { - mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or []; - }; + bsdNoLinkAsNeeded = finalAttrs: prevAttrs: + lib.optionalAttrs stdenv.hostPlatform.isBSD { + mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or []; + }; + + miscGoodPractice = finalAttrs: prevAttrs: + { + strictDeps = prevAttrs.strictDeps or true; + enableParallelBuilding = true; + }; in scope: { @@ -136,8 +143,14 @@ scope: { inherit resolvePath filesetToSource; - mkMesonDerivation = f: stdenv.mkDerivation - (lib.extends - (lib.composeExtensions bsdNoLinkAsNeeded localSourceLayer) - f); + mkMesonDerivation = f: let + exts = [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + ]; + in stdenv.mkDerivation + (lib.extends + (lib.foldr lib.composeExtensions (_: _: {}) exts) + f); } diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix index da136bbe1fc..743b3e9b758 100644 --- a/src/external-api-docs/package.nix +++ b/src/external-api-docs/package.nix @@ -53,10 +53,6 @@ mkMesonDerivation (finalAttrs: { echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products ''; - enableParallelBuilding = true; - - strictDeps = true; - meta = { platforms = lib.platforms.all; }; diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix index f2077dcafa4..07ca6d4d945 100644 --- a/src/internal-api-docs/package.nix +++ b/src/internal-api-docs/package.nix @@ -48,10 +48,6 @@ mkMesonDerivation (finalAttrs: { echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products ''; - enableParallelBuilding = true; - - strictDeps = true; - meta = { platforms = lib.platforms.all; }; diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix index ec3aa46600d..cde4949013e 100644 --- a/src/libcmd/package.nix +++ b/src/libcmd/package.nix @@ -93,14 +93,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix index 0b895437b6a..eb42195a438 100644 --- a/src/libexpr-c/package.nix +++ b/src/libexpr-c/package.nix @@ -63,12 +63,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index 704456c96cd..4d10079ff64 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -102,12 +102,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index b4abb144b50..9b5d8bff7a6 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -67,14 +67,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libflake/package.nix b/src/libflake/package.nix index af6f5da9416..851adf07e32 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -67,14 +67,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix index 478e34a856a..ce6f673001a 100644 --- a/src/libmain-c/package.nix +++ b/src/libmain-c/package.nix @@ -68,12 +68,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libmain/package.nix b/src/libmain/package.nix index bbd97ec3e58..47513dbdc65 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -62,14 +62,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index fc34c1bda72..e4f37223665 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -64,12 +64,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libstore/package.nix b/src/libstore/package.nix index 0a2ace91e40..02ff4194a8f 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -101,12 +101,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix index 53451998dca..ccfafd4d300 100644 --- a/src/libutil-c/package.nix +++ b/src/libutil-c/package.nix @@ -62,12 +62,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 28d7d8f0edb..4ce1a75b04c 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -88,12 +88,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/nix/package.nix b/src/nix/package.nix index fe83c6969d3..ef7265458fb 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -84,8 +84,6 @@ mkMesonDerivation (finalAttrs: { ] ); - outputs = [ "out" "dev" ]; - nativeBuildInputs = [ meson ninja @@ -114,12 +112,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/perl/package.nix b/src/perl/package.nix index 26856e631a1..0b9343fba06 100644 --- a/src/perl/package.nix +++ b/src/perl/package.nix @@ -73,5 +73,5 @@ perl.pkgs.toPerlModule (mkMesonDerivation (finalAttrs: { "--print-errorlogs" ]; - enableParallelBuilding = true; + strictDeps = false; })) diff --git a/tests/unit/libexpr-support/package.nix b/tests/unit/libexpr-support/package.nix index 0c966c55af1..f53aa842f64 100644 --- a/tests/unit/libexpr-support/package.nix +++ b/tests/unit/libexpr-support/package.nix @@ -66,12 +66,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/tests/unit/libexpr/package.nix b/tests/unit/libexpr/package.nix index 015e3fbc6b4..e70ed783682 100644 --- a/tests/unit/libexpr/package.nix +++ b/tests/unit/libexpr/package.nix @@ -71,12 +71,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { diff --git a/tests/unit/libfetchers/package.nix b/tests/unit/libfetchers/package.nix index cf75f68e50f..ad512f5624b 100644 --- a/tests/unit/libfetchers/package.nix +++ b/tests/unit/libfetchers/package.nix @@ -69,12 +69,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { diff --git a/tests/unit/libflake/package.nix b/tests/unit/libflake/package.nix index d2c9fdb89a4..0d63d2ff7d3 100644 --- a/tests/unit/libflake/package.nix +++ b/tests/unit/libflake/package.nix @@ -69,12 +69,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { diff --git a/tests/unit/libstore-support/package.nix b/tests/unit/libstore-support/package.nix index cb15cdd5f2f..f512db3eeb2 100644 --- a/tests/unit/libstore-support/package.nix +++ b/tests/unit/libstore-support/package.nix @@ -66,12 +66,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/tests/unit/libstore/package.nix b/tests/unit/libstore/package.nix index 39bf7758587..7560a5b7949 100644 --- a/tests/unit/libstore/package.nix +++ b/tests/unit/libstore/package.nix @@ -73,12 +73,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { diff --git a/tests/unit/libutil-support/package.nix b/tests/unit/libutil-support/package.nix index fdecdec7226..1665804cba3 100644 --- a/tests/unit/libutil-support/package.nix +++ b/tests/unit/libutil-support/package.nix @@ -64,12 +64,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/tests/unit/libutil/package.nix b/tests/unit/libutil/package.nix index c7827e74ff9..2fce5bfa846 100644 --- a/tests/unit/libutil/package.nix +++ b/tests/unit/libutil/package.nix @@ -70,12 +70,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { From 8a7e31362ad0b232f4de098573370f6994834397 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 25 Jul 2024 05:57:06 +0200 Subject: [PATCH 111/284] rl-next: Add credit --- doc/manual/rl-next/drop-vendored-toml11.md | 2 ++ doc/manual/rl-next/harden-user-sandboxing.md | 5 +++++ doc/manual/rl-next/nix-shell-looks-for-shell-nix.md | 2 ++ doc/manual/rl-next/repl-doc-renders-doc-comments.md | 4 +++- doc/manual/rl-next/shebang-relative.md | 2 ++ 5 files changed, 14 insertions(+), 1 deletion(-) diff --git a/doc/manual/rl-next/drop-vendored-toml11.md b/doc/manual/rl-next/drop-vendored-toml11.md index d1feeb7031a..8dd786c4452 100644 --- a/doc/manual/rl-next/drop-vendored-toml11.md +++ b/doc/manual/rl-next/drop-vendored-toml11.md @@ -4,3 +4,5 @@ synopsis: Stop vendoring toml11 We don't apply any patches to it, and vendoring it locks users into bugs (it hasn't been updated since its introduction in late 2021). + +Author: [**Winter (@winterqt)**](https://github.com/winterqt) diff --git a/doc/manual/rl-next/harden-user-sandboxing.md b/doc/manual/rl-next/harden-user-sandboxing.md index a647acf2540..ff81c9cb174 100644 --- a/doc/manual/rl-next/harden-user-sandboxing.md +++ b/doc/manual/rl-next/harden-user-sandboxing.md @@ -5,3 +5,8 @@ issues: --- The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. + +This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531), that was handled through the GitHub Security Advisories interface, and hence was merged directly in commit [2dd7f8f42](https://github.com/NixOS/nix/commit/2dd7f8f42da374d9fee4d424c1c6f82bcb36b393) instead of a PR. + +Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) +Co-authors: [**@edolstra**](https://github.com/edolstra) diff --git a/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md b/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md index 99be4148bf1..b9e4b3fb3ef 100644 --- a/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md +++ b/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md @@ -26,3 +26,5 @@ This also applies to `nix-shell` shebang scripts. Consider the following example This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. + +Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) diff --git a/doc/manual/rl-next/repl-doc-renders-doc-comments.md b/doc/manual/rl-next/repl-doc-renders-doc-comments.md index 05023697c96..fa241ebc187 100644 --- a/doc/manual/rl-next/repl-doc-renders-doc-comments.md +++ b/doc/manual/rl-next/repl-doc-renders-doc-comments.md @@ -48,6 +48,8 @@ Known limitations: - It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. - Some extensions to markdown are not yet supported, as you can see in the example above. -We'd like to acknowledge Yingchi Long for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as @sternenseemann and Johannes Kirschbauer for their contributions, proposals, and their work on [RFC 145]. +We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. + +Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) [RFC 145]: https://github.com/NixOS/rfcs/pull/145 diff --git a/doc/manual/rl-next/shebang-relative.md b/doc/manual/rl-next/shebang-relative.md index d12c0f8dce2..dd96bf20375 100644 --- a/doc/manual/rl-next/shebang-relative.md +++ b/doc/manual/rl-next/shebang-relative.md @@ -60,3 +60,5 @@ Example: #!nix -c bash hello ``` + +Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) From 7275d68d3b28115e2e2096aef7e6025292d4d58e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 25 Jul 2024 05:57:53 +0200 Subject: [PATCH 112/284] rl-next: Add top 10 by +1 reactions on PRs We should use a metric that weighs the related issues. Counterbalancing time doesn't make much sense to me. If it's around for longer, the fix will be relevant to more people. --- .../10564-attrcursor-remove-forceerrors.md | 9 ++++++ ...03-run-the-flake-regressions-test-suite.md | 8 +++++ ...unit-prefixes-in-configuration-settings.md | 10 ++++++ ...ild-show-all-fod-errors-with-keep-going.md | 10 ++++++ doc/manual/rl-next/10855-meson.md | 31 +++++++++++++++++++ .../11086-eval-cache-fix-cache-regressions.md | 14 +++++++++ .../rl-next/9063-introduce-libnixflake.md | 12 +++++++ 7 files changed, 94 insertions(+) create mode 100644 doc/manual/rl-next/10564-attrcursor-remove-forceerrors.md create mode 100644 doc/manual/rl-next/10603-run-the-flake-regressions-test-suite.md create mode 100644 doc/manual/rl-next/10668-support-unit-prefixes-in-configuration-settings.md create mode 100644 doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md create mode 100644 doc/manual/rl-next/10855-meson.md create mode 100644 doc/manual/rl-next/11086-eval-cache-fix-cache-regressions.md create mode 100644 doc/manual/rl-next/9063-introduce-libnixflake.md diff --git a/doc/manual/rl-next/10564-attrcursor-remove-forceerrors.md b/doc/manual/rl-next/10564-attrcursor-remove-forceerrors.md new file mode 100644 index 00000000000..864a55b51f7 --- /dev/null +++ b/doc/manual/rl-next/10564-attrcursor-remove-forceerrors.md @@ -0,0 +1,9 @@ +--- +synopsis: "Solve `cached failure of attribute X`" +prs: 10564 +issues: 10513 9165 +--- + +This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. + +Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) diff --git a/doc/manual/rl-next/10603-run-the-flake-regressions-test-suite.md b/doc/manual/rl-next/10603-run-the-flake-regressions-test-suite.md new file mode 100644 index 00000000000..42864323c06 --- /dev/null +++ b/doc/manual/rl-next/10603-run-the-flake-regressions-test-suite.md @@ -0,0 +1,8 @@ +--- +synopsis: "Run the flake regressions test suite" +prs: 10603 +--- + +This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. + +Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) diff --git a/doc/manual/rl-next/10668-support-unit-prefixes-in-configuration-settings.md b/doc/manual/rl-next/10668-support-unit-prefixes-in-configuration-settings.md new file mode 100644 index 00000000000..2caca9a815a --- /dev/null +++ b/doc/manual/rl-next/10668-support-unit-prefixes-in-configuration-settings.md @@ -0,0 +1,10 @@ +--- +synopsis: "Support unit prefixes in configuration settings" +prs: 10668 +--- + +Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + +This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). + +Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) diff --git a/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md b/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md new file mode 100644 index 00000000000..5c2797be843 --- /dev/null +++ b/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md @@ -0,0 +1,10 @@ +--- +synopsis: "nix3-build: show all FOD errors with `--keep-going`" +prs: 10734 +--- + +The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior of `nix build`. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. + +Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92) + +[`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) diff --git a/doc/manual/rl-next/10855-meson.md b/doc/manual/rl-next/10855-meson.md new file mode 100644 index 00000000000..0ab71390f7f --- /dev/null +++ b/doc/manual/rl-next/10855-meson.md @@ -0,0 +1,31 @@ +--- +synopsis: "Build with Meson" +prs: +- 10378 +- 10855 +- 10904 +- 10908 +- 10914 +- 10933 +- 10936 +- 10954 +- 10955 +- 10967 +- 10963 +- 10973 +- 11034 +- 11054 +- 11055 +- 11064 +- 11060 +- 11155 +issues: +- 2503 +--- + +These changes aim to replace the use of autotools and make with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + +Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). + +Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) +Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) diff --git a/doc/manual/rl-next/11086-eval-cache-fix-cache-regressions.md b/doc/manual/rl-next/11086-eval-cache-fix-cache-regressions.md new file mode 100644 index 00000000000..8a348a9adcc --- /dev/null +++ b/doc/manual/rl-next/11086-eval-cache-fix-cache-regressions.md @@ -0,0 +1,14 @@ +--- +synopsis: "Eval cache: fix cache regressions" +prs: 11086 +issues: 10570 +--- + +This update addresses two bugs in the evaluation cache system: + +1. Regression in #10570: The evaluation cache was not being persisted in `nix develop` because `evalCaches` retained references to the caches and was never freed. +2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. + +These bug fixes ensure that the evaluation cache is correctly managed and errors are appropriately handled. + +Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) diff --git a/doc/manual/rl-next/9063-introduce-libnixflake.md b/doc/manual/rl-next/9063-introduce-libnixflake.md new file mode 100644 index 00000000000..fd3645446c4 --- /dev/null +++ b/doc/manual/rl-next/9063-introduce-libnixflake.md @@ -0,0 +1,12 @@ +--- +synopsis: "Introduce `libnixflake`" +prs: 9063 +--- + +A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. + +See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. + +This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. + +Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) From e92dd06a7b5ff00e8908e9a7b5699de56d55e8d6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jul 2024 00:00:52 -0400 Subject: [PATCH 113/284] build-remote: Cope with long store URLs by falling back on hashing I hit this in the Meson port of the functional tests, because the use of standalone build directories. --- src/build-remote/build-remote.cc | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index a0a404e575a..8482b742d31 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -264,7 +264,20 @@ static int main_build_remote(int argc, char * * argv) auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri.render()) + ".upload-lock", true); + AutoCloseFD uploadLock; + { + auto setUpdateLock = [&](auto && fileName){ + uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true); + }; + try { + setUpdateLock(storeUri.render()); + } catch (SysError & e) { + if (e.errNo != ENAMETOOLONG) throw; + // Try again hashing the store URL so we have a shorter path + auto h = hashString(HashAlgorithm::MD5, storeUri.render()); + setUpdateLock(h.to_string(HashFormat::Base64, false)); + } + } { Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri.render())); From b711fcbef9701ae1fd11839aa911e7737f8a23cd Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 25 Jul 2024 06:00:59 +0200 Subject: [PATCH 114/284] rl-next: Drop zzz-other. Number soup. --- doc/manual/rl-next/zzz-other.md | 50 --------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 doc/manual/rl-next/zzz-other.md diff --git a/doc/manual/rl-next/zzz-other.md b/doc/manual/rl-next/zzz-other.md deleted file mode 100644 index f3721bd383e..00000000000 --- a/doc/manual/rl-next/zzz-other.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -synopsis: Other changes ---- - -- [#9063](https://github.com/NixOS/nix/pull/9063): introduce `libnixflake` and move flakes-specific code there (Nix is internally composed of a set of libraries, and this better reflects the architecture wrt flakes) -- [#10852](https://github.com/NixOS/nix/pull/10852): make Nix commands respond better to interruption - Ctrl+C in the terminal -- [#10853](https://github.com/NixOS/nix/pull/10853): fix docs of `builtins.importNative` -- [#10858](https://github.com/NixOS/nix/pull/10858): `flake check`: Recognize well known `homeModule`/`homeModules` attribute -- [#10865](https://github.com/NixOS/nix/pull/10865): Update dependencies to Nixpkgs 24.05 (when using the `nix` flake), and support bdwgc 8.2.6 ([#11141](https://github.com/NixOS/nix/issues/11141), [#10880](https://github.com/NixOS/nix/pull/10880)) -- [#10883](https://github.com/NixOS/nix/pull/10883): Use `TMP` instead of `XDG_RUNTIME_DIR` -- [#10994](https://github.com/NixOS/nix/pull/10994): fix minor bug in elided item counts when printing values lazily -- [#10988](https://github.com/NixOS/nix/pull/10988): restore `commit-lockfile-summary` alias -- [#10941](https://github.com/NixOS/nix/pull/10941): invalid derivation name now causes an actionable error message -- Changes to the interaction between Nix's I/O coroutines and the garbage collector: -- [#10878](https://github.com/NixOS/nix/pull/10878): allow `ipc-sysv*` in the Darwin build sandbox -- [#11031](https://github.com/NixOS/nix/pull/11031): fix Darwin build sandbox -- [#10907](https://github.com/NixOS/nix/pull/10907): use opaque struct instead of `void *` in the C API -- [#10947](https://github.com/NixOS/nix/issues/10947): fix evaluation cache accidentally persisting disallowed IFD errors -- [#11020](https://github.com/NixOS/nix/pull/11020): enable fetch and eval caching for tarballs -- [#11041](https://github.com/NixOS/nix/pull/11041): add discovered attribute paths to the `--show-trace` error trace in `nix-build`, `nix-env`, OfBorg, and other callers of `getDerivations` -- [#11056](https://github.com/NixOS/nix/pull/11056): `s3` store now uses system defined proxy settings -- [#11077](https://github.com/NixOS/nix/pull/11077): support hardlinks in tarballs -- [#11100](https://github.com/NixOS/nix/pull/11100): pretty print values consistently regardless of prior thunk state -- [#11086](https://github.com/NixOS/nix/pull/11086): fix loss of evaluation cache additions in `nix env run`, `nix shell`, `nix develop`, and `nix fmt` -- [#11149](https://github.com/NixOS/nix/pull/11149): report GC time and number of GC cycles in `NIX_SHOW_STATS=1` report -- [#11142](https://github.com/NixOS/nix/pull/11142): aliased options can now also be passed as flags, just like their "normal" counterparts, e.g. `--build-max-jobs` now works -- [#11043](https://github.com/NixOS/nix/pull/11043): `assert a == b; e` now reports some detail about why `a` and `b` are different when they are -- [#11159](https://github.com/NixOS/nix/pull/11159): don't crash a nix-daemon worker process when the client disconnects -- Stability improvements and fixes [#10861](https://github.com/NixOS/nix/pull/10861), [#10865](https://github.com/NixOS/nix/pull/10865), [#10918](https://github.com/NixOS/nix/pull/10918), [#10916](https://github.com/NixOS/nix/pull/10916), [#10884](https://github.com/NixOS/nix/pull/10884), [#10943](https://github.com/NixOS/nix/pull/10943), [#11019](https://github.com/NixOS/nix/pull/11019), [#11122](https://github.com/NixOS/nix/pull/11122), [#11117](https://github.com/NixOS/nix/pull/11117) -- User documentation improvements [#10888](https://github.com/NixOS/nix/pull/10888), [#10966](https://github.com/NixOS/nix/pull/10966), [#10974](https://github.com/NixOS/nix/pull/10974), [#10997](https://github.com/NixOS/nix/pull/10997), [#11013](https://github.com/NixOS/nix/pull/11013), [#11059](https://github.com/NixOS/nix/pull/11059), [#11119](https://github.com/NixOS/nix/pull/11119), [#11116](https://github.com/NixOS/nix/pull/11116), [#11061](https://github.com/NixOS/nix/pull/11061), [#11102](https://github.com/NixOS/nix/pull/11102) -- BSD support: [#10896](https://github.com/NixOS/nix/pull/10896) [#11022](https://github.com/NixOS/nix/pull/11022) [#11156](https://github.com/NixOS/nix/pull/11156) -- Windows support: [#10769](https://github.com/NixOS/nix/pull/10769), [#10975](https://github.com/NixOS/nix/pull/10975) [#11153](https://github.com/NixOS/nix/pull/11153) -- Portability: [#7048](https://github.com/NixOS/nix/pull/7048) [#11090](https://github.com/NixOS/nix/pull/11090) -- Installer improvements [#10902](https://github.com/NixOS/nix/pull/10902) -- Performance improvements [#10853](https://github.com/NixOS/nix/pull/10853), [#10854](https://github.com/NixOS/nix/pull/10854), [#11082](https://github.com/NixOS/nix/pull/11082), [#11092](https://github.com/NixOS/nix/pull/11092), [#11113](https://github.com/NixOS/nix/pull/11113) - -Contributor experience improvements: - -Use Meson to build Nix (nearing completion) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10967](https://github.com/NixOS/nix/pull/10967) [#10963](https://github.com/NixOS/nix/pull/10963) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11064](https://github.com/NixOS/nix/pull/11064) [#11060](https://github.com/NixOS/nix/pull/11060) [#11155](https://github.com/NixOS/nix/pull/11155) -- Testing improvements [#10864](https://github.com/NixOS/nix/pull/10864), [#10903](https://github.com/NixOS/nix/pull/10903), [#10874](https://github.com/NixOS/nix/pull/10874), [#10922](https://github.com/NixOS/nix/pull/10922), [#11006](https://github.com/NixOS/nix/pull/11006), [#11110](https://github.com/NixOS/nix/pull/11110), [#10931](https://github.com/NixOS/nix/pull/10931), [#11123](https://github.com/NixOS/nix/pull/11123) - - [#10603](https://github.com/NixOS/nix/pull/10603): We now evaluate a set of flakes in CI - - [#10922](https://github.com/NixOS/nix/pull/10922): The functional test suite is now run in both in the build sandbox and in a NixOS environment -- CI improvements [#10929](https://github.com/NixOS/nix/pull/10929) [#10999](https://github.com/NixOS/nix/pull/10999) [#11009](https://github.com/NixOS/nix/pull/11009) [#11065](https://github.com/NixOS/nix/pull/11065) [#11071](https://github.com/NixOS/nix/pull/11071) -- Contributor documentation improvements [#10869](https://github.com/NixOS/nix/pull/10869), [#9871](https://github.com/NixOS/nix/pull/9871), [#10960](https://github.com/NixOS/nix/pull/10960), [#11147](https://github.com/NixOS/nix/pull/11147) -- Error message improvements: [#11050](https://github.com/NixOS/nix/pull/11050) [#11154](https://github.com/NixOS/nix/pull/11154) -- Cleaning up the Settings system (`nix.conf` and related architectural cleanups): [#10913](https://github.com/NixOS/nix/pull/10913), [#10951](https://github.com/NixOS/nix/pull/10951), [#11007](https://github.com/NixOS/nix/pull/11007), [#11108](https://github.com/NixOS/nix/pull/11108), [#11014](https://github.com/NixOS/nix/pull/11014), [#11109](https://github.com/NixOS/nix/pull/11109), [#11112](https://github.com/NixOS/nix/pull/11112) -- Other cleanups and refactors [#10857](https://github.com/NixOS/nix/pull/10857) [#10935](https://github.com/NixOS/nix/pull/10935) [#10873](https://github.com/NixOS/nix/pull/10873) [#10745](https://github.com/NixOS/nix/pull/10745) [#10961](https://github.com/NixOS/nix/pull/10961) [#10962](https://github.com/NixOS/nix/pull/10962) [#10972](https://github.com/NixOS/nix/pull/10972) [#11018](https://github.com/NixOS/nix/pull/11018) [#11035](https://github.com/NixOS/nix/pull/11035) [#11037](https://github.com/NixOS/nix/pull/11037) [#11081](https://github.com/NixOS/nix/pull/11081) [#11089](https://github.com/NixOS/nix/pull/11089) [#11093](https://github.com/NixOS/nix/pull/11093) [#11114](https://github.com/NixOS/nix/pull/11114) [#11103](https://github.com/NixOS/nix/pull/11103) [#11126](https://github.com/NixOS/nix/pull/11126) [#11125](https://github.com/NixOS/nix/pull/11125) [#11120](https://github.com/NixOS/nix/pull/11120) -- Scheduler/builder refactoring [#11005](https://github.com/NixOS/nix/pull/11005) -- [#11011](https://github.com/NixOS/nix/pull/11011): enable `-Werror=unused-result` - From 1ae573831756ac08fd30eb9cfd90692b1c06ed3b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jul 2024 00:02:43 -0400 Subject: [PATCH 115/284] Fix some warnings I think they came from the last Nixpkgs bump. --- src/libexpr/primops/fromTOML.cc | 4 ++++ src/libstore/s3-binary-cache-store.cc | 3 +++ src/libstore/s3.hh | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 6c7d303e85a..70755f9e0b5 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -2,7 +2,11 @@ #include "eval-inline.hh" #include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" #include +#pragma GCC diagnostic pop namespace nix { diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 92ab47cd66d..1082657bb0e 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -10,6 +10,8 @@ #include "compression.hh" #include "filetransfer.hh" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" #include #include #include @@ -25,6 +27,7 @@ #include #include #include +#pragma GCC diagnostic pop using namespace Aws::Transfer; diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index f0aeb3bedd2..18de115aeb1 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -8,7 +8,7 @@ #include #include -namespace Aws { namespace Client { class ClientConfiguration; } } +namespace Aws { namespace Client { struct ClientConfiguration; } } namespace Aws { namespace S3 { class S3Client; } } namespace nix { From 6c38bc09526eab2ad66083e339f0c10cf7ce813a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 25 Jul 2024 07:34:08 +0200 Subject: [PATCH 116/284] {src/perl,build-utils-meson/diagnostics}: sort cflags This makes them easier to copy between places. --- build-utils-meson/diagnostics/meson.build | 6 +++--- src/perl/meson.build | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/build-utils-meson/diagnostics/meson.build b/build-utils-meson/diagnostics/meson.build index 2b79f6566dc..4548f93adce 100644 --- a/build-utils-meson/diagnostics/meson.build +++ b/build-utils-meson/diagnostics/meson.build @@ -1,11 +1,11 @@ add_project_arguments( - '-Wno-deprecated-declarations', - '-Wimplicit-fallthrough', + '-Wdeprecated-copy', '-Werror=switch', '-Werror=switch-enum', '-Werror=unused-result', - '-Wdeprecated-copy', '-Wignored-qualifiers', + '-Wimplicit-fallthrough', + '-Wno-deprecated-declarations', # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked # at ~1% overhead in `nix search`. # diff --git a/src/perl/meson.build b/src/perl/meson.build index 5fe7e1e28fa..02e0e68e5ad 100644 --- a/src/perl/meson.build +++ b/src/perl/meson.build @@ -23,21 +23,21 @@ nix_perl_conf.set('PACKAGE_VERSION', meson.project_version()) # set error arguments #------------------------------------------------- error_args = [ - '-Werror=unused-result', '-Wdeprecated-copy', '-Wdeprecated-declarations', + '-Werror=unused-result', '-Wignored-qualifiers', - '-Wno-pedantic', - '-Wno-non-virtual-dtor', - '-Wno-unused-parameter', - '-Wno-variadic-macros', + '-Wno-duplicate-decl-specifier', + '-Wno-literal-suffix', '-Wno-missing-field-initializers', + '-Wno-non-virtual-dtor', + '-Wno-pedantic', + '-Wno-pointer-bool-conversion', + '-Wno-reserved-user-defined-literal', '-Wno-unknown-warning-option', + '-Wno-unused-parameter', '-Wno-unused-variable', - '-Wno-literal-suffix', - '-Wno-reserved-user-defined-literal', - '-Wno-duplicate-decl-specifier', - '-Wno-pointer-bool-conversion', + '-Wno-variadic-macros', ] add_project_arguments( From 2c07ea8abbb2804b476b9df21a9c7c3c2e8dbb11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 25 Jul 2024 07:40:31 +0200 Subject: [PATCH 117/284] build-utils-meson: remove oudated meson build comment --- build-utils-meson/diagnostics/meson.build | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-utils-meson/diagnostics/meson.build b/build-utils-meson/diagnostics/meson.build index 4548f93adce..e81f19eff32 100644 --- a/build-utils-meson/diagnostics/meson.build +++ b/build-utils-meson/diagnostics/meson.build @@ -6,8 +6,5 @@ add_project_arguments( '-Wignored-qualifiers', '-Wimplicit-fallthrough', '-Wno-deprecated-declarations', - # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked - # at ~1% overhead in `nix search`. - # language : 'cpp', ) From 63e50a4b56a2e21e854636a622941fd36d2e78da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 25 Jul 2024 07:32:03 +0200 Subject: [PATCH 118/284] add werror=suggest-override Improves code readability by making overrides explicit. Inspired by lix code-base --- Makefile | 2 +- build-utils-meson/diagnostics/meson.build | 1 + src/libexpr/json-to-value.cc | 26 +++++++++---------- src/libexpr/nixexpr.hh | 4 +-- src/libstore/daemon.cc | 2 +- .../unix/build/local-derivation-goal.cc | 2 +- src/libutil/serialise.cc | 4 +-- src/libutil/serialise.hh | 6 ++--- src/perl/meson.build | 1 + 9 files changed, 25 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index bb64a104e72..ac76b1532a7 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ ifdef HOST_WINDOWS GLOBAL_LDFLAGS += -Wl,--export-all-symbols endif -GLOBAL_CXXFLAGS += -g -Wall -Wdeprecated-copy -Wignored-qualifiers -Wimplicit-fallthrough -Werror=unused-result -include $(buildprefix)config.h -std=c++2a -I src +GLOBAL_CXXFLAGS += -g -Wall -Wdeprecated-copy -Wignored-qualifiers -Wimplicit-fallthrough -Werror=unused-result -Werror=suggest-override -include $(buildprefix)config.h -std=c++2a -I src # Include the main lib, causing rules to be defined diff --git a/build-utils-meson/diagnostics/meson.build b/build-utils-meson/diagnostics/meson.build index e81f19eff32..30eedfc13f8 100644 --- a/build-utils-meson/diagnostics/meson.build +++ b/build-utils-meson/diagnostics/meson.build @@ -1,5 +1,6 @@ add_project_arguments( '-Wdeprecated-copy', + '-Werror=suggest-override', '-Werror=switch', '-Werror=switch-enum', '-Werror=unused-result', diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 20bee193faa..33ab55ee949 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -80,42 +80,42 @@ class JSONSax : nlohmann::json_sax { public: JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {}; - bool null() + bool null() override { rs->value(state).mkNull(); rs->add(); return true; } - bool boolean(bool val) + bool boolean(bool val) override { rs->value(state).mkBool(val); rs->add(); return true; } - bool number_integer(number_integer_t val) + bool number_integer(number_integer_t val) override { rs->value(state).mkInt(val); rs->add(); return true; } - bool number_unsigned(number_unsigned_t val) + bool number_unsigned(number_unsigned_t val) override { rs->value(state).mkInt(val); rs->add(); return true; } - bool number_float(number_float_t val, const string_t & s) + bool number_float(number_float_t val, const string_t & s) override { rs->value(state).mkFloat(val); rs->add(); return true; } - bool string(string_t & val) + bool string(string_t & val) override { rs->value(state).mkString(val); rs->add(); @@ -123,7 +123,7 @@ class JSONSax : nlohmann::json_sax { } #if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8 - bool binary(binary_t&) + bool binary(binary_t&) override { // This function ought to be unreachable assert(false); @@ -131,35 +131,35 @@ class JSONSax : nlohmann::json_sax { } #endif - bool start_object(std::size_t len) + bool start_object(std::size_t len) override { rs = std::make_unique(std::move(rs)); return true; } - bool key(string_t & name) + bool key(string_t & name) override { dynamic_cast(rs.get())->key(name, state); return true; } - bool end_object() { + bool end_object() override { rs = rs->resolve(state); rs->add(); return true; } - bool end_array() { + bool end_array() override { return end_object(); } - bool start_array(size_t len) { + bool start_array(size_t len) override { rs = std::make_unique(std::move(rs), len != std::numeric_limits::max() ? len : 128); return true; } - bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) { + bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) override { throw JSONParseError("%s", ex.what()); } }; diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 1bcc962c56a..3279e3d48b3 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -186,7 +186,7 @@ struct ExprInheritFrom : ExprVar this->fromWith = nullptr; } - void bindVars(EvalState & es, const std::shared_ptr & env); + void bindVars(EvalState & es, const std::shared_ptr & env) override; }; struct ExprSelect : Expr @@ -203,7 +203,7 @@ struct ExprSelect : Expr * * @param[out] v The attribute set that should contain the last attribute name (if it exists). * @return The last attribute name in `attrPath` - * + * * @note This does *not* evaluate the final attribute, and does not fail if that's the only attribute that does not exist. */ Symbol evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 6533b2f58c4..f28f92fce37 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -167,7 +167,7 @@ struct TunnelSink : Sink { Sink & to; TunnelSink(Sink & to) : to(to) { } - void operator () (std::string_view data) + void operator () (std::string_view data) override { to << STDERR_WRITE; writeString(data, to); diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 0dd102200a5..d30caaf5179 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -1258,7 +1258,7 @@ bool LocalDerivationGoal::isAllowed(const DerivedPath & req) struct RestrictedStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } + const std::string name() override { return "Restricted Store"; } }; /* A wrapper around LocalStore that only allows building/querying of diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 36b99905aab..1be7fa37a29 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -190,11 +190,11 @@ struct VirtualStackAllocator { class DefaultStackAllocator : public StackAllocator { boost::coroutines2::default_stack stack; - boost::context::stack_context allocate() { + boost::context::stack_context allocate() override { return stack.allocate(); } - void deallocate(boost::context::stack_context sctx) { + void deallocate(boost::context::stack_context sctx) override { stack.deallocate(sctx); } }; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 8137db5f40e..c7290dcef9d 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -204,7 +204,7 @@ struct TeeSink : Sink { Sink & sink1, & sink2; TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { } - virtual void operator () (std::string_view data) + virtual void operator () (std::string_view data) override { sink1(data); sink2(data); @@ -221,7 +221,7 @@ struct TeeSource : Source Sink & sink; TeeSource(Source & orig, Sink & sink) : orig(orig), sink(sink) { } - size_t read(char * data, size_t len) + size_t read(char * data, size_t len) override { size_t n = orig.read(data, len); sink({data, n}); @@ -238,7 +238,7 @@ struct SizedSource : Source size_t remain; SizedSource(Source & orig, size_t size) : orig(orig), remain(size) { } - size_t read(char * data, size_t len) + size_t read(char * data, size_t len) override { if (this->remain <= 0) { throw EndOfFile("sized: unexpected end-of-file"); diff --git a/src/perl/meson.build b/src/perl/meson.build index 02e0e68e5ad..dcb6a68a46d 100644 --- a/src/perl/meson.build +++ b/src/perl/meson.build @@ -25,6 +25,7 @@ nix_perl_conf.set('PACKAGE_VERSION', meson.project_version()) error_args = [ '-Wdeprecated-copy', '-Wdeprecated-declarations', + '-Werror=suggest-override', '-Werror=unused-result', '-Wignored-qualifiers', '-Wno-duplicate-decl-specifier', From 9b5ce9acc29a98bedf0530d5611f25f57eae6197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 25 Jul 2024 09:21:31 +0200 Subject: [PATCH 119/284] build-remote: only allocate storeUri once also it's probably not much overhead compared to the networking stuff it, but it's less code at least. --- src/build-remote/build-remote.cc | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 8482b742d31..82ad7d86212 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -101,7 +101,7 @@ static int main_build_remote(int argc, char * * argv) } std::optional drvPath; - StoreReference storeUri; + std::string storeUri; while (true) { @@ -234,17 +234,16 @@ static int main_build_remote(int argc, char * * argv) lock = -1; try { + storeUri = bestMachine->storeUri.render(); - Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri.render())); + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", storeUri)); sshStore = bestMachine->openStore(); sshStore->connect(); - storeUri = bestMachine->storeUri; - } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); printError("cannot build on '%s': %s%s", - bestMachine->storeUri.render(), e.what(), + storeUri, e.what(), msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; @@ -259,7 +258,7 @@ static int main_build_remote(int argc, char * * argv) assert(sshStore); - std::cerr << "# accept\n" << storeUri.render() << "\n"; + std::cerr << "# accept\n" << storeUri << "\n"; auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); @@ -270,17 +269,17 @@ static int main_build_remote(int argc, char * * argv) uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true); }; try { - setUpdateLock(storeUri.render()); + setUpdateLock(storeUri); } catch (SysError & e) { if (e.errNo != ENAMETOOLONG) throw; // Try again hashing the store URL so we have a shorter path - auto h = hashString(HashAlgorithm::MD5, storeUri.render()); + auto h = hashString(HashAlgorithm::MD5, storeUri); setUpdateLock(h.to_string(HashFormat::Base64, false)); } } { - Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri.render())); + Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); auto old = signal(SIGALRM, handleAlarm); alarm(15 * 60); @@ -293,7 +292,7 @@ static int main_build_remote(int argc, char * * argv) auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute; { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri.render())); + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute); } @@ -331,7 +330,7 @@ static int main_build_remote(int argc, char * * argv) optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv); auto & result = *optResult; if (!result.success()) - throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri.render(), result.errorMsg); + throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); } else { copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute); auto res = sshStore->buildPathsWithResults({ @@ -374,7 +373,7 @@ static int main_build_remote(int argc, char * * argv) } if (!missingPaths.empty()) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri.render())); + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); if (auto localStore = store.dynamic_pointer_cast()) for (auto & path : missingPaths) localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */ From baa28159d316e2c506fee8a82f66726c9305f68a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 25 Jul 2024 15:38:02 +0200 Subject: [PATCH 120/284] Update tests/functional/test-infra.sh Co-authored-by: John Ericson --- tests/functional/test-infra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test-infra.sh b/tests/functional/test-infra.sh index 1dab069fb2f..d02a11b4641 100755 --- a/tests/functional/test-infra.sh +++ b/tests/functional/test-infra.sh @@ -111,7 +111,7 @@ unset res # `grepQuiet` does not allow newlines in its arguments, because grep quietly # treats them as multiple queries. -( echo foo; echo bar; ) | expectStderr -101 grepQuiet $'foo\nbar' \ +{ echo foo; echo bar; } | expectStderr -101 grepQuiet $'foo\nbar' \ | grepQuiet -E 'test-infra\.sh:[0-9]+: in call to grepQuiet: newline not allowed in arguments; grep would try each line individually as if connected by an OR operator' # We took the blue pill and woke up in a world where `grep` is moderately safe. From f0fe1d880ded3b5c1e6d44ad0cee9105a50ebd65 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 25 Jul 2024 15:39:15 +0200 Subject: [PATCH 121/284] Update doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jörg Thalheim --- .../10734-nix3-build-show-all-fod-errors-with-keep-going.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md b/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md index 5c2797be843..e4e2f797c5c 100644 --- a/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md +++ b/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md @@ -5,6 +5,6 @@ prs: 10734 The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior of `nix build`. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. -Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92) +Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) [`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) From 55a654abfd0f9a0d8e70b9d2dec410888a3e76db Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 17:53:17 +0200 Subject: [PATCH 122/284] Make panic() and unreachable() robust Plus one or two tweaks. --- src/libutil/error.cc | 28 +++++++++++++++++++++++++--- src/libutil/error.hh | 8 ++------ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index b1858911aa5..ccd008c7c33 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,3 +1,5 @@ +#include + #include "error.hh" #include "environment-variables.hh" #include "signals.hh" @@ -430,16 +432,36 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s return out; } +/** Write to stderr in a robust and minimal way, considering that the process + * may be in a bad state. + */ +static void writeErr(std::string_view buf) +{ + while (!buf.empty()) { + auto n = write(STDERR_FILENO, buf.data(), buf.size()); + if (n < 0) { + if (errno == EINTR) continue; + abort(); + } + buf = buf.substr(n); + } +} + void panic(std::string_view msg) { - printError(msg); - printError("This was a fatal error, aborting."); + writeErr("\n\n" ANSI_RED "terminating due to unexpected unrecoverable internal error: " ANSI_NORMAL ); + writeErr(msg); + writeErr("\n"); abort(); } void panic(const char * file, int line, const char * func) { - panic(std::string("Unexpected condition in ") + func + " at " + file + ":" + std::to_string(line)); + char buf[512]; + int n = snprintf(buf, sizeof(buf), "Unexpected condition in %s at %s:%d", func, file, line); + if (n < 0) + panic("Unexpected condition and could not format error message"); + panic(std::string_view(buf, std::min(static_cast(sizeof(buf)), n))); } } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 572a1baf729..58d9026222f 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -275,17 +275,13 @@ void throwExceptionSelfCheck(); /** * Print a message and abort(). - * - * @note: This assumes that the logger is operational */ [[noreturn]] void panic(std::string_view msg); /** * Print a basic error message with source position and abort(). - * Use the unreachable macro to call this. - * - * @note: This assumes that the logger is operational + * Use the unreachable() macro to call this. */ [[noreturn]] void panic(const char * file, int line, const char * func); @@ -295,6 +291,6 @@ void panic(const char * file, int line, const char * func); * * @note: This assumes that the logger is operational */ -#define unreachable() (panic(__FILE__, __LINE__, __func__)) +#define unreachable() (::nix::panic(__FILE__, __LINE__, __func__)) } From 6e178cd899dc9e25e8da6a018f7cfdb233b762ea Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 25 Jul 2024 11:38:45 -0700 Subject: [PATCH 123/284] Fix reference to experimental features docs Arose because https://github.com/NixOS/nix/pull/9014 merged before https://github.com/NixOS/nix/pull/11131, but the latter did not rebase / merge against the latest master. --- doc/manual/src/language/operators.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index e96a2898804..e1c020781c3 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -198,11 +198,11 @@ Equivalent to `!`*b1* `||` *b2*. > **Warning** > > This syntax is part of an -> [experimental feature](@docroot@/contributing/experimental-features.md) +> [experimental feature](@docroot@/development/experimental-features.md) > and may change in future releases. > > To use this syntax, make sure the -> [`pipe-operators` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-pipe-operators) +> [`pipe-operators` experimental feature](@docroot@/development/experimental-features.md#xp-feature-pipe-operators) > is enabled. > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > From 90459e60dcfedae27bd7da5615fe1aac25cc4c4e Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 25 Jul 2024 11:38:45 -0700 Subject: [PATCH 124/284] Fix reference to experimental features docs Arose because https://github.com/NixOS/nix/pull/9014 merged before https://github.com/NixOS/nix/pull/11131, but the latter did not rebase / merge against the latest master. --- doc/manual/src/language/operators.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index e96a2898804..e1c020781c3 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -198,11 +198,11 @@ Equivalent to `!`*b1* `||` *b2*. > **Warning** > > This syntax is part of an -> [experimental feature](@docroot@/contributing/experimental-features.md) +> [experimental feature](@docroot@/development/experimental-features.md) > and may change in future releases. > > To use this syntax, make sure the -> [`pipe-operators` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-pipe-operators) +> [`pipe-operators` experimental feature](@docroot@/development/experimental-features.md#xp-feature-pipe-operators) > is enabled. > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > From 492715c0bb589f0df963cb9902a9f46230104d44 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Tue, 18 Jun 2024 00:54:05 -0700 Subject: [PATCH 125/284] diff-closures: fix a use after free Found by looking for interesting asan reports from the test suite. What happened here is that name got overwritten, but it was what actually held the backing memory for the thing it got overwritten by, which was a by-reference value coming out of std::regex. Due to absurd reasons I cannot seem to use a string_view iterator here, so I just copy the string with a longer lifetime instead. idk lol ==3796364==ERROR: AddressSanitizer: heap-use-after-free on address 0x503000014c61 at pc 0x74843523bf1d bp 0x7ffc68351330 sp 0x7ffc68350af0 READ of size 3 at 0x503000014c61 thread T0 0 0x74843523bf1c in __asan_memcpy (/nix/store/mzhqknx2mc94jdz4n320hn1lml86398y-clang-wrapper-17.0.6/resource-root/lib/linux/libclang_rt.asan-x86_64.so+0x159f1c) 1 0x6403cf6cbff4 in std::char_traits::copy(char*, char const*, unsigned long) /nix/store/14c6s4xzhy14i2b05s00rjns2j93gzz4-gcc-13.2.0/include/c++/13.2.0/bits/char_traits.h:445:33 <...> 7 0x6403cf6cbff4 in std::__cxx11::sub_match<__gnu_cxx::__normal_iterator, std::allocator>>>::str() const /nix/store/14c6s4xzhy14i2b05s00rjns2j93gzz4-gcc-13.2.0/include/c++/13.2.0/bits/regex.h:966:6 8 0x6403cf6cbff4 in std::__cxx11::sub_match<__gnu_cxx::__normal_iterator, std::allocator>>>::operator std::__cxx11::basic_string, std::allocator>() const /nix/store/14c6s4xzhy14i2b05s00rjns2j93gzz4-gcc-13.2.0/include/c++/13.2.0/bits/regex.h:955:16 9 0x6403cf6cbff4 in nix::getClosureInfo[abi:cxx11](nix::ref, nix::StorePath const&) /home/jade/lix/lix2/build/src/nix/diff-closures.cc:37:26 10 0x6403cf6cd70c in nix::printClosureDiff(nix::ref, nix::StorePath const&, nix::StorePath const&, std::basic_string_view>) /home/jade/lix/lix2/build/src/nix/diff-closures.cc:54:25 11 0x6403cf873331 in CmdProfileDiffClosures::run(nix::ref) /home/jade/lix/lix2/build/src/nix/profile.cc:479:17 <...> 0x503000014c61 is located 17 bytes inside of 21-byte region [0x503000014c50,0x503000014c65) freed by thread T0 here: 0 0x748435250470 in operator delete(void*) (/nix/store/mzhqknx2mc94jdz4n320hn1lml86398y-clang-wrapper-17.0.6/resource-root/lib/linux/libclang_rt.asan-x86_64.so+0x16e470) <...> 6 0x6403cf6cbda2 in std::__cxx11::basic_string, std::allocator>::~basic_string() /nix/store/14c6s4xzhy14i2b05s00rjns2j93gzz4-gcc-13.2.0/include/c++/13.2.0/bits/basic_string.h:792:9 7 0x6403cf6cbda2 in nix::getClosureInfo[abi:cxx11](nix::ref, nix::StorePath const&) /home/jade/lix/lix2/build/src/nix/diff-closures.cc:36:13 8 0x6403cf6cd70c in nix::printClosureDiff(nix::ref, nix::StorePath const&, nix::StorePath const&, std::basic_string_view>) /home/jade/lix/lix2/build/src/nix/diff-closures.cc:54:25 <...> previously allocated by thread T0 here: 0 0x74843524fa38 in operator new(unsigned long) (/nix/store/mzhqknx2mc94jdz4n320hn1lml86398y-clang-wrapper-17.0.6/resource-root/lib/linux/libclang_rt.asan-x86_64.so+0x16da38) <...> 9 0x6403cf6cb68c in std::__cxx11::basic_string, std::allocator>::basic_string>, void>(std::basic_string_view> const&, std::allocator const&) /nix/store/14c6s4xzhy14i2b05s00rjns2j93gzz4-gcc-13.2.0/include/c++/13.2.0/bits/basic_string.h:784:4 10 0x6403cf6cb68c in nix::getClosureInfo[abi:cxx11](nix::ref, nix::StorePath const&) /home/jade/lix/lix2/build/src/nix/diff-closures.cc:33:21 11 0x6403cf6cd70c in nix::printClosureDiff(nix::ref, nix::StorePath const&, nix::StorePath const&, std::basic_string_view>) /home/jade/lix/lix2/build/src/nix/diff-closures.cc:54:25 12 0x6403cf873331 in CmdProfileDiffClosures::run(nix::ref) /home/jade/lix/lix2/build/src/nix/profile.cc:479:17 <...> (cherry-picked from https://git.lix.systems/lix-project/lix/commit/b9b1bbd22fc3b34a4d8e2090f0d033f742c32ee0) --- src/nix/diff-closures.cc | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 46c94b211f9..4e12dc60a34 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -31,9 +31,18 @@ GroupedPaths getClosureInfo(ref store, const StorePath & toplevel) version suffixes like "unstable"). */ static std::regex regex("(.*)-([a-z]+|lib32|lib64)"); std::smatch match; - std::string name(path.name()); + std::string name{path.name()}; + // Used to keep name alive through being potentially overwritten below + // (to not invalidate the references from the regex result) + // + // n.b. cannot be just path.name().{begin,end}() since that returns const + // char *, which does not, for some reason, convert as required on + // libstdc++. Seems like a libstdc++ bug or standard bug to me... we + // can afford the allocation in any case. + const std::string origName{path.name()}; std::string outputName; - if (std::regex_match(name, match, regex)) { + + if (std::regex_match(origName, match, regex)) { name = match[1]; outputName = match[2]; } From 07aeedd37e2e6c2b2d48fd5cf59917adc241f949 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Tue, 23 Jul 2024 21:45:30 +0200 Subject: [PATCH 126/284] diff-closures: remove gratuitous copy This was done originally because std::smatch does not accept `const char *` as iterators. However, this was because we should have been using std::cmatch instead. (cherry picked from commit https://git.lix.systems/lix-project/lix/commit/12a5838d11f790d36e2ac626e8916a2c19bcb80b) --- src/nix/diff-closures.cc | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 4e12dc60a34..2bc7fe82b1b 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -25,24 +25,17 @@ GroupedPaths getClosureInfo(ref store, const StorePath & toplevel) GroupedPaths groupedPaths; - for (auto & path : closure) { + for (auto const & path : closure) { /* Strip the output name. Unfortunately this is ambiguous (we can't distinguish between output names like "bin" and version suffixes like "unstable"). */ static std::regex regex("(.*)-([a-z]+|lib32|lib64)"); - std::smatch match; + std::cmatch match; std::string name{path.name()}; - // Used to keep name alive through being potentially overwritten below - // (to not invalidate the references from the regex result) - // - // n.b. cannot be just path.name().{begin,end}() since that returns const - // char *, which does not, for some reason, convert as required on - // libstdc++. Seems like a libstdc++ bug or standard bug to me... we - // can afford the allocation in any case. - const std::string origName{path.name()}; + std::string_view const origName = path.name(); std::string outputName; - if (std::regex_match(origName, match, regex)) { + if (std::regex_match(origName.begin(), origName.end(), match, regex)) { name = match[1]; outputName = match[2]; } From 429a197d246782e286fe7d9f6fad0b9daf2941ec Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Sat, 20 Jul 2024 12:04:25 -0400 Subject: [PATCH 127/284] parser.y: use names where I'll be refactoring --- src/libexpr/parser.y | 72 ++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 9ad41c148dd..256244a75c2 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -180,22 +180,22 @@ expr_function $$ = me; SET_DOC_POS(me, @1); } - | '{' formals '}' ':' expr_function - { auto me = new ExprLambda(CUR_POS, state->validateFormals($2), $5); + | '{' formals '}' ':' expr_function[body] + { auto me = new ExprLambda(CUR_POS, state->validateFormals($formals), $body); $$ = me; SET_DOC_POS(me, @1); } - | '{' formals '}' '@' ID ':' expr_function + | '{' formals '}' '@' ID ':' expr_function[body] { - auto arg = state->symbols.create($5); - auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7); + auto arg = state->symbols.create($ID); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formals, CUR_POS, arg), $body); $$ = me; SET_DOC_POS(me, @1); } - | ID '@' '{' formals '}' ':' expr_function + | ID '@' '{' formals '}' ':' expr_function[body] { - auto arg = state->symbols.create($1); - auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7); + auto arg = state->symbols.create($ID); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formals, CUR_POS, arg), $body); $$ = me; SET_DOC_POS(me, @1); } @@ -364,50 +364,50 @@ ind_string_parts ; binds - : binds attrpath '=' expr ';' { - $$ = $1; + : binds[accum] attrpath '=' expr ';' { + $$ = $accum; - auto pos = state->at(@2); - auto exprPos = state->at(@4); + auto pos = state->at(@attrpath); + auto exprPos = state->at(@expr); { auto it = state->lexerState.positionToDocComment.find(pos); if (it != state->lexerState.positionToDocComment.end()) { - $4->setDocComment(it->second); + $expr->setDocComment(it->second); state->lexerState.positionToDocComment.emplace(exprPos, it->second); } } - state->addAttr($$, std::move(*$2), $4, pos); - delete $2; + state->addAttr($$, std::move(*$attrpath), $expr, pos); + delete $attrpath; } - | binds INHERIT attrs ';' - { $$ = $1; - for (auto & [i, iPos] : *$3) { - if ($$->attrs.find(i.symbol) != $$->attrs.end()) - state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos); - $$->attrs.emplace( + | binds[accum] INHERIT attrs ';' + { $$ = $accum; + for (auto & [i, iPos] : *$attrs) { + if ($accum->attrs.find(i.symbol) != $accum->attrs.end()) + state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos); + $accum->attrs.emplace( i.symbol, ExprAttrs::AttrDef(new ExprVar(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited)); } - delete $3; + delete $attrs; } - | binds INHERIT '(' expr ')' attrs ';' - { $$ = $1; - if (!$$->inheritFromExprs) - $$->inheritFromExprs = std::make_unique>(); - $$->inheritFromExprs->push_back($4); - auto from = new nix::ExprInheritFrom(state->at(@4), $$->inheritFromExprs->size() - 1); - for (auto & [i, iPos] : *$6) { - if ($$->attrs.find(i.symbol) != $$->attrs.end()) - state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos); - $$->attrs.emplace( + | binds[accum] INHERIT '(' expr ')' attrs ';' + { $$ = $accum; + if (!$accum->inheritFromExprs) + $accum->inheritFromExprs = std::make_unique>(); + $accum->inheritFromExprs->push_back($expr); + auto from = new nix::ExprInheritFrom(state->at(@expr), $accum->inheritFromExprs->size() - 1); + for (auto & [i, iPos] : *$attrs) { + if ($accum->attrs.find(i.symbol) != $accum->attrs.end()) + state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos); + $accum->attrs.emplace( i.symbol, ExprAttrs::AttrDef( new ExprSelect(iPos, from, i.symbol), iPos, ExprAttrs::AttrDef::Kind::InheritedFrom)); } - delete $6; + delete $attrs; } | { $$ = new ExprAttrs(state->at(@0)); } ; @@ -468,10 +468,10 @@ expr_list ; formals - : formal ',' formals - { $$ = $3; $$->formals.emplace_back(*$1); delete $1; } + : formal ',' formals[accum] + { $$ = $accum; $$->formals.emplace_back(*$formal); delete $formal; } | formal - { $$ = new Formals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; } + { $$ = new Formals; $$->formals.emplace_back(*$formal); $$->ellipsis = false; delete $formal; } | { $$ = new Formals; $$->ellipsis = false; } | ELLIPSIS From b0a8430e851790ebac4e2324084bd6d86b995316 Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Sat, 20 Jul 2024 12:04:25 -0400 Subject: [PATCH 128/284] parser.y: move attr doc setting into addAttr --- src/libexpr/parser-state.hh | 11 +++++++++-- src/libexpr/parser.y | 13 +------------ 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 4bb5c92046b..c23ef32a560 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -86,7 +86,7 @@ struct ParserState void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); - void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos); + void addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc); Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {}); Expr * stripIndentation(const PosIdx pos, std::vector>> && es); @@ -110,11 +110,12 @@ inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx pre }); } -inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos) +inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc) { AttrPath::iterator i; // All attrpaths have at least one attr assert(!attrPath.empty()); + auto pos = at(loc); // Checking attrPath validity. // =========================== for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { @@ -179,6 +180,12 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * } else { attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); } + + auto it = lexerState.positionToDocComment.find(pos); + if (it != lexerState.positionToDocComment.end()) { + e->setDocComment(it->second); + lexerState.positionToDocComment.emplace(at(exprLoc), it->second); + } } inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 256244a75c2..6386747f5f3 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -366,18 +366,7 @@ ind_string_parts binds : binds[accum] attrpath '=' expr ';' { $$ = $accum; - - auto pos = state->at(@attrpath); - auto exprPos = state->at(@expr); - { - auto it = state->lexerState.positionToDocComment.find(pos); - if (it != state->lexerState.positionToDocComment.end()) { - $expr->setDocComment(it->second); - state->lexerState.positionToDocComment.emplace(exprPos, it->second); - } - } - - state->addAttr($$, std::move(*$attrpath), $expr, pos); + state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr); delete $attrpath; } | binds[accum] INHERIT attrs ';' From 6e3b9e6a4de7430c8b130a7c87d7f5df68cf6c86 Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Sat, 20 Jul 2024 12:04:25 -0400 Subject: [PATCH 129/284] parser.y: eliminate conflicts --- src/libexpr/parser.y | 61 ++++++++++++------- .../lang/parse-fail-undef-var-2.err.exp | 2 +- 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 6386747f5f3..2069931e164 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -8,8 +8,8 @@ %parse-param { nix::ParserState * state } %lex-param { void * scanner } %lex-param { nix::ParserState * state } -%expect 1 -%expect-rr 1 +%expect 0 +%expect-rr 0 %code requires { @@ -133,8 +133,8 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) { %type expr_select expr_simple expr_app %type expr_pipe_from expr_pipe_into %type expr_list -%type binds -%type formals +%type binds binds1 +%type formals formal_set %type formal %type attrpath %type attrs @@ -180,22 +180,22 @@ expr_function $$ = me; SET_DOC_POS(me, @1); } - | '{' formals '}' ':' expr_function[body] - { auto me = new ExprLambda(CUR_POS, state->validateFormals($formals), $body); + | formal_set ':' expr_function[body] + { auto me = new ExprLambda(CUR_POS, state->validateFormals($formal_set), $body); $$ = me; SET_DOC_POS(me, @1); } - | '{' formals '}' '@' ID ':' expr_function[body] + | formal_set '@' ID ':' expr_function[body] { auto arg = state->symbols.create($ID); - auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formals, CUR_POS, arg), $body); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formal_set, CUR_POS, arg), $body); $$ = me; SET_DOC_POS(me, @1); } - | ID '@' '{' formals '}' ':' expr_function[body] + | ID '@' formal_set ':' expr_function[body] { auto arg = state->symbols.create($ID); - auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formals, CUR_POS, arg), $body); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formal_set, CUR_POS, arg), $body); $$ = me; SET_DOC_POS(me, @1); } @@ -311,11 +311,13 @@ expr_simple /* Let expressions `let {..., body = ...}' are just desugared into `(rec {..., body = ...}).body'. */ | LET '{' binds '}' - { $3->recursive = true; $$ = new ExprSelect(noPos, $3, state->s.body); } + { $3->recursive = true; $3->pos = CUR_POS; $$ = new ExprSelect(noPos, $3, state->s.body); } | REC '{' binds '}' - { $3->recursive = true; $$ = $3; } - | '{' binds '}' - { $$ = $2; } + { $3->recursive = true; $3->pos = CUR_POS; $$ = $3; } + | '{' binds1 '}' + { $2->pos = CUR_POS; $$ = $2; } + | '{' '}' + { $$ = new ExprAttrs(CUR_POS); } | '[' expr_list ']' { $$ = $2; } ; @@ -364,8 +366,13 @@ ind_string_parts ; binds - : binds[accum] attrpath '=' expr ';' { - $$ = $accum; + : binds1 + | { $$ = new ExprAttrs; } + ; + +binds1 + : binds1[accum] attrpath '=' expr ';' + { $$ = $accum; state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr); delete $attrpath; } @@ -398,7 +405,11 @@ binds } delete $attrs; } - | { $$ = new ExprAttrs(state->at(@0)); } + | attrpath '=' expr ';' + { $$ = new ExprAttrs; + state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr); + delete $attrpath; + } ; attrs @@ -456,15 +467,19 @@ expr_list | { $$ = new ExprList; } ; +formal_set + : '{' formals ',' ELLIPSIS '}' { $$ = $formals; $$->ellipsis = true; } + | '{' ELLIPSIS '}' { $$ = new Formals; $$->ellipsis = true; } + | '{' formals ',' '}' { $$ = $formals; $$->ellipsis = false; } + | '{' formals '}' { $$ = $formals; $$->ellipsis = false; } + | '{' '}' { $$ = new Formals; $$->ellipsis = false; } + ; + formals - : formal ',' formals[accum] + : formals[accum] ',' formal { $$ = $accum; $$->formals.emplace_back(*$formal); delete $formal; } | formal - { $$ = new Formals; $$->formals.emplace_back(*$formal); $$->ellipsis = false; delete $formal; } - | - { $$ = new Formals; $$->ellipsis = false; } - | ELLIPSIS - { $$ = new Formals; $$->ellipsis = true; } + { $$ = new Formals; $$->formals.emplace_back(*$formal); delete $formal; } ; formal diff --git a/tests/functional/lang/parse-fail-undef-var-2.err.exp b/tests/functional/lang/parse-fail-undef-var-2.err.exp index 393c454dd31..96e87b2aa91 100644 --- a/tests/functional/lang/parse-fail-undef-var-2.err.exp +++ b/tests/functional/lang/parse-fail-undef-var-2.err.exp @@ -1,4 +1,4 @@ -error: syntax error, unexpected ':', expecting '}' +error: syntax error, unexpected ':', expecting '}' or ',' at «stdin»:3:13: 2| 3| f = {x, y : ["baz" "bar" z "bat"]}: x + y; From 18db46a6cb72acaae748833b09b428a7794bd9c8 Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Mon, 22 Jul 2024 11:36:09 -0400 Subject: [PATCH 130/284] parser.y: GLR -> LALR --- src/libexpr/parser-state.hh | 1 + src/libexpr/parser.y | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index c23ef32a560..8ad0d9ad714 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -20,6 +20,7 @@ struct StringToken operator std::string_view() const { return {p, l}; } }; +// This type must be trivially copyable; see YYLTYPE_IS_TRIVIAL in parser.y. struct ParserLocation { int beginOffset; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 2069931e164..f2ccca7fcdd 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -1,4 +1,4 @@ -%glr-parser +%define api.location.type { ::nix::ParserLocation } %define api.pure %locations %define parse.error verbose @@ -9,7 +9,6 @@ %lex-param { void * scanner } %lex-param { nix::ParserState * state } %expect 0 -%expect-rr 0 %code requires { @@ -27,7 +26,17 @@ #include "eval-settings.hh" #include "parser-state.hh" -#define YYLTYPE ::nix::ParserLocation +// Bison seems to have difficulty growing the parser stack when using C++ with +// a custom location type. This undocumented macro tells Bison that our +// location type is "trivially copyable" in C++-ese, so it is safe to use the +// same memcpy macro it uses to grow the stack that it uses with its own +// default location type. Without this, we get "error: memory exhausted" when +// parsing some large Nix files. Our other options are to increase the initial +// stack size (200 by default) to be as large as we ever want to support (so +// that growing the stack is unnecessary), or redefine the stack-relocation +// macro ourselves (which is also undocumented). +#define YYLTYPE_IS_TRIVIAL 1 + #define YY_DECL int yylex \ (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state) @@ -77,7 +86,7 @@ YY_DECL; using namespace nix; -#define CUR_POS state->at(*yylocp) +#define CUR_POS state->at(yyloc) void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error) From 2141a52ca3e150079bce8bc12a0a992b16c046f1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jul 2024 15:38:44 +0200 Subject: [PATCH 131/284] nix repl: Remove unnecessary call to evalString This crashes with the multithreaded evaluator, which checks against attempts to finish an already finished value. --- src/libcmd/repl.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index b5d0816dd2c..f5e836f8c04 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -644,9 +644,6 @@ ProcessLineResult NixRepl::processLine(std::string line) fallbackPos = attr->pos; fallbackDoc = state->getDocCommentForPos(fallbackPos); } - - } else { - evalString(arg, v); } evalString(arg, v); From 6d843ce9fec37e854e518db19ce880895f7de2ac Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 6 Jun 2024 19:32:46 +0200 Subject: [PATCH 132/284] Provide std::hash --- src/libexpr/json-to-value.cc | 2 +- src/libexpr/symbol-table.hh | 11 +++++++++++ src/libexpr/value.hh | 4 ++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 33ab55ee949..21074bdd807 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -42,7 +42,7 @@ class JSONSax : nlohmann::json_sax { auto attrs2 = state.buildBindings(attrs.size()); for (auto & i : attrs) attrs2.insert(i.first, i.second); - parent->value(state).mkAttrs(attrs2.alreadySorted()); + parent->value(state).mkAttrs(attrs2); return std::move(parent); } void add() override { v = nullptr; } diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index c7a3563b040..8f7257e01ef 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -69,6 +69,8 @@ public: auto operator<=>(const Symbol other) const { return id <=> other.id; } bool operator==(const Symbol other) const { return id == other.id; } + + friend class std::hash; }; /** @@ -132,3 +134,12 @@ public: }; } + +template<> +struct std::hash +{ + std::size_t operator()(const nix::Symbol & s) const noexcept + { + return std::hash{}(s.id); + } +}; diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 1f4d72d3926..bfe526db327 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -494,11 +494,11 @@ void Value::mkBlackhole() #if HAVE_BOEHMGC typedef std::vector> ValueVector; -typedef std::map, traceable_allocator>> ValueMap; +typedef std::unordered_map, std::equal_to, traceable_allocator>> ValueMap; typedef std::map, traceable_allocator>> ValueVectorMap; #else typedef std::vector ValueVector; -typedef std::map ValueMap; +typedef std::unordered_map ValueMap; typedef std::map ValueVectorMap; #endif From ce663d75e36f1bb41c11a9d666d5b649328bee92 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 4 Jan 2024 15:05:20 +0100 Subject: [PATCH 133/284] LRUCache: Mark size() as const --- src/libutil/lru-cache.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 0e19517ed2c..6e14cac3519 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -89,7 +89,7 @@ public: return i->second.second; } - size_t size() + size_t size() const { return data.size(); } From ea46264bd35d479ca60b3f725d31f0735863281d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jul 2024 16:14:03 +0200 Subject: [PATCH 134/284] Store: Use SharedSync for state --- src/libstore/store-api.cc | 2 +- src/libstore/store-api.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2c4dee518b4..21fa2939d5e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -920,7 +920,7 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor const Store::Stats & Store::getStats() { { - auto state_(state.lock()); + auto state_(state.read()); stats.pathInfoCacheSize = state_->pathInfoCache.size(); } return stats; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7d5f533c5a3..8288cfdf01f 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -201,7 +201,7 @@ protected: LRUCache pathInfoCache; }; - Sync state; + SharedSync state; std::shared_ptr diskCache; From d9ba2a1634865b2d7306b97902ce5accb89a9bb2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jul 2024 19:06:49 +0200 Subject: [PATCH 135/284] Fix error message --- src/libfetchers/git-utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index ecc71ae471d..1a64acc5040 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -131,7 +131,7 @@ T peelObject(git_repository * repo, git_object * obj, git_object_t type) T obj2; if (git_object_peel((git_object * *) (typename T::pointer *) Setter(obj2), obj, type)) { auto err = git_error_last(); - throw Error("peeling Git object '%s': %s", git_object_id(obj), err->message); + throw Error("peeling Git object '%s': %s", *git_object_id(obj), err->message); } return obj2; } From 06b686b62dae4db86bb4c449c3418e271dff1b68 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jul 2024 20:24:58 +0200 Subject: [PATCH 136/284] Handle tarballs that don't consist of a single top-level directory Fixes #4785 (top-level directories are no longer merged into one). Fixes #10983 (top-level non-directories are no longer discarded). --- src/libfetchers/git-utils.cc | 77 ++++++++++++++++++++++++++---------- tests/functional/tarball.sh | 25 ++++++++++++ 2 files changed, 82 insertions(+), 20 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 1a64acc5040..032d8e0bd3f 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -126,7 +126,7 @@ Object lookupObject(git_repository * repo, const git_oid & oid, git_object_t typ } template -T peelObject(git_repository * repo, git_object * obj, git_object_t type) +T peelObject(git_object * obj, git_object_t type) { T obj2; if (git_object_peel((git_object * *) (typename T::pointer *) Setter(obj2), obj, type)) { @@ -136,6 +136,29 @@ T peelObject(git_repository * repo, git_object * obj, git_object_t type) return obj2; } +template +T dupObject(typename T::pointer obj) +{ + T obj2; + if (git_object_dup((git_object * *) (typename T::pointer *) Setter(obj2), (git_object *) obj)) + throw Error("duplicating object '%s': %s", *git_object_id((git_object *) obj), git_error_last()->message); + return obj2; +} + +/** + * Peel the specified object (i.e. follow tag and commit objects) to + * either a blob or a tree. + */ +static Object peelToTreeOrBlob(git_object * obj) +{ + /* git_object_peel() doesn't handle blob objects, so handle those + specially. */ + if (git_object_type(obj) == GIT_OBJECT_BLOB) + return dupObject(obj); + else + return peelObject(obj, GIT_OBJECT_TREE); +} + struct GitRepoImpl : GitRepo, std::enable_shared_from_this { /** Location of the repository on disk. */ @@ -166,7 +189,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this std::unordered_set done; std::queue todo; - todo.push(peelObject(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT)); + todo.push(peelObject(lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT)); while (auto commit = pop(todo)) { if (!done.insert(*git_commit_id(commit->get())).second) continue; @@ -184,7 +207,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this uint64_t getLastModified(const Hash & rev) override { - auto commit = peelObject(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT); + auto commit = peelObject(lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT); return git_commit_time(commit.get()); } @@ -476,11 +499,11 @@ ref GitRepo::openRepo(const std::filesystem::path & path, bool create, struct GitSourceAccessor : SourceAccessor { ref repo; - Tree root; + Object root; GitSourceAccessor(ref repo_, const Hash & rev) : repo(repo_) - , root(peelObject(*repo, lookupObject(*repo, hashToOID(rev)).get(), GIT_OBJECT_TREE)) + , root(peelToTreeOrBlob(lookupObject(*repo, hashToOID(rev)).get())) { } @@ -506,7 +529,7 @@ struct GitSourceAccessor : SourceAccessor std::optional maybeLstat(const CanonPath & path) override { if (path.isRoot()) - return Stat { .type = tDirectory }; + return Stat { .type = git_object_type(root.get()) == GIT_OBJECT_TREE ? tDirectory : tRegular }; auto entry = lookup(path); if (!entry) @@ -616,10 +639,10 @@ struct GitSourceAccessor : SourceAccessor std::optional lookupTree(const CanonPath & path) { if (path.isRoot()) { - Tree tree; - if (git_tree_dup(Setter(tree), root.get())) - throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message); - return tree; + if (git_object_type(root.get()) == GIT_OBJECT_TREE) + return dupObject((git_tree *) &*root); + else + return std::nullopt; } auto entry = lookup(path); @@ -646,10 +669,10 @@ struct GitSourceAccessor : SourceAccessor std::variant getTree(const CanonPath & path) { if (path.isRoot()) { - Tree tree; - if (git_tree_dup(Setter(tree), root.get())) - throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message); - return tree; + if (git_object_type(root.get()) == GIT_OBJECT_TREE) + return dupObject((git_tree *) &*root); + else + throw Error("Git root object '%s' is not a directory", *git_object_id(root.get())); } auto entry = need(path); @@ -669,6 +692,9 @@ struct GitSourceAccessor : SourceAccessor Blob getBlob(const CanonPath & path, bool expectSymlink) { + if (!expectSymlink && git_object_type(root.get()) == GIT_OBJECT_BLOB) + return dupObject((git_blob *) &*root); + auto notExpected = [&]() { throw Error( @@ -782,8 +808,6 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink std::vector pendingDirs; - size_t componentsToStrip = 1; - void pushBuilder(std::string name) { git_treebuilder * b; @@ -839,9 +863,6 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink { std::span pathComponents2{pathComponents}; - if (pathComponents2.size() <= componentsToStrip) return false; - pathComponents2 = pathComponents2.subspan(componentsToStrip); - updateBuilders( isDir ? pathComponents2 @@ -964,11 +985,27 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink git_tree_entry_filemode(entry)); } - Hash sync() override { + Hash sync() override + { updateBuilders({}); auto [oid, _name] = popBuilder(); + /* If the root directory contains a single entry that is a + directory or a non-executable regular file, return that as + the top-level object. We don't do this for executables + because they don't have a tree hash in the Git object + model. */ + auto _tree = lookupObject(*repo, oid, GIT_OBJECT_TREE); + auto tree = (const git_tree *) &*_tree; + + if (git_tree_entrycount(tree) == 1) { + auto entry = git_tree_entry_byindex(tree, 0); + auto mode = git_tree_entry_filemode(entry); + if (mode == GIT_FILEMODE_BLOB || mode == GIT_FILEMODE_TREE) + oid = *git_tree_entry_id(entry); + } + return toHash(oid); } }; diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index ab357ac7895..6799831ce4a 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -83,3 +83,28 @@ path="$(nix flake prefetch --json "tarball+file://$(pwd)/tree.tar.gz" | jq -r .s [[ $(cat "$path/a/zzz") = bar ]] [[ $(cat "$path/c/aap") = bar ]] [[ $(cat "$path/fnord") = bar ]] + +# Test a tarball that has multiple top-level directories. +rm -rf "$TEST_ROOT/tar_root" +mkdir -p "$TEST_ROOT/tar_root" "$TEST_ROOT/tar_root/foo" "$TEST_ROOT/tar_root/bar" +tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . +path="$(nix flake prefetch --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" +[[ -d "$path/foo" ]] +[[ -d "$path/bar" ]] + +# Test a tarball that has a single non-executable regular file. +rm -rf "$TEST_ROOT/tar_root" +mkdir -p "$TEST_ROOT/tar_root" +echo bar > "$TEST_ROOT/tar_root/foo" +tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . +path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" +[[ $(cat "$path") = bar ]] + +# Test a tarball that has a single executable regular file. +rm -rf "$TEST_ROOT/tar_root" +mkdir -p "$TEST_ROOT/tar_root" +echo bar > "$TEST_ROOT/tar_root/foo" +chmod +x "$TEST_ROOT/tar_root/foo" +tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . +path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" +[[ $(cat "$path/foo") = bar ]] From b88950ec777ab55a5e924c0b3758bad06a6f0f33 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jul 2024 20:34:04 +0200 Subject: [PATCH 137/284] Update fetchTree docs --- src/libexpr/primops/fetchTree.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e59d7fe6799..9a3b81bc998 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -559,11 +559,11 @@ static RegisterPrimOp primop_fetchTarball({ .doc = R"( Download the specified URL, unpack it and return the path of the unpacked tree. The file must be a tape archive (`.tar`) compressed - with `gzip`, `bzip2` or `xz`. The top-level path component of the - files in the tarball is removed, so it is best if the tarball - contains a single directory at top level. The typical use of the - function is to obtain external Nix expression dependencies, such as - a particular version of Nixpkgs, e.g. + with `gzip`, `bzip2` or `xz`. If the tarball consists of a + single directory or non-executable file, then the top-level path + component of the files in the tarball is removed. The typical + use of the function is to obtain external Nix expression + dependencies, such as a particular version of Nixpkgs, e.g. ```nix with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {}; From 5e83c0427f701df33a37a7993945d7db99f6267e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jul 2024 20:46:07 +0200 Subject: [PATCH 138/284] Fix test --- tests/unit/libfetchers/git-utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/libfetchers/git-utils.cc b/tests/unit/libfetchers/git-utils.cc index d3547ec6a31..f0d38d50c2e 100644 --- a/tests/unit/libfetchers/git-utils.cc +++ b/tests/unit/libfetchers/git-utils.cc @@ -103,7 +103,7 @@ TEST_F(GitUtilsTest, sink_hardlink) sink->createHardlink(CanonPath("foo-1.1/link"), CanonPath("hello")); FAIL() << "Expected an exception"; } catch (const nix::Error & e) { - ASSERT_THAT(e.msg(), testing::HasSubstr("invalid hard link target")); + ASSERT_THAT(e.msg(), testing::HasSubstr("cannot find hard link target")); ASSERT_THAT(e.msg(), testing::HasSubstr("/hello")); ASSERT_THAT(e.msg(), testing::HasSubstr("foo-1.1/link")); } From 6af40f488a26fda5de55e16c6ba950c6f99762ee Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 27 Jul 2024 01:38:18 +0200 Subject: [PATCH 139/284] Rename SyncBase::read() -> readLock() Make it explicit so it's clear what it's about when I and other contributors read its call sites. --- src/libstore/store-api.cc | 2 +- src/libutil/posix-source-accessor.cc | 2 +- src/libutil/sync.hh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 21fa2939d5e..b3e5ad014cf 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -920,7 +920,7 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor const Store::Stats & Store::getStats() { { - auto state_(state.read()); + auto state_(state.readLock()); stats.pathInfoCacheSize = state_->pathInfoCache.size(); } return stats; diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 35047f89eb8..2b1a485d55c 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -97,7 +97,7 @@ std::optional PosixSourceAccessor::cachedLstat(const CanonPath & pa Path absPath = makeAbsPath(path).string(); { - auto cache(_cache.read()); + auto cache(_cache.readLock()); auto i = cache->find(absPath); if (i != cache->end()) return i->second; } diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh index c1b699ffc30..d340f3d9760 100644 --- a/src/libutil/sync.hh +++ b/src/libutil/sync.hh @@ -106,7 +106,7 @@ public: * Acquire read access to the inner value. When using * `std::shared_mutex`, this will use a shared lock. */ - ReadLock read() const { return ReadLock(const_cast(this)); } + ReadLock readLock() const { return ReadLock(const_cast(this)); } }; template From 22f943bb1fe3feefdf4e469270de79406a46f5c3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 27 Jul 2024 02:10:31 +0200 Subject: [PATCH 140/284] dependencies: Centralize aws-sdk-cpp and sync with Nixpkgs By syncing with Nixpkgs, we reuse the same derivation, which is generally a good idea, and has the benefit that it is transitively a channel blocker. Changes: - https://github.com/NixOS/nixpkgs/pull/163313 (SuperSandro2000) > nix: disable big-parallel for aws-sdk-cpp > aws-sdk-cpp only takes ~1m52s on a 4 core machine under 50% load > which does not justify the requirement on big parallel. > Tested with `nix-build -A nixVersions.nix_2_6.aws-sdk-cpp`. > I can finally build nix without requiring a big-parallel machine. - https://github.com/NixOS/nixpkgs/pull/227506 (Artturin) > nix: use [ ] instead null to empty requiredSystemFeatures > fixes 'error: value is null while a list was expected' with 'nixpkgs.hostPlatform.gcc.arch = "x86_64";' --- package.nix | 5 +---- packaging/dependencies.nix | 9 +++++++++ src/libstore/package.nix | 5 +---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/package.nix b/package.nix index c3e565399e8..2499d4370b5 100644 --- a/package.nix +++ b/package.nix @@ -237,10 +237,7 @@ in { ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) + aws-sdk-cpp ; propagatedBuildInputs = [ diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index b77e1d14f38..e0737593fd4 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -79,6 +79,15 @@ scope: { inherit stdenv versionSuffix; version = lib.fileContents ../.version + versionSuffix; + aws-sdk-cpp = (pkgs.aws-sdk-cpp.override { + apis = [ "s3" "transfer" ]; + customMemoryManagement = false; + }).overrideAttrs { + # only a stripped down version is built, which takes a lot less resources + # to build, so we don't need a "big-parallel" machine. + requiredSystemFeatures = [ ]; + }; + libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { version = "2.5.5"; src = pkgs.fetchurl { diff --git a/src/libstore/package.nix b/src/libstore/package.nix index 02ff4194a8f..4582ba0d2b0 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -66,10 +66,7 @@ mkMesonDerivation (finalAttrs: { ] ++ lib.optional stdenv.hostPlatform.isLinux libseccomp # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) + aws-sdk-cpp ; propagatedBuildInputs = [ From 17b5d404457018ab1634fbbc38211607682ee37a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 27 Jul 2024 02:39:55 +0200 Subject: [PATCH 141/284] package.nix: Empty build inputs if not doBuild --- package.nix | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/package.nix b/package.nix index 2499d4370b5..a7c8923e8b4 100644 --- a/package.nix +++ b/package.nix @@ -216,7 +216,8 @@ in { ] ++ lib.optional stdenv.hostPlatform.isStatic unixtools.hexdump ; - buildInputs = lib.optionals doBuild [ + buildInputs = lib.optionals doBuild ( + [ brotli bzip2 curl @@ -238,12 +239,13 @@ in { # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) aws-sdk-cpp - ; + ); - propagatedBuildInputs = [ + propagatedBuildInputs = lib.optionals doBuild ([ boost nlohmann_json - ] ++ lib.optional enableGC boehmgc; + ] ++ lib.optional enableGC boehmgc + ); dontBuild = !attrs.doBuild; doCheck = attrs.doCheck; From f4464873f514a1aa05ca43178dfca6e1c978f0c9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 27 Jul 2024 13:01:56 +0200 Subject: [PATCH 142/284] tests/nixos/remote-builds: Print hello world to stderr Trying to learn more about enigmatic spurious hang at https://hydra.nixos.org/build/267517233/nixlog/8 - builder1 seems to have started properly - ssh connection and session are established - ssh client doesn't exit or client.succeed does not return for some reason. Seeing the stdout on the console might give a tiny bit more info. --- tests/nixos/remote-builds.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/nixos/remote-builds.nix b/tests/nixos/remote-builds.nix index 1661203ec16..8813832f5b4 100644 --- a/tests/nixos/remote-builds.nix +++ b/tests/nixos/remote-builds.nix @@ -104,7 +104,10 @@ in builder.succeed("mkdir -p -m 700 /root/.ssh") builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") builder.wait_for_unit("sshd") - client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") + client.succeed(f""" + ssh -o StrictHostKeyChecking=no {builder.name} \ + 'echo hello world on $(hostname)' >&2 + """) # Perform a build and check that it was performed on the builder. out = client.succeed( From 7c5a0b06a4a7908560bca273049e561c0f8193b2 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 27 Jul 2024 13:08:30 +0200 Subject: [PATCH 143/284] tests/nixos/remote-builds: Wait for multi-user This should make the test more robust, considering the strange hang in https://hydra.nixos.org/build/267517233/nixlog/8 `builder` seems to have reached `multi-user.target` before the SSH connection was established, but this seems to be coincidental. This does tell us that enforcing this has a minimal cost in terms of runtime. Waiting for `multi-user.target` on the client is honestly paranoid, but flaky tests are very bad for productivity. --- tests/nixos/remote-builds.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/nixos/remote-builds.nix b/tests/nixos/remote-builds.nix index 8813832f5b4..8ddf6ad0281 100644 --- a/tests/nixos/remote-builds.nix +++ b/tests/nixos/remote-builds.nix @@ -104,6 +104,11 @@ in builder.succeed("mkdir -p -m 700 /root/.ssh") builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") builder.wait_for_unit("sshd") + # Make sure the builder can handle our login correctly + builder.wait_for_unit("multi-user.target") + # Make sure there's no funny business on the client either + # (should not be necessary, but we have reason to be careful) + client.wait_for_unit("multi-user.target") client.succeed(f""" ssh -o StrictHostKeyChecking=no {builder.name} \ 'echo hello world on $(hostname)' >&2 From cc5b8cdc858467a759405ff8bba074bb9464b3b8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 27 Jul 2024 13:42:03 +0200 Subject: [PATCH 144/284] buildNoTests: Disable unit tests This seems to have been the intent all along. The odd combination of unit tests, but no functional tests caused a build error where some data for the unit test was source-filtered out. Apparently. It's unclear to me why that happened, so I'm proposing this alternate "fix" to get the buildNoTests to pass. It would be nice to test more configurations, but this mode of building is on the way out anyway, so let's just make it pass and see what configurations make sense to test as part of the meson migration. --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 2384c29741b..674c5ae8cee 100644 --- a/flake.nix +++ b/flake.nix @@ -143,6 +143,7 @@ nix_noTests = final.nix.override { doInstallCheck = false; + doCheck = false; }; # See https://github.com/NixOS/nixpkgs/pull/214409 From aa2b1d10e29b3839760f02b1e30e70d70cb44ed2 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 27 Jul 2024 14:58:57 +0200 Subject: [PATCH 145/284] Update doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md --- .../10734-nix3-build-show-all-fod-errors-with-keep-going.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md b/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md index e4e2f797c5c..1d623e952e9 100644 --- a/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md +++ b/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md @@ -3,7 +3,7 @@ synopsis: "nix3-build: show all FOD errors with `--keep-going`" prs: 10734 --- -The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior of `nix build`. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. +The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) From e0198c513accf7f3b481ca71ff482840f294a1da Mon Sep 17 00:00:00 2001 From: Pino Toscano Date: Sun, 28 Jul 2024 11:40:16 +0200 Subject: [PATCH 146/284] libcmd: do not compile editline helpers when building w/ readline The internal "completionCallback" and "listPossibleCallback" helpers are used only when building with editline; hence, do not build then when using readline, matching their usage in "ReadlineLikeInteracter::init()". --- src/libcmd/repl-interacter.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 76fe38780b6..187af46eaa4 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -35,6 +35,7 @@ void sigintHandler(int signo) static detail::ReplCompleterMixin * curRepl; // ugly +#ifndef USE_READLINE static char * completionCallback(char * s, int * match) { auto possible = curRepl->completePrefix(s); @@ -101,6 +102,7 @@ static int listPossibleCallback(char * s, char *** avp) return ac; } +#endif ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl) { From 1b47748e5a839c714864d51c9ec1ce86ea353456 Mon Sep 17 00:00:00 2001 From: Ivan Trubach Date: Sun, 28 Jul 2024 13:08:33 +0300 Subject: [PATCH 147/284] libstore: return ENOTSUP for getxattr functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change updates the seccomp profile to return ENOTSUP for getxattr functions family. This reflects the behavior of filesystems that don’t support extended attributes (or have an option to disable them), e.g. ext2. The current behavior is confusing for some programs because we can read extended attributes, but only get to know that they are not supported when setting them. In addition to that, ACLs on Linux are implemented via extended attributes internally and if we don’t return ENOTSUP, acl library converts file mode to ACL. https://git.savannah.nongnu.org/cgit/acl.git/tree/libacl/acl_get_file.c?id=d9bb1759d4dad2f28a6dcc8c1742ff75d16dd10d#n69 --- src/libstore/unix/build/local-derivation-goal.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 364e9a14851..d3482df17a1 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -1702,10 +1702,13 @@ void setupSeccomp() throw SysError("unable to add seccomp rule"); } - /* Prevent builders from creating EAs or ACLs. Not all filesystems + /* Prevent builders from using EAs or ACLs. Not all filesystems support these, and they're not allowed in the Nix store because they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(getxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lgetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fgetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) throw SysError("unable to add seccomp rule"); From 933f2c086a96d8ce358cc787cf06c50a07263e9e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sun, 28 Jul 2024 15:34:48 +0200 Subject: [PATCH 148/284] docs: fix link to building instructions (#11207) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ab647e53b1f..021e54a3b23 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Full reference documentation can be found in the [Nix manual](https://nix.dev/re ## Building and developing -Follow instructions in the Nix reference manual to [set up a development environment and build Nix from source](https://nix.dev/manual/nix/development/development/building.html). +Follow instructions in the Nix reference manual to [set up a development environment and build Nix from source](https://nix.dev/manual/nix/development/building.html). ## Contributing From 96e06b2b06266684d5d74f5f809bdd82ebeb7b04 Mon Sep 17 00:00:00 2001 From: Pino Toscano Date: Sun, 28 Jul 2024 11:31:28 +0200 Subject: [PATCH 149/284] libutil: remove template id from constructors This is not allowed in C++20, and GCC 14 warns about it: ../src/libutil/ref.hh:26:20: warning: template-id not allowed for constructor in C++20 [-Wtemplate-id-cdtor] 26 | explicit ref(const std::shared_ptr & p) | ^ ../src/libutil/ref.hh:26:20: note: remove the '< >' ../src/libutil/ref.hh:33:21: warning: template-id not allowed for constructor in C++20 [-Wtemplate-id-cdtor] 33 | explicit ref(T * p) | ^ ../src/libutil/ref.hh:33:21: note: remove the '< >' --- src/libutil/ref.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index 016fdd74a39..3d0e64ab477 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -23,14 +23,14 @@ public: : p(r.p) { } - explicit ref(const std::shared_ptr & p) + explicit ref(const std::shared_ptr & p) : p(p) { if (!p) throw std::invalid_argument("null pointer cast to ref"); } - explicit ref(T * p) + explicit ref(T * p) : p(p) { if (!p) From c34077578ea48af7534f4dd2aaeef39cb64d9814 Mon Sep 17 00:00:00 2001 From: Pino Toscano Date: Sun, 28 Jul 2024 17:33:24 +0200 Subject: [PATCH 150/284] libutil: fix/improve includes in current-process.cc - move from a __linux__ block to a !_WIN32 block: this matches what the actual code does, using getrlimit() & setrlimit() in !_WIN32 blocks - drop , which is not portable, and it is not used --- src/libutil/current-process.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 6ca48220d21..c2b1ac500f4 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -15,13 +15,12 @@ #if __linux__ # include -# include # include "cgroup.hh" # include "namespaces.hh" #endif #ifndef _WIN32 -# include +# include #endif namespace nix { From e0012b97abb4c97ccf7fb20299d7b62dd906e89d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Jul 2024 14:26:25 +0200 Subject: [PATCH 151/284] Split tarball-specific logic from GitFileSystemObjectSink --- src/libfetchers/git-utils.cc | 33 ++++++++++++++++------------- src/libfetchers/git-utils.hh | 11 ++++++++++ src/libfetchers/github.cc | 5 +++-- src/libfetchers/tarball.cc | 6 ++++-- tests/unit/libfetchers/git-utils.cc | 2 +- 5 files changed, 37 insertions(+), 20 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 032d8e0bd3f..c2d4fe6aa1d 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -486,6 +486,24 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return narHash; } + + Hash dereferenceSingletonDirectory(const Hash & oid_) override + { + auto oid = hashToOID(oid_); + + /* If the root directory contains */ + auto _tree = lookupObject(*this, oid, GIT_OBJECT_TREE); + auto tree = (const git_tree *) &*_tree; + + if (git_tree_entrycount(tree) == 1) { + auto entry = git_tree_entry_byindex(tree, 0); + auto mode = git_tree_entry_filemode(entry); + if (mode == GIT_FILEMODE_BLOB || mode == GIT_FILEMODE_TREE) + oid = *git_tree_entry_id(entry); + } + + return toHash(oid); + } }; ref GitRepo::openRepo(const std::filesystem::path & path, bool create, bool bare) @@ -991,21 +1009,6 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink auto [oid, _name] = popBuilder(); - /* If the root directory contains a single entry that is a - directory or a non-executable regular file, return that as - the top-level object. We don't do this for executables - because they don't have a tree hash in the Git object - model. */ - auto _tree = lookupObject(*repo, oid, GIT_OBJECT_TREE); - auto tree = (const git_tree *) &*_tree; - - if (git_tree_entrycount(tree) == 1) { - auto entry = git_tree_entry_byindex(tree, 0); - auto mode = git_tree_entry_filemode(entry); - if (mode == GIT_FILEMODE_BLOB || mode == GIT_FILEMODE_TREE) - oid = *git_tree_entry_id(entry); - } - return toHash(oid); } }; diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 495916f6283..644f22a0718 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -98,6 +98,17 @@ struct GitRepo * serialisation. This is memoised on-disk. */ virtual Hash treeHashToNarHash(const Hash & treeHash) = 0; + + /** + * If the specified Git object is a directory with a single entry + * that is a directory or a non-executable regular file, return + * the ID of that object. + * + * Note: We don't do this for executable files because they don't + * have a tree hash in the Git object model that distinguishes + * them from non-executable files. + */ + virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0; }; ref getTarballCache(); diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 2968d2df2a4..a2ac9247a8b 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -255,11 +255,12 @@ struct GitArchiveInputScheme : InputScheme }); TarArchive archive { *source }; - auto parseSink = getTarballCache()->getFileSystemObjectSink(); + auto tarballCache = getTarballCache(); + auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); TarballInfo tarballInfo { - .treeHash = parseSink->sync(), + .treeHash = tarballCache->dereferenceSingletonDirectory(parseSink->sync()), .lastModified = lastModified }; diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 55db3eafbc1..b09f628a4a4 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -164,7 +164,8 @@ DownloadTarballResult downloadTarball( TarArchive{path}; }) : TarArchive{*source}; - auto parseSink = getTarballCache()->getFileSystemObjectSink(); + auto tarballCache = getTarballCache(); + auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); auto res(_res->lock()); @@ -177,7 +178,8 @@ DownloadTarballResult downloadTarball( infoAttrs = cached->value; } else { infoAttrs.insert_or_assign("etag", res->etag); - infoAttrs.insert_or_assign("treeHash", parseSink->sync().gitRev()); + infoAttrs.insert_or_assign("treeHash", + tarballCache->dereferenceSingletonDirectory(parseSink->sync()).gitRev()); infoAttrs.insert_or_assign("lastModified", uint64_t(lastModified)); if (res->immutableUrl) infoAttrs.insert_or_assign("immutableUrl", *res->immutableUrl); diff --git a/tests/unit/libfetchers/git-utils.cc b/tests/unit/libfetchers/git-utils.cc index f0d38d50c2e..de5110cc39e 100644 --- a/tests/unit/libfetchers/git-utils.cc +++ b/tests/unit/libfetchers/git-utils.cc @@ -77,7 +77,7 @@ TEST_F(GitUtilsTest, sink_basic) // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello")); - auto result = sink->sync(); + auto result = repo->dereferenceSingletonDirectory(sink->sync()); auto accessor = repo->getAccessor(result, false); auto entries = accessor->readDirectory(CanonPath::root); ASSERT_EQ(entries.size(), 5); From 7c18b4d0600bfb8ad4159712025f1dcb790aa36f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Jul 2024 14:34:02 +0200 Subject: [PATCH 152/284] Don't dereference top-level regular files Since this yielded an empty directory as far back as Nix 2.3, we don't really need special handling for executables vs non-executables. --- src/libfetchers/git-utils.cc | 3 +-- src/libfetchers/git-utils.hh | 7 +------ tests/functional/tarball.sh | 10 +--------- 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index c2d4fe6aa1d..114aa4ec078 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -491,14 +491,13 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this { auto oid = hashToOID(oid_); - /* If the root directory contains */ auto _tree = lookupObject(*this, oid, GIT_OBJECT_TREE); auto tree = (const git_tree *) &*_tree; if (git_tree_entrycount(tree) == 1) { auto entry = git_tree_entry_byindex(tree, 0); auto mode = git_tree_entry_filemode(entry); - if (mode == GIT_FILEMODE_BLOB || mode == GIT_FILEMODE_TREE) + if (mode == GIT_FILEMODE_TREE) oid = *git_tree_entry_id(entry); } diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 644f22a0718..ca37f2e805f 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -101,12 +101,7 @@ struct GitRepo /** * If the specified Git object is a directory with a single entry - * that is a directory or a non-executable regular file, return - * the ID of that object. - * - * Note: We don't do this for executable files because they don't - * have a tree hash in the Git object model that distinguishes - * them from non-executable files. + * that is a directory, return the ID of that object. */ virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0; }; diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index 6799831ce4a..4d89456255f 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -92,15 +92,7 @@ path="$(nix flake prefetch --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .s [[ -d "$path/foo" ]] [[ -d "$path/bar" ]] -# Test a tarball that has a single non-executable regular file. -rm -rf "$TEST_ROOT/tar_root" -mkdir -p "$TEST_ROOT/tar_root" -echo bar > "$TEST_ROOT/tar_root/foo" -tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . -path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" -[[ $(cat "$path") = bar ]] - -# Test a tarball that has a single executable regular file. +# Test a tarball that has a single regular file. rm -rf "$TEST_ROOT/tar_root" mkdir -p "$TEST_ROOT/tar_root" echo bar > "$TEST_ROOT/tar_root/foo" From 71865dee2d2818a0955e784901de9c93a1f2baf7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Jul 2024 15:04:55 +0200 Subject: [PATCH 153/284] Fix fetchTarball docs --- src/libexpr/primops/fetchTree.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 9a3b81bc998..f79b6b7b83a 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -560,10 +560,10 @@ static RegisterPrimOp primop_fetchTarball({ Download the specified URL, unpack it and return the path of the unpacked tree. The file must be a tape archive (`.tar`) compressed with `gzip`, `bzip2` or `xz`. If the tarball consists of a - single directory or non-executable file, then the top-level path - component of the files in the tarball is removed. The typical - use of the function is to obtain external Nix expression - dependencies, such as a particular version of Nixpkgs, e.g. + single directory, then the top-level path component of the files + in the tarball is removed. The typical use of the function is to + obtain external Nix expression dependencies, such as a + particular version of Nixpkgs, e.g. ```nix with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {}; From e8bf2e74a5361033104670ab7148e6f8843073af Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Jul 2024 15:09:06 +0200 Subject: [PATCH 154/284] Add release note --- doc/manual/rl-next/tarball-fixes.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/manual/rl-next/tarball-fixes.md diff --git a/doc/manual/rl-next/tarball-fixes.md b/doc/manual/rl-next/tarball-fixes.md new file mode 100644 index 00000000000..c938e9db6c2 --- /dev/null +++ b/doc/manual/rl-next/tarball-fixes.md @@ -0,0 +1,9 @@ +--- +synopsis: "Improve handling of tarballs that don't consist of a single top-level directory" +prs: +- 11195 +--- + +In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). + +Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) From a3171cec541a8df3be19e1a5a60e7cd4ae978ef9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Jul 2024 15:12:01 +0200 Subject: [PATCH 155/284] Update src/libfetchers/git-utils.hh Co-authored-by: Robert Hensing --- src/libfetchers/git-utils.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index ca37f2e805f..9152528680c 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -102,6 +102,7 @@ struct GitRepo /** * If the specified Git object is a directory with a single entry * that is a directory, return the ID of that object. + * Otherwise, return the passed ID unchanged. */ virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0; }; From 12717325ccb5afeb686c8ddd02f9c9b623021f86 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 29 Jul 2024 12:16:05 -0400 Subject: [PATCH 156/284] Make sure we use `-isystem` with Meson on some deps Otherwise we get warnings on external code. --- src/libexpr/meson.build | 8 +++++++- src/libexpr/primops/fromTOML.cc | 3 --- src/libstore/meson.build | 3 ++- src/libstore/s3-binary-cache-store.cc | 3 --- src/libutil/meson.build | 1 + 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index fa90e7b41a3..4d8a38b435c 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -32,6 +32,7 @@ subdir('build-utils-meson/threads') boost = dependency( 'boost', modules : ['container', 'context'], + include_type: 'system', ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we # put in `deps_other`. @@ -55,7 +56,12 @@ if bdw_gc.found() endif configdata.set('HAVE_BOEHMGC', bdw_gc.found().to_int()) -toml11 = dependency('toml11', version : '>=3.7.0', method : 'cmake') +toml11 = dependency( + 'toml11', + version : '>=3.7.0', + method : 'cmake', + include_type: 'system', +) deps_other += toml11 config_h = configure_file( diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 70755f9e0b5..b4f1df7a826 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -3,10 +3,7 @@ #include -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wswitch-enum" #include -#pragma GCC diagnostic pop namespace nix { diff --git a/src/libstore/meson.build b/src/libstore/meson.build index cb8110f3fa7..50b15e15dc7 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -73,6 +73,7 @@ subdir('build-utils-meson/threads') boost = dependency( 'boost', modules : ['container'], + include_type: 'system', ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we # put in `deps_other`. @@ -113,7 +114,7 @@ if aws_s3.found() '-laws-cpp-sdk-core', '-laws-crt-cpp', ], - ) + ).as_system('system') endif deps_other += aws_s3 diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 1082657bb0e..92ab47cd66d 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -10,8 +10,6 @@ #include "compression.hh" #include "filetransfer.hh" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wswitch-enum" #include #include #include @@ -27,7 +25,6 @@ #include #include #include -#pragma GCC diagnostic pop using namespace Aws::Transfer; diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 04c778c3114..8552c4c9dad 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -62,6 +62,7 @@ endif boost = dependency( 'boost', modules : ['context', 'coroutine'], + include_type: 'system', ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we # put in `deps_other`. From f380becffacaabcfbace6b00e640cdc7bd0bbf64 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 29 Jul 2024 23:25:31 +0200 Subject: [PATCH 157/284] Credit all contributors in release notes --- .../data/release-credits-email-to-handle.json | 51 +++++ .../data/release-credits-handle-to-name.json | 44 +++++ maintainers/release-credits | 176 ++++++++++++++++++ maintainers/release-notes | 7 + maintainers/release-process.md | 4 + 5 files changed, 282 insertions(+) create mode 100644 maintainers/data/release-credits-email-to-handle.json create mode 100644 maintainers/data/release-credits-handle-to-name.json create mode 100755 maintainers/release-credits diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json new file mode 100644 index 00000000000..573ec2b3100 --- /dev/null +++ b/maintainers/data/release-credits-email-to-handle.json @@ -0,0 +1,51 @@ +{ + "bogus": "bogus", + "edolstra@gmail.com": "edolstra", + "roberth@users.noreply.github.com": "roberth", + "toscano.pino@tiscali.it": "pinotree", + "valentin@gagarin.work": "fricklerhandwerk", + "mr.trubach@icloud.com": "tie", + "robert@roberthensing.nl": "roberth", + "lix@jade.fyi": "lf-", + "cole.e.helbling@outlook.com": "cole-h", + "joerg@thalheim.io": "Mic92", + "John.Ericson@Obsidian.Systems": "Ericson2314", + "ryan.hendrickson@alum.mit.edu": "rhendric", + "67135060+poweredbypie@users.noreply.github.com": "poweredbypie", + "detroyejr@outlook.com": "detroyejr", + "silvan.mosberger@tweag.io": "infinisil", + "vcs@emily.moe": "emilazy", + "farid.m.zakaria@gmail.com": "fzakaria", + "22859658+RTUnreal@users.noreply.github.com": "RTUnreal", + "me@las.rs": "L-as", + "philip.taron@gmail.com": "philiptaron", + "root@goldstein.rs": "GoldsteinE", + "tomberek@users.noreply.github.com": "tomberek", + "lexi.mattick@neuralink.com": "kognise", + "andrew@johnandrewmarshall.com": "amarshall", + "contact@romain-neil.fr": "romain-neil", + "Mic92@users.noreply.github.com": "Mic92", + "valentin.gagarin@tweag.io": "fricklerhandwerk", + "siddhantk232@gmail.com": "siddhantk232", + "kn@openbsd.org": "klemensn", + "slyich@gmail.com": "trofi", + "theophane.hufschmitt@tweag.io": "thufschmitt", + "alicebob@lijzij.de": "alicebob", + "winter@winter.cafe": "winterqt", + "brian@brianmckenna.org": "puffnfresh", + "git@haenoe.party": "haenoe", + "peshogo@gmail.com": "pineapplehunter", + "poweredbypie@users.noreply.github.com": "poweredbypie", + "arthur200126@gmail.com": "Artoria2e5", + "tomberek@gmail.com": "tomberek", + "jaredbaur@fastmail.com": "jmbaur", + "andreas@rammhold.de": "andir", + "hamirmahal@gmail.com": "hamirmahal", + "git@JohnEricson.me": "Ericson2314", + "8763518+SkamDart@users.noreply.github.com": "SkamDart", + "kirillrdy@gmail.com": "kirillrdy", + "pennae@lix.systems": "pennae", + "delroth@gmail.com": "delroth", + "enno@nerdworks.de": "elohmeier", + "mjbauer95@gmail.com": "matthewbauer" +} \ No newline at end of file diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json new file mode 100644 index 00000000000..d68311dde10 --- /dev/null +++ b/maintainers/data/release-credits-handle-to-name.json @@ -0,0 +1,44 @@ +{ + "fzakaria": "Farid Zakaria", + "kognise": "Lexi Mattick", + "L-as": "Las Safin", + "haenoe": "HaeNoe", + "andir": "Andreas Rammhold", + "matthewbauer": "Matthew Bauer", + "emilazy": "Emily", + "pineapplehunter": "Shogo Takata", + "RTUnreal": null, + "jmbaur": "Jared Baur", + "Ericson2314": "John Ericson", + "pinotree": "Pino Toscano", + "tie": "Ivan Trubach", + "poweredbypie": null, + "fricklerhandwerk": "Valentin Gagarin", + "Mic92": "J\u00f6rg Thalheim", + "alicebob": "Harmen", + "elohmeier": "Enno Richter", + "delroth": "Pierre Bourdon", + "kirillrdy": null, + "thufschmitt": "Th\u00e9ophane Hufschmitt", + "detroyejr": "Jonathan De Troye", + "klemensn": "Klemens Nanni", + "tomberek": null, + "rhendric": "Ryan Hendrickson", + "philiptaron": "Philip Taron", + "puffnfresh": "Brian McKenna", + "lf-": "jade", + "romain-neil": "Romain Neil", + "hamirmahal": "Hamir Mahal", + "edolstra": "Eelco Dolstra", + "Artoria2e5": "Mingye Wang", + "SkamDart": "Cameron", + "roberth": "Robert Hensing", + "amarshall": "Andrew Marshall", + "trofi": "Sergei Trofimovich", + "cole-h": "Cole Helbling", + "infinisil": "Silvan Mosberger", + "siddhantk232": "Siddhant Kumar", + "winterqt": "Winter", + "GoldsteinE": "Max \u201cGoldstein\u201d Siling", + "pennae": null +} \ No newline at end of file diff --git a/maintainers/release-credits b/maintainers/release-credits new file mode 100755 index 00000000000..8350c4e2a18 --- /dev/null +++ b/maintainers/release-credits @@ -0,0 +1,176 @@ +#!/usr/bin/env nix +#!nix develop --impure --expr +#!nix `` +#!nix let flake = builtins.getFlake ("git+file://" + toString ../.); +#!nix pkgs = flake.inputs.nixpkgs.legacyPackages.${builtins.currentSystem}; +#!nix in pkgs.mkShell { nativeBuildInputs = [ +#!nix (pkgs.python3.withPackages (ps: with ps; [ requests ])) +#!nix ]; } +#!nix `` --command python3 + +# This script lists out the contributors for a given release. +# It must be run from the root of the Nix repository. + +import os +import sys +import json +import requests + +github_token = os.environ.get("GITHUB_TOKEN") +if not github_token: + print("GITHUB_TOKEN is not set. If you hit the rate limit, set it", file=sys.stderr) + # Might be ok, as we have a cache. + # raise ValueError("GITHUB_TOKEN must be set") + +# 1. Read the current version in .version +version = os.environ.get("VERSION") +if not version: + version = open(".version").read().strip() + +print(f"Generating release credits for Nix {version}", file=sys.stderr) + +# 2. Compute previous version +vcomponents = version.split(".") +if len(vcomponents) >= 2: + prev_version = f"{vcomponents[0]}.{int(vcomponents[1])-1}.0" +else: + raise ValueError(".version must have at least two components") + +# For unreleased versions +endref = "HEAD" +# For older releases +# endref = version + +# 2. Find the merge base between the current version and the previous version +mergeBase = os.popen(f"git merge-base {prev_version} {endref}").read().strip() +print(f"Merge base between {prev_version} and {endref} is {mergeBase}", file=sys.stderr) + +# 3. Find the date of the merge base +mergeBaseDate = os.popen(f"git show -s --format=%ci {mergeBase}").read().strip()[0:10] +print(f"Merge base date is {mergeBaseDate}", file=sys.stderr) + +# 4. Get the commits between the merge base and the current version + +def get_commits(): + raw = os.popen(f"git log --pretty=format:'%H\t%an\t%ae' {mergeBase}..{endref}").read().strip() + lines = raw.split("\n") + return [ { "hash": items[0], "author": items[1], "email": items[2] } + for line in lines + for items in (line.split("\t"),) + ] + +def commits_to_first_commit_by_email(commits): + by_email = dict() + for commit in commits: + email = commit["email"] + if email not in by_email: + by_email[email] = commit + return by_email + + +samples = commits_to_first_commit_by_email(get_commits()) + +# For quick testing, only pick two samples from the dict +# samples = dict(list(samples.items())[:2]) + +# Query the GitHub API to get handle +def get_github_commit(commit): + url = f"https://api.github.com/repos/NixOS/nix/commits/{commit['hash']}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + return response.json() + +class Cache: + def __init__(self, filename, require = True): + self.filename = filename + try: + with open(filename, "r") as f: + self.values = json.load(f) + except FileNotFoundError: + if require: + raise + self.values = dict() + def save(self): + with open(self.filename, "w") as f: + json.dump(self.values, f, indent=4) + print(f"Saved cache to {self.filename}", file=sys.stderr) + +# The email to handle cache maps email addresses to either +# - a handle (string) +# - None (if no handle was found) +email_to_handle_cache = Cache("maintainers/data/release-credits-email-to-handle.json") + +handles = set() +emails = dict() + +for sample in samples: + s = samples[sample] + email = s["email"] + if not email in email_to_handle_cache.values: + print(f"Querying GitHub API for {s['hash']}, to get handle for {s['email']}") + ghc = get_github_commit(samples[sample]) + gha = ghc["author"] + if gha and gha["login"]: + handle = gha["login"] + print(f"Handle: {handle}") + email_to_handle_cache.values[email] = handle + else: + print(f"Found no handle for {s['email']}") + email_to_handle_cache.values[email] = None + handle = email_to_handle_cache.values[email] + if handle is not None: + handles.add(handle) + else: + emails[email] = s["author"] + +# print(email_to_handle_cache.values) + +email_to_handle_cache.save() + +handle_to_name_cache = Cache("maintainers/data/release-credits-handle-to-name.json") + +print(f"Found {len(handles)} handles", file=sys.stderr) + +for handle in handles: + if not handle in handle_to_name_cache.values: + print(f"Querying GitHub API for {handle}, to get name", file=sys.stderr) + url = f"https://api.github.com/users/{handle}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + user = response.json() + name = user["name"] + print(f"Name: {name}", file=sys.stderr) + handle_to_name_cache.values[handle] = name + +handle_to_name_cache.save() + +entries = list() + +for handle in handles: + name = handle_to_name_cache.values[handle] + if name is None: + # This way it looks more regular + name = handle + entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] + +def shuffle(entries): + salt = os.urandom(16) + return sorted(entries, key=lambda x: hash((x, salt))) + +# Fair ordering is undecidable +entries = shuffle(entries) + +# For a sanity check, we could sort the entries by handle instead. +# entries = sorted(entries) + +print("") +print(f"This release was made possible by the following {len(entries)} contributors:") +print("") + +for entry in entries: + print(entry) + +for email in emails: + print(f"- {emails[email]}") diff --git a/maintainers/release-notes b/maintainers/release-notes index 0fca5abf27e..b7b5731c89e 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -151,6 +151,13 @@ section_title="Release $version_full ($DATE)" echo "# $section_title" echo changelog-d doc/manual/rl-next | sed -e 's/ *$//' + + if ! $IS_PATCH; then + echo + echo "# Contributors" + echo + VERSION=$version_full ./maintainers/release-credits + fi ) | tee -a $file log "Wrote $file" diff --git a/maintainers/release-process.md b/maintainers/release-process.md index da6886ea94d..7a2b3c0a79b 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -39,6 +39,10 @@ release: * Proof-read / edit / rearrange the release notes if needed. Breaking changes and highlights should go to the top. +* Run `maintainers/release-credits` to make sure the credits script works + and produces a sensible output. Some emails might not automatically map to + a GitHub handle. + * Push. ```console From 850e4779d6285feffd78dbf4c12eb203a3f10637 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 30 Jul 2024 00:15:49 +0200 Subject: [PATCH 158/284] release-credits: Remove bots --- maintainers/release-credits | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/maintainers/release-credits b/maintainers/release-credits index 8350c4e2a18..18641a07b2d 100755 --- a/maintainers/release-credits +++ b/maintainers/release-credits @@ -148,12 +148,19 @@ handle_to_name_cache.save() entries = list() +def is_bot(handle): + return handle.endswith("[bot]") + for handle in handles: name = handle_to_name_cache.values[handle] if name is None: # This way it looks more regular name = handle - entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] + + # Note to bots: I'm only doing this because it would distract humans, and + # therefore delay your creation. I'm sorry; it's for your benefit. + if not is_bot(handle): + entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] def shuffle(entries): salt = os.urandom(16) From d7f46cf28ea521eef0530a1a41af56a34dd89278 Mon Sep 17 00:00:00 2001 From: Pino Toscano Date: Tue, 30 Jul 2024 05:29:32 +0200 Subject: [PATCH 159/284] makefiles: recognize GNU/Hurd Set HOST_HURD & HOST_UNIX for GNU/Hurd in the makefile-based build system; the latter variable is important as it will include all the commit Unix bits. --- mk/platform.mk | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mk/platform.mk b/mk/platform.mk index fe960dedf73..22c114a2077 100644 --- a/mk/platform.mk +++ b/mk/platform.mk @@ -29,4 +29,8 @@ ifdef HOST_OS HOST_SOLARIS = 1 HOST_UNIX = 1 endif + ifeq ($(HOST_KERNEL), gnu) + HOST_HURD = 1 + HOST_UNIX = 1 + endif endif From 7442f4a16143fe69a71949f1edbbbcab6862ed23 Mon Sep 17 00:00:00 2001 From: Pino Toscano Date: Tue, 30 Jul 2024 05:31:42 +0200 Subject: [PATCH 160/284] libutil: use /proc/self/exe on Hurd as well Rely on the Linux-compatible procfs available on the Hurd to get the path of the current executable. --- src/libutil/current-process.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index c2b1ac500f4..0bc46d7464c 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -137,7 +137,7 @@ std::optional getSelfExe() { static auto cached = []() -> std::optional { - #if __linux__ + #if __linux__ || __GNU__ return readLink("/proc/self/exe"); #elif __APPLE__ char buf[1024]; From a1ccf6061396f398526bb99518cfd3b9c432aeb1 Mon Sep 17 00:00:00 2001 From: Pino Toscano Date: Tue, 30 Jul 2024 05:34:34 +0200 Subject: [PATCH 161/284] tests: define fallback PATH_MAX Few filesystem-related tests rely on PATH_MAX for buffers, and PATH_MAX is optional in POSIX (and not available on the Hurd). To make them build and pass, provide a fallback definition of PATH_MAX in case not available. Ideally speaking, the tests ought to not unconditionally rely on PATH_MAX, do alternative strategies (e.g. dynamically allocate buffers, expand them as needed, etc); OTOH this is test code, so it would be more work that what it would be worth, so IMHO the define fallback is good enough. --- tests/unit/libutil/tests.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index 8a3ca8561e5..2b73d323b8d 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -17,6 +17,10 @@ # define FS_ROOT FS_SEP #endif +#ifndef PATH_MAX +# define PATH_MAX 4096 +#endif + namespace nix { /* ----------- tests for util.hh ------------------------------------------------*/ From ee86e7f361c55c8c7dc2e45c3868802af249aeff Mon Sep 17 00:00:00 2001 From: Corbin Simpson Date: Tue, 30 Jul 2024 05:51:47 -0700 Subject: [PATCH 162/284] doc/command-ref/nix-shell: Shebangs can occur anywhere (#11202) Co-authored-by: Valentin Gagarin --- doc/manual/src/command-ref/nix-shell.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index ddec30f5b3e..69a711bd5e6 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -297,3 +297,8 @@ with import {}; runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" ``` + +The script's file name is passed as the first argument to the interpreter specified by the `-i` flag. + +Aside from the very first line, which is a directive to the operating system, the additional `#! nix-shell` lines do not need to be at the beginning of the file. +This allows wrapping them in block comments for languages where `#` does not start a comment, such as ECMAScript, Erlang, PHP, or Ruby. From f011cfd28dc359bb786990d94b63c2c482e1b8da Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 30 Jul 2024 17:35:46 +0200 Subject: [PATCH 163/284] maintainers/release-*: Add mode line This lets various tools figure out the language more easily. --- maintainers/release-credits | 1 + maintainers/release-notes | 1 + 2 files changed, 2 insertions(+) diff --git a/maintainers/release-credits b/maintainers/release-credits index 18641a07b2d..7a5c87d7dfb 100755 --- a/maintainers/release-credits +++ b/maintainers/release-credits @@ -1,4 +1,5 @@ #!/usr/bin/env nix +# vim: set filetype=python: #!nix develop --impure --expr #!nix `` #!nix let flake = builtins.getFlake ("git+file://" + toString ../.); diff --git a/maintainers/release-notes b/maintainers/release-notes index b7b5731c89e..c0c4ee734de 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -1,4 +1,5 @@ #!/usr/bin/env nix +# vim: set filetype=bash: #!nix shell .#changelog-d --command bash # --- CONFIGURATION --- From ef8021744855754a12a4b0d343d902b2a8a29d8c Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Mon, 29 Jul 2024 10:42:43 -0400 Subject: [PATCH 164/284] fix: add flake headers --- src/libflake/flake/nix-flake.pc.in | 10 ++++++++++ src/libflake/local.mk | 5 +++++ src/nix/meson.build | 1 + 3 files changed, 16 insertions(+) create mode 100644 src/libflake/flake/nix-flake.pc.in diff --git a/src/libflake/flake/nix-flake.pc.in b/src/libflake/flake/nix-flake.pc.in new file mode 100644 index 00000000000..10c52f5e9bb --- /dev/null +++ b/src/libflake/flake/nix-flake.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Package Manager +Version: @PACKAGE_VERSION@ +Requires: nix-util nix-store nix-expr +Libs: -L${libdir} -lnixflake +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libflake/local.mk b/src/libflake/local.mk index 2cceda2bf25..5e604ef3abf 100644 --- a/src/libflake/local.mk +++ b/src/libflake/local.mk @@ -15,3 +15,8 @@ libflake_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetcher libflake_LDFLAGS += $(THREAD_LDFLAGS) libflake_LIBS = libutil libstore libfetchers libexpr + +$(eval $(call install-file-in, $(buildprefix)$(d)/flake/nix-flake.pc, $(libdir)/pkgconfig, 0644)) + +$(foreach i, $(wildcard src/libflake/flake/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) diff --git a/src/nix/meson.build b/src/nix/meson.build index 53bb083a9c3..7405548de0c 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -20,6 +20,7 @@ deps_private_maybe_subproject = [ dependency('nix-util'), dependency('nix-store'), dependency('nix-expr'), + dependency('nix-flake'), dependency('nix-fetchers'), dependency('nix-main'), dependency('nix-cmd'), From dd75711895057328b00f92353730e7f873eafd52 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 12 Jul 2024 14:56:30 +0200 Subject: [PATCH 165/284] Use std::strong_ordering for version comparison The actual motive here is the avoidance of integer overflow if we were to make these use checked NixInts and retain the subtraction. However, the actual *intent* of this code is a three-way comparison, which can be done with operator<=>, so we should just do *that* instead. Change-Id: I7f9a7da1f3176424b528af6d1b4f1591e4ab26bf --- src/libexpr/primops.cc | 3 ++- src/libstore/names.cc | 8 ++++---- src/libstore/names.hh | 2 +- src/nix-env/nix-env.cc | 22 +++++++++++----------- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 0b3b19b5764..a23583f34c2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4446,7 +4446,8 @@ static void prim_compareVersions(EvalState & state, const PosIdx pos, Value * * { auto version1 = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.compareVersions"); auto version2 = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.compareVersions"); - v.mkInt(compareVersions(version1, version2)); + auto result = compareVersions(version1, version2); + v.mkInt(result < 0 ? -1 : result > 0 ? 1 : 0); } static RegisterPrimOp primop_compareVersions({ diff --git a/src/libstore/names.cc b/src/libstore/names.cc index 277aabf0f93..c0e1b1022ac 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -94,7 +94,7 @@ static bool componentsLT(const std::string_view c1, const std::string_view c2) } -int compareVersions(const std::string_view v1, const std::string_view v2) +std::strong_ordering compareVersions(const std::string_view v1, const std::string_view v2) { auto p1 = v1.begin(); auto p2 = v2.begin(); @@ -102,11 +102,11 @@ int compareVersions(const std::string_view v1, const std::string_view v2) while (p1 != v1.end() || p2 != v2.end()) { auto c1 = nextComponent(p1, v1.end()); auto c2 = nextComponent(p2, v2.end()); - if (componentsLT(c1, c2)) return -1; - else if (componentsLT(c2, c1)) return 1; + if (componentsLT(c1, c2)) return std::strong_ordering::less; + else if (componentsLT(c2, c1)) return std::strong_ordering::greater; } - return 0; + return std::strong_ordering::equal; } diff --git a/src/libstore/names.hh b/src/libstore/names.hh index d82b99bb439..a6909d54593 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -30,7 +30,7 @@ typedef std::list DrvNames; std::string_view nextComponent(std::string_view::const_iterator & p, const std::string_view::const_iterator end); -int compareVersions(const std::string_view v1, const std::string_view v2); +std::strong_ordering compareVersions(const std::string_view v1, const std::string_view v2); DrvNames drvNamesFromArgs(const Strings & opArgs); } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 40c20054268..ba2baccee85 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -204,15 +204,15 @@ static void loadDerivations(EvalState & state, const SourcePath & nixExprPath, } -static long getPriority(EvalState & state, PackageInfo & drv) +static NixInt getPriority(EvalState & state, PackageInfo & drv) { - return drv.queryMetaInt("priority", 0); + return drv.queryMetaInt("priority", NixInt(0)); } -static long comparePriorities(EvalState & state, PackageInfo & drv1, PackageInfo & drv2) +static std::strong_ordering comparePriorities(EvalState & state, PackageInfo & drv1, PackageInfo & drv2) { - return getPriority(state, drv2) - getPriority(state, drv1); + return getPriority(state, drv2) <=> getPriority(state, drv1); } @@ -280,7 +280,7 @@ std::vector pickNewestOnly(EvalState & state, std::vector matches) auto & oneDrv = match.packageInfo; const auto drvName = DrvName { oneDrv.queryName() }; - long comparison = 1; + std::strong_ordering comparison = std::strong_ordering::greater; const auto itOther = newest.find(drvName.name); @@ -288,9 +288,9 @@ std::vector pickNewestOnly(EvalState & state, std::vector matches) auto & newestDrv = itOther->second.packageInfo; comparison = - oneDrv.querySystem() == newestDrv.querySystem() ? 0 : - oneDrv.querySystem() == settings.thisSystem ? 1 : - newestDrv.querySystem() == settings.thisSystem ? -1 : 0; + oneDrv.querySystem() == newestDrv.querySystem() ? std::strong_ordering::equal : + oneDrv.querySystem() == settings.thisSystem ? std::strong_ordering::greater : + newestDrv.querySystem() == settings.thisSystem ? std::strong_ordering::less : std::strong_ordering::equal; if (comparison == 0) comparison = comparePriorities(state, oneDrv, newestDrv); if (comparison == 0) @@ -625,13 +625,13 @@ static void upgradeDerivations(Globals & globals, continue; DrvName newName(j->queryName()); if (newName.name == drvName.name) { - int d = compareVersions(drvName.version, newName.version); + std::strong_ordering d = compareVersions(drvName.version, newName.version); if ((upgradeType == utLt && d < 0) || (upgradeType == utLeq && d <= 0) || (upgradeType == utEq && d == 0) || upgradeType == utAlways) { - long d2 = -1; + std::strong_ordering d2 = std::strong_ordering::less; if (bestElem != availElems.end()) { d2 = comparePriorities(*globals.state, *bestElem, *j); if (d2 == 0) d2 = compareVersions(bestVersion, newName.version); @@ -902,7 +902,7 @@ static VersionDiff compareVersionAgainstSet( for (auto & i : elems) { DrvName name2(i.queryName()); if (name.name == name2.name) { - int d = compareVersions(name.version, name2.version); + std::strong_ordering d = compareVersions(name.version, name2.version); if (d < 0) { diff = cvGreater; version = name2.version; From e28cb67d41643bf2073e46149e6f5d10d1fd0ef5 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 5 Jul 2024 15:52:55 +0200 Subject: [PATCH 166/284] libutil: add checked arithmetic tools This is in preparation for adding checked arithmetic to the evaluator. Change-Id: I6e115ce8f5411feda1706624977a4dcd5efd4d13 --- src/libutil/checked-arithmetic.hh | 182 ++++++++++++++++++ src/libutil/meson.build | 1 + .../tests/gtest-with-params.hh | 54 ++++++ tests/unit/libutil/checked-arithmetic.cc | 158 +++++++++++++++ 4 files changed, 395 insertions(+) create mode 100644 src/libutil/checked-arithmetic.hh create mode 100644 tests/unit/libutil-support/tests/gtest-with-params.hh create mode 100644 tests/unit/libutil/checked-arithmetic.cc diff --git a/src/libutil/checked-arithmetic.hh b/src/libutil/checked-arithmetic.hh new file mode 100644 index 00000000000..55d6ad2059d --- /dev/null +++ b/src/libutil/checked-arithmetic.hh @@ -0,0 +1,182 @@ +#pragma once +/** + * @file Checked arithmetic with classes that make it hard to accidentally make something an unchecked operation. + */ + +#include +#include // IWYU pragma: keep +#include +#include +#include +#include +#include + +namespace nix::checked { + +class DivideByZero : std::exception +{}; + +/** + * Numeric value enforcing checked arithmetic. Performing mathematical operations on such values will return a Result + * type which needs to be checked. + */ +template +struct Checked +{ + using Inner = T; + + // TODO: this must be a "trivial default constructor", which means it + // cannot set the value to NOT DO UB on uninit. + T value; + + Checked() = default; + explicit Checked(T const value) + : value{value} + { + } + Checked(Checked const & other) = default; + Checked(Checked && other) = default; + Checked & operator=(Checked const & other) = default; + + std::strong_ordering operator<=>(Checked const & other) const = default; + std::strong_ordering operator<=>(T const & other) const + { + return value <=> other; + } + + explicit operator T() const + { + return value; + } + + enum class OverflowKind { + NoOverflow, + Overflow, + DivByZero, + }; + + class Result + { + T value; + OverflowKind overflowed_; + + public: + Result(T value, bool overflowed) + : value{value} + , overflowed_{overflowed ? OverflowKind::Overflow : OverflowKind::NoOverflow} + { + } + Result(T value, OverflowKind overflowed) + : value{value} + , overflowed_{overflowed} + { + } + + bool operator==(Result other) const + { + return value == other.value && overflowed_ == other.overflowed_; + } + + std::optional valueChecked() const + { + if (overflowed_ != OverflowKind::NoOverflow) { + return std::nullopt; + } else { + return value; + } + } + + /** + * Returns the result as if the arithmetic were performed as wrapping arithmetic. + * + * \throws DivideByZero if the operation was a divide by zero. + */ + T valueWrapping() const + { + if (overflowed_ == OverflowKind::DivByZero) { + throw DivideByZero{}; + } + return value; + } + + bool overflowed() const + { + return overflowed_ == OverflowKind::Overflow; + } + + bool divideByZero() const + { + return overflowed_ == OverflowKind::DivByZero; + } + }; + + Result operator+(Checked const other) const + { + return (*this) + other.value; + } + Result operator+(T const other) const + { + T result; + bool overflowed = __builtin_add_overflow(value, other, &result); + return Result{result, overflowed}; + } + + Result operator-(Checked const other) const + { + return (*this) - other.value; + } + Result operator-(T const other) const + { + T result; + bool overflowed = __builtin_sub_overflow(value, other, &result); + return Result{result, overflowed}; + } + + Result operator*(Checked const other) const + { + return (*this) * other.value; + } + Result operator*(T const other) const + { + T result; + bool overflowed = __builtin_mul_overflow(value, other, &result); + return Result{result, overflowed}; + } + + Result operator/(Checked const other) const + { + return (*this) / other.value; + } + /** + * Performs a checked division. + * + * If the right hand side is zero, the result is marked as a DivByZero and + * valueWrapping will throw. + */ + Result operator/(T const other) const + { + constexpr T const minV = std::numeric_limits::min(); + + // It's only possible to overflow with signed division since doing so + // requires crossing the two's complement limits by MIN / -1 (since + // two's complement has one more in range in the negative direction + // than in the positive one). + if (std::is_signed() && (value == minV && other == -1)) { + return Result{minV, true}; + } else if (other == 0) { + return Result{0, OverflowKind::DivByZero}; + } else { + T result = value / other; + return Result{result, false}; + } + } +}; + +template +std::ostream & operator<<(std::ostream & ios, Checked v) +{ + ios << v.value; + return ios; +} + +} diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 8552c4c9dad..e0299177f35 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -171,6 +171,7 @@ headers = [config_h] + files( 'args/root.hh', 'callback.hh', 'canon-path.hh', + 'checked-arithmetic.hh', 'chunked-vector.hh', 'closure.hh', 'comparator.hh', diff --git a/tests/unit/libutil-support/tests/gtest-with-params.hh b/tests/unit/libutil-support/tests/gtest-with-params.hh new file mode 100644 index 00000000000..323a083feaf --- /dev/null +++ b/tests/unit/libutil-support/tests/gtest-with-params.hh @@ -0,0 +1,54 @@ +#pragma once +// SPDX-FileCopyrightText: 2014 Emil Eriksson +// +// SPDX-License-Identifier: BSD-2-Clause +// +// The lion's share of this code is copy pasted directly out of RapidCheck +// headers, so the copyright is set accordingly. +/** + * @file Implements the ability to run a RapidCheck test under gtest with changed + * test parameters such as the number of tests to run. This is useful for + * running very large numbers of the extremely cheap property tests. + */ + +#include +#include +#include + +namespace rc::detail { + +using MakeTestParams = TestParams (*)(); + +template +void checkGTestWith(Testable && testable, MakeTestParams makeTestParams) +{ + const auto testInfo = ::testing::UnitTest::GetInstance()->current_test_info(); + detail::TestMetadata metadata; + metadata.id = std::string(testInfo->test_case_name()) + "/" + std::string(testInfo->name()); + metadata.description = std::string(testInfo->name()); + + const auto result = checkTestable(std::forward(testable), metadata, makeTestParams()); + + if (result.template is()) { + const auto success = result.template get(); + if (!success.distribution.empty()) { + printResultMessage(result, std::cout); + std::cout << std::endl; + } + } else { + std::ostringstream ss; + printResultMessage(result, ss); + FAIL() << ss.str() << std::endl; + } +} +} + +#define RC_GTEST_PROP_WITH_PARAMS(TestCase, Name, MakeParams, ArgList) \ + void rapidCheck_propImpl_##TestCase##_##Name ArgList; \ + \ + TEST(TestCase, Name) \ + { \ + ::rc::detail::checkGTestWith(&rapidCheck_propImpl_##TestCase##_##Name, MakeParams); \ + } \ + \ + void rapidCheck_propImpl_##TestCase##_##Name ArgList diff --git a/tests/unit/libutil/checked-arithmetic.cc b/tests/unit/libutil/checked-arithmetic.cc new file mode 100644 index 00000000000..75018660dc8 --- /dev/null +++ b/tests/unit/libutil/checked-arithmetic.cc @@ -0,0 +1,158 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "tests/gtest-with-params.hh" + +namespace rc { +using namespace nix; + +template +struct Arbitrary> +{ + static Gen> arbitrary() + { + return gen::arbitrary(); + } +}; + +} + +namespace nix::checked { + +// Pointer to member function! Mildly gross. +template +using Oper = Checked::Result (Checked::*)(T const other) const; + +template +using ReferenceOper = T (*)(T a, T b); + +/** + * Checks that performing an operation that overflows into an inaccurate result + * has the desired behaviour. + * + * TBig is a type large enough to represent all results of TSmall operations. + */ +template +void checkType(TSmall a_, TSmall b, Oper oper, ReferenceOper reference) +{ + // Sufficient to fit all values + TBig referenceResult = reference(a_, b); + constexpr const TSmall minV = std::numeric_limits::min(); + constexpr const TSmall maxV = std::numeric_limits::max(); + + Checked a{a_}; + auto result = (a.*(oper))(b); + + // Just truncate it to get the in-range result + RC_ASSERT(result.valueWrapping() == static_cast(referenceResult)); + + if (referenceResult > maxV || referenceResult < minV) { + RC_ASSERT(result.overflowed()); + RC_ASSERT(!result.valueChecked().has_value()); + } else { + RC_ASSERT(!result.overflowed()); + RC_ASSERT(result.valueChecked().has_value()); + RC_ASSERT(*result.valueChecked() == referenceResult); + } +} + +/** + * Checks that performing an operation that overflows into an inaccurate result + * has the desired behaviour. + * + * TBig is a type large enough to represent all results of TSmall operations. + */ +template +void checkDivision(TSmall a_, TSmall b) +{ + // Sufficient to fit all values + constexpr const TSmall minV = std::numeric_limits::min(); + + Checked a{a_}; + auto result = a / b; + + if (std::is_signed() && a_ == minV && b == -1) { + // This is the only possible overflow condition + RC_ASSERT(result.valueWrapping() == minV); + RC_ASSERT(result.overflowed()); + } else if (b == 0) { + RC_ASSERT(result.divideByZero()); + RC_ASSERT_THROWS_AS(result.valueWrapping(), nix::checked::DivideByZero); + RC_ASSERT(result.valueChecked() == std::nullopt); + } else { + TBig referenceResult = a_ / b; + auto result_ = result.valueChecked(); + RC_ASSERT(result_.has_value()); + RC_ASSERT(*result_ == referenceResult); + RC_ASSERT(result.valueWrapping() == referenceResult); + } +} + +/** Creates parameters that perform a more adequate number of checks to validate + * extremely cheap tests such as arithmetic tests */ +static rc::detail::TestParams makeParams() +{ + auto const & conf = rc::detail::configuration(); + auto newParams = conf.testParams; + newParams.maxSuccess = 10000; + return newParams; +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, add_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkType(a, b, &Checked::operator+, [](int32_t a, int32_t b) { return a + b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, add_signed, makeParams, (int16_t a, int16_t b)) +{ + checkType(a, b, &Checked::operator+, [](int32_t a, int32_t b) { return a + b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, sub_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkType(a, b, &Checked::operator-, [](int32_t a, int32_t b) { return a - b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, sub_signed, makeParams, (int16_t a, int16_t b)) +{ + checkType(a, b, &Checked::operator-, [](int32_t a, int32_t b) { return a - b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, mul_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkType(a, b, &Checked::operator*, [](int64_t a, int64_t b) { return a * b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, mul_signed, makeParams, (int16_t a, int16_t b)) +{ + checkType(a, b, &Checked::operator*, [](int64_t a, int64_t b) { return a * b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, div_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkDivision(a, b); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, div_signed, makeParams, (int16_t a, int16_t b)) +{ + checkDivision(a, b); +} + +// Make absolutely sure that we check the special cases if the proptest +// generator does not come up with them. This one is especially important +// because it has very specific pairs required for the edge cases unlike the +// others. +TEST(Checked, div_signed_special_cases) +{ + checkDivision(std::numeric_limits::min(), -1); + checkDivision(std::numeric_limits::min(), 0); + checkDivision(0, 0); +} + +} From 7b6622d73317f0cd093cde96b5898b8af07abea7 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 12 Jul 2024 16:22:34 +0200 Subject: [PATCH 167/284] language: cleanly ban integer overflows This also bans various sneaking of negative numbers from the language into unsuspecting builtins as was exposed while auditing the consequences of changing the Nix language integer type to a newtype. It's unlikely that this change comprehensively ensures correctness when passing integers out of the Nix language and we should probably add a checked-narrowing function or something similar, but that's out of scope for the immediate change. During the development of this I found a few fun facts about the language: - You could overflow integers by converting from unsigned JSON values. - You could overflow unsigned integers by converting negative numbers into them when going into Nix config, into fetchTree, and into flake inputs. The flake inputs and Nix config cannot actually be tested properly since they both ban thunks, however, we put in checks anyway because it's possible these could somehow be used to do such shenanigans some other way. Note that Lix has banned Nix language integer overflows since the very first public beta, but threw a SIGILL about them because we run with -fsanitize=signed-overflow -fsanitize-undefined-trap-on-error in production builds. Since the Nix language uses signed integers, overflow was simply undefined behaviour, and since we defined that to trap, it did. Trapping on it was a bad UX, but we didn't even entirely notice that we had done this at all until it was reported as a bug a couple of months later (which is, to be fair, that flag working as intended), and it's got enough production time that, aside from code that is IMHO buggy (and which is, in any case, not in nixpkgs) such as https://git.lix.systems/lix-project/lix/issues/445, we don't think anyone doing anything reasonable actually depends on wrapping overflow. Even for weird use cases such as doing funny bit crimes, it doesn't make sense IMO to have wrapping behaviour, since two's complement arithmetic overflow behaviour is so *aggressively* not what you want for *any* kind of mathematics/algorithms. The Nix language exists for package management, a domain where bit crimes are already only dubiously in scope to begin with, and it makes a lot more sense for that domain for the integers to never lose precision, either by throwing errors if they would, or by being arbitrary-precision. Fixes: https://github.com/NixOS/nix/issues/10968 Original-CL: https://gerrit.lix.systems/c/lix/+/1596 Change-Id: I51f253840c4af2ea5422b8a420aa5fafbf8fae75 --- src/libcmd/installable-flake.cc | 4 +- src/libcmd/installable-value.hh | 2 +- src/libexpr-c/nix_api_value.cc | 2 +- src/libexpr/eval-cache.cc | 4 +- src/libexpr/eval.cc | 21 ++-- src/libexpr/get-drvs.cc | 4 +- src/libexpr/json-to-value.cc | 7 +- src/libexpr/lexer.l | 2 +- src/libexpr/nixexpr.hh | 1 + src/libexpr/primops.cc | 102 ++++++++++++------ src/libexpr/primops/fetchTree.cc | 12 ++- src/libexpr/value-to-json.cc | 2 +- src/libexpr/value.hh | 10 +- src/libflake/flake/flake.cc | 22 +++- src/libstore/globals.hh | 10 +- .../lang/eval-fail-fetchTree-negative.err.exp | 8 ++ .../lang/eval-fail-fetchTree-negative.nix | 5 + ...ake-ref-to-string-negative-integer.err.exp | 14 +++ ...l-flake-ref-to-string-negative-integer.nix | 7 ++ .../eval-fail-fromJSON-overflowing.err.exp | 8 ++ .../lang/eval-fail-fromJSON-overflowing.nix | 1 + .../lang/eval-fail-overflowing-add.err.exp | 6 ++ .../lang/eval-fail-overflowing-add.nix | 4 + .../lang/eval-fail-overflowing-div.err.exp | 23 ++++ .../lang/eval-fail-overflowing-div.nix | 7 ++ .../lang/eval-fail-overflowing-mul.err.exp | 16 +++ .../lang/eval-fail-overflowing-mul.nix | 3 + .../lang/eval-fail-overflowing-sub.err.exp | 9 ++ .../lang/eval-fail-overflowing-sub.nix | 4 + tests/unit/libexpr-support/tests/libexpr.hh | 2 +- 30 files changed, 260 insertions(+), 62 deletions(-) create mode 100644 tests/functional/lang/eval-fail-fetchTree-negative.err.exp create mode 100644 tests/functional/lang/eval-fail-fetchTree-negative.nix create mode 100644 tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.err.exp create mode 100644 tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.nix create mode 100644 tests/functional/lang/eval-fail-fromJSON-overflowing.err.exp create mode 100644 tests/functional/lang/eval-fail-fromJSON-overflowing.nix create mode 100644 tests/functional/lang/eval-fail-overflowing-add.err.exp create mode 100644 tests/functional/lang/eval-fail-overflowing-add.nix create mode 100644 tests/functional/lang/eval-fail-overflowing-div.err.exp create mode 100644 tests/functional/lang/eval-fail-overflowing-div.nix create mode 100644 tests/functional/lang/eval-fail-overflowing-mul.err.exp create mode 100644 tests/functional/lang/eval-fail-overflowing-mul.nix create mode 100644 tests/functional/lang/eval-fail-overflowing-sub.err.exp create mode 100644 tests/functional/lang/eval-fail-overflowing-sub.nix diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 8796ad5ba79..6c9ee674808 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -104,12 +104,12 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() auto drvPath = attr->forceDerivation(); - std::optional priority; + std::optional priority; if (attr->maybeGetAttr(state->sOutputSpecified)) { } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { if (auto aPriority = aMeta->maybeGetAttr("priority")) - priority = aPriority->getInt(); + priority = aPriority->getInt().value; } return {{ diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh index 798cb5e1a00..60207cd235f 100644 --- a/src/libcmd/installable-value.hh +++ b/src/libcmd/installable-value.hh @@ -40,7 +40,7 @@ struct ExtraPathInfoValue : ExtraPathInfo /** * An optional priority for use with "build envs". See Package */ - std::optional priority; + std::optional priority; /** * The attribute path associated with this value. The idea is diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 845e879352f..fa2a9cbe2ae 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -306,7 +306,7 @@ int64_t nix_get_int(nix_c_context * context, const nix_value * value) try { auto & v = check_value_in(value); assert(v.type() == nix::nInt); - return v.integer(); + return v.integer().value; } NIXC_CATCH_ERRS_RES(0); } diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 26352187e72..95b484606fa 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -326,7 +326,7 @@ struct AttrDb case AttrType::Bool: return {{rowId, queryAttribute.getInt(2) != 0}}; case AttrType::Int: - return {{rowId, int_t{queryAttribute.getInt(2)}}}; + return {{rowId, int_t{NixInt{queryAttribute.getInt(2)}}}}; case AttrType::ListOfStrings: return {{rowId, tokenizeString>(queryAttribute.getStr(2), "\t")}}; case AttrType::Missing: @@ -469,7 +469,7 @@ Value & AttrCursor::forceValue() else if (v.type() == nBool) cachedValue = {root->db->setBool(getKey(), v.boolean()), v.boolean()}; else if (v.type() == nInt) - cachedValue = {root->db->setInt(getKey(), v.integer()), int_t{v.integer()}}; + cachedValue = {root->db->setInt(getKey(), v.integer().value), int_t{v.integer()}}; else if (v.type() == nAttrs) ; // FIXME: do something? else diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 746ccab2ace..76c54936fa0 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1979,7 +1979,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) NixStringContext context; std::vector s; size_t sSize = 0; - NixInt n = 0; + NixInt n{0}; NixFloat nf = 0; bool first = !forceString; @@ -2023,17 +2023,22 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) if (firstType == nInt) { if (vTmp.type() == nInt) { - n += vTmp.integer(); + auto newN = n + vTmp.integer(); + if (auto checked = newN.valueChecked(); checked.has_value()) { + n = NixInt(*checked); + } else { + state.error("integer overflow in adding %1% + %2%", n, vTmp.integer()).atPos(i_pos).debugThrow(); + } } else if (vTmp.type() == nFloat) { // Upgrade the type from int to float; firstType = nFloat; - nf = n; + nf = n.value; nf += vTmp.fpoint(); } else state.error("cannot add %1% to an integer", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow(); } else if (firstType == nFloat) { if (vTmp.type() == nInt) { - nf += vTmp.integer(); + nf += vTmp.integer().value; } else if (vTmp.type() == nFloat) { nf += vTmp.fpoint(); } else @@ -2158,7 +2163,7 @@ NixFloat EvalState::forceFloat(Value & v, const PosIdx pos, std::string_view err try { forceValue(v, pos); if (v.type() == nInt) - return v.integer(); + return v.integer().value; else if (v.type() != nFloat) error( "expected a float but found %1%: %2%", @@ -2345,7 +2350,7 @@ BackedStringView EvalState::coerceToString( shell scripting convenience, just like `null'. */ if (v.type() == nBool && v.boolean()) return "1"; if (v.type() == nBool && !v.boolean()) return ""; - if (v.type() == nInt) return std::to_string(v.integer()); + if (v.type() == nInt) return std::to_string(v.integer().value); if (v.type() == nFloat) return std::to_string(v.fpoint()); if (v.type() == nNull) return ""; @@ -2728,9 +2733,9 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v // Special case type-compatibility between float and int if (v1.type() == nInt && v2.type() == nFloat) - return v1.integer() == v2.fpoint(); + return v1.integer().value == v2.fpoint(); if (v1.type() == nFloat && v2.type() == nInt) - return v1.fpoint() == v2.integer(); + return v1.fpoint() == v2.integer().value; // All other types are not compatible with each other. if (v1.type() != v2.type()) return false; diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 7041a3932ee..20963ec91ce 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -246,8 +246,8 @@ NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def) if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ - if (auto n = string2Int(v->c_str())) - return *n; + if (auto n = string2Int(v->c_str())) + return NixInt{*n}; } return def; } diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 21074bdd807..9ac56541ab9 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -2,6 +2,7 @@ #include "value.hh" #include "eval.hh" +#include #include #include @@ -101,8 +102,12 @@ class JSONSax : nlohmann::json_sax { return true; } - bool number_unsigned(number_unsigned_t val) override + bool number_unsigned(number_unsigned_t val_) override { + if (val_ > std::numeric_limits::max()) { + throw Error("unsigned json number %1% outside of Nix integer range", val_); + } + NixInt::Inner val = val_; rs->value(state).mkInt(val); rs->add(); return true; diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index eb1825b7c3f..a7e44cb7288 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -138,7 +138,7 @@ or { return OR_KW; } {INT} { errno = 0; std::optional numMay = string2Int(yytext); if (numMay.has_value()) { - yylval->n = *numMay; + yylval->n = NixInt{*numMay}; } else { throw ParseError(ErrorInfo{ .msg = HintFmt("invalid integer '%1%'", yytext), diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 3279e3d48b3..7868834f195 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -107,6 +107,7 @@ struct ExprInt : Expr { Value v; ExprInt(NixInt n) { v.mkInt(n); }; + ExprInt(NixInt::Inner n) { v.mkInt(n); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a23583f34c2..acc0d6c1c4f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -587,9 +587,9 @@ struct CompareValues { try { if (v1->type() == nFloat && v2->type() == nInt) - return v1->fpoint() < v2->integer(); + return v1->fpoint() < v2->integer().value; if (v1->type() == nInt && v2->type() == nFloat) - return v1->integer() < v2->fpoint(); + return v1->integer().value < v2->fpoint(); if (v1->type() != v2->type()) state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow(); // Allow selecting a subset of enum values @@ -2677,13 +2677,13 @@ static struct LazyPosAcessors { PrimOp primop_lineOfPos{ .arity = 1, .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.positions[PosIdx(args[0]->integer())].line); + v.mkInt(state.positions[PosIdx(args[0]->integer().value)].line); } }; PrimOp primop_columnOfPos{ .arity = 1, .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.positions[PosIdx(args[0]->integer())].column); + v.mkInt(state.positions[PosIdx(args[0]->integer().value)].column); } }; @@ -3159,7 +3159,8 @@ static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Val /* Return the n-1'th element of a list. */ static void prim_elemAt(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - elemAt(state, pos, *args[0], state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt"), v); + NixInt::Inner elem = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt").value; + elemAt(state, pos, *args[0], elem, v); } static RegisterPrimOp primop_elemAt({ @@ -3453,10 +3454,12 @@ static RegisterPrimOp primop_all({ static void prim_genList(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto len = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList"); + auto len_ = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList").value; - if (len < 0) - state.error("cannot create list of size %1%", len).atPos(pos).debugThrow(); + if (len_ < 0) + state.error("cannot create list of size %1%", len_).atPos(pos).debugThrow(); + + size_t len = size_t(len_); // More strict than striclty (!) necessary, but acceptable // as evaluating map without accessing any values makes little sense. @@ -3713,9 +3716,17 @@ static void prim_add(EvalState & state, const PosIdx pos, Value * * args, Value if (args[0]->type() == nFloat || args[1]->type() == nFloat) v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the addition") + state.forceFloat(*args[1], pos, "while evaluating the second argument of the addition")); - else - v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the addition") - + state.forceInt(*args[1], pos, "while evaluating the second argument of the addition")); + else { + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the addition"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the addition"); + + auto result_ = i1 + i2; + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in adding %1% + %2%", i1, i2).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_add({ @@ -3734,9 +3745,18 @@ static void prim_sub(EvalState & state, const PosIdx pos, Value * * args, Value if (args[0]->type() == nFloat || args[1]->type() == nFloat) v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the subtraction") - state.forceFloat(*args[1], pos, "while evaluating the second argument of the subtraction")); - else - v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the subtraction") - - state.forceInt(*args[1], pos, "while evaluating the second argument of the subtraction")); + else { + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the subtraction"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the subtraction"); + + auto result_ = i1 - i2; + + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in subtracting %1% - %2%", i1, i2).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_sub({ @@ -3755,9 +3775,18 @@ static void prim_mul(EvalState & state, const PosIdx pos, Value * * args, Value if (args[0]->type() == nFloat || args[1]->type() == nFloat) v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first of the multiplication") * state.forceFloat(*args[1], pos, "while evaluating the second argument of the multiplication")); - else - v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the multiplication") - * state.forceInt(*args[1], pos, "while evaluating the second argument of the multiplication")); + else { + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the multiplication"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the multiplication"); + + auto result_ = i1 * i2; + + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in multiplying %1% * %2%", i1, i2).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_mul({ @@ -3784,10 +3813,12 @@ static void prim_div(EvalState & state, const PosIdx pos, Value * * args, Value NixInt i1 = state.forceInt(*args[0], pos, "while evaluating the first operand of the division"); NixInt i2 = state.forceInt(*args[1], pos, "while evaluating the second operand of the division"); /* Avoid division overflow as it might raise SIGFPE. */ - if (i1 == std::numeric_limits::min() && i2 == -1) - state.error("overflow in integer division").atPos(pos).debugThrow(); - - v.mkInt(i1 / i2); + auto result_ = i1 / i2; + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in dividing %1% / %2%", i1, i2).atPos(pos).debugThrow(); + } } } @@ -3802,8 +3833,9 @@ static RegisterPrimOp primop_div({ static void prim_bitAnd(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitAnd") - & state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitAnd")); + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitAnd"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitAnd"); + v.mkInt(i1.value & i2.value); } static RegisterPrimOp primop_bitAnd({ @@ -3817,8 +3849,10 @@ static RegisterPrimOp primop_bitAnd({ static void prim_bitOr(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitOr") - | state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitOr")); + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitOr"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitOr"); + + v.mkInt(i1.value | i2.value); } static RegisterPrimOp primop_bitOr({ @@ -3832,8 +3866,10 @@ static RegisterPrimOp primop_bitOr({ static void prim_bitXor(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitXor") - ^ state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitXor")); + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitXor"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitXor"); + + v.mkInt(i1.value ^ i2.value); } static RegisterPrimOp primop_bitXor({ @@ -3913,13 +3949,19 @@ static RegisterPrimOp primop_toString({ non-negative. */ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring"); + NixInt::Inner start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring").value; if (start < 0) state.error("negative start position in 'substring'").atPos(pos).debugThrow(); - int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring"); + NixInt::Inner len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring").value; + + // Negative length may be idiomatically passed to builtins.substring to get + // the tail of the string. + if (len < 0) { + len = std::numeric_limits::max(); + } // Special-case on empty substring to avoid O(n) strlen // This allows for the use of empty substrings to efficently capture string context @@ -3962,7 +4004,7 @@ static void prim_stringLength(EvalState & state, const PosIdx pos, Value * * arg { NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.stringLength"); - v.mkInt(s->size()); + v.mkInt(NixInt::Inner(s->size())); } static RegisterPrimOp primop_stringLength({ diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index f79b6b7b83a..78328701dcd 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -122,9 +122,15 @@ static void fetchTree( } else if (attr.value->type() == nBool) attrs.emplace(state.symbols[attr.name], Explicit{attr.value->boolean()}); - else if (attr.value->type() == nInt) - attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer())); - else if (state.symbols[attr.name] == "publicKeys") { + else if (attr.value->type() == nInt) { + auto intValue = attr.value->integer().value; + + if (intValue < 0) { + state.error("negative value given for fetchTree attr %1%: %2%", state.symbols[attr.name], intValue).atPos(pos).debugThrow(); + } + + attrs.emplace(state.symbols[attr.name], uint64_t(intValue)); + } else if (state.symbols[attr.name] == "publicKeys") { experimentalFeatureSettings.require(Xp::VerifiedFetches); attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump()); } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index f8cc056161e..8044fe3472e 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -22,7 +22,7 @@ json printValueAsJSON(EvalState & state, bool strict, switch (v.type()) { case nInt: - out = v.integer(); + out = v.integer().value; break; case nBool: diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 257da1d2d69..1510abfea12 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -8,6 +8,7 @@ #include "value/context.hh" #include "source-path.hh" #include "print-options.hh" +#include "checked-arithmetic.hh" #if HAVE_BOEHMGC #include @@ -73,8 +74,8 @@ class EvalState; class XMLWriter; class Printer; -typedef int64_t NixInt; -typedef double NixFloat; +using NixInt = checked::Checked; +using NixFloat = double; /** * External values must descend from ExternalValueBase, so that @@ -304,6 +305,11 @@ public: return internalType != tUninitialized; } + inline void mkInt(NixInt::Inner n) + { + mkInt(NixInt{n}); + } + inline void mkInt(NixInt n) { finishValue(tInt, { .integer = n }); diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 627dfe83093..fd1183514c9 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -140,9 +140,16 @@ static FlakeInput parseFlakeInput(EvalState & state, case nBool: attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean() }); break; - case nInt: - attrs.emplace(state.symbols[attr.name], (long unsigned int) attr.value->integer()); + case nInt: { + auto intValue = attr.value->integer().value; + + if (intValue < 0) { + state.error("negative value given for flake input attribute %1%: %2%", state.symbols[attr.name], intValue).debugThrow(); + } + + attrs.emplace(state.symbols[attr.name], uint64_t(intValue)); break; + } default: if (attr.name == state.symbols.create("publicKeys")) { experimentalFeatureSettings.require(Xp::VerifiedFetches); @@ -272,7 +279,7 @@ static Flake readFlake( else if (setting.value->type() == nInt) flake.config.settings.emplace( state.symbols[setting.name], - state.forceInt(*setting.value, setting.pos, "")); + state.forceInt(*setting.value, setting.pos, "").value); else if (setting.value->type() == nBool) flake.config.settings.emplace( state.symbols[setting.name], @@ -904,8 +911,13 @@ static void prim_flakeRefToString( for (const auto & attr : *args[0]->attrs()) { auto t = attr.value->type(); if (t == nInt) { - attrs.emplace(state.symbols[attr.name], - (uint64_t) attr.value->integer()); + auto intValue = attr.value->integer().value; + + if (intValue < 0) { + state.error("negative value given for flake ref attr %1%: %2%", state.symbols[attr.name], intValue).atPos(pos).debugThrow(); + } + + attrs.emplace(state.symbols[attr.name], uint64_t(intValue)); } else if (t == nBool) { attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean() }); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 8760c9d145b..1ca9f02f5ab 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1131,7 +1131,10 @@ public: )"}; Setting maxFree{ - this, std::numeric_limits::max(), "max-free", + // n.b. this is deliberately int64 max rather than uint64 max because + // this goes through the Nix language JSON parser and thus needs to be + // representable in Nix language integers. + this, std::numeric_limits::max(), "max-free", R"( When a garbage collection is triggered by the `min-free` option, it stops as soon as `max-free` bytes are available. The default is @@ -1221,7 +1224,10 @@ public: Setting warnLargePathThreshold{ this, - std::numeric_limits::max(), + // n.b. this is deliberately int64 max rather than uint64 max because + // this goes through the Nix language JSON parser and thus needs to be + // representable in Nix language integers. + std::numeric_limits::max(), "warn-large-path-threshold", R"( Warn when copying a path larger than this number of bytes to the Nix store diff --git a/tests/functional/lang/eval-fail-fetchTree-negative.err.exp b/tests/functional/lang/eval-fail-fetchTree-negative.err.exp new file mode 100644 index 00000000000..d9ba1f0b2f8 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchTree-negative.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchTree' builtin + at /pwd/lang/eval-fail-fetchTree-negative.nix:1:1: + 1| builtins.fetchTree { + | ^ + 2| type = "file"; + + error: negative value given for fetchTree attr owner: -1 diff --git a/tests/functional/lang/eval-fail-fetchTree-negative.nix b/tests/functional/lang/eval-fail-fetchTree-negative.nix new file mode 100644 index 00000000000..90bcab5d884 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchTree-negative.nix @@ -0,0 +1,5 @@ +builtins.fetchTree { + type = "file"; + url = "file://eval-fail-fetchTree-negative.nix"; + owner = -1; +} diff --git a/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.err.exp b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.err.exp new file mode 100644 index 00000000000..25c8d7eaaa8 --- /dev/null +++ b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.err.exp @@ -0,0 +1,14 @@ +error: + … while calling the 'seq' builtin + at /pwd/lang/eval-fail-flake-ref-to-string-negative-integer.nix:1:16: + 1| let n = -1; in builtins.seq n (builtins.flakeRefToString { + | ^ + 2| type = "github"; + + … while calling the 'flakeRefToString' builtin + at /pwd/lang/eval-fail-flake-ref-to-string-negative-integer.nix:1:32: + 1| let n = -1; in builtins.seq n (builtins.flakeRefToString { + | ^ + 2| type = "github"; + + error: negative value given for flake ref attr repo: -1 diff --git a/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.nix b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.nix new file mode 100644 index 00000000000..e0208eb2519 --- /dev/null +++ b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.nix @@ -0,0 +1,7 @@ +let n = -1; in builtins.seq n (builtins.flakeRefToString { + type = "github"; + owner = "NixOS"; + repo = n; + ref = "23.05"; + dir = "lib"; +}) diff --git a/tests/functional/lang/eval-fail-fromJSON-overflowing.err.exp b/tests/functional/lang/eval-fail-fromJSON-overflowing.err.exp new file mode 100644 index 00000000000..a39082b450a --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-overflowing.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromJSON' builtin + at /pwd/lang/eval-fail-fromJSON-overflowing.nix:1:1: + 1| builtins.fromJSON ''{"attr": 18446744073709551615}'' + | ^ + 2| + + error: unsigned json number 18446744073709551615 outside of Nix integer range diff --git a/tests/functional/lang/eval-fail-fromJSON-overflowing.nix b/tests/functional/lang/eval-fail-fromJSON-overflowing.nix new file mode 100644 index 00000000000..6dfbce3f68a --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-overflowing.nix @@ -0,0 +1 @@ +builtins.fromJSON ''{"attr": 18446744073709551615}'' diff --git a/tests/functional/lang/eval-fail-overflowing-add.err.exp b/tests/functional/lang/eval-fail-overflowing-add.err.exp new file mode 100644 index 00000000000..6458cf1c933 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-add.err.exp @@ -0,0 +1,6 @@ +error: integer overflow in adding 9223372036854775807 + 1 + at /pwd/lang/eval-fail-overflowing-add.nix:4:8: + 3| b = 1; + 4| in a + b + | ^ + 5| diff --git a/tests/functional/lang/eval-fail-overflowing-add.nix b/tests/functional/lang/eval-fail-overflowing-add.nix new file mode 100644 index 00000000000..24258fc200e --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-add.nix @@ -0,0 +1,4 @@ +let + a = 9223372036854775807; + b = 1; +in a + b diff --git a/tests/functional/lang/eval-fail-overflowing-div.err.exp b/tests/functional/lang/eval-fail-overflowing-div.err.exp new file mode 100644 index 00000000000..8ce07d4d662 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-div.err.exp @@ -0,0 +1,23 @@ +error: + … while calling the 'seq' builtin + at /pwd/lang/eval-fail-overflowing-div.nix:7:4: + 6| b = -1; + 7| in builtins.seq intMin (builtins.seq b (intMin / b)) + | ^ + 8| + + … while calling the 'seq' builtin + at /pwd/lang/eval-fail-overflowing-div.nix:7:25: + 6| b = -1; + 7| in builtins.seq intMin (builtins.seq b (intMin / b)) + | ^ + 8| + + … while calling the 'div' builtin + at /pwd/lang/eval-fail-overflowing-div.nix:7:48: + 6| b = -1; + 7| in builtins.seq intMin (builtins.seq b (intMin / b)) + | ^ + 8| + + error: integer overflow in dividing -9223372036854775808 / -1 diff --git a/tests/functional/lang/eval-fail-overflowing-div.nix b/tests/functional/lang/eval-fail-overflowing-div.nix new file mode 100644 index 00000000000..44fbe9d7e31 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-div.nix @@ -0,0 +1,7 @@ +let + # lol, this has to be written as an expression like this because negative + # numbers use unary negation rather than parsing directly, and 2**63 is out + # of range + intMin = -9223372036854775807 - 1; + b = -1; +in builtins.seq intMin (builtins.seq b (intMin / b)) diff --git a/tests/functional/lang/eval-fail-overflowing-mul.err.exp b/tests/functional/lang/eval-fail-overflowing-mul.err.exp new file mode 100644 index 00000000000..f42b39d4db9 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-mul.err.exp @@ -0,0 +1,16 @@ +error: + … while calling the 'mul' builtin + at /pwd/lang/eval-fail-overflowing-mul.nix:3:10: + 2| a = 4294967297; + 3| in a * a * a + | ^ + 4| + + … while calling the 'mul' builtin + at /pwd/lang/eval-fail-overflowing-mul.nix:3:6: + 2| a = 4294967297; + 3| in a * a * a + | ^ + 4| + + error: integer overflow in multiplying 4294967297 * 4294967297 diff --git a/tests/functional/lang/eval-fail-overflowing-mul.nix b/tests/functional/lang/eval-fail-overflowing-mul.nix new file mode 100644 index 00000000000..6081d9c7b14 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-mul.nix @@ -0,0 +1,3 @@ +let + a = 4294967297; +in a * a * a diff --git a/tests/functional/lang/eval-fail-overflowing-sub.err.exp b/tests/functional/lang/eval-fail-overflowing-sub.err.exp new file mode 100644 index 00000000000..66a3a03f885 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-sub.err.exp @@ -0,0 +1,9 @@ +error: + … while calling the 'sub' builtin + at /pwd/lang/eval-fail-overflowing-sub.nix:4:6: + 3| b = 2; + 4| in a - b + | ^ + 5| + + error: integer overflow in subtracting -9223372036854775807 - 2 diff --git a/tests/functional/lang/eval-fail-overflowing-sub.nix b/tests/functional/lang/eval-fail-overflowing-sub.nix new file mode 100644 index 00000000000..229b8c6d264 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-sub.nix @@ -0,0 +1,4 @@ +let + a = -9223372036854775807; + b = 2; +in a - b diff --git a/tests/unit/libexpr-support/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh index 54a4a0e9e3b..045607e875e 100644 --- a/tests/unit/libexpr-support/tests/libexpr.hh +++ b/tests/unit/libexpr-support/tests/libexpr.hh @@ -85,7 +85,7 @@ namespace nix { if (arg.type() != nInt) { return false; } - return arg.integer() == v; + return arg.integer().value == v; } MATCHER_P(IsFloatEq, v, fmt("The float is equal to \"%1%\"", v)) { From bf050d9e966534eeb92188da1a93ac459bed0f97 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 12 Jul 2024 18:20:14 +0200 Subject: [PATCH 168/284] docs: update to define integer overflow Change-Id: Ie8a1b31035f2d27a220e5df2e9e178ec3b39ee68 --- doc/manual/src/language/operators.md | 8 ++++++-- doc/manual/src/language/syntax.md | 7 +++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index e1c020781c3..2c2dcc276ac 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -67,8 +67,12 @@ After evaluating *attrset* and *attrpath*, the computational complexity is O(log ## Arithmetic -Numbers are type-compatible: -Pure integer operations will always return integers, whereas any operation involving at least one floating point number return a floating point number. +Numbers will retain their type unless mixed with other numeric types: +Pure integer operations will always return integers, whereas any operation involving at least one floating point number returns a floating point number. + +Evaluation of the following numeric operations throws an evaluation error: +- Division by zero +- Integer overflow, that is, any operation yielding a result outside of the representable range of [Nix language integers](./syntax.md#number-literal) See also [Comparison] and [Equality]. diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md index 6108bacd676..f05935502d2 100644 --- a/doc/manual/src/language/syntax.md +++ b/doc/manual/src/language/syntax.md @@ -183,6 +183,13 @@ This section covers syntax and semantics of the Nix language. Numbers, which can be *integers* (like `123`) or *floating point* (like `123.43` or `.27e13`). + Integers in the Nix language are 64-bit [two's complement] signed integers, with a range of -9223372036854775808 to 9223372036854775807, inclusive. + + [two's complement]: https://en.wikipedia.org/wiki/Two%27s_complement + + Note that negative numeric literals are actually parsed as unary negation of positive numeric literals. + This means that the minimum integer `-9223372036854775808` cannot be written as-is as a literal, since the positive number `9223372036854775808` is one past the maximum range. + See [arithmetic] and [comparison] operators for semantics. [arithmetic]: ./operators.md#arithmetic From 5878b1475f1c50b42672ed85e7259e7d328920e9 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 25 Jul 2024 23:06:53 +0200 Subject: [PATCH 169/284] doc: release notes for banning integer overflow Change-Id: Ib75ab5b8b4d879035d7ee7678f9cd0c491a39c0a --- doc/manual/rl-next/ban-integer-overflow.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/manual/rl-next/ban-integer-overflow.md diff --git a/doc/manual/rl-next/ban-integer-overflow.md b/doc/manual/rl-next/ban-integer-overflow.md new file mode 100644 index 00000000000..49ecfe4c6fe --- /dev/null +++ b/doc/manual/rl-next/ban-integer-overflow.md @@ -0,0 +1,21 @@ +--- +synopsis: Define integer overflow in the Nix language as an error +issues: [10968] +prs: [11188] +--- + +Previously, integer overflow in the Nix language invoked C++ level signed overflow, which was undefined behaviour, but *probably* manifested as wrapping around on overflow. + +Since prior to the public release of Lix, Lix had C++ signed overflow defined to crash the process and nobody noticed this having accidentally removed overflow from the Nix language for three months until it was caught by fiddling around. +Given the significant body of actual Nix code that has been evaluated by Lix in that time, it does not appear that nixpkgs or much of importance depends on integer overflow, so it is safe to turn into an error. + +Some other overflows were fixed: +- `builtins.fromJSON` of values greater than the maximum representable value in a signed 64-bit integer will generate an error. +- `nixConfig` in flakes will no longer accept negative values for configuration options. + +Integer overflow now looks like the following: + +``` +» nix eval --expr '9223372036854775807 + 1' +error: integer overflow in adding 9223372036854775807 + 1 +``` From db5bacb63772138036bdab4cb89a83bb52bbbcfa Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 31 Jul 2024 21:41:26 +0200 Subject: [PATCH 170/284] reword documentation on `nix-path` config option (#7772) * docs: unify documentation on search paths - put all the information on search path semantics into `builtins.findFile` - put all the information on determining the value of `builtins.nixPath` into the `nix-path` setting maybe `builtins.nixPath` is a better place for this, but those bits can still be moved around now that it's all next to each other. - link to the syntax page for lookup paths from all places that are concerned with it - add or clarify examples - add a test verifying a claim from documentation --- doc/manual/src/command-ref/env-common.md | 26 +++-- doc/manual/src/command-ref/opt-common.md | 9 +- src/libcmd/common-eval-args.cc | 70 +------------ src/libexpr/eval-settings.hh | 21 ++-- src/libexpr/primops.cc | 122 ++++++++++++++++++++--- tests/functional/nix_path.sh | 16 +-- 6 files changed, 152 insertions(+), 112 deletions(-) diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index d3f5f9c1443..0b501788293 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -9,22 +9,26 @@ Most Nix commands interpret the following environment variables: - [`NIX_PATH`](#env-NIX_PATH) - A colon-separated list of directories used to look up the location of Nix - expressions using [paths](@docroot@/language/types.md#type-path) - enclosed in angle brackets (i.e., ``), - e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the - [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). + A colon-separated list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). - If `NIX_PATH` is not set at all, Nix will fall back to the following list in [impure](@docroot@/command-ref/conf-file.md#conf-pure-eval) and [unrestricted](@docroot@/command-ref/conf-file.md#conf-restrict-eval) evaluation mode: + This environment variable overrides the value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path). - 1. `$HOME/.nix-defexpr/channels` - 2. `nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixpkgs` - 3. `/nix/var/nix/profiles/per-user/root/channels` + It can be extended using the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). + + > **Example** + > + > ```bash + > $ export NIX_PATH=`/home/eelco/Dev:nixos-config=/etc/nixos + > ``` If `NIX_PATH` is set to an empty string, resolving search paths will always fail. - For example, attempting to use `` will produce: - error: file 'nixpkgs' was not found in the Nix search path + > **Example** + > + > ```bash + > $ NIX_PATH= nix-instantiate --eval '' + > error: file 'nixpkgs' was not found in the Nix search path (add it using $NIX_PATH or -I) + > ``` - [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE) diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index a42909e2d13..69a7002072d 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -37,7 +37,7 @@ Most Nix commands accept the following command-line options: Print even more informational messages. - `4` “Debug” - + Print debug information. - `5` “Vomit” @@ -187,11 +187,12 @@ Most Nix commands accept the following command-line options: For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell --packages ` convenience flag instead. -- [`-I`](#opt-I) *path* +- [`-I` / `--include`](#opt-I) *path* - Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + Add an entry to the list of search paths used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). - [`--option`](#opt-option) *name* *value* diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index decadd751cc..fcef92487cb 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -91,75 +91,11 @@ MixEvalArgs::MixEvalArgs() .longName = "include", .shortName = 'I', .description = R"( - Add *path* to the Nix search path. The Nix search path is - initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/types.md#type-path) enclosed in angle - brackets (i.e., ``). + Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md) - For instance, passing + This option may be given multiple times. - ``` - -I /home/eelco/Dev - -I /etc/nixos - ``` - - will cause Nix to look for paths relative to `/home/eelco/Dev` and - `/etc/nixos`, in that order. This is equivalent to setting the - `NIX_PATH` environment variable to - - ``` - /home/eelco/Dev:/etc/nixos - ``` - - It is also possible to match paths against a prefix. For example, - passing - - ``` - -I nixpkgs=/home/eelco/Dev/nixpkgs-branch - -I /etc/nixos - ``` - - will cause Nix to search for `` in - `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - - If a path in the Nix search path starts with `http://` or `https://`, - it is interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must consist of a single - top-level directory. For example, passing - - ``` - -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz - ``` - - tells Nix to download and use the current contents of the `master` - branch in the `nixpkgs` repository. - - The URLs of the tarballs from the official `nixos.org` channels - (see [the manual page for `nix-channel`](../nix-channel.md)) can be - abbreviated as `channel:`. For instance, the - following two flags are equivalent: - - ``` - -I nixpkgs=channel:nixos-21.05 - -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz - ``` - - You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the - search path. For instance, - - ``` - -I nixpkgs=flake:nixpkgs - ``` - - specifies that the prefix `nixpkgs` shall refer to the source tree - downloaded from the `nixpkgs` entry in the flake registry. Similarly, - - ``` - -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 - ``` - - makes `` refer to a particular branch of the - `NixOS/nixpkgs` repository on GitHub. + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). )", .category = category, .labels = {"path"}, diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 30a8c5c588a..0cfc14c1b74 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -79,19 +79,24 @@ struct EvalSettings : Config This setting determines the value of [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) and can be used with [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile). - The default value is + - The configuration setting is overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) + environment variable. + - `NIX_PATH` is overridden by [specifying the setting as the command line flag](@docroot@/command-ref/conf-file.md#command-line-flags) `--nix-path`. + - Any current value is extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path`. - ``` - $HOME/.nix-defexpr/channels - nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs - $NIX_STATE_DIR/profiles/per-user/root/channels - ``` + If the respective paths are accessible, the default values are: - It can be overridden with the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH) or the [`-I` command line option](@docroot@/command-ref/opt-common.md#opt-I). + - `$HOME/.nix-defexpr/channels` + - `nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs` + - `$NIX_STATE_DIR/profiles/per-user/root/channels` + + See [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR) for details. > **Note** > - > If [pure evaluation](#conf-pure-eval) is enabled, `nixPath` evaluates to the empty list `[ ]`. + > If [restricted evaluation](@docroot@/command-ref/conf-file.md#conf-restrict-eval) is enabled, the default value is empty. + > + > If [pure evaluation](#conf-pure-eval) is enabled, `builtins.nixPath` *always* evaluates to the empty list `[ ]`. )", {}, false}; Setting currentSystem{ diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 0b3b19b5764..7ceb84f0e39 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1843,45 +1843,130 @@ static RegisterPrimOp primop_findFile(PrimOp { .doc = R"( Find *lookup-path* in *search-path*. - A search path is represented list of [attribute sets](./types.md#attribute-set) with two attributes: + [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](#builtins-nixPath): + + ```nix + + ``` + + is equivalent to: + + ```nix + builtins.findFile builtins.nixPath "nixpkgs" + ``` + + A search path is represented as a list of [attribute sets](./types.md#attribute-set) with two attributes: - `prefix` is a relative path. - `path` denotes a file system location - The exact syntax depends on the command line interface. Examples of search path attribute sets: + - ``` + { + prefix = ""; + path = "/nix/var/nix/profiles/per-user/root/channels"; + } + ``` - ``` { prefix = "nixos-config"; path = "/etc/nixos/configuration.nix"; } ``` - - ``` { - prefix = ""; - path = "/nix/var/nix/profiles/per-user/root/channels"; + prefix = "nixpkgs"; + path = "https://github.com/NixOS/nixpkgs/tarballs/master"; + } + ``` + - ``` + { + prefix = "nixpkgs"; + path = "channel:nixpkgs-unstable"; + } + ``` + - ``` + { + prefix = "flake-compat"; + path = "flake:github:edolstra/flake-compat"; } ``` The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/types.md#type-path) of the match: - - If *lookup-path* matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`. - Note that the `path` may need to be downloaded at this point to look inside. + - If a prefix of `lookup-path` matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`. + The contents of `path` may need to be downloaded at this point to look inside. + - If the suffix is found inside that directory, then the entry is a match. The combined absolute path of the directory (now downloaded if need be) and the suffix is returned. - [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](#builtins-nixPath): + > **Example** + > + > A *search-path* value + > + > ``` + > [ + > { + > prefix = ""; + > path = "/home/eelco/Dev"; + > } + > { + > prefix = "nixos-config"; + > path = "/etc/nixos"; + > } + > ] + > ``` + > + > and a *lookup-path* value `"nixos-config"` will cause Nix to try `/home/eelco/Dev/nixos-config` and `/etc/nixos` in that order and return the first path that exists. - ```nix - - ``` + If `path` starts with `http://` or `https://`, it is interpreted as the URL of a tarball that will be downloaded and unpacked to a temporary location. + The tarball must consist of a single top-level directory. - is equivalent to: + The URLs of the tarballs from the official `nixos.org` channels can be abbreviated as `channel:`. + See [documentation on `nix-channel`](@docroot@/command-ref/nix-channel.md) for details about channels. - ```nix - builtins.findFile builtins.nixPath "nixpkgs" - ``` + > **Example** + > + > These two search path entries are equivalent: + > + > - ``` + > { + > prefix = "nixpkgs"; + > path = "channel:nixpkgs-unstable"; + > } + > ``` + > - ``` + > { + > prefix = "nixpkgs"; + > path = "https://nixos.org/channels/nixos-unstable/nixexprs.tar.xz"; + > } + > ``` + + Search paths can also point to source trees using [flake URLs](@docroot@/command-ref/new-cli/nix3-flake.md#url-like-syntax). + + + > **Example** + > + > The search path entry + > + > ``` + > { + > prefix = "nixpkgs"; + > path = "flake:nixpkgs"; + > } + > ``` + > specifies that the prefix `nixpkgs` shall refer to the source tree downloaded from the `nixpkgs` entry in the flake registry. + > + > Similarly + > + > ``` + > { + > prefix = "nixpkgs"; + > path = "flake:github:nixos/nixpkgs/nixos-22.05"; + > } + > ``` + > + > makes `` refer to a particular branch of the `NixOS/nixpkgs` repository on GitHub. )", .fun = prim_findFile, }); @@ -4731,6 +4816,13 @@ void EvalState::createBaseEnv() .doc = R"( The value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path): a list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). + > **Example** + > + > ```bash + > $ NIX_PATH= nix-instantiate --eval --expr "builtins.nixPath" -I foo=bar --no-pure-eval + > [ { path = "bar"; prefix = "foo"; } ] + > ``` + Lookup path expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.findFile`](./builtins.html#builtins-findFile): diff --git a/tests/functional/nix_path.sh b/tests/functional/nix_path.sh index 7e6a0458d10..90cba1f0c9c 100755 --- a/tests/functional/nix_path.sh +++ b/tests/functional/nix_path.sh @@ -22,13 +22,13 @@ nix-instantiate --eval -E '' --restrict-eval # # | precedence | hard-coded | nix-path in file | extra-nix-path in file | nix-path in env | extra-nix-path in env | NIX_PATH | nix-path | extra-nix-path | -I | # |------------------------|------------|------------------|------------------------|-----------------|-----------------------|-----------|-----------|-----------------|-----------------| -# | hard-coded | x | ^override | ^append | ^override | ^append | ^override | ^override | ^append | ^append | -# | nix-path in file | | last wins | ^append | ^override | ^append | ^override | ^override | ^append | ^append | -# | extra-nix-path in file | | | append in order | ^override | ^append | ^override | ^override | ^append | ^append | -# | nix-path in env | | | | last wins | ^append | ^override | ^override | ^append | ^append | -# | extra-nix-path in env | | | | | append in order | ^override | ^override | ^append | ^append | -# | NIX_PATH | | | | | | x | ^override | ^append | ^append | -# | nix-path | | | | | | | last wins | ^append | ^append | +# | hard-coded | x | ^override | ^append | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | nix-path in file | | last wins | ^append | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | extra-nix-path in file | | | append in order | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | nix-path in env | | | | last wins | ^append | ^override | ^override | ^append | ^prepend | +# | extra-nix-path in env | | | | | append in order | ^override | ^override | ^append | ^prepend | +# | NIX_PATH | | | | | | x | ^override | ^append | ^prepend | +# | nix-path | | | | | | | last wins | ^append | ^prepend | # | extra-nix-path | | | | | | | | append in order | append in order | # | -I | | | | | | | | | append in order | @@ -59,6 +59,8 @@ echo "nix-path = test=$TEST_ROOT/from-nix-path-file" >> "$test_nix_conf" # -I extends NIX_PATH [[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +# -I takes precedence over NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test) = $TEST_ROOT/from-I ]] # if -I does not have the desired entry, the value from NIX_PATH is used [[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] From c952d933e574dbc17304b18b33a9afeed7ebd966 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 31 Jul 2024 21:57:31 +0200 Subject: [PATCH 171/284] release notes: 2.24.0 --- .../10564-attrcursor-remove-forceerrors.md | 9 - ...03-run-the-flake-regressions-test-suite.md | 8 - ...unit-prefixes-in-configuration-settings.md | 10 - ...ild-show-all-fod-errors-with-keep-going.md | 10 - doc/manual/rl-next/10855-meson.md | 31 -- .../11086-eval-cache-fix-cache-regressions.md | 14 - .../rl-next/9063-introduce-libnixflake.md | 12 - doc/manual/rl-next/drop-vendored-toml11.md | 8 - doc/manual/rl-next/harden-user-sandboxing.md | 12 - .../rl-next/nix-shell-looks-for-shell-nix.md | 30 -- doc/manual/rl-next/pipe-operators.md | 28 -- .../rl-next/repl-doc-renders-doc-comments.md | 55 ---- doc/manual/rl-next/shebang-relative.md | 64 ---- doc/manual/rl-next/tarball-fixes.md | 9 - doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/release-notes/rl-2.24.md | 300 ++++++++++++++++++ 16 files changed, 301 insertions(+), 300 deletions(-) delete mode 100644 doc/manual/rl-next/10564-attrcursor-remove-forceerrors.md delete mode 100644 doc/manual/rl-next/10603-run-the-flake-regressions-test-suite.md delete mode 100644 doc/manual/rl-next/10668-support-unit-prefixes-in-configuration-settings.md delete mode 100644 doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md delete mode 100644 doc/manual/rl-next/10855-meson.md delete mode 100644 doc/manual/rl-next/11086-eval-cache-fix-cache-regressions.md delete mode 100644 doc/manual/rl-next/9063-introduce-libnixflake.md delete mode 100644 doc/manual/rl-next/drop-vendored-toml11.md delete mode 100644 doc/manual/rl-next/harden-user-sandboxing.md delete mode 100644 doc/manual/rl-next/nix-shell-looks-for-shell-nix.md delete mode 100644 doc/manual/rl-next/pipe-operators.md delete mode 100644 doc/manual/rl-next/repl-doc-renders-doc-comments.md delete mode 100644 doc/manual/rl-next/shebang-relative.md delete mode 100644 doc/manual/rl-next/tarball-fixes.md create mode 100644 doc/manual/src/release-notes/rl-2.24.md diff --git a/doc/manual/rl-next/10564-attrcursor-remove-forceerrors.md b/doc/manual/rl-next/10564-attrcursor-remove-forceerrors.md deleted file mode 100644 index 864a55b51f7..00000000000 --- a/doc/manual/rl-next/10564-attrcursor-remove-forceerrors.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -synopsis: "Solve `cached failure of attribute X`" -prs: 10564 -issues: 10513 9165 ---- - -This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. - -Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) diff --git a/doc/manual/rl-next/10603-run-the-flake-regressions-test-suite.md b/doc/manual/rl-next/10603-run-the-flake-regressions-test-suite.md deleted file mode 100644 index 42864323c06..00000000000 --- a/doc/manual/rl-next/10603-run-the-flake-regressions-test-suite.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -synopsis: "Run the flake regressions test suite" -prs: 10603 ---- - -This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. - -Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) diff --git a/doc/manual/rl-next/10668-support-unit-prefixes-in-configuration-settings.md b/doc/manual/rl-next/10668-support-unit-prefixes-in-configuration-settings.md deleted file mode 100644 index 2caca9a815a..00000000000 --- a/doc/manual/rl-next/10668-support-unit-prefixes-in-configuration-settings.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -synopsis: "Support unit prefixes in configuration settings" -prs: 10668 ---- - -Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. - -This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). - -Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) diff --git a/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md b/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md deleted file mode 100644 index 1d623e952e9..00000000000 --- a/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -synopsis: "nix3-build: show all FOD errors with `--keep-going`" -prs: 10734 ---- - -The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. - -Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) - -[`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) diff --git a/doc/manual/rl-next/10855-meson.md b/doc/manual/rl-next/10855-meson.md deleted file mode 100644 index 0ab71390f7f..00000000000 --- a/doc/manual/rl-next/10855-meson.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -synopsis: "Build with Meson" -prs: -- 10378 -- 10855 -- 10904 -- 10908 -- 10914 -- 10933 -- 10936 -- 10954 -- 10955 -- 10967 -- 10963 -- 10973 -- 11034 -- 11054 -- 11055 -- 11064 -- 11060 -- 11155 -issues: -- 2503 ---- - -These changes aim to replace the use of autotools and make with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. - -Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). - -Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) -Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) diff --git a/doc/manual/rl-next/11086-eval-cache-fix-cache-regressions.md b/doc/manual/rl-next/11086-eval-cache-fix-cache-regressions.md deleted file mode 100644 index 8a348a9adcc..00000000000 --- a/doc/manual/rl-next/11086-eval-cache-fix-cache-regressions.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -synopsis: "Eval cache: fix cache regressions" -prs: 11086 -issues: 10570 ---- - -This update addresses two bugs in the evaluation cache system: - -1. Regression in #10570: The evaluation cache was not being persisted in `nix develop` because `evalCaches` retained references to the caches and was never freed. -2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. - -These bug fixes ensure that the evaluation cache is correctly managed and errors are appropriately handled. - -Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) diff --git a/doc/manual/rl-next/9063-introduce-libnixflake.md b/doc/manual/rl-next/9063-introduce-libnixflake.md deleted file mode 100644 index fd3645446c4..00000000000 --- a/doc/manual/rl-next/9063-introduce-libnixflake.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -synopsis: "Introduce `libnixflake`" -prs: 9063 ---- - -A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. - -See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. - -This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. - -Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) diff --git a/doc/manual/rl-next/drop-vendored-toml11.md b/doc/manual/rl-next/drop-vendored-toml11.md deleted file mode 100644 index 8dd786c4452..00000000000 --- a/doc/manual/rl-next/drop-vendored-toml11.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -synopsis: Stop vendoring toml11 ---- - -We don't apply any patches to it, and vendoring it locks users into -bugs (it hasn't been updated since its introduction in late 2021). - -Author: [**Winter (@winterqt)**](https://github.com/winterqt) diff --git a/doc/manual/rl-next/harden-user-sandboxing.md b/doc/manual/rl-next/harden-user-sandboxing.md deleted file mode 100644 index ff81c9cb174..00000000000 --- a/doc/manual/rl-next/harden-user-sandboxing.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -synopsis: Harden the user sandboxing -significance: significant -issues: ---- - -The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. - -This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531), that was handled through the GitHub Security Advisories interface, and hence was merged directly in commit [2dd7f8f42](https://github.com/NixOS/nix/commit/2dd7f8f42da374d9fee4d424c1c6f82bcb36b393) instead of a PR. - -Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) -Co-authors: [**@edolstra**](https://github.com/edolstra) diff --git a/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md b/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md deleted file mode 100644 index b9e4b3fb3ef..00000000000 --- a/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -synopsis: "`nix-shell ` looks for `shell.nix`" -significance: significant -issues: -- 496 -- 2279 -- 4529 -- 5431 -- 11053 -prs: -- 11057 ---- - -`nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. - -Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. -This adjustment addresses a commonly reported problem. - -This also applies to `nix-shell` shebang scripts. Consider the following example: - -```shell -#!/usr/bin/env nix-shell -#!nix-shell -i bash -``` - -This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. - -The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. - -Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) diff --git a/doc/manual/rl-next/pipe-operators.md b/doc/manual/rl-next/pipe-operators.md deleted file mode 100644 index b4cbe30e354..00000000000 --- a/doc/manual/rl-next/pipe-operators.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -synopsis: "Add `pipe-operators` experimental feature" -prs: -- 11131 ---- - -This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). - -The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. -*a* `|>` *b* is equivalent to the function application *b* *a*, and -*a* `<|` *b* is equivalent to the function application *a* *b*. - -For example: - -``` -nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 -9 - -nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 -7 -``` - -`<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. -These properties may change in future releases. - -See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. - -[pipe operators]: @docroot@/language/operators.md#pipe-operators diff --git a/doc/manual/rl-next/repl-doc-renders-doc-comments.md b/doc/manual/rl-next/repl-doc-renders-doc-comments.md deleted file mode 100644 index fa241ebc187..00000000000 --- a/doc/manual/rl-next/repl-doc-renders-doc-comments.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -synopsis: "`nix-repl`'s `:doc` shows documentation comments" -significance: significant -issues: -- 3904 -- 10771 -prs: -- 1652 -- 9054 -- 11072 ---- - -`nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. -This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. - -Example: - -``` -nix-repl> :doc lib.toFunction -Function toFunction - … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 - - Turns any non-callable values into constant functions. Returns - callable values as is. - -Inputs - - v - - : Any value - -Examples - - :::{.example} - -## lib.trivial.toFunction usage example - - | nix-repl> lib.toFunction 1 2 - | 1 - | - | nix-repl> lib.toFunction (x: x + 1) 2 - | 3 - - ::: -``` - -Known limitations: -- It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. -- Some extensions to markdown are not yet supported, as you can see in the example above. - -We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. - -Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) - -[RFC 145]: https://github.com/NixOS/rfcs/pull/145 diff --git a/doc/manual/rl-next/shebang-relative.md b/doc/manual/rl-next/shebang-relative.md deleted file mode 100644 index dd96bf20375..00000000000 --- a/doc/manual/rl-next/shebang-relative.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -synopsis: "`nix-shell` shebang uses relative path" -prs: -- 5088 -- 11058 -issues: -- 4232 ---- - - -Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). -Previously they were resolved relative to the current working directory. - -For example, consider the following script in `~/myproject/say-hi`: - -```shell -#!/usr/bin/env nix-shell -#!nix-shell --expr 'import ./shell.nix' -#!nix-shell --arg toolset './greeting-tools.nix' -#!nix-shell -i bash -hello -``` - -Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory; home in this example: - -```console -[hostname:~]$ ./myproject/say-hi -error: - … while calling the 'import' builtin - at «string»:1:2: - 1| (import ./shell.nix) - | ^ - - error: path '/home/user/shell.nix' does not exist -``` - -Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. - -```console -$ ./myproject/say-hi -Hello, world! -``` - -**Opt-out** - -This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. -The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. -This option will be removed in a future release. - -**`nix` command shebang** - -The experimental [`nix` command shebang](@docroot@/command-ref/new-cli/nix.md?highlight=shebang#shebang-interpreter) already behaves in this script-relative manner. - -Example: - -```shell -#!/usr/bin/env nix -#!nix develop -#!nix --expr ``import ./shell.nix`` -#!nix -c bash -hello -``` - -Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) diff --git a/doc/manual/rl-next/tarball-fixes.md b/doc/manual/rl-next/tarball-fixes.md deleted file mode 100644 index c938e9db6c2..00000000000 --- a/doc/manual/rl-next/tarball-fixes.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -synopsis: "Improve handling of tarballs that don't consist of a single top-level directory" -prs: -- 11195 ---- - -In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). - -Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 3918faeb2dd..8739599a03e 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -127,6 +127,7 @@ - [Contributing](development/contributing.md) - [Releases](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} + - [Release 2.24 (2024-07-31)](release-notes/rl-2.24.md) - [Release 2.23 (2024-06-03)](release-notes/rl-2.23.md) - [Release 2.22 (2024-04-23)](release-notes/rl-2.22.md) - [Release 2.21 (2024-03-11)](release-notes/rl-2.21.md) diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md new file mode 100644 index 00000000000..c6807257379 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -0,0 +1,300 @@ +# Release 2.24.0 (2024-07-31) + +### Significant changes + +- Harden the user sandboxing + + The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. + + This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531), that was handled through the GitHub Security Advisories interface, and hence was merged directly in commit [2dd7f8f42](https://github.com/NixOS/nix/commit/2dd7f8f42da374d9fee4d424c1c6f82bcb36b393) instead of a PR. + + Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) + Co-authors: [**@edolstra**](https://github.com/edolstra) + +- `nix-shell ` looks for `shell.nix` [#496](https://github.com/NixOS/nix/issues/496) [#2279](https://github.com/NixOS/nix/issues/2279) [#4529](https://github.com/NixOS/nix/issues/4529) [#5431](https://github.com/NixOS/nix/issues/5431) [#11053](https://github.com/NixOS/nix/issues/11053) [#11057](https://github.com/NixOS/nix/pull/11057) + + `nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. + + Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. + This adjustment addresses a commonly reported problem. + + This also applies to `nix-shell` shebang scripts. Consider the following example: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell -i bash + ``` + + This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. + + The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- `nix-repl`'s `:doc` shows documentation comments [#3904](https://github.com/NixOS/nix/issues/3904) [#10771](https://github.com/NixOS/nix/issues/10771) [#1652](https://github.com/NixOS/nix/pull/1652) [#9054](https://github.com/NixOS/nix/pull/9054) [#11072](https://github.com/NixOS/nix/pull/11072) + + `nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. + This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. + + Example: + + ``` + nix-repl> :doc lib.toFunction + Function toFunction + … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 + + Turns any non-callable values into constant functions. Returns + callable values as is. + + Inputs + + v + + : Any value + + Examples + + :::{.example} + + ## lib.trivial.toFunction usage example + + | nix-repl> lib.toFunction 1 2 + | 1 + | + | nix-repl> lib.toFunction (x: x + 1) 2 + | 3 + + ::: + ``` + + Known limitations: + - It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. + - Some extensions to markdown are not yet supported, as you can see in the example above. + + We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + + [RFC 145]: https://github.com/NixOS/rfcs/pull/145 + +### Other changes + +- Solve `cached failure of attribute X` [#9165](https://github.com/NixOS/nix/issues/9165) [#10513](https://github.com/NixOS/nix/issues/10513) [#10564](https://github.com/NixOS/nix/pull/10564) + + This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Run the flake regressions test suite [#10603](https://github.com/NixOS/nix/pull/10603) + + This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Support unit prefixes in configuration settings [#10668](https://github.com/NixOS/nix/pull/10668) + + Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + + This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- nix3-build: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) + + The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. + + Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) + + [`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) + +- Build with Meson [#2503](https://github.com/NixOS/nix/issues/2503) [#10378](https://github.com/NixOS/nix/pull/10378) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10963](https://github.com/NixOS/nix/pull/10963) [#10967](https://github.com/NixOS/nix/pull/10967) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11060](https://github.com/NixOS/nix/pull/11060) [#11064](https://github.com/NixOS/nix/pull/11064) [#11155](https://github.com/NixOS/nix/pull/11155) + + These changes aim to replace the use of autotools and make with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + + Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). + + Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) + Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) + +- Eval cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) + + This update addresses two bugs in the evaluation cache system: + + 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop` because `evalCaches` retained references to the caches and was never freed. + 2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. + + These bug fixes ensure that the evaluation cache is correctly managed and errors are appropriately handled. + + Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) + +- Introduce `libnixflake` [#9063](https://github.com/NixOS/nix/pull/9063) + + A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. + + See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. + + This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. + + Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) + +- CL options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) + + The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Make `nix store gc` use the auto-GC policy [#7851](https://github.com/NixOS/nix/pull/7851) + + + +- Stop vendoring toml11 + + We don't apply any patches to it, and vendoring it locks users into + bugs (it hasn't been updated since its introduction in late 2021). + + Author: [**Winter (@winterqt)**](https://github.com/winterqt) + +- Rename hash format `base32` to `nix32` [#8678](https://github.com/NixOS/nix/pull/8678) + + Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for + [Base32](https://en.wikipedia.org/wiki/Base32). + + ## Deprecation: Use `nix32` instead of `base32` as `toHashFormat` + + For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` + parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value + remains as a deprecated alias for `"base32"`. Please convert your code from: + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} + ``` + + to + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} + ``` + +- Add `pipe-operators` experimental feature [#11131](https://github.com/NixOS/nix/pull/11131) + + This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). + + The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. + *a* `|>` *b* is equivalent to the function application *b* *a*, and + *a* `<|` *b* is equivalent to the function application *a* *b*. + + For example: + + ``` + nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 + 9 + + nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 + 7 + ``` + + `<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. + These properties may change in future releases. + + See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. + + [pipe operators]: @docroot@/language/operators.md#pipe-operators + +- `nix-shell` shebang uses relative path [#4232](https://github.com/NixOS/nix/issues/4232) [#5088](https://github.com/NixOS/nix/pull/5088) [#11058](https://github.com/NixOS/nix/pull/11058) + + + Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). + Previously they were resolved relative to the current working directory. + + For example, consider the following script in `~/myproject/say-hi`: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell --expr 'import ./shell.nix' + #!nix-shell --arg toolset './greeting-tools.nix' + #!nix-shell -i bash + hello + ``` + + Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory; home in this example: + + ```console + [hostname:~]$ ./myproject/say-hi + error: + … while calling the 'import' builtin + at «string»:1:2: + 1| (import ./shell.nix) + | ^ + + error: path '/home/user/shell.nix' does not exist + ``` + + Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. + + ```console + $ ./myproject/say-hi + Hello, world! + ``` + + **Opt-out** + + This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. + The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. + This option will be removed in a future release. + + **`nix` command shebang** + + The experimental [`nix` command shebang](@docroot@/command-ref/new-cli/nix.md?highlight=shebang#shebang-interpreter) already behaves in this script-relative manner. + + Example: + + ```shell + #!/usr/bin/env nix + #!nix develop + #!nix --expr ``import ./shell.nix`` + #!nix -c bash + hello + ``` + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Setting to warn about large paths [#10778](https://github.com/NixOS/nix/pull/10778) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using the `warn-large-path-threshold` setting, + e.g. `--warn-large-path-threshold 100M`. + + +# Contributors + +Querying GitHub API for ee86e7f361c55c8c7dc2e45c3868802af249aeff, to get handle for MostAwesomeDude@gmail.com From 733c816d3493459e0f7f899111b9ad19fe04147a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 31 Jul 2024 15:04:18 -0500 Subject: [PATCH 172/284] Small windows cross fixes (#11230) --- src/libexpr/eval.cc | 4 +++- src/libstore/gc.cc | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 746ccab2ace..de5d85821ef 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2860,8 +2860,10 @@ void EvalState::printStatistics() topObj["cpuTime"] = cpuTime; #endif topObj["time"] = { +#ifndef _WIN32 // TODO implement {"cpu", cpuTime}, -#ifdef HAVE_BOEHMGC +#endif +#if HAVE_BOEHMGC {GC_is_incremental_mode() ? "gcNonIncremental" : "gc", gcFullOnlyTime}, {GC_is_incremental_mode() ? "gcNonIncrementalFraction" : "gcFraction", gcFullOnlyTime / cpuTime}, #endif diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index c865fddd7db..1494712dab4 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -891,7 +891,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) void LocalStore::autoGC(bool sync) { -#ifdef HAVE_STATVFS +#if HAVE_STATVFS static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE"); auto getAvail = [this]() -> uint64_t { From 22ad0e653f9abff03de97fe928eaf13a99f7e8a1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 31 Jul 2024 22:14:27 +0200 Subject: [PATCH 173/284] Edit release notes --- doc/manual/src/release-notes/rl-2.24.md | 42 +++++++------------------ 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md index c6807257379..cb82b1def48 100644 --- a/doc/manual/src/release-notes/rl-2.24.md +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -2,11 +2,11 @@ ### Significant changes -- Harden the user sandboxing +- Harden user sandboxing The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. - This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531), that was handled through the GitHub Security Advisories interface, and hence was merged directly in commit [2dd7f8f42](https://github.com/NixOS/nix/commit/2dd7f8f42da374d9fee4d424c1c6f82bcb36b393) instead of a PR. + This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531). Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) Co-authors: [**@edolstra**](https://github.com/edolstra) @@ -99,7 +99,7 @@ Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) -- nix3-build: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) +- `nix build`: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. @@ -109,22 +109,20 @@ - Build with Meson [#2503](https://github.com/NixOS/nix/issues/2503) [#10378](https://github.com/NixOS/nix/pull/10378) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10963](https://github.com/NixOS/nix/pull/10963) [#10967](https://github.com/NixOS/nix/pull/10967) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11060](https://github.com/NixOS/nix/pull/11060) [#11064](https://github.com/NixOS/nix/pull/11064) [#11155](https://github.com/NixOS/nix/pull/11155) - These changes aim to replace the use of autotools and make with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + These changes aim to replace the use of autotools and `make` with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) -- Eval cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) +- Evaluation cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) This update addresses two bugs in the evaluation cache system: - 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop` because `evalCaches` retained references to the caches and was never freed. + 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop`. 2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. - These bug fixes ensure that the evaluation cache is correctly managed and errors are appropriately handled. - Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) - Introduce `libnixflake` [#9063](https://github.com/NixOS/nix/pull/9063) @@ -137,9 +135,9 @@ Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) -- CL options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) +- CLI options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) - The `--debugger` now prints source location information, instead of the +- The `--debugger` now prints source location information, instead of the pointers of source location information. Before: ``` @@ -160,11 +158,7 @@ 133| in ``` -- Make `nix store gc` use the auto-GC policy [#7851](https://github.com/NixOS/nix/pull/7851) - - - -- Stop vendoring toml11 +- Stop vendoring `toml11` We don't apply any patches to it, and vendoring it locks users into bugs (it hasn't been updated since its introduction in late 2021). @@ -176,7 +170,7 @@ Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for [Base32](https://en.wikipedia.org/wiki/Base32). - ## Deprecation: Use `nix32` instead of `base32` as `toHashFormat` + **Deprecation**: Use `nix32` instead of `base32` as `toHashFormat` For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value @@ -233,7 +227,7 @@ hello ``` - Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory; home in this example: + Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory, such as the user's home directory in this example: ```console [hostname:~]$ ./myproject/say-hi @@ -259,20 +253,6 @@ The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. This option will be removed in a future release. - **`nix` command shebang** - - The experimental [`nix` command shebang](@docroot@/command-ref/new-cli/nix.md?highlight=shebang#shebang-interpreter) already behaves in this script-relative manner. - - Example: - - ```shell - #!/usr/bin/env nix - #!nix develop - #!nix --expr ``import ./shell.nix`` - #!nix -c bash - hello - ``` - Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) - Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) From f136ec5290128470244006579d935653df355bdf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 31 Jul 2024 22:16:44 +0200 Subject: [PATCH 174/284] Add contributors --- doc/manual/src/release-notes/rl-2.24.md | 46 ++++++++++++++++++- .../data/release-credits-email-to-handle.json | 3 +- .../data/release-credits-handle-to-name.json | 3 +- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md index cb82b1def48..5479cd3b9f5 100644 --- a/doc/manual/src/release-notes/rl-2.24.md +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -277,4 +277,48 @@ # Contributors -Querying GitHub API for ee86e7f361c55c8c7dc2e45c3868802af249aeff, to get handle for MostAwesomeDude@gmail.com +This release was made possible by the following 43 contributors: + +- Andreas Rammhold [**(@andir)**](https://github.com/andir) +- Andrew Marshall [**(@amarshall)**](https://github.com/amarshall) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Cameron [**(@SkamDart)**](https://github.com/SkamDart) +- Cole Helbling [**(@cole-h)**](https://github.com/cole-h) +- Corbin Simpson [**(@MostAwesomeDude)**](https://github.com/MostAwesomeDude) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- Emily [**(@emilazy)**](https://github.com/emilazy) +- Enno Richter [**(@elohmeier)**](https://github.com/elohmeier) +- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria) +- HaeNoe [**(@haenoe)**](https://github.com/haenoe) +- Hamir Mahal [**(@hamirmahal)**](https://github.com/hamirmahal) +- Harmen [**(@alicebob)**](https://github.com/alicebob) +- Ivan Trubach [**(@tie)**](https://github.com/tie) +- Jared Baur [**(@jmbaur)**](https://github.com/jmbaur) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Jonathan De Troye [**(@detroyejr)**](https://github.com/detroyejr) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Klemens Nanni [**(@klemensn)**](https://github.com/klemensn) +- Las Safin [**(@L-as)**](https://github.com/L-as) +- Lexi Mattick [**(@kognise)**](https://github.com/kognise) +- Matthew Bauer [**(@matthewbauer)**](https://github.com/matthewbauer) +- Max “Goldstein” Siling [**(@GoldsteinE)**](https://github.com/GoldsteinE) +- Mingye Wang [**(@Artoria2e5)**](https://github.com/Artoria2e5) +- Philip Taron [**(@philiptaron)**](https://github.com/philiptaron) +- Pierre Bourdon [**(@delroth)**](https://github.com/delroth) +- Pino Toscano [**(@pinotree)**](https://github.com/pinotree) +- RTUnreal [**(@RTUnreal)**](https://github.com/RTUnreal) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- Romain Neil [**(@romain-neil)**](https://github.com/romain-neil) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Shogo Takata [**(@pineapplehunter)**](https://github.com/pineapplehunter) +- Siddhant Kumar [**(@siddhantk232)**](https://github.com/siddhantk232) +- Silvan Mosberger [**(@infinisil)**](https://github.com/infinisil) +- Théophane Hufschmitt [**(@thufschmitt)**](https://github.com/thufschmitt) +- Valentin Gagarin [**(@fricklerhandwerk)**](https://github.com/fricklerhandwerk) +- Winter [**(@winterqt)**](https://github.com/winterqt) +- jade [**(@lf-)**](https://github.com/lf-) +- kirillrdy [**(@kirillrdy)**](https://github.com/kirillrdy) +- pennae [**(@pennae)**](https://github.com/pennae) +- poweredbypie [**(@poweredbypie)**](https://github.com/poweredbypie) +- tomberek [**(@tomberek)**](https://github.com/tomberek) diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json index 573ec2b3100..cddc1a6e70a 100644 --- a/maintainers/data/release-credits-email-to-handle.json +++ b/maintainers/data/release-credits-email-to-handle.json @@ -47,5 +47,6 @@ "pennae@lix.systems": "pennae", "delroth@gmail.com": "delroth", "enno@nerdworks.de": "elohmeier", - "mjbauer95@gmail.com": "matthewbauer" + "mjbauer95@gmail.com": "matthewbauer", + "MostAwesomeDude@gmail.com": "MostAwesomeDude" } \ No newline at end of file diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json index d68311dde10..abf9ed05b97 100644 --- a/maintainers/data/release-credits-handle-to-name.json +++ b/maintainers/data/release-credits-handle-to-name.json @@ -40,5 +40,6 @@ "siddhantk232": "Siddhant Kumar", "winterqt": "Winter", "GoldsteinE": "Max \u201cGoldstein\u201d Siling", - "pennae": null + "pennae": null, + "MostAwesomeDude": "Corbin Simpson" } \ No newline at end of file From 8ff169715dde088a4c286b3379d3f548eafe3221 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Mon, 29 Apr 2024 08:15:16 -0600 Subject: [PATCH 175/284] docs: clarify how the different kinds of installables are selected Change-Id: I146736bb97ebe035e04be69ce9fb60a557e38c6c --- src/nix/nix.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/nix/nix.md b/src/nix/nix.md index f958ce09acc..34f73d03284 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -59,9 +59,13 @@ These are command line arguments that represent something that can be realised i The following types of installable are supported by most commands: - [Flake output attribute](#flake-output-attribute) (experimental) + - This is the default - [Store path](#store-path) + - This is assumed if the argument is a Nix store path or a symlink to a Nix store path - [Nix file](#nix-file), optionally qualified by an attribute path + - Specified with `--file`/`-f` - [Nix expression](#nix-expression), optionally qualified by an attribute path + - Specified with `--expr`/`-E` For most commands, if no installable is specified, `.` is assumed. That is, Nix will operate on the default flake output attribute of the flake in the current directory. From cb5a5dd4f3064e3235fa908a44a5d074ef3c4205 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Mon, 29 Apr 2024 08:09:50 -0600 Subject: [PATCH 176/284] docs: clarify how ^ works for -E/-f installables We didn't even realize you *could* use this syntax with -E and -f, much less that the attribute path could be *empty*. Change-Id: Id1a6715609f3a76a5ce477bd43a7832effbbe07b --- src/nix/nix.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/nix/nix.md b/src/nix/nix.md index 34f73d03284..56587d0b233 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -182,9 +182,10 @@ that contains programs, and a `dev` output that provides development artifacts like C/C++ header files. The outputs on which `nix` commands operate are determined as follows: -* You can explicitly specify the desired outputs using the syntax - *installable*`^`*output1*`,`*...*`,`*outputN*. For example, you can - obtain the `dev` and `static` outputs of the `glibc` package: +* You can explicitly specify the desired outputs using the syntax *installable*`^`*output1*`,`*...*`,`*outputN* — that is, a caret followed immediately by a comma-separated list of derivation outputs to select. + For installables specified as [Flake output attributes](#flake-output-attribute) or [Store paths](#store-path), the output is specified in the same argument: + + For example, you can obtain the `dev` and `static` outputs of the `glibc` package: ```console # nix build 'nixpkgs#glibc^dev,static' @@ -199,6 +200,19 @@ operate are determined as follows: … ``` + For `-e`/`--expr` and `-f`/`--file`, the derivation output is specified as part of the attribute path: + + ```console + $ nix build -f '' 'glibc^dev,static' + $ nix build --impure -E 'import { }' 'glibc^dev,static' + ``` + + This syntax is the same even if the actual attribute path is empty: + + ```console + $ nix build -E 'let pkgs = import { }; in pkgs.glibc' '^dev,static' + ``` + * You can also specify that *all* outputs should be used using the syntax *installable*`^*`. For example, the following shows the size of all outputs of the `glibc` package in the binary cache: From 794a50065b33cbaaaf1f6ff9dbf954bb5eedfc14 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 31 Jul 2024 22:33:41 +0200 Subject: [PATCH 177/284] base32 -> nix32 --- doc/manual/src/release-notes/rl-2.24.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md index 5479cd3b9f5..5bcc1d79ced 100644 --- a/doc/manual/src/release-notes/rl-2.24.md +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -174,7 +174,7 @@ For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value - remains as a deprecated alias for `"base32"`. Please convert your code from: + remains as a deprecated alias for `"nix32"`. Please convert your code from: ```nix builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} From 6ed67d35ed51905434eba7d4a167b43aa581478f Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Wed, 31 Jul 2024 17:39:43 -0400 Subject: [PATCH 178/284] docs: add variables; rework scope (#11062) Co-authored-by: Valentin Gagarin --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/language/identifiers.md | 2 +- doc/manual/src/language/scope.md | 32 ++++++++++++++++++-------- doc/manual/src/language/variables.md | 10 ++++++++ 4 files changed, 35 insertions(+), 10 deletions(-) create mode 100644 doc/manual/src/language/variables.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 8739599a03e..7661f5f6287 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -28,6 +28,7 @@ - [Data Types](language/types.md) - [String context](language/string-context.md) - [Syntax and semantics](language/syntax.md) + - [Variables](language/variables.md) - [Identifiers](language/identifiers.md) - [Scoping rules](language/scope.md) - [String interpolation](language/string-interpolation.md) diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md index c9e981da626..bd58a9b365b 100644 --- a/doc/manual/src/language/identifiers.md +++ b/doc/manual/src/language/identifiers.md @@ -22,7 +22,7 @@ A name can be an [identifier](#identifier) or a [string literal](./syntax.md#str > > *name* → *identifier* | *string* -Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting attributes). +Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting-attributes). # Keywords diff --git a/doc/manual/src/language/scope.md b/doc/manual/src/language/scope.md index 5c6aed38d31..96c7468eb53 100644 --- a/doc/manual/src/language/scope.md +++ b/doc/manual/src/language/scope.md @@ -1,14 +1,28 @@ # Scoping rules -Nix is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope), but with multiple scopes and shadowing rules. +A *scope* in the Nix language is a dictionary keyed by [name](./identifiers.md#names), mapping each name to an expression and a *definition type*. +The definition type is either *explicit* or *implicit*. +Each entry in this dictionary is a *definition*. -* primary scope: explicitly-bound variables - * [`let`](./syntax.md#let-expressions) - * [`inherit`](./syntax.md#inheriting-attributes) - * [function](./syntax.md#functions) arguments +Explicit definitions are created by the following expressions: +- [let-expressions](syntax.md#let-expressions) +- [recursive attribute set literals](syntax.md#recursive-sets) (`rec`) +- [function literals](syntax.md#functions) -* secondary scope: implicitly-bound variables - * [`with`](./syntax.md#with-expressions) +Implicit definitions are only created by [with-expressions](./syntax.md#with-expressions). -Primary scope takes precedence over secondary scope. -See [`with`](./syntax.md#with-expressions) for a detailed example. +Every expression is *enclosed* by a scope. +The outermost expression is enclosed by the [built-in, global scope](./builtins.md), which contains only explicit definitions. +The respective definition types *extend* their enclosing scope by adding new definitions, or replacing existing ones with the same name. +An explicit definition can replace a definition of any type; an implicit definition can only replace another implicit definition. + +Each of the above expressions defines which of its subexpressions are enclosed by the extended scope. +In all other cases, the same scope that encloses an expression is the enclosing scope for its subexpressions. + +The Nix language is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope); +the value of a variable is determined only by the variable's enclosing scope, and not by the dynamic context in which the variable is evaluated. + +> **Note** +> +> Expressions entered into the [Nix REPL](@docroot@/command-ref/new-cli/nix3-repl.md) are enclosed by a scope that can be extended by command line arguments or previous REPL commands. +> These ways of extending scope are not, strictly speaking, part of the Nix language. diff --git a/doc/manual/src/language/variables.md b/doc/manual/src/language/variables.md new file mode 100644 index 00000000000..af6aff8a2c5 --- /dev/null +++ b/doc/manual/src/language/variables.md @@ -0,0 +1,10 @@ +# Variables + +A *variable* is an [identifier](identifiers.md) used as an expression. + +> **Syntax** +> +> *expression* → *identifier* + +A variable must have the same name as a definition in the [scope](./scope.md) that encloses it. +The value of a variable is the value of the corresponding expression in the enclosing scope. From 9e8afc68e554c5c45b8e9f387d4fcf4ac0174a6a Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Wed, 31 Jul 2024 19:07:57 -0400 Subject: [PATCH 179/284] docs: add language/string-literals.md --- doc/manual/redirects.js | 1 + doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/language/identifiers.md | 2 +- .../src/language/string-interpolation.md | 4 + doc/manual/src/language/string-literals.md | 189 ++++++++++++++++++ doc/manual/src/language/syntax.md | 172 +--------------- doc/manual/src/language/types.md | 2 +- 7 files changed, 199 insertions(+), 172 deletions(-) create mode 100644 doc/manual/src/language/string-literals.md diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index beef6ef4ab5..cb8cd18fa14 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -344,6 +344,7 @@ const redirects = { }, "language/syntax.html": { "scoping-rules": "scoping.html", + "string-literal": "string-literals.html", }, "installation/installing-binary.html": { "linux": "uninstall.html#linux", diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 7661f5f6287..eef7d189cbb 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -29,6 +29,7 @@ - [String context](language/string-context.md) - [Syntax and semantics](language/syntax.md) - [Variables](language/variables.md) + - [String literals](language/string-literals.md) - [Identifiers](language/identifiers.md) - [Scoping rules](language/scope.md) - [String interpolation](language/string-interpolation.md) diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md index bd58a9b365b..a4153b5886f 100644 --- a/doc/manual/src/language/identifiers.md +++ b/doc/manual/src/language/identifiers.md @@ -16,7 +16,7 @@ An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character seq # Names -A name can be an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). +A name can be an [identifier](#identifier) or a [string literal](string-literals.md). > **Syntax** > diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 1778bdfa06b..27780dcbb39 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -8,6 +8,10 @@ Such a construct is called *interpolated string*, and the expression inside is a [path]: ./types.md#type-path [attribute set]: ./types.md#attribute-set +> **Syntax** +> +> *interpolation_element* → `${` *expression* `}` + ## Examples ### String diff --git a/doc/manual/src/language/string-literals.md b/doc/manual/src/language/string-literals.md new file mode 100644 index 00000000000..7605e8c61db --- /dev/null +++ b/doc/manual/src/language/string-literals.md @@ -0,0 +1,189 @@ +# String literals + +A *string literal* represents a [string](types.md#type-string) value. + +> **Syntax** +> +> *expression* → *string* +> +> *string* → `"` ( *string_char*\* [*interpolation_element*][string interpolation] )* *string_char*\* `"` +> +> *string* → `''` ( *indented_string_char*\* [*interpolation_element*][string interpolation] )* *indented_string_char*\* `''` +> +> *string* → *uri* +> +> *string_char* ~ `[^"$\\]|\$(?!\{)|\\.` +> +> *indented_string_char* ~ `[^$']|\$\$|\$(?!\{)|''[$']|''\\.|'(?!')` +> +> *uri* ~ `[A-Za-z][+\-.0-9A-Za-z]*:[!$%&'*+,\-./0-9:=?@A-Z_a-z~]+` + +Strings can be written in three ways. + +The most common way is to enclose the string between double quotes, e.g., `"foo bar"`. +Strings can span multiple lines. +The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation]. + +[string interpolation]: ./string-interpolation.md + +The following must be escaped to represent them within a string, by prefixing with a backslash (`\`): + +- Double quote (`"`) + +> **Example** +> +> ```nix +> "\"" +> ``` +> +> "\"" + +- Backslash (`\`) + +> **Example** +> +> ```nix +> "\\" +> ``` +> +> "\\" + +- Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" + +> **Example** +> +> ```nix +> "\${" +> ``` +> +> "\${" + +The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. + +A "double-dollar-curly" (`$${`) can be written literally. + +> **Example** +> +> ```nix +> "$${" +> ``` +> +> "$\${" + +String values are output on the terminal with Nix-specific escaping. +Strings written to files will contain the characters encoded by the escaping. + +The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so: + +```nix +'' +This is the first line. +This is the second line. + This is the third line. +'' +``` + +This kind of string literal intelligently strips indentation from +the start of each line. To be precise, it strips from each line a +number of spaces equal to the minimal indentation of the string as a +whole (disregarding the indentation of empty lines). For instance, +the first and second line are indented two spaces, while the third +line is indented four spaces. Thus, two spaces are stripped from +each line, so the resulting string is + +```nix +"This is the first line.\nThis is the second line.\n This is the third line.\n" +``` + +> **Note** +> +> Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line. + +> **Warning** +> +> Prefixed tab characters are not stripped. +> +> > **Example** +> > +> > The following indented string is prefixed with tabs: +> > +> > '' +> > all: +> > @echo hello +> > '' +> > +> > "\tall:\n\t\t@echo hello\n" + +Indented strings support [string interpolation]. + +The following must be escaped to represent them in an indented string: + +- `$` is escaped by prefixing it with two single quotes (`''`) + +> **Example** +> +> ```nix +> '' +> ''$ +> '' +> ``` +> +> "$\n" + +- `''` is escaped by prefixing it with one single quote (`'`) + +> **Example** +> +> ```nix +> '' +> ''' +> '' +> ``` +> +> "''\n" + +These special characters are escaped as follows: +- Linefeed (`\n`): `''\n` +- Carriage return (`\r`): `''\r` +- Tab (`\t`): `''\t` + +`''\` escapes any other character. + +A "double-dollar-curly" (`$${`) can be written literally. + +> **Example** +> +> ```nix +> '' +> $${ +> '' +> ``` +> +> "$\${\n" + +Indented strings are primarily useful in that they allow multi-line +string literals to follow the indentation of the enclosing Nix +expression, and that less escaping is typically necessary for +strings representing languages such as shell scripts and +configuration files because `''` is much less common than `"`. +Example: + +```nix +stdenv.mkDerivation { +... +postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; +... +} +``` + +Finally, as a convenience, *URIs* as defined in appendix B of +[RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as +is*, without quotes. For instance, the string +`"http://example.org/foo.tar.bz2"` can also be written as +`http://example.org/foo.tar.bz2`. diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md index 6108bacd676..daf073aeff8 100644 --- a/doc/manual/src/language/syntax.md +++ b/doc/manual/src/language/syntax.md @@ -6,175 +6,7 @@ This section covers syntax and semantics of the Nix language. ### String {#string-literal} - *Strings* can be written in three ways. - - The most common way is to enclose the string between double quotes, e.g., `"foo bar"`. - Strings can span multiple lines. - The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation]. - - [string interpolation]: ./string-interpolation.md - - The following must be escaped to represent them within a string, by prefixing with a backslash (`\`): - - - Double quote (`"`) - - > **Example** - > - > ```nix - > "\"" - > ``` - > - > "\"" - - - Backslash (`\`) - - > **Example** - > - > ```nix - > "\\" - > ``` - > - > "\\" - - - Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" - - > **Example** - > - > ```nix - > "\${" - > ``` - > - > "\${" - - The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. - - A "double-dollar-curly" (`$${`) can be written literally. - - > **Example** - > - > ```nix - > "$${" - > ``` - > - > "$\${" - - String values are output on the terminal with Nix-specific escaping. - Strings written to files will contain the characters encoded by the escaping. - - The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so: - - ```nix - '' - This is the first line. - This is the second line. - This is the third line. - '' - ``` - - This kind of string literal intelligently strips indentation from - the start of each line. To be precise, it strips from each line a - number of spaces equal to the minimal indentation of the string as a - whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two spaces, while the third - line is indented four spaces. Thus, two spaces are stripped from - each line, so the resulting string is - - ```nix - "This is the first line.\nThis is the second line.\n This is the third line.\n" - ``` - - > **Note** - > - > Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line. - - > **Warning** - > - > Prefixed tab characters are not stripped. - > - > > **Example** - > > - > > The following indented string is prefixed with tabs: - > > - > > '' - > > all: - > > @echo hello - > > '' - > > - > > "\tall:\n\t\t@echo hello\n" - - Indented strings support [string interpolation]. - - The following must be escaped to represent them in an indented string: - - - `$` is escaped by prefixing it with two single quotes (`''`) - - > **Example** - > - > ```nix - > '' - > ''$ - > '' - > ``` - > - > "$\n" - - - `''` is escaped by prefixing it with one single quote (`'`) - - > **Example** - > - > ```nix - > '' - > ''' - > '' - > ``` - > - > "''\n" - - These special characters are escaped as follows: - - Linefeed (`\n`): `''\n` - - Carriage return (`\r`): `''\r` - - Tab (`\t`): `''\t` - - `''\` escapes any other character. - - A "double-dollar-curly" (`$${`) can be written literally. - - > **Example** - > - > ```nix - > '' - > $${ - > '' - > ``` - > - > "$\${\n" - - Indented strings are primarily useful in that they allow multi-line - string literals to follow the indentation of the enclosing Nix - expression, and that less escaping is typically necessary for - strings representing languages such as shell scripts and - configuration files because `''` is much less common than `"`. - Example: - - ```nix - stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... - } - ``` - - Finally, as a convenience, *URIs* as defined in appendix B of - [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as - is*, without quotes. For instance, the string - `"http://example.org/foo.tar.bz2"` can also be written as - `http://example.org/foo.tar.bz2`. +See [String literals](string-literals.md). ### Number {#number-literal} @@ -253,7 +85,7 @@ Attribute sets are written enclosed in curly brackets (`{ }`). Attribute names and attribute values are separated by an equal sign (`=`). Each value can be an arbitrary expression, terminated by a semicolon (`;`) -An attribute name is a string without context, and is denoted by a [name] (an [identifier](./identifiers.md#identifiers) or [string literal](#string-literal)). +An attribute name is a string without context, and is denoted by a [name] (an [identifier](./identifiers.md#identifiers) or [string literal](string-literals.md)). [name]: ./identifiers.md#names diff --git a/doc/manual/src/language/types.md b/doc/manual/src/language/types.md index 229756e6be2..82184a8b030 100644 --- a/doc/manual/src/language/types.md +++ b/doc/manual/src/language/types.md @@ -45,7 +45,7 @@ The function [`builtins.isBool`](builtins.md#builtins-isBool) can be used to det A _string_ in the Nix language is an immutable, finite-length sequence of bytes, along with a [string context](string-context.md). Nix does not assume or support working natively with character encodings. -String values without string context can be expressed as [string literals](syntax.md#string-literal). +String values without string context can be expressed as [string literals](string-literals.md). The function [`builtins.isString`](builtins.md#builtins-isString) can be used to determine if a value is a string. ### Path {#type-path} From 17318bc70dd0a3712d0a2c94a180dd9f614c97b0 Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Wed, 31 Jul 2024 19:22:17 -0400 Subject: [PATCH 180/284] docs: fix string literal example formatting --- doc/manual/src/language/string-literals.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/language/string-literals.md b/doc/manual/src/language/string-literals.md index 7605e8c61db..8f4b75f3e04 100644 --- a/doc/manual/src/language/string-literals.md +++ b/doc/manual/src/language/string-literals.md @@ -36,7 +36,7 @@ The following must be escaped to represent them within a string, by prefixing wi > "\"" > ``` > -> "\"" +> "\"" - Backslash (`\`) @@ -46,7 +46,7 @@ The following must be escaped to represent them within a string, by prefixing wi > "\\" > ``` > -> "\\" +> "\\" - Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" @@ -56,7 +56,7 @@ The following must be escaped to represent them within a string, by prefixing wi > "\${" > ``` > -> "\${" +> "\${" The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. @@ -68,7 +68,7 @@ A "double-dollar-curly" (`$${`) can be written literally. > "$${" > ``` > -> "$\${" +> "$\${" String values are output on the terminal with Nix-specific escaping. Strings written to files will contain the characters encoded by the escaping. @@ -107,10 +107,11 @@ each line, so the resulting string is > > > > The following indented string is prefixed with tabs: > > -> > '' +> > 
''
 > > 	all:
 > > 		@echo hello
 > > ''
+> >
> > > > "\tall:\n\t\t@echo hello\n" From 617e711820762d27285eb8710ce42d52f6448acd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 Aug 2024 10:41:42 +0200 Subject: [PATCH 181/284] 'build' is now 'build.nix' --- maintainers/upload-release.pl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 4c4e2bd6f5e..73198856858 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -42,7 +42,7 @@ sub fetch { my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; -my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json')); +my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix.x86_64-linux", 'application/json')); #print Dumper($buildInfo); my $releaseName = $buildInfo->{nixname}; @@ -91,7 +91,7 @@ sub getStorePath { sub copyManual { my $manual; eval { - $manual = getStorePath("build.x86_64-linux", "doc"); + $manual = getStorePath("build.nix.x86_64-linux", "doc"); }; if ($@) { warn "$@"; @@ -240,12 +240,12 @@ sub downloadFile { # Upload nix-fallback-paths.nix. write_file("$tmpDir/fallback-paths.nix", "{\n" . - " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . - " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . - " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . - " riscv64-linux = \"" . getStorePath("buildCross.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . - " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . - " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . + " x86_64-linux = \"" . getStorePath("build.nix.x86_64-linux") . "\";\n" . + " i686-linux = \"" . getStorePath("build.nix.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.nix.aarch64-linux") . "\";\n" . + " riscv64-linux = \"" . getStorePath("buildCross.nix.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . + " x86_64-darwin = \"" . getStorePath("build.nix.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.nix.aarch64-darwin") . "\";\n" . "}\n"); # Upload release files to S3. From 30aca6f243d8d5dcdbbfcf2a36bcdefa9d2aeeee Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 Aug 2024 10:43:00 +0200 Subject: [PATCH 182/284] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index ad2261920c0..5c18f9195b5 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.24.0 +2.25.0 From b291b61089da1121145d92d6747ad2b18737ad9f Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Thu, 1 Aug 2024 05:14:49 -0400 Subject: [PATCH 183/284] docs: editorial quibbles (#11232) --- doc/manual/src/language/identifiers.md | 3 ++- doc/manual/src/language/scope.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md index bd58a9b365b..861ee3e2092 100644 --- a/doc/manual/src/language/identifiers.md +++ b/doc/manual/src/language/identifiers.md @@ -16,13 +16,14 @@ An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character seq # Names -A name can be an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). +A *name* can be written as an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). > **Syntax** > > *name* → *identifier* | *string* Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting-attributes). +Two names are the same if they represent the same sequence of characters, regardless of whether they are written as identifiers or strings. # Keywords diff --git a/doc/manual/src/language/scope.md b/doc/manual/src/language/scope.md index 96c7468eb53..9373324e2ff 100644 --- a/doc/manual/src/language/scope.md +++ b/doc/manual/src/language/scope.md @@ -13,7 +13,7 @@ Implicit definitions are only created by [with-expressions](./syntax.md#with-exp Every expression is *enclosed* by a scope. The outermost expression is enclosed by the [built-in, global scope](./builtins.md), which contains only explicit definitions. -The respective definition types *extend* their enclosing scope by adding new definitions, or replacing existing ones with the same name. +The expressions listed above *extend* their enclosing scope by adding new definitions, or replacing existing ones with the same name. An explicit definition can replace a definition of any type; an implicit definition can only replace another implicit definition. Each of the above expressions defines which of its subexpressions are enclosed by the extended scope. From 9b5b7b796341eca437fe08bb278c49dfbae2deaa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 Aug 2024 16:51:57 +0200 Subject: [PATCH 184/284] Fix the S3 store It was failing with: error: AWS error fetching 'nix-cache-info': The specified bucket does not exist because `S3BinaryCacheStoreImpl` had a `bucketName` field that shadowed the inherited `bucketName from `S3BinaryCacheStoreConfig`. --- src/libstore/s3-binary-cache-store.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 92ab47cd66d..21175b1ebfd 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -220,8 +220,6 @@ std::string S3BinaryCacheStoreConfig::doc() struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore { - std::string bucketName; - Stats stats; S3Helper s3Helper; From 739418504c4d2f28fb5f45151b1c83707c3571e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 2 Aug 2024 11:12:06 +0200 Subject: [PATCH 185/284] allow to c api with older c versions In the FFI world we have many tools that are not gcc/clang and therefore not always support the latest C standard. This fixes support with cffi i.e. used in https://github.com/tweag/python-nix --- src/libexpr-c/nix_api_expr.h | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index adf8b65b1a3..1764b49f321 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -14,6 +14,16 @@ #include "nix_api_util.h" #include +#ifndef __has_c_attribute +# define __has_c_attribute(x) 0 +#endif + +#if __has_c_attribute(deprecated) +# define NIX_DEPRECATED(msg) [[deprecated(msg)]] +#else +# define NIX_DEPRECATED(msg) +#endif + #ifdef __cplusplus extern "C" { #endif @@ -45,7 +55,7 @@ typedef struct EvalState EvalState; // nix::EvalState * @see nix_value_incref, nix_value_decref */ typedef struct nix_value nix_value; -[[deprecated("use nix_value instead")]] typedef nix_value Value; +NIX_DEPRECATED("use nix_value instead") typedef nix_value Value; // Function prototypes /** From 088fa815d324e9c6fddf2f0358ac9b64ba6d42c7 Mon Sep 17 00:00:00 2001 From: 0x5a4 <54070204+0x5a4@users.noreply.github.com> Date: Thu, 1 Aug 2024 23:37:45 +0200 Subject: [PATCH 186/284] fix: bash mangles flake ref completion --- misc/bash/completion.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/misc/bash/completion.sh b/misc/bash/completion.sh index 9af695f5a43..c4ba96cd32c 100644 --- a/misc/bash/completion.sh +++ b/misc/bash/completion.sh @@ -12,9 +12,16 @@ function _complete_nix { elif [[ $completion == attrs ]]; then compopt -o nospace fi - else - COMPREPLY+=("$completion") + continue fi + + if [[ "${cur}" =~ "=" ]]; then + # drop everything up to the first =. if a = is included, bash assumes this to be + # an arg=value argument and the completion gets mangled (see #11208) + completion="${completion#*=}" + fi + + COMPREPLY+=("${completion}") done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}" 2>/dev/null) __ltrim_colon_completions "$cur" } From 5a6e28e1667386b6ce4310bb07ed196432d7a9ec Mon Sep 17 00:00:00 2001 From: a-kenji Date: Mon, 5 Aug 2024 01:03:21 +0200 Subject: [PATCH 187/284] docs: installable remove alternate expression flag (#11254) --- src/nix/nix.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nix/nix.md b/src/nix/nix.md index 56587d0b233..443757a0523 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -65,7 +65,7 @@ The following types of installable are supported by most commands: - [Nix file](#nix-file), optionally qualified by an attribute path - Specified with `--file`/`-f` - [Nix expression](#nix-expression), optionally qualified by an attribute path - - Specified with `--expr`/`-E` + - Specified with `--expr` For most commands, if no installable is specified, `.` is assumed. That is, Nix will operate on the default flake output attribute of the flake in the current directory. @@ -200,17 +200,17 @@ operate are determined as follows: … ``` - For `-e`/`--expr` and `-f`/`--file`, the derivation output is specified as part of the attribute path: + For `--expr` and `-f`/`--file`, the derivation output is specified as part of the attribute path: ```console $ nix build -f '' 'glibc^dev,static' - $ nix build --impure -E 'import { }' 'glibc^dev,static' + $ nix build --impure --expr 'import { }' 'glibc^dev,static' ``` This syntax is the same even if the actual attribute path is empty: ```console - $ nix build -E 'let pkgs = import { }; in pkgs.glibc' '^dev,static' + $ nix build --impure --expr 'let pkgs = import { }; in pkgs.glibc' '^dev,static' ``` * You can also specify that *all* outputs should be used using the From 2950f9e18af1bd57b566b8c0b4df71022edb3b80 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 5 Aug 2024 11:38:38 +0200 Subject: [PATCH 188/284] Add a VM test for S3BinaryCacheStore Fixes #11238. --- tests/nixos/default.nix | 2 + tests/nixos/nix-copy-closure.nix | 2 +- tests/nixos/s3-binary-cache-store.nix | 63 +++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 tests/nixos/s3-binary-cache-store.nix diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index c0c7b42fd9b..66174c928e5 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -146,4 +146,6 @@ in functional_root = runNixOSTestFor "x86_64-linux" ./functional/as-root.nix; user-sandboxing = runNixOSTestFor "x86_64-linux" ./user-sandboxing; + + s3-binary-cache-store = runNixOSTestFor "x86_64-linux" ./s3-binary-cache-store.nix; } diff --git a/tests/nixos/nix-copy-closure.nix b/tests/nixos/nix-copy-closure.nix index 66cbfb0338d..b9daa0a1f90 100644 --- a/tests/nixos/nix-copy-closure.nix +++ b/tests/nixos/nix-copy-closure.nix @@ -1,6 +1,6 @@ # Test ‘nix-copy-closure’. -{ lib, config, nixpkgs, hostPkgs, ... }: +{ lib, config, nixpkgs, ... }: let pkgs = config.nodes.client.nixpkgs.pkgs; diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix new file mode 100644 index 00000000000..0154579680e --- /dev/null +++ b/tests/nixos/s3-binary-cache-store.nix @@ -0,0 +1,63 @@ +{ lib, config, nixpkgs, ... }: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + pkgA = pkgs.cowsay; + + accessKey = "BKIKJAA5BMMU2RHO6IBB"; + secretKey = "V7f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12"; + env = "AWS_ACCESS_KEY_ID=${accessKey} AWS_SECRET_ACCESS_KEY=${secretKey}"; + + storeUrl = "s3://my-cache?endpoint=http://server:9000®ion=eu-west-1"; + +in { + name = "nix-copy-closure"; + + nodes = + { server = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + virtualisation.additionalPaths = [ pkgA ]; + environment.systemPackages = [ pkgs.minio-client ]; + nix.extraOptions = "experimental-features = nix-command"; + services.minio = { + enable = true; + region = "eu-west-1"; + rootCredentialsFile = pkgs.writeText "minio-credentials-full" '' + MINIO_ROOT_USER=${accessKey} + MINIO_ROOT_PASSWORD=${secretKey} + ''; + }; + networking.firewall.allowedTCPPorts = [ 9000 ]; + }; + + client = + { config, pkgs, ... }: + { virtualisation.writableStore = true; + nix.extraOptions = "experimental-features = nix-command"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + start_all() + + # Create a binary cache. + server.wait_for_unit("minio") + + server.succeed("mc config host add minio http://localhost:9000 ${accessKey} ${secretKey} --api s3v4") + server.succeed("mc mb minio/my-cache") + + server.succeed("${env} nix copy --to '${storeUrl}' ${pkgA}") + + # Copy a package from the binary cache. + client.fail("nix path-info ${pkgA}") + + client.succeed("${env} nix store info --store '${storeUrl}' >&2") + + client.succeed("${env} nix copy --no-check-sigs --from '${storeUrl}' ${pkgA}") + + client.succeed("nix path-info ${pkgA}") + ''; +} From 547e808a7528e9705fb2fec8014113a483b2006d Mon Sep 17 00:00:00 2001 From: Jeremy Kolb Date: Thu, 27 Jun 2024 15:35:55 -0400 Subject: [PATCH 189/284] nix flake show: add the description if it exists --- src/nix/flake.cc | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3f9f8f99b06..8eb88d708cc 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1243,25 +1243,30 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto showDerivation = [&]() { auto name = visitor.getAttr(state->sName)->getString(); + std::optional description; + if (auto aMeta = visitor.maybeGetAttr(state->sMeta)) { + if (auto aDescription = aMeta->maybeGetAttr(state->sDescription)) + description = aDescription->getString(); + } + if (json) { - std::optional description; - if (auto aMeta = visitor.maybeGetAttr(state->sMeta)) { - if (auto aDescription = aMeta->maybeGetAttr(state->sDescription)) - description = aDescription->getString(); - } j.emplace("type", "derivation"); j.emplace("name", name); if (description) j.emplace("description", *description); } else { - logger->cout("%s: %s '%s'", - headerPrefix, + auto type = attrPath.size() == 2 && attrPathS[0] == "devShell" ? "development environment" : attrPath.size() >= 2 && attrPathS[0] == "devShells" ? "development environment" : attrPath.size() == 3 && attrPathS[0] == "checks" ? "derivation" : attrPath.size() >= 1 && attrPathS[0] == "hydraJobs" ? "derivation" : - "package", - name); + "package"; + if (description) { + logger->cout("%s: %s '%s' - '%s'", headerPrefix, type, name, *description); + } + else { + logger->cout("%s: %s '%s'", headerPrefix, type, name); + } } }; From 07d0527c0c2154c0868e242fb1d0e8da34199eeb Mon Sep 17 00:00:00 2001 From: Jeremy Kolb Date: Mon, 8 Jul 2024 11:56:41 -0400 Subject: [PATCH 190/284] nix flake show: Only print up to the first new line if it exists. --- src/nix/flake.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 8eb88d708cc..9c6469a0866 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1262,7 +1262,11 @@ struct CmdFlakeShow : FlakeCommand, MixJSON attrPath.size() >= 1 && attrPathS[0] == "hydraJobs" ? "derivation" : "package"; if (description) { - logger->cout("%s: %s '%s' - '%s'", headerPrefix, type, name, *description); + // Handle new lines in descriptions. + auto index = description->find('\n'); + std::string_view sanitized_description(description->data(), index != std::string::npos ? index : description->size()); + + logger->cout("%s: %s '%s' - '%s'", headerPrefix, type, name, sanitized_description); } else { logger->cout("%s: %s '%s'", headerPrefix, type, name); From 59b6aafadb66fcc6b2b7a2a6a72680ef58f684c7 Mon Sep 17 00:00:00 2001 From: Jeremy Kolb Date: Mon, 8 Jul 2024 15:25:17 -0400 Subject: [PATCH 191/284] add tests --- tests/functional/flakes/show.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/functional/flakes/show.sh b/tests/functional/flakes/show.sh index 22e1f419376..2911790debb 100755 --- a/tests/functional/flakes/show.sh +++ b/tests/functional/flakes/show.sh @@ -87,3 +87,22 @@ assert show_output.legacyPackages.${builtins.currentSystem}.AAAAAASomeThingsFail assert show_output.legacyPackages.${builtins.currentSystem}.simple.name == "simple"; true ' + +cat >flake.nix< ./show-output.txt +test "$(awk -F '[:] ' '/aNoDescription/{print $NF}' ./show-output.txt)" = "package 'simple'" +test "$(awk -F '[:] ' '/bOneLineDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - 'one line'" +test "$(awk -F '[:] ' '/cMultiLineDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - 'line one'" From f22cf1fd3851ebac8a0a2040428ba9ce92a209c6 Mon Sep 17 00:00:00 2001 From: Jeremy Kolb Date: Mon, 8 Jul 2024 16:31:33 -0400 Subject: [PATCH 192/284] Handle long strings, embedded new lines and empty descriptions --- src/nix/flake.cc | 21 ++++++++++++++++----- tests/functional/flakes/show.sh | 8 +++++++- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 9c6469a0866..89a1326fd0e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1261,12 +1261,23 @@ struct CmdFlakeShow : FlakeCommand, MixJSON attrPath.size() == 3 && attrPathS[0] == "checks" ? "derivation" : attrPath.size() >= 1 && attrPathS[0] == "hydraJobs" ? "derivation" : "package"; - if (description) { - // Handle new lines in descriptions. - auto index = description->find('\n'); - std::string_view sanitized_description(description->data(), index != std::string::npos ? index : description->size()); + if (description && !description->empty()) { + // Trim the string and only display the first line of the description. + auto trimmed = nix::trim(*description); + auto newLinePos = trimmed.find('\n'); + auto length = newLinePos != std::string::npos ? newLinePos : trimmed.size(); + + // If the string is too long then resize add ellipses + std::string desc; + if (length > 80) { + trimmed.resize(80); + desc = trimmed.append("..."); + } + else { + desc = trimmed.substr(0, length); + } - logger->cout("%s: %s '%s' - '%s'", headerPrefix, type, name, sanitized_description); + logger->cout("%s: %s '%s' - '%s'", headerPrefix, type, name, desc); } else { logger->cout("%s: %s '%s'", headerPrefix, type, name); diff --git a/tests/functional/flakes/show.sh b/tests/functional/flakes/show.sh index 2911790debb..d60adb99f45 100755 --- a/tests/functional/flakes/show.sh +++ b/tests/functional/flakes/show.sh @@ -95,9 +95,13 @@ cat >flake.nix< ./show-output.txt test "$(awk -F '[:] ' '/aNoDescription/{print $NF}' ./show-output.txt)" = "package 'simple'" test "$(awk -F '[:] ' '/bOneLineDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - 'one line'" test "$(awk -F '[:] ' '/cMultiLineDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - 'line one'" +test "$(awk -F '[:] ' '/dLongDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - '01234567890123456789012345678901234567890123456789012345678901234567890123456789...'" +test "$(awk -F '[:] ' '/eEmptyDescription/{print $NF}' ./show-output.txt)" = "package 'simple'" From 930818bb1daf97d5751b67fc6399323b3557bb7a Mon Sep 17 00:00:00 2001 From: Jeremy Kolb Date: Thu, 18 Jul 2024 09:42:48 -0400 Subject: [PATCH 193/284] Account for total length of 80 --- src/nix/flake.cc | 4 ++-- tests/functional/flakes/show.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 89a1326fd0e..48bec08c1be 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1269,8 +1269,8 @@ struct CmdFlakeShow : FlakeCommand, MixJSON // If the string is too long then resize add ellipses std::string desc; - if (length > 80) { - trimmed.resize(80); + if (length > 77) { + trimmed.resize(77); desc = trimmed.append("..."); } else { diff --git a/tests/functional/flakes/show.sh b/tests/functional/flakes/show.sh index d60adb99f45..3d91613ee4a 100755 --- a/tests/functional/flakes/show.sh +++ b/tests/functional/flakes/show.sh @@ -110,5 +110,5 @@ nix flake show > ./show-output.txt test "$(awk -F '[:] ' '/aNoDescription/{print $NF}' ./show-output.txt)" = "package 'simple'" test "$(awk -F '[:] ' '/bOneLineDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - 'one line'" test "$(awk -F '[:] ' '/cMultiLineDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - 'line one'" -test "$(awk -F '[:] ' '/dLongDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - '01234567890123456789012345678901234567890123456789012345678901234567890123456789...'" +test "$(awk -F '[:] ' '/dLongDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - '01234567890123456789012345678901234567890123456789012345678901234567890123456...'" test "$(awk -F '[:] ' '/eEmptyDescription/{print $NF}' ./show-output.txt)" = "package 'simple'" From 9d2d4d11e6fad32d97cb3605614835a16c5b5cdf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Aug 2024 12:34:05 -0400 Subject: [PATCH 194/284] Split tests, organize more string functions The test split matches PR #8920, so the utility files and tests files are once again to 1-1. The string changes continues what was started in PR #11093. --- src/libexpr/eval-cache.cc | 2 + src/libutil/strings-inline.hh | 41 ++ src/libutil/strings.cc | 13 +- src/libutil/strings.hh | 28 ++ src/libutil/util.cc | 18 - src/libutil/util.hh | 44 +- tests/unit/libutil/file-system.cc | 258 +++++++++++ tests/unit/libutil/meson.build | 5 +- tests/unit/libutil/processes.cc | 17 + tests/unit/libutil/strings.cc | 151 +++++++ tests/unit/libutil/terminal.cc | 60 +++ tests/unit/libutil/tests.cc | 703 ------------------------------ tests/unit/libutil/util.cc | 385 ++++++++++++++++ 13 files changed, 959 insertions(+), 766 deletions(-) create mode 100644 tests/unit/libutil/file-system.cc create mode 100644 tests/unit/libutil/processes.cc create mode 100644 tests/unit/libutil/terminal.cc delete mode 100644 tests/unit/libutil/tests.cc create mode 100644 tests/unit/libutil/util.cc diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 26352187e72..3d77c029e2a 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -4,6 +4,8 @@ #include "eval.hh" #include "eval-inline.hh" #include "store-api.hh" +// Need specialization involving `SymbolStr` just in this one module. +#include "strings-inline.hh" namespace nix::eval_cache { diff --git a/src/libutil/strings-inline.hh b/src/libutil/strings-inline.hh index 10c1b19e688..d254d486dd4 100644 --- a/src/libutil/strings-inline.hh +++ b/src/libutil/strings-inline.hh @@ -4,6 +4,21 @@ namespace nix { +template +C tokenizeString(std::string_view s, std::string_view separators) +{ + C result; + auto pos = s.find_first_not_of(separators, 0); + while (pos != s.npos) { + auto end = s.find_first_of(separators, pos + 1); + if (end == s.npos) + end = s.size(); + result.insert(result.end(), std::string(s, pos, end - pos)); + pos = s.find_first_not_of(separators, end); + } + return result; +} + template std::string concatStringsSep(const std::string_view sep, const C & ss) { @@ -28,4 +43,30 @@ std::string concatStringsSep(const std::string_view sep, const C & ss) return s; } +template +std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss) +{ + size_t size = 0; + + // TODO? remove to make sure we don't rely on the empty item ignoring behavior, + // or just get rid of this function by understanding the remaining calls. + // for (auto & i : ss) { + // // Make sure we don't rely on the empty item ignoring behavior + // assert(!i.empty()); + // break; + // } + + // need a cast to string_view since this is also called with Symbols + for (const auto & s : ss) + size += sep.size() + std::string_view(s).size(); + std::string s; + s.reserve(size); + for (auto & i : ss) { + if (s.size() != 0) + s += sep; + s += i; + } + return s; +} + } // namespace nix diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc index 7ec618bf449..2bb7f8c0aed 100644 --- a/src/libutil/strings.cc +++ b/src/libutil/strings.cc @@ -1,12 +1,15 @@ #include #include "strings-inline.hh" -#include "util.hh" namespace nix { -template std::string concatStringsSep(std::string_view, const Strings &); -template std::string concatStringsSep(std::string_view, const StringSet &); +template std::list tokenizeString(std::string_view s, std::string_view separators); +template std::set tokenizeString(std::string_view s, std::string_view separators); +template std::vector tokenizeString(std::string_view s, std::string_view separators); + +template std::string concatStringsSep(std::string_view, const std::list &); +template std::string concatStringsSep(std::string_view, const std::set &); template std::string concatStringsSep(std::string_view, const std::vector &); typedef std::string_view strings_2[2]; @@ -16,4 +19,8 @@ template std::string concatStringsSep(std::string_view, const strings_3 &); typedef std::string_view strings_4[4]; template std::string concatStringsSep(std::string_view, const strings_4 &); +template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::list &); +template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); +template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); + } // namespace nix diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh index 3b112c4096d..6e991e490e1 100644 --- a/src/libutil/strings.hh +++ b/src/libutil/strings.hh @@ -8,6 +8,18 @@ namespace nix { +/** + * String tokenizer. + * + * See also `basicSplitString()`, which preserves empty strings between separators, as well as at the start and end. + */ +template +C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); + +extern template std::list tokenizeString(std::string_view s, std::string_view separators); +extern template std::set tokenizeString(std::string_view s, std::string_view separators); +extern template std::vector tokenizeString(std::string_view s, std::string_view separators); + /** * Concatenate the given strings with a separator between the elements. */ @@ -18,4 +30,20 @@ extern template std::string concatStringsSep(std::string_view, const std::list &); extern template std::string concatStringsSep(std::string_view, const std::vector &); +/** + * Ignore any empty strings at the start of the list, and then concatenate the + * given strings with a separator between the elements. + * + * @deprecated This function exists for historical reasons. You probably just + * want to use `concatStringsSep`. + */ +template +[[deprecated( + "Consider removing the empty string dropping behavior. If acceptable, use concatStringsSep instead.")]] std::string +dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss); + +extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::list &); +extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); +extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); + } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 698e181a1d1..460802be3d6 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -53,24 +53,6 @@ std::vector stringsToCharPtrs(const Strings & ss) ////////////////////////////////////////////////////////////////////// -template C tokenizeString(std::string_view s, std::string_view separators) -{ - C result; - auto pos = s.find_first_not_of(separators, 0); - while (pos != s.npos) { - auto end = s.find_first_of(separators, pos + 1); - if (end == s.npos) end = s.size(); - result.insert(result.end(), std::string(s, pos, end - pos)); - pos = s.find_first_not_of(separators, end); - } - return result; -} - -template Strings tokenizeString(std::string_view s, std::string_view separators); -template StringSet tokenizeString(std::string_view s, std::string_view separators); -template std::vector tokenizeString(std::string_view s, std::string_view separators); - - std::string chomp(std::string_view s) { size_t i = s.find_last_not_of(" \n\r\t"); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 877d1527945..25128a9009f 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -28,49 +28,11 @@ std::vector stringsToCharPtrs(const Strings & ss); MakeError(FormatError, Error); -/** - * String tokenizer. - */ -template C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); - - -/** - * Ignore any empty strings at the start of the list, and then concatenate the - * given strings with a separator between the elements. - * - * @deprecated This function exists for historical reasons. You probably just - * want to use `concatStringsSep`. - */ -template -[[deprecated("Consider removing the empty string dropping behavior. If acceptable, use concatStringsSep instead.")]] -std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss) -{ - size_t size = 0; - - // TODO? remove to make sure we don't rely on the empty item ignoring behavior, - // or just get rid of this function by understanding the remaining calls. - // for (auto & i : ss) { - // // Make sure we don't rely on the empty item ignoring behavior - // assert(!i.empty()); - // break; - // } - - // need a cast to string_view since this is also called with Symbols - for (const auto & s : ss) size += sep.size() + std::string_view(s).size(); - std::string s; - s.reserve(size); - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - -template -auto concatStrings(Parts && ... parts) +template +auto concatStrings(Parts &&... parts) -> std::enable_if_t<(... && std::is_convertible_v), std::string> { - std::string_view views[sizeof...(parts)] = { parts... }; + std::string_view views[sizeof...(parts)] = {parts...}; return concatStringsSep({}, views); } diff --git a/tests/unit/libutil/file-system.cc b/tests/unit/libutil/file-system.cc new file mode 100644 index 00000000000..cfddaae1cd3 --- /dev/null +++ b/tests/unit/libutil/file-system.cc @@ -0,0 +1,258 @@ +#include "util.hh" +#include "types.hh" +#include "file-system.hh" +#include "processes.hh" +#include "terminal.hh" +#include "strings.hh" + +#include +#include +#include + +#include + +#ifdef _WIN32 +# define FS_SEP "\\" +# define FS_ROOT "C:" FS_SEP // Need a mounted one, C drive is likely +#else +# define FS_SEP "/" +# define FS_ROOT FS_SEP +#endif + +#ifndef PATH_MAX +# define PATH_MAX 4096 +#endif + +namespace nix { + +/* ----------- tests for file-system.hh -------------------------------------*/ + +/* ---------------------------------------------------------------------------- + * absPath + * --------------------------------------------------------------------------*/ + +TEST(absPath, doesntChangeRoot) +{ + auto p = absPath(FS_ROOT); + + ASSERT_EQ(p, FS_ROOT); +} + +TEST(absPath, turnsEmptyPathIntoCWD) +{ + char cwd[PATH_MAX + 1]; + auto p = absPath(""); + + ASSERT_EQ(p, getcwd((char *) &cwd, PATH_MAX)); +} + +TEST(absPath, usesOptionalBasePathWhenGiven) +{ + char _cwd[PATH_MAX + 1]; + char * cwd = getcwd((char *) &_cwd, PATH_MAX); + + auto p = absPath("", cwd); + + ASSERT_EQ(p, cwd); +} + +TEST(absPath, isIdempotent) +{ + char _cwd[PATH_MAX + 1]; + char * cwd = getcwd((char *) &_cwd, PATH_MAX); + auto p1 = absPath(cwd); + auto p2 = absPath(p1); + + ASSERT_EQ(p1, p2); +} + +TEST(absPath, pathIsCanonicalised) +{ + auto path = FS_ROOT "some/path/with/trailing/dot/."; + auto p1 = absPath(path); + auto p2 = absPath(p1); + + ASSERT_EQ(p1, FS_ROOT "some" FS_SEP "path" FS_SEP "with" FS_SEP "trailing" FS_SEP "dot"); + ASSERT_EQ(p1, p2); +} + +/* ---------------------------------------------------------------------------- + * canonPath + * --------------------------------------------------------------------------*/ + +TEST(canonPath, removesTrailingSlashes) +{ + auto path = FS_ROOT "this/is/a/path//"; + auto p = canonPath(path); + + ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); +} + +TEST(canonPath, removesDots) +{ + auto path = FS_ROOT "this/./is/a/path/./"; + auto p = canonPath(path); + + ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); +} + +TEST(canonPath, removesDots2) +{ + auto path = FS_ROOT "this/a/../is/a////path/foo/.."; + auto p = canonPath(path); + + ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); +} + +TEST(canonPath, requiresAbsolutePath) +{ + ASSERT_ANY_THROW(canonPath(".")); + ASSERT_ANY_THROW(canonPath("..")); + ASSERT_ANY_THROW(canonPath("../")); + ASSERT_DEATH({ canonPath(""); }, "path != \"\""); +} + +/* ---------------------------------------------------------------------------- + * dirOf + * --------------------------------------------------------------------------*/ + +TEST(dirOf, returnsEmptyStringForRoot) +{ + auto p = dirOf("/"); + + ASSERT_EQ(p, "/"); +} + +TEST(dirOf, returnsFirstPathComponent) +{ + auto p1 = dirOf("/dir/"); + ASSERT_EQ(p1, "/dir"); + auto p2 = dirOf("/dir"); + ASSERT_EQ(p2, "/"); + auto p3 = dirOf("/dir/.."); + ASSERT_EQ(p3, "/dir"); + auto p4 = dirOf("/dir/../"); + ASSERT_EQ(p4, "/dir/.."); +} + +/* ---------------------------------------------------------------------------- + * baseNameOf + * --------------------------------------------------------------------------*/ + +TEST(baseNameOf, emptyPath) +{ + auto p1 = baseNameOf(""); + ASSERT_EQ(p1, ""); +} + +TEST(baseNameOf, pathOnRoot) +{ + auto p1 = baseNameOf("/dir"); + ASSERT_EQ(p1, "dir"); +} + +TEST(baseNameOf, relativePath) +{ + auto p1 = baseNameOf("dir/foo"); + ASSERT_EQ(p1, "foo"); +} + +TEST(baseNameOf, pathWithTrailingSlashRoot) +{ + auto p1 = baseNameOf("/"); + ASSERT_EQ(p1, ""); +} + +TEST(baseNameOf, trailingSlash) +{ + auto p1 = baseNameOf("/dir/"); + ASSERT_EQ(p1, "dir"); +} + +TEST(baseNameOf, trailingSlashes) +{ + auto p1 = baseNameOf("/dir//"); + ASSERT_EQ(p1, "dir"); +} + +TEST(baseNameOf, absoluteNothingSlashNothing) +{ + auto p1 = baseNameOf("//"); + ASSERT_EQ(p1, ""); +} + +/* ---------------------------------------------------------------------------- + * isInDir + * --------------------------------------------------------------------------*/ + +TEST(isInDir, trivialCase) +{ + auto p1 = isInDir("/foo/bar", "/foo"); + ASSERT_EQ(p1, true); +} + +TEST(isInDir, notInDir) +{ + auto p1 = isInDir("/zes/foo/bar", "/foo"); + ASSERT_EQ(p1, false); +} + +// XXX: hm, bug or feature? :) Looking at the implementation +// this might be problematic. +TEST(isInDir, emptyDir) +{ + auto p1 = isInDir("/zes/foo/bar", ""); + ASSERT_EQ(p1, true); +} + +/* ---------------------------------------------------------------------------- + * isDirOrInDir + * --------------------------------------------------------------------------*/ + +TEST(isDirOrInDir, trueForSameDirectory) +{ + ASSERT_EQ(isDirOrInDir("/nix", "/nix"), true); + ASSERT_EQ(isDirOrInDir("/", "/"), true); +} + +TEST(isDirOrInDir, trueForEmptyPaths) +{ + ASSERT_EQ(isDirOrInDir("", ""), true); +} + +TEST(isDirOrInDir, falseForDisjunctPaths) +{ + ASSERT_EQ(isDirOrInDir("/foo", "/bar"), false); +} + +TEST(isDirOrInDir, relativePaths) +{ + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), true); +} + +// XXX: while it is possible to use "." or ".." in the +// first argument this doesn't seem to work in the second. +TEST(isDirOrInDir, DISABLED_shouldWork) +{ + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), true); +} + +/* ---------------------------------------------------------------------------- + * pathExists + * --------------------------------------------------------------------------*/ + +TEST(pathExists, rootExists) +{ + ASSERT_TRUE(pathExists(FS_ROOT)); +} + +TEST(pathExists, cwdExists) +{ + ASSERT_TRUE(pathExists(".")); +} + +TEST(pathExists, bogusPathDoesNotExist) +{ + ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); +} +} diff --git a/tests/unit/libutil/meson.build b/tests/unit/libutil/meson.build index 7f024e6f25b..f4f2ae7f9bb 100644 --- a/tests/unit/libutil/meson.build +++ b/tests/unit/libutil/meson.build @@ -61,11 +61,14 @@ sources = files( 'lru-cache.cc', 'nix_api_util.cc', 'pool.cc', + 'processes.cc', 'references.cc', 'spawn.cc', + 'strings.cc', 'suggestions.cc', - 'tests.cc', + 'terminal.cc', 'url.cc', + 'util.cc', 'xml-writer.cc', ) diff --git a/tests/unit/libutil/processes.cc b/tests/unit/libutil/processes.cc new file mode 100644 index 00000000000..9033595e85c --- /dev/null +++ b/tests/unit/libutil/processes.cc @@ -0,0 +1,17 @@ +#include "processes.hh" + +#include + +namespace nix { + +/* ---------------------------------------------------------------------------- + * statusOk + * --------------------------------------------------------------------------*/ + +TEST(statusOk, zeroIsOk) +{ + ASSERT_EQ(statusOk(0), true); + ASSERT_EQ(statusOk(1), false); +} + +} // namespace nix diff --git a/tests/unit/libutil/strings.cc b/tests/unit/libutil/strings.cc index 47a20770e2e..0bd2fe0a517 100644 --- a/tests/unit/libutil/strings.cc +++ b/tests/unit/libutil/strings.cc @@ -80,4 +80,155 @@ TEST(concatStringsSep, buildSingleString) ASSERT_EQ(concatStringsSep(",", strings), "this"); } +/* ---------------------------------------------------------------------------- + * dropEmptyInitThenConcatStringsSep + * --------------------------------------------------------------------------*/ + +TEST(dropEmptyInitThenConcatStringsSep, empty) +{ + Strings strings; + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), ""); +} + +TEST(dropEmptyInitThenConcatStringsSep, buildCommaSeparatedString) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this,is,great"); +} + +TEST(dropEmptyInitThenConcatStringsSep, buildStringWithEmptySeparator) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep("", strings), "thisisgreat"); +} + +TEST(dropEmptyInitThenConcatStringsSep, buildSingleString) +{ + Strings strings; + strings.push_back("this"); + strings.push_back(""); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this,"); +} + +TEST(dropEmptyInitThenConcatStringsSep, emptyStrings) +{ + Strings strings; + strings.push_back(""); + strings.push_back(""); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), ""); +} + +/* ---------------------------------------------------------------------------- + * tokenizeString + * --------------------------------------------------------------------------*/ + +TEST(tokenizeString, empty) +{ + Strings expected = {}; + + ASSERT_EQ(tokenizeString(""), expected); +} + +TEST(tokenizeString, oneSep) +{ + Strings expected = {}; + + ASSERT_EQ(tokenizeString(" "), expected); +} + +TEST(tokenizeString, twoSep) +{ + Strings expected = {}; + + ASSERT_EQ(tokenizeString(" \n"), expected); +} + +TEST(tokenizeString, tokenizeSpacesWithDefaults) +{ + auto s = "foo bar baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsWithDefaults) +{ + auto s = "foo\tbar\tbaz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsSpacesWithDefaults) +{ + auto s = "foo\t bar\t baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsSpacesNewlineWithDefaults) +{ + auto s = "foo\t\n bar\t\n baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsSpacesNewlineRetWithDefaults) +{ + auto s = "foo\t\n\r bar\t\n\r baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); + + auto s2 = "foo \t\n\r bar \t\n\r baz"; + Strings expected2 = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s2), expected2); +} + +TEST(tokenizeString, tokenizeWithCustomSep) +{ + auto s = "foo\n,bar\n,baz\n"; + Strings expected = {"foo\n", "bar\n", "baz\n"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + +TEST(tokenizeString, tokenizeSepAtStart) +{ + auto s = ",foo,bar,baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + +TEST(tokenizeString, tokenizeSepAtEnd) +{ + auto s = "foo,bar,baz,"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + +TEST(tokenizeString, tokenizeSepEmpty) +{ + auto s = "foo,,baz"; + Strings expected = {"foo", "baz"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + } // namespace nix diff --git a/tests/unit/libutil/terminal.cc b/tests/unit/libutil/terminal.cc new file mode 100644 index 00000000000..cdeb9fd945f --- /dev/null +++ b/tests/unit/libutil/terminal.cc @@ -0,0 +1,60 @@ +#include "util.hh" +#include "types.hh" +#include "terminal.hh" +#include "strings.hh" + +#include +#include + +#include + +namespace nix { + +/* ---------------------------------------------------------------------------- + * filterANSIEscapes + * --------------------------------------------------------------------------*/ + +TEST(filterANSIEscapes, emptyString) +{ + auto s = ""; + auto expected = ""; + + ASSERT_EQ(filterANSIEscapes(s), expected); +} + +TEST(filterANSIEscapes, doesntChangePrintableChars) +{ + auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f"; + + ASSERT_EQ(filterANSIEscapes(s), s); +} + +TEST(filterANSIEscapes, filtersColorCodes) +{ + auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m"; + + ASSERT_EQ(filterANSIEscapes(s, true, 2), " A"); + ASSERT_EQ(filterANSIEscapes(s, true, 3), " A "); + ASSERT_EQ(filterANSIEscapes(s, true, 4), " A "); + ASSERT_EQ(filterANSIEscapes(s, true, 5), " A B"); + ASSERT_EQ(filterANSIEscapes(s, true, 8), " A B C"); +} + +TEST(filterANSIEscapes, expandsTabs) +{ + auto s = "foo\tbar\tbaz"; + + ASSERT_EQ(filterANSIEscapes(s, true), "foo bar baz"); +} + +TEST(filterANSIEscapes, utf8) +{ + ASSERT_EQ(filterANSIEscapes("foobar", true, 5), "fooba"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 6), "fóóbär"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 5), "fóóbä"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 3), "fóó"); + ASSERT_EQ(filterANSIEscapes("f€€bär", true, 4), "f€€b"); + ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b"); +} + +} // namespace nix diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc deleted file mode 100644 index 2b73d323b8d..00000000000 --- a/tests/unit/libutil/tests.cc +++ /dev/null @@ -1,703 +0,0 @@ -#include "util.hh" -#include "types.hh" -#include "file-system.hh" -#include "processes.hh" -#include "terminal.hh" - -#include -#include - -#include - -#ifdef _WIN32 -# define FS_SEP "\\" -# define FS_ROOT "C:" FS_SEP // Need a mounted one, C drive is likely -#else -# define FS_SEP "/" -# define FS_ROOT FS_SEP -#endif - -#ifndef PATH_MAX -# define PATH_MAX 4096 -#endif - -namespace nix { - -/* ----------- tests for util.hh ------------------------------------------------*/ - - /* ---------------------------------------------------------------------------- - * absPath - * --------------------------------------------------------------------------*/ - - TEST(absPath, doesntChangeRoot) { - auto p = absPath(FS_ROOT); - - ASSERT_EQ(p, FS_ROOT); - } - - - - - TEST(absPath, turnsEmptyPathIntoCWD) { - char cwd[PATH_MAX+1]; - auto p = absPath(""); - - ASSERT_EQ(p, getcwd((char*)&cwd, PATH_MAX)); - } - - TEST(absPath, usesOptionalBasePathWhenGiven) { - char _cwd[PATH_MAX+1]; - char* cwd = getcwd((char*)&_cwd, PATH_MAX); - - auto p = absPath("", cwd); - - ASSERT_EQ(p, cwd); - } - - TEST(absPath, isIdempotent) { - char _cwd[PATH_MAX+1]; - char* cwd = getcwd((char*)&_cwd, PATH_MAX); - auto p1 = absPath(cwd); - auto p2 = absPath(p1); - - ASSERT_EQ(p1, p2); - } - - - TEST(absPath, pathIsCanonicalised) { - auto path = FS_ROOT "some/path/with/trailing/dot/."; - auto p1 = absPath(path); - auto p2 = absPath(p1); - - ASSERT_EQ(p1, FS_ROOT "some" FS_SEP "path" FS_SEP "with" FS_SEP "trailing" FS_SEP "dot"); - ASSERT_EQ(p1, p2); - } - - /* ---------------------------------------------------------------------------- - * canonPath - * --------------------------------------------------------------------------*/ - - TEST(canonPath, removesTrailingSlashes) { - auto path = FS_ROOT "this/is/a/path//"; - auto p = canonPath(path); - - ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); - } - - TEST(canonPath, removesDots) { - auto path = FS_ROOT "this/./is/a/path/./"; - auto p = canonPath(path); - - ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); - } - - TEST(canonPath, removesDots2) { - auto path = FS_ROOT "this/a/../is/a////path/foo/.."; - auto p = canonPath(path); - - ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); - } - - TEST(canonPath, requiresAbsolutePath) { - ASSERT_ANY_THROW(canonPath(".")); - ASSERT_ANY_THROW(canonPath("..")); - ASSERT_ANY_THROW(canonPath("../")); - ASSERT_DEATH({ canonPath(""); }, "path != \"\""); - } - - /* ---------------------------------------------------------------------------- - * dirOf - * --------------------------------------------------------------------------*/ - - TEST(dirOf, returnsEmptyStringForRoot) { - auto p = dirOf("/"); - - ASSERT_EQ(p, "/"); - } - - TEST(dirOf, returnsFirstPathComponent) { - auto p1 = dirOf("/dir/"); - ASSERT_EQ(p1, "/dir"); - auto p2 = dirOf("/dir"); - ASSERT_EQ(p2, "/"); - auto p3 = dirOf("/dir/.."); - ASSERT_EQ(p3, "/dir"); - auto p4 = dirOf("/dir/../"); - ASSERT_EQ(p4, "/dir/.."); - } - - /* ---------------------------------------------------------------------------- - * baseNameOf - * --------------------------------------------------------------------------*/ - - TEST(baseNameOf, emptyPath) { - auto p1 = baseNameOf(""); - ASSERT_EQ(p1, ""); - } - - TEST(baseNameOf, pathOnRoot) { - auto p1 = baseNameOf("/dir"); - ASSERT_EQ(p1, "dir"); - } - - TEST(baseNameOf, relativePath) { - auto p1 = baseNameOf("dir/foo"); - ASSERT_EQ(p1, "foo"); - } - - TEST(baseNameOf, pathWithTrailingSlashRoot) { - auto p1 = baseNameOf("/"); - ASSERT_EQ(p1, ""); - } - - TEST(baseNameOf, trailingSlash) { - auto p1 = baseNameOf("/dir/"); - ASSERT_EQ(p1, "dir"); - } - - TEST(baseNameOf, trailingSlashes) { - auto p1 = baseNameOf("/dir//"); - ASSERT_EQ(p1, "dir"); - } - - TEST(baseNameOf, absoluteNothingSlashNothing) { - auto p1 = baseNameOf("//"); - ASSERT_EQ(p1, ""); - } - - /* ---------------------------------------------------------------------------- - * isInDir - * --------------------------------------------------------------------------*/ - - TEST(isInDir, trivialCase) { - auto p1 = isInDir("/foo/bar", "/foo"); - ASSERT_EQ(p1, true); - } - - TEST(isInDir, notInDir) { - auto p1 = isInDir("/zes/foo/bar", "/foo"); - ASSERT_EQ(p1, false); - } - - // XXX: hm, bug or feature? :) Looking at the implementation - // this might be problematic. - TEST(isInDir, emptyDir) { - auto p1 = isInDir("/zes/foo/bar", ""); - ASSERT_EQ(p1, true); - } - - /* ---------------------------------------------------------------------------- - * isDirOrInDir - * --------------------------------------------------------------------------*/ - - TEST(isDirOrInDir, trueForSameDirectory) { - ASSERT_EQ(isDirOrInDir("/nix", "/nix"), true); - ASSERT_EQ(isDirOrInDir("/", "/"), true); - } - - TEST(isDirOrInDir, trueForEmptyPaths) { - ASSERT_EQ(isDirOrInDir("", ""), true); - } - - TEST(isDirOrInDir, falseForDisjunctPaths) { - ASSERT_EQ(isDirOrInDir("/foo", "/bar"), false); - } - - TEST(isDirOrInDir, relativePaths) { - ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), true); - } - - // XXX: while it is possible to use "." or ".." in the - // first argument this doesn't seem to work in the second. - TEST(isDirOrInDir, DISABLED_shouldWork) { - ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), true); - - } - - /* ---------------------------------------------------------------------------- - * pathExists - * --------------------------------------------------------------------------*/ - - TEST(pathExists, rootExists) { - ASSERT_TRUE(pathExists(FS_ROOT)); - } - - TEST(pathExists, cwdExists) { - ASSERT_TRUE(pathExists(".")); - } - - TEST(pathExists, bogusPathDoesNotExist) { - ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); - } - - /* ---------------------------------------------------------------------------- - * dropEmptyInitThenConcatStringsSep - * --------------------------------------------------------------------------*/ - - TEST(dropEmptyInitThenConcatStringsSep, buildCommaSeparatedString) { - Strings strings; - strings.push_back("this"); - strings.push_back("is"); - strings.push_back("great"); - - ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this,is,great"); - } - - TEST(dropEmptyInitThenConcatStringsSep, buildStringWithEmptySeparator) { - Strings strings; - strings.push_back("this"); - strings.push_back("is"); - strings.push_back("great"); - - ASSERT_EQ(dropEmptyInitThenConcatStringsSep("", strings), "thisisgreat"); - } - - TEST(dropEmptyInitThenConcatStringsSep, buildSingleString) { - Strings strings; - strings.push_back("this"); - - ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this"); - } - - /* ---------------------------------------------------------------------------- - * hasPrefix - * --------------------------------------------------------------------------*/ - - TEST(hasPrefix, emptyStringHasNoPrefix) { - ASSERT_FALSE(hasPrefix("", "foo")); - } - - TEST(hasPrefix, emptyStringIsAlwaysPrefix) { - ASSERT_TRUE(hasPrefix("foo", "")); - ASSERT_TRUE(hasPrefix("jshjkfhsadf", "")); - } - - TEST(hasPrefix, trivialCase) { - ASSERT_TRUE(hasPrefix("foobar", "foo")); - } - - /* ---------------------------------------------------------------------------- - * hasSuffix - * --------------------------------------------------------------------------*/ - - TEST(hasSuffix, emptyStringHasNoSuffix) { - ASSERT_FALSE(hasSuffix("", "foo")); - } - - TEST(hasSuffix, trivialCase) { - ASSERT_TRUE(hasSuffix("foo", "foo")); - ASSERT_TRUE(hasSuffix("foobar", "bar")); - } - - /* ---------------------------------------------------------------------------- - * base64Encode - * --------------------------------------------------------------------------*/ - - TEST(base64Encode, emptyString) { - ASSERT_EQ(base64Encode(""), ""); - } - - TEST(base64Encode, encodesAString) { - ASSERT_EQ(base64Encode("quod erat demonstrandum"), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="); - } - - TEST(base64Encode, encodeAndDecode) { - auto s = "quod erat demonstrandum"; - auto encoded = base64Encode(s); - auto decoded = base64Decode(encoded); - - ASSERT_EQ(decoded, s); - } - - TEST(base64Encode, encodeAndDecodeNonPrintable) { - char s[256]; - std::iota(std::rbegin(s), std::rend(s), 0); - - auto encoded = base64Encode(s); - auto decoded = base64Decode(encoded); - - EXPECT_EQ(decoded.length(), 255); - ASSERT_EQ(decoded, s); - } - - /* ---------------------------------------------------------------------------- - * base64Decode - * --------------------------------------------------------------------------*/ - - TEST(base64Decode, emptyString) { - ASSERT_EQ(base64Decode(""), ""); - } - - TEST(base64Decode, decodeAString) { - ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum"); - } - - TEST(base64Decode, decodeThrowsOnInvalidChar) { - ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error); - } - - /* ---------------------------------------------------------------------------- - * getLine - * --------------------------------------------------------------------------*/ - - TEST(getLine, all) { - { - auto [line, rest] = getLine("foo\nbar\nxyzzy"); - ASSERT_EQ(line, "foo"); - ASSERT_EQ(rest, "bar\nxyzzy"); - } - - { - auto [line, rest] = getLine("foo\r\nbar\r\nxyzzy"); - ASSERT_EQ(line, "foo"); - ASSERT_EQ(rest, "bar\r\nxyzzy"); - } - - { - auto [line, rest] = getLine("foo\n"); - ASSERT_EQ(line, "foo"); - ASSERT_EQ(rest, ""); - } - - { - auto [line, rest] = getLine("foo"); - ASSERT_EQ(line, "foo"); - ASSERT_EQ(rest, ""); - } - - { - auto [line, rest] = getLine(""); - ASSERT_EQ(line, ""); - ASSERT_EQ(rest, ""); - } - } - - /* ---------------------------------------------------------------------------- - * toLower - * --------------------------------------------------------------------------*/ - - TEST(toLower, emptyString) { - ASSERT_EQ(toLower(""), ""); - } - - TEST(toLower, nonLetters) { - auto s = "!@(*$#)(@#=\\234_"; - ASSERT_EQ(toLower(s), s); - } - - // std::tolower() doesn't handle unicode characters. In the context of - // store paths this isn't relevant but doesn't hurt to record this behavior - // here. - TEST(toLower, umlauts) { - auto s = "ÄÖÜ"; - ASSERT_EQ(toLower(s), "ÄÖÜ"); - } - - /* ---------------------------------------------------------------------------- - * string2Float - * --------------------------------------------------------------------------*/ - - TEST(string2Float, emptyString) { - ASSERT_EQ(string2Float(""), std::nullopt); - } - - TEST(string2Float, trivialConversions) { - ASSERT_EQ(string2Float("1.0"), 1.0); - - ASSERT_EQ(string2Float("0.0"), 0.0); - - ASSERT_EQ(string2Float("-100.25"), -100.25); - } - - /* ---------------------------------------------------------------------------- - * string2Int - * --------------------------------------------------------------------------*/ - - TEST(string2Int, emptyString) { - ASSERT_EQ(string2Int(""), std::nullopt); - } - - TEST(string2Int, trivialConversions) { - ASSERT_EQ(string2Int("1"), 1); - - ASSERT_EQ(string2Int("0"), 0); - - ASSERT_EQ(string2Int("-100"), -100); - } - - /* ---------------------------------------------------------------------------- - * renderSize - * --------------------------------------------------------------------------*/ - - TEST(renderSize, misc) { - ASSERT_EQ(renderSize(0, true), " 0.0 KiB"); - ASSERT_EQ(renderSize(100, true), " 0.1 KiB"); - ASSERT_EQ(renderSize(100), "0.1 KiB"); - ASSERT_EQ(renderSize(972, true), " 0.9 KiB"); - ASSERT_EQ(renderSize(973, true), " 1.0 KiB"); // FIXME: should round down - ASSERT_EQ(renderSize(1024, true), " 1.0 KiB"); - ASSERT_EQ(renderSize(1024 * 1024, true), "1024.0 KiB"); - ASSERT_EQ(renderSize(1100 * 1024, true), " 1.1 MiB"); - ASSERT_EQ(renderSize(2ULL * 1024 * 1024 * 1024, true), " 2.0 GiB"); - ASSERT_EQ(renderSize(2100ULL * 1024 * 1024 * 1024, true), " 2.1 TiB"); - } - -#ifndef _WIN32 // TODO re-enable on Windows, once we can start processes - /* ---------------------------------------------------------------------------- - * statusOk - * --------------------------------------------------------------------------*/ - - TEST(statusOk, zeroIsOk) { - ASSERT_EQ(statusOk(0), true); - ASSERT_EQ(statusOk(1), false); - } -#endif - - - /* ---------------------------------------------------------------------------- - * rewriteStrings - * --------------------------------------------------------------------------*/ - - TEST(rewriteStrings, emptyString) { - StringMap rewrites; - rewrites["this"] = "that"; - - ASSERT_EQ(rewriteStrings("", rewrites), ""); - } - - TEST(rewriteStrings, emptyRewrites) { - StringMap rewrites; - - ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); - } - - TEST(rewriteStrings, successfulRewrite) { - StringMap rewrites; - rewrites["this"] = "that"; - - ASSERT_EQ(rewriteStrings("this and that", rewrites), "that and that"); - } - - TEST(rewriteStrings, doesntOccur) { - StringMap rewrites; - rewrites["foo"] = "bar"; - - ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); - } - - /* ---------------------------------------------------------------------------- - * replaceStrings - * --------------------------------------------------------------------------*/ - - TEST(replaceStrings, emptyString) { - ASSERT_EQ(replaceStrings("", "this", "that"), ""); - ASSERT_EQ(replaceStrings("this and that", "", ""), "this and that"); - } - - TEST(replaceStrings, successfulReplace) { - ASSERT_EQ(replaceStrings("this and that", "this", "that"), "that and that"); - } - - TEST(replaceStrings, doesntOccur) { - ASSERT_EQ(replaceStrings("this and that", "foo", "bar"), "this and that"); - } - - /* ---------------------------------------------------------------------------- - * trim - * --------------------------------------------------------------------------*/ - - TEST(trim, emptyString) { - ASSERT_EQ(trim(""), ""); - } - - TEST(trim, removesWhitespace) { - ASSERT_EQ(trim("foo"), "foo"); - ASSERT_EQ(trim(" foo "), "foo"); - ASSERT_EQ(trim(" foo bar baz"), "foo bar baz"); - ASSERT_EQ(trim(" \t foo bar baz\n"), "foo bar baz"); - } - - /* ---------------------------------------------------------------------------- - * chomp - * --------------------------------------------------------------------------*/ - - TEST(chomp, emptyString) { - ASSERT_EQ(chomp(""), ""); - } - - TEST(chomp, removesWhitespace) { - ASSERT_EQ(chomp("foo"), "foo"); - ASSERT_EQ(chomp("foo "), "foo"); - ASSERT_EQ(chomp(" foo "), " foo"); - ASSERT_EQ(chomp(" foo bar baz "), " foo bar baz"); - ASSERT_EQ(chomp("\t foo bar baz\n"), "\t foo bar baz"); - } - - /* ---------------------------------------------------------------------------- - * quoteStrings - * --------------------------------------------------------------------------*/ - - TEST(quoteStrings, empty) { - Strings s = { }; - Strings expected = { }; - - ASSERT_EQ(quoteStrings(s), expected); - } - - TEST(quoteStrings, emptyStrings) { - Strings s = { "", "", "" }; - Strings expected = { "''", "''", "''" }; - ASSERT_EQ(quoteStrings(s), expected); - - } - - TEST(quoteStrings, trivialQuote) { - Strings s = { "foo", "bar", "baz" }; - Strings expected = { "'foo'", "'bar'", "'baz'" }; - - ASSERT_EQ(quoteStrings(s), expected); - } - - TEST(quoteStrings, quotedStrings) { - Strings s = { "'foo'", "'bar'", "'baz'" }; - Strings expected = { "''foo''", "''bar''", "''baz''" }; - - ASSERT_EQ(quoteStrings(s), expected); - } - - /* ---------------------------------------------------------------------------- - * tokenizeString - * --------------------------------------------------------------------------*/ - - TEST(tokenizeString, empty) { - Strings expected = { }; - - ASSERT_EQ(tokenizeString(""), expected); - } - - TEST(tokenizeString, tokenizeSpacesWithDefaults) { - auto s = "foo bar baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsWithDefaults) { - auto s = "foo\tbar\tbaz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsSpacesWithDefaults) { - auto s = "foo\t bar\t baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsSpacesNewlineWithDefaults) { - auto s = "foo\t\n bar\t\n baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsSpacesNewlineRetWithDefaults) { - auto s = "foo\t\n\r bar\t\n\r baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - - auto s2 = "foo \t\n\r bar \t\n\r baz"; - Strings expected2 = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s2), expected2); - } - - TEST(tokenizeString, tokenizeWithCustomSep) { - auto s = "foo\n,bar\n,baz\n"; - Strings expected = { "foo\n", "bar\n", "baz\n" }; - - ASSERT_EQ(tokenizeString(s, ","), expected); - } - - /* ---------------------------------------------------------------------------- - * get - * --------------------------------------------------------------------------*/ - - TEST(get, emptyContainer) { - StringMap s = { }; - auto expected = nullptr; - - ASSERT_EQ(get(s, "one"), expected); - } - - TEST(get, getFromContainer) { - StringMap s; - s["one"] = "yi"; - s["two"] = "er"; - auto expected = "yi"; - - ASSERT_EQ(*get(s, "one"), expected); - } - - TEST(getOr, emptyContainer) { - StringMap s = { }; - auto expected = "yi"; - - ASSERT_EQ(getOr(s, "one", "yi"), expected); - } - - TEST(getOr, getFromContainer) { - StringMap s; - s["one"] = "yi"; - s["two"] = "er"; - auto expected = "yi"; - - ASSERT_EQ(getOr(s, "one", "nope"), expected); - } - - /* ---------------------------------------------------------------------------- - * filterANSIEscapes - * --------------------------------------------------------------------------*/ - - TEST(filterANSIEscapes, emptyString) { - auto s = ""; - auto expected = ""; - - ASSERT_EQ(filterANSIEscapes(s), expected); - } - - TEST(filterANSIEscapes, doesntChangePrintableChars) { - auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f"; - - ASSERT_EQ(filterANSIEscapes(s), s); - } - - TEST(filterANSIEscapes, filtersColorCodes) { - auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m"; - - ASSERT_EQ(filterANSIEscapes(s, true, 2), " A" ); - ASSERT_EQ(filterANSIEscapes(s, true, 3), " A " ); - ASSERT_EQ(filterANSIEscapes(s, true, 4), " A " ); - ASSERT_EQ(filterANSIEscapes(s, true, 5), " A B" ); - ASSERT_EQ(filterANSIEscapes(s, true, 8), " A B C" ); - } - - TEST(filterANSIEscapes, expandsTabs) { - auto s = "foo\tbar\tbaz"; - - ASSERT_EQ(filterANSIEscapes(s, true), "foo bar baz" ); - } - - TEST(filterANSIEscapes, utf8) { - ASSERT_EQ(filterANSIEscapes("foobar", true, 5), "fooba"); - ASSERT_EQ(filterANSIEscapes("fóóbär", true, 6), "fóóbär"); - ASSERT_EQ(filterANSIEscapes("fóóbär", true, 5), "fóóbä"); - ASSERT_EQ(filterANSIEscapes("fóóbär", true, 3), "fóó"); - ASSERT_EQ(filterANSIEscapes("f€€bär", true, 4), "f€€b"); - ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b"); - } - -} diff --git a/tests/unit/libutil/util.cc b/tests/unit/libutil/util.cc new file mode 100644 index 00000000000..a3f7c720a5c --- /dev/null +++ b/tests/unit/libutil/util.cc @@ -0,0 +1,385 @@ +#include "util.hh" +#include "types.hh" +#include "file-system.hh" +#include "terminal.hh" +#include "strings.hh" + +#include +#include + +#include + +namespace nix { + +/* ----------- tests for util.hh --------------------------------------------*/ + +/* ---------------------------------------------------------------------------- + * hasPrefix + * --------------------------------------------------------------------------*/ + +TEST(hasPrefix, emptyStringHasNoPrefix) +{ + ASSERT_FALSE(hasPrefix("", "foo")); +} + +TEST(hasPrefix, emptyStringIsAlwaysPrefix) +{ + ASSERT_TRUE(hasPrefix("foo", "")); + ASSERT_TRUE(hasPrefix("jshjkfhsadf", "")); +} + +TEST(hasPrefix, trivialCase) +{ + ASSERT_TRUE(hasPrefix("foobar", "foo")); +} + +/* ---------------------------------------------------------------------------- + * hasSuffix + * --------------------------------------------------------------------------*/ + +TEST(hasSuffix, emptyStringHasNoSuffix) +{ + ASSERT_FALSE(hasSuffix("", "foo")); +} + +TEST(hasSuffix, trivialCase) +{ + ASSERT_TRUE(hasSuffix("foo", "foo")); + ASSERT_TRUE(hasSuffix("foobar", "bar")); +} + +/* ---------------------------------------------------------------------------- + * base64Encode + * --------------------------------------------------------------------------*/ + +TEST(base64Encode, emptyString) +{ + ASSERT_EQ(base64Encode(""), ""); +} + +TEST(base64Encode, encodesAString) +{ + ASSERT_EQ(base64Encode("quod erat demonstrandum"), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="); +} + +TEST(base64Encode, encodeAndDecode) +{ + auto s = "quod erat demonstrandum"; + auto encoded = base64Encode(s); + auto decoded = base64Decode(encoded); + + ASSERT_EQ(decoded, s); +} + +TEST(base64Encode, encodeAndDecodeNonPrintable) +{ + char s[256]; + std::iota(std::rbegin(s), std::rend(s), 0); + + auto encoded = base64Encode(s); + auto decoded = base64Decode(encoded); + + EXPECT_EQ(decoded.length(), 255); + ASSERT_EQ(decoded, s); +} + +/* ---------------------------------------------------------------------------- + * base64Decode + * --------------------------------------------------------------------------*/ + +TEST(base64Decode, emptyString) +{ + ASSERT_EQ(base64Decode(""), ""); +} + +TEST(base64Decode, decodeAString) +{ + ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum"); +} + +TEST(base64Decode, decodeThrowsOnInvalidChar) +{ + ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error); +} + +/* ---------------------------------------------------------------------------- + * getLine + * --------------------------------------------------------------------------*/ + +TEST(getLine, all) +{ + { + auto [line, rest] = getLine("foo\nbar\nxyzzy"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, "bar\nxyzzy"); + } + + { + auto [line, rest] = getLine("foo\r\nbar\r\nxyzzy"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, "bar\r\nxyzzy"); + } + + { + auto [line, rest] = getLine("foo\n"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, ""); + } + + { + auto [line, rest] = getLine("foo"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, ""); + } + + { + auto [line, rest] = getLine(""); + ASSERT_EQ(line, ""); + ASSERT_EQ(rest, ""); + } +} + +/* ---------------------------------------------------------------------------- + * toLower + * --------------------------------------------------------------------------*/ + +TEST(toLower, emptyString) +{ + ASSERT_EQ(toLower(""), ""); +} + +TEST(toLower, nonLetters) +{ + auto s = "!@(*$#)(@#=\\234_"; + ASSERT_EQ(toLower(s), s); +} + +// std::tolower() doesn't handle unicode characters. In the context of +// store paths this isn't relevant but doesn't hurt to record this behavior +// here. +TEST(toLower, umlauts) +{ + auto s = "ÄÖÜ"; + ASSERT_EQ(toLower(s), "ÄÖÜ"); +} + +/* ---------------------------------------------------------------------------- + * string2Float + * --------------------------------------------------------------------------*/ + +TEST(string2Float, emptyString) +{ + ASSERT_EQ(string2Float(""), std::nullopt); +} + +TEST(string2Float, trivialConversions) +{ + ASSERT_EQ(string2Float("1.0"), 1.0); + + ASSERT_EQ(string2Float("0.0"), 0.0); + + ASSERT_EQ(string2Float("-100.25"), -100.25); +} + +/* ---------------------------------------------------------------------------- + * string2Int + * --------------------------------------------------------------------------*/ + +TEST(string2Int, emptyString) +{ + ASSERT_EQ(string2Int(""), std::nullopt); +} + +TEST(string2Int, trivialConversions) +{ + ASSERT_EQ(string2Int("1"), 1); + + ASSERT_EQ(string2Int("0"), 0); + + ASSERT_EQ(string2Int("-100"), -100); +} + +/* ---------------------------------------------------------------------------- + * renderSize + * --------------------------------------------------------------------------*/ + +TEST(renderSize, misc) +{ + ASSERT_EQ(renderSize(0, true), " 0.0 KiB"); + ASSERT_EQ(renderSize(100, true), " 0.1 KiB"); + ASSERT_EQ(renderSize(100), "0.1 KiB"); + ASSERT_EQ(renderSize(972, true), " 0.9 KiB"); + ASSERT_EQ(renderSize(973, true), " 1.0 KiB"); // FIXME: should round down + ASSERT_EQ(renderSize(1024, true), " 1.0 KiB"); + ASSERT_EQ(renderSize(1024 * 1024, true), "1024.0 KiB"); + ASSERT_EQ(renderSize(1100 * 1024, true), " 1.1 MiB"); + ASSERT_EQ(renderSize(2ULL * 1024 * 1024 * 1024, true), " 2.0 GiB"); + ASSERT_EQ(renderSize(2100ULL * 1024 * 1024 * 1024, true), " 2.1 TiB"); +} + +/* ---------------------------------------------------------------------------- + * rewriteStrings + * --------------------------------------------------------------------------*/ + +TEST(rewriteStrings, emptyString) +{ + StringMap rewrites; + rewrites["this"] = "that"; + + ASSERT_EQ(rewriteStrings("", rewrites), ""); +} + +TEST(rewriteStrings, emptyRewrites) +{ + StringMap rewrites; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); +} + +TEST(rewriteStrings, successfulRewrite) +{ + StringMap rewrites; + rewrites["this"] = "that"; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "that and that"); +} + +TEST(rewriteStrings, doesntOccur) +{ + StringMap rewrites; + rewrites["foo"] = "bar"; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); +} + +/* ---------------------------------------------------------------------------- + * replaceStrings + * --------------------------------------------------------------------------*/ + +TEST(replaceStrings, emptyString) +{ + ASSERT_EQ(replaceStrings("", "this", "that"), ""); + ASSERT_EQ(replaceStrings("this and that", "", ""), "this and that"); +} + +TEST(replaceStrings, successfulReplace) +{ + ASSERT_EQ(replaceStrings("this and that", "this", "that"), "that and that"); +} + +TEST(replaceStrings, doesntOccur) +{ + ASSERT_EQ(replaceStrings("this and that", "foo", "bar"), "this and that"); +} + +/* ---------------------------------------------------------------------------- + * trim + * --------------------------------------------------------------------------*/ + +TEST(trim, emptyString) +{ + ASSERT_EQ(trim(""), ""); +} + +TEST(trim, removesWhitespace) +{ + ASSERT_EQ(trim("foo"), "foo"); + ASSERT_EQ(trim(" foo "), "foo"); + ASSERT_EQ(trim(" foo bar baz"), "foo bar baz"); + ASSERT_EQ(trim(" \t foo bar baz\n"), "foo bar baz"); +} + +/* ---------------------------------------------------------------------------- + * chomp + * --------------------------------------------------------------------------*/ + +TEST(chomp, emptyString) +{ + ASSERT_EQ(chomp(""), ""); +} + +TEST(chomp, removesWhitespace) +{ + ASSERT_EQ(chomp("foo"), "foo"); + ASSERT_EQ(chomp("foo "), "foo"); + ASSERT_EQ(chomp(" foo "), " foo"); + ASSERT_EQ(chomp(" foo bar baz "), " foo bar baz"); + ASSERT_EQ(chomp("\t foo bar baz\n"), "\t foo bar baz"); +} + +/* ---------------------------------------------------------------------------- + * quoteStrings + * --------------------------------------------------------------------------*/ + +TEST(quoteStrings, empty) +{ + Strings s = {}; + Strings expected = {}; + + ASSERT_EQ(quoteStrings(s), expected); +} + +TEST(quoteStrings, emptyStrings) +{ + Strings s = {"", "", ""}; + Strings expected = {"''", "''", "''"}; + ASSERT_EQ(quoteStrings(s), expected); +} + +TEST(quoteStrings, trivialQuote) +{ + Strings s = {"foo", "bar", "baz"}; + Strings expected = {"'foo'", "'bar'", "'baz'"}; + + ASSERT_EQ(quoteStrings(s), expected); +} + +TEST(quoteStrings, quotedStrings) +{ + Strings s = {"'foo'", "'bar'", "'baz'"}; + Strings expected = {"''foo''", "''bar''", "''baz''"}; + + ASSERT_EQ(quoteStrings(s), expected); +} + +/* ---------------------------------------------------------------------------- + * get + * --------------------------------------------------------------------------*/ + +TEST(get, emptyContainer) +{ + StringMap s = {}; + auto expected = nullptr; + + ASSERT_EQ(get(s, "one"), expected); +} + +TEST(get, getFromContainer) +{ + StringMap s; + s["one"] = "yi"; + s["two"] = "er"; + auto expected = "yi"; + + ASSERT_EQ(*get(s, "one"), expected); +} + +TEST(getOr, emptyContainer) +{ + StringMap s = {}; + auto expected = "yi"; + + ASSERT_EQ(getOr(s, "one", "yi"), expected); +} + +TEST(getOr, getFromContainer) +{ + StringMap s; + s["one"] = "yi"; + s["two"] = "er"; + auto expected = "yi"; + + ASSERT_EQ(getOr(s, "one", "nope"), expected); +} + +} // namespace nix From 0a00bd07b2421acfb21751a718292affa8c6e837 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 5 Aug 2024 18:56:02 +0200 Subject: [PATCH 195/284] PathSubstitutionGoal: Fix spurious "failed" count in the progress bar It is not an error if queryPathInfo() indicates that a path does not exist in the substituter. Fixes #11198. This was broken in 846869da0ed0580beb7f827b303fef9a8386de37. --- src/libstore/build/substitution-goal.cc | 8 +++++--- src/libstore/build/substitution-goal.hh | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 7deeb47487d..0152f180828 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -145,8 +145,10 @@ Goal::Co PathSubstitutionGoal::init() /* None left. Terminate this goal and let someone else deal with it. */ - worker.failedSubstitutions++; - worker.updateProgress(); + if (substituterFailed) { + worker.failedSubstitutions++; + worker.updateProgress(); + } /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a @@ -158,7 +160,7 @@ Goal::Co PathSubstitutionGoal::init() } -Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool& substituterFailed) +Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool & substituterFailed) { trace("all references realised"); diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index c1de45379f1..f2cf797e5d2 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -66,7 +66,7 @@ public: */ Co init() override; Co gotInfo(); - Co tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool& substituterFailed); + Co tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool & substituterFailed); Co finished(); /** From 1c5f1de43f3497e47d638bb04fdf0a033de2036d Mon Sep 17 00:00:00 2001 From: Jeremy Kolb Date: Mon, 5 Aug 2024 14:15:14 -0400 Subject: [PATCH 196/284] copy string using filterANSIEscapes and enforce the max length --- src/nix/flake.cc | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 48bec08c1be..a5c6ff876d6 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -17,6 +17,7 @@ #include "eval-cache.hh" #include "markdown.hh" #include "users.hh" +#include "terminal.hh" #include #include @@ -1263,18 +1264,16 @@ struct CmdFlakeShow : FlakeCommand, MixJSON "package"; if (description && !description->empty()) { // Trim the string and only display the first line of the description. + const size_t maxLength = 77; auto trimmed = nix::trim(*description); auto newLinePos = trimmed.find('\n'); - auto length = newLinePos != std::string::npos ? newLinePos : trimmed.size(); + auto length = newLinePos != std::string::npos ? newLinePos : trimmed.length(); - // If the string is too long then resize add ellipses - std::string desc; - if (length > 77) { - trimmed.resize(77); - desc = trimmed.append("..."); - } - else { - desc = trimmed.substr(0, length); + // Resize/sanitize the string and if it's too long add ellipses + std::string desc = filterANSIEscapes(trimmed, false, length); + if (desc.length() > maxLength) { + desc.resize(maxLength); + desc = desc.append("..."); } logger->cout("%s: %s '%s' - '%s'", headerPrefix, type, name, desc); From 9bf6684b08368f850f6c451ec7313da0ae88738e Mon Sep 17 00:00:00 2001 From: Jeremy Kolb Date: Tue, 6 Aug 2024 09:39:42 -0400 Subject: [PATCH 197/284] Use window size --- src/nix/flake.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index a5c6ff876d6..8ef19e3eaf1 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1263,8 +1263,11 @@ struct CmdFlakeShow : FlakeCommand, MixJSON attrPath.size() >= 1 && attrPathS[0] == "hydraJobs" ? "derivation" : "package"; if (description && !description->empty()) { + // Maximum length to print + size_t maxLength = getWindowSize().second; + if (maxLength == 0) + maxLength = 77; // Trim the string and only display the first line of the description. - const size_t maxLength = 77; auto trimmed = nix::trim(*description); auto newLinePos = trimmed.find('\n'); auto length = newLinePos != std::string::npos ? newLinePos : trimmed.length(); From 6c861b9c51b6dece17dbd94e9016d7984159c9a8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Aug 2024 12:05:29 -0400 Subject: [PATCH 198/284] Factor out `lookupExecutable` and other PATH improvments This ended up motivating a good deal of other infra improvements in order to get Windows right: - `OsString` to complement `std::filesystem::path` - env var code for working with the underlying `OsString`s - Rename `PATHNG_LITERAL` to `OS_STR` - `NativePathTrait` renamed to `OsPathTrait`, given a character template parameter until #9205 is complete. Split `tests.cc` matching split of `util.{cc,hh}` last year. Co-authored-by: Robert Hensing --- .clang-format | 1 + maintainers/flake-module.nix | 3 - src/libutil/environment-variables.cc | 11 +- src/libutil/environment-variables.hh | 11 ++ src/libutil/executable-path.cc | 79 +++++++++++++ src/libutil/executable-path.hh | 64 +++++++++++ src/libutil/file-path-impl.hh | 9 +- src/libutil/file-path.hh | 25 +--- src/libutil/file-system.cc | 23 +++- src/libutil/file-system.hh | 6 + src/libutil/meson.build | 3 + src/libutil/os-string.hh | 43 +++++++ src/libutil/strings-inline.hh | 46 +++++++- src/libutil/strings.cc | 9 ++ src/libutil/strings.hh | 20 ++++ src/libutil/unix/environment-variables.cc | 10 ++ src/libutil/unix/file-path.cc | 10 -- src/libutil/unix/meson.build | 1 + src/libutil/unix/os-string.cc | 21 ++++ src/libutil/util.cc | 1 + src/libutil/windows/environment-variables.cc | 30 ++++- src/libutil/windows/file-path.cc | 12 -- src/libutil/windows/meson.build | 1 + src/libutil/windows/os-string.cc | 24 ++++ src/nix/config-check.cc | 36 ++++-- src/nix/develop.cc | 6 +- src/nix/env.cc | 9 +- src/nix/search.cc | 1 + src/nix/upgrade-nix.cc | 19 ++-- tests/unit/libutil/executable-path.cc | 64 +++++++++++ tests/unit/libutil/meson.build | 1 + tests/unit/libutil/strings.cc | 114 +++++++++++++++++++ 32 files changed, 616 insertions(+), 97 deletions(-) create mode 100644 src/libutil/executable-path.cc create mode 100644 src/libutil/executable-path.hh create mode 100644 src/libutil/os-string.hh create mode 100644 src/libutil/unix/os-string.cc create mode 100644 src/libutil/windows/os-string.cc create mode 100644 tests/unit/libutil/executable-path.cc diff --git a/.clang-format b/.clang-format index 3067583e106..4f191fc18b5 100644 --- a/.clang-format +++ b/.clang-format @@ -31,3 +31,4 @@ AlwaysBreakBeforeMultilineStrings: true IndentPPDirectives: AfterHash PPIndentWidth: 2 BinPackArguments: false +BreakBeforeTernaryOperators: true diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index be91df536ad..c3eaf671c1a 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -275,7 +275,6 @@ ''^src/libutil/current-process\.hh$'' ''^src/libutil/english\.cc$'' ''^src/libutil/english\.hh$'' - ''^src/libutil/environment-variables\.cc$'' ''^src/libutil/error\.cc$'' ''^src/libutil/error\.hh$'' ''^src/libutil/exit\.hh$'' @@ -357,7 +356,6 @@ ''^src/libutil/util\.cc$'' ''^src/libutil/util\.hh$'' ''^src/libutil/variant-wrapper\.hh$'' - ''^src/libutil/windows/environment-variables\.cc$'' ''^src/libutil/windows/file-descriptor\.cc$'' ''^src/libutil/windows/file-path\.cc$'' ''^src/libutil/windows/processes\.cc$'' @@ -485,7 +483,6 @@ ''^tests/unit/libutil/pool\.cc'' ''^tests/unit/libutil/references\.cc'' ''^tests/unit/libutil/suggestions\.cc'' - ''^tests/unit/libutil/tests\.cc'' ''^tests/unit/libutil/url\.cc'' ''^tests/unit/libutil/xml-writer\.cc'' ]; diff --git a/src/libutil/environment-variables.cc b/src/libutil/environment-variables.cc index d43197aa087..5947cf742ac 100644 --- a/src/libutil/environment-variables.cc +++ b/src/libutil/environment-variables.cc @@ -1,20 +1,23 @@ #include "util.hh" #include "environment-variables.hh" -extern char * * environ __attribute__((weak)); +extern char ** environ __attribute__((weak)); namespace nix { std::optional getEnv(const std::string & key) { char * value = getenv(key.c_str()); - if (!value) return {}; + if (!value) + return {}; return std::string(value); } -std::optional getEnvNonEmpty(const std::string & key) { +std::optional getEnvNonEmpty(const std::string & key) +{ auto value = getEnv(key); - if (value == "") return {}; + if (value == "") + return {}; return value; } diff --git a/src/libutil/environment-variables.hh b/src/libutil/environment-variables.hh index e0649adac34..879e1f30492 100644 --- a/src/libutil/environment-variables.hh +++ b/src/libutil/environment-variables.hh @@ -9,6 +9,7 @@ #include #include "types.hh" +#include "file-path.hh" namespace nix { @@ -17,6 +18,11 @@ namespace nix { */ std::optional getEnv(const std::string & key); +/** + * Like `getEnv`, but using `OsString` to avoid coercions. + */ +std::optional getEnvOs(const OsString & key); + /** * @return a non empty environment variable. Returns nullopt if the env * variable is set to "" @@ -43,6 +49,11 @@ int unsetenv(const char * name); */ int setEnv(const char * name, const char * value); +/** + * Like `setEnv`, but using `OsString` to avoid coercions. + */ +int setEnvOs(const OsString & name, const OsString & value); + /** * Clear the environment. */ diff --git a/src/libutil/executable-path.cc b/src/libutil/executable-path.cc new file mode 100644 index 00000000000..1658e36676d --- /dev/null +++ b/src/libutil/executable-path.cc @@ -0,0 +1,79 @@ +#include "environment-variables.hh" +#include "executable-path.hh" +#include "strings-inline.hh" +#include "util.hh" +#include "file-path-impl.hh" + +namespace nix { + +namespace fs = std::filesystem; + +constexpr static const OsStringView path_var_separator{ + &ExecutablePath::separator, + 1, +}; + +ExecutablePath ExecutablePath::load() +{ + // "If PATH is unset or is set to null, the path search is + // implementation-defined." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + return ExecutablePath::parse(getEnvOs(OS_STR("PATH")).value_or(OS_STR(""))); +} + +ExecutablePath ExecutablePath::parse(const OsString & path) +{ + auto strings = path.empty() ? (std::list{}) + : basicSplitString, OsString::value_type>(path, path_var_separator); + + std::vector ret; + ret.reserve(strings.size()); + + std::transform( + std::make_move_iterator(strings.begin()), + std::make_move_iterator(strings.end()), + std::back_inserter(ret), + [](auto && str) { + return fs::path{ + str.empty() + // "A zero-length prefix is a legacy feature that + // indicates the current working directory. It + // appears as two adjacent characters + // ("::"), as an initial preceding the rest + // of the list, or as a trailing following + // the rest of the list." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + ? OS_STR(".") + : std::move(str), + }; + }); + + return {ret}; +} + +OsString ExecutablePath::render() const +{ + std::vector path2; + for (auto & p : directories) + path2.push_back(p.native()); + return basicConcatStringsSep(path_var_separator, path2); +} + +std::optional +ExecutablePath::find(const OsString & exe, std::function isExecutable) const +{ + // "If the pathname being sought contains a , the search + // through the path prefixes shall not be performed." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + assert(OsPathTrait::rfindPathSep(exe) == exe.npos); + + for (auto & dir : directories) { + auto candidate = dir / exe; + if (isExecutable(candidate)) + return std::filesystem::canonical(candidate); + } + + return std::nullopt; +} + +} // namespace nix diff --git a/src/libutil/executable-path.hh b/src/libutil/executable-path.hh new file mode 100644 index 00000000000..bcb8d28e8e6 --- /dev/null +++ b/src/libutil/executable-path.hh @@ -0,0 +1,64 @@ +#pragma once +///@file + +#include "file-system.hh" + +namespace nix { + +struct ExecutablePath +{ + std::vector directories; + + constexpr static const OsString::value_type separator = +#ifdef WIN32 + L';' +#else + ':' +#endif + ; + + /** + * Parse `path` into a list of paths. + * + * On Unix we split on `:`, on Windows we split on `;`. + * + * For Unix, this is according to the POSIX spec for `PATH`. + * https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + */ + static ExecutablePath parse(const OsString & path); + + /** + * Load the `PATH` environment variable and `parse` it. + */ + static ExecutablePath load(); + + /** + * Opposite of `parse` + */ + OsString render() const; + + /** + * Search for an executable. + * + * For Unix, this is according to the POSIX spec for `PATH`. + * https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + * + * @param exe This must just be a name, and not contain any `/` (or + * `\` on Windows). in case it does, per the spec no lookup should + * be perfomed, and the path (it is not just a file name) as is. + * This is the caller's respsonsibility. + * + * This is a pure function, except for the default `isExecutable` + * argument, which uses the ambient file system to check if a file is + * executable (and exists). + * + * @return path to a resolved executable + */ + std::optional find( + const OsString & exe, + std::function isExecutableFile = isExecutableFileAmbient) const; + + bool operator==(const ExecutablePath &) const = default; +}; + +} // namespace nix diff --git a/src/libutil/file-path-impl.hh b/src/libutil/file-path-impl.hh index 4c90150fdc9..d7c823fd0f1 100644 --- a/src/libutil/file-path-impl.hh +++ b/src/libutil/file-path-impl.hh @@ -91,13 +91,10 @@ struct WindowsPathTrait }; -/** - * @todo Revisit choice of `char` or `wchar_t` for `WindowsPathTrait` - * argument. - */ -using NativePathTrait = +template +using OsPathTrait = #ifdef _WIN32 - WindowsPathTrait + WindowsPathTrait #else UnixPathTrait #endif diff --git a/src/libutil/file-path.hh b/src/libutil/file-path.hh index 6589c4060bb..8e4a88b9d56 100644 --- a/src/libutil/file-path.hh +++ b/src/libutil/file-path.hh @@ -1,10 +1,10 @@ #pragma once ///@file -#include #include #include "types.hh" +#include "os-string.hh" namespace nix { @@ -22,39 +22,26 @@ typedef std::set PathSetNG; * * @todo drop `NG` suffix and replace the one in `types.hh`. */ -struct PathViewNG : std::basic_string_view +struct PathViewNG : OsStringView { - using string_view = std::basic_string_view; + using string_view = OsStringView; using string_view::string_view; PathViewNG(const std::filesystem::path & path) - : std::basic_string_view(path.native()) + : OsStringView{path.native()} { } - PathViewNG(const std::filesystem::path::string_type & path) - : std::basic_string_view(path) + PathViewNG(const OsString & path) + : OsStringView{path} { } const string_view & native() const { return *this; } string_view & native() { return *this; } }; -std::string os_string_to_string(PathViewNG::string_view path); - -std::filesystem::path::string_type string_to_os_string(std::string_view s); - std::optional maybePath(PathView path); std::filesystem::path pathNG(PathView path); -/** - * Create string literals with the native character width of paths - */ -#ifndef _WIN32 -# define PATHNG_LITERAL(s) s -#else -# define PATHNG_LITERAL(s) L ## s -#endif - } diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 060a806fbc5..156dddf98dd 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -92,7 +92,7 @@ Path canonPath(PathView path, bool resolveSymlinks) arbitrary (but high) limit to prevent infinite loops. */ unsigned int followCount = 0, maxFollow = 1024; - auto ret = canonPathInner( + auto ret = canonPathInner>( path, [&followCount, &temp, maxFollow, resolveSymlinks] (std::string & result, std::string_view & remaining) { @@ -122,7 +122,7 @@ Path canonPath(PathView path, bool resolveSymlinks) Path dirOf(const PathView path) { - Path::size_type pos = NativePathTrait::rfindPathSep(path); + Path::size_type pos = OsPathTrait::rfindPathSep(path); if (pos == path.npos) return "."; return fs::path{path}.parent_path().string(); @@ -135,10 +135,10 @@ std::string_view baseNameOf(std::string_view path) return ""; auto last = path.size() - 1; - while (last > 0 && NativePathTrait::isPathSep(path[last])) + while (last > 0 && OsPathTrait::isPathSep(path[last])) last -= 1; - auto pos = NativePathTrait::rfindPathSep(path, last); + auto pos = OsPathTrait::rfindPathSep(path, last); if (pos == path.npos) pos = 0; else @@ -569,7 +569,7 @@ void replaceSymlink(const Path & target, const Path & link) } void setWriteTime( - const std::filesystem::path & path, + const fs::path & path, time_t accessedTime, time_t modificationTime, std::optional optIsSymlink) @@ -685,4 +685,17 @@ void moveFile(const Path & oldName, const Path & newName) ////////////////////////////////////////////////////////////////////// +bool isExecutableFileAmbient(const fs::path & exe) { + // Check file type, because directory being executable means + // something completely different. + return std::filesystem::is_regular_file(exe) + && access(exe.string().c_str(), +#ifdef WIN32 + 0 // TODO do better +#else + X_OK +#endif + ) == 0; +} + } diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 4b215162de6..5350ceb7e12 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -263,6 +263,12 @@ std::pair createTempFile(const Path & prefix = "nix"); */ Path defaultTempDir(); +/** + * Interpret `exe` as a location in the ambient file system and return + * whether it exists AND is executable. + */ +bool isExecutableFileAmbient(const std::filesystem::path & exe); + /** * Used in various places. */ diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 8552c4c9dad..8ff7ee51ff0 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -129,6 +129,7 @@ sources = files( 'english.cc', 'environment-variables.cc', 'error.cc', + 'executable-path.cc', 'exit.cc', 'experimental-features.cc', 'file-content-address.cc', @@ -183,6 +184,7 @@ headers = [config_h] + files( 'english.hh', 'environment-variables.hh', 'error.hh', + 'executable-path.hh', 'exit.hh', 'experimental-features.hh', 'file-content-address.hh', @@ -202,6 +204,7 @@ headers = [config_h] + files( 'lru-cache.hh', 'memory-source-accessor.hh', 'muxable-pipe.hh', + 'os-string.hh', 'pool.hh', 'position.hh', 'posix-source-accessor.hh', diff --git a/src/libutil/os-string.hh b/src/libutil/os-string.hh new file mode 100644 index 00000000000..0d75173e50e --- /dev/null +++ b/src/libutil/os-string.hh @@ -0,0 +1,43 @@ +#pragma once +///@file + +#include +#include +#include + +namespace nix { + +/** + * Named because it is similar to the Rust type, except it is in the + * native encoding not WTF-8. + * + * Same as `std::filesystem::path::string_type`, but manually defined to + * avoid including a much more complex header. + */ +using OsString = std::basic_string< +#if defined(_WIN32) && !defined(__CYGWIN__) + wchar_t +#else + char +#endif + >; + +/** + * `std::string_view` counterpart for `OsString`. + */ +using OsStringView = std::basic_string_view; + +std::string os_string_to_string(OsStringView path); + +OsString string_to_os_string(std::string_view s); + +/** + * Create string literals with the native character width of paths + */ +#ifndef _WIN32 +# define OS_STR(s) s +#else +# define OS_STR(s) L##s +#endif + +} diff --git a/src/libutil/strings-inline.hh b/src/libutil/strings-inline.hh index d254d486dd4..25b8e0ff67e 100644 --- a/src/libutil/strings-inline.hh +++ b/src/libutil/strings-inline.hh @@ -4,8 +4,8 @@ namespace nix { -template -C tokenizeString(std::string_view s, std::string_view separators) +template +C basicTokenizeString(std::basic_string_view s, std::basic_string_view separators) { C result; auto pos = s.find_first_not_of(separators, 0); @@ -13,14 +13,42 @@ C tokenizeString(std::string_view s, std::string_view separators) auto end = s.find_first_of(separators, pos + 1); if (end == s.npos) end = s.size(); - result.insert(result.end(), std::string(s, pos, end - pos)); + result.insert(result.end(), std::basic_string(s, pos, end - pos)); pos = s.find_first_not_of(separators, end); } return result; } template -std::string concatStringsSep(const std::string_view sep, const C & ss) +C tokenizeString(std::string_view s, std::string_view separators) +{ + return basicTokenizeString(s, separators); +} + +template +C basicSplitString(std::basic_string_view s, std::basic_string_view separators) +{ + C result; + size_t pos = 0; + while (pos <= s.size()) { + auto end = s.find_first_of(separators, pos); + if (end == s.npos) + end = s.size(); + result.insert(result.end(), std::basic_string(s, pos, end - pos)); + pos = end + 1; + } + + return result; +} + +template +C splitString(std::string_view s, std::string_view separators) +{ + return basicSplitString(s, separators); +} + +template +std::basic_string basicConcatStringsSep(const std::basic_string_view sep, const C & ss) { size_t size = 0; bool tail = false; @@ -28,10 +56,10 @@ std::string concatStringsSep(const std::string_view sep, const C & ss) for (const auto & s : ss) { if (tail) size += sep.size(); - size += std::string_view(s).size(); + size += std::basic_string_view{s}.size(); tail = true; } - std::string s; + std::basic_string s; s.reserve(size); tail = false; for (auto & i : ss) { @@ -43,6 +71,12 @@ std::string concatStringsSep(const std::string_view sep, const C & ss) return s; } +template +std::string concatStringsSep(const std::string_view sep, const C & ss) +{ + return basicConcatStringsSep(sep, ss); +} + template std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss) { diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc index 2bb7f8c0aed..60297228ead 100644 --- a/src/libutil/strings.cc +++ b/src/libutil/strings.cc @@ -1,6 +1,8 @@ +#include #include #include "strings-inline.hh" +#include "os-string.hh" namespace nix { @@ -8,6 +10,13 @@ template std::list tokenizeString(std::string_view s, std::string_v template std::set tokenizeString(std::string_view s, std::string_view separators); template std::vector tokenizeString(std::string_view s, std::string_view separators); +template std::list splitString(std::string_view s, std::string_view separators); +template std::set splitString(std::string_view s, std::string_view separators); +template std::vector splitString(std::string_view s, std::string_view separators); + +template std::list basicSplitString( + std::basic_string_view s, std::basic_string_view separators); + template std::string concatStringsSep(std::string_view, const std::list &); template std::string concatStringsSep(std::string_view, const std::set &); template std::string concatStringsSep(std::string_view, const std::vector &); diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh index 6e991e490e1..88b48d7703a 100644 --- a/src/libutil/strings.hh +++ b/src/libutil/strings.hh @@ -13,6 +13,12 @@ namespace nix { * * See also `basicSplitString()`, which preserves empty strings between separators, as well as at the start and end. */ +template +C basicTokenizeString(std::basic_string_view s, std::basic_string_view separators); + +/** + * Like `basicTokenizeString` but specialized to the default `char` + */ template C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); @@ -20,6 +26,20 @@ extern template std::list tokenizeString(std::string_view s, std::s extern template std::set tokenizeString(std::string_view s, std::string_view separators); extern template std::vector tokenizeString(std::string_view s, std::string_view separators); +/** + * Split a string, preserving empty strings between separators, as well as at the start and end. + * + * Returns a non-empty collection of strings. + */ +template +C basicSplitString(std::basic_string_view s, std::basic_string_view separators); +template +C splitString(std::string_view s, std::string_view separators); + +extern template std::list splitString(std::string_view s, std::string_view separators); +extern template std::set splitString(std::string_view s, std::string_view separators); +extern template std::vector splitString(std::string_view s, std::string_view separators); + /** * Concatenate the given strings with a separator between the elements. */ diff --git a/src/libutil/unix/environment-variables.cc b/src/libutil/unix/environment-variables.cc index 9c6fd3b1808..cd7c8f5e566 100644 --- a/src/libutil/unix/environment-variables.cc +++ b/src/libutil/unix/environment-variables.cc @@ -9,4 +9,14 @@ int setEnv(const char * name, const char * value) return ::setenv(name, value, 1); } +std::optional getEnvOs(const std::string & key) +{ + return getEnv(key); +} + +int setEnvOs(const OsString & name, const OsString & value) +{ + return setEnv(name.c_str(), value.c_str()); +} + } diff --git a/src/libutil/unix/file-path.cc b/src/libutil/unix/file-path.cc index 294048a2f8f..cccee86a1d7 100644 --- a/src/libutil/unix/file-path.cc +++ b/src/libutil/unix/file-path.cc @@ -8,16 +8,6 @@ namespace nix { -std::string os_string_to_string(PathViewNG::string_view path) -{ - return std::string { path }; -} - -std::filesystem::path::string_type string_to_os_string(std::string_view s) -{ - return std::string { s }; -} - std::optional maybePath(PathView path) { return { path }; diff --git a/src/libutil/unix/meson.build b/src/libutil/unix/meson.build index 38e5cd3aa47..1c5bf27fb14 100644 --- a/src/libutil/unix/meson.build +++ b/src/libutil/unix/meson.build @@ -4,6 +4,7 @@ sources += files( 'file-path.cc', 'file-system.cc', 'muxable-pipe.cc', + 'os-string.cc', 'processes.cc', 'signals.cc', 'users.cc', diff --git a/src/libutil/unix/os-string.cc b/src/libutil/unix/os-string.cc new file mode 100644 index 00000000000..8378afde292 --- /dev/null +++ b/src/libutil/unix/os-string.cc @@ -0,0 +1,21 @@ +#include +#include +#include +#include + +#include "file-path.hh" +#include "util.hh" + +namespace nix { + +std::string os_string_to_string(PathViewNG::string_view path) +{ + return std::string{path}; +} + +std::filesystem::path::string_type string_to_os_string(std::string_view s) +{ + return std::string{s}; +} + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 460802be3d6..db3ed1ddfa8 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,5 +1,6 @@ #include "util.hh" #include "fmt.hh" +#include "file-path.hh" #include #include diff --git a/src/libutil/windows/environment-variables.cc b/src/libutil/windows/environment-variables.cc index 25ab9d63ad9..525d08c6422 100644 --- a/src/libutil/windows/environment-variables.cc +++ b/src/libutil/windows/environment-variables.cc @@ -4,7 +4,30 @@ namespace nix { -int unsetenv(const char *name) +std::optional getEnvOs(const OsString & key) +{ + // Determine the required buffer size for the environment variable value + DWORD bufferSize = GetEnvironmentVariableW(key.c_str(), nullptr, 0); + if (bufferSize == 0) { + return std::nullopt; + } + + // Allocate a buffer to hold the environment variable value + std::wstring value{L'\0', bufferSize}; + + // Retrieve the environment variable value + DWORD resultSize = GetEnvironmentVariableW(key.c_str(), &value[0], bufferSize); + if (resultSize == 0) { + return std::nullopt; + } + + // Resize the string to remove the extra null characters + value.resize(resultSize); + + return value; +} + +int unsetenv(const char * name) { return -SetEnvironmentVariableA(name, nullptr); } @@ -14,4 +37,9 @@ int setEnv(const char * name, const char * value) return -SetEnvironmentVariableA(name, value); } +int setEnvOs(const OsString & name, const OsString & value) +{ + return -SetEnvironmentVariableW(name.c_str(), value.c_str()); +} + } diff --git a/src/libutil/windows/file-path.cc b/src/libutil/windows/file-path.cc index 3114ac4dfa7..7405c426b62 100644 --- a/src/libutil/windows/file-path.cc +++ b/src/libutil/windows/file-path.cc @@ -9,18 +9,6 @@ namespace nix { -std::string os_string_to_string(PathViewNG::string_view path) -{ - std::wstring_convert> converter; - return converter.to_bytes(std::filesystem::path::string_type { path }); -} - -std::filesystem::path::string_type string_to_os_string(std::string_view s) -{ - std::wstring_convert> converter; - return converter.from_bytes(std::string { s }); -} - std::optional maybePath(PathView path) { if (path.length() >= 3 && (('A' <= path[0] && path[0] <= 'Z') || ('a' <= path[0] && path[0] <= 'z')) && path[1] == ':' && WindowsPathTrait::isPathSep(path[2])) { diff --git a/src/libutil/windows/meson.build b/src/libutil/windows/meson.build index 00320877ff9..1c645fe0573 100644 --- a/src/libutil/windows/meson.build +++ b/src/libutil/windows/meson.build @@ -4,6 +4,7 @@ sources += files( 'file-path.cc', 'file-system.cc', 'muxable-pipe.cc', + 'os-string.cc', 'processes.cc', 'users.cc', 'windows-async-pipe.cc', diff --git a/src/libutil/windows/os-string.cc b/src/libutil/windows/os-string.cc new file mode 100644 index 00000000000..7507f9030da --- /dev/null +++ b/src/libutil/windows/os-string.cc @@ -0,0 +1,24 @@ +#include +#include +#include +#include + +#include "file-path.hh" +#include "file-path-impl.hh" +#include "util.hh" + +namespace nix { + +std::string os_string_to_string(PathViewNG::string_view path) +{ + std::wstring_convert> converter; + return converter.to_bytes(std::filesystem::path::string_type{path}); +} + +std::filesystem::path::string_type string_to_os_string(std::string_view s) +{ + std::wstring_convert> converter; + return converter.from_bytes(std::string{s}); +} + +} diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index 09d14073386..1a6574de2ea 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -8,6 +8,7 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "worker-protocol.hh" +#include "executable-path.hh" using namespace nix; @@ -39,6 +40,8 @@ void checkInfo(const std::string & msg) { } +namespace fs = std::filesystem; + struct CmdConfigCheck : StoreCommand { bool success = true; @@ -75,11 +78,13 @@ struct CmdConfigCheck : StoreCommand bool checkNixInPath() { - PathSet dirs; + std::set dirs; - for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) - if (pathExists(dir + "/nix-env")) - dirs.insert(dirOf(canonPath(dir + "/nix-env", true))); + for (auto & dir : ExecutablePath::load().directories) { + auto candidate = dir / "nix-env"; + if (fs::exists(candidate)) + dirs.insert(fs::canonical(candidate).parent_path() ); + } if (dirs.size() != 1) { std::stringstream ss; @@ -94,18 +99,25 @@ struct CmdConfigCheck : StoreCommand bool checkProfileRoots(ref store) { - PathSet dirs; + std::set dirs; - for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) { - Path profileDir = dirOf(dir); + for (auto & dir : ExecutablePath::load().directories) { + auto profileDir = dir.parent_path(); try { - Path userEnv = canonPath(profileDir, true); + auto userEnv = fs::weakly_canonical(profileDir); + + auto noContainsProfiles = [&]{ + for (auto && part : profileDir) + if (part == "profiles") return false; + return true; + }; - if (store->isStorePath(userEnv) && hasSuffix(userEnv, "user-environment")) { - while (profileDir.find("/profiles/") == std::string::npos && std::filesystem::is_symlink(profileDir)) - profileDir = absPath(readLink(profileDir), dirOf(profileDir)); + if (store->isStorePath(userEnv.string()) && hasSuffix(userEnv.string(), "user-environment")) { + while (noContainsProfiles() && std::filesystem::is_symlink(profileDir)) + profileDir = fs::weakly_canonical( + profileDir.parent_path() / fs::read_symlink(profileDir)); - if (profileDir.find("/profiles/") == std::string::npos) + if (noContainsProfiles()) dirs.insert(dir); } } catch (SystemError &) { diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 92ec3b78ac3..effc86a0a72 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -415,7 +415,7 @@ struct Common : InstallableCommand, MixProfile if (buildEnvironment.providesStructuredAttrs()) { fixupStructuredAttrs( - PATHNG_LITERAL("sh"), + OS_STR("sh"), "NIX_ATTRS_SH_FILE", buildEnvironment.getAttrsSH(), rewrites, @@ -423,7 +423,7 @@ struct Common : InstallableCommand, MixProfile tmpDir ); fixupStructuredAttrs( - PATHNG_LITERAL("json"), + OS_STR("json"), "NIX_ATTRS_JSON_FILE", buildEnvironment.getAttrsJSON(), rewrites, @@ -447,7 +447,7 @@ struct Common : InstallableCommand, MixProfile const BuildEnvironment & buildEnvironment, const std::filesystem::path & tmpDir) { - auto targetFilePath = tmpDir / PATHNG_LITERAL(".attrs."); + auto targetFilePath = tmpDir / OS_STR(".attrs."); targetFilePath += ext; writeFile(targetFilePath.string(), content); diff --git a/src/nix/env.cc b/src/nix/env.cc index 9db03ca3780..832320320ae 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -5,6 +5,7 @@ #include "eval.hh" #include "run.hh" #include "strings.hh" +#include "executable-path.hh" using namespace nix; @@ -95,10 +96,10 @@ struct CmdShell : InstallablesCommand, MixEnvironment } // TODO: split losslessly; empty means . - auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); - unixPath.insert(unixPath.begin(), pathAdditions.begin(), pathAdditions.end()); - auto unixPathString = concatStringsSep(":", unixPath); - setEnv("PATH", unixPathString.c_str()); + auto unixPath = ExecutablePath::load(); + unixPath.directories.insert(unixPath.directories.begin(), pathAdditions.begin(), pathAdditions.end()); + auto unixPathString = unixPath.render(); + setEnvOs(OS_STR("PATH"), unixPathString.c_str()); Strings args; for (auto & arg : command) diff --git a/src/nix/search.cc b/src/nix/search.cc index 7f8504d3f1e..c8d0b9e9641 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -10,6 +10,7 @@ #include "eval-cache.hh" #include "attr-path.hh" #include "hilite.hh" +#include "strings-inline.hh" #include #include diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 7b3357700e8..9ca3f6087e0 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -8,6 +8,7 @@ #include "attr-path.hh" #include "names.hh" #include "progress-bar.hh" +#include "executable-path.hh" using namespace nix; @@ -102,23 +103,17 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand /* Return the profile in which Nix is installed. */ Path getProfileDir(ref store) { - Path where; - - for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) - if (pathExists(dir + "/nix-env")) { - where = dir; - break; - } - - if (where == "") + auto whereOpt = ExecutablePath::load().find(OS_STR("nix-env")); + if (!whereOpt) throw Error("couldn't figure out how Nix is installed, so I can't upgrade it"); + auto & where = *whereOpt; printInfo("found Nix in '%s'", where); - if (hasPrefix(where, "/run/current-system")) + if (hasPrefix(where.string(), "/run/current-system")) throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'"); - Path profileDir = dirOf(where); + Path profileDir = where.parent_path().string(); // Resolve profile to /nix/var/nix/profiles/ link. while (canonPath(profileDir).find("/profiles/") == std::string::npos && std::filesystem::is_symlink(profileDir)) @@ -128,7 +123,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Path userEnv = canonPath(profileDir, true); - if (baseNameOf(where) != "bin" || + if (where.filename() != "bin" || !hasSuffix(userEnv, "user-environment")) throw Error("directory '%s' does not appear to be part of a Nix profile", where); diff --git a/tests/unit/libutil/executable-path.cc b/tests/unit/libutil/executable-path.cc new file mode 100644 index 00000000000..8d182357dab --- /dev/null +++ b/tests/unit/libutil/executable-path.cc @@ -0,0 +1,64 @@ +#include + +#include "executable-path.hh" + +namespace nix { + +#ifdef WIN32 +# define PATH_VAR_SEP L";" +#else +# define PATH_VAR_SEP ":" +#endif + +#define PATH_ENV_ROUND_TRIP(NAME, STRING_LIT, CXX_LIT) \ + TEST(ExecutablePath, NAME) \ + { \ + OsString s = STRING_LIT; \ + auto v = ExecutablePath::parse(s); \ + EXPECT_EQ(v, (ExecutablePath CXX_LIT)); \ + auto s2 = v.render(); \ + EXPECT_EQ(s2, s); \ + } + +PATH_ENV_ROUND_TRIP(emptyRoundTrip, OS_STR(""), ({})) + +PATH_ENV_ROUND_TRIP( + oneElemRoundTrip, + OS_STR("/foo"), + ({ + OS_STR("/foo"), + })) + +PATH_ENV_ROUND_TRIP( + twoElemsRoundTrip, + OS_STR("/foo" PATH_VAR_SEP "/bar"), + ({ + OS_STR("/foo"), + OS_STR("/bar"), + })) + +PATH_ENV_ROUND_TRIP( + threeElemsRoundTrip, + OS_STR("/foo" PATH_VAR_SEP "." PATH_VAR_SEP "/bar"), + ({ + OS_STR("/foo"), + OS_STR("."), + OS_STR("/bar"), + })) + +TEST(ExecutablePath, elementyElemNormalize) +{ + auto v = ExecutablePath::parse(PATH_VAR_SEP PATH_VAR_SEP PATH_VAR_SEP); + EXPECT_EQ( + v, + (ExecutablePath{{ + OS_STR("."), + OS_STR("."), + OS_STR("."), + OS_STR("."), + }})); + auto s2 = v.render(); + EXPECT_EQ(s2, OS_STR("." PATH_VAR_SEP "." PATH_VAR_SEP "." PATH_VAR_SEP ".")); +} + +} diff --git a/tests/unit/libutil/meson.build b/tests/unit/libutil/meson.build index f4f2ae7f9bb..83cec13ec06 100644 --- a/tests/unit/libutil/meson.build +++ b/tests/unit/libutil/meson.build @@ -52,6 +52,7 @@ sources = files( 'closure.cc', 'compression.cc', 'config.cc', + 'executable-path.cc', 'file-content-address.cc', 'git.cc', 'hash.cc', diff --git a/tests/unit/libutil/strings.cc b/tests/unit/libutil/strings.cc index 0bd2fe0a517..8ceb1676760 100644 --- a/tests/unit/libutil/strings.cc +++ b/tests/unit/libutil/strings.cc @@ -1,4 +1,5 @@ #include +#include #include "strings.hh" @@ -231,4 +232,117 @@ TEST(tokenizeString, tokenizeSepEmpty) ASSERT_EQ(tokenizeString(s, ","), expected); } +/* ---------------------------------------------------------------------------- + * splitString + * --------------------------------------------------------------------------*/ + +TEST(splitString, empty) +{ + Strings expected = {""}; + + ASSERT_EQ(splitString("", " \t\n\r"), expected); +} + +TEST(splitString, oneSep) +{ + Strings expected = {"", ""}; + + ASSERT_EQ(splitString(" ", " \t\n\r"), expected); +} + +TEST(splitString, twoSep) +{ + Strings expected = {"", "", ""}; + + ASSERT_EQ(splitString(" \n", " \t\n\r"), expected); +} + +TEST(splitString, tokenizeSpacesWithSpaces) +{ + auto s = "foo bar baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsWithDefaults) +{ + auto s = "foo\tbar\tbaz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsSpacesWithDefaults) +{ + auto s = "foo\t bar\t baz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "", "bar", "", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsSpacesNewlineWithDefaults) +{ + auto s = "foo\t\n bar\t\n baz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "", "", "bar", "", "", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsSpacesNewlineRetWithDefaults) +{ + auto s = "foo\t\n\r bar\t\n\r baz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "", "", "", "bar", "", "", "", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); + + auto s2 = "foo \t\n\r bar \t\n\r baz"; + Strings expected2 = {"foo", "", "", "", "", "bar", "", "", "", "", "baz"}; + + ASSERT_EQ(splitString(s2, " \t\n\r"), expected2); +} + +TEST(splitString, tokenizeWithCustomSep) +{ + auto s = "foo\n,bar\n,baz\n"; + Strings expected = {"foo\n", "bar\n", "baz\n"}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +TEST(splitString, tokenizeSepAtStart) +{ + auto s = ",foo,bar,baz"; + Strings expected = {"", "foo", "bar", "baz"}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +TEST(splitString, tokenizeSepAtEnd) +{ + auto s = "foo,bar,baz,"; + Strings expected = {"foo", "bar", "baz", ""}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +TEST(splitString, tokenizeSepEmpty) +{ + auto s = "foo,,baz"; + Strings expected = {"foo", "", "baz"}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +// concatStringsSep sep . splitString sep = id if sep is 1 char +RC_GTEST_PROP(splitString, recoveredByConcatStringsSep, (const std::string & s)) +{ + RC_ASSERT(concatStringsSep("/", splitString(s, "/")) == s); + RC_ASSERT(concatStringsSep("a", splitString(s, "a")) == s); +} + } // namespace nix From 0646b6cd610d901438fb66eadf09708d302e1c66 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 7 Aug 2024 22:29:56 -0500 Subject: [PATCH 199/284] Update comments / documentation. Co-authored-by: Robert Hensing --- src/libutil/file-system.cc | 1 + src/libutil/file-system.hh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 156dddf98dd..536ae29abe7 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -688,6 +688,7 @@ void moveFile(const Path & oldName, const Path & newName) bool isExecutableFileAmbient(const fs::path & exe) { // Check file type, because directory being executable means // something completely different. + // `is_regular_file` follows symlinks before checking. return std::filesystem::is_regular_file(exe) && access(exe.string().c_str(), #ifdef WIN32 diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 5350ceb7e12..1ae5fa13630 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -265,7 +265,7 @@ Path defaultTempDir(); /** * Interpret `exe` as a location in the ambient file system and return - * whether it exists AND is executable. + * whether it resolves to a file that is executable. */ bool isExecutableFileAmbient(const std::filesystem::path & exe); From 00f6db36fd72c9e82e923ce89d0ddb7d2e738528 Mon Sep 17 00:00:00 2001 From: Andrew Marshall Date: Thu, 8 Aug 2024 14:29:40 -0400 Subject: [PATCH 200/284] libstore: fix port binding in __darwinAllowLocalNetworking sandbox In d60c3f7f7c83134b5b4470ed84b6d5ed38e28753, this was changed to close a hole in the sandbox. Unfortunately, this was too restrictive such that it made local port binding fail, thus making derivations that needed `__darwinAllowLocalNetworking` gain nearly nothing, and thus largely fail (as the primary use for it is to enable port binding). This unfortunately does mean that a sandboxed build process can, in coordination with an actor outside the sandbox, escape the sandbox by binding a port and connecting to it externally to send data. I do not see a way around this with my experimentation and understanding of the (quite undocumented) macOS sandbox profile API. Notably it seems not possible to use the sandbox to do any of: - Restrict the remote IP of inbound network requests - Restrict the address being bound to As such, the `(local ip "*:*")` here appears to be functionally no different than `(local ip "localhost:*")` (however it *should* be different than removing the filter entirely, as that would make it also apply to non-IP networking). Doing `(allow network-inbound (require-all (local ip "localhost:*") (remote ip "localhost:*")))` causes listening to fail. Note that `network-inbound` implies `network-bind`. --- src/libstore/unix/build/sandbox-defaults.sb | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/unix/build/sandbox-defaults.sb b/src/libstore/unix/build/sandbox-defaults.sb index 6da01b7356b..15cd6daf5e0 100644 --- a/src/libstore/unix/build/sandbox-defaults.sb +++ b/src/libstore/unix/build/sandbox-defaults.sb @@ -49,6 +49,7 @@ R""( (if (param "_ALLOW_LOCAL_NETWORKING") (begin (allow network* (remote ip "localhost:*")) + (allow network-inbound (local ip "*:*")) ; required to bind and listen ; Allow access to /etc/resolv.conf (which is a symlink to ; /private/var/run/resolv.conf). From abbaba91223b47c87d270666d9c1a90ca0f55b34 Mon Sep 17 00:00:00 2001 From: Jeremy Kolb Date: Thu, 8 Aug 2024 14:41:25 -0400 Subject: [PATCH 201/284] Use the window size for the entire length --- src/nix/flake.cc | 40 +++++++++++++++++++++++---------- tests/functional/flakes/show.sh | 4 ++-- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 8ef19e3eaf1..839085b0417 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1264,22 +1264,38 @@ struct CmdFlakeShow : FlakeCommand, MixJSON "package"; if (description && !description->empty()) { // Maximum length to print - size_t maxLength = getWindowSize().second; - if (maxLength == 0) - maxLength = 77; - // Trim the string and only display the first line of the description. - auto trimmed = nix::trim(*description); + size_t maxLength = getWindowSize().second > 0 ? getWindowSize().second : 80; + + // Trim the description and only use the first line + auto trimmed = trim(*description); auto newLinePos = trimmed.find('\n'); auto length = newLinePos != std::string::npos ? newLinePos : trimmed.length(); - // Resize/sanitize the string and if it's too long add ellipses - std::string desc = filterANSIEscapes(trimmed, false, length); - if (desc.length() > maxLength) { - desc.resize(maxLength); - desc = desc.append("..."); + // Sanitize the description and calculate the two parts of the line + // In order to get the length of the printable characters we need to + // filter out escape sequences. + auto beginningOfLine = fmt("%s: %s '%s'", headerPrefix, type, name); + auto beginningOfLineLength = filterANSIEscapes(beginningOfLine, true).length(); + auto restOfLine = fmt(" - '%s'", filterANSIEscapes(trimmed, false, length)); + + // If we are already over the maximum length then do not trim + // and don't print the description (preserves existing behavior) + if (beginningOfLineLength >= maxLength) { + logger->cout("%s", beginningOfLine); + } + else { + auto line = beginningOfLine + restOfLine; + // FIXME: Specifying `true` here gives the correct length + // BUT removes colors/bold so something is not quite right here. + line = filterANSIEscapes(line, true, maxLength); + + // NOTE: This test might be incorrect since I get things like: + // 168 or 161 > maxLength. + if (line.length() > maxLength) { + line = line.replace(line.length() - 3, 3, "..."); + } + logger->cout("%s", line); } - - logger->cout("%s: %s '%s' - '%s'", headerPrefix, type, name, desc); } else { logger->cout("%s: %s '%s'", headerPrefix, type, name); diff --git a/tests/functional/flakes/show.sh b/tests/functional/flakes/show.sh index 3d91613ee4a..0edc450c345 100755 --- a/tests/functional/flakes/show.sh +++ b/tests/functional/flakes/show.sh @@ -110,5 +110,5 @@ nix flake show > ./show-output.txt test "$(awk -F '[:] ' '/aNoDescription/{print $NF}' ./show-output.txt)" = "package 'simple'" test "$(awk -F '[:] ' '/bOneLineDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - 'one line'" test "$(awk -F '[:] ' '/cMultiLineDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - 'line one'" -test "$(awk -F '[:] ' '/dLongDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - '01234567890123456789012345678901234567890123456789012345678901234567890123456...'" -test "$(awk -F '[:] ' '/eEmptyDescription/{print $NF}' ./show-output.txt)" = "package 'simple'" +test "$(awk -F '[:] ' '/dLongDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - '012345678901234567890123456..." +test "$(awk -F '[:] ' '/eEmptyDescription/{print $NF}' ./show-output.txt)" = "package 'simple'" \ No newline at end of file From 3cc2e2a0ac7df557f5cd958a0736ce4eedf5ae07 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 11 Aug 2024 03:31:44 +0200 Subject: [PATCH 202/284] Edit docs --- doc/manual/rl-next/ban-integer-overflow.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/rl-next/ban-integer-overflow.md b/doc/manual/rl-next/ban-integer-overflow.md index 49ecfe4c6fe..0e553af76f3 100644 --- a/doc/manual/rl-next/ban-integer-overflow.md +++ b/doc/manual/rl-next/ban-integer-overflow.md @@ -4,10 +4,10 @@ issues: [10968] prs: [11188] --- -Previously, integer overflow in the Nix language invoked C++ level signed overflow, which was undefined behaviour, but *probably* manifested as wrapping around on overflow. +Previously, integer overflow in the Nix language invoked C++ level signed overflow, which was undefined behaviour, but *usually* manifested as wrapping around on overflow. Since prior to the public release of Lix, Lix had C++ signed overflow defined to crash the process and nobody noticed this having accidentally removed overflow from the Nix language for three months until it was caught by fiddling around. -Given the significant body of actual Nix code that has been evaluated by Lix in that time, it does not appear that nixpkgs or much of importance depends on integer overflow, so it is safe to turn into an error. +Given the significant body of actual Nix code that has been evaluated by Lix in that time, it does not appear that nixpkgs or much of importance depends on integer overflow, so it appears safe to turn into an error. Some other overflows were fixed: - `builtins.fromJSON` of values greater than the maximum representable value in a signed 64-bit integer will generate an error. @@ -16,6 +16,6 @@ Some other overflows were fixed: Integer overflow now looks like the following: ``` -» nix eval --expr '9223372036854775807 + 1' +$ nix eval --expr '9223372036854775807 + 1' error: integer overflow in adding 9223372036854775807 + 1 ``` From f7c86d1a2f4460f58e8a91d004e438e69a3e133c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 11 Aug 2024 12:47:35 +0200 Subject: [PATCH 203/284] CONTRIBUTING.md: Add attribution and context rules We've recently had an incident where these rules were not followed, so let's add guidelines to increase the chances of contributors getting this right. Relevant discussion: https://discourse.nixos.org/t/code-attribution-policy/50445/2 --- CONTRIBUTING.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 12423366a74..1c9ee9b3518 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -52,6 +52,20 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). Link related issues to inform interested parties and future contributors about your change. If your pull request closes one or multiple issues, mention that in the description using `Closes: #`, as it will then happen automatically when your change is merged. + * Credit original authors when you're reusing or building on their work. + * Link to relevant changes in other projects, so that others can understand the full context of the change in the future when you or someone else will change or troubleshoot the code. + This is especially important when your change is based on work done in other repositories. + + Example: + ``` + This is based on the work of @user in . + This solution took inspiration from . + + Co-authored-by: User Name + ``` + + Use the `git cherry-pick -x` flag, and amend the commits to link to forks when applicable. + * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request). * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. From 70dde8c70c937498f85f1f54372608bac6b1a5c9 Mon Sep 17 00:00:00 2001 From: siddhantCodes Date: Sun, 11 Aug 2024 16:30:59 +0530 Subject: [PATCH 204/284] Use `std::filesystem::path` in build.cc --- src/nix/build.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nix/build.cc b/src/nix/build.cc index 47910018659..da9132d02f0 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -43,22 +43,22 @@ static nlohmann::json builtPathsWithResultToJSON(const std::vector& buildables, LocalFSStore& store2) +static void createOutLinks(const std::filesystem::path& outLink, const std::vector& buildables, LocalFSStore& store2) { for (const auto & [_i, buildable] : enumerate(buildables)) { auto i = _i; std::visit(overloaded { [&](const BuiltPath::Opaque & bo) { - std::string symlink = outLink; + auto symlink = outLink; if (i) symlink += fmt("-%d", i); - store2.addPermRoot(bo.path, absPath(symlink)); + store2.addPermRoot(bo.path, absPath(symlink.string())); }, [&](const BuiltPath::Built & bfd) { for (auto & output : bfd.outputs) { - std::string symlink = outLink; + auto symlink = outLink; if (i) symlink += fmt("-%d", i); if (output.first != "out") symlink += fmt("-%s", output.first); - store2.addPermRoot(output.second, absPath(symlink)); + store2.addPermRoot(output.second, absPath(symlink.string())); } }, }, buildable.path.raw()); From b0b19389821e0ab7aef35684bb91c65717136cce Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 11 Aug 2024 13:15:58 +0200 Subject: [PATCH 205/284] Urge contributors to read about contributing --- .github/PULL_REQUEST_TEMPLATE.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d12a4d36c5c..69da87db727 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,3 +1,22 @@ + + # Motivation From bd4e5a375b62ccc2f3eda22f8fa7f9d57910620f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 11 Aug 2024 13:16:36 +0200 Subject: [PATCH 206/284] Refer contributors to the matrix room We were basically sending contributors into the woods with that page. --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 12423366a74..9fc119708c2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,4 +83,4 @@ For larger changes see the [Nix reference manual](https://nix.dev/manual/nix/dev ## Getting help Whenever you're stuck or do not know how to proceed, you can always ask for help. -The appropriate channels to do so can be found on the [NixOS Community](https://nixos.org/community/) page. +We invite you to use our [Matrix room](https://matrix.to/#/#nix-dev:nixos.org) to ask questions. From 2c12a6962eb916fac8ebe0ebf335bb65b1da48b1 Mon Sep 17 00:00:00 2001 From: siddhantCodes Date: Sun, 11 Aug 2024 19:18:04 +0530 Subject: [PATCH 207/284] Remove unused variable --- src/nix/bundle.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 7d9aa771124..e152c26f2de 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -117,8 +117,6 @@ struct CmdBundle : InstallableValueCommand }, }); - auto outPathS = store->printStorePath(outPath); - if (!outLink) { auto * attr = vRes->attrs()->get(evalState->sName); if (!attr) From 0abc664a789b2403420041d34695ce4f8e78d1ad Mon Sep 17 00:00:00 2001 From: siddhantCodes Date: Sun, 11 Aug 2024 19:19:46 +0530 Subject: [PATCH 208/284] Use `std::filesystem` in `eval` and `flake` ... ... executables --- src/nix/eval.cc | 8 ++++---- src/nix/flake.cc | 25 +++++++++++++------------ 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 494735516f8..f8a9da73665 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -78,14 +78,14 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption if (pathExists(*writeTo)) throw Error("path '%s' already exists", *writeTo); - std::function recurse; + std::function recurse; - recurse = [&](Value & v, const PosIdx pos, const Path & path) + recurse = [&](Value & v, const PosIdx pos, const std::filesystem::path & path) { state->forceValue(v, pos); if (v.type() == nString) // FIXME: disallow strings with contexts? - writeFile(path, v.string_view()); + writeFile(path.string(), v.string_view()); else if (v.type() == nAttrs) { if (mkdir(path.c_str() #ifndef _WIN32 // TODO abstract mkdir perms for Windows @@ -98,7 +98,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption try { if (name == "." || name == "..") throw Error("invalid file name '%s'", name); - recurse(*attr.value, attr.pos, concatStrings(path, "/", name)); + recurse(*attr.value, attr.pos, path / name); } catch (Error & e) { e.addTrace( state->positions[attr.pos], diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3f9f8f99b06..7cfbf48a110 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -18,6 +18,7 @@ #include "markdown.hh" #include "users.hh" +#include #include #include @@ -870,27 +871,27 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand "If you've set '%s' to a string, try using a path instead.", templateDir, templateDirAttr->getAttrPathStr()).debugThrow(); - std::vector changedFiles; - std::vector conflictedFiles; + std::vector changedFiles; + std::vector conflictedFiles; - std::function copyDir; - copyDir = [&](const Path & from, const Path & to) + std::function copyDir; + copyDir = [&](const std::filesystem::path & from, const std::filesystem::path & to) { createDirs(to); for (auto & entry : std::filesystem::directory_iterator{from}) { checkInterrupt(); - auto from2 = entry.path().string(); - auto to2 = to + "/" + entry.path().filename().string(); - auto st = lstat(from2); - if (S_ISDIR(st.st_mode)) + auto from2 = entry.path(); + auto to2 = to / entry.path().filename(); + auto st = entry.symlink_status(); + if (std::filesystem::is_directory(st)) copyDir(from2, to2); - else if (S_ISREG(st.st_mode)) { + else if (std::filesystem::is_regular_file(st)) { auto contents = readFile(from2); if (pathExists(to2)) { auto contents2 = readFile(to2); if (contents != contents2) { - printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2, from2); + printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2.string()); conflictedFiles.push_back(to2); } else { notice("skipping identical file: %s", from2); @@ -899,11 +900,11 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand } else writeFile(to2, contents); } - else if (S_ISLNK(st.st_mode)) { + else if (std::filesystem::is_symlink(st)) { auto target = readLink(from2); if (pathExists(to2)) { if (readLink(to2) != target) { - printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2, from2); + printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2.string()); conflictedFiles.push_back(to2); } else { notice("skipping identical file: %s", from2); From 8e70f6f85021fb906b81114525cabbb2943d052a Mon Sep 17 00:00:00 2001 From: siddhantCodes Date: Sun, 11 Aug 2024 19:56:06 +0530 Subject: [PATCH 209/284] Use `std::filesystem::path` in `profile.cc` ... ...and `run.cc` --- src/nix/profile.cc | 12 ++++++------ src/nix/run.cc | 13 +++++++------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 1096f4386be..d751abdb1f7 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -122,9 +122,9 @@ struct ProfileManifest ProfileManifest() { } - ProfileManifest(EvalState & state, const Path & profile) + ProfileManifest(EvalState & state, const std::filesystem::path & profile) { - auto manifestPath = profile + "/manifest.json"; + auto manifestPath = profile / "manifest.json"; if (pathExists(manifestPath)) { auto json = nlohmann::json::parse(readFile(manifestPath)); @@ -176,12 +176,12 @@ struct ProfileManifest } } - else if (pathExists(profile + "/manifest.nix")) { + else if (pathExists(profile / "manifest.nix")) { // FIXME: needed because of pure mode; ugly. - state.allowPath(state.store->followLinksToStore(profile)); - state.allowPath(state.store->followLinksToStore(profile + "/manifest.nix")); + state.allowPath(state.store->followLinksToStore(profile.string())); + state.allowPath(state.store->followLinksToStore((profile / "manifest.nix").string())); - auto packageInfos = queryInstalled(state, state.store->followLinksToStore(profile)); + auto packageInfos = queryInstalled(state, state.store->followLinksToStore(profile.string())); for (auto & packageInfo : packageInfos) { ProfileElement element; diff --git a/src/nix/run.cc b/src/nix/run.cc index ec6a4d1e82e..dfe7f374f8d 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -11,6 +11,7 @@ #include "source-accessor.hh" #include "progress-bar.hh" #include "eval.hh" +#include #if __linux__ # include @@ -169,7 +170,7 @@ void chrootHelper(int argc, char * * argv) if (!pathExists(storeDir)) { // FIXME: Use overlayfs? - Path tmpDir = createTempDir(); + std::filesystem::path tmpDir = createTempDir(); createDirs(tmpDir + storeDir); @@ -178,16 +179,16 @@ void chrootHelper(int argc, char * * argv) for (auto entry : std::filesystem::directory_iterator{"/"}) { checkInterrupt(); - auto src = entry.path().string(); - Path dst = tmpDir + "/" + entry.path().filename().string(); + auto src = entry.path(); + Path dst = tmpDir / entry.path().filename(); if (pathExists(dst)) continue; - auto st = lstat(src); - if (S_ISDIR(st.st_mode)) { + auto st = entry.symlink_status(); + if (std::filesystem::is_directory(st)) { if (mkdir(dst.c_str(), 0700) == -1) throw SysError("creating directory '%s'", dst); if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) throw SysError("mounting '%s' on '%s'", src, dst); - } else if (S_ISLNK(st.st_mode)) + } else if (std::filesystem::is_symlink(st)) createSymlink(readLink(src), dst); } From d98e06a58130cbfeecbe03a51a829d81e46bea93 Mon Sep 17 00:00:00 2001 From: siddhantCodes Date: Sun, 11 Aug 2024 20:13:47 +0530 Subject: [PATCH 210/284] Use `std::filesystem::path` in more executables --- src/nix-channel/nix-channel.cc | 6 +++--- src/nix-collect-garbage/nix-collect-garbage.cc | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 9f7f557b59d..e2bedcbcb16 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -17,7 +17,7 @@ using namespace nix; typedef std::map Channels; static Channels channels; -static Path channelsList; +static std::filesystem::path channelsList; // Reads the list of channels. static void readChannels() @@ -41,7 +41,7 @@ static void writeChannels() { auto channelsFD = AutoCloseFD{open(channelsList.c_str(), O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, 0644)}; if (!channelsFD) - throw SysError("opening '%1%' for writing", channelsList); + throw SysError("opening '%1%' for writing", channelsList.string()); for (const auto & channel : channels) writeFull(channelsFD.get(), channel.second + " " + channel.first + "\n"); } @@ -165,7 +165,7 @@ static int main_nix_channel(int argc, char ** argv) { // Figure out the name of the `.nix-channels' file to use auto home = getHome(); - channelsList = settings.useXDGBaseDirectories ? createNixStateDir() + "/channels" : home + "/.nix-channels"; + channelsList = settings.useXDGBaseDirectories ? createNixStateDir() + OS_STR("/channels") : home + OS_STR("/.nix-channels"); nixDefExpr = getNixDefExpr(); // Figure out the name of the channels profile. diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 91209c97898..66acd60bc4a 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -21,7 +21,7 @@ bool dryRun = false; * Of course, this makes rollbacks to before this point in time * impossible. */ -void removeOldGenerations(std::string dir) +void removeOldGenerations(std::filesystem::path dir) { if (access(dir.c_str(), R_OK) != 0) return; @@ -81,7 +81,7 @@ static int main_nix_collect_garbage(int argc, char * * argv) }); if (removeOld) { - std::set dirsToClean = { + std::set dirsToClean = { profilesDir(), settings.nixStateDir + "/profiles", dirOf(getDefaultProfile())}; for (auto & dir : dirsToClean) removeOldGenerations(dir); From b64d6aa7b0fcc801d8ee496c41658d9883af37aa Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 11 Aug 2024 18:07:19 +0200 Subject: [PATCH 211/284] CONTRIBUTING.md: Clarify use of cherry-pick on forks --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1c9ee9b3518..da56653b80d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,7 +64,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). Co-authored-by: User Name ``` - Use the `git cherry-pick -x` flag, and amend the commits to link to forks when applicable. + When cherry-picking from a different repository, use the `-x` flag, and then amend the commits to turn the hashes into URLs. * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request). * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. From 9f6ee93f488c8935b560588ad7ba321d9618f588 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 12 Aug 2024 15:47:02 +0200 Subject: [PATCH 212/284] fetchers::downloadTarball(): Return a cacheable accessor downloadTarball() is used by `-I foo=` etc. fetchToStore() needs the accessor to have a fingerprint to enable caching. Fixes #11271. --- src/libcmd/common-eval-args.cc | 4 +++- src/libexpr/eval.cc | 4 +++- src/libexpr/primops/fetchTree.cc | 6 +++++- src/libfetchers/tarball.cc | 20 ++++++++++++++++++-- src/libfetchers/tarball.hh | 9 ++++++--- 5 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index fcef92487cb..ae9994a05f6 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -171,7 +171,9 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas { if (EvalSettings::isPseudoUrl(s)) { auto accessor = fetchers::downloadTarball( - EvalSettings::resolvePseudoUrl(s)).accessor; + state.store, + state.fetchSettings, + EvalSettings::resolvePseudoUrl(s)); auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy); return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index c9101678ce7..92320b554e8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -3088,7 +3088,9 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pa if (EvalSettings::isPseudoUrl(value)) { try { auto accessor = fetchers::downloadTarball( - EvalSettings::resolvePseudoUrl(value)).accessor; + store, + fetchSettings, + EvalSettings::resolvePseudoUrl(value)); auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy); return finish(store->toRealPath(storePath)); } catch (Error & e) { diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 78328701dcd..5d074e62303 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -507,7 +507,11 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v // https://github.com/NixOS/nix/issues/4313 auto storePath = unpack - ? fetchToStore(*state.store, fetchers::downloadTarball(*url).accessor, FetchMode::Copy, name) + ? fetchToStore( + *state.store, + fetchers::downloadTarball(state.store, state.fetchSettings, *url), + FetchMode::Copy, + name) : fetchers::downloadFile(state.store, *url, name).storePath; if (expectedHash) { diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 457210542dc..dd4f3b78086 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -102,7 +102,7 @@ DownloadFileResult downloadFile( }; } -DownloadTarballResult downloadTarball( +static DownloadTarballResult downloadTarball_( const std::string & url, const Headers & headers) { @@ -202,6 +202,22 @@ DownloadTarballResult downloadTarball( return attrsToResult(infoAttrs); } +ref downloadTarball( + ref store, + const Settings & settings, + const std::string & url) +{ + /* Go through Input::getAccessor() to ensure that the resulting + accessor has a fingerprint. */ + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "tarball"); + attrs.insert_or_assign("url", url); + + auto input = Input::fromAttrs(settings, std::move(attrs)); + + return input.getAccessor(store).first; +} + // An input scheme corresponding to a curl-downloadable resource. struct CurlInputScheme : InputScheme { @@ -353,7 +369,7 @@ struct TarballInputScheme : CurlInputScheme { auto input(_input); - auto result = downloadTarball(getStrAttr(input.attrs, "url"), {}); + auto result = downloadTarball_(getStrAttr(input.attrs, "url"), {}); result.accessor->setPathDisplay("«" + input.to_string() + "»"); diff --git a/src/libfetchers/tarball.hh b/src/libfetchers/tarball.hh index d9bdd123d58..2042041d5ad 100644 --- a/src/libfetchers/tarball.hh +++ b/src/libfetchers/tarball.hh @@ -14,6 +14,8 @@ struct SourceAccessor; namespace nix::fetchers { +struct Settings; + struct DownloadFileResult { StorePath storePath; @@ -40,8 +42,9 @@ struct DownloadTarballResult * Download and import a tarball into the Git cache. The result is the * Git tree hash of the root directory. */ -DownloadTarballResult downloadTarball( - const std::string & url, - const Headers & headers = {}); +ref downloadTarball( + ref store, + const Settings & settings, + const std::string & url); } From 58b03ef1cd73385699e3ea5c46d0dc0d36c3bbcd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jul 2024 23:14:36 -0400 Subject: [PATCH 213/284] Move `NIX_BIN_DIR` and all logic using it to the Nix executable itself This is because with the split packages of the Meson build, we simply have no idea what directory the binaries will be installed in when we build the library. In the process of doing so, consolidate and make more sophisticated the logic to cope with a few corner cases (e.g. `NIX_BIN_DIR` exists, but no binaries are inside it). Co-authored-by: Robert Hensing --- doc/manual/rl-next/build-hook-default.md | 22 ++++++++++++++ src/libcmd/repl.cc | 38 +++++++++++------------- src/libcmd/repl.hh | 14 +++++++-- src/libstore/globals.cc | 29 ------------------ src/libstore/globals.hh | 7 +---- src/libstore/local.mk | 1 - src/libstore/meson.build | 2 -- src/libstore/unix/build/hook-instance.cc | 14 +++++++-- src/libutil/executable-path.cc | 18 ++++++++++- src/libutil/executable-path.hh | 15 +++++++++- src/nix-channel/nix-channel.cc | 11 +++---- src/nix/local.mk | 2 ++ src/nix/main.cc | 12 ++++++++ src/nix/meson.build | 19 +++++++++++- src/nix/repl.cc | 23 +++++++++++++- src/nix/self-exe.cc | 38 ++++++++++++++++++++++++ src/nix/self-exe.hh | 31 +++++++++++++++++++ src/nix/upgrade-nix.cc | 5 ++-- src/perl/lib/Nix/Config.pm.in | 1 - src/perl/lib/Nix/Store.pm | 2 +- src/perl/lib/Nix/Store.xs | 5 ---- 21 files changed, 227 insertions(+), 82 deletions(-) create mode 100644 doc/manual/rl-next/build-hook-default.md create mode 100644 src/nix/self-exe.cc create mode 100644 src/nix/self-exe.hh diff --git a/doc/manual/rl-next/build-hook-default.md b/doc/manual/rl-next/build-hook-default.md new file mode 100644 index 00000000000..197290536c3 --- /dev/null +++ b/doc/manual/rl-next/build-hook-default.md @@ -0,0 +1,22 @@ +--- +synopsis: |- + The `build-hook` setting's default is less useful when using `libnixstore` as a library +prs: +- 11178 +--- + +*This is an obscure issue that only affects usage of the `libnixstore` library outside of the Nix executable.* + +As part the ongoing [rewrite of the build system](https://github.com/NixOS/nix/issues/2503) to use [Meson](https://mesonbuild.com/), we are also switching to packaging individual Nix components separately (and building them in separate derivations). +This means that when building `libnixstore` we do not know where the Nix binaries will be installed --- `libnixstore` doesn't know about downstream consumers like the Nix binaries at all. + +*This is also unrelated to the _`post`_-`build-hook`*, which is often used for pushing to a cache.* + +This has a small adverse affect on remote building --- the `build-remote` executable that is specified from the [`build-hook`](@docroot@/command-ref/conf-file.md#conf-build-hook) setting will not be gotten from the (presumed) installation location, but instead looked up on the `PATH`. +This means that other applications linking `libnixstore` that wish to use remote building must arrange for the `nix` command to be on the PATH (or manually overriding `build-hook`) in order for that to work. + +Long term we don't envision this being a downside, because we plan to [get rid of `build-remote` and the build hook setting entirely](https://github.com/NixOS/nix/issues/1221). +There is simply no need to add a second layer of remote-procedure-calling when we want to connect to a remote builder. +The build hook protocol did in principle support custom ways of remote building, but that can also be accomplished with a custom service for the ssh or daemon/ssh-ng protocols, or with a custom [store type](@docroot@/store/types/) i.e. `Store` subclass. + +The Perl bindings no longer expose `getBinDir` either, since they libraries those bindings wrap no longer know the location of installed binaries as described above. diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index efc04b029fc..e7c43367c24 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -7,7 +7,6 @@ #include "ansicolor.hh" #include "shared.hh" -#include "config-global.hh" #include "eval.hh" #include "eval-settings.hh" #include "attr-path.hh" @@ -77,10 +76,14 @@ struct NixRepl int displ; StringSet varNames; + RunNix * runNixPtr; + + void runNix(Path program, const Strings & args, const std::optional & input = {}); + std::unique_ptr interacter; NixRepl(const LookupPath & lookupPath, nix::ref store,ref state, - std::function getValues); + std::function getValues, RunNix * runNix); virtual ~NixRepl() = default; ReplExitStatus mainLoop() override; @@ -125,32 +128,16 @@ std::string removeWhitespace(std::string s) NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref store, ref state, - std::function getValues) + std::function getValues, RunNix * runNix = nullptr) : AbstractNixRepl(state) , debugTraceIndex(0) , getValues(getValues) , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get())) + , runNixPtr{runNix} , interacter(make_unique(getDataDir() + "/nix/repl-history")) { } -void runNix(Path program, const Strings & args, - const std::optional & input = {}) -{ - auto subprocessEnv = getEnv(); - subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); - //isInteractive avoid grabling interactive commands - runProgram2(RunOptions { - .program = settings.nixBinDir+ "/" + program, - .args = args, - .environment = subprocessEnv, - .input = input, - .isInteractive = true, - }); - - return; -} - static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt) { if (dt.isError) @@ -833,9 +820,18 @@ void NixRepl::evalString(std::string s, Value & v) } +void NixRepl::runNix(Path program, const Strings & args, const std::optional & input) +{ + if (runNixPtr) + (*runNixPtr)(program, args, input); + else + throw Error("Cannot run '%s', no method of calling the Nix CLI provided", program); +} + + std::unique_ptr AbstractNixRepl::create( const LookupPath & lookupPath, nix::ref store, ref state, - std::function getValues) + std::function getValues, RunNix * runNix) { return std::make_unique( lookupPath, diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 3fd4b2c391a..11d1820f504 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -19,9 +19,19 @@ struct AbstractNixRepl typedef std::vector> AnnotatedValues; + using RunNix = void(Path program, const Strings & args, const std::optional & input); + + /** + * @param runNix Function to run the nix CLI to support various + * `:` commands. Optional; if not provided, + * everything else will still work fine, but those commands won't. + */ static std::unique_ptr create( - const LookupPath & lookupPath, nix::ref store, ref state, - std::function getValues); + const LookupPath & lookupPath, + nix::ref store, + ref state, + std::function getValues, + RunNix * runNix = nullptr); static ReplExitStatus runSimple( ref evalState, diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 439a6f97c08..52ab35b4ce5 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -64,7 +64,6 @@ Settings::Settings() , nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) , nixConfDir(canonPath(getEnvNonEmpty("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) , nixUserConfFiles(getUserConfigFiles()) - , nixBinDir(canonPath(getEnvNonEmpty("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) , nixManDir(canonPath(NIX_MAN_DIR)) , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { @@ -95,34 +94,6 @@ Settings::Settings() sandboxPaths = tokenizeString("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /bin/bash /private/tmp /private/var/tmp /usr/lib"); allowedImpureHostPrefixes = tokenizeString("/System/Library /usr/lib /dev /bin/sh"); #endif - - /* Set the build hook location - - For builds we perform a self-invocation, so Nix has to be self-aware. - That is, it has to know where it is installed. We don't think it's sentient. - - Normally, nix is installed according to `nixBinDir`, which is set at compile time, - but can be overridden. This makes for a great default that works even if this - code is linked as a library into some other program whose main is not aware - that it might need to be a build remote hook. - - However, it may not have been installed at all. For example, if it's a static build, - there's a good chance that it has been moved out of its installation directory. - That makes `nixBinDir` useless. Instead, we'll query the OS for the path to the - current executable, using `getSelfExe()`. - - As a last resort, we resort to `PATH`. Hopefully we find a `nix` there that's compatible. - If you're porting Nix to a new platform, that might be good enough for a while, but - you'll want to improve `getSelfExe()` to work on your platform. - */ - std::string nixExePath = nixBinDir + "/nix"; - if (!pathExists(nixExePath)) { - nixExePath = getSelfExe().value_or("nix"); - } - buildHook = { - nixExePath, - "__build-remote", - }; } void loadConfFile(AbstractConfig & config) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 1ca9f02f5ab..65c7bf3bca1 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -84,11 +84,6 @@ public: */ std::vector nixUserConfFiles; - /** - * The directory where the main programs are stored. - */ - Path nixBinDir; - /** * The directory where the man pages are stored. */ @@ -246,7 +241,7 @@ public: )", {"build-timeout"}}; - Setting buildHook{this, {}, "build-hook", + Setting buildHook{this, {"nix", "__build-remote"}, "build-hook", R"( The path to the helper program that executes remote builds. diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 5dc8f3370bc..88be6a36628 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -71,7 +71,6 @@ libstore_CXXFLAGS += \ -DNIX_STATE_DIR=\"$(NIX_ROOT)$(localstatedir)/nix\" \ -DNIX_LOG_DIR=\"$(NIX_ROOT)$(localstatedir)/log/nix\" \ -DNIX_CONF_DIR=\"$(NIX_ROOT)$(sysconfdir)/nix\" \ - -DNIX_BIN_DIR=\"$(NIX_ROOT)$(bindir)\" \ -DNIX_MAN_DIR=\"$(NIX_ROOT)$(mandir)\" \ -DLSOF=\"$(NIX_ROOT)$(lsof)\" diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 50b15e15dc7..d2cc235fde3 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -328,7 +328,6 @@ prefix = get_option('prefix') path_opts = [ # Meson built-ins. 'datadir', - 'bindir', 'mandir', 'libdir', 'includedir', @@ -373,7 +372,6 @@ cpp_str_defines = { 'NIX_STATE_DIR': state_dir / 'nix', 'NIX_LOG_DIR': log_dir, 'NIX_CONF_DIR': sysconfdir / 'nix', - 'NIX_BIN_DIR': bindir, 'NIX_MAN_DIR': mandir, } diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index d73d86ff27f..4f8492fe92a 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -4,6 +4,7 @@ #include "file-system.hh" #include "child.hh" #include "strings.hh" +#include "executable-path.hh" namespace nix { @@ -16,11 +17,18 @@ HookInstance::HookInstance() if (buildHookArgs.empty()) throw Error("'build-hook' setting is empty"); - auto buildHook = canonPath(buildHookArgs.front()); + std::filesystem::path buildHook = buildHookArgs.front(); buildHookArgs.pop_front(); + try { + buildHook = ExecutablePath::load().findPath(buildHook); + } catch (ExecutableLookupError & e) { + e.addTrace(nullptr, "while resolving the 'build-hook' setting'"); + throw; + } + Strings args; - args.push_back(std::string(baseNameOf(buildHook))); + args.push_back(buildHook.filename().string()); for (auto & arg : buildHookArgs) args.push_back(arg); @@ -59,7 +67,7 @@ HookInstance::HookInstance() if (dup2(builderOut.readSide.get(), 5) == -1) throw SysError("dupping builder's stdout/stderr"); - execv(buildHook.c_str(), stringsToCharPtrs(args).data()); + execv(buildHook.native().c_str(), stringsToCharPtrs(args).data()); throw SysError("executing '%s'", buildHook); }); diff --git a/src/libutil/executable-path.cc b/src/libutil/executable-path.cc index 1658e36676d..8005a19bed4 100644 --- a/src/libutil/executable-path.cc +++ b/src/libutil/executable-path.cc @@ -60,7 +60,7 @@ OsString ExecutablePath::render() const } std::optional -ExecutablePath::find(const OsString & exe, std::function isExecutable) const +ExecutablePath::findName(const OsString & exe, std::function isExecutable) const { // "If the pathname being sought contains a , the search // through the path prefixes shall not be performed." @@ -76,4 +76,20 @@ ExecutablePath::find(const OsString & exe, std::function return std::nullopt; } +fs::path ExecutablePath::findPath(const fs::path & exe, std::function isExecutable) const +{ + // "If the pathname being sought contains a , the search + // through the path prefixes shall not be performed." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + if (exe.filename() == exe) { + auto resOpt = findName(exe, isExecutable); + if (resOpt) + return *resOpt; + else + throw ExecutableLookupError("Could not find executable '%s'", exe.native()); + } else { + return exe; + } +} + } // namespace nix diff --git a/src/libutil/executable-path.hh b/src/libutil/executable-path.hh index bcb8d28e8e6..f46d5e2128c 100644 --- a/src/libutil/executable-path.hh +++ b/src/libutil/executable-path.hh @@ -5,6 +5,8 @@ namespace nix { +MakeError(ExecutableLookupError, Error); + struct ExecutablePath { std::vector directories; @@ -54,10 +56,21 @@ struct ExecutablePath * * @return path to a resolved executable */ - std::optional find( + std::optional findName( const OsString & exe, std::function isExecutableFile = isExecutableFileAmbient) const; + /** + * Like the `findName` but also allows a file path as input. + * + * This implements the full POSIX spec: if the path is just a name, + * it searches like the above. Otherwise, it returns the path as is. + * If (in the name case) the search fails, an exception is thrown. + */ + std::filesystem::path findPath( + const std::filesystem::path & exe, + std::function isExecutable = isExecutableFileAmbient) const; + bool operator==(const ExecutablePath &) const = default; }; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 9f7f557b59d..878f3c82803 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -7,6 +7,7 @@ #include "eval-settings.hh" // for defexpr #include "users.hh" #include "tarball.hh" +#include "self-exe.hh" #include #include @@ -67,7 +68,7 @@ static void removeChannel(const std::string & name) channels.erase(name); writeChannels(); - runProgram(settings.nixBinDir + "/nix-env", true, { "--profile", profile, "--uninstall", name }); + runProgram(getNixBin("nix-env").string(), true, { "--profile", profile, "--uninstall", name }); } static Path nixDefExpr; @@ -118,7 +119,7 @@ static void update(const StringSet & channelNames) bool unpacked = false; if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { - runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + unpackChannelPath + + runProgram(getNixBin("nix-build").string(), false, { "--no-out-link", "--expr", "import " + unpackChannelPath + "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); unpacked = true; } @@ -143,7 +144,7 @@ static void update(const StringSet & channelNames) for (auto & expr : exprs) envArgs.push_back(std::move(expr)); envArgs.push_back("--quiet"); - runProgram(settings.nixBinDir + "/nix-env", false, envArgs); + runProgram(getNixBin("nix-env").string(), false, envArgs); // Make the channels appear in nix-env. struct stat st; @@ -244,7 +245,7 @@ static int main_nix_channel(int argc, char ** argv) case cListGenerations: if (!args.empty()) throw UsageError("'--list-generations' expects no arguments"); - std::cout << runProgram(settings.nixBinDir + "/nix-env", false, {"--profile", profile, "--list-generations"}) << std::flush; + std::cout << runProgram(getNixBin("nix-env").string(), false, {"--profile", profile, "--list-generations"}) << std::flush; break; case cRollback: if (args.size() > 1) @@ -256,7 +257,7 @@ static int main_nix_channel(int argc, char ** argv) } else { envArgs.push_back("--rollback"); } - runProgram(settings.nixBinDir + "/nix-env", false, envArgs); + runProgram(getNixBin("nix-env").string(), false, envArgs); break; } diff --git a/src/nix/local.mk b/src/nix/local.mk index 28b30b58619..b57f6b3e2f9 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -26,6 +26,8 @@ endif nix_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain) -I src/libcmd -I doc/manual $(INCLUDE_nix) +nix_CXXFLAGS += -DNIX_BIN_DIR=\"$(NIX_ROOT)$(bindir)\" + nix_LIBS = libexpr libmain libfetchers libflake libstore libutil libcmd nix_LDFLAGS = $(THREAD_LDFLAGS) $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS) diff --git a/src/nix/main.cc b/src/nix/main.cc index 9d7d617ccda..34de79ac877 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -19,6 +19,7 @@ #include "network-proxy.hh" #include "eval-cache.hh" #include "flake/flake.hh" +#include "self-exe.hh" #include #include @@ -366,6 +367,17 @@ void mainWrapped(int argc, char * * argv) initGC(); flake::initLib(flakeSettings); + /* Set the build hook location + + For builds we perform a self-invocation, so Nix has to be + self-aware. That is, it has to know where it is installed. We + don't think it's sentient. + */ + settings.buildHook.setDefault(Strings { + getNixBin({}).string(), + "__build-remote", + }); + #if __linux__ if (isRootUser()) { try { diff --git a/src/nix/meson.build b/src/nix/meson.build index 7405548de0c..05bee6a8701 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -33,6 +33,21 @@ subdir('build-utils-meson/threads') subdir('build-utils-meson/export-all-symbols') +configdata = configuration_data() + +fs = import('fs') + +bindir = get_option('bindir') +if not fs.is_absolute(bindir) + bindir = get_option('prefix') / bindir +endif +configdata.set_quoted('NIX_BIN_DIR', bindir) + +config_h = configure_file( + configuration : configdata, + output : 'config-nix-cli.hh', +) + add_project_arguments( # TODO(Qyriad): Yes this is how the autoconf+Make system did it. # It would be nice for our headers to be idempotent instead. @@ -42,15 +57,17 @@ add_project_arguments( #'-include', 'config-fetchers.hh', '-include', 'config-main.hh', '-include', 'config-cmd.hh', + '-include', 'config-nix-cli.hh', language : 'cpp', ) subdir('build-utils-meson/diagnostics') subdir('build-utils-meson/generate-header') -nix_sources = files( +nix_sources = [config_h] + files( 'add-to-store.cc', 'app.cc', + 'self-exe.cc', 'build.cc', 'bundle.cc', 'cat.cc', diff --git a/src/nix/repl.cc b/src/nix/repl.cc index a2f3e033e72..5a570749f4c 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -1,12 +1,32 @@ #include "eval.hh" #include "eval-settings.hh" +#include "config-global.hh" #include "globals.hh" #include "command.hh" #include "installable-value.hh" #include "repl.hh" +#include "processes.hh" +#include "self-exe.hh" namespace nix { +void runNix(Path program, const Strings & args, + const std::optional & input = {}) +{ + auto subprocessEnv = getEnv(); + subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); + //isInteractive avoid grabling interactive commands + runProgram2(RunOptions { + .program = getNixBin(program).string(), + .args = args, + .environment = subprocessEnv, + .input = input, + .isInteractive = true, + }); + + return; +} + struct CmdRepl : RawInstallablesCommand { CmdRepl() { @@ -81,7 +101,8 @@ struct CmdRepl : RawInstallablesCommand lookupPath, openStore(), state, - getValues + getValues, + runNix ); repl->autoArgs = getAutoArgs(*repl->state); repl->initEnv(); diff --git a/src/nix/self-exe.cc b/src/nix/self-exe.cc new file mode 100644 index 00000000000..a260bafd589 --- /dev/null +++ b/src/nix/self-exe.cc @@ -0,0 +1,38 @@ +#include "current-process.hh" +#include "file-system.hh" +#include "globals.hh" +#include "self-exe.hh" + +namespace nix { + +namespace fs = std::filesystem; + +fs::path getNixBin(std::optional binaryNameOpt) +{ + auto getBinaryName = [&] { return binaryNameOpt ? *binaryNameOpt : "nix"; }; + + // If the environment variable is set, use it unconditionally + if (auto envOpt = getEnvNonEmpty("NIX_BIN_DIR")) + return fs::path{*envOpt} / std::string{getBinaryName()}; + + // Use some-times avaiable OS tricks to get to the path of this Nix, and try that + if (auto selfOpt = getSelfExe()) { + fs::path path{*selfOpt}; + if (binaryNameOpt) + path = path.parent_path() / std::string{*binaryNameOpt}; + if (fs::exists(path)) + return path; + } + + // If `nix` exists at the hardcoded fallback path, use it. + { + auto path = fs::path{NIX_BIN_DIR} / std::string{getBinaryName()}; + if (fs::exists(path)) + return path; + } + + // return just the name, hoping the exe is on the `PATH` + return getBinaryName(); +} + +} diff --git a/src/nix/self-exe.hh b/src/nix/self-exe.hh new file mode 100644 index 00000000000..0772afa67ef --- /dev/null +++ b/src/nix/self-exe.hh @@ -0,0 +1,31 @@ +#pragma once +///@file + +#include + +namespace nix { + +/** + * Get a path to the given Nix binary. + * + * Normally, nix is installed according to `NIX_BIN_DIR`, which is set + * at compile time, but can be overridden. + * + * However, it may not have been installed at all. For example, if it's + * a static build, there's a good chance that it has been moved out of + * its installation directory. That makes `NIX_BIN_DIR` useless. + * Instead, we'll query the OS for the path to the current executable, + * using `getSelfExe()`. + * + * As a last resort, we resort to `PATH`. Hopefully we find a `nix` + * there that's compatible. If you're porting Nix to a new platform, + * that might be good enough for a while, but you'll want to improve + * `getSelfExe()` to work on your platform. + * + * @param binary_name the exact binary name we're looking up. Might be + * `nix-*` instead of `nix` for the legacy CLI commands. Optional to use + * current binary name. + */ +std::filesystem::path getNixBin(std::optional binary_name = {}); + +} diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 9ca3f6087e0..f54cc59d036 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -9,6 +9,7 @@ #include "names.hh" #include "progress-bar.hh" #include "executable-path.hh" +#include "self-exe.hh" using namespace nix; @@ -93,7 +94,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand { Activity act(*logger, lvlInfo, actUnknown, fmt("installing '%s' into profile '%s'...", store->printStorePath(storePath), profileDir)); - runProgram(settings.nixBinDir + "/nix-env", false, + runProgram(getNixBin("nix-env").string(), false, {"--profile", profileDir, "-i", store->printStorePath(storePath), "--no-sandbox"}); } @@ -103,7 +104,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand /* Return the profile in which Nix is installed. */ Path getProfileDir(ref store) { - auto whereOpt = ExecutablePath::load().find(OS_STR("nix-env")); + auto whereOpt = ExecutablePath::load().findName(OS_STR("nix-env")); if (!whereOpt) throw Error("couldn't figure out how Nix is installed, so I can't upgrade it"); auto & where = *whereOpt; diff --git a/src/perl/lib/Nix/Config.pm.in b/src/perl/lib/Nix/Config.pm.in index 508a15e1598..ad51cff3b28 100644 --- a/src/perl/lib/Nix/Config.pm.in +++ b/src/perl/lib/Nix/Config.pm.in @@ -5,7 +5,6 @@ use Nix::Store; $version = "@PACKAGE_VERSION@"; -$binDir = Nix::Store::getBinDir; $storeDir = Nix::Store::getStoreDir; %config = (); diff --git a/src/perl/lib/Nix/Store.pm b/src/perl/lib/Nix/Store.pm index 16f2e17c81a..f2ae7e88f81 100644 --- a/src/perl/lib/Nix/Store.pm +++ b/src/perl/lib/Nix/Store.pm @@ -24,7 +24,7 @@ our @EXPORT = qw( hashPath hashFile hashString convertHash signString checkSignature - getBinDir getStoreDir + getStoreDir setVerbosity ); diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index f951437c899..172c3500de0 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -424,11 +424,6 @@ StoreWrapper::addTempRoot(char * storePath) } -SV * getBinDir() - PPCODE: - XPUSHs(sv_2mortal(newSVpv(settings.nixBinDir.c_str(), 0))); - - SV * getStoreDir() PPCODE: XPUSHs(sv_2mortal(newSVpv(settings.nixStore.c_str(), 0))); From d49e14ba4ad4880a679f7265387ec83d01945b4d Mon Sep 17 00:00:00 2001 From: Jeremy Kolb Date: Mon, 12 Aug 2024 14:49:52 -0400 Subject: [PATCH 214/284] Take ANSI and tree characters into account --- src/nix/flake.cc | 68 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 16 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 839085b0417..fdbadd39097 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1263,6 +1263,46 @@ struct CmdFlakeShow : FlakeCommand, MixJSON attrPath.size() >= 1 && attrPathS[0] == "hydraJobs" ? "derivation" : "package"; if (description && !description->empty()) { + + // Takes a string and returns the # of characters displayed + auto columnLengthOfString = [](std::string_view s) -> unsigned int { + unsigned int columnCount = 0; + for (auto i = s.begin(); i < s.end();) { + // Test first character to determine if it is one of + // treeConn, treeLast, treeLine + if (*i == -30) { + i += 3; + ++columnCount; + } + // Escape sequences + // https://en.wikipedia.org/wiki/ANSI_escape_code + else if (*i == '\e') { + // Eat '[' + if (*(++i) == '[') { + ++i; + // Eat parameter bytes + while(*i >= 0x30 && *i <= 0x3f) ++i; + + // Eat intermediate bytes + while(*i >= 0x20 && *i <= 0x2f) ++i; + + // Eat final byte + if(*i >= 0x40 && *i <= 0x73) ++i; + } + else { + // Eat Fe Escape sequence + if (*i >= 0x40 && *i <= 0x5f) ++i; + } + } + else { + ++i; + ++columnCount; + } + } + + return columnCount; + }; + // Maximum length to print size_t maxLength = getWindowSize().second > 0 ? getWindowSize().second : 80; @@ -1271,29 +1311,25 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto newLinePos = trimmed.find('\n'); auto length = newLinePos != std::string::npos ? newLinePos : trimmed.length(); - // Sanitize the description and calculate the two parts of the line - // In order to get the length of the printable characters we need to - // filter out escape sequences. auto beginningOfLine = fmt("%s: %s '%s'", headerPrefix, type, name); - auto beginningOfLineLength = filterANSIEscapes(beginningOfLine, true).length(); - auto restOfLine = fmt(" - '%s'", filterANSIEscapes(trimmed, false, length)); + auto line = fmt("%s: %s '%s' - '%s'", headerPrefix, type, name, trimmed.substr(0, length)); // If we are already over the maximum length then do not trim // and don't print the description (preserves existing behavior) - if (beginningOfLineLength >= maxLength) { + if (columnLengthOfString(beginningOfLine) >= maxLength) { logger->cout("%s", beginningOfLine); } + // If the entire line fits then print that + else if (columnLengthOfString(line) < maxLength) { + logger->cout("%s", line); + } + // Otherwise we need to truncate else { - auto line = beginningOfLine + restOfLine; - // FIXME: Specifying `true` here gives the correct length - // BUT removes colors/bold so something is not quite right here. - line = filterANSIEscapes(line, true, maxLength); - - // NOTE: This test might be incorrect since I get things like: - // 168 or 161 > maxLength. - if (line.length() > maxLength) { - line = line.replace(line.length() - 3, 3, "..."); - } + auto lineLength = columnLengthOfString(line); + auto chopOff = lineLength - maxLength; + line.resize(line.length() - chopOff); + line = line.replace(line.length() - 3, 3, "..."); + logger->cout("%s", line); } } From f22bf867eb41a98441ab44b3667d3abb146ee603 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Mon, 12 Aug 2024 22:18:14 -0400 Subject: [PATCH 215/284] fix: use SymbolStr in constructor --- src/libexpr/value.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 257da1d2d69..39144fe15ed 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -325,9 +325,9 @@ public: void mkStringMove(const char * s, const NixStringContext & context); - inline void mkString(const Symbol & s) + inline void mkString(const SymbolStr & s) { - mkString(((const std::string &) s).c_str()); + mkString(s.c_str()); } void mkPath(const SourcePath & path); From 95fe9f5ba18bb5fe025a06cbba0e18740ad101dd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 9 Jul 2024 14:46:26 -0400 Subject: [PATCH 216/284] Fix Meson installation of the Nix CLI Co-Authored-By: Qyriad --- src/nix/meson.build | 52 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/nix/meson.build b/src/nix/meson.build index 05bee6a8701..798c98e3367 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -192,3 +192,55 @@ this_exe = executable( link_args: linker_export_flags, install : true, ) + +meson.override_find_program('nix', this_exe) + +nix_symlinks = [ + 'nix-build', + 'nix-channel', + 'nix-collect-garbage', + 'nix-copy-closure', + 'nix-daemon', + 'nix-env', + 'nix-hash', + 'nix-instantiate', + 'nix-prefetch-url', + 'nix-shell', + 'nix-store', +] + +foreach linkname : nix_symlinks + install_symlink( + linkname, + # TODO(Qyriad): should these continue to be relative symlinks? + pointing_to : 'nix', + install_dir : get_option('bindir'), + # The 'runtime' tag is what executables default to, which we want to emulate here. + install_tag : 'runtime' + ) + t = custom_target( + command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'], + output: linkname, + # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working) + build_by_default: true + ) + # TODO(Ericson3214): Dosen't yet work + #meson.override_find_program(linkname, t) +endforeach + +install_symlink( + 'build-remote', + pointing_to : '..' / '..'/ get_option('bindir') / 'nix', + install_dir : get_option('libexecdir') / 'nix', + # The 'runtime' tag is what executables default to, which we want to emulate here. + install_tag : 'runtime' +) + +custom_target( + command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'], + output: 'build-remote', + # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working) + build_by_default: true +) +# TODO(Ericson3214): Dosen't yet work +#meson.override_find_program(linkname, t) From 4956e7c44c5018c2c73a125913e264558cd638d6 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 13 Aug 2024 19:22:32 +0200 Subject: [PATCH 217/284] add cross-references to `nix-path` overriding (#11288) * add cross-references to `nix-path` overriding while this information is already present in the settings, it's more likely to be first accessed through the "lookup path" page, which currently requires following two links to get to the practically important bits. Co-authored-by: Robert Hensing --- doc/manual/src/language/constructs/lookup-path.md | 7 ++----- src/libexpr/primops.cc | 5 ++++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md index 11b9fe88c2a..a2e80280b8a 100644 --- a/doc/manual/src/language/constructs/lookup-path.md +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -4,11 +4,8 @@ > > *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>` -A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry. - -The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath). - -See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution. +A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry in [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath). +The algorithm for lookup path resolution is described in the documentation on [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile). > **Example** > diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 2de6cbf4b8b..9de8ff599eb 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4857,7 +4857,10 @@ void EvalState::createBaseEnv() addConstant("__nixPath", v, { .type = nList, .doc = R"( - The value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path): a list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). + A list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). + Its value is primarily determined by the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path), which are + - Overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment variable or the `--nix-path` option + - Extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path` > **Example** > From 77d84a8d8b82864fcd5dd7de6295ab3fc5f8e120 Mon Sep 17 00:00:00 2001 From: Noam Yorav-Raphael Date: Wed, 14 Aug 2024 14:35:42 +0300 Subject: [PATCH 218/284] /homeless-shelter -> /proc/homeless/shelter This makes it so even root can't create $HOME, for example by running `mkdir -p $HOME/.cache/foo`. --- doc/manual/src/language/derivations.md | 2 +- src/libstore/unix/build/local-derivation-goal.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 8e3f0f79174..c78c7b8c87d 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -264,7 +264,7 @@ The [`builder`](#attr-builder) is executed as follows: - `PATH` is set to `/path-not-set` to prevent shells from initialising it to their built-in default value. - - `HOME` is set to `/homeless-shelter` to prevent programs from + - `HOME` is set to `/proc/homeless-shelter` to prevent programs from using `/etc/passwd` or the like to find the user's home directory, which could cause impurity. Usually, when `HOME` is set, it is used as the location of the home directory, even if diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index d3482df17a1..f6bbba8b443 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -102,7 +102,7 @@ void handleDiffHook( } } -const Path LocalDerivationGoal::homeDir = "/homeless-shelter"; +const Path LocalDerivationGoal::homeDir = "/proc/homeless-shelter"; LocalDerivationGoal::~LocalDerivationGoal() From 612fc76020495cefee6a5f5259fd83a531b0ef31 Mon Sep 17 00:00:00 2001 From: bryango Date: Wed, 14 Aug 2024 20:27:12 +0800 Subject: [PATCH 219/284] doc/manual: fix misaligned icons in custom.css (#11296) --- doc/manual/custom.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/custom.css b/doc/manual/custom.css index 9e8e3886f6e..7af150be391 100644 --- a/doc/manual/custom.css +++ b/doc/manual/custom.css @@ -12,8 +12,8 @@ h1.menu-title::before { } -h1.menu-title { - padding: 0.5em; +.menu-bar { + padding: 0.5em 0em; } .sidebar .sidebar-scrollbox { From cc9fe4dee713717c88b22c8701f9bdffc786baa1 Mon Sep 17 00:00:00 2001 From: Bryan Honof Date: Fri, 2 Aug 2024 00:10:08 +0200 Subject: [PATCH 220/284] Fix a few shellcheck tests Ref nixos/nix#10795 --- maintainers/flake-module.nix | 11 --- tests/functional/signing.sh | 94 +++++++++---------- tests/functional/simple.builder.sh | 7 +- tests/functional/simple.sh | 10 +- tests/functional/ssh-relay.sh | 8 +- tests/functional/store-info.sh | 4 +- tests/functional/structured-attrs.sh | 11 ++- .../functional/substitute-with-invalid-ca.sh | 8 +- tests/functional/suggestions.sh | 2 +- tests/functional/tarball.sh | 44 ++++----- tests/functional/test-infra.sh | 5 +- tests/functional/test-libstoreconsumer.sh | 2 +- tests/functional/timeout.sh | 2 +- 13 files changed, 102 insertions(+), 106 deletions(-) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index c3eaf671c1a..0b83e56962d 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -641,19 +641,8 @@ ''^tests/functional/selfref-gc\.sh$'' ''^tests/functional/shell\.sh$'' ''^tests/functional/shell\.shebang\.sh$'' - ''^tests/functional/signing\.sh$'' ''^tests/functional/simple\.builder\.sh$'' - ''^tests/functional/simple\.sh$'' - ''^tests/functional/ssh-relay\.sh$'' - ''^tests/functional/store-info\.sh$'' - ''^tests/functional/structured-attrs\.sh$'' - ''^tests/functional/substitute-with-invalid-ca\.sh$'' - ''^tests/functional/suggestions\.sh$'' ''^tests/functional/supplementary-groups\.sh$'' - ''^tests/functional/tarball\.sh$'' - ''^tests/functional/test-infra\.sh$'' - ''^tests/functional/test-libstoreconsumer\.sh$'' - ''^tests/functional/timeout\.sh$'' ''^tests/functional/toString-path\.sh$'' ''^tests/functional/user-envs-migration\.sh$'' ''^tests/functional/user-envs-test-case\.sh$'' diff --git a/tests/functional/signing.sh b/tests/functional/signing.sh index 890d1446f4f..8ec093a4803 100755 --- a/tests/functional/signing.sh +++ b/tests/functional/signing.sh @@ -5,108 +5,108 @@ source common.sh clearStoreIfPossible clearCache -nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 -pk1=$(cat $TEST_ROOT/pk1) -nix-store --generate-binary-cache-key cache2.example.org $TEST_ROOT/sk2 $TEST_ROOT/pk2 -pk2=$(cat $TEST_ROOT/pk2) +nix-store --generate-binary-cache-key cache1.example.org "$TEST_ROOT"/sk1 "$TEST_ROOT"/pk1 +pk1=$(cat "$TEST_ROOT"/pk1) +nix-store --generate-binary-cache-key cache2.example.org "$TEST_ROOT"/sk2 "$TEST_ROOT"/pk2 +pk2=$(cat "$TEST_ROOT"/pk2) # Build a path. outPath=$(nix-build dependencies.nix --no-out-link --secret-key-files "$TEST_ROOT/sk1 $TEST_ROOT/sk2") # Verify that the path got signed. -info=$(nix path-info --json $outPath) -echo $info | jq -e '.[] | .ultimate == true' +info=$(nix path-info --json "$outPath") +echo "$info" | jq -e '.[] | .ultimate == true' TODO_NixOS # looks like an actual bug? Following line fails on NixOS: -echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' -echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' # Test "nix store verify". -nix store verify -r $outPath +nix store verify -r "$outPath" -expect 2 nix store verify -r $outPath --sigs-needed 1 +expect 2 nix store verify -r "$outPath" --sigs-needed 1 -nix store verify -r $outPath --sigs-needed 1 --trusted-public-keys $pk1 +nix store verify -r "$outPath" --sigs-needed 1 --trusted-public-keys "$pk1" -expect 2 nix store verify -r $outPath --sigs-needed 2 --trusted-public-keys $pk1 +expect 2 nix store verify -r "$outPath" --sigs-needed 2 --trusted-public-keys "$pk1" -nix store verify -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" +nix store verify -r "$outPath" --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" nix store verify --all --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" # Build something unsigned. outPath2=$(nix-build simple.nix --no-out-link) -nix store verify -r $outPath +nix store verify -r "$outPath" # Verify that the path did not get signed but does have the ultimate bit. -info=$(nix path-info --json $outPath2) -echo $info | jq -e '.[] | .ultimate == true' -echo $info | jq -e '.[] | .signatures == []' +info=$(nix path-info --json "$outPath2") +echo "$info" | jq -e '.[] | .ultimate == true' +echo "$info" | jq -e '.[] | .signatures == []' # Test "nix store verify". -nix store verify -r $outPath2 +nix store verify -r "$outPath2" -expect 2 nix store verify -r $outPath2 --sigs-needed 1 +expect 2 nix store verify -r "$outPath2" --sigs-needed 1 -expect 2 nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 +expect 2 nix store verify -r "$outPath2" --sigs-needed 1 --trusted-public-keys "$pk1" # Test "nix store sign". -nix store sign --key-file $TEST_ROOT/sk1 $outPath2 +nix store sign --key-file "$TEST_ROOT"/sk1 "$outPath2" -nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 +nix store verify -r "$outPath2" --sigs-needed 1 --trusted-public-keys "$pk1" # Build something content-addressed. outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no-out-link) -nix path-info --json $outPathCA | jq -e '.[] | .ca | startswith("fixed:md5:")' +nix path-info --json "$outPathCA" | jq -e '.[] | .ca | startswith("fixed:md5:")' # Content-addressed paths don't need signatures, so they verify # regardless of --sigs-needed. -nix store verify $outPathCA -nix store verify $outPathCA --sigs-needed 1000 +nix store verify "$outPathCA" +nix store verify "$outPathCA" --sigs-needed 1000 # Check that signing a content-addressed path doesn't overflow validSigs -nix store sign --key-file $TEST_ROOT/sk1 $outPathCA -nix store verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1 +nix store sign --key-file "$TEST_ROOT"/sk1 "$outPathCA" +nix store verify -r "$outPathCA" --sigs-needed 1000 --trusted-public-keys "$pk1" # Copy to a binary cache. -nix copy --to file://$cacheDir $outPath2 +nix copy --to file://"$cacheDir" "$outPath2" # Verify that signatures got copied. -info=$(nix path-info --store file://$cacheDir --json $outPath2) -echo $info | jq -e '.[] | .ultimate == false' -echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' -echo $info | expect 4 jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' +info=$(nix path-info --store file://"$cacheDir" --json "$outPath2") +echo "$info" | jq -e '.[] | .ultimate == false' +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' +echo "$info" | expect 4 jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' # Verify that adding a signature to a path in a binary cache works. -nix store sign --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2 -info=$(nix path-info --store file://$cacheDir --json $outPath2) -echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' -echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' +nix store sign --store file://"$cacheDir" --key-file "$TEST_ROOT"/sk2 "$outPath2" +info=$(nix path-info --store file://"$cacheDir" --json "$outPath2") +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' # Copying to a diverted store should fail due to a lack of signatures by trusted keys. -chmod -R u+w $TEST_ROOT/store0 || true -rm -rf $TEST_ROOT/store0 +chmod -R u+w "$TEST_ROOT"/store0 || true +rm -rf "$TEST_ROOT"/store0 # Fails or very flaky only on GHA + macOS: # expectStderr 1 nix copy --to $TEST_ROOT/store0 $outPath | grepQuiet -E 'cannot add path .* because it lacks a signature by a trusted key' # but this works: -(! nix copy --to $TEST_ROOT/store0 $outPath) +(! nix copy --to "$TEST_ROOT"/store0 "$outPath") # But succeed if we supply the public keys. -nix copy --to $TEST_ROOT/store0 $outPath --trusted-public-keys $pk1 +nix copy --to "$TEST_ROOT"/store0 "$outPath" --trusted-public-keys "$pk1" -expect 2 nix store verify --store $TEST_ROOT/store0 -r $outPath +expect 2 nix store verify --store "$TEST_ROOT"/store0 -r "$outPath" -nix store verify --store $TEST_ROOT/store0 -r $outPath --trusted-public-keys $pk1 -nix store verify --store $TEST_ROOT/store0 -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" +nix store verify --store "$TEST_ROOT"/store0 -r "$outPath" --trusted-public-keys "$pk1" +nix store verify --store "$TEST_ROOT"/store0 -r "$outPath" --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" # It should also succeed if we disable signature checking. -(! nix copy --to $TEST_ROOT/store0 $outPath2) -nix copy --to $TEST_ROOT/store0?require-sigs=false $outPath2 +(! nix copy --to "$TEST_ROOT"/store0 "$outPath2") +nix copy --to "$TEST_ROOT"/store0?require-sigs=false "$outPath2" # But signatures should still get copied. -nix store verify --store $TEST_ROOT/store0 -r $outPath2 --trusted-public-keys $pk1 +nix store verify --store "$TEST_ROOT"/store0 -r "$outPath2" --trusted-public-keys "$pk1" # Content-addressed stuff can be copied without signatures. -nix copy --to $TEST_ROOT/store0 $outPathCA +nix copy --to "$TEST_ROOT"/store0 "$outPathCA" diff --git a/tests/functional/simple.builder.sh b/tests/functional/simple.builder.sh index 569e8ca88c1..97abf06763d 100644 --- a/tests/functional/simple.builder.sh +++ b/tests/functional/simple.builder.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + echo "PATH=$PATH" # Verify that the PATH is empty. @@ -5,7 +7,6 @@ if mkdir foo 2> /dev/null; then exit 1; fi # Set a PATH (!!! impure). export PATH=$goodPath +mkdir "$out" -mkdir $out - -echo "Hello World!" > $out/hello \ No newline at end of file +echo "Hello World!" > "$out"/hello diff --git a/tests/functional/simple.sh b/tests/functional/simple.sh index 86acca0c2d1..8afa369c2e2 100755 --- a/tests/functional/simple.sh +++ b/tests/functional/simple.sh @@ -12,7 +12,7 @@ outPath=$(nix-store -rvv "$drvPath") echo "output path is $outPath" -(! [ -w $outPath ]) +[[ ! -w $outPath ]] text=$(cat "$outPath/hello") if test "$text" != "Hello World!"; then exit 1; fi @@ -21,16 +21,16 @@ TODO_NixOS # Directed delete: $outPath is not reachable from a root, so it should # be deleteable. -nix-store --delete $outPath -(! [ -e $outPath/hello ]) +nix-store --delete "$outPath" +[[ ! -e $outPath/hello ]] -outPath="$(NIX_REMOTE=local?store=/foo\&real=$TEST_ROOT/real-store nix-instantiate --readonly-mode hash-check.nix)" +outPath="$(NIX_REMOTE='local?store=/foo&real='"$TEST_ROOT"'/real-store' nix-instantiate --readonly-mode hash-check.nix)" if test "$outPath" != "/foo/lfy1s6ca46rm5r6w4gg9hc0axiakjcnm-dependencies.drv"; then echo "hashDerivationModulo appears broken, got $outPath" exit 1 fi -outPath="$(NIX_REMOTE=local?store=/foo\&real=$TEST_ROOT/real-store nix-instantiate --readonly-mode big-derivation-attr.nix)" +outPath="$(NIX_REMOTE='local?store=/foo&real='"$TEST_ROOT"'/real-store' nix-instantiate --readonly-mode big-derivation-attr.nix)" if test "$outPath" != "/foo/xxiwa5zlaajv6xdjynf9yym9g319d6mn-big-derivation-attr.drv"; then echo "big-derivation-attr.nix hash appears broken, got $outPath. Memory corruption in large drv attr?" exit 1 diff --git a/tests/functional/ssh-relay.sh b/tests/functional/ssh-relay.sh index 059c664348a..71b8ae9abe1 100755 --- a/tests/functional/ssh-relay.sh +++ b/tests/functional/ssh-relay.sh @@ -2,10 +2,10 @@ source common.sh -echo foo > $TEST_ROOT/hello.sh +echo foo > "$TEST_ROOT"/hello.sh ssh_localhost=ssh://localhost -remote_store=?remote-store=$ssh_localhost +remote_store="?remote-store=$ssh_localhost" store=$ssh_localhost @@ -13,6 +13,6 @@ store+=$remote_store store+=$remote_store store+=$remote_store -out=$(nix store add-path --store "$store" $TEST_ROOT/hello.sh) +out=$(nix store add-path --store "$store" "$TEST_ROOT"/hello.sh) -[ foo = $(< $out) ] +[ foo = "$(< "$out")" ] diff --git a/tests/functional/store-info.sh b/tests/functional/store-info.sh index f37889fbb1e..beecc2dd9dc 100755 --- a/tests/functional/store-info.sh +++ b/tests/functional/store-info.sh @@ -8,12 +8,12 @@ STORE_INFO_JSON=$(nix store info --json) echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then - DAEMON_VERSION=$($NIX_DAEMON_PACKAGE/bin/nix daemon --version | cut -d' ' -f3) + DAEMON_VERSION=$("$NIX_DAEMON_PACKAGE"/bin/nix daemon --version | cut -d' ' -f3) echo "$STORE_INFO" | grep "Version: $DAEMON_VERSION" [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]] fi -expect 127 NIX_REMOTE=unix:$PWD/store nix store info || \ +expect 127 NIX_REMOTE=unix:"$PWD"/store nix store info || \ fail "nix store info on a non-existent store should fail" TODO_NixOS diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index ec1282668fd..64d136e993a 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -8,17 +8,19 @@ requireDaemonNewerThan "2.4pre20210712" clearStoreIfPossible -rm -f $TEST_ROOT/result +rm -f "$TEST_ROOT"/result -nix-build structured-attrs.nix -A all -o $TEST_ROOT/result +nix-build structured-attrs.nix -A all -o "$TEST_ROOT"/result -[[ $(cat $TEST_ROOT/result/foo) = bar ]] -[[ $(cat $TEST_ROOT/result-dev/foo) = foo ]] +[[ $(cat "$TEST_ROOT"/result/foo) = bar ]] +[[ $(cat "$TEST_ROOT"/result-dev/foo) = foo ]] export NIX_BUILD_SHELL=$SHELL +# shellcheck disable=SC2016 env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ --run 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' +# shellcheck disable=SC2016 nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' TODO_NixOS # following line fails. @@ -26,6 +28,7 @@ TODO_NixOS # following line fails. # `nix develop` is a slightly special way of dealing with environment vars, it parses # these from a shell-file exported from a derivation. This is to test especially `outputs` # (which is an associative array in thsi case) being fine. +# shellcheck disable=SC2016 nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"' nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_JSON_FILE=' diff --git a/tests/functional/substitute-with-invalid-ca.sh b/tests/functional/substitute-with-invalid-ca.sh index d8af67237b8..33432e95da9 100755 --- a/tests/functional/substitute-with-invalid-ca.sh +++ b/tests/functional/substitute-with-invalid-ca.sh @@ -11,16 +11,16 @@ getRemoteNarInfo () { echo "$cacheDir/$(getHash "$1").narinfo" } -cat < $TEST_HOME/good.txt +cat < "$TEST_HOME"/good.txt I’m a good path EOF -cat < $TEST_HOME/bad.txt +cat < "$TEST_HOME"/bad.txt I’m a bad path EOF -good=$(nix-store --add $TEST_HOME/good.txt) -bad=$(nix-store --add $TEST_HOME/bad.txt) +good=$(nix-store --add "$TEST_HOME"/good.txt) +bad=$(nix-store --add "$TEST_HOME"/bad.txt) nix copy --to "$BINARY_CACHE" "$good" nix copy --to "$BINARY_CACHE" "$bad" nix-collect-garbage >/dev/null 2>&1 diff --git a/tests/functional/suggestions.sh b/tests/functional/suggestions.sh index 8db6f7b97f3..fbca93da859 100755 --- a/tests/functional/suggestions.sh +++ b/tests/functional/suggestions.sh @@ -37,7 +37,7 @@ NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION=$(! nix build .\#bar 2>&1 1>/dev/null) [[ ! "$NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION" =~ "Did you mean" ]] || \ fail "The nix build stderr shouldn’t suggest anything if there’s nothing relevant to suggest" -NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '(builtins.getFlake (builtins.toPath ./.)).packages.'$system'.fob' 2>&1 1>/dev/null) +NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '(builtins.getFlake (builtins.toPath ./.)).packages.'"$system"'.fob' 2>&1 1>/dev/null) [[ "$NIX_EVAL_STDERR_WITH_SUGGESTIONS" =~ "Did you mean one of fo1, fo2, foo or fooo?" ]] || \ fail "The evaluator should suggest the three closest possiblities" diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index 4d89456255f..dee0a98f1cf 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -4,51 +4,51 @@ source common.sh clearStoreIfPossible -rm -rf $TEST_HOME +rm -rf "$TEST_HOME" tarroot=$TEST_ROOT/tarball -rm -rf $tarroot -mkdir -p $tarroot -cp dependencies.nix $tarroot/default.nix -cp config.nix dependencies.builder*.sh $tarroot/ -touch -d '@1000000000' $tarroot $tarroot/* +rm -rf "$tarroot" +mkdir -p "$tarroot" +cp dependencies.nix "$tarroot/default.nix" +cp config.nix dependencies.builder*.sh "$tarroot/" +touch -d '@1000000000' "$tarroot" "$tarroot"/* -hash=$(nix hash path $tarroot) +hash=$(nix hash path "$tarroot") test_tarball() { local ext="$1" local compressor="$2" tarball=$TEST_ROOT/tarball.tar$ext - (cd $TEST_ROOT && GNUTAR_REPRODUCIBLE= tar --mtime=$tarroot/default.nix --owner=0 --group=0 --numeric-owner --sort=name -c -f - tarball) | $compressor > $tarball + (cd "$TEST_ROOT" && GNUTAR_REPRODUCIBLE=1 tar --mtime="$tarroot"/default.nix --owner=0 --group=0 --numeric-owner --sort=name -c -f - tarball) | $compressor > "$tarball" - nix-env -f file://$tarball -qa --out-path | grepQuiet dependencies + nix-env -f file://"$tarball" -qa --out-path | grepQuiet dependencies - nix-build -o $TEST_ROOT/result file://$tarball + nix-build -o "$TEST_ROOT"/result file://"$tarball" - nix-build -o $TEST_ROOT/result '' -I foo=file://$tarball + nix-build -o "$TEST_ROOT"/result '' -I foo=file://"$tarball" - nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTarball file://$tarball)" # Do not re-fetch paths already present - nix-build -o $TEST_ROOT/result -E "import (fetchTarball { url = file:///does-not-exist/must-remain-unused/$tarball; sha256 = \"$hash\"; })" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTarball { url = file:///does-not-exist/must-remain-unused/$tarball; sha256 = \"$hash\"; })" - nix-build -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)" - nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" - nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree file://$tarball)" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" # Do not re-fetch paths already present - nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file:///does-not-exist/must-remain-unused/$tarball; narHash = \"$hash\"; })" - expectStderr 102 nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" | grep 'NAR hash mismatch in input' + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file:///does-not-exist/must-remain-unused/$tarball; narHash = \"$hash\"; })" + expectStderr 102 nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" | grep 'NAR hash mismatch in input' [[ $(nix eval --impure --expr "(fetchTree file://$tarball).lastModified") = 1000000000 ]] nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2 nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true' - nix-instantiate --eval -E '1 + 2' -I fnord=file:///no-such-tarball.tar$ext - nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file:///no-such-tarball$ext - (! nix-instantiate --eval -E ' 1' -I fnord=file:///no-such-tarball$ext) + nix-instantiate --eval -E '1 + 2' -I fnord=file:///no-such-tarball.tar"$ext" + nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file:///no-such-tarball"$ext" + (! nix-instantiate --eval -E ' 1' -I fnord=file:///no-such-tarball"$ext") - nix-instantiate --eval -E '' -I fnord=file:///no-such-tarball$ext -I fnord=. + nix-instantiate --eval -E '' -I fnord=file:///no-such-tarball"$ext" -I fnord=. # Ensure that the `name` attribute isn’t accepted as that would mess # with the content-addressing diff --git a/tests/functional/test-infra.sh b/tests/functional/test-infra.sh index d02a11b4641..2da26b08ccd 100755 --- a/tests/functional/test-infra.sh +++ b/tests/functional/test-infra.sh @@ -14,7 +14,7 @@ expect 1 false expect 1 expect 0 false function ret() { - return $1 + return "$1" } # `expect` can call functions, not just executables @@ -48,6 +48,7 @@ expectStderr 1 noisyFalse | grepQuiet NAY # `set -o pipefile` is enabled +# shellcheck disable=SC2317# shellcheck disable=SC2317 pipefailure () { # shellcheck disable=SC2216 true | false | true @@ -55,6 +56,7 @@ pipefailure () { expect 1 pipefailure unset pipefailure +# shellcheck disable=SC2317 pipefailure () { # shellcheck disable=SC2216 false | true | true @@ -82,6 +84,7 @@ expect 1 useUnbound # ! alone unfortunately negates `set -e`, but it works in functions: # shellcheck disable=SC2251 ! true +# shellcheck disable=SC2317 funBang () { ! true } diff --git a/tests/functional/test-libstoreconsumer.sh b/tests/functional/test-libstoreconsumer.sh index d1a1accb607..2adead1c021 100755 --- a/tests/functional/test-libstoreconsumer.sh +++ b/tests/functional/test-libstoreconsumer.sh @@ -5,4 +5,4 @@ source common.sh drv="$(nix-instantiate simple.nix)" cat "$drv" out="$(./test-libstoreconsumer/test-libstoreconsumer "$drv")" -cat "$out/hello" | grep -F "Hello World!" +grep -F "Hello World!" < "$out/hello" diff --git a/tests/functional/timeout.sh b/tests/functional/timeout.sh index f4235453816..ae47fdc9684 100755 --- a/tests/functional/timeout.sh +++ b/tests/functional/timeout.sh @@ -9,7 +9,7 @@ needLocalStore "see #4813" messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) && status=0 || status=$? -if [ $status -ne 101 ]; then +if [ "$status" -ne 101 ]; then echo "error: 'nix-store' exited with '$status'; should have exited 101" # FIXME: https://github.com/NixOS/nix/issues/4813 From 93f58150c9cd5f2fc9345c2730ce8a6da47dc474 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 13 Aug 2024 16:15:56 -0400 Subject: [PATCH 221/284] Coarse versions for constituent packages As discussed in our meeting, we should use a simplified version for the libraries without the date or commit hash. This will make rebuilding a lot faster in many cases. Progress on #10379 Co-Authored-By: Robert Hensing --- flake.nix | 19 +++++++++---------- package.nix | 5 ++--- packaging/components.nix | 31 +++++++++++++++++++++++++++---- packaging/dependencies.nix | 5 +---- packaging/hydra.nix | 9 ++++++++- 5 files changed, 47 insertions(+), 22 deletions(-) diff --git a/flake.nix b/flake.nix index 9e8592e3a6d..638f6b4bda8 100644 --- a/flake.nix +++ b/flake.nix @@ -26,12 +26,6 @@ officialRelease = false; - version = lib.fileContents ./.version + versionSuffix; - versionSuffix = - if officialRelease - then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - linux32BitSystems = [ "i686-linux" ]; linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ]; linuxSystems = linux32BitSystems ++ linux64BitSystems; @@ -130,12 +124,16 @@ # without "polluting" the top level "`pkgs`" attrset. # This also has the benefit of providing us with a distinct set of packages # we can iterate over. - nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix); + nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix { + inherit (final) lib; + inherit officialRelease; + src = self; + }); # The dependencies are in their own scope, so that they don't have to be # in Nixpkgs top level `pkgs` or `nixComponents`. nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { - inherit inputs stdenv versionSuffix; + inherit inputs stdenv; pkgs = final; }); @@ -170,6 +168,7 @@ linux64BitSystems nixpkgsFor self + officialRelease ; }; @@ -253,10 +252,10 @@ dockerImage = let pkgs = nixpkgsFor.${system}.native; - image = import ./docker.nix { inherit pkgs; tag = version; }; + image = import ./docker.nix { inherit pkgs; tag = pkgs.nix.version; }; in pkgs.runCommand - "docker-image-tarball-${version}" + "docker-image-tarball-${pkgs.nix.version}" { meta.description = "Docker image with Nix for ${system}"; } '' mkdir -p $out/nix-support diff --git a/package.nix b/package.nix index a7c8923e8b4..d41748b7cf5 100644 --- a/package.nix +++ b/package.nix @@ -47,7 +47,8 @@ , pname ? "nix" -, versionSuffix ? "" +, version +, versionSuffix # Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix , doBuild ? true @@ -112,8 +113,6 @@ let inherit (lib) fileset; - version = lib.fileContents ./.version + versionSuffix; - # selected attributes with defaults, will be used to define some # things which should instead be gotten via `finalAttrs` in order to # work with overriding. diff --git a/packaging/components.nix b/packaging/components.nix index 870e9ae615f..0e8334d1005 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -1,11 +1,34 @@ +{ + lib, + src, + officialRelease, +}: + scope: + let inherit (scope) callPackage; + + baseVersion = lib.fileContents ../.version; + + versionSuffix = lib.optionalString (!officialRelease) "pre"; + + fineVersionSuffix = lib.optionalString + (!officialRelease) + "pre${builtins.substring 0 8 (src.lastModifiedDate or src.lastModified or "19700101")}_${src.shortRev or "dirty"}"; + + fineVersion = baseVersion + fineVersionSuffix; in # This becomes the pkgs.nixComponents attribute set { - nix = callPackage ../package.nix { }; + version = baseVersion + versionSuffix; + inherit versionSuffix; + + nix = callPackage ../package.nix { + version = fineVersion; + versionSuffix = fineVersionSuffix; + }; nix-util = callPackage ../src/libutil/package.nix { }; nix-util-c = callPackage ../src/libutil-c/package.nix { }; @@ -34,10 +57,10 @@ in nix-cmd = callPackage ../src/libcmd/package.nix { }; # Will replace `nix` once the old build system is gone. - nix-ng = callPackage ../src/nix/package.nix { }; + nix-ng = callPackage ../src/nix/package.nix { version = fineVersion; }; - nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { }; - nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { }; + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { version = fineVersion; }; + nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { version = fineVersion; }; nix-perl-bindings = callPackage ../src/perl/package.nix { }; } diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index e0737593fd4..21c48e5cc82 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -8,7 +8,6 @@ pkgs, stdenv, - versionSuffix, }: let @@ -73,11 +72,9 @@ let strictDeps = prevAttrs.strictDeps or true; enableParallelBuilding = true; }; - in scope: { - inherit stdenv versionSuffix; - version = lib.fileContents ../.version + versionSuffix; + inherit stdenv; aws-sdk-cpp = (pkgs.aws-sdk-cpp.override { apis = [ "s3" "transfer" ]; diff --git a/packaging/hydra.nix b/packaging/hydra.nix index dbe99247675..9752d90e3b1 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -6,6 +6,7 @@ , linux64BitSystems , nixpkgsFor , self +, officialRelease }: let inherit (inputs) nixpkgs nixpkgs-regression; @@ -16,7 +17,7 @@ let }; testNixVersions = pkgs: client: daemon: - pkgs.callPackage ../package.nix { + pkgs.nixComponents.callPackage ../package.nix { pname = "nix-tests" + lib.optionalString @@ -28,6 +29,12 @@ let test-daemon = daemon; doBuild = false; + + # This could be more accurate, but a shorter version will match the + # fine version with rev. This functionality is already covered in + # the normal test, so it's fine. + version = pkgs.nixComponents.version; + versionSuffix = pkgs.nixComponents.versionSuffix; }; # Technically we could just return `pkgs.nixComponents`, but for Hydra it's From 34fe2478a2b26569e5e0b244e927d3b50da0056d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 Jul 2024 16:07:06 -0400 Subject: [PATCH 222/284] Build Functional tests with Meson Co-Authored-By: Qyriad Co-authored-by: Robert Hensing --- flake.nix | 4 + meson.build | 1 + packaging/components.nix | 2 + packaging/hydra.nix | 7 +- src/libstore/meson.build | 2 +- src/nix-functional-tests | 1 + tests/functional/.version | 1 + tests/functional/ca/meson.build | 33 +++ tests/functional/common.sh | 3 +- .../{vars-and-functions.sh => functions.sh} | 107 ++----- tests/functional/common/init.sh | 4 +- tests/functional/common/meson.build | 5 + tests/functional/common/paths.sh | 19 +- tests/functional/common/subst-vars.sh.in | 20 +- tests/functional/common/vars.sh | 72 +++++ .../derivation-advanced-attributes.sh | 2 +- tests/functional/dyn-drv/meson.build | 19 ++ tests/functional/flakes/meson.build | 28 ++ tests/functional/git-hashing/meson.build | 8 + .../functional/local-overlay-store/common.sh | 3 +- .../local-overlay-store/meson.build | 18 ++ tests/functional/meson.build | 266 ++++++++++++++++++ tests/functional/nested-sandboxing/command.sh | 2 + tests/functional/nested-sandboxing/runner.nix | 7 +- tests/functional/package.nix | 117 ++++++++ tests/functional/plugins.sh | 9 +- tests/functional/plugins/meson.build | 16 ++ tests/functional/restricted.sh | 3 - .../test-libstoreconsumer/meson.build | 14 + 29 files changed, 678 insertions(+), 115 deletions(-) create mode 120000 src/nix-functional-tests create mode 120000 tests/functional/.version create mode 100644 tests/functional/ca/meson.build rename tests/functional/common/{vars-and-functions.sh => functions.sh} (76%) create mode 100644 tests/functional/common/meson.build create mode 100644 tests/functional/common/vars.sh create mode 100644 tests/functional/dyn-drv/meson.build create mode 100644 tests/functional/flakes/meson.build create mode 100644 tests/functional/git-hashing/meson.build create mode 100644 tests/functional/local-overlay-store/meson.build create mode 100644 tests/functional/meson.build create mode 100644 tests/functional/package.nix create mode 100644 tests/functional/plugins/meson.build create mode 100644 tests/functional/test-libstoreconsumer/meson.build diff --git a/flake.nix b/flake.nix index 638f6b4bda8..22ca5411822 100644 --- a/flake.nix +++ b/flake.nix @@ -210,6 +210,9 @@ "${nixpkgsPrefix}${pkgName}-${testName}" = test; }) ) + // lib.optionalAttrs (nixpkgs.stdenv.hostPlatform == nixpkgs.stdenv.buildPlatform) { + "${nixpkgsPrefix}nix-functional-tests" = nixpkgs.nixComponents.nix-functional-tests; + } ) // devFlake.checks.${system} or {} ); @@ -323,6 +326,7 @@ ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-functional-tests.baseNativeBuildInputs ++ lib.optional (!stdenv.buildPlatform.canExecute stdenv.hostPlatform # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 diff --git a/meson.build b/meson.build index 1554244ab66..715a3862d4f 100644 --- a/meson.build +++ b/meson.build @@ -42,3 +42,4 @@ subproject('nix-fetchers-tests') subproject('nix-expr-test-support') subproject('nix-expr-tests') subproject('nix-flake-tests') +subproject('nix-functional-tests') diff --git a/packaging/components.nix b/packaging/components.nix index 0e8334d1005..f14613e8a1d 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -59,6 +59,8 @@ in # Will replace `nix` once the old build system is gone. nix-ng = callPackage ../src/nix/package.nix { version = fineVersion; }; + nix-functional-tests = callPackage ../src/nix-functional-tests/package.nix { version = fineVersion; }; + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { version = fineVersion; }; nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { version = fineVersion; }; diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 9752d90e3b1..46b4ff51d21 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -62,6 +62,7 @@ let "nix-main-c" "nix-cmd" "nix-ng" + "nix-functional-tests" ]; in { @@ -75,8 +76,10 @@ in lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName})); buildCross = forAllPackages (pkgName: - forAllCrossSystems (crossSystem: - lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}))); + # Hack to avoid non-evaling package + (if pkgName == "nix-functional-tests" then lib.flip builtins.removeAttrs ["x86_64-w64-mingw32"] else lib.id) + (forAllCrossSystems (crossSystem: + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName})))); buildNoGc = forAllSystems (system: self.packages.${system}.nix.override { enableGC = false; } diff --git a/src/libstore/meson.build b/src/libstore/meson.build index d2cc235fde3..8e30845e122 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -21,7 +21,7 @@ configdata = configuration_data() # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) -configdata.set_quoted('SYSTEM', host_machine.system()) +configdata.set_quoted('SYSTEM', host_machine.cpu_family() + '-' + host_machine.system()) deps_private_maybe_subproject = [ ] diff --git a/src/nix-functional-tests b/src/nix-functional-tests new file mode 120000 index 00000000000..ed0cdf60bf9 --- /dev/null +++ b/src/nix-functional-tests @@ -0,0 +1 @@ +../tests/functional \ No newline at end of file diff --git a/tests/functional/.version b/tests/functional/.version new file mode 120000 index 00000000000..b7badcd0cc8 --- /dev/null +++ b/tests/functional/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/tests/functional/ca/meson.build b/tests/functional/ca/meson.build new file mode 100644 index 00000000000..f682ab28f02 --- /dev/null +++ b/tests/functional/ca/meson.build @@ -0,0 +1,33 @@ +configure_file( + input : 'config.nix.in', + output : 'config.nix', + configuration : test_confdata, +) + +suites += { + 'name': 'ca', + 'deps': [], + 'tests': [ + 'build-with-garbage-path.sh', + 'build.sh', + 'build-cache.sh', + 'concurrent-builds.sh', + 'derivation-json.sh', + 'duplicate-realisation-in-closure.sh', + 'eval-store.sh', + 'gc.sh', + 'import-derivation.sh', + 'new-build-cmd.sh', + 'nix-copy.sh', + 'nix-run.sh', + 'nix-shell.sh', + 'post-hook.sh', + 'recursive.sh', + 'repl.sh', + 'selfref-gc.sh', + 'signatures.sh', + 'substitute.sh', + 'why-depends.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/common.sh b/tests/functional/common.sh index d038aaf59dd..325fac44c41 100644 --- a/tests/functional/common.sh +++ b/tests/functional/common.sh @@ -8,7 +8,8 @@ COMMON_SH_SOURCED=1 functionalTestsDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")" -source "$functionalTestsDir/common/vars-and-functions.sh" +source "$functionalTestsDir/common/vars.sh" +source "$functionalTestsDir/common/functions.sh" source "$functionalTestsDir/common/init.sh" if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/functions.sh similarity index 76% rename from tests/functional/common/vars-and-functions.sh rename to tests/functional/common/functions.sh index 632c81a82a5..d05fac4e7f8 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/functions.sh @@ -1,10 +1,10 @@ -# NOTE: instances of @variable@ are substituted as defined in /mk/templates.mk +# shellcheck shell=bash set -eu -o pipefail -if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then +if [[ -z "${COMMON_FUNCTIONS_SH_SOURCED-}" ]]; then -COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1 +COMMON_FUNCTIONS_SH_SOURCED=1 isTestOnNixOS() { [[ "${isTestOnNixOS:-}" == 1 ]] @@ -15,64 +15,14 @@ die() { exit 1 } -set +x - -commonDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")" - -source "$commonDir/subst-vars.sh" -# Make sure shellcheck knows all these will be defined by the above generated snippet -: "${bindir?} ${coreutils?} ${dot?} ${SHELL?} ${PAGER?} ${busybox?} ${version?} ${system?} ${BUILD_SHARED_LIBS?}" - -source "$commonDir/paths.sh" -source "$commonDir/test-root.sh" - -test_nix_conf_dir=$TEST_ROOT/etc -test_nix_conf=$test_nix_conf_dir/nix.conf - -export TEST_HOME=$TEST_ROOT/test-home - -if ! isTestOnNixOS; then - export NIX_STORE_DIR - if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then - # Maybe the build directory is symlinked. - export NIX_IGNORE_SYMLINK_STORE=1 - NIX_STORE_DIR=$TEST_ROOT/store - fi - export NIX_LOCALSTATE_DIR=$TEST_ROOT/var - export NIX_LOG_DIR=$TEST_ROOT/var/log/nix - export NIX_STATE_DIR=$TEST_ROOT/var/nix - export NIX_CONF_DIR=$test_nix_conf_dir - export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket - unset NIX_USER_CONF_FILES - export _NIX_TEST_SHARED=$TEST_ROOT/shared - if [[ -n $NIX_STORE ]]; then - export _NIX_TEST_NO_SANDBOX=1 - fi - export _NIX_IN_TEST=$TEST_ROOT/shared - export _NIX_TEST_NO_LSOF=1 - export NIX_REMOTE=${NIX_REMOTE_-} - -fi # ! isTestOnNixOS - -unset NIX_PATH -export HOME=$TEST_HOME -unset XDG_STATE_HOME -unset XDG_DATA_HOME -unset XDG_CONFIG_HOME -unset XDG_CONFIG_DIRS -unset XDG_CACHE_HOME - -export IMPURE_VAR1=foo -export IMPURE_VAR2=bar - -cacheDir=$TEST_ROOT/binary-cache - readLink() { + # TODO fix this + # shellcheck disable=SC2012 ls -l "$1" | sed 's/.*->\ //' } clearProfiles() { - profiles="$HOME"/.local/state/nix/profiles + profiles="$HOME/.local/state/nix/profiles" rm -rf "$profiles" } @@ -105,11 +55,11 @@ doClearStore() { } clearCache() { - rm -rf "$cacheDir" + rm -rf "${cacheDir?}" } clearCacheCache() { - rm -f $TEST_HOME/.cache/nix/binary-cache* + rm -f "$TEST_HOME/.cache/nix/binary-cache"* } startDaemon() { @@ -122,7 +72,7 @@ startDaemon() { return fi # Start the daemon, wait for the socket to appear. - rm -f $NIX_DAEMON_SOCKET_PATH + rm -f "$NIX_DAEMON_SOCKET_PATH" PATH=$DAEMON_PATH nix --extra-experimental-features 'nix-command' daemon & _NIX_TEST_DAEMON_PID=$! export _NIX_TEST_DAEMON_PID @@ -151,14 +101,14 @@ killDaemon() { if [[ "${_NIX_TEST_DAEMON_PID-}" == '' ]]; then return fi - kill $_NIX_TEST_DAEMON_PID + kill "$_NIX_TEST_DAEMON_PID" for i in {0..100}; do - kill -0 $_NIX_TEST_DAEMON_PID 2> /dev/null || break + kill -0 "$_NIX_TEST_DAEMON_PID" 2> /dev/null || break sleep 0.1 done - kill -9 $_NIX_TEST_DAEMON_PID 2> /dev/null || true - wait $_NIX_TEST_DAEMON_PID || true - rm -f $NIX_DAEMON_SOCKET_PATH + kill -9 "$_NIX_TEST_DAEMON_PID" 2> /dev/null || true + wait "$_NIX_TEST_DAEMON_PID" || true + rm -f "$NIX_DAEMON_SOCKET_PATH" # Indicate daemon is stopped unset _NIX_TEST_DAEMON_PID # Restore old nix remote @@ -177,14 +127,11 @@ restartDaemon() { startDaemon } -if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then - _canUseSandbox=1 -fi - isDaemonNewer () { [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0 local requiredVersion="$1" - local daemonVersion=$($NIX_DAEMON_PACKAGE/bin/nix daemon --version | cut -d' ' -f3) + local daemonVersion + daemonVersion=$("$NIX_DAEMON_PACKAGE/bin/nix" daemon --version | cut -d' ' -f3) [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]] } @@ -237,7 +184,7 @@ expect() { shift "$@" && res=0 || res="$?" # also match "negative" codes, which wrap around to >127 - if [[ $res -ne $expected && $res -ne $[256 + expected] ]]; then + if [[ $res -ne $expected && $res -ne $((256 + expected)) ]]; then echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi @@ -252,7 +199,7 @@ expectStderr() { shift "$@" 2>&1 && res=0 || res="$?" # also match "negative" codes, which wrap around to >127 - if [[ $res -ne $expected && $res -ne $[256 + expected] ]]; then + if [[ $res -ne $expected && $res -ne $((256 + expected)) ]]; then echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi @@ -267,7 +214,7 @@ expectStderr() { # error: This error is expected # EOF assertStderr() { - diff -u /dev/stdin <($@ 2>/dev/null 2>&1) + diff -u /dev/stdin <("$@" 2>/dev/null 2>&1) } needLocalStore() { @@ -283,11 +230,9 @@ buggyNeedLocalStore() { enableFeatures() { local features="$1" - sed -i 's/experimental-features .*/& '"$features"'/' "$test_nix_conf_dir"/nix.conf + sed -i 's/experimental-features .*/& '"$features"'/' "${test_nix_conf?}" } -set -x - onError() { set +x echo "$0: test failed at:" >&2 @@ -311,15 +256,15 @@ callerPrefix() { local i file line fn savedFn # Use `caller` for i in $(seq 0 100); do - caller $i > /dev/null || { + caller "$i" > /dev/null || { if [[ -n "${file:-}" ]]; then echo "$file:$line: ${savedFn+in call to $savedFn: }" fi break } - line="$(caller $i | cut -d' ' -f1)" - fn="$(caller $i | cut -d' ' -f2)" - file="$(caller $i | cut -d' ' -f3)" + line="$(caller "$i" | cut -d' ' -f1)" + fn="$(caller "$i" | cut -d' ' -f2)" + file="$(caller "$i" | cut -d' ' -f3)" if [[ $file != "${BASH_SOURCE[0]}" ]]; then echo "$file:$line: ${savedFn+in call to $savedFn: }" return @@ -342,7 +287,7 @@ checkGrepArgs() { for arg in "$@"; do if [[ "$arg" != "${arg//$'\n'/_}" ]]; then echo "$(callerPrefix)newline not allowed in arguments; grep would try each line individually as if connected by an OR operator" >&2 - return -101 + return 155 # = -101 mod 256 fi done } @@ -400,4 +345,4 @@ count() { trap onError ERR -fi # COMMON_VARS_AND_FUNCTIONS_SH_SOURCED +fi # COMMON_FUNCTIONS_SH_SOURCED diff --git a/tests/functional/common/init.sh b/tests/functional/common/init.sh index d33ad5d5744..d849c07340f 100755 --- a/tests/functional/common/init.sh +++ b/tests/functional/common/init.sh @@ -7,10 +7,10 @@ if isTestOnNixOS; then mkdir -p "$test_nix_conf_dir" "$TEST_HOME" - export NIX_USER_CONF_FILES="$test_nix_conf_dir/nix.conf" + export NIX_USER_CONF_FILES="$test_nix_conf" mkdir -p "$test_nix_conf_dir" "$TEST_HOME" ! test -e "$test_nix_conf" - cat > "$test_nix_conf_dir/nix.conf" < "$test_nix_conf" < /dev/null); then + # Maybe the build directory is symlinked. + export NIX_IGNORE_SYMLINK_STORE=1 + NIX_STORE_DIR=$TEST_ROOT/store + fi + export NIX_LOCALSTATE_DIR=$TEST_ROOT/var + export NIX_LOG_DIR=$TEST_ROOT/var/log/nix + export NIX_STATE_DIR=$TEST_ROOT/var/nix + export NIX_CONF_DIR=$test_nix_conf_dir + export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket + unset NIX_USER_CONF_FILES + export _NIX_TEST_SHARED=$TEST_ROOT/shared + if [[ -n $NIX_STORE ]]; then + export _NIX_TEST_NO_SANDBOX=1 + fi + export _NIX_IN_TEST=$TEST_ROOT/shared + export _NIX_TEST_NO_LSOF=1 + export NIX_REMOTE=${NIX_REMOTE_-} + +fi # ! isTestOnNixOS + +unset NIX_PATH +export HOME=$TEST_HOME +unset XDG_STATE_HOME +unset XDG_DATA_HOME +unset XDG_CONFIG_HOME +unset XDG_CONFIG_DIRS +unset XDG_CACHE_HOME + +export IMPURE_VAR1=foo +export IMPURE_VAR2=bar + +# Used in other files +# shellcheck disable=SC2034 +cacheDir=$TEST_ROOT/binary-cache + +if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then + _canUseSandbox=1 +fi + +fi # COMMON_VARS_SH_SOURCED diff --git a/tests/functional/derivation-advanced-attributes.sh b/tests/functional/derivation-advanced-attributes.sh index 6c0c76b4cee..271f17dc69c 100755 --- a/tests/functional/derivation-advanced-attributes.sh +++ b/tests/functional/derivation-advanced-attributes.sh @@ -3,7 +3,7 @@ source common/test-root.sh source common/paths.sh -set -o pipefail +set -eu -o pipefail source characterisation/framework.sh diff --git a/tests/functional/dyn-drv/meson.build b/tests/functional/dyn-drv/meson.build new file mode 100644 index 00000000000..3c671d0133e --- /dev/null +++ b/tests/functional/dyn-drv/meson.build @@ -0,0 +1,19 @@ +configure_file( + input : 'config.nix.in', + output : 'config.nix', + configuration : test_confdata, +) + +suites += { + 'name': 'dyn-drv', + 'deps': [], + 'tests': [ + 'text-hashed-output.sh', + 'recursive-mod-json.sh', + 'build-built-drv.sh', + 'eval-outputOf.sh', + 'dep-built-drv.sh', + 'old-daemon-error-hack.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/flakes/meson.build b/tests/functional/flakes/meson.build new file mode 100644 index 00000000000..8c1afd6ff05 --- /dev/null +++ b/tests/functional/flakes/meson.build @@ -0,0 +1,28 @@ +suites += { + 'name': 'flakes', + 'deps': [], + 'tests': [ + 'flakes.sh', + 'develop.sh', + 'edit.sh', + 'run.sh', + 'mercurial.sh', + 'circular.sh', + 'init.sh', + 'inputs.sh', + 'follow-paths.sh', + 'bundle.sh', + 'check.sh', + 'unlocked-override.sh', + 'absolute-paths.sh', + 'absolute-attr-paths.sh', + 'build-paths.sh', + 'flake-in-submodule.sh', + 'prefetch.sh', + 'eval-cache.sh', + 'search-root.sh', + 'config.sh', + 'show.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/git-hashing/meson.build b/tests/functional/git-hashing/meson.build new file mode 100644 index 00000000000..7486bfb8fcc --- /dev/null +++ b/tests/functional/git-hashing/meson.build @@ -0,0 +1,8 @@ +suites += { + 'name': 'git-hashing', + 'deps': [], + 'tests': [ + 'simple.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/local-overlay-store/common.sh b/tests/functional/local-overlay-store/common.sh index b171f91f499..27338ea23e8 100644 --- a/tests/functional/local-overlay-store/common.sh +++ b/tests/functional/local-overlay-store/common.sh @@ -1,4 +1,5 @@ -source ../common/vars-and-functions.sh +source ../common/vars.sh +source ../common/functions.sh TODO_NixOS diff --git a/tests/functional/local-overlay-store/meson.build b/tests/functional/local-overlay-store/meson.build new file mode 100644 index 00000000000..6ff5d3169f9 --- /dev/null +++ b/tests/functional/local-overlay-store/meson.build @@ -0,0 +1,18 @@ +suites += { + 'name': 'local-overlay-store', + 'deps': [], + 'tests': [ + 'check-post-init.sh', + 'redundant-add.sh', + 'build.sh', + 'bad-uris.sh', + 'add-lower.sh', + 'delete-refs.sh', + 'delete-duplicate.sh', + 'gc.sh', + 'verify.sh', + 'optimise.sh', + 'stale-file-handle.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/meson.build b/tests/functional/meson.build new file mode 100644 index 00000000000..ebecdd9e85d --- /dev/null +++ b/tests/functional/meson.build @@ -0,0 +1,266 @@ +project('nix-functional-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.3', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +# Need to combine source and build trees +run_command( + 'rsync', + '-a', + '--copy-unsafe-links', + meson.current_source_dir() / '', + meson.current_build_dir() / '', +) +# This current-source-escaping relative is no good because we don't know +# where the build directory will be, therefore we fix it up. Once the +# Make build system is gone, we should think about doing this better. +scripts_dir = fs.relative_to( + meson.current_source_dir() / '..' / '..' / 'scripts', + meson.current_build_dir(), +) +run_command( + 'sed', + '-i', meson.current_build_dir() / 'bash-profile.sh', + '-e', 's^../../scripts^@0@^'.format(scripts_dir), +) + +nix = find_program('nix') +bash = find_program('bash', native : true) +busybox = find_program('busybox', native : true, required : false) +coreutils = find_program('coreutils', native : true) +dot = find_program('dot', native : true, required : false) + +nix_bin_dir = fs.parent(nix.full_path()) + +test_confdata = { + 'bindir': nix_bin_dir, + 'coreutils': fs.parent(coreutils.full_path()), + 'dot': dot.found() ? dot.full_path() : '', + 'bash': bash.full_path(), + 'sandbox_shell': busybox.found() ? busybox.full_path() : '', + 'PACKAGE_VERSION': meson.project_version(), + 'system': host_machine.cpu_family() + '-' + host_machine.system(), +} + +# Just configures `common/vars-and-functions.sh.in`. +# Done as a subdir() so Meson places it under `common` in the build directory as well. +subdir('common') + +config_nix_in = configure_file( + input : 'config.nix.in', + output : 'config.nix', + configuration : test_confdata, +) + +suites = [ + { + 'name' : 'main', + 'deps': [], + 'tests': [ + 'test-infra.sh', + 'gc.sh', + 'nix-collect-garbage-d.sh', + 'remote-store.sh', + 'legacy-ssh-store.sh', + 'lang.sh', + 'lang-gc.sh', + 'characterisation-test-infra.sh', + 'experimental-features.sh', + 'fetchMercurial.sh', + 'gc-auto.sh', + 'user-envs.sh', + 'user-envs-migration.sh', + 'binary-cache.sh', + 'multiple-outputs.sh', + 'nix-build.sh', + 'gc-concurrent.sh', + 'repair.sh', + 'fixed.sh', + 'export-graph.sh', + 'timeout.sh', + 'fetchGitRefs.sh', + 'gc-runtime.sh', + 'tarball.sh', + 'fetchGit.sh', + 'fetchurl.sh', + 'fetchPath.sh', + 'fetchTree-file.sh', + 'simple.sh', + 'referrers.sh', + 'optimise-store.sh', + 'substitute-with-invalid-ca.sh', + 'signing.sh', + 'hash-convert.sh', + 'hash-path.sh', + 'gc-non-blocking.sh', + 'check.sh', + 'nix-shell.sh', + 'check-refs.sh', + 'build-remote-input-addressed.sh', + 'secure-drv-outputs.sh', + 'restricted.sh', + 'fetchGitSubmodules.sh', + 'fetchGitVerification.sh', + 'readfile-context.sh', + 'nix-channel.sh', + 'recursive.sh', + 'dependencies.sh', + 'check-reqs.sh', + 'build-remote-content-addressed-fixed.sh', + 'build-remote-content-addressed-floating.sh', + 'build-remote-trustless-should-pass-0.sh', + 'build-remote-trustless-should-pass-1.sh', + 'build-remote-trustless-should-pass-2.sh', + 'build-remote-trustless-should-pass-3.sh', + 'build-remote-trustless-should-fail-0.sh', + 'build-remote-with-mounted-ssh-ng.sh', + 'nar-access.sh', + 'impure-eval.sh', + 'pure-eval.sh', + 'eval.sh', + 'repl.sh', + 'binary-cache-build-remote.sh', + 'search.sh', + 'logging.sh', + 'export.sh', + 'config.sh', + 'add.sh', + 'chroot-store.sh', + 'filter-source.sh', + 'misc.sh', + 'dump-db.sh', + 'linux-sandbox.sh', + 'supplementary-groups.sh', + 'build-dry.sh', + 'structured-attrs.sh', + 'shell.sh', + 'brotli.sh', + 'zstd.sh', + 'compression-levels.sh', + 'nix-copy-ssh.sh', + 'nix-copy-ssh-ng.sh', + 'post-hook.sh', + 'function-trace.sh', + 'fmt.sh', + 'eval-store.sh', + 'why-depends.sh', + 'derivation-json.sh', + 'derivation-advanced-attributes.sh', + 'import-derivation.sh', + 'nix_path.sh', + 'case-hack.sh', + 'placeholders.sh', + 'ssh-relay.sh', + 'build.sh', + 'build-delete.sh', + 'output-normalization.sh', + 'selfref-gc.sh', + 'db-migration.sh', + 'bash-profile.sh', + 'pass-as-file.sh', + 'nix-profile.sh', + 'suggestions.sh', + 'store-info.sh', + 'fetchClosure.sh', + 'completions.sh', + 'impure-derivations.sh', + 'path-from-hash-part.sh', + 'path-info.sh', + 'toString-path.sh', + 'read-only-store.sh', + 'nested-sandboxing.sh', + 'impure-env.sh', + 'debugger.sh', + 'extra-sandbox-profile.sh', + 'help.sh', + ], + 'workdir': meson.current_build_dir(), + }, +] + +nix_store = dependency('nix-store', required : false) +if nix_store.found() + subdir('test-libstoreconsumer') + suites += { + 'name': 'libstoreconsumer', + 'deps': [ + libstoreconsumer_tester, + ], + 'tests': [ + 'test-libstoreconsumer.sh', + ], + 'workdir': meson.current_build_dir(), + } + +endif + +# Plugin tests require shared libraries support. +nix_expr = dependency('nix-expr', required : false) +if nix_expr.found() and get_option('default_library') != 'static' + subdir('plugins') + suites += { + 'name': 'plugins', + 'deps': [ + libplugintest, + ], + 'tests': [ + 'plugins.sh', + ], + 'workdir': meson.current_build_dir(), + } +endif + +subdir('ca') +subdir('dyn-drv') +subdir('flakes') +subdir('git-hashing') +subdir('local-overlay-store') + +foreach suite : suites + foreach script : suite['tests'] + workdir = suite['workdir'] + prefix = fs.relative_to(workdir, meson.project_build_root()) + + script = script + # Turns, e.g., `tests/functional/flakes/show.sh` into a Meson test target called + # `functional-flakes-show`. + name = fs.replace_suffix(prefix / script, '') + + test( + name, + bash, + args: [ + '-x', + '-e', + '-u', + '-o', 'pipefail', + script, + ], + suite : suite['name'], + env : { + 'TEST_NAME': name, + 'NIX_REMOTE': '', + 'PS4': '+(${BASH_SOURCE[0]-$0}:$LINENO) ', + }, + # some tests take 15+ seconds even on an otherwise idle machine, on a loaded machine + # this can easily drive them to failure. give them more time than default of 30sec + timeout : 300, + # Used for target dependency/ordering tracking, not adding compiler flags or anything. + depends : suite['deps'], + workdir : workdir, + # Won't pass until man pages are generated + should_fail : suite['name'] == 'main' and script == 'help.sh' + ) + endforeach +endforeach diff --git a/tests/functional/nested-sandboxing/command.sh b/tests/functional/nested-sandboxing/command.sh index 69366486cec..e9c40a5d9cd 100644 --- a/tests/functional/nested-sandboxing/command.sh +++ b/tests/functional/nested-sandboxing/command.sh @@ -1,3 +1,5 @@ +set -eu -o pipefail + export NIX_BIN_DIR=$(dirname $(type -p nix)) # TODO Get Nix and its closure more flexibly export EXTRA_SANDBOX="/nix/store $(dirname $NIX_BIN_DIR)" diff --git a/tests/functional/nested-sandboxing/runner.nix b/tests/functional/nested-sandboxing/runner.nix index 9a5822c881a..1e79d5065b6 100644 --- a/tests/functional/nested-sandboxing/runner.nix +++ b/tests/functional/nested-sandboxing/runner.nix @@ -6,7 +6,10 @@ mkDerivation { name = "nested-sandboxing"; busybox = builtins.getEnv "busybox"; EXTRA_SANDBOX = builtins.getEnv "EXTRA_SANDBOX"; - buildCommand = if altitude == 0 then '' + buildCommand = '' + set -x + set -eu -o pipefail + '' + (if altitude == 0 then '' echo Deep enough! > $out '' else '' cp -r ${../common} ./common @@ -20,5 +23,5 @@ mkDerivation { source ./nested-sandboxing/command.sh runNixBuild ${storeFun} ${toString altitude} >> $out - ''; + ''); } diff --git a/tests/functional/package.nix b/tests/functional/package.nix new file mode 100644 index 00000000000..205b03614cc --- /dev/null +++ b/tests/functional/package.nix @@ -0,0 +1,117 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config +, rsync + +, jq +, git +, mercurial +, util-linux + +, nix-store +, nix-expr +, nix-ng + +, rapidcheck +, gtest +, runCommand + +, busybox-sandbox-shell ? null + +# Configuration Options + +, version + +# For running the functional tests against a different pre-built Nix. +, test-daemon ? null +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-functional-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../scripts/nix-profile.sh.in + ../../.version + ../../tests/functional + ./. + ]; + + # Hack for sake of the dev shell + passthru.baseNativeBuildInputs = [ + meson + ninja + pkg-config + rsync + + jq + git + mercurial + ] ++ lib.optionals stdenv.hostPlatform.isLinux [ + # For various sandboxing tests that needs a statically-linked shell, + # etc. + busybox-sandbox-shell + # For Overlay FS tests need `mount`, `umount`, and `unshare`. + # TODO use `unixtools` to be precise over which executables instead? + util-linux + ]; + + nativeBuildInputs = finalAttrs.passthru.baseNativeBuildInputs ++ [ + nix-ng + ]; + + buildInputs = [ + nix-store + nix-expr + ]; + + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + '' + # TEMP hack for Meson before make is gone, where + # `src/nix-functional-tests` is during the transition a symlink and + # not the actual directory directory. + + '' + cd $(readlink -e $PWD) + echo $PWD | grep tests/functional + ''; + + mesonCheckFlags = [ + "--print-errorlogs" + ]; + + preCheck = + # See https://github.com/NixOS/nix/issues/2523 + # Occurs often in tests since https://github.com/NixOS/nix/pull/9900 + lib.optionalString stdenv.hostPlatform.isDarwin '' + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + ''; + + doCheck = true; + + installPhase = '' + touch $out + ''; + + meta = { + platforms = lib.platforms.unix; + }; + +} // lib.optionalAttrs (test-daemon != null) { + NIX_DAEMON_PACKAGE = test-daemon; +}) diff --git a/tests/functional/plugins.sh b/tests/functional/plugins.sh index ab4876df9da..fc2d1907cec 100755 --- a/tests/functional/plugins.sh +++ b/tests/functional/plugins.sh @@ -2,10 +2,11 @@ source common.sh -if [[ $BUILD_SHARED_LIBS != 1 ]]; then - skipTest "Plugins are not supported" -fi +for ext in so dylib; do + plugin="$PWD/plugins/libplugintest.$ext" + [[ -f "$plugin" ]] && break +done -res=$(nix --option setting-set true --option plugin-files $PWD/plugins/libplugintest* eval --expr builtins.anotherNull) +res=$(nix --option setting-set true --option plugin-files "$plugin" eval --expr builtins.anotherNull) [ "$res"x = "nullx" ] diff --git a/tests/functional/plugins/meson.build b/tests/functional/plugins/meson.build new file mode 100644 index 00000000000..3d6b2f0e1d8 --- /dev/null +++ b/tests/functional/plugins/meson.build @@ -0,0 +1,16 @@ +libplugintest = shared_module( + 'plugintest', + 'plugintest.cc', + cpp_args : [ + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.hh', + '-include', 'config-expr.hh', + ], + dependencies : [ + dependency('nix-expr'), + ], + build_by_default : false, +) diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index 591367e9f2d..e5fe9c13637 100755 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -16,9 +16,6 @@ nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.n (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix') nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=../.. -(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel') -nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src - expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' | grepQuiet "forbidden in restricted mode" nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' -I src=. diff --git a/tests/functional/test-libstoreconsumer/meson.build b/tests/functional/test-libstoreconsumer/meson.build new file mode 100644 index 00000000000..7076127f70a --- /dev/null +++ b/tests/functional/test-libstoreconsumer/meson.build @@ -0,0 +1,14 @@ +libstoreconsumer_tester = executable( + 'test-libstoreconsumer', + 'main.cc', + cpp_args : [ + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + ], + dependencies : [ + dependency('nix-store'), + ], + build_by_default : false, +) From 6f3045c2a225dca7b1ed8a9c9dc27ab50f575900 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 14 Aug 2024 15:40:32 -0400 Subject: [PATCH 223/284] Remove unit tests from old build system Now that we can run all tests with Meson, we want developers making code changes to use it. (Only the manual needs to be built with the build system, and that will change shortly.) This reverts commit b0bc2a97bfe007fbc32f584ed9de5e7cb75a521c. --- Makefile | 19 ------------ Makefile.config.in | 1 - configure.ac | 22 -------------- package.nix | 33 +------------------- tests/unit/libexpr-support/local.mk | 23 -------------- tests/unit/libexpr/local.mk | 45 ---------------------------- tests/unit/libfetchers/local.mk | 37 ----------------------- tests/unit/libflake/local.mk | 43 -------------------------- tests/unit/libstore-support/local.mk | 21 ------------- tests/unit/libstore/local.mk | 38 ----------------------- tests/unit/libutil-support/local.mk | 19 ------------ tests/unit/libutil/local.mk | 37 ----------------------- 12 files changed, 1 insertion(+), 337 deletions(-) delete mode 100644 tests/unit/libexpr-support/local.mk delete mode 100644 tests/unit/libexpr/local.mk delete mode 100644 tests/unit/libfetchers/local.mk delete mode 100644 tests/unit/libflake/local.mk delete mode 100644 tests/unit/libstore-support/local.mk delete mode 100644 tests/unit/libstore/local.mk delete mode 100644 tests/unit/libutil-support/local.mk delete mode 100644 tests/unit/libutil/local.mk diff --git a/Makefile b/Makefile index dbf510a3e9c..b51ae6cc730 100644 --- a/Makefile +++ b/Makefile @@ -38,18 +38,6 @@ makefiles += \ endif endif -ifeq ($(ENABLE_UNIT_TESTS), yes) -makefiles += \ - tests/unit/libutil/local.mk \ - tests/unit/libutil-support/local.mk \ - tests/unit/libstore/local.mk \ - tests/unit/libstore-support/local.mk \ - tests/unit/libfetchers/local.mk \ - tests/unit/libexpr/local.mk \ - tests/unit/libexpr-support/local.mk \ - tests/unit/libflake/local.mk -endif - ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes) ifdef HOST_UNIX makefiles += \ @@ -104,13 +92,6 @@ include mk/lib.mk # These must be defined after `mk/lib.mk`. Otherwise the first rule # incorrectly becomes the default target. -ifneq ($(ENABLE_UNIT_TESTS), yes) -.PHONY: check -check: - @echo "Unit tests are disabled. Configure without '--disable-unit-tests', or avoid calling 'make check'." - @exit 1 -endif - ifneq ($(ENABLE_FUNCTIONAL_TESTS), yes) .PHONY: installcheck installcheck: diff --git a/Makefile.config.in b/Makefile.config.in index 3100d207365..e131484f61d 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -12,7 +12,6 @@ ENABLE_BUILD = @ENABLE_BUILD@ ENABLE_DOC_GEN = @ENABLE_DOC_GEN@ ENABLE_FUNCTIONAL_TESTS = @ENABLE_FUNCTIONAL_TESTS@ ENABLE_S3 = @ENABLE_S3@ -ENABLE_UNIT_TESTS = @ENABLE_UNIT_TESTS@ GTEST_LIBS = @GTEST_LIBS@ HAVE_LIBCPUID = @HAVE_LIBCPUID@ HAVE_SECCOMP = @HAVE_SECCOMP@ diff --git a/configure.ac b/configure.ac index 5c22ed17636..18d718c0737 100644 --- a/configure.ac +++ b/configure.ac @@ -141,18 +141,6 @@ AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]), ENABLE_BUILD=$enableval, ENABLE_BUILD=yes) AC_SUBST(ENABLE_BUILD) -# Building without unit tests is useful for bootstrapping with a smaller footprint -# or running the tests in a separate derivation. Otherwise, we do compile and -# run them. - -AC_ARG_ENABLE(unit-tests, AS_HELP_STRING([--disable-unit-tests],[Do not build the tests]), - ENABLE_UNIT_TESTS=$enableval, ENABLE_UNIT_TESTS=$ENABLE_BUILD) -AC_SUBST(ENABLE_UNIT_TESTS) - -AS_IF( - [test "$ENABLE_BUILD" == "no" && test "$ENABLE_UNIT_TESTS" == "yes"], - [AC_MSG_ERROR([Cannot enable unit tests when building overall is disabled. Please do not pass '--enable-unit-tests' or do not pass '--disable-build'.])]) - AC_ARG_ENABLE(functional-tests, AS_HELP_STRING([--disable-functional-tests],[Do not build the tests]), ENABLE_FUNCTIONAL_TESTS=$enableval, ENABLE_FUNCTIONAL_TESTS=yes) AC_SUBST(ENABLE_FUNCTIONAL_TESTS) @@ -358,16 +346,6 @@ if test "$gc" = yes; then CFLAGS="$old_CFLAGS" fi -AS_IF([test "$ENABLE_UNIT_TESTS" == "yes"],[ - -# Look for gtest. -PKG_CHECK_MODULES([GTEST], [gtest_main gmock_main]) - -# Look for rapidcheck. -PKG_CHECK_MODULES([RAPIDCHECK], [rapidcheck rapidcheck_gtest]) - -]) - # Look for nlohmann/json. PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9]) diff --git a/package.nix b/package.nix index d41748b7cf5..c0d04179df7 100644 --- a/package.nix +++ b/package.nix @@ -53,10 +53,6 @@ # Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix , doBuild ? true -# Run the unit tests as part of the build. See `installUnitTests` for an -# alternative to this. -, doCheck ? __forDefaults.canRunInstalled - # Run the functional tests as part of the build. , doInstallCheck ? test-client != null || __forDefaults.canRunInstalled @@ -89,11 +85,6 @@ # - readline , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" -# Whether to install unit tests. This is useful when cross compiling -# since we cannot run them natively during the build, but can do so -# later. -, installUnitTests ? doBuild && !__forDefaults.canExecuteHost - # For running the functional tests against a pre-built Nix. Probably # want to use in conjunction with `doBuild = false;`. , test-daemon ? null @@ -117,7 +108,7 @@ let # things which should instead be gotten via `finalAttrs` in order to # work with overriding. attrs = { - inherit doBuild doCheck doInstallCheck; + inherit doBuild doInstallCheck; }; mkDerivation = @@ -133,16 +124,11 @@ in mkDerivation (finalAttrs: let inherit (finalAttrs) - doCheck doInstallCheck ; doBuild = !finalAttrs.dontBuild; - # Either running the unit tests during the build, or installing them - # to be run later, requiresthe unit tests to be built. - buildUnitTests = doCheck || installUnitTests; - in { inherit pname version; @@ -176,8 +162,6 @@ in { ./scripts/local.mk ] ++ lib.optionals enableManual [ ./doc/manual - ] ++ lib.optionals buildUnitTests [ - ./tests/unit ] ++ lib.optionals doInstallCheck [ ./tests/functional ])); @@ -190,8 +174,6 @@ in { # If we are doing just build or just docs, the one thing will use # "out". We only need additional outputs if we are doing both. ++ lib.optional (doBuild && enableManual) "doc" - ++ lib.optional installUnitTests "check" - ++ lib.optional doCheck "testresults" ; nativeBuildInputs = [ @@ -230,9 +212,6 @@ in { ({ inherit readline editline; }.${readlineFlavor}) ] ++ lib.optionals enableMarkdown [ lowdown - ] ++ lib.optionals buildUnitTests [ - gtest - rapidcheck ] ++ lib.optional stdenv.isLinux libseccomp ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies @@ -247,22 +226,16 @@ in { ); dontBuild = !attrs.doBuild; - doCheck = attrs.doCheck; configureFlags = [ (lib.enableFeature doBuild "build") - (lib.enableFeature buildUnitTests "unit-tests") (lib.enableFeature doInstallCheck "functional-tests") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature enableGC "gc") (lib.enableFeature enableMarkdown "markdown") - (lib.enableFeature installUnitTests "install-unit-tests") (lib.withFeatureAs true "readline-flavor" readlineFlavor) ] ++ lib.optionals (!forDevShell) [ "--sysconfdir=/etc" - ] ++ lib.optionals installUnitTests [ - "--with-check-bin-dir=${builtins.placeholder "check"}/bin" - "--with-check-lib-dir=${builtins.placeholder "check"}/lib" ] ++ lib.optionals (doBuild) [ "--with-boost=${boost}/lib" ] ++ lib.optionals (doBuild && stdenv.isLinux) [ @@ -343,10 +316,6 @@ in { platforms = lib.platforms.unix ++ lib.platforms.windows; mainProgram = "nix"; broken = !(lib.all (a: a) [ - # We cannot run or install unit tests if we don't build them or - # Nix proper (which they depend on). - (installUnitTests -> doBuild) - (doCheck -> doBuild) # The build process for the manual currently requires extracting # data from the Nix executable we are trying to document. (enableManual -> doBuild) diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk deleted file mode 100644 index 0501de33c43..00000000000 --- a/tests/unit/libexpr-support/local.mk +++ /dev/null @@ -1,23 +0,0 @@ -libraries += libexpr-test-support - -libexpr-test-support_NAME = libnixexpr-test-support - -libexpr-test-support_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libexpr-test-support_INSTALL_DIR := $(checklibdir) -else - libexpr-test-support_INSTALL_DIR := -endif - -libexpr-test-support_SOURCES := \ - $(wildcard $(d)/tests/*.cc) \ - $(wildcard $(d)/tests/value/*.cc) - -libexpr-test-support_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) - -libexpr-test-support_LIBS = \ - libstore-test-support libutil-test-support \ - libexpr libstore libutil - -libexpr-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk deleted file mode 100644 index 1617e282351..00000000000 --- a/tests/unit/libexpr/local.mk +++ /dev/null @@ -1,45 +0,0 @@ -check: libexpr-tests_RUN - -programs += libexpr-tests - -libexpr-tests_NAME := libnixexpr-tests - -libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libexpr-tests.xml - -libexpr-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libexpr-tests_INSTALL_DIR := $(checkbindir) -else - libexpr-tests_INSTALL_DIR := -endif - -libexpr-tests_SOURCES := \ - $(wildcard $(d)/*.cc) \ - $(wildcard $(d)/value/*.cc) \ - $(wildcard $(d)/flake/*.cc) - -libexpr-tests_EXTRA_INCLUDES = \ - -I tests/unit/libexpr-support \ - -I tests/unit/libstore-support \ - -I tests/unit/libutil-support \ - $(INCLUDE_libexpr) \ - $(INCLUDE_libexprc) \ - $(INCLUDE_libfetchers) \ - $(INCLUDE_libstore) \ - $(INCLUDE_libstorec) \ - $(INCLUDE_libutil) \ - $(INCLUDE_libutilc) - -libexpr-tests_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) - -libexpr-tests_LIBS = \ - libexpr-test-support libstore-test-support libutil-test-support \ - libexpr libexprc libfetchers libstore libstorec libutil libutilc - -libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock - -ifdef HOST_WINDOWS - # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space - libexpr-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) -endif diff --git a/tests/unit/libfetchers/local.mk b/tests/unit/libfetchers/local.mk deleted file mode 100644 index 30aa142a5e2..00000000000 --- a/tests/unit/libfetchers/local.mk +++ /dev/null @@ -1,37 +0,0 @@ -check: libfetchers-tests_RUN - -programs += libfetchers-tests - -libfetchers-tests_NAME = libnixfetchers-tests - -libfetchers-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libfetchers-tests.xml - -libfetchers-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libfetchers-tests_INSTALL_DIR := $(checkbindir) -else - libfetchers-tests_INSTALL_DIR := -endif - -libfetchers-tests_SOURCES := $(wildcard $(d)/*.cc) - -libfetchers-tests_EXTRA_INCLUDES = \ - -I tests/unit/libstore-support \ - -I tests/unit/libutil-support \ - $(INCLUDE_libfetchers) \ - $(INCLUDE_libstore) \ - $(INCLUDE_libutil) - -libfetchers-tests_CXXFLAGS += $(libfetchers-tests_EXTRA_INCLUDES) - -libfetchers-tests_LIBS = \ - libstore-test-support libutil-test-support \ - libfetchers libstore libutil - -libfetchers-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) $(LIBGIT2_LIBS) - -ifdef HOST_WINDOWS - # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space - libfetchers-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) -endif diff --git a/tests/unit/libflake/local.mk b/tests/unit/libflake/local.mk deleted file mode 100644 index 590bcf7c031..00000000000 --- a/tests/unit/libflake/local.mk +++ /dev/null @@ -1,43 +0,0 @@ -check: libflake-tests_RUN - -programs += libflake-tests - -libflake-tests_NAME := libnixflake-tests - -libflake-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libflake-tests.xml - -libflake-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libflake-tests_INSTALL_DIR := $(checkbindir) -else - libflake-tests_INSTALL_DIR := -endif - -libflake-tests_SOURCES := \ - $(wildcard $(d)/*.cc) \ - $(wildcard $(d)/value/*.cc) \ - $(wildcard $(d)/flake/*.cc) - -libflake-tests_EXTRA_INCLUDES = \ - -I tests/unit/libflake-support \ - -I tests/unit/libstore-support \ - -I tests/unit/libutil-support \ - $(INCLUDE_libflake) \ - $(INCLUDE_libexpr) \ - $(INCLUDE_libfetchers) \ - $(INCLUDE_libstore) \ - $(INCLUDE_libutil) \ - -libflake-tests_CXXFLAGS += $(libflake-tests_EXTRA_INCLUDES) - -libflake-tests_LIBS = \ - libexpr-test-support libstore-test-support libutil-test-support \ - libflake libexpr libfetchers libstore libutil - -libflake-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock - -ifdef HOST_WINDOWS - # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space - libflake-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) -endif diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk deleted file mode 100644 index 56dedd825d8..00000000000 --- a/tests/unit/libstore-support/local.mk +++ /dev/null @@ -1,21 +0,0 @@ -libraries += libstore-test-support - -libstore-test-support_NAME = libnixstore-test-support - -libstore-test-support_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libstore-test-support_INSTALL_DIR := $(checklibdir) -else - libstore-test-support_INSTALL_DIR := -endif - -libstore-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) - -libstore-test-support_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) - -libstore-test-support_LIBS = \ - libutil-test-support \ - libstore libutil - -libstore-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libstore/local.mk b/tests/unit/libstore/local.mk deleted file mode 100644 index 8d3d6b0afe2..00000000000 --- a/tests/unit/libstore/local.mk +++ /dev/null @@ -1,38 +0,0 @@ -check: libstore-tests_RUN - -programs += libstore-tests - -libstore-tests_NAME = libnixstore-tests - -libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libstore-tests.xml - -libstore-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libstore-tests_INSTALL_DIR := $(checkbindir) -else - libstore-tests_INSTALL_DIR := -endif - -libstore-tests_SOURCES := $(wildcard $(d)/*.cc) - -libstore-tests_EXTRA_INCLUDES = \ - -I tests/unit/libstore-support \ - -I tests/unit/libutil-support \ - $(INCLUDE_libstore) \ - $(INCLUDE_libstorec) \ - $(INCLUDE_libutil) \ - $(INCLUDE_libutilc) - -libstore-tests_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) - -libstore-tests_LIBS = \ - libstore-test-support libutil-test-support \ - libstore libstorec libutil libutilc - -libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) - -ifdef HOST_WINDOWS - # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space - libstore-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) -endif diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk deleted file mode 100644 index 5f7835c9f61..00000000000 --- a/tests/unit/libutil-support/local.mk +++ /dev/null @@ -1,19 +0,0 @@ -libraries += libutil-test-support - -libutil-test-support_NAME = libnixutil-test-support - -libutil-test-support_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libutil-test-support_INSTALL_DIR := $(checklibdir) -else - libutil-test-support_INSTALL_DIR := -endif - -libutil-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) - -libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) - -libutil-test-support_LIBS = libutil - -libutil-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libutil/local.mk b/tests/unit/libutil/local.mk deleted file mode 100644 index 404f35cf1ac..00000000000 --- a/tests/unit/libutil/local.mk +++ /dev/null @@ -1,37 +0,0 @@ -check: libutil-tests_RUN - -programs += libutil-tests - -libutil-tests_NAME = libnixutil-tests - -libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libutil-tests.xml - -libutil-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libutil-tests_INSTALL_DIR := $(checkbindir) -else - libutil-tests_INSTALL_DIR := -endif - -libutil-tests_SOURCES := $(wildcard $(d)/*.cc) - -libutil-tests_EXTRA_INCLUDES = \ - -I tests/unit/libutil-support \ - $(INCLUDE_libutil) \ - $(INCLUDE_libutilc) - -libutil-tests_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) - -libutil-tests_LIBS = libutil-test-support libutil libutilc - -libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) - -ifdef HOST_WINDOWS - # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space - libutil-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) -endif - -check: $(d)/data/git/check-data.sh.test - -$(eval $(call run-test,$(d)/data/git/check-data.sh)) From b41cc1a7555e59d5753bf4f8cbbfa9137107d2fe Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 14 Aug 2024 16:04:35 -0400 Subject: [PATCH 224/284] Make wrapper derivation This ensures just `nix build`-ing the flake doesn't forget to run all tests. One can still specifiy specific attributes to just build one thing. Co-authored-by: Robert Hensing --- .github/workflows/ci.yml | 14 ------ flake.nix | 51 ++++++++++++++------ packaging/components.nix | 6 ++- packaging/everything.nix | 93 ++++++++++++++++++++++++++++++++++++ packaging/hydra.nix | 5 +- tests/functional/package.nix | 6 +-- tests/nixos/quick-build.nix | 6 +-- 7 files changed, 142 insertions(+), 39 deletions(-) create mode 100644 packaging/everything.nix diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4eb9cf10dc7..9831d0e0e5e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -195,20 +195,6 @@ jobs: - uses: DeterminateSystems/magic-nix-cache-action@main - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes .#hydraJobs.tests.functional_user - meson_build: - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main - # Only meson packages that don't have a tests.run derivation. - # Those that have it are already built and tested as part of nix flake check. - - run: nix build -L .#hydraJobs.build.{nix-cmd,nix-main}.$(nix-instantiate --eval --expr builtins.currentSystem | sed -e 's/"//g') - flake_regressions: needs: vm_tests runs-on: ubuntu-22.04 diff --git a/flake.nix b/flake.nix index 22ca5411822..cded6c3a94b 100644 --- a/flake.nix +++ b/flake.nix @@ -139,11 +139,6 @@ nix = final.nixComponents.nix; - nix_noTests = final.nix.override { - doInstallCheck = false; - doCheck = false; - }; - # See https://github.com/NixOS/nixpkgs/pull/214409 # Remove when fixed in this flake's nixpkgs pre-commit = @@ -222,7 +217,7 @@ # for which we don't apply the full build matrix such as cross or static. inherit (nixpkgsFor.${system}.native) changelog-d; - default = self.packages.${system}.nix; + default = self.packages.${system}.nix-ng; nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; } @@ -230,22 +225,48 @@ // flatMapAttrs { # Components we'll iterate over in the upcoming lambda "nix" = { }; - # Temporarily disabled because GitHub Actions OOM issues. Once - # the old build system is gone and we are back to one build - # system, we should reenable these. - #"nix-util" = { }; - #"nix-store" = { }; - #"nix-fetchers" = { }; + "nix-util" = { }; + "nix-util-c" = { }; + "nix-util-test-support" = { }; + "nix-util-tests" = { }; + + "nix-store" = { }; + "nix-store-c" = { }; + "nix-store-test-support" = { }; + "nix-store-tests" = { }; + + "nix-fetchers" = { }; + "nix-fetchers-tests" = { }; + + "nix-expr" = { }; + "nix-expr-c" = { }; + "nix-expr-test-support" = { }; + "nix-expr-tests" = { }; + + "nix-flake" = { }; + "nix-flake-tests" = { }; + + "nix-main" = { }; + "nix-main-c" = { }; + + "nix-cmd" = { }; + + "nix-cli" = { }; + + "nix-functional-tests" = { supportsCross = false; }; + + "nix-perl-bindings" = { supportsCross = false; }; + "nix-ng" = { }; } - (pkgName: {}: { + (pkgName: { supportsCross ? true }: { # These attributes go right into `packages.`. "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; } - // flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { + // lib.optionalAttrs supportsCross (flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { # These attributes go right into `packages.`. "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}; - }) + })) // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: { # These attributes go right into `packages.`. "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName}; diff --git a/packaging/components.nix b/packaging/components.nix index f14613e8a1d..5fc3236cf7b 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -56,8 +56,7 @@ in nix-cmd = callPackage ../src/libcmd/package.nix { }; - # Will replace `nix` once the old build system is gone. - nix-ng = callPackage ../src/nix/package.nix { version = fineVersion; }; + nix-cli = callPackage ../src/nix/package.nix { version = fineVersion; }; nix-functional-tests = callPackage ../src/nix-functional-tests/package.nix { version = fineVersion; }; @@ -65,4 +64,7 @@ in nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { version = fineVersion; }; nix-perl-bindings = callPackage ../src/perl/package.nix { }; + + # Will replace `nix` once the old build system is gone. + nix-ng = callPackage ../packaging/everything.nix { }; } diff --git a/packaging/everything.nix b/packaging/everything.nix new file mode 100644 index 00000000000..8c8ce66113a --- /dev/null +++ b/packaging/everything.nix @@ -0,0 +1,93 @@ +{ + lib, + stdenv, + buildEnv, + + nix-util, + nix-util-c, + nix-util-test-support, + nix-util-tests, + + nix-store, + nix-store-c, + nix-store-test-support, + nix-store-tests, + + nix-fetchers, + nix-fetchers-tests, + + nix-expr, + nix-expr-c, + nix-expr-test-support, + nix-expr-tests, + + nix-flake, + nix-flake-tests, + + nix-main, + nix-main-c, + + nix-cmd, + + nix-cli, + + nix-functional-tests, + + nix-internal-api-docs, + nix-external-api-docs, + + nix-perl-bindings, +}: + +(buildEnv rec { + name = "nix-${nix-cli.version}"; + paths = [ + nix-util + nix-util-c + nix-util-test-support + nix-util-tests + + nix-store + nix-store-c + nix-store-test-support + nix-store-tests + + nix-fetchers + nix-fetchers-tests + + nix-expr + nix-expr-c + nix-expr-test-support + nix-expr-tests + + nix-flake + nix-flake-tests + + nix-main + nix-main-c + + nix-cmd + + nix-cli + + nix-internal-api-docs + nix-external-api-docs + + ] ++ lib.optionals (stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ + nix-perl-bindings + ]; +}).overrideAttrs (_: { + doCheck = true; + doInstallCheck = true; + + checkInputs = [ + # Actually run the unit tests too + nix-util-tests.tests.run + nix-store-tests.tests.run + nix-expr-tests.tests.run + nix-flake-tests.tests.run + ]; + installCheckInputs = [ + nix-functional-tests + ]; +}) diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 46b4ff51d21..65978835c30 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -61,8 +61,9 @@ let "nix-main" "nix-main-c" "nix-cmd" - "nix-ng" + "nix-cli" "nix-functional-tests" + "nix-ng" ]; in { @@ -85,7 +86,7 @@ in self.packages.${system}.nix.override { enableGC = false; } ); - buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nix_noTests); + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-cli); # Toggles some settings for better coverage. Windows needs these # library combinations, and Debian build Nix with GNU readline too. diff --git a/tests/functional/package.nix b/tests/functional/package.nix index 205b03614cc..277711123eb 100644 --- a/tests/functional/package.nix +++ b/tests/functional/package.nix @@ -15,7 +15,7 @@ , nix-store , nix-expr -, nix-ng +, nix-cli , rapidcheck , gtest @@ -67,7 +67,7 @@ mkMesonDerivation (finalAttrs: { ]; nativeBuildInputs = finalAttrs.passthru.baseNativeBuildInputs ++ [ - nix-ng + nix-cli ]; buildInputs = [ @@ -105,7 +105,7 @@ mkMesonDerivation (finalAttrs: { doCheck = true; installPhase = '' - touch $out + mkdir $out ''; meta = { diff --git a/tests/nixos/quick-build.nix b/tests/nixos/quick-build.nix index 37169b4e2e0..57e3b9cdbd3 100644 --- a/tests/nixos/quick-build.nix +++ b/tests/nixos/quick-build.nix @@ -27,13 +27,13 @@ in }; config = { - passthru.quickBuild = + passthru.quickBuild = let withQuickBuild = extendModules { modules = [{ quickBuild = true; }]; }; in withQuickBuild.config.test; defaults = { pkgs, ... }: { config = lib.mkIf test.config.quickBuild { - nix.package = pkgs.nix_noTests; + nix.package = pkgs.nixComponents.nix-cli; system.forbiddenDependenciesRegexes = [ # This would indicate that the quickBuild feature is broken. @@ -44,4 +44,4 @@ in }; }; }; -} \ No newline at end of file +} From 0fabb348baf923bc10a61bc0dc3315023cba9588 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Thu, 18 Jul 2024 09:56:57 -0500 Subject: [PATCH 225/284] add script to migrate macOS 15 Sequoia nixbld UIDs While we don't have any easy way to forcibly notify everyone about the impending breakage (or forcibly migrate the users on their system), this script enables those who do hear about the problem to migrate their systems before they take the macOS update. It should also enable people who only discover it after the update when a build fails to ~fix their installs without a full reinstall. --- scripts/bigsur-nixbld-user-migration.sh | 4 +- scripts/sequoia-nixbld-user-migration.sh | 161 ++++++++++++++++++++--- 2 files changed, 146 insertions(+), 19 deletions(-) mode change 100644 => 100755 scripts/sequoia-nixbld-user-migration.sh diff --git a/scripts/bigsur-nixbld-user-migration.sh b/scripts/bigsur-nixbld-user-migration.sh index 0eb312e07cd..a2e2f30f002 100755 --- a/scripts/bigsur-nixbld-user-migration.sh +++ b/scripts/bigsur-nixbld-user-migration.sh @@ -2,7 +2,7 @@ ((NEW_NIX_FIRST_BUILD_UID=301)) -id_available(){ +id_unavailable(){ dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null } @@ -15,7 +15,7 @@ change_nixbld_names_and_ids(){ while read -r name uid; do echo " Checking $name (uid: $uid)" # iterate for a clean ID - while id_available "$next_id"; do + while id_unavailable "$next_id"; do ((next_id++)) if ((next_id >= 400)); then echo "We've hit UID 400 without placing all of your users :(" diff --git a/scripts/sequoia-nixbld-user-migration.sh b/scripts/sequoia-nixbld-user-migration.sh old mode 100644 new mode 100755 index 778a9568810..9208a360523 --- a/scripts/sequoia-nixbld-user-migration.sh +++ b/scripts/sequoia-nixbld-user-migration.sh @@ -1,36 +1,163 @@ #!/usr/bin/env bash -((NEW_NIX_FIRST_BUILD_UID=331)) +set -x -id_available(){ +((NEW_NIX_FIRST_BUILD_UID=350)) +((TEMP_NIX_FIRST_BUILD_UID=31000)) + +nix_user_n() { + printf "_nixbld%d" "$1" +} + +id_unavailable(){ dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null } -change_nixbld_names_and_ids(){ - local name uid next_id - ((next_id=NEW_NIX_FIRST_BUILD_UID)) - echo "Attempting to migrate _nixbld users." - echo "Each _nixbld# user should have its UID moved to $next_id+" +any_nixbld(){ + dscl . list /Users UniqueID | grep -E '\b_nixbld' >/dev/null +} + +re_create_nixbld_user(){ + local name uid + + name="$1" + uid="$2" + + sudo /usr/bin/dscl . -create "/Users/$name" "UniqueID" "$uid" + sudo /usr/bin/dscl . -create "/Users/$name" "IsHidden" "1" + sudo /usr/bin/dscl . -create "/Users/$name" "NFSHomeDirectory" "/var/empty" + sudo /usr/bin/dscl . -create "/Users/$name" "RealName" "Nix build user $name" + sudo /usr/bin/dscl . -create "/Users/$name" "UserShell" "/sbin/nologin" + sudo /usr/bin/dscl . -create "/Users/$name" "PrimaryGroupID" "30001" +} + +hit_id_cap(){ + echo "We've hit UID 400 without placing all of your users :(" + echo "You should use the commands in this script as a starting" + echo "point to review your UID-space and manually move the" + echo "remaining users (or delete them, if you don't need them)." +} + +# evacuate the role-uid space to simplify final placement logic +temporarily_move_existing_nixbld_uids(){ + local name uid next_id user_n + + ((next_id=TEMP_NIX_FIRST_BUILD_UID)) + + echo "" + echo "Step 1: move existing _nixbld users out of the destination UID range." + while read -r name uid; do - echo " Checking $name (uid: $uid)" # iterate for a clean ID - while id_available "$next_id"; do + while id_unavailable "$next_id"; do ((next_id++)) - if ((next_id >= 400)); then - echo "We've hit UID 400 without placing all of your users :(" + # We really want to get these all placed, but I guess there's + # some risk we iterate forever--so we'll give up after 9k uids. + if ((next_id >= 40000)); then + echo "We've hit UID 40000 without temporarily placing all of your users :(" echo "You should use the commands in this script as a starting" echo "point to review your UID-space and manually move the" - echo "remaining users (or delete them, if you don't need them)." + echo "remaining users to any open UID over 1000." exit 1 fi done + sudo dscl . -create "/Users/$name" UniqueID "$next_id" + echo " Temporarily moved $name from uid $uid -> $next_id" + + done < <(dscl . list /Users UniqueID | grep _nixbld | sort -n -k2) +} - # first 2 are cleanup, it's OK if they aren't here - sudo dscl . delete "/Users/$name" dsAttrTypeNative:_writers_passwd &>/dev/null || true - sudo dscl . change "/Users/$name" NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true - sudo dscl . change "/Users/$name" UniqueID "$uid" "$next_id" +change_nixbld_uids(){ + local name next_id user_n + + ((next_id=NEW_NIX_FIRST_BUILD_UID)) + ((user_n=1)) + name="$(nix_user_n "$user_n")" + + # we know that we have *some* nixbld users, but macOS may have + # already clobbered the first few users if this system has been + # upgraded + + echo "" + echo "Step 2: re-create missing early _nixbld# users." + + until dscl . read "/Users/$name" &>/dev/null; do + # iterate for a clean ID + while id_unavailable "$next_id"; do + ((next_id++)) + if ((next_id >= 400)); then + hit_id_cap + exit 1 + fi + done + + re_create_nixbld_user "$name" "$next_id" + echo " $name was missing; created with uid: $next_id" + + ((user_n++)) + name="$(nix_user_n "$user_n")" + done + + echo "" + echo "Step 3: relocate remaining _nixbld# UIDs to $next_id+" + + # start at first _nixbld# not re-created above and increment + # until _nixbld doesn't exist + while dscl . read "/Users/$name" &>/dev/null; do + # iterate for a clean ID + while id_unavailable "$next_id"; do + ((next_id++)) + if ((next_id >= 400)); then + hit_id_cap + exit 1 + fi + done + + sudo dscl . -create "/Users/$name" UniqueID "$next_id" echo " $name migrated to uid: $next_id" + + ((user_n++)) + name="$(nix_user_n "$user_n")" + done + + if ((user_n == 1)); then + echo "Didn't find _nixbld1. Perhaps you have single-user Nix?" + exit 1 + else + echo "Migrated $((user_n - 1)) users. If you want to double-check, try:" + echo "dscl . list /Users UniqueID | grep _nixbld | sort -n -k2" + fi +} +needs_migration(){ + local name uid next_id user_n + + ((next_id=NEW_NIX_FIRST_BUILD_UID)) + ((user_n=1)) + + while read -r name uid; do + expected_name="$(nix_user_n "$user_n")" + if [[ "$expected_name" != "$name" ]]; then + return 0 + fi + if [[ "$next_id" != "$uid" ]]; then + return 0 + fi + ((next_id++)) + ((user_n++)) done < <(dscl . list /Users UniqueID | grep _nixbld | sort -n -k2) + return 1 } -change_nixbld_names_and_ids + +if any_nixbld; then + if needs_migration; then + echo "Attempting to migrate _nixbld users." + temporarily_move_existing_nixbld_uids + change_nixbld_uids + else + echo "_nixbld users already appear to be migrated." + fi +else + echo "Didn't find any _nixbld users. Perhaps you have single-user Nix?" + exit 1 +fi From 6068e32aa790fc53e7123d8442282714af45770b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 15 Aug 2024 11:36:44 +0200 Subject: [PATCH 226/284] refactor: Extract EvalState::addCallDepth --- src/libexpr/eval-inline.hh | 8 ++++++++ src/libexpr/eval.cc | 19 +------------------ src/libexpr/eval.hh | 20 ++++++++++++++++++++ 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 6fa34b06279..d5ce238b2ed 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -4,6 +4,7 @@ #include "print.hh" #include "eval.hh" #include "eval-error.hh" +#include "eval-settings.hh" namespace nix { @@ -138,5 +139,12 @@ inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view e } } +[[gnu::always_inline]] +inline CallDepth EvalState::addCallDepth(const PosIdx pos) { + if (callDepth > settings.maxCallDepth) + error("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow(); + + return CallDepth(callDepth); +}; } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index c9101678ce7..0e70ca26254 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1471,26 +1471,9 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v) v.mkLambda(&env, this); } -namespace { -/** Increments a count on construction and decrements on destruction. - */ -class CallDepth { - size_t & count; -public: - CallDepth(size_t & count) : count(count) { - ++count; - } - ~CallDepth() { - --count; - } -}; -}; - void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos) { - if (callDepth > settings.maxCallDepth) - error("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow(); - CallDepth _level(callDepth); + auto _level = addCallDepth(pos); auto trace = settings.traceFunctionCalls ? std::make_unique(positions[pos]) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index ddf5dcf9478..9fd31e90423 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -41,6 +41,21 @@ namespace eval_cache { class EvalCache; } +/** + * Increments a count on construction and decrements on destruction. + */ +class CallDepth { + size_t & count; + +public: + CallDepth(size_t & count) : count(count) { + ++count; + } + ~CallDepth() { + --count; + } +}; + /** * Function that implements a primop. */ @@ -649,6 +664,11 @@ private: public: + /** + * Check that the call depth is within limits, and increment it, until the returned object is destroyed. + */ + inline CallDepth addCallDepth(const PosIdx pos); + /** * Do a deep equality test between two values. That is, list * elements and attributes are compared recursively. From 72a4d1f52d46f1472b79a04d3af2dee194f13d52 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 15 Aug 2024 12:29:59 +0200 Subject: [PATCH 227/284] Add :doc support for __functor --- src/libexpr/eval.cc | 14 +++ src/libexpr/eval.hh | 6 ++ tests/functional/repl/doc-functor.expected | 101 +++++++++++++++++++++ tests/functional/repl/doc-functor.in | 10 ++ tests/functional/repl/doc-functor.nix | 101 +++++++++++++++++++++ 5 files changed, 232 insertions(+) create mode 100644 tests/functional/repl/doc-functor.expected create mode 100644 tests/functional/repl/doc-functor.in create mode 100644 tests/functional/repl/doc-functor.nix diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 0e70ca26254..b87d96be09c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -616,6 +616,20 @@ std::optional EvalState::getDoc(Value & v) strdup(ss.data()), }; } + if (isFunctor(v)) { + try { + Value & functor = *v.attrs()->find(sFunctor)->value; + Value * vp = &v; + Value partiallyApplied; + callFunction(functor, 1, &vp, partiallyApplied, noPos); + auto _level = addCallDepth(noPos); + return getDoc(partiallyApplied); + } + catch (Error & e) { + e.addTrace(nullptr, "while partially calling '%1%' to retrieve documentation", "__functor"); + throw; + } + } return {}; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 9fd31e90423..da9dd2087ca 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -640,6 +640,12 @@ public: const char * doc; }; + /** + * Retrieve the documentation for a value. This will evaluate the value if + * it is a thunk, and it will partially apply __functor if applicable. + * + * @param v The value to get the documentation for. + */ std::optional getDoc(Value & v); private: diff --git a/tests/functional/repl/doc-functor.expected b/tests/functional/repl/doc-functor.expected new file mode 100644 index 00000000000..8cb2706ef0f --- /dev/null +++ b/tests/functional/repl/doc-functor.expected @@ -0,0 +1,101 @@ +Nix +Type :? for help. + +nix-repl> :l doc-functor.nix +Added variables. + +nix-repl> :doc multiplier +Function `__functor`\ + … defined at /path/to/tests/functional/repl/doc-functor.nix:12:23 + + +Multiply the argument by the factor stored in the factor attribute. + +nix-repl> :doc doubler +Function `multiply`\ + … defined at /path/to/tests/functional/repl/doc-functor.nix:5:17 + + +Look, it's just like a function! + +nix-repl> :doc recursive +Function `__functor`\ + … defined at /path/to/tests/functional/repl/doc-functor.nix:77:23 + + +This looks bad, but the docs are ok because of the eta expansion. + +nix-repl> :doc recursive2 +error: + … while partially calling '__functor' to retrieve documentation + + … while calling '__functor' + at /path/to/tests/functional/repl/doc-functor.nix:85:17: + 84| */ + 85| __functor = self: self.__functor self; + | ^ + 86| }; + + … from call site + at /path/to/tests/functional/repl/doc-functor.nix:85:23: + 84| */ + 85| __functor = self: self.__functor self; + | ^ + 86| }; + + (19999 duplicate frames omitted) + + error: stack overflow; max-call-depth exceeded + at /path/to/tests/functional/repl/doc-functor.nix:85:23: + 84| */ + 85| __functor = self: self.__functor self; + | ^ + 86| }; + +nix-repl> :doc diverging +error: + … while partially calling '__functor' to retrieve documentation + + (10000 duplicate frames omitted) + + … while calling '__functor' + at /path/to/tests/functional/repl/doc-functor.nix:97:19: + 96| f = x: { + 97| __functor = self: (f (x + 1)); + | ^ + 98| }; + + error: stack overflow; max-call-depth exceeded + at /path/to/tests/functional/repl/doc-functor.nix:97:26: + 96| f = x: { + 97| __functor = self: (f (x + 1)); + | ^ + 98| }; + +nix-repl> :doc helper +Function `square`\ + … defined at /path/to/tests/functional/repl/doc-functor.nix:36:12 + + +Compute x^2 + +nix-repl> :doc helper2 +Function `__functor`\ + … defined at /path/to/tests/functional/repl/doc-functor.nix:45:23 + + +This is a function that can be overridden. + +nix-repl> :doc lib.helper3 +Function `__functor`\ + … defined at /path/to/tests/functional/repl/doc-functor.nix:45:23 + + +This is a function that can be overridden. + +nix-repl> :doc helper3 +Function `__functor`\ + … defined at /path/to/tests/functional/repl/doc-functor.nix:45:23 + + +This is a function that can be overridden. diff --git a/tests/functional/repl/doc-functor.in b/tests/functional/repl/doc-functor.in new file mode 100644 index 00000000000..d2bb57a0248 --- /dev/null +++ b/tests/functional/repl/doc-functor.in @@ -0,0 +1,10 @@ +:l doc-functor.nix +:doc multiplier +:doc doubler +:doc recursive +:doc recursive2 +:doc diverging +:doc helper +:doc helper2 +:doc lib.helper3 +:doc helper3 diff --git a/tests/functional/repl/doc-functor.nix b/tests/functional/repl/doc-functor.nix new file mode 100644 index 00000000000..f526f453f19 --- /dev/null +++ b/tests/functional/repl/doc-functor.nix @@ -0,0 +1,101 @@ +rec { + /** + Look, it's just like a function! + */ + multiply = p: q: p * q; + + multiplier = { + factor = 2; + /** + Multiply the argument by the factor stored in the factor attribute. + */ + __functor = self: x: x * self.factor; + }; + + doubler = { + description = "bla"; + /** + Multiply by two. This doc probably won't be rendered because the + returned partial application won't have any reference to this location; + only pointing to the second lambda in the multiply function. + */ + __functor = self: multiply 2; + }; + + makeOverridable = f: { + /** + This is a function that can be overridden. + */ + __functor = self: f; + override = throw "not implemented"; + }; + + /** + Compute x^2 + */ + square = x: x * x; + + helper = makeOverridable square; + + # Somewhat analogous to the Nixpkgs makeOverridable function. + makeVeryOverridable = f: { + /** + This is a function that can be overridden. + */ + __functor = self: arg: f arg // { override = throw "not implemented"; overrideAttrs = throw "not implemented"; }; + override = throw "not implemented"; + }; + + helper2 = makeVeryOverridable square; + + # The RFC might be ambiguous here. The doc comment from makeVeryOverridable + # is "inner" in terms of values, but not inner in terms of expressions. + # Returning the following attribute comment might be allowed. + # TODO: I suppose we could look whether the attribute value expression + # contains a doc, and if not, return the attribute comment anyway? + + /** + Compute x^3 + */ + lib.helper3 = makeVeryOverridable (x: x * x * x); + + /** + Compute x^3... + */ + helper3 = makeVeryOverridable (x: x * x * x); + + + # ------ + + # getDoc traverses a potentially infinite structure in case of __functor, so + # we need to test with recursive inputs and diverging inputs. + + recursive = { + /** + This looks bad, but the docs are ok because of the eta expansion. + */ + __functor = self: x: self x; + }; + + recursive2 = { + /** + Docs probably won't work in this case, because the "partial" application + of self results in an infinite recursion. + */ + __functor = self: self.__functor self; + }; + + diverging = let + /** + Docs probably won't work in this case, because the "partial" application + of self results in an diverging computation that causes a stack overflow. + It's not an infinite recursion because each call is different. + This must be handled by the documentation retrieval logic, as it + reimplements the __functor invocation to be partial. + */ + f = x: { + __functor = self: (f (x + 1)); + }; + in f null; + +} From e225b630621712e407d610109d6e866c06e1e948 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 15 Aug 2024 13:51:15 +0200 Subject: [PATCH 228/284] doc: Document function application operator --- doc/manual/src/language/operators.md | 22 ++++++++++++++++++++-- doc/manual/src/language/syntax.md | 2 +- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 2c2dcc276ac..ce5b034c6ba 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -3,7 +3,7 @@ | Name | Syntax | Associativity | Precedence | |----------------------------------------|--------------------------------------------|---------------|------------| | [Attribute selection] | *attrset* `.` *attrpath* \[ `or` *expr* \] | none | 1 | -| Function application | *func* *expr* | left | 2 | +| [Function application] | *func* *expr* | left | 2 | | [Arithmetic negation][arithmetic] | `-` *number* | none | 3 | | [Has attribute] | *attrset* `?` *attrpath* | none | 4 | | List concatenation | *list* `++` *list* | right | 5 | @@ -32,7 +32,7 @@ [string]: ./types.md#type-string [path]: ./types.md#type-path [number]: ./types.md#type-float -[list]: ./types.md#list +[list]: ./types.md#type-list [attribute set]: ./types.md#attribute-set @@ -48,6 +48,22 @@ If the attribute doesn’t exist, return the *expr* after `or` if provided, othe [Attribute selection]: #attribute-selection +## Function application + +> **Syntax** +> +> *func* *expr* + +Apply the callable value *func* to the argument *expr*. Note the absence of a visible operator symbol. +A callable value is either: +- a [user-defined function][function] +- a [built-in][builtins] function +- an attribute set with a [`__functor` attribute](./syntax.md#attr-__functor) + +> **Warning** +> +> This "operator" also separates [list] items, which means that calls in list items must be enclosed by parentheses. + ## Has attribute > **Syntax** @@ -215,3 +231,5 @@ Equivalent to `!`*b1* `||` *b2*. > ``` [Pipe operator]: #pipe-operators +[builtins]: ./builtins.md +[Function application]: #function-application diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md index a6a3ea05223..506afbea130 100644 --- a/doc/manual/src/language/syntax.md +++ b/doc/manual/src/language/syntax.md @@ -218,7 +218,7 @@ a string), that attribute is simply not added to the set: This will evaluate to `{}` if `foo` evaluates to `false`. -A set that has a `__functor` attribute whose value is callable (i.e. is +A set that has a [`__functor`]{#attr-__functor} attribute whose value is callable (i.e. is itself a function or a set with a `__functor` attribute whose value is callable) can be applied as if it were a function, with the set itself passed in first , e.g., From ce62b766ef976f941b1be8a580afc2c1dc88e17f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 15 Aug 2024 15:25:50 +0200 Subject: [PATCH 229/284] fix link from the readme (#11307) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 021e54a3b23..ab647e53b1f 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Full reference documentation can be found in the [Nix manual](https://nix.dev/re ## Building and developing -Follow instructions in the Nix reference manual to [set up a development environment and build Nix from source](https://nix.dev/manual/nix/development/building.html). +Follow instructions in the Nix reference manual to [set up a development environment and build Nix from source](https://nix.dev/manual/nix/development/development/building.html). ## Contributing From 06b18cff2021d7cca21ad013fea6da0c43109032 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 15 Aug 2024 18:53:42 +0200 Subject: [PATCH 230/284] doc: Edit language/operators Co-authored-by: John Ericson Co-authored-by: Valentin Gagarin --- doc/manual/src/language/operators.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index ce5b034c6ba..27444258a15 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -54,7 +54,7 @@ If the attribute doesn’t exist, return the *expr* after `or` if provided, othe > > *func* *expr* -Apply the callable value *func* to the argument *expr*. Note the absence of a visible operator symbol. +Apply the callable value *func* to the argument *expr*. Note the absence of any visible operator symbol. A callable value is either: - a [user-defined function][function] - a [built-in][builtins] function @@ -62,7 +62,7 @@ A callable value is either: > **Warning** > -> This "operator" also separates [list] items, which means that calls in list items must be enclosed by parentheses. +> [List][list] items are also separated by whitespace, which means that function calls in list items must be enclosed by parentheses. ## Has attribute From 30af4a9e27cdafd4be6997c9764c5c9b229ea550 Mon Sep 17 00:00:00 2001 From: Michael Gallagher Date: Thu, 15 Aug 2024 17:56:05 -0700 Subject: [PATCH 231/284] nix-daemon.sh profile script: operate under `set -u` in bash see https://github.com/NixOS/nix/commit/d459d3307c7b1b10f6489ed048fff192e7834928 --- scripts/nix-profile-daemon.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index eb124c0b5c0..59c00d49191 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -52,7 +52,7 @@ elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS else # Fall back to what is in the nix profiles, favouring whatever is defined last. check_nix_profiles() { - if [ -n "$ZSH_VERSION" ]; then + if [ -n "${ZSH_VERSION:-}" ]; then # Zsh by default doesn't split words in unquoted parameter expansion. # Set local_options for these options to be reverted at the end of the function # and shwordsplit to force splitting words in $NIX_PROFILES below. From c4192a66174cb1c67840a97ba7bc0a706a2c1e2a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 16 Aug 2024 15:27:53 +0200 Subject: [PATCH 232/284] Add nix::execvpe --- src/libutil/unix/exec.hh | 13 +++++++++++++ src/libutil/unix/meson.build | 1 + src/libutil/unix/processes.cc | 7 +++++++ 3 files changed, 21 insertions(+) create mode 100644 src/libutil/unix/exec.hh diff --git a/src/libutil/unix/exec.hh b/src/libutil/unix/exec.hh new file mode 100644 index 00000000000..e6b80889aa9 --- /dev/null +++ b/src/libutil/unix/exec.hh @@ -0,0 +1,13 @@ +#pragma once + +namespace nix { + +/** + * `execvpe` is a GNU extension, so we need to implement it for other POSIX + * platforms. + * + * We use our own implementation unconditionally for consistency. + */ +int execvpe(const char * file0, char * const argv[], char * const envp[]); + +} diff --git a/src/libutil/unix/meson.build b/src/libutil/unix/meson.build index 1c5bf27fb14..d36152db973 100644 --- a/src/libutil/unix/meson.build +++ b/src/libutil/unix/meson.build @@ -13,6 +13,7 @@ sources += files( include_dirs += include_directories('.') headers += files( + 'exec.hh', 'monitor-fd.hh', 'signals-impl.hh', ) diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index c5ce74acc33..09acba35a95 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -1,5 +1,6 @@ #include "current-process.hh" #include "environment-variables.hh" +#include "executable-path.hh" #include "signals.hh" #include "processes.hh" #include "finally.hh" @@ -419,4 +420,10 @@ bool statusOk(int status) return WIFEXITED(status) && WEXITSTATUS(status) == 0; } +int execvpe(const char * file0, char * const argv[], char * const envp[]) +{ + auto file = ExecutablePath::load().findPath(file0).string(); + return execve(file.c_str(), argv, envp); +} + } From a03bb4455cee010bbfcf7e322b10ec7e35123032 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 3 Jul 2024 16:51:25 +0200 Subject: [PATCH 233/284] Fix SSH invocation when local SHELL misbehaves Setting it to /bin/sh will make it more predictable when users have their favorite shell in SHELL, which might not behave as expected. For instance, a bad rc file could send something to stdout before our LocalCommand gets to write "started". This may help https://github.com/NixOS/nix/issues/11010 --- src/libstore/ssh.cc | 33 ++++++++++++++++++++++++++++++--- tests/nixos/remote-builds.nix | 17 ++++++++++++++++- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index e5d623adf3a..b8c5f4d975e 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -3,6 +3,7 @@ #include "current-process.hh" #include "environment-variables.hh" #include "util.hh" +#include "exec.hh" namespace nix { @@ -44,6 +45,10 @@ void SSHMaster::addCommonSSHOpts(Strings & args) if (compress) args.push_back("-C"); + // We use this to make ssh signal back to us that the connection is established. + // It really does run locally; see createSSHEnv which sets up SHELL to make + // it launch more reliably. The local command runs synchronously, so presumably + // the remote session won't be garbled if the local command is slow. args.push_back("-oPermitLocalCommand=yes"); args.push_back("-oLocalCommand=echo started"); } @@ -56,6 +61,27 @@ bool SSHMaster::isMasterRunning() { return res.first == 0; } +Strings createSSHEnv() +{ + // Copy the environment and set SHELL=/bin/sh + std::map env = getEnv(); + + // SSH will invoke the "user" shell for -oLocalCommand, but that means + // $SHELL. To keep things simple and avoid potential issues with other + // shells, we set it to /bin/sh. + // Technically, we don't need that, and we could reinvoke ourselves to print + // "started". Self-reinvocation is tricky with library consumers, but mostly + // solved; refer to the development history of nixExePath in libstore/globals.cc. + env.insert_or_assign("SHELL", "/bin/sh"); + + Strings r; + for (auto & [k, v] : env) { + r.push_back(k + "=" + v); + } + + return r; +} + std::unique_ptr SSHMaster::startCommand( Strings && command, Strings && extraSshArgs) { @@ -104,8 +130,8 @@ std::unique_ptr SSHMaster::startCommand( } args.splice(args.end(), std::move(command)); - - execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); + auto env = createSSHEnv(); + nix::execvpe(args.begin()->c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(env).data()); // could not exec ssh/bash throw SysError("unable to execute '%s'", args.front()); @@ -172,7 +198,8 @@ Path SSHMaster::startMaster() if (verbosity >= lvlChatty) args.push_back("-v"); addCommonSSHOpts(args); - execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); + auto env = createSSHEnv(); + nix::execvpe(args.begin()->c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(env).data()); throw SysError("unable to execute '%s'", args.front()); }, options); diff --git a/tests/nixos/remote-builds.nix b/tests/nixos/remote-builds.nix index 8ddf6ad0281..ab159eaadf0 100644 --- a/tests/nixos/remote-builds.nix +++ b/tests/nixos/remote-builds.nix @@ -81,6 +81,17 @@ in virtualisation.additionalPaths = [ config.system.build.extraUtils ]; nix.settings.substituters = lib.mkForce [ ]; programs.ssh.extraConfig = "ConnectTimeout 30"; + environment.systemPackages = [ + # `bad-shell` is used to make sure Nix works an environment with a misbehaving shell. + # + # More realistically, a bad shell would still run the command ("echo started") + # but considering that our solution is to avoid this shell (set via $SHELL), we + # don't need to bother with a more functional mock shell. + (pkgs.writeScriptBin "bad-shell" '' + #!${pkgs.runtimeShell} + echo "Hello, I am a broken shell" + '') + ]; }; }; @@ -114,9 +125,13 @@ in 'echo hello world on $(hostname)' >&2 """) + # Check that SSH uses SHELL for LocalCommand, as expected, and check that + # our test setup here is working. The next test will use this bad SHELL. + client.succeed(f"SHELL=$(which bad-shell) ssh -oLocalCommand='true' -oPermitLocalCommand=yes {builder1.name} 'echo hello world' | grep -F 'Hello, I am a broken shell'") + # Perform a build and check that it was performed on the builder. out = client.succeed( - "nix-build ${expr nodes.client 1} 2> build-output", + "SHELL=$(which bad-shell) nix-build ${expr nodes.client 1} 2> build-output", "grep -q Hello build-output" ) builder1.succeed(f"test -e {out}") From 8866d2cd838902d45782541efe08efc1e1f1a2ab Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Fri, 16 Aug 2024 07:09:27 -0700 Subject: [PATCH 234/284] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/63d37ccd2d178d54e7fb691d7ec76000740ea24a?narHash=sha256-7cCC8%2BTdq1%2B3OPyc3%2BgVo9dzUNkNIQfwSDJ2HSi2u3o%3D' (2024-07-21) → 'github:NixOS/nixpkgs/c3d4ac725177c030b1e289015989da2ad9d56af0?narHash=sha256-sqLwJcHYeWLOeP/XoLwAtYjr01TISlkOfz%2BNG82pbdg%3D' (2024-08-15) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 2ac413a6986..b5d0b881c5c 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1721548954, - "narHash": "sha256-7cCC8+Tdq1+3OPyc3+gVo9dzUNkNIQfwSDJ2HSi2u3o=", + "lastModified": 1723688146, + "narHash": "sha256-sqLwJcHYeWLOeP/XoLwAtYjr01TISlkOfz+NG82pbdg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "63d37ccd2d178d54e7fb691d7ec76000740ea24a", + "rev": "c3d4ac725177c030b1e289015989da2ad9d56af0", "type": "github" }, "original": { From aa3d35c1f4145c9532620a20d6727c2214eab054 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Fri, 16 Aug 2024 07:22:30 -0700 Subject: [PATCH 235/284] ci: check that all outputs for all systems can evaluate --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9831d0e0e5e..3463335b95e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,7 @@ jobs: done ) & - run: nix --experimental-features 'nix-command flakes' flake check -L + - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json # Steps to test CI automation in your own fork. # Cachix: From d4aa7d5dc78f65b68b4c5f6c721b6fb3ae045874 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 14 Aug 2024 23:19:22 +0200 Subject: [PATCH 236/284] Use nixosTest.quickBuild behavior by default This wasn't the default behaviour because: > We don't enable this by default to avoid the mostly unnecessary work of > performing an additional build of the package in cases where we build > the package normally anyway, such as in our pre-merge CI. Since we have a componentized build, we've solved the duplication. In the new situation, building both with and without unit tests isn't any slow than just a build with unit tests, so there's no point in using the unit-tested build anymore. By using the otherwise untested build, we reduce the minimum build time towards the NixOS test, at no cost. If you want to run all tests, build all attributes. --- doc/manual/src/development/testing.md | 8 ++--- tests/nixos/default.nix | 13 ++++++-- tests/nixos/quick-build.nix | 47 --------------------------- 3 files changed, 12 insertions(+), 56 deletions(-) delete mode 100644 tests/nixos/quick-build.nix diff --git a/doc/manual/src/development/testing.md b/doc/manual/src/development/testing.md index 3949164d503..a1782d86c2e 100644 --- a/doc/manual/src/development/testing.md +++ b/doc/manual/src/development/testing.md @@ -276,14 +276,12 @@ To ensure that characterisation testing doesn't make it harder to intentionally We run the functional tests not just in the build, but also in VM tests. This helps us ensure that Nix works correctly on NixOS, and environments that have similar characteristics that are hard to reproduce in a build environment. -The recommended way to run these tests during development is: +These can be run with: ```shell -nix build .#hydraJobs.tests.functional_user.quickBuild +nix build .#hydraJobs.tests.functional_user ``` -The `quickBuild` attribute configures the test to use a `nix` package that's built without integration tests, so that you can iterate on the tests without performing recompilations due to the changed sources for `installCheck`. - Generally, this build is sufficient, but in nightly or CI we also test the attributes `functional_root` and `functional_trusted`, in which the test suite is run with different levels of authorization. ## Integration tests @@ -294,8 +292,6 @@ Because these tests are expensive and require more than what the standard github You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`. -If you are testing a build of `nix` that you haven't compiled yet, you may iterate faster by appending the `quickBuild` attribute: `nix build .#hydraJobs.tests.{testName}.quickBuild`. - ## Installer tests After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 66174c928e5..3fa341ef1e5 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -4,20 +4,24 @@ let nixos-lib = import (nixpkgs + "/nixos/lib") { }; + noTests = pkg: pkg.overrideAttrs ( + finalAttrs: prevAttrs: { + doCheck = false; + doInstallCheck = false; + }); + # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests runNixOSTestFor = system: test: (nixos-lib.runTest { imports = [ test - - # Add the quickBuild attribute to the check packages - ./quick-build.nix ]; hostPkgs = nixpkgsFor.${system}.native; defaults = { nixpkgs.pkgs = nixpkgsFor.${system}.native; nix.checkAllErrors = false; + nix.package = noTests nixpkgsFor.${system}.native.nix; }; _module.args.nixpkgs = nixpkgs; _module.args.system = system; @@ -29,6 +33,9 @@ let forNix = nixVersion: runNixOSTestFor system { imports = [test]; defaults.nixpkgs.overlays = [(curr: prev: { + # NOTE: noTests pkg might not have been built yet for some older versions of the package + # and in versions before 2.25, the untested build wasn't shared with the tested build yet + # Add noTests here when those versions become irrelevant. nix = (builtins.getFlake "nix/${nixVersion}").packages.${system}.nix; })]; }; diff --git a/tests/nixos/quick-build.nix b/tests/nixos/quick-build.nix deleted file mode 100644 index 57e3b9cdbd3..00000000000 --- a/tests/nixos/quick-build.nix +++ /dev/null @@ -1,47 +0,0 @@ -test@{ lib, extendModules, ... }: -let - inherit (lib) mkOption types; -in -{ - options = { - quickBuild = mkOption { - description = '' - Whether to perform a "quick" build of the Nix package to test. - - When iterating on the functional tests, it's recommended to "set" this - to `true`, so that changes to the functional tests don't require any - recompilation of the package. - You can do so by building the `.quickBuild` attribute on the check package, - e.g: - ```console - nix build .#hydraJobs.functional_user.quickBuild - ``` - - We don't enable this by default to avoid the mostly unnecessary work of - performing an additional build of the package in cases where we build - the package normally anyway, such as in our pre-merge CI. - ''; - type = types.bool; - default = false; - }; - }; - - config = { - passthru.quickBuild = - let withQuickBuild = extendModules { modules = [{ quickBuild = true; }]; }; - in withQuickBuild.config.test; - - defaults = { pkgs, ... }: { - config = lib.mkIf test.config.quickBuild { - nix.package = pkgs.nixComponents.nix-cli; - - system.forbiddenDependenciesRegexes = [ - # This would indicate that the quickBuild feature is broken. - # It could happen if NixOS has a dependency on pkgs.nix instead of - # config.nix.package somewhere. - (builtins.unsafeDiscardStringContext pkgs.nix.outPath) - ]; - }; - }; - }; -} From 4ba57c9eb28952039164d83fcd21f454794fa3fe Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Sat, 17 Aug 2024 02:46:58 -0400 Subject: [PATCH 237/284] ci: use attribute with version for docker --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3463335b95e..84e5ab998d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,7 +146,7 @@ jobs: with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV + - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV - uses: cachix/cachix-action@v15 if: needs.check_secrets.outputs.cachix == 'true' with: From 80f20fa4cb75ad48d74047ca060869bb9138f776 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 17 Aug 2024 08:31:41 +0200 Subject: [PATCH 238/284] replace backport github action with mergify The current backport action cannot automerge because the github action bot does not trigger github CI actions. Mergify instead does not have this limitation and can also use a merge queue. On top we have now a declarative configuration to allow contributers to add new tests to required without having access to the github org. An example pull request and backport can be seen here: https://github.com/Mic92/nix-1/pull/4 and here: https://github.com/Mic92/nix-1/pull/5 To complete the setup the mergify app must be enabled for this repository. It's already installed in the nixos organization for nixos-hardware and other repositories. --- .github/workflows/backport.yml | 32 ------------ .mergify.yml | 92 ++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 32 deletions(-) delete mode 100644 .github/workflows/backport.yml create mode 100644 .mergify.yml diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml deleted file mode 100644 index dd110de6c2a..00000000000 --- a/.github/workflows/backport.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Backport -on: - pull_request_target: - types: [closed, labeled] -permissions: - contents: read -jobs: - backport: - name: Backport Pull Request - permissions: - # for zeebe-io/backport-action - contents: write - pull-requests: write - if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name)) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - # required to find all branches - fetch-depth: 0 - - name: Create backport PRs - # should be kept in sync with `version` - uses: zeebe-io/backport-action@v3.0.2 - with: - # Config README: https://github.com/zeebe-io/backport-action#backport-action - github_token: ${{ secrets.GITHUB_TOKEN }} - github_workspace: ${{ github.workspace }} - pull_description: |- - Automatic backport to `${target_branch}`, triggered by a label in #${pull_number}. - # should be kept in sync with `uses` - version: v0.0.5 diff --git a/.mergify.yml b/.mergify.yml new file mode 100644 index 00000000000..663c45d92f4 --- /dev/null +++ b/.mergify.yml @@ -0,0 +1,92 @@ +queue_rules: + - name: default + # all required tests need to go here + merge_conditions: + - check-success=installer + - check-success=installer_test (macos-latest) + - check-success=installer_test (ubuntu-latest) + - check-success=tests (macos-latest) + - check-success=tests (ubuntu-latest) + - check-success=vm_tests + merge_method: rebase + batch_size: 5 + +pull_request_rules: + - name: merge using the merge queue + conditions: + - base=master + - label~=merge-queue|dependencies + actions: + queue: {} + +# The rules below will first create backport pull requests and put those in a merge queue. + + - name: backport patches to 2.18 + conditions: + - label=backport 2.18-maintenance + actions: + backport: + branches: + - 2.18-maintenance + labels: + - merge-queue + + - name: backport patches to 2.19 + conditions: + - label=backport 2.19-maintenance + actions: + backport: + branches: + - 2.19-maintenance + labels: + - merge-queue + + - name: backport patches to 2.20 + conditions: + - label=backport 2.20-maintenance + actions: + backport: + branches: + - 2.20-maintenance + labels: + - merge-queue + + - name: backport patches to 2.21 + conditions: + - label=backport 2.21-maintenance + actions: + backport: + branches: + - 2.21-maintenance + labels: + - merge-queue + + - name: backport patches to 2.22 + conditions: + - label=backport 2.22-maintenance + actions: + backport: + branches: + - 2.22-maintenance + labels: + - merge-queue + + - name: backport patches to 2.23 + conditions: + - label=backport 2.23-maintenance + actions: + backport: + branches: + - 2.23-maintenance + labels: + - merge-queue + + - name: backport patches to 2.24 + conditions: + - label=backport 2.24-maintenance + actions: + backport: + branches: + - "2.24-maintenance" + labels: + - merge-queue From a5f6ee8550b78badffe48993b0a236b7934c1be5 Mon Sep 17 00:00:00 2001 From: shivaraj-bh Date: Wed, 14 Aug 2024 14:57:34 +0530 Subject: [PATCH 239/284] `nix flake show`: Support `meta` attribute for `apps` Metadata information for flake apps will be useful while exploring a flake using `nix flake show` --- src/nix/flake.cc | 12 +++++++++--- src/nix/run.md | 3 +++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3f9f8f99b06..09f41dca8b3 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1251,8 +1251,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON } j.emplace("type", "derivation"); j.emplace("name", name); - if (description) - j.emplace("description", *description); + j.emplace("description", description ? *description : ""); } else { logger->cout("%s: %s '%s'", headerPrefix, @@ -1340,12 +1339,19 @@ struct CmdFlakeShow : FlakeCommand, MixJSON (attrPath.size() == 3 && attrPathS[0] == "apps")) { auto aType = visitor.maybeGetAttr("type"); + std::optional description; + if (auto aMeta = visitor.maybeGetAttr(state->sMeta)) { + if (auto aDescription = aMeta->maybeGetAttr(state->sDescription)) + description = aDescription->getString(); + } if (!aType || aType->getString() != "app") state->error("not an app definition").debugThrow(); if (json) { j.emplace("type", "app"); + if (description) + j.emplace("description", *description); } else { - logger->cout("%s: app", headerPrefix); + logger->cout("%s: app: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description ? *description : "no description"); } } diff --git a/src/nix/run.md b/src/nix/run.md index 250ea65aa7c..eb96e6b319e 100644 --- a/src/nix/run.md +++ b/src/nix/run.md @@ -80,6 +80,7 @@ An app is specified by a flake output attribute named apps.x86_64-linux.blender_2_79 = { type = "app"; program = "${self.packages.x86_64-linux.blender_2_79}/bin/blender"; + meta.description = "Run Blender, a free and open-source 3D creation suite."; }; ``` @@ -90,4 +91,6 @@ The only supported attributes are: * `program` (required): The full path of the executable to run. It must reside in the Nix store. +* `meta.description` (optional): A description of the app. + )"" From adabca6e4f4a1863344992b51a44f943690ba42d Mon Sep 17 00:00:00 2001 From: shivaraj-bh Date: Fri, 16 Aug 2024 03:12:02 +0530 Subject: [PATCH 240/284] `nix flake check`: Add apps check; Check if formatter is a derivation --- src/nix/flake.cc | 41 +++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 09f41dca8b3..10cb6af9b12 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -437,14 +437,39 @@ struct CmdFlakeCheck : FlakeCommand auto checkApp = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { - #if 0 - // FIXME - auto app = App(*state, v); - for (auto & i : app.context) { - auto [drvPathS, outputName] = NixStringContextElem::parse(i); - store->parseStorePath(drvPathS); + Activity act(*logger, lvlInfo, actUnknown, fmt("checking app '%s'", attrPath)); + state->forceAttrs(v, pos, ""); + if (auto attr = v.attrs()->get(state->symbols.create("type"))) + state->forceStringNoCtx(*attr->value, attr->pos, ""); + else + throw Error("app '%s' lacks attribute 'type'", attrPath); + + if (auto attr = v.attrs()->get(state->symbols.create("program"))) { + if (attr->name == state->symbols.create("program")) { + NixStringContext context; + state->forceString(*attr->value, context, attr->pos, ""); + } + } else + throw Error("app '%s' lacks attribute 'program'", attrPath); + + if (auto attr = v.attrs()->get(state->symbols.create("meta"))) { + state->forceAttrs(*attr->value, attr->pos, ""); + if (auto dAttr = attr->value->attrs()->get(state->symbols.create("description"))) + state->forceStringNoCtx(*dAttr->value, dAttr->pos, ""); + else + logWarning({ + .msg = HintFmt("app '%s' lacks attribute 'meta.description'", attrPath), + }); + } else + logWarning({ + .msg = HintFmt("app '%s' lacks attribute 'meta'", attrPath), + }); + + for (auto & attr : *v.attrs()) { + std::string_view name(state->symbols[attr.name]); + if (name != "type" && name != "program" && name != "meta") + throw Error("app '%s' has unsupported attribute '%s'", attrPath, name); } - #endif } catch (Error & e) { e.addTrace(resolve(pos), HintFmt("while checking the app definition '%s'", attrPath)); reportError(e); @@ -629,7 +654,7 @@ struct CmdFlakeCheck : FlakeCommand const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); if (checkSystemType(attr_name, attr.pos)) { - checkApp( + checkDerivation( fmt("%s.%s", name, attr_name), *attr.value, attr.pos); }; From 2ab93fd5fda3f61f6b1560db7da21a34dbd13b7d Mon Sep 17 00:00:00 2001 From: shivaraj-bh Date: Fri, 16 Aug 2024 03:12:24 +0530 Subject: [PATCH 241/284] `nix flake check`: Add functional tests for apps and formatter --- tests/functional/flakes/check.sh | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/functional/flakes/check.sh b/tests/functional/flakes/check.sh index 3b83dcafe4b..27e73444ae0 100755 --- a/tests/functional/flakes/check.sh +++ b/tests/functional/flakes/check.sh @@ -91,3 +91,47 @@ nix flake check $flakeDir checkRes=$(nix flake check --all-systems --keep-going $flakeDir 2>&1 && fail "nix flake check --all-systems should have failed" || true) echo "$checkRes" | grepQuiet "packages.system-1.default" echo "$checkRes" | grepQuiet "packages.system-2.default" + +cat > $flakeDir/flake.nix < $flakeDir/flake.nix <&1 && fail "nix flake check --all-systems should have failed" || true) +echo "$checkRes" | grepQuiet "unknown-attr" + +cat > $flakeDir/flake.nix <&1 && fail "nix flake check --all-systems should have failed" || true) +echo "$checkRes" | grepQuiet "formatter.system-1" From 9ccad9708eac3593defab8a8f74d3acf1bb60870 Mon Sep 17 00:00:00 2001 From: siddhantCodes Date: Sun, 18 Aug 2024 18:27:20 +0530 Subject: [PATCH 242/284] Don't use `OS_STR` in `nix-channel.cc` --- src/nix-channel/nix-channel.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 52fa3c4f659..56d1d7abb77 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -166,7 +166,7 @@ static int main_nix_channel(int argc, char ** argv) { // Figure out the name of the `.nix-channels' file to use auto home = getHome(); - channelsList = settings.useXDGBaseDirectories ? createNixStateDir() + OS_STR("/channels") : home + OS_STR("/.nix-channels"); + channelsList = settings.useXDGBaseDirectories ? createNixStateDir() + "/channels" : home + "/.nix-channels"; nixDefExpr = getNixDefExpr(); // Figure out the name of the channels profile. From 67de1932774b834377f704d9d358ef3d4951d0ef Mon Sep 17 00:00:00 2001 From: Sandro Date: Sun, 18 Aug 2024 18:44:59 +0200 Subject: [PATCH 243/284] Remove duplicated section (#11324) --- doc/manual/src/release-notes/rl-2.24.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md index 5bcc1d79ced..08ec65be902 100644 --- a/doc/manual/src/release-notes/rl-2.24.md +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -261,12 +261,6 @@ Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) -- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) - - In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. - - Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) - - Setting to warn about large paths [#10778](https://github.com/NixOS/nix/pull/10778) Nix can now warn when evaluation of a Nix expression causes a large From 59db8fd62b5300afbbabb1e8a12d547b336a3bdf Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Sun, 18 Aug 2024 22:35:54 -0400 Subject: [PATCH 244/284] fix: check to see if there are any lines before --- src/nix-build/nix-build.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 0ce987d8a5c..a5b9e1e548e 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -163,7 +163,7 @@ static void main_nix_build(int argc, char * * argv) script = argv[1]; try { auto lines = tokenizeString(readFile(script), "\n"); - if (std::regex_search(lines.front(), std::regex("^#!"))) { + if (!lines.empty() && std::regex_search(lines.front(), std::regex("^#!"))) { lines.pop_front(); inShebang = true; for (int i = 2; i < argc; ++i) From 93853833478122a200045e6fc9cf45c532eed80c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 19 Aug 2024 15:17:13 +0200 Subject: [PATCH 245/284] Revert "Remove unit tests from old build system" `make check` was reverted too soon. The hacking guide wasn't brought up to date with the new workflow, and it's not clear how to use meson for everything. This reverts commit 6f3045c2a225dca7b1ed8a9c9dc27ab50f575900. --- Makefile | 19 ++++++++++++ Makefile.config.in | 1 + configure.ac | 22 ++++++++++++++ package.nix | 33 +++++++++++++++++++- tests/unit/libexpr-support/local.mk | 23 ++++++++++++++ tests/unit/libexpr/local.mk | 45 ++++++++++++++++++++++++++++ tests/unit/libfetchers/local.mk | 37 +++++++++++++++++++++++ tests/unit/libflake/local.mk | 43 ++++++++++++++++++++++++++ tests/unit/libstore-support/local.mk | 21 +++++++++++++ tests/unit/libstore/local.mk | 38 +++++++++++++++++++++++ tests/unit/libutil-support/local.mk | 19 ++++++++++++ tests/unit/libutil/local.mk | 37 +++++++++++++++++++++++ 12 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 tests/unit/libexpr-support/local.mk create mode 100644 tests/unit/libexpr/local.mk create mode 100644 tests/unit/libfetchers/local.mk create mode 100644 tests/unit/libflake/local.mk create mode 100644 tests/unit/libstore-support/local.mk create mode 100644 tests/unit/libstore/local.mk create mode 100644 tests/unit/libutil-support/local.mk create mode 100644 tests/unit/libutil/local.mk diff --git a/Makefile b/Makefile index b51ae6cc730..dbf510a3e9c 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,18 @@ makefiles += \ endif endif +ifeq ($(ENABLE_UNIT_TESTS), yes) +makefiles += \ + tests/unit/libutil/local.mk \ + tests/unit/libutil-support/local.mk \ + tests/unit/libstore/local.mk \ + tests/unit/libstore-support/local.mk \ + tests/unit/libfetchers/local.mk \ + tests/unit/libexpr/local.mk \ + tests/unit/libexpr-support/local.mk \ + tests/unit/libflake/local.mk +endif + ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes) ifdef HOST_UNIX makefiles += \ @@ -92,6 +104,13 @@ include mk/lib.mk # These must be defined after `mk/lib.mk`. Otherwise the first rule # incorrectly becomes the default target. +ifneq ($(ENABLE_UNIT_TESTS), yes) +.PHONY: check +check: + @echo "Unit tests are disabled. Configure without '--disable-unit-tests', or avoid calling 'make check'." + @exit 1 +endif + ifneq ($(ENABLE_FUNCTIONAL_TESTS), yes) .PHONY: installcheck installcheck: diff --git a/Makefile.config.in b/Makefile.config.in index e131484f61d..3100d207365 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -12,6 +12,7 @@ ENABLE_BUILD = @ENABLE_BUILD@ ENABLE_DOC_GEN = @ENABLE_DOC_GEN@ ENABLE_FUNCTIONAL_TESTS = @ENABLE_FUNCTIONAL_TESTS@ ENABLE_S3 = @ENABLE_S3@ +ENABLE_UNIT_TESTS = @ENABLE_UNIT_TESTS@ GTEST_LIBS = @GTEST_LIBS@ HAVE_LIBCPUID = @HAVE_LIBCPUID@ HAVE_SECCOMP = @HAVE_SECCOMP@ diff --git a/configure.ac b/configure.ac index 18d718c0737..5c22ed17636 100644 --- a/configure.ac +++ b/configure.ac @@ -141,6 +141,18 @@ AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]), ENABLE_BUILD=$enableval, ENABLE_BUILD=yes) AC_SUBST(ENABLE_BUILD) +# Building without unit tests is useful for bootstrapping with a smaller footprint +# or running the tests in a separate derivation. Otherwise, we do compile and +# run them. + +AC_ARG_ENABLE(unit-tests, AS_HELP_STRING([--disable-unit-tests],[Do not build the tests]), + ENABLE_UNIT_TESTS=$enableval, ENABLE_UNIT_TESTS=$ENABLE_BUILD) +AC_SUBST(ENABLE_UNIT_TESTS) + +AS_IF( + [test "$ENABLE_BUILD" == "no" && test "$ENABLE_UNIT_TESTS" == "yes"], + [AC_MSG_ERROR([Cannot enable unit tests when building overall is disabled. Please do not pass '--enable-unit-tests' or do not pass '--disable-build'.])]) + AC_ARG_ENABLE(functional-tests, AS_HELP_STRING([--disable-functional-tests],[Do not build the tests]), ENABLE_FUNCTIONAL_TESTS=$enableval, ENABLE_FUNCTIONAL_TESTS=yes) AC_SUBST(ENABLE_FUNCTIONAL_TESTS) @@ -346,6 +358,16 @@ if test "$gc" = yes; then CFLAGS="$old_CFLAGS" fi +AS_IF([test "$ENABLE_UNIT_TESTS" == "yes"],[ + +# Look for gtest. +PKG_CHECK_MODULES([GTEST], [gtest_main gmock_main]) + +# Look for rapidcheck. +PKG_CHECK_MODULES([RAPIDCHECK], [rapidcheck rapidcheck_gtest]) + +]) + # Look for nlohmann/json. PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9]) diff --git a/package.nix b/package.nix index c0d04179df7..d41748b7cf5 100644 --- a/package.nix +++ b/package.nix @@ -53,6 +53,10 @@ # Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix , doBuild ? true +# Run the unit tests as part of the build. See `installUnitTests` for an +# alternative to this. +, doCheck ? __forDefaults.canRunInstalled + # Run the functional tests as part of the build. , doInstallCheck ? test-client != null || __forDefaults.canRunInstalled @@ -85,6 +89,11 @@ # - readline , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" +# Whether to install unit tests. This is useful when cross compiling +# since we cannot run them natively during the build, but can do so +# later. +, installUnitTests ? doBuild && !__forDefaults.canExecuteHost + # For running the functional tests against a pre-built Nix. Probably # want to use in conjunction with `doBuild = false;`. , test-daemon ? null @@ -108,7 +117,7 @@ let # things which should instead be gotten via `finalAttrs` in order to # work with overriding. attrs = { - inherit doBuild doInstallCheck; + inherit doBuild doCheck doInstallCheck; }; mkDerivation = @@ -124,11 +133,16 @@ in mkDerivation (finalAttrs: let inherit (finalAttrs) + doCheck doInstallCheck ; doBuild = !finalAttrs.dontBuild; + # Either running the unit tests during the build, or installing them + # to be run later, requiresthe unit tests to be built. + buildUnitTests = doCheck || installUnitTests; + in { inherit pname version; @@ -162,6 +176,8 @@ in { ./scripts/local.mk ] ++ lib.optionals enableManual [ ./doc/manual + ] ++ lib.optionals buildUnitTests [ + ./tests/unit ] ++ lib.optionals doInstallCheck [ ./tests/functional ])); @@ -174,6 +190,8 @@ in { # If we are doing just build or just docs, the one thing will use # "out". We only need additional outputs if we are doing both. ++ lib.optional (doBuild && enableManual) "doc" + ++ lib.optional installUnitTests "check" + ++ lib.optional doCheck "testresults" ; nativeBuildInputs = [ @@ -212,6 +230,9 @@ in { ({ inherit readline editline; }.${readlineFlavor}) ] ++ lib.optionals enableMarkdown [ lowdown + ] ++ lib.optionals buildUnitTests [ + gtest + rapidcheck ] ++ lib.optional stdenv.isLinux libseccomp ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies @@ -226,16 +247,22 @@ in { ); dontBuild = !attrs.doBuild; + doCheck = attrs.doCheck; configureFlags = [ (lib.enableFeature doBuild "build") + (lib.enableFeature buildUnitTests "unit-tests") (lib.enableFeature doInstallCheck "functional-tests") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature enableGC "gc") (lib.enableFeature enableMarkdown "markdown") + (lib.enableFeature installUnitTests "install-unit-tests") (lib.withFeatureAs true "readline-flavor" readlineFlavor) ] ++ lib.optionals (!forDevShell) [ "--sysconfdir=/etc" + ] ++ lib.optionals installUnitTests [ + "--with-check-bin-dir=${builtins.placeholder "check"}/bin" + "--with-check-lib-dir=${builtins.placeholder "check"}/lib" ] ++ lib.optionals (doBuild) [ "--with-boost=${boost}/lib" ] ++ lib.optionals (doBuild && stdenv.isLinux) [ @@ -316,6 +343,10 @@ in { platforms = lib.platforms.unix ++ lib.platforms.windows; mainProgram = "nix"; broken = !(lib.all (a: a) [ + # We cannot run or install unit tests if we don't build them or + # Nix proper (which they depend on). + (installUnitTests -> doBuild) + (doCheck -> doBuild) # The build process for the manual currently requires extracting # data from the Nix executable we are trying to document. (enableManual -> doBuild) diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk new file mode 100644 index 00000000000..0501de33c43 --- /dev/null +++ b/tests/unit/libexpr-support/local.mk @@ -0,0 +1,23 @@ +libraries += libexpr-test-support + +libexpr-test-support_NAME = libnixexpr-test-support + +libexpr-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-test-support_INSTALL_DIR := $(checklibdir) +else + libexpr-test-support_INSTALL_DIR := +endif + +libexpr-test-support_SOURCES := \ + $(wildcard $(d)/tests/*.cc) \ + $(wildcard $(d)/tests/value/*.cc) + +libexpr-test-support_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) + +libexpr-test-support_LIBS = \ + libstore-test-support libutil-test-support \ + libexpr libstore libutil + +libexpr-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk new file mode 100644 index 00000000000..1617e282351 --- /dev/null +++ b/tests/unit/libexpr/local.mk @@ -0,0 +1,45 @@ +check: libexpr-tests_RUN + +programs += libexpr-tests + +libexpr-tests_NAME := libnixexpr-tests + +libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libexpr-tests.xml + +libexpr-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-tests_INSTALL_DIR := $(checkbindir) +else + libexpr-tests_INSTALL_DIR := +endif + +libexpr-tests_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) \ + $(wildcard $(d)/flake/*.cc) + +libexpr-tests_EXTRA_INCLUDES = \ + -I tests/unit/libexpr-support \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + $(INCLUDE_libexpr) \ + $(INCLUDE_libexprc) \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) \ + $(INCLUDE_libstorec) \ + $(INCLUDE_libutil) \ + $(INCLUDE_libutilc) + +libexpr-tests_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) + +libexpr-tests_LIBS = \ + libexpr-test-support libstore-test-support libutil-test-support \ + libexpr libexprc libfetchers libstore libstorec libutil libutilc + +libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock + +ifdef HOST_WINDOWS + # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space + libexpr-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) +endif diff --git a/tests/unit/libfetchers/local.mk b/tests/unit/libfetchers/local.mk new file mode 100644 index 00000000000..30aa142a5e2 --- /dev/null +++ b/tests/unit/libfetchers/local.mk @@ -0,0 +1,37 @@ +check: libfetchers-tests_RUN + +programs += libfetchers-tests + +libfetchers-tests_NAME = libnixfetchers-tests + +libfetchers-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libfetchers-tests.xml + +libfetchers-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libfetchers-tests_INSTALL_DIR := $(checkbindir) +else + libfetchers-tests_INSTALL_DIR := +endif + +libfetchers-tests_SOURCES := $(wildcard $(d)/*.cc) + +libfetchers-tests_EXTRA_INCLUDES = \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) \ + $(INCLUDE_libutil) + +libfetchers-tests_CXXFLAGS += $(libfetchers-tests_EXTRA_INCLUDES) + +libfetchers-tests_LIBS = \ + libstore-test-support libutil-test-support \ + libfetchers libstore libutil + +libfetchers-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) $(LIBGIT2_LIBS) + +ifdef HOST_WINDOWS + # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space + libfetchers-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) +endif diff --git a/tests/unit/libflake/local.mk b/tests/unit/libflake/local.mk new file mode 100644 index 00000000000..590bcf7c031 --- /dev/null +++ b/tests/unit/libflake/local.mk @@ -0,0 +1,43 @@ +check: libflake-tests_RUN + +programs += libflake-tests + +libflake-tests_NAME := libnixflake-tests + +libflake-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libflake-tests.xml + +libflake-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libflake-tests_INSTALL_DIR := $(checkbindir) +else + libflake-tests_INSTALL_DIR := +endif + +libflake-tests_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) \ + $(wildcard $(d)/flake/*.cc) + +libflake-tests_EXTRA_INCLUDES = \ + -I tests/unit/libflake-support \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + $(INCLUDE_libflake) \ + $(INCLUDE_libexpr) \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) \ + $(INCLUDE_libutil) \ + +libflake-tests_CXXFLAGS += $(libflake-tests_EXTRA_INCLUDES) + +libflake-tests_LIBS = \ + libexpr-test-support libstore-test-support libutil-test-support \ + libflake libexpr libfetchers libstore libutil + +libflake-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock + +ifdef HOST_WINDOWS + # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space + libflake-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) +endif diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk new file mode 100644 index 00000000000..56dedd825d8 --- /dev/null +++ b/tests/unit/libstore-support/local.mk @@ -0,0 +1,21 @@ +libraries += libstore-test-support + +libstore-test-support_NAME = libnixstore-test-support + +libstore-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-test-support_INSTALL_DIR := $(checklibdir) +else + libstore-test-support_INSTALL_DIR := +endif + +libstore-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) + +libstore-test-support_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) + +libstore-test-support_LIBS = \ + libutil-test-support \ + libstore libutil + +libstore-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libstore/local.mk b/tests/unit/libstore/local.mk new file mode 100644 index 00000000000..8d3d6b0afe2 --- /dev/null +++ b/tests/unit/libstore/local.mk @@ -0,0 +1,38 @@ +check: libstore-tests_RUN + +programs += libstore-tests + +libstore-tests_NAME = libnixstore-tests + +libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libstore-tests.xml + +libstore-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-tests_INSTALL_DIR := $(checkbindir) +else + libstore-tests_INSTALL_DIR := +endif + +libstore-tests_SOURCES := $(wildcard $(d)/*.cc) + +libstore-tests_EXTRA_INCLUDES = \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + $(INCLUDE_libstore) \ + $(INCLUDE_libstorec) \ + $(INCLUDE_libutil) \ + $(INCLUDE_libutilc) + +libstore-tests_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) + +libstore-tests_LIBS = \ + libstore-test-support libutil-test-support \ + libstore libstorec libutil libutilc + +libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) + +ifdef HOST_WINDOWS + # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space + libstore-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) +endif diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk new file mode 100644 index 00000000000..5f7835c9f61 --- /dev/null +++ b/tests/unit/libutil-support/local.mk @@ -0,0 +1,19 @@ +libraries += libutil-test-support + +libutil-test-support_NAME = libnixutil-test-support + +libutil-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-test-support_INSTALL_DIR := $(checklibdir) +else + libutil-test-support_INSTALL_DIR := +endif + +libutil-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) + +libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) + +libutil-test-support_LIBS = libutil + +libutil-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libutil/local.mk b/tests/unit/libutil/local.mk new file mode 100644 index 00000000000..404f35cf1ac --- /dev/null +++ b/tests/unit/libutil/local.mk @@ -0,0 +1,37 @@ +check: libutil-tests_RUN + +programs += libutil-tests + +libutil-tests_NAME = libnixutil-tests + +libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libutil-tests.xml + +libutil-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-tests_INSTALL_DIR := $(checkbindir) +else + libutil-tests_INSTALL_DIR := +endif + +libutil-tests_SOURCES := $(wildcard $(d)/*.cc) + +libutil-tests_EXTRA_INCLUDES = \ + -I tests/unit/libutil-support \ + $(INCLUDE_libutil) \ + $(INCLUDE_libutilc) + +libutil-tests_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) + +libutil-tests_LIBS = libutil-test-support libutil libutilc + +libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) + +ifdef HOST_WINDOWS + # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space + libutil-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) +endif + +check: $(d)/data/git/check-data.sh.test + +$(eval $(call run-test,$(d)/data/git/check-data.sh)) From ceae25825fdb39fc973d23e7f0d2e355a6974bc9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 19 Aug 2024 10:24:38 -0400 Subject: [PATCH 246/284] Update documentation to refer to Meson not Make in most places This is necessary to make the Meson one the default and preferred one. Co-authored-by: Robert Hensing --- doc/manual/src/development/building.md | 88 +++++++++---------- doc/manual/src/development/testing.md | 58 ++++++------ .../src/installation/building-source.md | 21 ++--- maintainers/format.sh | 11 +++ maintainers/local.mk | 11 +-- 5 files changed, 89 insertions(+), 100 deletions(-) create mode 100755 maintainers/format.sh diff --git a/doc/manual/src/development/building.md b/doc/manual/src/development/building.md index 5a5fb33687f..dbf08029673 100644 --- a/doc/manual/src/development/building.md +++ b/doc/manual/src/development/building.md @@ -34,16 +34,21 @@ $ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages To build Nix itself in this shell: ```console -[nix-shell]$ autoreconfPhase -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out -[nix-shell]$ make -j $NIX_BUILD_CORES +[nix-shell]$ mesonFlags+=" --prefix=$(pwd)/outputs/out" +[nix-shell]$ dontAddPrefix=1 mesonConfigurePhase +[nix-shell]$ ninjaBuildPhase ``` -To install it in `$(pwd)/outputs` and test it: +To test it: ```console -[nix-shell]$ make install -[nix-shell]$ make installcheck -j $NIX_BUILD_CORES +[nix-shell]$ mesonCheckPhase +``` + +To install it in `$(pwd)/outputs`: + +```console +[nix-shell]$ ninjaInstallPhase [nix-shell]$ ./outputs/out/bin/nix --version nix (Nix) 2.12 ``` @@ -85,16 +90,20 @@ $ nix develop .#native-clangStdenvPackages To build Nix itself in this shell: ```console -[nix-shell]$ autoreconfPhase -[nix-shell]$ configurePhase -[nix-shell]$ make -j $NIX_BUILD_CORES OPTIMIZE=0 +[nix-shell]$ mesonConfigurePhase +[nix-shell]$ ninjaBuildPhase ``` -To install it in `$(pwd)/outputs` and test it: +To test it: ```console -[nix-shell]$ make install OPTIMIZE=0 -[nix-shell]$ make installcheck check -j $NIX_BUILD_CORES +[nix-shell]$ mesonCheckPhase +``` + +To install it in `$(pwd)/outputs`: + +```console +[nix-shell]$ ninjaInstallPhase [nix-shell]$ nix --version nix (Nix) 2.12 ``` @@ -110,25 +119,6 @@ $ nix build You can also build Nix for one of the [supported platforms](#platforms). -## Makefile variables - -You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run `make install`. - -Run `make` with [`-e` / `--environment-overrides`](https://www.gnu.org/software/make/manual/make.html#index-_002de) to allow environment variables to override `Makefile` variables: - -- `ENABLE_BUILD=yes` to enable building the C++ code. -- `ENABLE_DOC_GEN=yes` to enable building the documentation (manual, man pages, etc.). - - The docs can take a while to build, so you may want to disable this for local development. -- `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests. -- `OPTIMIZE=1` to enable optimizations. -- `libraries=libutil programs=` to only build a specific library. - - This will fail in the linking phase if the other libraries haven't been built, but is useful for checking types. -- `libraries= programs=nix` to only build a specific program. - - This will not work in general, because the programs need the libraries. - ## Platforms Nix can be built for various platforms, as specified in [`flake.nix`]: @@ -175,27 +165,38 @@ Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootst It is useful to perform multiple cross and native builds on the same source tree, for example to ensure that better support for one platform doesn't break the build for another. -In order to facilitate this, Nix has some support for being built out of tree – that is, placing build artefacts in a different directory than the source code: +Meson thankfully makes this very easy by confining all build products to the build directory --- one simple shares the source directory between multiple build directories, each of which contains the build for Nix to a different platform. + +Nixpkgs's `mesonConfigurePhase` always chooses `build` in the current directory as the name and location of the build. +This makes having multiple build directories slightly more inconvenient. +The good news is that Meson/Ninja seem to cope well with relocating the build directory after it is created. -1. Create a directory for the build, e.g. +Here's how to do that + +1. Configure as usual ```bash - mkdir build + mesonConfigurePhase ``` -2. Run the configure script from that directory, e.g. +2. Rename the build directory ```bash - cd build - ../configure + cd .. # since `mesonConfigurePhase` cd'd inside + mv build build-linux # or whatever name we want + cd build-linux ``` -3. Run make from the source directory, but with the build directory specified, e.g. +3. Build as usual ```bash - make builddir=build + ninjaBuildPhase ``` +> **N.B.** +> [`nixpkgs#335818`](https://github.com/NixOS/nixpkgs/issues/335818) tracks giving `mesonConfigurePhase` proper support for custom build directories. +> When it is fixed, we can simplify these instructions and then remove this notice. + ## System type Nix uses a string with the following format to identify the *system type* or *platform* it runs on: @@ -257,11 +258,8 @@ You can use any of the other supported environments in place of `nix-ccacheStden The `clangd` LSP server is installed by default on the `clang`-based `devShell`s. See [supported compilation environments](#compilation-environments) and instructions how to set up a shell [with flakes](#nix-with-flakes) or in [classic Nix](#classic-nix). -To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running: - -```console -make compile_commands.json -``` +To use the LSP with your editor, you will want a `compile_commands.json` file telling `clangd` how we are compiling the code. +Meson's configure always produces this inside the build directory. Configure your editor to use the `clangd` from the `.#native-clangStdenvPackages` shell. You can do that either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). @@ -276,7 +274,7 @@ Configure your editor to use the `clangd` from the `.#native-clangStdenvPackages You may run the formatters as a one-off using: ```console -make format +./maintainers/format.sh ``` If you'd like to run the formatters before every commit, install the hooks: diff --git a/doc/manual/src/development/testing.md b/doc/manual/src/development/testing.md index a1782d86c2e..8b8bc568d12 100644 --- a/doc/manual/src/development/testing.md +++ b/doc/manual/src/development/testing.md @@ -135,60 +135,52 @@ Functional tests are run during `installCheck` in the `nix` package build, as we ### Running the whole test suite -The whole test suite can be run with: +The whole test suite (functional and unit tests) can be run with: ```shell-session -$ make install && make installcheck -ran test tests/functional/foo.sh... [PASS] -ran test tests/functional/bar.sh... [PASS] -... +$ mesonCheckPhase ``` ### Grouping tests Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite. Each test group is in a subdirectory of `tests`. -For example, `tests/functional/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs. +For example, `tests/functional/ca/meson.build` defines a `ca` test group for content-addressed derivation outputs. That test group can be run like this: ```shell-session -$ make ca.test-group -j50 -ran test tests/functional/ca/nix-run.sh... [PASS] -ran test tests/functional/ca/import-derivation.sh... [PASS] -... -``` - -The test group is defined in Make like this: -```makefile -$(test-group-name)-tests := \ - $(d)/test0.sh \ - $(d)/test1.sh \ - ... - -install-tests-groups += $(test-group-name) +$ meson test --suite ca +ninja: Entering directory `/home/jcericson/src/nix/master/build' +ninja: no work to do. +[1-20/20] 🌑 nix-functional-tests:ca / ca/why-depends 1/20 nix-functional-tests:ca / ca/nix-run OK 0.16s +[2-20/20] 🌒 nix-functional-tests:ca / ca/why-depends 2/20 nix-functional-tests:ca / ca/import-derivation OK 0.17s ``` ### Running individual tests -Individual tests can be run with `make`: +Individual tests can be run with `meson`: ```shell-session -$ make tests/functional/${testName}.sh.test -ran test tests/functional/${testName}.sh... [PASS] +$ meson test ${testName} +ninja: Entering directory `/home/jcericson/src/nix/master/build' +ninja: no work to do. +1/1 nix-functional-tests:main / ${testName} OK 0.41s + +Ok: 1 +Expected Fail: 0 +Fail: 0 +Unexpected Pass: 0 +Skipped: 0 +Timeout: 0 + +Full log written to /home/jcericson/src/nix/master/build/meson-logs/testlog.txt ``` -or without `make`: +or without `meson`, showing the output: ```shell-session -$ ./mk/run-test.sh tests/functional/${testName}.sh -ran test tests/functional/${testName}.sh... [PASS] -``` - -To see the complete output, one can also run: - -```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh +$ TEST_NAME=${testName} NIX_REMOTE='' PS4='+(${BASH_SOURCE[0]-$0}:$LINENO) tests/functional/${testName}.sh +(${testName}.sh:1) foo output from foo +(${testName}.sh:2) bar @@ -254,7 +246,7 @@ It is frequently useful to regenerate the expected output. To do that, rerun the failed test(s) with `_NIX_TEST_ACCEPT=1`. For example: ```bash -_NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test +_NIX_TEST_ACCEPT=1 meson test lang ``` This convention is shared with the [characterisation unit tests](#characterisation-testing-unit) too. diff --git a/doc/manual/src/installation/building-source.md b/doc/manual/src/installation/building-source.md index 7dad9805a23..d35cc18c21d 100644 --- a/doc/manual/src/installation/building-source.md +++ b/doc/manual/src/installation/building-source.md @@ -1,31 +1,26 @@ # Building Nix from Source -After cloning Nix's Git repository, issue the following commands: +Nix is built with [Meson](https://mesonbuild.com/). +It is broken up into multiple Meson packages, which are optionally combined in a single project using Meson's [subprojects](https://mesonbuild.com/Subprojects.html) feature. -```console -$ autoreconf -vfi -$ ./configure options... -$ make -$ make install -``` +There are no mandatory extra steps to the building process: +generic Meson installation instructions like [this](https://mesonbuild.com/Quick-guide.html#using-meson-as-a-distro-packager) should work. -Nix requires GNU Make so you may need to invoke `gmake` instead. - -The installation path can be specified by passing the `--prefix=prefix` +The installation path can be specified by passing the `-Dprefix=prefix` to `configure`. The default installation directory is `/usr/local`. You can change this to any location you like. You must have write permission to the *prefix* path. Nix keeps its *store* (the place where packages are stored) in `/nix/store` by default. This can be changed using -`--with-store-dir=path`. +`-Dstore-dir=path`. > **Warning** -> +> > It is best *not* to change the Nix store from its default, since doing > so makes it impossible to use pre-built binaries from the standard > Nixpkgs channels — that is, all packages will need to be built from > source. Nix keeps state (such as its database and log files) in `/nix/var` by -default. This can be changed using `--localstatedir=path`. +default. This can be changed using `-Dlocalstatedir=path`. diff --git a/maintainers/format.sh b/maintainers/format.sh new file mode 100755 index 00000000000..a2a6d8b41af --- /dev/null +++ b/maintainers/format.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +if ! type -p pre-commit &>/dev/null; then + echo "format.sh: pre-commit not found. Please use \`nix develop\`."; + exit 1; +fi; +if test -z "$_NIX_PRE_COMMIT_HOOKS_CONFIG"; then + echo "format.sh: _NIX_PRE_COMMIT_HOOKS_CONFIG not set. Please use \`nix develop\`."; + exit 1; +fi; +pre-commit run --config "$_NIX_PRE_COMMIT_HOOKS_CONFIG" --all-files diff --git a/maintainers/local.mk b/maintainers/local.mk index 88d594d67d8..e81517eda14 100644 --- a/maintainers/local.mk +++ b/maintainers/local.mk @@ -3,13 +3,6 @@ print-top-help += echo ' format: Format source code' # This uses the cached .pre-commit-hooks.yaml file +fmt_script := $(d)/format.sh format: - @if ! type -p pre-commit &>/dev/null; then \ - echo "make format: pre-commit not found. Please use \`nix develop\`."; \ - exit 1; \ - fi; \ - if test -z "$$_NIX_PRE_COMMIT_HOOKS_CONFIG"; then \ - echo "make format: _NIX_PRE_COMMIT_HOOKS_CONFIG not set. Please use \`nix develop\`."; \ - exit 1; \ - fi; \ - pre-commit run --config $$_NIX_PRE_COMMIT_HOOKS_CONFIG --all-files + @$(fmt_script) From 2f0db04da08e67f29577c27fae3c0eb758fc0879 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 19 Aug 2024 16:41:40 +0200 Subject: [PATCH 247/284] tests.installer: Load profile with -o unset Tested with an ubuntu case. Might need revision depending on what hydra thinks of the rest. --- tests/installer/default.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/installer/default.nix b/tests/installer/default.nix index 4aed6eae489..7c82045ad2c 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -217,10 +217,16 @@ let $ssh < Date: Mon, 19 Aug 2024 16:48:33 +0200 Subject: [PATCH 248/284] Fix doc build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes ``` GEN /home/eelco/Dev/nix-master/outputs/out/share/doc/nix/manual/index.html error: File not found: ../store/types/ ┌─ release-notes/rl-next.md:60:197 │ 60 │ The build hook protocol did in principle support custom ways of remote building, but that can also be accomplished with a custom service for the ssh or daemon/ssh-ng protocols, or with a custom [store type](../store/types/) i.e. `Store` subclass. │ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File not found: ../store/types/ Error: One or more incorrect links 2024-08-19 16:47:57 [ERROR] (mdbook::renderer): Renderer exited with non-zero return code. 2024-08-19 16:47:57 [ERROR] (mdbook::utils): Error: Rendering failed 2024-08-19 16:47:57 [ERROR] (mdbook::utils): Caused By: The "linkcheck" renderer failed ``` --- doc/manual/rl-next/build-hook-default.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/rl-next/build-hook-default.md b/doc/manual/rl-next/build-hook-default.md index 197290536c3..0d5a130c091 100644 --- a/doc/manual/rl-next/build-hook-default.md +++ b/doc/manual/rl-next/build-hook-default.md @@ -17,6 +17,6 @@ This means that other applications linking `libnixstore` that wish to use remote Long term we don't envision this being a downside, because we plan to [get rid of `build-remote` and the build hook setting entirely](https://github.com/NixOS/nix/issues/1221). There is simply no need to add a second layer of remote-procedure-calling when we want to connect to a remote builder. -The build hook protocol did in principle support custom ways of remote building, but that can also be accomplished with a custom service for the ssh or daemon/ssh-ng protocols, or with a custom [store type](@docroot@/store/types/) i.e. `Store` subclass. +The build hook protocol did in principle support custom ways of remote building, but that can also be accomplished with a custom service for the ssh or daemon/ssh-ng protocols, or with a custom [store type](@docroot@/store/types/index.md) i.e. `Store` subclass. The Perl bindings no longer expose `getBinDir` either, since they libraries those bindings wrap no longer know the location of installed binaries as described above. From b52e58903e346a883c62d6af3b406e51e602f471 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 17 Aug 2024 13:34:45 +0200 Subject: [PATCH 249/284] doc: Manage expectations for eval-cache Incorrectly high expectations lead to frustration for users who stick around to experience how useless it is for e.g. a devShell https://functional.cafe/@arianvp/112976284363120036: > Flakes doesn't have eval caching. It has command line argument > caching. It literally just stores the cli argument you passed > in a sqlite database and yes that's as useless as it sounds > When I discovered flakes had no expression level caching whatsoever > I kind of felt lied to and betrayed. --- doc/manual/src/release-notes/rl-2.4.md | 2 ++ src/libexpr/eval-settings.hh | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 1201e53b6ae..dbec5a29d28 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -141,6 +141,8 @@ more than 2800 commits from 195 contributors since release 2.3. the evaluation cache. This is made possible by the hermetic evaluation model of flakes. + Intermediate results are not cached. + * The new `--offline` flag disables substituters and causes all locally cached tarballs and repositories to be considered up-to-date. diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 0cfc14c1b74..3d412bbbf57 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -185,7 +185,11 @@ struct EvalSettings : Config )"}; Setting useEvalCache{this, true, "eval-cache", - "Whether to use the flake evaluation cache."}; + R"( + Whether to use the flake evaluation cache. + Certain commands won't have to evaluate when invoked for the second time with a particular version of a flake. + Intermediate results are not cached. + )"}; Setting ignoreExceptionsDuringTry{this, false, "ignore-try", R"( From 3e5bf903413f420c1f997e4b55140761172b8434 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 2 Aug 2024 20:02:57 -0400 Subject: [PATCH 250/284] feat: better warning for common SSL errors --- src/libstore/filetransfer.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 5ea8b6f962c..58c52acc001 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -54,6 +54,8 @@ struct curlFileTransfer : public FileTransfer bool done = false; // whether either the success or failure function has been called Callback callback; CURL * req = 0; + // buffer to accompany the `req` above + char errbuf[CURL_ERROR_SIZE]; bool active = false; // whether the handle has been added to the multi object std::string statusMsg; @@ -370,6 +372,9 @@ struct curlFileTransfer : public FileTransfer if (writtenToSink) curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink); + curl_easy_setopt(req, CURLOPT_ERRORBUFFER, errbuf); + errbuf[0] = 0; + result.data.clear(); result.bodySize = 0; } @@ -484,8 +489,8 @@ struct curlFileTransfer : public FileTransfer code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) : FileTransferError(err, std::move(response), - "unable to %s '%s': %s (%d)", - request.verb(), request.uri, curl_easy_strerror(code), code); + "unable to %s '%s': %s (%d) %s", + request.verb(), request.uri, curl_easy_strerror(code), code, errbuf); /* If this is a transient error, then maybe retry the download after a while. If we're writing to a From 84ea12ad7fbedddf3f9bfc1d7c2159df43b9219b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 19 Aug 2024 11:02:46 -0400 Subject: [PATCH 251/284] Fix build errors on Windows --- src/libutil/executable-path.cc | 2 +- .../nix-collect-garbage.cc | 4 +- src/nix/eval.cc | 8 +--- src/nix/flake.cc | 43 ++++++++++--------- src/nix/profile.cc | 6 +-- 5 files changed, 31 insertions(+), 32 deletions(-) diff --git a/src/libutil/executable-path.cc b/src/libutil/executable-path.cc index 8005a19bed4..da71088e739 100644 --- a/src/libutil/executable-path.cc +++ b/src/libutil/executable-path.cc @@ -86,7 +86,7 @@ fs::path ExecutablePath::findPath(const fs::path & exe, std::functionsymbols[attr.name]; try { diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 6d440871813..b7bbb767b31 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -25,6 +25,8 @@ #include "strings-inline.hh" +namespace fs = std::filesystem; + using namespace nix; using namespace nix::flake; using json = nlohmann::json; @@ -897,25 +899,26 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand "If you've set '%s' to a string, try using a path instead.", templateDir, templateDirAttr->getAttrPathStr()).debugThrow(); - std::vector changedFiles; - std::vector conflictedFiles; + std::vector changedFiles; + std::vector conflictedFiles; - std::function copyDir; - copyDir = [&](const std::filesystem::path & from, const std::filesystem::path & to) + std::function copyDir; + copyDir = [&](const fs::path & from, const fs::path & to) { - createDirs(to); + fs::create_directories(to); - for (auto & entry : std::filesystem::directory_iterator{from}) { + for (auto & entry : fs::directory_iterator{from}) { checkInterrupt(); auto from2 = entry.path(); auto to2 = to / entry.path().filename(); auto st = entry.symlink_status(); - if (std::filesystem::is_directory(st)) + auto to_st = fs::symlink_status(to2); + if (fs::is_directory(st)) copyDir(from2, to2); - else if (std::filesystem::is_regular_file(st)) { - auto contents = readFile(from2); - if (pathExists(to2)) { - auto contents2 = readFile(to2); + else if (fs::is_regular_file(st)) { + auto contents = readFile(from2.string()); + if (fs::exists(to_st)) { + auto contents2 = readFile(to2.string()); if (contents != contents2) { printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2.string()); conflictedFiles.push_back(to2); @@ -924,12 +927,12 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand } continue; } else - writeFile(to2, contents); + writeFile(to2.string(), contents); } - else if (std::filesystem::is_symlink(st)) { - auto target = readLink(from2); - if (pathExists(to2)) { - if (readLink(to2) != target) { + else if (fs::is_symlink(st)) { + auto target = fs::read_symlink(from2); + if (fs::exists(to_st)) { + if (fs::read_symlink(to2) != target) { printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2.string()); conflictedFiles.push_back(to2); } else { @@ -937,7 +940,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand } continue; } else - createSymlink(target, to2); + fs::create_symlink(target, to2); } else throw Error("file '%s' has unsupported type", from2); @@ -948,9 +951,9 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand copyDir(templateDir, flakeDir); - if (!changedFiles.empty() && pathExists(flakeDir + "/.git")) { + if (!changedFiles.empty() && fs::exists(std::filesystem::path{flakeDir} / ".git")) { Strings args = { "-C", flakeDir, "add", "--intent-to-add", "--force", "--" }; - for (auto & s : changedFiles) args.push_back(s); + for (auto & s : changedFiles) args.emplace_back(s.string()); runProgram("git", true, args); } auto welcomeText = cursor->maybeGetAttr("welcomeText"); @@ -1275,7 +1278,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON if (auto aDescription = aMeta->maybeGetAttr(state->sDescription)) description = aDescription->getString(); } - + if (json) { j.emplace("type", "derivation"); j.emplace("name", name); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index d751abdb1f7..324fd633003 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -126,8 +126,8 @@ struct ProfileManifest { auto manifestPath = profile / "manifest.json"; - if (pathExists(manifestPath)) { - auto json = nlohmann::json::parse(readFile(manifestPath)); + if (std::filesystem::exists(manifestPath)) { + auto json = nlohmann::json::parse(readFile(manifestPath.string())); auto version = json.value("version", 0); std::string sUrl; @@ -176,7 +176,7 @@ struct ProfileManifest } } - else if (pathExists(profile / "manifest.nix")) { + else if (std::filesystem::exists(profile / "manifest.nix")) { // FIXME: needed because of pure mode; ugly. state.allowPath(state.store->followLinksToStore(profile.string())); state.allowPath(state.store->followLinksToStore((profile / "manifest.nix").string())); From 9243457cb24dd9734bbc8b9c4b532954ca6caba8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 19 Aug 2024 17:07:22 +0200 Subject: [PATCH 252/284] Fix umount failure Fixes ``` umount: /tmp/nix-shell.i3xRwX/nix-test/local-overlay-store/delete-refs/stores/merged-store/nix/store: filesystem was unmounted, but failed to update userspace mount table. make: *** [mk/lib.mk:93: tests/functional/local-overlay-store/delete-refs.sh.test] Error 16 ``` in a dev shell. Note: this previously worked before we didn't have umount in the dev shell, so we got /run/wrappers/bin/umount. --- tests/functional/local-overlay-store/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/local-overlay-store/common.sh b/tests/functional/local-overlay-store/common.sh index 27338ea23e8..cbdb3a1f77c 100644 --- a/tests/functional/local-overlay-store/common.sh +++ b/tests/functional/local-overlay-store/common.sh @@ -69,7 +69,7 @@ mountOverlayfs () { || skipTest "overlayfs is not supported" cleanupOverlay () { - umount "$storeBRoot/nix/store" + umount -n "$storeBRoot/nix/store" rm -r $storeVolume/workdir } trap cleanupOverlay EXIT From 39daa4a0d3e7451680e070d0bff7998ec0aea787 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 19 Aug 2024 17:49:26 +0200 Subject: [PATCH 253/284] withFramedSink(): Don't use a thread to monitor the other side Since withFramedSink() is now used a lot more than in the past (for every addToStore() variant), we were creating a lot of threads, e.g. nix flake show --no-eval-cache --all-systems github:NixOS/nix/afdd12be5e19c0001ff3297dea544301108d298 would create 46418 threads. While threads on Linux are cheap, this is still substantial overhead. So instead, just poll from FramedSink before every write whether there are pending messages from the daemon. This could slightly increase the latency on log messages from the daemon, but not on exceptions (which were only synchronously checked from FramedSink anyway). This speeds up the command above from 19.2s to 17.5s on my machine (a 9% speedup). --- src/libstore/remote-store-connection.hh | 2 +- src/libstore/remote-store.cc | 42 +++++----------------- src/libstore/worker-protocol-connection.cc | 15 +++++--- src/libstore/worker-protocol-connection.hh | 5 +-- src/libutil/serialise.cc | 20 +++++++++++ src/libutil/serialise.hh | 29 +++++++++------ 6 files changed, 61 insertions(+), 52 deletions(-) diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh index 405120ee926..513bd6838bf 100644 --- a/src/libstore/remote-store-connection.hh +++ b/src/libstore/remote-store-connection.hh @@ -49,7 +49,7 @@ struct RemoteStore::ConnectionHandle RemoteStore::Connection & operator * () { return *handle; } RemoteStore::Connection * operator -> () { return &*handle; } - void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); + void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true, bool block = true); void withFramedSink(std::function fun); }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 555936c186f..69bbc64fca3 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -153,9 +153,9 @@ RemoteStore::ConnectionHandle::~ConnectionHandle() } } -void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, bool flush) +void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, bool flush, bool block) { - handle->processStderr(&daemonException, sink, source, flush); + handle->processStderr(&daemonException, sink, source, flush, block); } @@ -926,43 +926,17 @@ void RemoteStore::ConnectionHandle::withFramedSink(std::functionto.flush(); - std::exception_ptr ex; - - /* Handle log messages / exceptions from the remote on a separate - thread. */ - std::thread stderrThread([&]() - { - try { - ReceiveInterrupts receiveInterrupts; - processStderr(nullptr, nullptr, false); - } catch (...) { - ex = std::current_exception(); - } - }); - - Finally joinStderrThread([&]() { - if (stderrThread.joinable()) { - stderrThread.join(); - if (ex) { - try { - std::rethrow_exception(ex); - } catch (...) { - ignoreException(); - } - } - } - }); - - { - FramedSink sink((*this)->to, ex); + FramedSink sink((*this)->to, [&]() { + /* Periodically process stderr messages and exceptions + from the daemon. */ + processStderr(nullptr, nullptr, false, false); + }); fun(sink); sink.flush(); } - stderrThread.join(); - if (ex) - std::rethrow_exception(ex); + processStderr(nullptr, nullptr, false); } } diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc index a47dbb689d8..ae434c7f062 100644 --- a/src/libstore/worker-protocol-connection.cc +++ b/src/libstore/worker-protocol-connection.cc @@ -32,7 +32,8 @@ static Logger::Fields readFields(Source & from) return fields; } -std::exception_ptr WorkerProto::BasicClientConnection::processStderrReturn(Sink * sink, Source * source, bool flush) +std::exception_ptr +WorkerProto::BasicClientConnection::processStderrReturn(Sink * sink, Source * source, bool flush, bool block) { if (flush) to.flush(); @@ -41,6 +42,9 @@ std::exception_ptr WorkerProto::BasicClientConnection::processStderrReturn(Sink while (true) { + if (!block && !from.hasData()) + break; + auto msg = readNum(from); if (msg == STDERR_WRITE) { @@ -95,8 +99,10 @@ std::exception_ptr WorkerProto::BasicClientConnection::processStderrReturn(Sink logger->result(act, type, fields); } - else if (msg == STDERR_LAST) + else if (msg == STDERR_LAST) { + assert(block); break; + } else throw Error("got unknown message type %x from Nix daemon", msg); @@ -130,9 +136,10 @@ std::exception_ptr WorkerProto::BasicClientConnection::processStderrReturn(Sink } } -void WorkerProto::BasicClientConnection::processStderr(bool * daemonException, Sink * sink, Source * source, bool flush) +void WorkerProto::BasicClientConnection::processStderr( + bool * daemonException, Sink * sink, Source * source, bool flush, bool block) { - auto ex = processStderrReturn(sink, source, flush); + auto ex = processStderrReturn(sink, source, flush, block); if (ex) { *daemonException = true; std::rethrow_exception(ex); diff --git a/src/libstore/worker-protocol-connection.hh b/src/libstore/worker-protocol-connection.hh index 9c96195b5f4..9665067ddef 100644 --- a/src/libstore/worker-protocol-connection.hh +++ b/src/libstore/worker-protocol-connection.hh @@ -70,9 +70,10 @@ struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection virtual void closeWrite() = 0; - std::exception_ptr processStderrReturn(Sink * sink = 0, Source * source = 0, bool flush = true); + std::exception_ptr processStderrReturn(Sink * sink = 0, Source * source = 0, bool flush = true, bool block = true); - void processStderr(bool * daemonException, Sink * sink = 0, Source * source = 0, bool flush = true); + void + processStderr(bool * daemonException, Sink * sink = 0, Source * source = 0, bool flush = true, bool block = true); /** * Establishes connection, negotiating version. diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 4899134d7c3..1e30d27b74a 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -10,6 +10,8 @@ #ifdef _WIN32 # include # include "windows-error.hh" +#else +# include #endif @@ -158,6 +160,24 @@ bool FdSource::good() } +bool FdSource::hasData() +{ + if (BufferedSource::hasData()) return true; + + while (true) { + struct pollfd fds[1]; + fds[0].fd = fd; + fds[0].events = POLLIN; + auto n = poll(fds, 1, 0); + if (n < 0) { + if (errno == EINTR) continue; + throw SysError("polling file descriptor"); + } + return n == 1 && (fds[0].events & POLLIN); + } +} + + size_t StringSource::read(char * data, size_t len) { if (pos == s.size()) throw EndOfFile("end of string reached"); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index c7290dcef9d..964b9a30dd6 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -104,6 +104,9 @@ struct BufferedSource : Source size_t read(char * data, size_t len) override; + /** + * Return true if the buffer is not empty. + */ bool hasData(); protected: @@ -162,6 +165,13 @@ struct FdSource : BufferedSource FdSource & operator=(FdSource && s) = default; bool good() override; + + /** + * Return true if the buffer is not empty after a non-blocking + * read. + */ + bool hasData(); + protected: size_t readUnbuffered(char * data, size_t len) override; private: @@ -522,15 +532,16 @@ struct FramedSource : Source /** * Write as chunks in the format expected by FramedSource. * - * The exception_ptr reference can be used to terminate the stream when you - * detect that an error has occurred on the remote end. + * The `checkError` function can be used to terminate the stream when you + * detect that an error has occurred. */ struct FramedSink : nix::BufferedSink { BufferedSink & to; - std::exception_ptr & ex; + std::function checkError; - FramedSink(BufferedSink & to, std::exception_ptr & ex) : to(to), ex(ex) + FramedSink(BufferedSink & to, std::function && checkError) + : to(to), checkError(checkError) { } ~FramedSink() @@ -545,13 +556,9 @@ struct FramedSink : nix::BufferedSink void writeUnbuffered(std::string_view data) override { - /* Don't send more data if the remote has - encountered an error. */ - if (ex) { - auto ex2 = ex; - ex = nullptr; - std::rethrow_exception(ex2); - } + /* Don't send more data if an error has occured. */ + checkError(); + to << data.size(); to(data); }; From 62b9a26f60753af73d08528cf39b80ba7b210cef Mon Sep 17 00:00:00 2001 From: Noam Yorav-Raphael Date: Mon, 19 Aug 2024 20:18:08 +0300 Subject: [PATCH 254/284] Set $HOME=/proc/homeless-shelter on Linux, and /homeless-shelter on OSX. --- doc/manual/src/language/derivations.md | 3 ++- src/libstore/unix/build/local-derivation-goal.cc | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index c78c7b8c87d..b57f6bb6a90 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -264,7 +264,8 @@ The [`builder`](#attr-builder) is executed as follows: - `PATH` is set to `/path-not-set` to prevent shells from initialising it to their built-in default value. - - `HOME` is set to `/proc/homeless-shelter` to prevent programs from + - `HOME` is set to `/proc/homeless-shelter` on Linux and `/homeless-shelter` + on OSX, to prevent programs from using `/etc/passwd` or the like to find the user's home directory, which could cause impurity. Usually, when `HOME` is set, it is used as the location of the home directory, even if diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index f6bbba8b443..e1035fbddd3 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -102,7 +102,14 @@ void handleDiffHook( } } +// We want $HOME to be un-creatable in the sandbox. On Linux, +// you can't create anything inside /proc since it's a virtual filesystem. +// On Darwin it seems that `/homeless-shelter` is good enough. +#if __linux__ const Path LocalDerivationGoal::homeDir = "/proc/homeless-shelter"; +#else +const Path LocalDerivationGoal::homeDir = "/homeless-shelter"; +#endif LocalDerivationGoal::~LocalDerivationGoal() From 2926a859a36befed99bcf8f09031869aa71fba54 Mon Sep 17 00:00:00 2001 From: Jeremy Kolb Date: Mon, 19 Aug 2024 14:46:46 -0400 Subject: [PATCH 255/284] Release note for nix flake show change --- .../rl-next/nix-flake-show-description.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 doc/manual/rl-next/nix-flake-show-description.md diff --git a/doc/manual/rl-next/nix-flake-show-description.md b/doc/manual/rl-next/nix-flake-show-description.md new file mode 100644 index 00000000000..7feb0848357 --- /dev/null +++ b/doc/manual/rl-next/nix-flake-show-description.md @@ -0,0 +1,25 @@ +--- +synopsis: Show package descriptions with `nix flake show` +issues: [10977] +prs: [10980] +--- + +`nix flake show` will now display a package's `meta.description` if it exists. If the description does not fit in the terminal it will be truncated to fit the terminal width. If the size of the terminal width is unknown the description will be capped at 80 characters. + +``` +$ nix flake show +└───packages + └───x86_64-linux + ├───builderImage: package 'docker-image-ara-builder-image.tar.gz' - 'Docker image hosting the nix build environment' + └───runnerImage: package 'docker-image-gitlab-runner.tar.gz' - 'Docker image hosting the gitlab-runner executable' +``` + +In a narrower terminal: + +``` +$ nix flake show +└───packages + └───x86_64-linux + ├───builderImage: package 'docker-image-ara-builder-image.tar.gz' - 'Docker image hosting the nix b... + └───runnerImage: package 'docker-image-gitlab-runner.tar.gz' - 'Docker image hosting the gitlab-run... +``` From 03b258bf97f1740b90cdcdcadfa65266180a01a0 Mon Sep 17 00:00:00 2001 From: eldritch horrors Date: Fri, 9 Aug 2024 21:17:52 +0200 Subject: [PATCH 256/284] libutil: rename and optimize closeMostFDs this is only used to close non-stdio files in derivation sandboxes. we may as well encode that in its name, drop the unnecessary integer set, and use close_range to deal with the actual closing of files. not only is this clearer, it also makes sandbox setup on linux fast by 1ms each (cherry-picked and adapted from https://git.lix.systems/lix-project/lix/commit/c7d97802e4f59b8621e67cf62275d6a7fde8fe62) Co-authored-by: Eelco Dolstra Co-authored-by: Cole Helbling Co-authored-by: John Ericson --- .../unix/build/local-derivation-goal.cc | 2 +- src/libutil/file-descriptor.hh | 4 +-- src/libutil/unix/file-descriptor.cc | 30 +++++++++++++++---- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index e1035fbddd3..43a9a919121 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -1993,7 +1993,7 @@ void LocalDerivationGoal::runChild() throw SysError("changing into '%1%'", tmpDir); /* Close all other file descriptors. */ - unix::closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); + unix::closeExtraFDs(); #if __linux__ linux::setPersonality(drv->platform); diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh index be61375f64f..bf8354087fa 100644 --- a/src/libutil/file-descriptor.hh +++ b/src/libutil/file-descriptor.hh @@ -143,10 +143,10 @@ public: namespace unix { /** - * Close all file descriptors except those listed in the given set. + * Close all file descriptors except stdio fds (ie 0, 1, 2). * Good practice in child processes. */ -void closeMostFDs(const std::set & exceptions); +void closeExtraFDs(); /** * Set the close-on-exec flag for the given file descriptor. diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc index a3af1623fc9..f867199c04e 100644 --- a/src/libutil/unix/file-descriptor.cc +++ b/src/libutil/unix/file-descriptor.cc @@ -120,14 +120,35 @@ void Pipe::create() ////////////////////////////////////////////////////////////////////// -void unix::closeMostFDs(const std::set & exceptions) +#if __linux__ || __FreeBSD__ +// In future we can use a syscall wrapper, but at the moment musl and older glibc version don't support it. +static int unix_close_range(unsigned int first, unsigned int last, int flags) { + return syscall(SYS_close_range, first, last, (unsigned int)flags); +} +#endif + +void unix::closeExtraFDs() +{ + constexpr int MAX_KEPT_FD = 2; + static_assert(std::max({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}) == MAX_KEPT_FD); + +#if __linux__ || __FreeBSD__ + // first try to close_range everything we don't care about. if this + // returns an error with these parameters we're running on a kernel + // that does not implement close_range (i.e. pre 5.9) and fall back + // to the old method. we should remove that though, in some future. + if (unix_close_range(MAX_KEPT_FD + 1, ~0U, 0) == 0) { + return; + } +#endif + #if __linux__ try { for (auto & s : std::filesystem::directory_iterator{"/proc/self/fd"}) { checkInterrupt(); auto fd = std::stoi(s.path().filename()); - if (!exceptions.count(fd)) { + if (fd > MAX_KEPT_FD) { debug("closing leaked FD %d", fd); close(fd); } @@ -142,9 +163,8 @@ void unix::closeMostFDs(const std::set & exceptions) #if HAVE_SYSCONF maxFD = sysconf(_SC_OPEN_MAX); #endif - for (int fd = 0; fd < maxFD; ++fd) - if (!exceptions.count(fd)) - close(fd); /* ignore result */ + for (int fd = MAX_KEPT_FD + 1; fd < maxFD; ++fd) + close(fd); /* ignore result */ } From 3fe1b60c57074ee7aad17e0147f1c53bc83c9a7c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 21 Aug 2024 16:45:46 +0200 Subject: [PATCH 257/284] Remove dead makeMutable() function This code wasn't being compiled because the #if conditional didn't trigger. Anyway, Nix 0.15 is 15 years old so we don't need to keep this around. --- src/libstore/local-store.cc | 59 ------------------------------------- src/libstore/local-store.hh | 2 -- 2 files changed, 61 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 819cee34532..eeecde6e73c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -42,7 +42,6 @@ # include # include # include -# include #endif #ifdef __CYGWIN__ @@ -366,8 +365,6 @@ LocalStore::LocalStore( have performed the upgrade already. */ curSchema = getSchema(); - if (curSchema < 7) { upgradeStore7(); } - openDB(*state, false); if (curSchema < 8) { @@ -1596,62 +1593,6 @@ std::optional LocalStore::isTrustedClient() } -#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) - -static void makeMutable(const Path & path) -{ - checkInterrupt(); - - auto st = lstat(path); - - if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return; - - if (S_ISDIR(st.st_mode)) { - for (auto & i : readDirectory(path)) - makeMutable(path + "/" + i.name); - } - - /* The O_NOFOLLOW is important to prevent us from changing the - mutable bit on the target of a symlink (which would be a - security hole). */ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW -#ifndef _WIN32 - | O_CLOEXEC -#endif - ); - if (fd == INVALID_DESCRIPTOR) { - if (errno == ELOOP) return; // it's a symlink - throw SysError("opening file '%1%'", path); - } - - unsigned int flags = 0, old; - - /* Silently ignore errors getting/setting the immutable flag so - that we work correctly on filesystems that don't support it. */ - if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) return; - old = flags; - flags &= ~FS_IMMUTABLE_FL; - if (old == flags) return; - if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) return; -} - -/* Upgrade from schema 6 (Nix 0.15) to schema 7 (Nix >= 1.3). */ -void LocalStore::upgradeStore7() -{ - if (!isRootUser()) return; - printInfo("removing immutable bits from the Nix store (this may take a while)..."); - makeMutable(realStoreDir); -} - -#else - -void LocalStore::upgradeStore7() -{ -} - -#endif - - void LocalStore::vacuumDB() { auto state(_state.lock()); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index a03cfc03b30..21848cc4d10 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -373,8 +373,6 @@ private: void updatePathInfo(State & state, const ValidPathInfo & info); - void upgradeStore6(); - void upgradeStore7(); PathSet queryValidPathsOld(); ValidPathInfo queryPathInfoOld(const Path & path); From efbf4996355f4ce846c38dbeb9b5b7a4b418f322 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 21 Aug 2024 16:50:18 +0200 Subject: [PATCH 258/284] Remove redundant " --- src/libstore/globals.hh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 7a307f89439..ec0c69851cc 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -401,9 +401,9 @@ public: Setting fsyncStorePaths{this, false, "fsync-store-paths", R"( - "Whether to call `fsync()` on store paths before registering them, to - flush them to disk. This improves robustness in case of system crashes, - but reduces performance. The default is `false`. + Whether to call `fsync()` on store paths before registering them, to + flush them to disk. This improves robustness in case of system crashes, + but reduces performance. The default is `false`. )"}; Setting useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal", From e8752ca57a92ec31e250d740307423134b96814f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 21 Aug 2024 17:05:36 +0200 Subject: [PATCH 259/284] Add FIXME --- src/libstore/globals.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ec0c69851cc..e5e7024cbe7 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -410,6 +410,7 @@ public: "Whether SQLite should use WAL mode."}; #ifndef _WIN32 + // FIXME: remove this option, `fsync-store-paths` is faster. Setting syncBeforeRegistering{this, false, "sync-before-registering", "Whether to call `sync()` before registering a path as valid."}; #endif From 02446918f410b57bd7c1182bc1799639ba798556 Mon Sep 17 00:00:00 2001 From: Noam Yorav-Raphael Date: Wed, 21 Aug 2024 18:29:45 +0300 Subject: [PATCH 260/284] Add changelog for homeless-shelter-to-proc --- doc/manual/rl-next/homeless-shelter-to-proc.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/manual/rl-next/homeless-shelter-to-proc.md diff --git a/doc/manual/rl-next/homeless-shelter-to-proc.md b/doc/manual/rl-next/homeless-shelter-to-proc.md new file mode 100644 index 00000000000..4f9519eb411 --- /dev/null +++ b/doc/manual/rl-next/homeless-shelter-to-proc.md @@ -0,0 +1,9 @@ +--- +synopsis: On linux, set $HOME=/proc/homeless-shelter instead of /homeless-shelter +issues: [8313, 11295] +prs: [11300] +--- + +When building, $HOME is set to a non-existing dir. Previously it was always set to `/homeless-shelter`. After a build, the builder verifies that it doesn't exist. In some scenarios (specifically when using the linux sandbox with a single-user installation), it is possible to create the `/homeless-shelter` directory, and some tools will create it, resulting in a build error. + +Now, on Linux, $HOME is set to `/proc/homeless-shelter`. This directory can never be created, since `/proc` is a virtual filesystem. This resolves the issue. From a643c9b1f9270183fd5b0fe49711cf507272101d Mon Sep 17 00:00:00 2001 From: Noam Yorav-Raphael Date: Wed, 21 Aug 2024 19:58:16 +0300 Subject: [PATCH 261/284] Update doc/manual/rl-next/homeless-shelter-to-proc.md Co-authored-by: Eelco Dolstra --- doc/manual/rl-next/homeless-shelter-to-proc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/rl-next/homeless-shelter-to-proc.md b/doc/manual/rl-next/homeless-shelter-to-proc.md index 4f9519eb411..73213c0df4d 100644 --- a/doc/manual/rl-next/homeless-shelter-to-proc.md +++ b/doc/manual/rl-next/homeless-shelter-to-proc.md @@ -4,6 +4,6 @@ issues: [8313, 11295] prs: [11300] --- -When building, $HOME is set to a non-existing dir. Previously it was always set to `/homeless-shelter`. After a build, the builder verifies that it doesn't exist. In some scenarios (specifically when using the linux sandbox with a single-user installation), it is possible to create the `/homeless-shelter` directory, and some tools will create it, resulting in a build error. +When building, $HOME is set to a non-existing directory. Previously it was always set to `/homeless-shelter`. Before a build, Nix verifies that it doesn't exist. In some scenarios (specifically when using the Linux sandbox with a single-user installation), it is possible to create the `/homeless-shelter` directory, and some tools will create it, resulting in a build error. Now, on Linux, $HOME is set to `/proc/homeless-shelter`. This directory can never be created, since `/proc` is a virtual filesystem. This resolves the issue. From ce7cf4a2d32d3221eed50f087fe53f17f5c5ca12 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 21 Aug 2024 19:50:24 +0200 Subject: [PATCH 262/284] Update src/libutil/serialise.hh Co-authored-by: John Ericson --- src/libutil/serialise.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 964b9a30dd6..36e3e68e3d9 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -533,7 +533,7 @@ struct FramedSource : Source * Write as chunks in the format expected by FramedSource. * * The `checkError` function can be used to terminate the stream when you - * detect that an error has occurred. + * detect that an error has occurred. It does so by throwing an exception. */ struct FramedSink : nix::BufferedSink { From 270c8469d712a232915e736eeecacca696f77a01 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 21 Aug 2024 20:54:02 +0200 Subject: [PATCH 263/284] select() -> poll() for Windows compat --- src/libutil/serialise.cc | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 1e30d27b74a..056c3353121 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -165,15 +165,20 @@ bool FdSource::hasData() if (BufferedSource::hasData()) return true; while (true) { - struct pollfd fds[1]; - fds[0].fd = fd; - fds[0].events = POLLIN; - auto n = poll(fds, 1, 0); + fd_set fds; + FD_ZERO(&fds); + FD_SET(fd, &fds); + + struct timeval timeout; + timeout.tv_sec = 0; + timeout.tv_usec = 0; + + auto n = select(fd + 1, &fds, nullptr, nullptr, &timeout); if (n < 0) { if (errno == EINTR) continue; throw SysError("polling file descriptor"); } - return n == 1 && (fds[0].events & POLLIN); + return FD_ISSET(fd, &fds); } } From fac756fed4c0c5c4c78924ed8ecc900809f8c596 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 21 Aug 2024 21:08:26 +0200 Subject: [PATCH 264/284] Add FIXME --- src/libstore/daemon.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index ef3326cd64d..6079eae7ba4 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -402,6 +402,9 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); + // FIXME: this means that if there is an error + // half-way through, the client will keep sending + // data, since we haven't sent it the error yet. auto [contentAddressMethod, hashAlgo] = ContentAddressMethod::parseWithAlgo(camStr); FramedSource source(conn.from); FileSerialisationMethod dumpMethod; From 528e4904181072f9abc1c5657ba40fa181a27e48 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 22 Aug 2024 13:47:33 +0200 Subject: [PATCH 265/284] Disable the static build in CI GHA builds are now frequently timing out, so let's not do this. If wanted, it could be done in a separate job. --- flake.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index cded6c3a94b..d7a6eead9c1 100644 --- a/flake.nix +++ b/flake.nix @@ -195,7 +195,8 @@ } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) { # TODO: enable static builds for darwin, blocked on: # https://github.com/NixOS/nixpkgs/issues/320448 - "static-" = nixpkgsFor.${system}.static; + # TODO: disabled to speed up GHA CI. + #"static-" = nixpkgsFor.${system}.static; }) (nixpkgsPrefix: nixpkgs: flatMapAttrs nixpkgs.nixComponents From 3af73bae5c3d21d74718d5da08c93eb1f7fe8636 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 22 Aug 2024 13:48:46 +0200 Subject: [PATCH 266/284] Remove checks for nixComponents We are currently building Nix twice in the main GHA CI job, which is frequently timing out. Obviously, we want this to be fast, so only do the main build for now. --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index d7a6eead9c1..f5893d8f6fd 100644 --- a/flake.nix +++ b/flake.nix @@ -189,6 +189,7 @@ # system, we should reenable this. #perlBindings = self.hydraJobs.perlBindings.${system}; } + /* # Add "passthru" tests // flatMapAttrs ({ "" = nixpkgsFor.${system}.native; @@ -210,6 +211,7 @@ "${nixpkgsPrefix}nix-functional-tests" = nixpkgs.nixComponents.nix-functional-tests; } ) + */ // devFlake.checks.${system} or {} ); From 21a164aa0399b16f8484f9f2d70036cf7e6a28f9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 22 Aug 2024 15:26:34 +0200 Subject: [PATCH 267/284] Fix hang Signed-off-by: Eelco Dolstra --- src/libutil/file-system.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index ded7335f9f9..aa5f3670c75 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -339,7 +339,7 @@ void recursiveSync(const Path & path) while (!dirsToEnumerate.empty()) { auto currentDir = dirsToEnumerate.back(); dirsToEnumerate.pop_back(); - for (auto & entry : std::filesystem::directory_iterator(path)) { + for (auto & entry : std::filesystem::directory_iterator(currentDir)) { auto st = entry.symlink_status(); if (fs::is_directory(st)) { dirsToEnumerate.emplace_back(entry.path()); From 9ff0b55d4ef8c810455035e5735488349912999f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 22 Aug 2024 15:34:19 +0200 Subject: [PATCH 268/284] Add a VM test for fsync-store-paths Based on https://github.com/squalus/nix-durability-tests/blob/master/flake.nix. --- tests/nixos/default.nix | 2 ++ tests/nixos/fsync.nix | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 tests/nixos/fsync.nix diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 3fa341ef1e5..40d29b37128 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -155,4 +155,6 @@ in user-sandboxing = runNixOSTestFor "x86_64-linux" ./user-sandboxing; s3-binary-cache-store = runNixOSTestFor "x86_64-linux" ./s3-binary-cache-store.nix; + + fsync = runNixOSTestFor "x86_64-linux" ./fsync.nix; } diff --git a/tests/nixos/fsync.nix b/tests/nixos/fsync.nix new file mode 100644 index 00000000000..99ac2b25d50 --- /dev/null +++ b/tests/nixos/fsync.nix @@ -0,0 +1,39 @@ +{ lib, config, nixpkgs, pkgs, ... }: + +let + pkg1 = pkgs.go; +in + +{ + name = "fsync"; + + nodes.machine = + { config, lib, pkgs, ... }: + { virtualisation.emptyDiskImages = [ 1024 ]; + environment.systemPackages = [ pkg1 ]; + nix.settings.experimental-features = [ "nix-command" ]; + nix.settings.fsync-store-paths = true; + nix.settings.require-sigs = false; + boot.supportedFilesystems = [ "ext4" "btrfs" "xfs" ]; + }; + + testScript = { nodes }: '' + # fmt: off + for fs in ("ext4", "btrfs", "xfs"): + machine.succeed("mkfs.{} {} /dev/vdb".format(fs, "-F" if fs == "ext4" else "-f")) + machine.succeed("mkdir -p /mnt") + machine.succeed("mount /dev/vdb /mnt") + machine.succeed("sync") + machine.succeed("nix copy --offline ${pkg1} --to /mnt") + machine.crash() + + machine.start() + machine.wait_for_unit("multi-user.target") + machine.succeed("mkdir -p /mnt") + machine.succeed("mount /dev/vdb /mnt") + machine.succeed("nix path-info --offline --store /mnt ${pkg1}") + machine.succeed("nix store verify --all --store /mnt --no-trust") + + machine.succeed("umount /dev/vdb") + ''; +} From 71e7188e07b807497c5d93fa20d34c6bf731f9c1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 22 Aug 2024 15:48:36 +0200 Subject: [PATCH 269/284] Add release note --- doc/manual/rl-next/fsync-store-paths.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/manual/rl-next/fsync-store-paths.md diff --git a/doc/manual/rl-next/fsync-store-paths.md b/doc/manual/rl-next/fsync-store-paths.md new file mode 100644 index 00000000000..0e9e7f7f2c2 --- /dev/null +++ b/doc/manual/rl-next/fsync-store-paths.md @@ -0,0 +1,9 @@ +--- +synopsis: Add setting `fsync-store-paths` +issues: [1218] +prs: [7126] +--- + +Nix now has a setting `fsync-store-paths` that ensures that new store paths are durably written to disk before they are registered as "valid" in Nix's database. This can prevent Nix store corruption if the system crashes or there is a power loss. This setting defaults to `false`. + +Author: [**@squalus**](https://github.com/squalus) From 168bf9c3cdcf458060b17b4b8df06a28e28d5c69 Mon Sep 17 00:00:00 2001 From: aleksana Date: Thu, 22 Aug 2024 23:06:43 +0800 Subject: [PATCH 270/284] README: update CI badge The `Test` workflow was renamed to `CI` in https://github.com/NixOS/nix/commit/9aa486c4be2801a25847b9c75012054d04e792d0. It still seems to be showing the status it was last running on the master branch. This information is misleading and should be corrected. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ab647e53b1f..54a6fcc3949 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Nix [![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos) -[![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions) +[![CI](https://github.com/NixOS/nix/workflows/CI/badge.svg)](https://github.com/NixOS/nix/actions/workflows/ci.yml) Nix is a powerful package manager for Linux and other Unix systems that makes package management reliable and reproducible. Please refer to the [Nix manual](https://nix.dev/reference/nix-manual) From 5d28a0032282970788f723910f40a67e402d2e09 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Mon, 19 Aug 2024 16:33:23 +0300 Subject: [PATCH 271/284] Add nix_store_copy_closure to libstore-c --- src/libstore-c/nix_api_store.cc | 12 ++++++++++++ src/libstore-c/nix_api_store.h | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index 79841ca49a1..fb7391276c4 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -144,3 +144,15 @@ StorePath * nix_store_path_clone(const StorePath * p) { return new StorePath{p->path}; } + +nix_err nix_store_copy_closure(nix_c_context * context, Store * srcStore, Store * dstStore, StorePath * path) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::RealisedPath::Set paths; + paths.insert(path->path); + nix::copyClosure(*srcStore->ptr, *dstStore->ptr, paths); + } + NIXC_CATCH_ERRS +} diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index 4b213445778..93208cb7c28 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -161,6 +161,16 @@ nix_err nix_store_realise( nix_err nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data); +/** + * @brief Copy the closure of `path` from `srcStore` to `dstStore`. + * + * @param[out] context Optional, stores error information + * @param[in] srcStore nix source store reference + * @param[in] srcStore nix destination store reference + * @param[in] path Path to copy + */ +nix_err nix_store_copy_closure(nix_c_context * context, Store * srcStore, Store * dstStore, StorePath * path); + // cffi end #ifdef __cplusplus } From 2a14a20bb3c8db049e48b6c7887909d378e92ef4 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 23 Aug 2024 20:47:22 +0200 Subject: [PATCH 272/284] packaging: Expose libs This exposes the libraries in a way that is easily replicated in Nixpkgs, without having to create a package set within a package set. --- packaging/everything.nix | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/packaging/everything.nix b/packaging/everything.nix index 8c8ce66113a..6dae7f1c208 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -76,7 +76,7 @@ ] ++ lib.optionals (stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ nix-perl-bindings ]; -}).overrideAttrs (_: { +}).overrideAttrs (finalAttrs: prevAttrs: { doCheck = true; doInstallCheck = true; @@ -90,4 +90,37 @@ installCheckInputs = [ nix-functional-tests ]; + passthru = prevAttrs.passthru // { + /** + These are the libraries that are part of the Nix project. They are used + by the Nix CLI and other tools. + + If you need to use these libraries in your project, we recommend to use + the `-c` C API libraries exclusively, if possible. + + We also recommend that you build the complete package to ensure that the unit tests pass. + You could do this in CI, or by passing it in an unused environment variable. e.g in a `mkDerivation` call: + + ```nix + buildInputs = [ nix.libs.nix-util-c nix.libs.nix-store-c ]; + # Make sure the nix libs we use are ok + unusedInputsForTests = [ nix ]; + disallowedReferences = nix.all; + ``` + */ + libs = { + inherit + nix-util + nix-util-c + nix-store + nix-store-c + nix-fetchers + nix-expr + nix-expr-c + nix-flake + nix-main + nix-main-c + ; + }; + }; }) From c9f45677b585dabb3a83570e21426257d92746bd Mon Sep 17 00:00:00 2001 From: Bryan Honof Date: Wed, 21 Aug 2024 16:57:06 +0200 Subject: [PATCH 273/284] fix: Error on malformed URI query parameter Signed-off-by: Bryan Honof --- src/libutil/url.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libutil/url.cc b/src/libutil/url.cc index bcbe9ea4eb2..78c83244007 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -79,10 +79,15 @@ std::map decodeQuery(const std::string & query) for (auto s : tokenizeString(query, "&")) { auto e = s.find('='); - if (e != std::string::npos) - result.emplace( - s.substr(0, e), - percentDecode(std::string_view(s).substr(e + 1))); + + if (e == std::string::npos) { + warn("invalid URI query '%s', did you forget an equals sign `=`?", s); + continue; + } + + result.emplace( + s.substr(0, e), + percentDecode(std::string_view(s).substr(e + 1))); } return result; From 096bec8eb27a534b54c4cd810a5a9d8250703a0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 25 Aug 2024 09:04:47 +0200 Subject: [PATCH 274/284] Revert "Merge pull request #11300 from noamraph/homeless-shelter-to-proc" This reverts commit 43e82c944671ad7ce5da1b75991a4c1f48b545c4, reversing changes made to d79b9bdec0557315b0897707394339082cec004c. Since /proc/homeless-shelter returns a different errno than /homeless-shelter (ENOENT vs EACCES), we need to revert this change. Software depends on this error code i.e. cargo and therefore breaks. --- doc/manual/src/language/derivations.md | 3 +-- src/libstore/unix/build/local-derivation-goal.cc | 7 ------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index b57f6bb6a90..8e3f0f79174 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -264,8 +264,7 @@ The [`builder`](#attr-builder) is executed as follows: - `PATH` is set to `/path-not-set` to prevent shells from initialising it to their built-in default value. - - `HOME` is set to `/proc/homeless-shelter` on Linux and `/homeless-shelter` - on OSX, to prevent programs from + - `HOME` is set to `/homeless-shelter` to prevent programs from using `/etc/passwd` or the like to find the user's home directory, which could cause impurity. Usually, when `HOME` is set, it is used as the location of the home directory, even if diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index e1035fbddd3..d3482df17a1 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -102,14 +102,7 @@ void handleDiffHook( } } -// We want $HOME to be un-creatable in the sandbox. On Linux, -// you can't create anything inside /proc since it's a virtual filesystem. -// On Darwin it seems that `/homeless-shelter` is good enough. -#if __linux__ -const Path LocalDerivationGoal::homeDir = "/proc/homeless-shelter"; -#else const Path LocalDerivationGoal::homeDir = "/homeless-shelter"; -#endif LocalDerivationGoal::~LocalDerivationGoal() From 90560eeccc990315fa44bf18e78ed69aa7d552a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 25 Aug 2024 09:03:59 +0200 Subject: [PATCH 275/284] Revert "Merge pull request #11350 from noamraph/homeless-shelter-to-proc-changelog" This reverts commit fa49d2e356d44d416ac86d2286189e8f5f6878ba, reversing changes made to af26fe39344faff70e009d980820b8667c319cb2. --- doc/manual/rl-next/homeless-shelter-to-proc.md | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 doc/manual/rl-next/homeless-shelter-to-proc.md diff --git a/doc/manual/rl-next/homeless-shelter-to-proc.md b/doc/manual/rl-next/homeless-shelter-to-proc.md deleted file mode 100644 index 73213c0df4d..00000000000 --- a/doc/manual/rl-next/homeless-shelter-to-proc.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -synopsis: On linux, set $HOME=/proc/homeless-shelter instead of /homeless-shelter -issues: [8313, 11295] -prs: [11300] ---- - -When building, $HOME is set to a non-existing directory. Previously it was always set to `/homeless-shelter`. Before a build, Nix verifies that it doesn't exist. In some scenarios (specifically when using the Linux sandbox with a single-user installation), it is possible to create the `/homeless-shelter` directory, and some tools will create it, resulting in a build error. - -Now, on Linux, $HOME is set to `/proc/homeless-shelter`. This directory can never be created, since `/proc` is a virtual filesystem. This resolves the issue. From 93a8b06070306df388a168a2285a3476070c7399 Mon Sep 17 00:00:00 2001 From: Emily Date: Mon, 26 Aug 2024 05:26:49 +0100 Subject: [PATCH 276/284] sequoia-nixbld-user-migration: increment base UID by 1 --- scripts/sequoia-nixbld-user-migration.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sequoia-nixbld-user-migration.sh b/scripts/sequoia-nixbld-user-migration.sh index 9208a360523..644249192dd 100755 --- a/scripts/sequoia-nixbld-user-migration.sh +++ b/scripts/sequoia-nixbld-user-migration.sh @@ -2,7 +2,7 @@ set -x -((NEW_NIX_FIRST_BUILD_UID=350)) +((NEW_NIX_FIRST_BUILD_UID=351)) ((TEMP_NIX_FIRST_BUILD_UID=31000)) nix_user_n() { From 77ddcbe12e169051f9bad1ab5e7581b148a94883 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 26 Aug 2024 16:15:13 +0200 Subject: [PATCH 277/284] getDoc: Explain why we partially apply __functor --- src/libexpr/eval.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b87d96be09c..ca41a9944e9 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -621,6 +621,11 @@ std::optional EvalState::getDoc(Value & v) Value & functor = *v.attrs()->find(sFunctor)->value; Value * vp = &v; Value partiallyApplied; + // The first paramater is not user-provided, and may be + // handled by code that is opaque to the user, like lib.const = x: y: y; + // So preferably we show docs that are relevant to the + // "partially applied" function returned by e.g. `const`. + // We apply the first argument: callFunction(functor, 1, &vp, partiallyApplied, noPos); auto _level = addCallDepth(noPos); return getDoc(partiallyApplied); From dbabfc92d4e1864f793f167a438e532673afdc14 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 26 Aug 2024 15:42:09 -0400 Subject: [PATCH 278/284] Make sure we have an `execvpe` on Windows too Necessary to fix a build (that was already broken in other ways) after PR #11021. --- src/libutil/{unix => }/exec.hh | 7 ++++++- src/libutil/meson.build | 1 + src/libutil/unix/meson.build | 1 - src/libutil/unix/processes.cc | 8 +++++--- src/libutil/windows/processes.cc | 8 ++++++++ 5 files changed, 20 insertions(+), 5 deletions(-) rename src/libutil/{unix => }/exec.hh (53%) diff --git a/src/libutil/unix/exec.hh b/src/libutil/exec.hh similarity index 53% rename from src/libutil/unix/exec.hh rename to src/libutil/exec.hh index e6b80889aa9..405e1926849 100644 --- a/src/libutil/unix/exec.hh +++ b/src/libutil/exec.hh @@ -1,5 +1,7 @@ #pragma once +#include "os-string.hh" + namespace nix { /** @@ -8,6 +10,9 @@ namespace nix { * * We use our own implementation unconditionally for consistency. */ -int execvpe(const char * file0, char * const argv[], char * const envp[]); +int execvpe( + const OsString::value_type * file0, + const OsString::value_type * const argv[], + const OsString::value_type * const envp[]); } diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 200eeb4e913..72ff461ca5e 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -129,6 +129,7 @@ sources = files( 'english.cc', 'environment-variables.cc', 'error.cc', + 'exec.hh', 'executable-path.cc', 'exit.cc', 'experimental-features.cc', diff --git a/src/libutil/unix/meson.build b/src/libutil/unix/meson.build index d36152db973..1c5bf27fb14 100644 --- a/src/libutil/unix/meson.build +++ b/src/libutil/unix/meson.build @@ -13,7 +13,6 @@ sources += files( include_dirs += include_directories('.') headers += files( - 'exec.hh', 'monitor-fd.hh', 'signals-impl.hh', ) diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 09acba35a95..43d9179d9d7 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -420,10 +420,12 @@ bool statusOk(int status) return WIFEXITED(status) && WEXITSTATUS(status) == 0; } -int execvpe(const char * file0, char * const argv[], char * const envp[]) +int execvpe(const char * file0, const char * const argv[], const char * const envp[]) { - auto file = ExecutablePath::load().findPath(file0).string(); - return execve(file.c_str(), argv, envp); + auto file = ExecutablePath::load().findPath(file0); + // `const_cast` is safe. See the note in + // https://pubs.opengroup.org/onlinepubs/9799919799/functions/exec.html + return execve(file.c_str(), const_cast(argv), const_cast(envp)); } } diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index 9cd714f8419..7f34c5632e8 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -1,6 +1,7 @@ #include "current-process.hh" #include "environment-variables.hh" #include "error.hh" +#include "executable-path.hh" #include "file-descriptor.hh" #include "file-path.hh" #include "signals.hh" @@ -377,4 +378,11 @@ bool statusOk(int status) { return status == 0; } + +int execvpe(const wchar_t * file0, const wchar_t * const argv[], const wchar_t * const envp[]) +{ + auto file = ExecutablePath::load().findPath(file0); + return _wexecve(file.c_str(), argv, envp); +} + } From a97a08411c4e34323b48e95ed593bb53d67bde23 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 26 Aug 2024 12:24:37 -0400 Subject: [PATCH 279/284] More support for `std::filepath` in libnixutil We're not replacing `Path` in exposed definitions in many cases, but just adding alternatives. This will allow us to "top down" change `Path` to `std::fileysystem::path`, and then we can remove the `Path`-using utilities which will become unused. Also add some test files which we forgot to include in the libutil unit tests `meson.build`. Co-Authored-By: siddhantCodes --- src/libcmd/common-eval-args.cc | 6 +- src/libcmd/installables.cc | 4 +- src/libcmd/repl.cc | 4 +- src/libexpr/parser.y | 2 +- src/libstore/gc.cc | 2 +- src/libstore/local-overlay-store.cc | 2 +- src/libutil/args.hh | 22 ++++++ src/libutil/exec.hh | 5 +- src/libutil/executable-path.cc | 6 +- src/libutil/executable-path.hh | 6 +- src/libutil/file-system.cc | 32 ++++++--- src/libutil/file-system.hh | 69 ++++++++++++++++++- src/libutil/linux/namespaces.cc | 2 +- src/libutil/os-string.hh | 17 +++-- src/libutil/posix-source-accessor.cc | 2 +- src/libutil/serialise.cc | 6 +- src/libutil/strings.cc | 4 +- src/libutil/unix/users.cc | 2 + src/nix/bundle.cc | 4 +- src/nix/config-check.cc | 4 +- src/nix/develop.cc | 6 +- src/nix/flake.cc | 10 +-- src/nix/run.cc | 18 ++--- src/nix/self-exe.cc | 4 +- tests/unit/libfetchers/public-key.cc | 6 +- .../libstore-support/tests/nix_api_store.hh | 2 +- tests/unit/libstore-support/tests/protocol.hh | 6 +- .../libstore/derivation-advanced-attrs.cc | 6 +- tests/unit/libstore/derivation.cc | 6 +- tests/unit/libstore/machines.cc | 20 +++--- tests/unit/libstore/nar-info.cc | 6 +- tests/unit/libstore/path-info.cc | 6 +- tests/unit/libstore/store-reference.cc | 6 +- .../libutil-support/tests/characterization.hh | 10 +-- tests/unit/libutil/file-system.cc | 56 ++++++++------- tests/unit/libutil/git.cc | 6 +- tests/unit/libutil/meson.build | 3 + 37 files changed, 258 insertions(+), 120 deletions(-) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index ae9994a05f6..ccbf957d97e 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -18,6 +18,8 @@ namespace nix { +namespace fs { using namespace std::filesystem; } + fetchers::Settings fetchSettings; static GlobalConfig::Register rFetchSettings(&fetchSettings); @@ -119,8 +121,8 @@ MixEvalArgs::MixEvalArgs() .category = category, .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { - auto from = parseFlakeRef(fetchSettings, _from, absPath(".")); - auto to = parseFlakeRef(fetchSettings, _to, absPath(".")); + auto from = parseFlakeRef(fetchSettings, _from, fs::current_path().string()); + auto to = parseFlakeRef(fetchSettings, _to, fs::current_path().string()); fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 0fe956ec023..22e7eb546c8 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -31,6 +31,8 @@ namespace nix { +namespace fs { using namespace std::filesystem; } + void completeFlakeInputPath( AddCompletions & completions, ref evalState, @@ -341,7 +343,7 @@ void completeFlakeRefWithFragment( auto flakeRefS = std::string(prefix.substr(0, hash)); // TODO: ideally this would use the command base directory instead of assuming ".". - auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), absPath(".")); + auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), fs::current_path().string()); auto evalCache = openEvalCache(*evalState, std::make_shared(lockFlake( diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index e7c43367c24..63f6c1bdd0d 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -622,7 +622,7 @@ ProcessLineResult NixRepl::processLine(std::string line) // When missing, trigger the normal exception // e.g. :doc builtins.foo // behaves like - // nix-repl> builtins.foo + // nix-repl> builtins.foo // error: attribute 'foo' missing evalString(arg, v); assert(false); @@ -720,7 +720,7 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, absPath("."), true); + auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, std::filesystem::current_path().string(), true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index f2ccca7fcdd..a79abbf168e 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -350,7 +350,7 @@ string_parts_interpolated path_start : PATH { - Path path(absPath({$1.p, $1.l}, state->basePath.path.abs())); + Path path(absPath(std::string_view{$1.p, $1.l}, state->basePath.path.abs())); /* add back in the trailing '/' to the first segment */ if ($1.p[$1.l-1] == '/' && $1.l > 1) path += "/"; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 1494712dab4..91cf7636616 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -333,7 +333,7 @@ static std::string quoteRegexChars(const std::string & raw) } #if __linux__ -static void readFileRoots(const char * path, UncheckedRoots & roots) +static void readFileRoots(const std::filesystem::path & path, UncheckedRoots & roots) { try { roots[readFile(path)].emplace(path); diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index ec2c5f4e936..b86beba2caf 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -31,7 +31,7 @@ LocalOverlayStore::LocalOverlayStore(std::string_view scheme, PathView path, con if (checkMount.get()) { std::smatch match; std::string mountInfo; - auto mounts = readFile("/proc/self/mounts"); + auto mounts = readFile(std::filesystem::path{"/proc/self/mounts"}); auto regex = std::regex(R"((^|\n)overlay )" + realStoreDir.get() + R"( .*(\n|$))"); // Mount points can be stacked, so there might be multiple matching entries. diff --git a/src/libutil/args.hh b/src/libutil/args.hh index c0236ee3d72..513b8d81169 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -113,6 +113,16 @@ protected: , arity(1) { } + Handler(std::filesystem::path * dest) + : fun([dest](std::vector ss) { *dest = ss[0]; }) + , arity(1) + { } + + Handler(std::optional * dest) + : fun([dest](std::vector ss) { *dest = ss[0]; }) + , arity(1) + { } + template Handler(T * dest, const T & val) : fun([dest, val](std::vector ss) { *dest = val; }) @@ -283,6 +293,18 @@ public: }); } + /** + * Expect a path argument. + */ + void expectArg(const std::string & label, std::filesystem::path * dest, bool optional = false) + { + expectArgs({ + .label = label, + .optional = optional, + .handler = {dest} + }); + } + /** * Expect 0 or more arguments. */ diff --git a/src/libutil/exec.hh b/src/libutil/exec.hh index 405e1926849..cbbe80c4e9b 100644 --- a/src/libutil/exec.hh +++ b/src/libutil/exec.hh @@ -10,9 +10,6 @@ namespace nix { * * We use our own implementation unconditionally for consistency. */ -int execvpe( - const OsString::value_type * file0, - const OsString::value_type * const argv[], - const OsString::value_type * const envp[]); +int execvpe(const OsChar * file0, const OsChar * const argv[], const OsChar * const envp[]); } diff --git a/src/libutil/executable-path.cc b/src/libutil/executable-path.cc index da71088e739..9fb5214b2d7 100644 --- a/src/libutil/executable-path.cc +++ b/src/libutil/executable-path.cc @@ -6,7 +6,9 @@ namespace nix { -namespace fs = std::filesystem; +namespace fs { +using namespace std::filesystem; +} constexpr static const OsStringView path_var_separator{ &ExecutablePath::separator, @@ -24,7 +26,7 @@ ExecutablePath ExecutablePath::load() ExecutablePath ExecutablePath::parse(const OsString & path) { auto strings = path.empty() ? (std::list{}) - : basicSplitString, OsString::value_type>(path, path_var_separator); + : basicSplitString, OsChar>(path, path_var_separator); std::vector ret; ret.reserve(strings.size()); diff --git a/src/libutil/executable-path.hh b/src/libutil/executable-path.hh index f46d5e2128c..c5cfa1c3918 100644 --- a/src/libutil/executable-path.hh +++ b/src/libutil/executable-path.hh @@ -7,11 +7,15 @@ namespace nix { MakeError(ExecutableLookupError, Error); +/** + * @todo rename, it is not just good for execuatable paths, but also + * other lists of paths. + */ struct ExecutablePath { std::vector directories; - constexpr static const OsString::value_type separator = + constexpr static const OsChar separator = #ifdef WIN32 L';' #else diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index aa5f3670c75..edcacb50a82 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -26,10 +26,10 @@ #include "strings-inline.hh" -namespace fs = std::filesystem; - namespace nix { +namespace fs { using namespace std::filesystem; } + /** * Treat the string as possibly an absolute path, by inspecting the * start of it. Return whether it was probably intended to be @@ -73,6 +73,10 @@ Path absPath(PathView path, std::optional dir, bool resolveSymlinks) return canonPath(path, resolveSymlinks); } +std::filesystem::path absPath(const std::filesystem::path & path, bool resolveSymlinks) +{ + return absPath(path.string(), std::nullopt, resolveSymlinks); +} Path canonPath(PathView path, bool resolveSymlinks) { @@ -206,10 +210,10 @@ bool pathExists(const Path & path) return maybeLstat(path).has_value(); } -bool pathAccessible(const Path & path) +bool pathAccessible(const std::filesystem::path & path) { try { - return pathExists(path); + return pathExists(path.string()); } catch (SysError & e) { // swallow EPERM if (e.errNo == EPERM) return false; @@ -238,6 +242,11 @@ std::string readFile(const Path & path) return readFile(fd.get()); } +std::string readFile(const std::filesystem::path & path) +{ + return readFile(os_string_to_string(PathViewNG { path })); +} + void readFile(const Path & path, Sink & sink) { @@ -324,7 +333,7 @@ void recursiveSync(const Path & path) /* If it's a file, just fsync and return. */ auto st = lstat(path); if (S_ISREG(st.st_mode)) { - AutoCloseFD fd = open(path.c_str(), O_RDONLY, 0); + AutoCloseFD fd = toDescriptor(open(path.c_str(), O_RDONLY, 0)); if (!fd) throw SysError("opening file '%1%'", path); fd.fsync(); @@ -344,7 +353,7 @@ void recursiveSync(const Path & path) if (fs::is_directory(st)) { dirsToEnumerate.emplace_back(entry.path()); } else if (fs::is_regular_file(st)) { - AutoCloseFD fd = open(entry.path().c_str(), O_RDONLY, 0); + AutoCloseFD fd = toDescriptor(open(entry.path().string().c_str(), O_RDONLY, 0)); if (!fd) throw SysError("opening file '%1%'", entry.path()); fd.fsync(); @@ -355,7 +364,7 @@ void recursiveSync(const Path & path) /* Fsync all the directories. */ for (auto dir = dirsToFsync.rbegin(); dir != dirsToFsync.rend(); ++dir) { - AutoCloseFD fd = open(dir->c_str(), O_RDONLY, 0); + AutoCloseFD fd = toDescriptor(open(dir->string().c_str(), O_RDONLY, 0)); if (!fd) throw SysError("opening directory '%1%'", *dir); fd.fsync(); @@ -595,19 +604,20 @@ void createSymlink(const Path & target, const Path & link) fs::create_symlink(target, link); } -void replaceSymlink(const Path & target, const Path & link) +void replaceSymlink(const fs::path & target, const fs::path & link) { for (unsigned int n = 0; true; n++) { - Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + auto tmp = link.parent_path() / fs::path{fmt(".%d_%s", n, link.filename().string())}; + tmp = tmp.lexically_normal(); try { - createSymlink(target, tmp); + fs::create_symlink(target, tmp); } catch (fs::filesystem_error & e) { if (e.code() == std::errc::file_exists) continue; throw; } - std::filesystem::rename(tmp, link); + fs::rename(tmp, link); break; } diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 0f406a2de0a..eb3e4ec6630 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -46,16 +46,33 @@ struct Source; * @return An absolutized path, resolving paths relative to the * specified directory, or the current directory otherwise. The path * is also canonicalised. + * + * In the process of being deprecated for `std::filesystem::absolute`. */ Path absPath(PathView path, std::optional dir = {}, bool resolveSymlinks = false); +inline Path absPath(const Path & path, + std::optional dir = {}, + bool resolveSymlinks = false) +{ + return absPath(PathView{path}, dir, resolveSymlinks); +} + +std::filesystem::path absPath(const std::filesystem::path & path, + bool resolveSymlinks = false); + /** * Canonicalise a path by removing all `.` or `..` components and * double or trailing slashes. Optionally resolves all symlink * components such that each component of the resulting path is *not* * a symbolic link. + * + * In the process of being deprecated for + * `std::filesystem::path::lexically_normal` (for the `resolveSymlinks = + * false` case), and `std::filesystem::weakly_canonical` (for the + * `resolveSymlinks = true` case). */ Path canonPath(PathView path, bool resolveSymlinks = false); @@ -64,12 +81,18 @@ Path canonPath(PathView path, bool resolveSymlinks = false); * everything before the final `/`. If the path is the root or an * immediate child thereof (e.g., `/foo`), this means `/` * is returned. + * + * In the process of being deprecated for + * `std::filesystem::path::parent_path`. */ Path dirOf(const PathView path); /** * @return the base name of the given canonical path, i.e., everything * following the final `/` (trailing slashes are removed). + * + * In the process of being deprecated for + * `std::filesystem::path::filename`. */ std::string_view baseNameOf(std::string_view path); @@ -98,20 +121,42 @@ std::optional maybeLstat(const Path & path); /** * @return true iff the given path exists. + * + * In the process of being deprecated for `fs::symlink_exists`. */ bool pathExists(const Path & path); +namespace fs { + +/** + * ``` + * symlink_exists(p) = std::filesystem::exists(std::filesystem::symlink_status(p)) + * ``` + * Missing convenience analogous to + * ``` + * std::filesystem::exists(p) = std::filesystem::exists(std::filesystem::status(p)) + * ``` + */ +inline bool symlink_exists(const std::filesystem::path & path) { + return std::filesystem::exists(std::filesystem::symlink_status(path)); +} + +} // namespace fs + /** * A version of pathExists that returns false on a permission error. * Useful for inferring default paths across directories that might not * be readable. * @return true iff the given path can be accessed and exists */ -bool pathAccessible(const Path & path); +bool pathAccessible(const std::filesystem::path & path); /** * Read the contents (target) of a symbolic link. The result is not * in any way canonicalised. + * + * In the process of being deprecated for + * `std::filesystem::read_symlink`. */ Path readLink(const Path & path); @@ -124,14 +169,23 @@ Descriptor openDirectory(const std::filesystem::path & path); * Read the contents of a file into a string. */ std::string readFile(const Path & path); +std::string readFile(const std::filesystem::path & path); void readFile(const Path & path, Sink & sink); /** * Write a string to a file. */ void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); +static inline void writeFile(const std::filesystem::path & path, std::string_view s, mode_t mode = 0666, bool sync = false) +{ + return writeFile(path.string(), s, mode, sync); +} void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); +static inline void writeFile(const std::filesystem::path & path, Source & source, mode_t mode = 0666, bool sync = false) +{ + return writeFile(path.string(), source, mode, sync); +} /** * Flush a path's parent directory to disk. @@ -154,6 +208,9 @@ void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed); /** * Create a directory and all its parents, if necessary. + * + * In the process of being deprecated for + * `std::filesystem::create_directories`. */ void createDirs(const Path & path); inline void createDirs(PathView path) @@ -192,13 +249,21 @@ void setWriteTime(const std::filesystem::path & path, const struct stat & st); /** * Create a symlink. + * + * In the process of being deprecated for + * `std::filesystem::create_symlink`. */ void createSymlink(const Path & target, const Path & link); /** * Atomically create or replace a symlink. */ -void replaceSymlink(const Path & target, const Path & link); +void replaceSymlink(const std::filesystem::path & target, const std::filesystem::path & link); + +inline void replaceSymlink(const Path & target, const Path & link) +{ + return replaceSymlink(std::filesystem::path{target}, std::filesystem::path{link}); +} /** * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` diff --git a/src/libutil/linux/namespaces.cc b/src/libutil/linux/namespaces.cc index d4766cbba91..c5e21dffcb3 100644 --- a/src/libutil/linux/namespaces.cc +++ b/src/libutil/linux/namespaces.cc @@ -118,7 +118,7 @@ void saveMountNamespace() void restoreMountNamespace() { try { - auto savedCwd = absPath("."); + auto savedCwd = std::filesystem::current_path(); if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) throw SysError("restoring parent mount namespace"); diff --git a/src/libutil/os-string.hh b/src/libutil/os-string.hh index 0d75173e50e..3e24763fb56 100644 --- a/src/libutil/os-string.hh +++ b/src/libutil/os-string.hh @@ -11,21 +11,30 @@ namespace nix { * Named because it is similar to the Rust type, except it is in the * native encoding not WTF-8. * - * Same as `std::filesystem::path::string_type`, but manually defined to + * Same as `std::filesystem::path::value_type`, but manually defined to * avoid including a much more complex header. */ -using OsString = std::basic_string< +using OsChar = #if defined(_WIN32) && !defined(__CYGWIN__) wchar_t #else char #endif - >; + ; + +/** + * Named because it is similar to the Rust type, except it is in the + * native encoding not WTF-8. + * + * Same as `std::filesystem::path::string_type`, but manually defined + * for the same reason as `OsChar`. + */ +using OsString = std::basic_string; /** * `std::string_view` counterpart for `OsString`. */ -using OsStringView = std::basic_string_view; +using OsStringView = std::basic_string_view; std::string os_string_to_string(OsStringView path); diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 2b1a485d55c..8cec3388d7b 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -20,7 +20,7 @@ PosixSourceAccessor::PosixSourceAccessor() SourcePath PosixSourceAccessor::createAtRoot(const std::filesystem::path & path) { - std::filesystem::path path2 = absPath(path.string()); + std::filesystem::path path2 = absPath(path); return { make_ref(path2.root_path()), CanonPath { path2.relative_path().string() }, diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 8a57858f57b..4aa5ae385d1 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -9,6 +9,7 @@ #ifdef _WIN32 # include +# include # include "windows-error.hh" #else # include @@ -167,13 +168,14 @@ bool FdSource::hasData() while (true) { fd_set fds; FD_ZERO(&fds); - FD_SET(fd, &fds); + int fd_ = fromDescriptorReadOnly(fd); + FD_SET(fd_, &fds); struct timeval timeout; timeout.tv_sec = 0; timeout.tv_usec = 0; - auto n = select(fd + 1, &fds, nullptr, nullptr, &timeout); + auto n = select(fd_ + 1, &fds, nullptr, nullptr, &timeout); if (n < 0) { if (errno == EINTR) continue; throw SysError("polling file descriptor"); diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc index 60297228ead..5cad95758cc 100644 --- a/src/libutil/strings.cc +++ b/src/libutil/strings.cc @@ -14,8 +14,8 @@ template std::list splitString(std::string_view s, std::string_view template std::set splitString(std::string_view s, std::string_view separators); template std::vector splitString(std::string_view s, std::string_view separators); -template std::list basicSplitString( - std::basic_string_view s, std::basic_string_view separators); +template std::list +basicSplitString(std::basic_string_view s, std::basic_string_view separators); template std::string concatStringsSep(std::string_view, const std::list &); template std::string concatStringsSep(std::string_view, const std::set &); diff --git a/src/libutil/unix/users.cc b/src/libutil/unix/users.cc index 58063a953c3..107a6e04f98 100644 --- a/src/libutil/unix/users.cc +++ b/src/libutil/unix/users.cc @@ -9,6 +9,8 @@ namespace nix { +namespace fs { using namespace std::filesystem; } + std::string getUserName() { auto pw = getpwuid(geteuid()); diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index e152c26f2de..5b7862c4e0c 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -6,6 +6,8 @@ #include "local-fs-store.hh" #include "eval-inline.hh" +namespace nix::fs { using namespace std::filesystem; } + using namespace nix; struct CmdBundle : InstallableValueCommand @@ -78,7 +80,7 @@ struct CmdBundle : InstallableValueCommand auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec( - fetchSettings, bundler, absPath(".")); + fetchSettings, bundler, fs::current_path().string()); const flake::LockFlags lockFlags{ .writeLockFile = false }; InstallableFlake bundler{this, evalState, std::move(bundlerFlakeRef), bundlerName, std::move(extendedOutputsSpec), diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index 1a6574de2ea..6cf73785ea2 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -10,6 +10,8 @@ #include "worker-protocol.hh" #include "executable-path.hh" +namespace nix::fs { using namespace std::filesystem; } + using namespace nix; namespace { @@ -40,8 +42,6 @@ void checkInfo(const std::string & msg) { } -namespace fs = std::filesystem; - struct CmdConfigCheck : StoreCommand { bool success = true; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index effc86a0a72..04672e2ada1 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -21,6 +21,8 @@ #include "strings.hh" +namespace nix::fs { using namespace std::filesystem; } + using namespace nix; struct DevelopSettings : Config @@ -341,7 +343,7 @@ struct Common : InstallableCommand, MixProfile ref store, const BuildEnvironment & buildEnvironment, const std::filesystem::path & tmpDir, - const std::filesystem::path & outputsDir = std::filesystem::path { absPath(".") } / "outputs") + const std::filesystem::path & outputsDir = fs::path { fs::current_path() } / "outputs") { // A list of colon-separated environment variables that should be // prepended to, rather than overwritten, in order to keep the shell usable. @@ -450,7 +452,7 @@ struct Common : InstallableCommand, MixProfile auto targetFilePath = tmpDir / OS_STR(".attrs."); targetFilePath += ext; - writeFile(targetFilePath.string(), content); + writeFile(targetFilePath, content); auto fileInBuilderEnv = buildEnvironment.vars.find(envVar); assert(fileInBuilderEnv != buildEnvironment.vars.end()); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b7bbb767b31..2db1e039ec9 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -25,7 +25,7 @@ #include "strings-inline.hh" -namespace fs = std::filesystem; +namespace nix::fs { using namespace std::filesystem; } using namespace nix; using namespace nix::flake; @@ -53,7 +53,7 @@ class FlakeCommand : virtual Args, public MixFlakeOptions FlakeRef getFlakeRef() { - return parseFlakeRef(fetchSettings, flakeUrl, absPath(".")); //FIXME + return parseFlakeRef(fetchSettings, flakeUrl, fs::current_path().string()); //FIXME } LockedFlake lockFlake() @@ -65,7 +65,7 @@ class FlakeCommand : virtual Args, public MixFlakeOptions { return { // Like getFlakeRef but with expandTilde calld first - parseFlakeRef(fetchSettings, expandTilde(flakeUrl), absPath(".")) + parseFlakeRef(fetchSettings, expandTilde(flakeUrl), fs::current_path().string()) }; } }; @@ -880,7 +880,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto evalState = getEvalState(); auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment( - fetchSettings, templateUrl, absPath(".")); + fetchSettings, templateUrl, fs::current_path().string()); auto installable = InstallableFlake(nullptr, evalState, std::move(templateFlakeRef), templateName, ExtendedOutputsSpec::Default(), @@ -927,7 +927,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand } continue; } else - writeFile(to2.string(), contents); + writeFile(to2, contents); } else if (fs::is_symlink(st)) { auto target = fs::read_symlink(from2); diff --git a/src/nix/run.cc b/src/nix/run.cc index dfe7f374f8d..63ae8a19579 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -20,6 +20,8 @@ #include +namespace nix::fs { using namespace std::filesystem; } + using namespace nix; std::string chrootHelperName = "__run_in_chroot"; @@ -170,25 +172,25 @@ void chrootHelper(int argc, char * * argv) if (!pathExists(storeDir)) { // FIXME: Use overlayfs? - std::filesystem::path tmpDir = createTempDir(); + fs::path tmpDir = createTempDir(); createDirs(tmpDir + storeDir); if (mount(realStoreDir.c_str(), (tmpDir + storeDir).c_str(), "", MS_BIND, 0) == -1) throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); - for (auto entry : std::filesystem::directory_iterator{"/"}) { + for (auto entry : fs::directory_iterator{"/"}) { checkInterrupt(); auto src = entry.path(); - Path dst = tmpDir / entry.path().filename(); + fs::path dst = tmpDir / entry.path().filename(); if (pathExists(dst)) continue; auto st = entry.symlink_status(); - if (std::filesystem::is_directory(st)) { + if (fs::is_directory(st)) { if (mkdir(dst.c_str(), 0700) == -1) throw SysError("creating directory '%s'", dst); if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) throw SysError("mounting '%s' on '%s'", src, dst); - } else if (std::filesystem::is_symlink(st)) + } else if (fs::is_symlink(st)) createSymlink(readLink(src), dst); } @@ -205,9 +207,9 @@ void chrootHelper(int argc, char * * argv) if (mount(realStoreDir.c_str(), storeDir.c_str(), "", MS_BIND, 0) == -1) throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); - writeFile("/proc/self/setgroups", "deny"); - writeFile("/proc/self/uid_map", fmt("%d %d %d", uid, uid, 1)); - writeFile("/proc/self/gid_map", fmt("%d %d %d", gid, gid, 1)); + writeFile(fs::path{"/proc/self/setgroups"}, "deny"); + writeFile(fs::path{"/proc/self/uid_map"}, fmt("%d %d %d", uid, uid, 1)); + writeFile(fs::path{"/proc/self/gid_map"}, fmt("%d %d %d", gid, gid, 1)); #if __linux__ if (system != "") diff --git a/src/nix/self-exe.cc b/src/nix/self-exe.cc index a260bafd589..81a117e604e 100644 --- a/src/nix/self-exe.cc +++ b/src/nix/self-exe.cc @@ -5,7 +5,9 @@ namespace nix { -namespace fs = std::filesystem; +namespace fs { +using namespace std::filesystem; +} fs::path getNixBin(std::optional binaryNameOpt) { diff --git a/tests/unit/libfetchers/public-key.cc b/tests/unit/libfetchers/public-key.cc index 8a639da9f6a..80796bd0fc9 100644 --- a/tests/unit/libfetchers/public-key.cc +++ b/tests/unit/libfetchers/public-key.cc @@ -10,11 +10,11 @@ using nlohmann::json; class PublicKeyTest : public CharacterizationTest { - Path unitTestData = getUnitTestData() + "/public-key"; + std::filesystem::path unitTestData = getUnitTestData() / "public-key"; public: - Path goldenMaster(std::string_view testStem) const override { - return unitTestData + "/" + testStem; + std::filesystem::path goldenMaster(std::string_view testStem) const override { + return unitTestData / testStem; } }; diff --git a/tests/unit/libstore-support/tests/nix_api_store.hh b/tests/unit/libstore-support/tests/nix_api_store.hh index a2d35d083d4..193b44970d0 100644 --- a/tests/unit/libstore-support/tests/nix_api_store.hh +++ b/tests/unit/libstore-support/tests/nix_api_store.hh @@ -10,7 +10,7 @@ #include #include -namespace fs = std::filesystem; +namespace fs { using namespace std::filesystem; } namespace nixC { class nix_api_store_test : public nix_api_util_context diff --git a/tests/unit/libstore-support/tests/protocol.hh b/tests/unit/libstore-support/tests/protocol.hh index 3c9e52c117a..3f6799d1ccb 100644 --- a/tests/unit/libstore-support/tests/protocol.hh +++ b/tests/unit/libstore-support/tests/protocol.hh @@ -12,10 +12,10 @@ namespace nix { template class ProtoTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/" + protocolDir; + std::filesystem::path unitTestData = getUnitTestData() / protocolDir; - Path goldenMaster(std::string_view testStem) const override { - return unitTestData + "/" + testStem + ".bin"; + std::filesystem::path goldenMaster(std::string_view testStem) const override { + return unitTestData / (std::string { testStem + ".bin" }); } }; diff --git a/tests/unit/libstore/derivation-advanced-attrs.cc b/tests/unit/libstore/derivation-advanced-attrs.cc index 26cf947a853..4d839ddaba0 100644 --- a/tests/unit/libstore/derivation-advanced-attrs.cc +++ b/tests/unit/libstore/derivation-advanced-attrs.cc @@ -16,12 +16,12 @@ using nlohmann::json; class DerivationAdvancedAttrsTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/derivation"; + std::filesystem::path unitTestData = getUnitTestData() / "derivation"; public: - Path goldenMaster(std::string_view testStem) const override + std::filesystem::path goldenMaster(std::string_view testStem) const override { - return unitTestData + "/" + testStem; + return unitTestData / testStem; } }; diff --git a/tests/unit/libstore/derivation.cc b/tests/unit/libstore/derivation.cc index 71979f88598..14652921abc 100644 --- a/tests/unit/libstore/derivation.cc +++ b/tests/unit/libstore/derivation.cc @@ -13,11 +13,11 @@ using nlohmann::json; class DerivationTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/derivation"; + std::filesystem::path unitTestData = getUnitTestData() / "derivation"; public: - Path goldenMaster(std::string_view testStem) const override { - return unitTestData + "/" + testStem; + std::filesystem::path goldenMaster(std::string_view testStem) const override { + return unitTestData / testStem; } /** diff --git a/tests/unit/libstore/machines.cc b/tests/unit/libstore/machines.cc index 2307f4d629d..2d66e953408 100644 --- a/tests/unit/libstore/machines.cc +++ b/tests/unit/libstore/machines.cc @@ -13,6 +13,8 @@ using testing::Eq; using testing::Field; using testing::SizeIs; +namespace nix::fs { using namespace std::filesystem; } + using namespace nix; TEST(machines, getMachinesWithEmptyBuilders) { @@ -135,10 +137,10 @@ TEST(machines, getMachinesWithIncorrectFormat) { } TEST(machines, getMachinesWithCorrectFileReference) { - auto path = absPath(getUnitTestData() + "/machines/valid"); - ASSERT_TRUE(pathExists(path)); + auto path = fs::weakly_canonical(getUnitTestData() / "machines/valid"); + ASSERT_TRUE(fs::exists(path)); - auto actual = Machine::parseConfig({}, "@" + path); + auto actual = Machine::parseConfig({}, "@" + path.string()); ASSERT_THAT(actual, SizeIs(3)); EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl")))); @@ -146,20 +148,22 @@ TEST(machines, getMachinesWithCorrectFileReference) { } TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) { - auto path = "/dev/null"; - ASSERT_TRUE(pathExists(path)); + fs::path path = "/dev/null"; + ASSERT_TRUE(fs::exists(path)); - auto actual = Machine::parseConfig({}, std::string{"@"} + path); + auto actual = Machine::parseConfig({}, "@" + path.string()); ASSERT_THAT(actual, SizeIs(0)); } TEST(machines, getMachinesWithIncorrectFileReference) { - auto actual = Machine::parseConfig({}, "@" + absPath("/not/a/file")); + auto path = fs::weakly_canonical("/not/a/file"); + ASSERT_TRUE(!fs::exists(path)); + auto actual = Machine::parseConfig({}, "@" + path.string()); ASSERT_THAT(actual, SizeIs(0)); } TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) { EXPECT_THROW( - Machine::parseConfig({}, "@" + absPath(getUnitTestData() + "/machines/bad_format")), + Machine::parseConfig({}, "@" + fs::weakly_canonical(getUnitTestData() / "machines" / "bad_format").string()), FormatError); } diff --git a/tests/unit/libstore/nar-info.cc b/tests/unit/libstore/nar-info.cc index a6cb62de4ac..0d155743d91 100644 --- a/tests/unit/libstore/nar-info.cc +++ b/tests/unit/libstore/nar-info.cc @@ -13,10 +13,10 @@ using nlohmann::json; class NarInfoTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/nar-info"; + std::filesystem::path unitTestData = getUnitTestData() / "nar-info"; - Path goldenMaster(PathView testStem) const override { - return unitTestData + "/" + testStem + ".json"; + std::filesystem::path goldenMaster(PathView testStem) const override { + return unitTestData / (testStem + ".json"); } }; diff --git a/tests/unit/libstore/path-info.cc b/tests/unit/libstore/path-info.cc index 9e9c6303d90..d6c4c2a7f7e 100644 --- a/tests/unit/libstore/path-info.cc +++ b/tests/unit/libstore/path-info.cc @@ -12,10 +12,10 @@ using nlohmann::json; class PathInfoTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/path-info"; + std::filesystem::path unitTestData = getUnitTestData() / "path-info"; - Path goldenMaster(PathView testStem) const override { - return unitTestData + "/" + testStem + ".json"; + std::filesystem::path goldenMaster(PathView testStem) const override { + return unitTestData / (testStem + ".json"); } }; diff --git a/tests/unit/libstore/store-reference.cc b/tests/unit/libstore/store-reference.cc index 052cd7beddd..d4c42f0fda1 100644 --- a/tests/unit/libstore/store-reference.cc +++ b/tests/unit/libstore/store-reference.cc @@ -13,11 +13,11 @@ using nlohmann::json; class StoreReferenceTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/store-reference"; + std::filesystem::path unitTestData = getUnitTestData() / "store-reference"; - Path goldenMaster(PathView testStem) const override + std::filesystem::path goldenMaster(PathView testStem) const override { - return unitTestData + "/" + testStem + ".txt"; + return unitTestData / (testStem + ".txt"); } }; diff --git a/tests/unit/libutil-support/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh index 19ba824ac74..5e790e75ba6 100644 --- a/tests/unit/libutil-support/tests/characterization.hh +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -13,7 +13,7 @@ namespace nix { * The path to the unit test data directory. See the contributing guide * in the manual for further details. */ -static inline Path getUnitTestData() { +static inline std::filesystem::path getUnitTestData() { return getEnv("_NIX_TEST_UNIT_DATA").value(); } @@ -36,7 +36,7 @@ protected: * While the "golden master" for this characterization test is * located. It should not be shared with any other test. */ - virtual Path goldenMaster(PathView testStem) const = 0; + virtual std::filesystem::path goldenMaster(PathView testStem) const = 0; public: /** @@ -77,7 +77,7 @@ public: if (testAccept()) { - createDirs(dirOf(file)); + std::filesystem::create_directories(file.parent_path()); writeFile2(file, got); GTEST_SKIP() << "Updating golden master " @@ -97,10 +97,10 @@ public: { writeTest( testStem, test, - [](const Path & f) -> std::string { + [](const std::filesystem::path & f) -> std::string { return readFile(f); }, - [](const Path & f, const std::string & c) { + [](const std::filesystem::path & f, const std::string & c) { return writeFile(f, c); }); } diff --git a/tests/unit/libutil/file-system.cc b/tests/unit/libutil/file-system.cc index cfddaae1cd3..7ef804f3403 100644 --- a/tests/unit/libutil/file-system.cc +++ b/tests/unit/libutil/file-system.cc @@ -12,8 +12,8 @@ #include #ifdef _WIN32 -# define FS_SEP "\\" -# define FS_ROOT "C:" FS_SEP // Need a mounted one, C drive is likely +# define FS_SEP L"\\" +# define FS_ROOT L"C:" FS_SEP // Need a mounted one, C drive is likely #else # define FS_SEP "/" # define FS_ROOT FS_SEP @@ -23,6 +23,12 @@ # define PATH_MAX 4096 #endif +#ifdef _WIN32 +# define GET_CWD _wgetcwd +#else +# define GET_CWD getcwd +#endif + namespace nix { /* ----------- tests for file-system.hh -------------------------------------*/ @@ -33,34 +39,34 @@ namespace nix { TEST(absPath, doesntChangeRoot) { - auto p = absPath(FS_ROOT); + auto p = absPath(std::filesystem::path{FS_ROOT}); ASSERT_EQ(p, FS_ROOT); } TEST(absPath, turnsEmptyPathIntoCWD) { - char cwd[PATH_MAX + 1]; - auto p = absPath(""); + OsChar cwd[PATH_MAX + 1]; + auto p = absPath(std::filesystem::path{""}); - ASSERT_EQ(p, getcwd((char *) &cwd, PATH_MAX)); + ASSERT_EQ(p, GET_CWD((OsChar *) &cwd, PATH_MAX)); } TEST(absPath, usesOptionalBasePathWhenGiven) { - char _cwd[PATH_MAX + 1]; - char * cwd = getcwd((char *) &_cwd, PATH_MAX); + OsChar _cwd[PATH_MAX + 1]; + OsChar * cwd = GET_CWD((OsChar *) &_cwd, PATH_MAX); - auto p = absPath("", cwd); + auto p = absPath(std::filesystem::path{""}.string(), std::filesystem::path{cwd}.string()); - ASSERT_EQ(p, cwd); + ASSERT_EQ(p, std::filesystem::path{cwd}.string()); } TEST(absPath, isIdempotent) { - char _cwd[PATH_MAX + 1]; - char * cwd = getcwd((char *) &_cwd, PATH_MAX); - auto p1 = absPath(cwd); + OsChar _cwd[PATH_MAX + 1]; + OsChar * cwd = GET_CWD((OsChar *) &_cwd, PATH_MAX); + auto p1 = absPath(std::filesystem::path{cwd}); auto p2 = absPath(p1); ASSERT_EQ(p1, p2); @@ -68,8 +74,8 @@ TEST(absPath, isIdempotent) TEST(absPath, pathIsCanonicalised) { - auto path = FS_ROOT "some/path/with/trailing/dot/."; - auto p1 = absPath(path); + auto path = FS_ROOT OS_STR("some/path/with/trailing/dot/."); + auto p1 = absPath(std::filesystem::path{path}); auto p2 = absPath(p1); ASSERT_EQ(p1, FS_ROOT "some" FS_SEP "path" FS_SEP "with" FS_SEP "trailing" FS_SEP "dot"); @@ -82,26 +88,26 @@ TEST(absPath, pathIsCanonicalised) TEST(canonPath, removesTrailingSlashes) { - auto path = FS_ROOT "this/is/a/path//"; - auto p = canonPath(path); + std::filesystem::path path = FS_ROOT "this/is/a/path//"; + auto p = canonPath(path.string()); - ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); + ASSERT_EQ(p, std::filesystem::path{FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"}.string()); } TEST(canonPath, removesDots) { - auto path = FS_ROOT "this/./is/a/path/./"; - auto p = canonPath(path); + std::filesystem::path path = FS_ROOT "this/./is/a/path/./"; + auto p = canonPath(path.string()); - ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); + ASSERT_EQ(p, std::filesystem::path{FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"}.string()); } TEST(canonPath, removesDots2) { - auto path = FS_ROOT "this/a/../is/a////path/foo/.."; - auto p = canonPath(path); + std::filesystem::path path = FS_ROOT "this/a/../is/a////path/foo/.."; + auto p = canonPath(path.string()); - ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); + ASSERT_EQ(p, std::filesystem::path{FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"}.string()); } TEST(canonPath, requiresAbsolutePath) @@ -243,7 +249,7 @@ TEST(isDirOrInDir, DISABLED_shouldWork) TEST(pathExists, rootExists) { - ASSERT_TRUE(pathExists(FS_ROOT)); + ASSERT_TRUE(pathExists(std::filesystem::path{FS_ROOT}.string())); } TEST(pathExists, cwdExists) diff --git a/tests/unit/libutil/git.cc b/tests/unit/libutil/git.cc index 3d01d980609..9232de5b99a 100644 --- a/tests/unit/libutil/git.cc +++ b/tests/unit/libutil/git.cc @@ -11,12 +11,12 @@ using namespace git; class GitTest : public CharacterizationTest { - Path unitTestData = getUnitTestData() + "/git"; + std::filesystem::path unitTestData = getUnitTestData() / "git"; public: - Path goldenMaster(std::string_view testStem) const override { - return unitTestData + "/" + testStem; + std::filesystem::path goldenMaster(std::string_view testStem) const override { + return unitTestData / std::string(testStem); } /** diff --git a/tests/unit/libutil/meson.build b/tests/unit/libutil/meson.build index 83cec13ec06..c39db8cda1d 100644 --- a/tests/unit/libutil/meson.build +++ b/tests/unit/libutil/meson.build @@ -48,12 +48,14 @@ subdir('build-utils-meson/diagnostics') sources = files( 'args.cc', 'canon-path.cc', + 'checked-arithmetic.cc', 'chunked-vector.cc', 'closure.cc', 'compression.cc', 'config.cc', 'executable-path.cc', 'file-content-address.cc', + 'file-system.cc', 'git.cc', 'hash.cc', 'hilite.cc', @@ -62,6 +64,7 @@ sources = files( 'lru-cache.cc', 'nix_api_util.cc', 'pool.cc', + 'position.cc', 'processes.cc', 'references.cc', 'spawn.cc', From 8bce63f30a773611f782ea3a53b9ab46180392c7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 26 Aug 2024 16:44:19 -0400 Subject: [PATCH 280/284] More `std::filesystem` for `nix-collect-garbage` Co-Authored-By: siddhantCodes --- .../nix-collect-garbage.cc | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 457e5f3c9d2..20d5161df09 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -11,6 +11,8 @@ #include #include +namespace nix::fs { using namespace std::filesystem; } + using namespace nix; std::string deleteOlderThan; @@ -21,23 +23,23 @@ bool dryRun = false; * Of course, this makes rollbacks to before this point in time * impossible. */ -void removeOldGenerations(std::filesystem::path dir) +void removeOldGenerations(fs::path dir) { if (access(dir.string().c_str(), R_OK) != 0) return; bool canWrite = access(dir.string().c_str(), W_OK) == 0; - for (auto & i : std::filesystem::directory_iterator{dir}) { + for (auto & i : fs::directory_iterator{dir}) { checkInterrupt(); auto path = i.path().string(); auto type = i.symlink_status().type(); - if (type == std::filesystem::file_type::symlink && canWrite) { + if (type == fs::file_type::symlink && canWrite) { std::string link; try { link = readLink(path); - } catch (std::filesystem::filesystem_error & e) { + } catch (fs::filesystem_error & e) { if (e.code() == std::errc::no_such_file_or_directory) continue; throw; } @@ -49,7 +51,7 @@ void removeOldGenerations(std::filesystem::path dir) } else deleteOldGenerations(path, dryRun); } - } else if (type == std::filesystem::file_type::directory) { + } else if (type == fs::file_type::directory) { removeOldGenerations(path); } } @@ -81,8 +83,11 @@ static int main_nix_collect_garbage(int argc, char * * argv) }); if (removeOld) { - std::set dirsToClean = { - profilesDir(), settings.nixStateDir + "/profiles", dirOf(getDefaultProfile())}; + std::set dirsToClean = { + profilesDir(), + fs::path{settings.nixStateDir} / "profiles", + fs::path{getDefaultProfile()}.parent_path(), + }; for (auto & dir : dirsToClean) removeOldGenerations(dir); } From c7ec33605e8c2dff0ebe40e4a1beba7a98530432 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 14 Aug 2024 16:19:35 -0400 Subject: [PATCH 281/284] Meson misc things Meson-ify a few things, scripts, completions, etc. Should make our Meson build complete except for docs. Co-Authored-By: Qyriad Co-Authored-By: eldritch horrors --- build-utils-meson/export/meson.build | 3 +++ flake.nix | 3 ++- meson.build | 4 +++- misc/bash/meson.build | 8 +++++++ misc/fish/meson.build | 8 +++++++ misc/meson.build | 5 +++++ misc/systemd/meson.build | 25 ++++++++++++++++++++++ misc/zsh/meson.build | 10 +++++++++ scripts/meson.build | 29 ++++++++++++++++++++++++++ src/libstore/meson.build | 22 ++++++++++++++----- src/libstore/meson.options | 4 ---- src/libutil/meson.build | 2 +- src/nix/meson.build | 14 ++++++++++++- src/nix/meson.options | 6 ++++++ src/nix/misc | 1 + src/nix/package.nix | 8 ++++++- src/nix/scripts | 1 + subprojects | 1 + tests/unit/libutil-support/meson.build | 1 + 19 files changed, 141 insertions(+), 14 deletions(-) create mode 100644 misc/bash/meson.build create mode 100644 misc/fish/meson.build create mode 100644 misc/meson.build create mode 100644 misc/systemd/meson.build create mode 100644 misc/zsh/meson.build create mode 100644 scripts/meson.build create mode 100644 src/nix/meson.options create mode 120000 src/nix/misc create mode 120000 src/nix/scripts create mode 120000 subprojects diff --git a/build-utils-meson/export/meson.build b/build-utils-meson/export/meson.build index 40f6dcd59a0..9f59505721e 100644 --- a/build-utils-meson/export/meson.build +++ b/build-utils-meson/export/meson.build @@ -10,6 +10,7 @@ foreach dep : deps_public_subproject endforeach requires_public += deps_public +extra_pkg_config_variables = get_variable('extra_pkg_config_variables', {}) import('pkgconfig').generate( this_library, filebase : meson.project_name(), @@ -20,6 +21,7 @@ import('pkgconfig').generate( requires : requires_public, requires_private : requires_private, libraries_private : libraries_private, + variables : extra_pkg_config_variables, ) meson.override_dependency(meson.project_name(), declare_dependency( @@ -27,4 +29,5 @@ meson.override_dependency(meson.project_name(), declare_dependency( link_with : this_library, compile_args : ['-std=c++2a'], dependencies : deps_public_subproject + deps_public, + variables : extra_pkg_config_variables, )) diff --git a/flake.nix b/flake.nix index f5893d8f6fd..5ca9c1a45ce 100644 --- a/flake.nix +++ b/flake.nix @@ -295,6 +295,7 @@ devShells = let makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: let + buildCanExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform; modular = devFlake.getSystem stdenv.buildPlatform.system; transformFlag = prefix: flag: assert builtins.isString flag; @@ -352,7 +353,7 @@ ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs ++ pkgs.nixComponents.nix-functional-tests.baseNativeBuildInputs ++ lib.optional - (!stdenv.buildPlatform.canExecute stdenv.hostPlatform + (!buildCanExecuteHost # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages diff --git a/meson.build b/meson.build index 715a3862d4f..8dd44cc1063 100644 --- a/meson.build +++ b/meson.build @@ -3,7 +3,9 @@ project('nix-dev-shell', 'cpp', version : files('.version'), - subproject_dir : 'src', + default_options : [ + 'localstatedir=/nix/var', + ] ) # Internal Libraries diff --git a/misc/bash/meson.build b/misc/bash/meson.build new file mode 100644 index 00000000000..8a97a02cbf0 --- /dev/null +++ b/misc/bash/meson.build @@ -0,0 +1,8 @@ +configure_file( + input : 'completion.sh', + output : 'nix', + install : true, + install_dir : get_option('datadir') / 'bash-completion' / 'completions', + install_mode : 'rw-r--r--', + copy : true, +) diff --git a/misc/fish/meson.build b/misc/fish/meson.build new file mode 100644 index 00000000000..e7e89b43827 --- /dev/null +++ b/misc/fish/meson.build @@ -0,0 +1,8 @@ +configure_file( + input : 'completion.fish', + output : 'nix.fish', + install : true, + install_dir : get_option('datadir') / 'fish' / 'vendor_completions.d', + install_mode : 'rw-r--r--', + copy : true, +) diff --git a/misc/meson.build b/misc/meson.build new file mode 100644 index 00000000000..a6d1f944bbc --- /dev/null +++ b/misc/meson.build @@ -0,0 +1,5 @@ +subdir('bash') +subdir('fish') +subdir('zsh') + +subdir('systemd') diff --git a/misc/systemd/meson.build b/misc/systemd/meson.build new file mode 100644 index 00000000000..58b30f30bff --- /dev/null +++ b/misc/systemd/meson.build @@ -0,0 +1,25 @@ +foreach config : [ 'nix-daemon.socket', 'nix-daemon.service' ] + configure_file( + input : config + '.in', + output : config, + install : true, + install_dir : get_option('prefix') / 'lib/systemd/system', + install_mode : 'rw-r--r--', + configuration : { + 'storedir' : store_dir, + 'localstatedir' : localstatedir, + 'bindir' : get_option('datadir'), + }, + ) +endforeach + +configure_file( + input : 'nix-daemon.conf.in', + output : 'nix-daemon.conf', + install : true, + install_dir : get_option('prefix') / 'lib/tmpfiles.d', + install_mode : 'rw-r--r--', + configuration : { + 'localstatedir' : localstatedir, + }, +) diff --git a/misc/zsh/meson.build b/misc/zsh/meson.build new file mode 100644 index 00000000000..f3d0426e792 --- /dev/null +++ b/misc/zsh/meson.build @@ -0,0 +1,10 @@ +foreach script : [ [ 'completion.zsh', '_nix' ], [ 'run-help-nix' ] ] + configure_file( + input : script[0], + output : script.get(1, script[0]), + install : true, + install_dir : get_option('datadir') / 'zsh/site-functions', + install_mode : 'rw-r--r--', + copy : true, + ) +endforeach diff --git a/scripts/meson.build b/scripts/meson.build new file mode 100644 index 00000000000..2671e6a13d4 --- /dev/null +++ b/scripts/meson.build @@ -0,0 +1,29 @@ +# configures `scripts/nix-profile.sh.in` (and copies the original to the build directory). +# this is only needed for tests, but running it unconditionally does not hurt enough to care. +configure_file( + input : 'nix-profile.sh.in', + output : 'nix-profile.sh', + configuration : { + 'localstatedir': localstatedir, + } +) + +# https://github.com/mesonbuild/meson/issues/860 +configure_file( + input : 'nix-profile.sh.in', + output : 'nix-profile.sh.in', + copy : true, +) + +foreach rc : [ '.sh', '.fish', '-daemon.sh', '-daemon.fish' ] + configure_file( + input : 'nix-profile' + rc + '.in', + output : 'nix' + rc, + install : true, + install_dir : get_option('profile-dir'), + install_mode : 'rw-r--r--', + configuration : { + 'localstatedir': localstatedir, + }, + ) +endforeach diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 8e30845e122..2adc9b3e4ff 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -7,6 +7,7 @@ project('nix-store', 'cpp', 'debug=true', 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail + 'localstatedir=/nix/var', ], meson_version : '>= 1.1', license : 'LGPL-2.1-or-later', @@ -324,7 +325,7 @@ fs = import('fs') prefix = get_option('prefix') # For each of these paths, assume that it is relative to the prefix unless -# it is already an absolute path (which is the default for store-dir, state-dir, and log-dir). +# it is already an absolute path (which is the default for store-dir, localstatedir, and log-dir). path_opts = [ # Meson built-ins. 'datadir', @@ -334,13 +335,13 @@ path_opts = [ 'libexecdir', # Homecooked Nix directories. 'store-dir', - 'state-dir', + 'localstatedir', 'log-dir', ] # For your grepping pleasure, this loop sets the following variables that aren't mentioned # literally above: # store_dir -# state_dir +# localstatedir # log_dir # profile_dir foreach optname : path_opts @@ -364,12 +365,12 @@ lsof = find_program('lsof', required : false) # Aside from prefix itself, each of these was made into an absolute path # by joining it with prefix, unless it was already an absolute path -# (which is the default for store-dir, state-dir, and log-dir). +# (which is the default for store-dir, localstatedir, and log-dir). cpp_str_defines = { 'NIX_PREFIX': prefix, 'NIX_STORE_DIR': store_dir, 'NIX_DATA_DIR': datadir, - 'NIX_STATE_DIR': state_dir / 'nix', + 'NIX_STATE_DIR': localstatedir / 'nix', 'NIX_LOG_DIR': log_dir, 'NIX_CONF_DIR': sysconfdir / 'nix', 'NIX_MAN_DIR': mandir, @@ -421,4 +422,15 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] +extra_pkg_config_variables = { + 'storedir' : get_option('store-dir'), +} + +# Working around https://github.com/mesonbuild/meson/issues/13584 +if host_machine.system() != 'macos' + extra_pkg_config_variables += { + 'localstatedir' : get_option('localstatedir'), + } +endif + subdir('build-utils-meson/export') diff --git a/src/libstore/meson.options b/src/libstore/meson.options index 723a8e020fd..ebad24dc47f 100644 --- a/src/libstore/meson.options +++ b/src/libstore/meson.options @@ -16,10 +16,6 @@ option('store-dir', type : 'string', value : '/nix/store', description : 'path of the Nix store', ) -option('state-dir', type : 'string', value : '/nix/var', - description : 'path to store state in for Nix', -) - option('log-dir', type : 'string', value : '/nix/var/log/nix', description : 'path to store logs in for Nix', ) diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 72ff461ca5e..797dcae6d7e 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -129,7 +129,6 @@ sources = files( 'english.cc', 'environment-variables.cc', 'error.cc', - 'exec.hh', 'executable-path.cc', 'exit.cc', 'experimental-features.cc', @@ -186,6 +185,7 @@ headers = [config_h] + files( 'english.hh', 'environment-variables.hh', 'error.hh', + 'exec.hh', 'executable-path.hh', 'exit.hh', 'experimental-features.hh', diff --git a/src/nix/meson.build b/src/nix/meson.build index 798c98e3367..6edb768e31c 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -7,6 +7,7 @@ project('nix', 'cpp', 'debug=true', 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail + 'localstatedir=/nix/var', ], meson_version : '>= 1.1', license : 'LGPL-2.1-or-later', @@ -16,9 +17,11 @@ cxx = meson.get_compiler('cpp') subdir('build-utils-meson/deps-lists') +nix_store = dependency('nix-store') + deps_private_maybe_subproject = [ dependency('nix-util'), - dependency('nix-store'), + nix_store, dependency('nix-expr'), dependency('nix-flake'), dependency('nix-fetchers'), @@ -244,3 +247,12 @@ custom_target( ) # TODO(Ericson3214): Dosen't yet work #meson.override_find_program(linkname, t) + +localstatedir = nix_store.get_variable( + 'localstatedir', + default_value : get_option('localstatedir'), +) +assert(localstatedir == get_option('localstatedir')) +store_dir = nix_store.get_variable('storedir') +subdir('scripts') +subdir('misc') diff --git a/src/nix/meson.options b/src/nix/meson.options new file mode 100644 index 00000000000..8430dd66907 --- /dev/null +++ b/src/nix/meson.options @@ -0,0 +1,6 @@ +# vim: filetype=meson + +# A relative path means it gets appended to prefix. +option('profile-dir', type : 'string', value : 'etc/profile.d', + description : 'the path to install shell profile files', +) diff --git a/src/nix/misc b/src/nix/misc new file mode 120000 index 00000000000..2825552c9b6 --- /dev/null +++ b/src/nix/misc @@ -0,0 +1 @@ +../../misc \ No newline at end of file diff --git a/src/nix/package.nix b/src/nix/package.nix index ef7265458fb..3e19c6dca87 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -36,9 +36,10 @@ mkMesonDerivation (finalAttrs: { ../../.version ./.version ./meson.build - # ./meson.options + ./meson.options # Symbolic links to other dirs + ## exes ./build-remote ./doc ./nix-build @@ -48,6 +49,11 @@ mkMesonDerivation (finalAttrs: { ./nix-env ./nix-instantiate ./nix-store + ## dirs + ./scripts + ../../scripts + ./misc + ../../misc # Doc nix files for --help ../../doc/manual/generate-manpage.nix diff --git a/src/nix/scripts b/src/nix/scripts new file mode 120000 index 00000000000..c5efc95ebfa --- /dev/null +++ b/src/nix/scripts @@ -0,0 +1 @@ +../../scripts \ No newline at end of file diff --git a/subprojects b/subprojects new file mode 120000 index 00000000000..e8310385c56 --- /dev/null +++ b/subprojects @@ -0,0 +1 @@ +src \ No newline at end of file diff --git a/tests/unit/libutil-support/meson.build b/tests/unit/libutil-support/meson.build index 6be4972c6bd..42b49a6a0b3 100644 --- a/tests/unit/libutil-support/meson.build +++ b/tests/unit/libutil-support/meson.build @@ -46,6 +46,7 @@ include_dirs = [include_directories('.')] headers = files( 'tests/characterization.hh', + 'tests/gtest-with-params.hh', 'tests/hash.hh', 'tests/nix_api_util.hh', 'tests/string_callback.hh', From 8e63dc4455b80c096c74a97f07cfc15a1264502d Mon Sep 17 00:00:00 2001 From: Seggy Umboh Date: Tue, 27 Aug 2024 17:23:27 -0700 Subject: [PATCH 282/284] Fix diskutil deleteVolume command in Uninstall guide (#11371) Co-authored-by: Valentin Gagarin --- doc/manual/src/installation/uninstall.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/installation/uninstall.md b/doc/manual/src/installation/uninstall.md index 590327fea1b..91fb90bc0fb 100644 --- a/doc/manual/src/installation/uninstall.md +++ b/doc/manual/src/installation/uninstall.md @@ -133,7 +133,9 @@ which you may remove. diskutil list ``` - If you _do_ find a "Nix Store" volume, delete it by running `diskutil deleteVolume` with the store volume's `diskXsY` identifier. + If you _do_ find a "Nix Store" volume, delete it by running `diskutil apfs deleteVolume` with the store volume's `diskXsY` identifier. + + If you get an error that the volume is in use by the kernel, reboot and immediately delete the volume before starting any other process. > **Note** > From cb4b9be4587efcee0c642eeb0aafab006b314d95 Mon Sep 17 00:00:00 2001 From: Jan Hrcek <2716069+jhrcek@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:01:56 +0200 Subject: [PATCH 283/284] Fix few duplicate word occurrences (#11381) --- doc/manual/src/development/json-guideline.md | 2 +- doc/manual/src/protocols/store-path.md | 2 +- doc/manual/src/release-notes/rl-2.23.md | 2 +- package.nix | 2 +- src/libstore/local-overlay-store.md | 4 ++-- src/nix/flake.md | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/development/json-guideline.md b/doc/manual/src/development/json-guideline.md index b4bc92af961..309b4b3a06e 100644 --- a/doc/manual/src/development/json-guideline.md +++ b/doc/manual/src/development/json-guideline.md @@ -90,7 +90,7 @@ This representation is extensible and preserves the ordering: ## Self-describing values -As described in the previous section, it's crucial that schemas can be extended with with new fields without breaking compatibility. +As described in the previous section, it's crucial that schemas can be extended with new fields without breaking compatibility. However, that should *not* mean we use the presence/absence of fields to indicate optional information *within* a version of the schema. Instead, always include the field, and use `null` to indicate the "nothing" case. diff --git a/doc/manual/src/protocols/store-path.md b/doc/manual/src/protocols/store-path.md index 52352d358a1..8ec6f8201ff 100644 --- a/doc/manual/src/protocols/store-path.md +++ b/doc/manual/src/protocols/store-path.md @@ -82,7 +82,7 @@ where - if `type` = `"source:" ...`: - the the hash of the [Nix Archive (NAR)] serialization of the [file system object](@docroot@/store/file-system-object.md) of the store object. + the hash of the [Nix Archive (NAR)] serialization of the [file system object](@docroot@/store/file-system-object.md) of the store object. - if `type` = `"output:" id`: diff --git a/doc/manual/src/release-notes/rl-2.23.md b/doc/manual/src/release-notes/rl-2.23.md index ac842fdc058..76e9534a89d 100644 --- a/doc/manual/src/release-notes/rl-2.23.md +++ b/doc/manual/src/release-notes/rl-2.23.md @@ -85,7 +85,7 @@ - Store object info JSON format now uses `null` rather than omitting fields [#9995](https://github.com/NixOS/nix/pull/9995) The [store object info JSON format](@docroot@/protocols/json/store-object-info.md), used for e.g. `nix path-info`, no longer omits fields to indicate absent information, but instead includes the fields with a `null` value. - For example, `"ca": null` is used to to indicate a store object that isn't content-addressed rather than omitting the `ca` field entirely. + For example, `"ca": null` is used to indicate a store object that isn't content-addressed rather than omitting the `ca` field entirely. This makes records of this sort more self-describing, and easier to consume programmatically. We will follow this design principle going forward; diff --git a/package.nix b/package.nix index d41748b7cf5..5c8d9f9b65d 100644 --- a/package.nix +++ b/package.nix @@ -60,7 +60,7 @@ # Run the functional tests as part of the build. , doInstallCheck ? test-client != null || __forDefaults.canRunInstalled -# Check test coverage of Nix. Probably want to use with with at least +# Check test coverage of Nix. Probably want to use with at least # one of `doCHeck` or `doInstallCheck` enabled. , withCoverageChecks ? false diff --git a/src/libstore/local-overlay-store.md b/src/libstore/local-overlay-store.md index 1e1a3d26c7b..baa3958951c 100644 --- a/src/libstore/local-overlay-store.md +++ b/src/libstore/local-overlay-store.md @@ -77,13 +77,13 @@ The parts of a local overlay store are as follows: The lower store directory and upper layer directory are combined via OverlayFS to create this directory. Nix doesn't do this itself, because it typically wouldn't have the permissions to do so, so it is the responsibility of the user to set this up first. - Nix can, however, optionally check that that the OverlayFS mount settings appear as expected, matching Nix's own settings. + Nix can, however, optionally check that the OverlayFS mount settings appear as expected, matching Nix's own settings. - **Upper SQLite database**: > Not directly specified. > The location of the database instead depends on the [`state`](#store-experimental-local-overlay-store-state) setting. - > It is is always `${state}/db`. + > It is always `${state}/db`. This contains the metadata of all of the upper layer [store objects][store object] (everything beyond their file system objects), and also duplicate copies of some lower layer store object's metadta. The duplication is so the metadata for the [closure](@docroot@/glossary.md#gloss-closure) of upper layer [store objects][store object] can be found entirely within the upper layer. diff --git a/src/nix/flake.md b/src/nix/flake.md index 46d5a3867bf..d8ee4ac717a 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -120,7 +120,7 @@ Contrary to URL-like references, path-like flake references can contain arbitrar ### Examples -* `.`: The flake to which the current directory belongs to. +* `.`: The flake to which the current directory belongs. * `/home/alice/src/patchelf`: A flake in some other directory. * `./../sub directory/with Ûñî©ôδ€`: A flake in another relative directory that has Unicode characters in its name. From 9b1cefe27e542d890aa346996a03dfecd9793dfe Mon Sep 17 00:00:00 2001 From: Bryan Honof Date: Wed, 28 Aug 2024 18:48:18 +0200 Subject: [PATCH 284/284] Update src/libutil/url.cc Co-authored-by: Robert Hensing --- src/libutil/url.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 78c83244007..8ef1857bb50 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -81,7 +81,7 @@ std::map decodeQuery(const std::string & query) auto e = s.find('='); if (e == std::string::npos) { - warn("invalid URI query '%s', did you forget an equals sign `=`?", s); + warn("dubious URI query '%s' is missing equal sign '%s'", s, "="); continue; }