From e764ed31f61308690a9a8a5dc6f20e570b7de262 Mon Sep 17 00:00:00 2001 From: Lexi Mattick Date: Fri, 12 Jul 2024 09:45:35 -0700 Subject: [PATCH 1/5] Eval cache: fix cache regressions - Fix eval cache not being persisted in `nix develop` (since #10570) - Don't attempt to commit cache transaction if there is no active transaction, which will spew errors in edge cases - Drive-by: trivial typo fix --- src/libcmd/installable-value.hh | 2 +- src/libexpr/eval-cache.cc | 2 +- src/nix/develop.cc | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh index f300d392bb5..798cb5e1a00 100644 --- a/src/libcmd/installable-value.hh +++ b/src/libcmd/installable-value.hh @@ -66,7 +66,7 @@ struct ExtraPathInfoValue : ExtraPathInfo }; /** - * An Installable which corresponds a Nix langauge value, in addition to + * An Installable which corresponds a Nix language value, in addition to * a collection of \ref DerivedPath "derived paths". */ struct InstallableValue : Installable diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 2630c34d563..5085fedc2f0 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -95,7 +95,7 @@ struct AttrDb { try { auto state(_state->lock()); - if (!failed) + if (!failed && state->txn->active) state->txn->commit(); state->txn.reset(); } catch (...) { diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 6bd3dc9efc6..b89de5d5c1d 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -697,6 +697,10 @@ struct CmdDevelop : Common, MixEnvironment } } + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + getEvalState()->evalCaches.clear(); + runProgramInStore(store, UseLookupPath::Use, shell, args, buildEnvironment.getSystem()); #endif } From 6c4470ec2a495fc201447aa8fca63c71f9849fe6 Mon Sep 17 00:00:00 2001 From: Lexi Mattick Date: Fri, 12 Jul 2024 11:54:12 -0700 Subject: [PATCH 2/5] Clean up cache for all commands --- src/nix/develop.cc | 2 +- src/nix/env.cc | 7 ++++++- src/nix/fmt.cc | 7 ++++++- src/nix/run.cc | 8 ++++++-- src/nix/run.hh | 2 +- 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index b89de5d5c1d..84237be85dd 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -701,7 +701,7 @@ struct CmdDevelop : Common, MixEnvironment // we are about to exec out of this process without running C++ destructors. getEvalState()->evalCaches.clear(); - runProgramInStore(store, UseLookupPath::Use, shell, args, buildEnvironment.getSystem()); + execProgramInStore(store, UseLookupPath::Use, shell, args, buildEnvironment.getSystem()); #endif } }; diff --git a/src/nix/env.cc b/src/nix/env.cc index 021c47cbbd0..84a87eaeeb7 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "eval.hh" #include "run.hh" #include @@ -99,7 +100,11 @@ struct CmdShell : InstallablesCommand, MixEnvironment for (auto & arg : command) args.push_back(arg); - runProgramInStore(store, UseLookupPath::Use, *command.begin(), args); + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + getEvalState()->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::Use, *command.begin(), args); } }; diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc index 4b0fbb89dfb..d6583449575 100644 --- a/src/nix/fmt.cc +++ b/src/nix/fmt.cc @@ -1,5 +1,6 @@ #include "command.hh" #include "installable-value.hh" +#include "eval.hh" #include "run.hh" using namespace nix; @@ -49,7 +50,11 @@ struct CmdFmt : SourceExprCommand { } } - runProgramInStore(store, UseLookupPath::DontUse, app.program, programArgs); + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + evalState->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::DontUse, app.program, programArgs); }; }; diff --git a/src/nix/run.cc b/src/nix/run.cc index c1aae168577..ec6a4d1e82e 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -25,7 +25,7 @@ std::string chrootHelperName = "__run_in_chroot"; namespace nix { -void runProgramInStore(ref store, +void execProgramInStore(ref store, UseLookupPath useLookupPath, const std::string & program, const Strings & args, @@ -128,7 +128,11 @@ struct CmdRun : InstallableValueCommand Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); - runProgramInStore(store, UseLookupPath::DontUse, app.program, allArgs); + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + state->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::DontUse, app.program, allArgs); } }; diff --git a/src/nix/run.hh b/src/nix/run.hh index 2fe6ed86ae6..51517fdc94a 100644 --- a/src/nix/run.hh +++ b/src/nix/run.hh @@ -10,7 +10,7 @@ enum struct UseLookupPath { DontUse }; -void runProgramInStore(ref store, +void execProgramInStore(ref store, UseLookupPath useLookupPath, const std::string & program, const Strings & args, From bb7bf73682fee2b57e0d08f7b728358c1b1ecc42 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 12:01:10 -0400 Subject: [PATCH 3/5] More progress converting --- src/libcmd/common-eval-args.cc | 2 +- src/libcmd/repl.cc | 4 +- src/libstore/binary-cache-store.cc | 72 +++++++++++++++++++++++-- src/libstore/binary-cache-store.hh | 60 ++++++++++----------- src/libstore/build/substitution-goal.cc | 1 + src/libstore/common-ssh-store-config.cc | 49 ++++++++++++++++- src/libstore/common-ssh-store-config.hh | 43 +++++++++------ src/libstore/local-fs-store.cc | 10 ++-- src/libstore/local-fs-store.hh | 3 -- src/libstore/local-store.cc | 4 +- src/libstore/local-store.hh | 1 - src/libstore/remote-fs-accessor.hh | 2 +- src/libstore/remote-store.cc | 18 +++++++ src/libstore/remote-store.hh | 30 ++++++++--- src/libstore/store-api.cc | 10 ++-- src/libstore/store-api.hh | 4 -- src/libstore/store-dir-config.cc | 2 +- src/libstore/uds-remote-store.hh | 28 ++++------ src/libutil/config-abstract.hh | 12 +++++ 19 files changed, 251 insertions(+), 104 deletions(-) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index decadd751cc..fc75f047f4b 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -9,7 +9,7 @@ #include "registry.hh" #include "flake/flakeref.hh" #include "flake/settings.hh" -#include "store-api.hh" +#include "store-open.hh" #include "command.hh" #include "tarball.hh" #include "fetch-to-store.hh" diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index b5d0816dd2c..4dd41ca663d 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -12,7 +12,7 @@ #include "eval-settings.hh" #include "attr-path.hh" #include "signals.hh" -#include "store-api.hh" +#include "store-open.hh" #include "log-store.hh" #include "common-eval-args.hh" #include "get-drvs.hh" @@ -635,7 +635,7 @@ ProcessLineResult NixRepl::processLine(std::string line) // When missing, trigger the normal exception // e.g. :doc builtins.foo // behaves like - // nix-repl> builtins.foo + // nix-repl> builtins.foo // error: attribute 'foo' missing evalString(arg, v); assert(false); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index e8c8892b337..4adc599007e 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -13,6 +13,7 @@ #include "callback.hh" #include "signals.hh" #include "archive.hh" +#include "config-parse-impl.hh" #include #include @@ -24,9 +25,74 @@ namespace nix { -BinaryCacheStore::BinaryCacheStore(const Params & params) - : BinaryCacheStoreConfig(params) - , Store(params) +BinaryCacheStore::Config::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , BinaryCacheStoreConfigT{ + .compression = { + .name = "compression", + .description = "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`).", + }, + .writeNARListing = { + .name = "write-nar-listing", + .description = "Whether to write a JSON file that lists the files in each NAR.", + }, + .writeDebugInfo = { + .name = "index-debug-info", + .description = R"( + Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to + fetch debug info on demand + )", + }, + .secretKeyFile{ + .name = "secret-key", + .description = "Path to the secret key used to sign the binary cache.", + }, + .localNarCache{ + .name = "local-nar-cache", + .description = "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`.", + }, + .parallelCompression{ + .name = "parallel-compression", + .description = "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`.", + }, + .compressionLevel{ + .name = "compression-level", + .description = R"( + The *preset level* to be used when compressing NARs. + The meaning and accepted values depend on the compression method selected. + `-1` specifies that the default compression level should be used. + )", + }, + } +{} + +const BinaryCacheStore::Config::Descriptions BinaryCacheStore::Config::descriptions{}; + +decltype(BinaryCacheStore::Config::defaults) BinaryCacheStore::Config::defaults = { + .compression = {"xz"}, + .writeNARListing{false}, + .writeDebugInfo{false}, + .secretKeyFile{""}, + .localNarCache{""}, + .parallelCompression{false}, + .compressionLevel{-1}, +}; + +BinaryCacheStore::Config::BinaryCacheStoreConfig(const StoreReference::Params & params) + : StoreConfig{params} + , BinaryCacheStoreConfigT{ + CONFIG_ROW(compression), + CONFIG_ROW(secretKeyFile), + CONFIG_ROW(localNarCache), + CONFIG_ROW(parallelCompression), + CONFIG_ROW(compressionLevel), + } +{ +} + +BinaryCacheStore::BinaryCacheStore(const Config & config) + : Config{config} + , Store{static_cast(*this)} { if (secretKeyFile != "") signer = std::make_unique( diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 234c39caf3a..27d7a845ab7 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -13,48 +13,46 @@ namespace nix { struct NarInfo; -struct BinaryCacheStoreConfig : virtual StoreConfig +template class F> +struct BinaryCacheStoreConfigT { - using StoreConfig::StoreConfig; - - const Setting compression{this, "xz", "compression", - "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; - - const Setting writeNARListing{this, false, "write-nar-listing", - "Whether to write a JSON file that lists the files in each NAR."}; - - const Setting writeDebugInfo{this, false, "index-debug-info", - R"( - Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to - fetch debug info on demand - )"}; + const F compression; + const F writeNARListing; + const F writeDebugInfo; + const F secretKeyFile; + const F localNarCache; + const F parallelCompression; + const F compressionLevel; +}; - const Setting secretKeyFile{this, "", "secret-key", - "Path to the secret key used to sign the binary cache."}; +struct BinaryCacheStoreConfig : + virtual Store::Config, + BinaryCacheStoreConfigT +{ + struct Descriptions : + virtual Store::Config::Descriptions, + BinaryCacheStoreConfigT + { + Descriptions(); + }; - const Setting localNarCache{this, "", "local-nar-cache", - "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; + static const Descriptions descriptions; - const Setting parallelCompression{this, false, "parallel-compression", - "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; + static BinaryCacheStoreConfigT defaults; - const Setting compressionLevel{this, -1, "compression-level", - R"( - The *preset level* to be used when compressing NARs. - The meaning and accepted values depend on the compression method selected. - `-1` specifies that the default compression level should be used. - )"}; + BinaryCacheStoreConfig(const StoreReference::Params &); }; - /** * @note subclasses must implement at least one of the two * virtual getFile() methods. */ -class BinaryCacheStore : public virtual BinaryCacheStoreConfig, - public virtual Store, - public virtual LogStore +struct BinaryCacheStore : + virtual BinaryCacheStoreConfig, + virtual Store, + virtual LogStore { + using Config = BinaryCacheStoreConfig; private: std::unique_ptr signer; @@ -64,7 +62,7 @@ protected: // The prefix under which realisation infos will be stored const std::string realisationsPrefix = "realisations"; - BinaryCacheStore(const Params & params); + BinaryCacheStore(const Config &); public: diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 7deeb47487d..32189ca7ae5 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -1,4 +1,5 @@ #include "worker.hh" +#include "store-open.hh" #include "substitution-goal.hh" #include "nar-info.hh" #include "finally.hh" diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index 05332b9bb5c..e1448b4b4eb 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -2,9 +2,45 @@ #include "common-ssh-store-config.hh" #include "ssh.hh" +#include "config-parse-impl.hh" namespace nix { +CommonSSHStoreConfig::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , CommonSSHStoreConfigT{ + .sshKey{ + .name = "ssh-key", + .description = "Path to the SSH private key used to authenticate to the remote machine.", + }, + .sshPublicHostKey = { + .name = "base64-ssh-public-host-key", + .description = "The public host key of the remote machine.", + }, + .compress = { + .name = "compress", + .description = "Whether to enable SSH compression.", + }, + .remoteStore{ + .name = "remote-store", + .description = R"( + [Store URL](@docroot@/store/types/index.md#store-url-format) + to be used on the remote machine. The default is `auto` + (i.e. use the Nix daemon or `/nix/store` directly). + )", + }, + } +{} + +const CommonSSHStoreConfig::Descriptions CommonSSHStoreConfig::descriptions{}; + +decltype(CommonSSHStoreConfig::defaults) CommonSSHStoreConfig::defaults = { + .sshKey{""}, + .sshPublicHostKey{""}, + .compress{false}, + .remoteStore{""}, +}; + static std::string extractConnStr(std::string_view scheme, std::string_view _connStr) { if (_connStr.empty()) @@ -22,8 +58,17 @@ static std::string extractConnStr(std::string_view scheme, std::string_view _con return connStr; } -CommonSSHStoreConfig::CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params) - : StoreConfig(params) +CommonSSHStoreConfig::CommonSSHStoreConfig( + std::string_view scheme, + std::string_view host, + const StoreReference::Params & params) + : Store::Config(params) + , CommonSSHStoreConfigT{ + CONFIG_ROW(sshKey), + CONFIG_ROW(sshPublicHostKey), + CONFIG_ROW(compress), + CONFIG_ROW(remoteStore), + } , host(extractConnStr(scheme, host)) { } diff --git a/src/libstore/common-ssh-store-config.hh b/src/libstore/common-ssh-store-config.hh index 5deb6f4c9e9..4def51da0e5 100644 --- a/src/libstore/common-ssh-store-config.hh +++ b/src/libstore/common-ssh-store-config.hh @@ -7,27 +7,38 @@ namespace nix { class SSHMaster; -struct CommonSSHStoreConfig : virtual StoreConfig +template class F> +struct CommonSSHStoreConfigT { - using StoreConfig::StoreConfig; - - CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params); + const F sshKey; + const F sshPublicHostKey; + const F compress; + const F remoteStore; +}; - const Setting sshKey{this, "", "ssh-key", - "Path to the SSH private key used to authenticate to the remote machine."}; +struct CommonSSHStoreConfig : + virtual Store::Config, + CommonSSHStoreConfigT +{ + struct Descriptions : + virtual Store::Config::Descriptions, + CommonSSHStoreConfigT + { + Descriptions(); + }; - const Setting sshPublicHostKey{this, "", "base64-ssh-public-host-key", - "The public host key of the remote machine."}; + static const Descriptions descriptions; - const Setting compress{this, false, "compress", - "Whether to enable SSH compression."}; + static CommonSSHStoreConfigT defaults; - const Setting remoteStore{this, "", "remote-store", - R"( - [Store URL](@docroot@/store/types/index.md#store-url-format) - to be used on the remote machine. The default is `auto` - (i.e. use the Nix daemon or `/nix/store` directly). - )"}; + /** + * @param scheme Note this isn't stored by this mix-in class, but + * just used for better error messages. + */ + CommonSSHStoreConfig( + std::string_view scheme, + std::string_view host, + const StoreReference::Params & params); /** * The `parseURL` function supports both IPv6 URIs as defined in diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index a8b23996159..12035c1efe7 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -33,15 +33,15 @@ LocalFSStore::Config::Descriptions::Descriptions() const LocalFSStore::Config::Descriptions LocalFSStore::Config::descriptions{}; -LocalFSStoreConfigT LocalFSStoreConfig::defaults( +LocalFSStoreConfigT LocalFSStore::Config::defaults( const Store::Config & storeConfig, const std::optional rootDir) { return { - .rootDir = {.value = std::nullopt }, - .stateDir = {.value = rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir }, - .logDir = {.value = rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir }, - .realStoreDir = {.value = rootDir ? *rootDir + "/nix/store" : storeConfig.storeDir }, + .rootDir = {std::nullopt}, + .stateDir = {rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir}, + .logDir = {rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir}, + .realStoreDir = {rootDir ? *rootDir + "/nix/store" : storeConfig.storeDir}, }; } diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 654dd210f40..051e4494012 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -11,11 +11,8 @@ template class F> struct LocalFSStoreConfigT { const F> rootDir; - const F stateDir; - const F logDir; - const F realStoreDir; }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2dfc78ebda3..aff70589d18 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -89,8 +89,8 @@ LocalStore::Config::Descriptions::Descriptions() const LocalStore::Config::Descriptions LocalStore::Config::descriptions{}; decltype(LocalStore::Config::defaults) LocalStore::Config::defaults = { - .requireSigs = {.value = settings.requireSigs }, - .readOnly = {.value = false }, + .requireSigs = {settings.requireSigs}, + .readOnly = {false}, }; LocalStore::Config::LocalStoreConfig( diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 13e26922c0a..06e402f80f9 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -38,7 +38,6 @@ template class F> struct LocalStoreConfigT { const F requireSigs; - const F readOnly; }; diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index d09762a53c4..6946074affa 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -19,7 +19,7 @@ class RemoteFSAccessor : public SourceAccessor std::pair, CanonPath> fetch(const CanonPath & path); - friend class BinaryCacheStore; + friend struct BinaryCacheStore; Path makeCacheFile(std::string_view hashPart, const std::string & ext); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 6e8931ca2e4..30d1523b550 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -23,6 +23,24 @@ namespace nix { +#if 0 + const Setting maxConnections{this, 1, "max-connections", + "Maximum number of concurrent connections to the Nix daemon."}; + + const Setting maxConnectionAge{this, + std::numeric_limits::max(), + "max-connection-age", + "Maximum age of a connection before it is closed."}; + + const Setting maxConnections{this, 1, "max-connections", + "Maximum number of concurrent connections to the Nix daemon."}; + + const Setting maxConnectionAge{this, + std::numeric_limits::max(), + "max-connection-age", + "Maximum age of a connection before it is closed."}; +#endif + /* TODO: Separate these store types into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) : RemoteStoreConfig(params) diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index f19aa0fc57b..437c4aee734 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -18,17 +18,31 @@ struct FdSink; struct FdSource; template class Pool; -struct RemoteStoreConfig : virtual StoreConfig +template class F> +struct RemoteStoreConfigT { - using StoreConfig::StoreConfig; + const F maxConnections; - const Setting maxConnections{this, 1, "max-connections", - "Maximum number of concurrent connections to the Nix daemon."}; + const F maxConnectionAge; +}; + +struct RemoteStoreConfig : + virtual Store::Config, + RemoteStoreConfigT +{ + struct Descriptions : + virtual Store::Config::Descriptions, + RemoteStoreConfigT + { + Descriptions(); + }; - const Setting maxConnectionAge{this, - std::numeric_limits::max(), - "max-connection-age", - "Maximum age of a connection before it is closed."}; + static const Descriptions descriptions; + + /** + * The other defaults depend on the choice of `storeDir` and `rootDir` + */ + static RemoteStoreConfigT defaults; }; /** diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c4bb9d594b0..b67faf38a5e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -239,11 +239,11 @@ const Store::Config::Descriptions Store::Config::descriptions{}; decltype(Store::Config::defaults) Store::Config::defaults = { - .pathInfoCacheSize = { .value = 65536 }, - .isTrusted = { .value = false }, - .priority = { .value = 0 }, - .wantMassQuery = { .value = false }, - .systemFeatures = { .value = Store::Config::getDefaultSystemFeatures() }, + .pathInfoCacheSize = {65536}, + .isTrusted = {false}, + .priority = {0}, + .wantMassQuery = {false}, + .systemFeatures = {Store::Config::getDefaultSystemFeatures()}, }; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 09352e601ba..7e9a5fa53f5 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -100,13 +100,9 @@ template class F> struct StoreConfigT { const F pathInfoCacheSize; - const F isTrusted; - F priority; - F wantMassQuery; - F systemFeatures; }; diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc index a51962d7fa9..026a6cd9837 100644 --- a/src/libstore/store-dir-config.cc +++ b/src/libstore/store-dir-config.cc @@ -17,7 +17,7 @@ const StoreDirConfigT StoreDirConfig::descriptions = { }; const StoreDirConfigT StoreDirConfig::defaults = { - ._storeDir = {.value = settings.nixStore}, + ._storeDir = {settings.nixStore}, }; StoreDirConfig::StoreDirConfig(const StoreReference::Params & params) diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index a8e57166416..8d19ad5f5bc 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -7,7 +7,9 @@ namespace nix { -struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig +struct UDSRemoteStoreConfig : + virtual LocalFSStoreConfig, + virtual RemoteStoreConfig { // TODO(fzakaria): Delete this constructor once moved over to the factory pattern // outlined in https://github.com/NixOS/nix/issues/10766 @@ -34,7 +36,7 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon */ Path path; -protected: +private: static constexpr char const * scheme = "unix"; public: @@ -42,24 +44,12 @@ public: { return {scheme}; } }; -class UDSRemoteStore : public virtual UDSRemoteStoreConfig - , public virtual IndirectRootStore - , public virtual RemoteStore +struct UDSRemoteStore : + virtual UDSRemoteStoreConfig, + virtual IndirectRootStore, + virtual RemoteStore { -public: - - /** - * @deprecated This is the old API to construct the store. - */ - UDSRemoteStore(const Params & params); - - /** - * @param authority is the socket path. - */ - UDSRemoteStore( - std::string_view scheme, - std::string_view authority, - const Params & params); + UDSRemoteStore(const UDSRemoteStoreConfig &); std::string getUri() override; diff --git a/src/libutil/config-abstract.hh b/src/libutil/config-abstract.hh index 10c98f0a73f..fcde0696b98 100644 --- a/src/libutil/config-abstract.hh +++ b/src/libutil/config-abstract.hh @@ -12,14 +12,26 @@ struct JustValue { return value; } + operator T &() { return value; } + const T & get() const { return value; } + + bool operator==(auto && v2) const + { + return value == v2; + } + + bool operator!=(auto && v2) const + { + return value != v2; + } }; template From ca1e89513141503450b3ee9fb371060f9cc92d9e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 12:31:46 -0400 Subject: [PATCH 4/5] WIP --- src/libstore/common-ssh-store-config.cc | 43 ++++----- src/libstore/dummy-store.cc | 4 +- src/libstore/filetransfer.cc | 2 +- src/libstore/http-binary-cache-store.cc | 32 ++++--- src/libstore/http-binary-cache-store.hh | 7 +- src/libstore/local-binary-cache-store.cc | 30 +++---- src/libstore/local-binary-cache-store.hh | 11 ++- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 5 +- src/libstore/machines.cc | 2 +- src/libstore/misc.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 76 ++++++++++++++-- src/libstore/s3-binary-cache-store.hh | 109 +++++++---------------- src/libstore/store-api.hh | 2 +- 14 files changed, 175 insertions(+), 152 deletions(-) diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index e1448b4b4eb..a2d7e2ebad6 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -9,28 +9,29 @@ namespace nix { CommonSSHStoreConfig::Descriptions::Descriptions() : Store::Config::Descriptions{Store::Config::descriptions} , CommonSSHStoreConfigT{ - .sshKey{ - .name = "ssh-key", - .description = "Path to the SSH private key used to authenticate to the remote machine.", - }, - .sshPublicHostKey = { - .name = "base64-ssh-public-host-key", - .description = "The public host key of the remote machine.", - }, - .compress = { - .name = "compress", - .description = "Whether to enable SSH compression.", - }, - .remoteStore{ - .name = "remote-store", - .description = R"( - [Store URL](@docroot@/store/types/index.md#store-url-format) - to be used on the remote machine. The default is `auto` - (i.e. use the Nix daemon or `/nix/store` directly). + .sshKey{ + .name = "ssh-key", + .description = "Path to the SSH private key used to authenticate to the remote machine.", + }, + .sshPublicHostKey{ + .name = "base64-ssh-public-host-key", + .description = "The public host key of the remote machine.", + }, + .compress{ + .name = "compress", + .description = "Whether to enable SSH compression.", + }, + .remoteStore{ + .name = "remote-store", + .description = R"( + [Store URL](@docroot@/store/types/index.md#store-url-format) + to be used on the remote machine. The default is `auto` + (i.e. use the Nix daemon or `/nix/store` directly). )", - }, - } -{} + }, + } +{ +} const CommonSSHStoreConfig::Descriptions CommonSSHStoreConfig::descriptions{}; diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index c6dd71ba669..968c2d3fbac 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -36,7 +36,7 @@ struct DummyStoreConfig : virtual StoreConfig { return {"dummy"}; } - std::shared_ptr openStore() override; + std::shared_ptr openStore() const override; }; @@ -100,7 +100,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store { unsupported("getFSAccessor"); } }; -std::shared_ptr DummyStore::Config::openStore() +std::shared_ptr DummyStore::Config::openStore() const { return std::make_shared(*this); } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index cbbb0fe7a34..ce296668798 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -715,7 +715,7 @@ struct curlFileTransfer : public FileTransfer } #if ENABLE_S3 - std::tuple parseS3Uri(std::string uri) + std::tuple parseS3Uri(std::string uri) { auto [path, params] = splitUriAndParams(uri); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index b15ef4e4cba..0a663ec036d 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -3,6 +3,7 @@ #include "globals.hh" #include "nar-info-disk-cache.hh" #include "callback.hh" +#include "store-registration.hh" namespace nix { @@ -12,7 +13,7 @@ MakeError(UploadToHTTP, Error); HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( std::string_view scheme, std::string_view _cacheUri, - const Params & params) + const StoreReference::Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) , cacheUri( @@ -35,10 +36,10 @@ std::string HttpBinaryCacheStoreConfig::doc() } -class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore +class HttpBinaryCacheStore : + public virtual HttpBinaryCacheStoreConfig, + public virtual BinaryCacheStore { -private: - struct State { bool enabled = true; @@ -49,15 +50,14 @@ class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public v public: - HttpBinaryCacheStore( - std::string_view scheme, - PathView cacheUri, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , HttpBinaryCacheStoreConfig(scheme, cacheUri, params) - , Store(params) - , BinaryCacheStore(params) + using Config = HttpBinaryCacheStoreConfig; + + HttpBinaryCacheStore(const Config & config) + : Store::Config{config} + , BinaryCacheStore::Config{config} + , HttpBinaryCacheStore::Config{config} + , Store{static_cast(*this)} + , BinaryCacheStore{static_cast(*this)} { diskCache = getNarInfoDiskCache(); } @@ -207,6 +207,10 @@ class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public v } }; -static RegisterStoreImplementation regHttpBinaryCacheStore; +std::shared_ptr HttpBinaryCacheStore::Config::openStore() const { + return std::make_shared(*this); +} + +static RegisterStoreImplementation regHttpBinaryCacheStore; } diff --git a/src/libstore/http-binary-cache-store.hh b/src/libstore/http-binary-cache-store.hh index d2fc43210a2..d15c6e3d0db 100644 --- a/src/libstore/http-binary-cache-store.hh +++ b/src/libstore/http-binary-cache-store.hh @@ -4,9 +4,8 @@ namespace nix { struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - - HttpBinaryCacheStoreConfig(std::string_view scheme, std::string_view _cacheUri, const Params & params); + HttpBinaryCacheStoreConfig( + std::string_view scheme, std::string_view cacheUri, const StoreReference::Params & params); Path cacheUri; @@ -25,6 +24,8 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig } std::string doc() override; + + std::shared_ptr openStore() const override; }; } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index dcc6affe4a1..8fb163e3064 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -2,6 +2,7 @@ #include "globals.hh" #include "nar-info-disk-cache.hh" #include "signals.hh" +#include "store-registration.hh" #include @@ -10,7 +11,7 @@ namespace nix { LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig( std::string_view scheme, PathView binaryCacheDir, - const Params & params) + const StoreReference::Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) , binaryCacheDir(binaryCacheDir) @@ -26,21 +27,18 @@ std::string LocalBinaryCacheStoreConfig::doc() } -struct LocalBinaryCacheStore : virtual LocalBinaryCacheStoreConfig, virtual BinaryCacheStore +struct LocalBinaryCacheStore : + virtual LocalBinaryCacheStoreConfig, + virtual BinaryCacheStore { - /** - * @param binaryCacheDir `file://` is a short-hand for `file:///` - * for now. - */ - LocalBinaryCacheStore( - std::string_view scheme, - PathView binaryCacheDir, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , LocalBinaryCacheStoreConfig(scheme, binaryCacheDir, params) - , Store(params) - , BinaryCacheStore(params) + using Config = LocalBinaryCacheStoreConfig; + + LocalBinaryCacheStore(const Config & config) + : Store::Config{config} + , BinaryCacheStore::Config{config} + , LocalBinaryCacheStore::Config{config} + , Store{static_cast(*this)} + , BinaryCacheStore{static_cast(*this)} { } @@ -127,6 +125,6 @@ std::set LocalBinaryCacheStoreConfig::uriSchemes() return {"file"}; } -static RegisterStoreImplementation regLocalBinaryCacheStore; +static RegisterStoreImplementation regLocalBinaryCacheStore; } diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh index 997e8ecbb51..01f7969eb61 100644 --- a/src/libstore/local-binary-cache-store.hh +++ b/src/libstore/local-binary-cache-store.hh @@ -4,9 +4,12 @@ namespace nix { struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - - LocalBinaryCacheStoreConfig(std::string_view scheme, PathView binaryCacheDir, const Params & params); + /** + * @param binaryCacheDir `file://` is a short-hand for `file:///` + * for now. + */ + LocalBinaryCacheStoreConfig( + std::string_view scheme, PathView binaryCacheDir, const StoreReference::Params & params); Path binaryCacheDir; @@ -18,6 +21,8 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig static std::set uriSchemes(); std::string doc() override; + + std::shared_ptr openStore() const override; }; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index aff70589d18..d3677fdeb39 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -113,7 +113,7 @@ std::string LocalStoreConfig::doc() ; } -std::shared_ptr LocalStore::Config::openStore() +std::shared_ptr LocalStore::Config::openStore() const { return std::make_shared(*this); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 06e402f80f9..45cb04b7159 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -56,9 +56,6 @@ struct LocalStoreConfig : static const Descriptions descriptions; - /** - * The other defaults depend on the choice of `storeDir` and `rootDir` - */ static LocalStoreConfigT defaults; LocalStoreConfig(const StoreReference::Params &); @@ -75,7 +72,7 @@ struct LocalStoreConfig : std::string doc() override; - std::shared_ptr openStore() override; + std::shared_ptr openStore() const override; }; class LocalStore : diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 256cf918892..9e4145e0ac5 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -1,6 +1,6 @@ #include "machines.hh" #include "globals.hh" -#include "store-api.hh" +#include "store-open.hh" #include diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index bcc02206bc9..31762f34d2c 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -3,7 +3,7 @@ #include "derivations.hh" #include "parsed-derivations.hh" #include "globals.hh" -#include "store-api.hh" +#include "store-open.hh" #include "thread-pool.hh" #include "realisation.hh" #include "topo-sort.hh" diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 1a0ec11112e..abd2727194b 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -1,7 +1,5 @@ #if ENABLE_S3 -#include - #include "s3.hh" #include "s3-binary-cache-store.hh" #include "nar-info.hh" @@ -187,6 +185,75 @@ S3Helper::FileTransferResult S3Helper::getObject( return res; } + const Setting profile{ + this, + "", + "profile", + R"( + The name of the AWS configuration profile to use. By default + Nix will use the `default` profile. + )"}; + + const Setting region{ + this, + Aws::Region::US_EAST_1, + "region", + R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )"}; + + const Setting scheme{ + this, + "", + "scheme", + R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )"}; + + const Setting endpoint{ + this, + "", + "endpoint", + R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and will use path-based + > addressing instead of virtual host based addressing. + )"}; + + const Setting narinfoCompression{ + this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; + + const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; + + const Setting logCompression{ + this, + "", + "log-compression", + R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )"}; + + const Setting multipartUpload{this, false, "multipart-upload", "Whether to use multi-part uploads."}; + + const Setting bufferSize{ + this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; + + S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) : BinaryCacheStoreConfig(params) , BinaryCacheStore(params) @@ -201,11 +268,6 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( , BinaryCacheStoreConfig(params) , bucketName(bucketName) { - // Don't want to use use AWS SDK in header, so we check the default - // here. TODO do this better after we overhaul the store settings - // system. - assert(std::string{defaultRegion} == std::string{Aws::Region::US_EAST_1}); - if (bucketName.empty()) throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme); } diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index 7d303a115f4..c28e9320378 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -7,87 +7,40 @@ namespace nix { -struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +template class F> +struct S3BinaryCacheStoreConfigT { - std::string bucketName; - - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - - S3BinaryCacheStoreConfig(std::string_view uriScheme, std::string_view bucketName, const Params & params); + const F profile; + const F region; + const F scheme; + const F endpoint; + const F narinfoCompression; + const F lsCompression; + const F logCompression; + const F multipartUpload; + const F bufferSize; +}; - const Setting profile{ - this, - "", - "profile", - R"( - The name of the AWS configuration profile to use. By default - Nix will use the `default` profile. - )"}; +struct S3BinaryCacheStoreConfig : + virtual BinaryCacheStoreConfig, + S3BinaryCacheStoreConfigT +{ + struct Descriptions : + virtual Store::Config::Descriptions, + virtual BinaryCacheStore::Config::Descriptions, + S3BinaryCacheStoreConfigT + { + Descriptions(); + }; -protected: + static const Descriptions descriptions; - constexpr static const char * defaultRegion = "us-east-1"; + static S3BinaryCacheStoreConfigT defaults; -public: + S3BinaryCacheStoreConfig( + std::string_view uriScheme, std::string_view bucketName, const StoreReference::Params & params); - const Setting region{ - this, - defaultRegion, - "region", - R"( - The region of the S3 bucket. If your bucket is not in - `us–east-1`, you should always explicitly specify the region - parameter. - )"}; - - const Setting scheme{ - this, - "", - "scheme", - R"( - The scheme used for S3 requests, `https` (default) or `http`. This - option allows you to disable HTTPS for binary caches which don't - support it. - - > **Note** - > - > HTTPS should be used if the cache might contain sensitive - > information. - )"}; - - const Setting endpoint{ - this, - "", - "endpoint", - R"( - The URL of the endpoint of an S3-compatible service such as MinIO. - Do not specify this setting if you're using Amazon S3. - - > **Note** - > - > This endpoint must support HTTPS and will use path-based - > addressing instead of virtual host based addressing. - )"}; - - const Setting narinfoCompression{ - this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; - - const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; - - const Setting logCompression{ - this, - "", - "log-compression", - R"( - Compression method for `log/*` files. It is recommended to - use a compression method supported by most web browsers - (e.g. `brotli`). - )"}; - - const Setting multipartUpload{this, false, "multipart-upload", "Whether to use multi-part uploads."}; - - const Setting bufferSize{ - this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; + std::string bucketName; const std::string name() override { @@ -102,11 +55,13 @@ public: std::string doc() override; }; -class S3BinaryCacheStore : public virtual BinaryCacheStore +struct S3BinaryCacheStore : virtual BinaryCacheStore { + using Config = S3BinaryCacheStoreConfig; + protected: - S3BinaryCacheStore(const Params & params); + S3BinaryCacheStore(const Config &); public: diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7e9a5fa53f5..6571fd37c10 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -154,7 +154,7 @@ struct StoreConfig : * Open a store of the type corresponding to this configuration * type. */ - virtual std::shared_ptr openStore() = 0; + virtual std::shared_ptr openStore() const = 0; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig From 901b34fb71ff224a3dde41d0cf24a428511f5246 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 13:22:29 -0400 Subject: [PATCH 5/5] WIP --- src/libstore/dummy-store.cc | 6 +- src/libstore/http-binary-cache-store.cc | 5 +- src/libstore/http-binary-cache-store.hh | 2 +- src/libstore/local-binary-cache-store.hh | 2 +- src/libstore/local-store.cc | 4 +- src/libstore/local-store.hh | 2 +- src/libstore/s3-binary-cache-store.cc | 216 ++++++++++-------- src/libstore/s3-binary-cache-store.hh | 2 + src/libstore/store-api.hh | 2 +- .../unix/build/local-derivation-goal.cc | 63 +++-- 10 files changed, 180 insertions(+), 124 deletions(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 968c2d3fbac..3d2931cd9ba 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -36,7 +36,7 @@ struct DummyStoreConfig : virtual StoreConfig { return {"dummy"}; } - std::shared_ptr openStore() const override; + ref openStore() const override; }; @@ -100,9 +100,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store { unsupported("getFSAccessor"); } }; -std::shared_ptr DummyStore::Config::openStore() const +ref DummyStore::Config::openStore() const { - return std::make_shared(*this); + return make_ref(*this); } static RegisterStoreImplementation regDummyStore; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 0a663ec036d..accd84899f1 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -207,8 +207,9 @@ class HttpBinaryCacheStore : } }; -std::shared_ptr HttpBinaryCacheStore::Config::openStore() const { - return std::make_shared(*this); +ref HttpBinaryCacheStore::Config::openStore() const +{ + return make_ref(*this); } static RegisterStoreImplementation regHttpBinaryCacheStore; diff --git a/src/libstore/http-binary-cache-store.hh b/src/libstore/http-binary-cache-store.hh index d15c6e3d0db..397bd0592a6 100644 --- a/src/libstore/http-binary-cache-store.hh +++ b/src/libstore/http-binary-cache-store.hh @@ -25,7 +25,7 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig std::string doc() override; - std::shared_ptr openStore() const override; + ref openStore() const override; }; } diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh index 01f7969eb61..647cd83d099 100644 --- a/src/libstore/local-binary-cache-store.hh +++ b/src/libstore/local-binary-cache-store.hh @@ -22,7 +22,7 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig std::string doc() override; - std::shared_ptr openStore() const override; + ref openStore() const override; }; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d3677fdeb39..2a930051d4f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -113,9 +113,9 @@ std::string LocalStoreConfig::doc() ; } -std::shared_ptr LocalStore::Config::openStore() const +ref LocalStore::Config::openStore() const { - return std::make_shared(*this); + return make_ref(*this); } struct LocalStore::State::Stmts { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 45cb04b7159..3d5aa1d3dbd 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -72,7 +72,7 @@ struct LocalStoreConfig : std::string doc() override; - std::shared_ptr openStore() const override; + ref openStore() const override; }; class LocalStore : diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index abd2727194b..6b9a809af96 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -7,6 +7,8 @@ #include "globals.hh" #include "compression.hh" #include "filetransfer.hh" +#include "config-parse-impl.hh" +#include "store-registration.hh" #include #include @@ -185,93 +187,119 @@ S3Helper::FileTransferResult S3Helper::getObject( return res; } - const Setting profile{ - this, - "", - "profile", - R"( - The name of the AWS configuration profile to use. By default - Nix will use the `default` profile. - )"}; - - const Setting region{ - this, - Aws::Region::US_EAST_1, - "region", - R"( - The region of the S3 bucket. If your bucket is not in - `us–east-1`, you should always explicitly specify the region - parameter. - )"}; - - const Setting scheme{ - this, - "", - "scheme", - R"( - The scheme used for S3 requests, `https` (default) or `http`. This - option allows you to disable HTTPS for binary caches which don't - support it. - - > **Note** - > - > HTTPS should be used if the cache might contain sensitive - > information. - )"}; - - const Setting endpoint{ - this, - "", - "endpoint", - R"( - The URL of the endpoint of an S3-compatible service such as MinIO. - Do not specify this setting if you're using Amazon S3. - - > **Note** - > - > This endpoint must support HTTPS and will use path-based - > addressing instead of virtual host based addressing. - )"}; - - const Setting narinfoCompression{ - this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; - - const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; - - const Setting logCompression{ - this, - "", - "log-compression", - R"( - Compression method for `log/*` files. It is recommended to - use a compression method supported by most web browsers - (e.g. `brotli`). - )"}; - - const Setting multipartUpload{this, false, "multipart-upload", "Whether to use multi-part uploads."}; - - const Setting bufferSize{ - this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; - - -S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) - : BinaryCacheStoreConfig(params) - , BinaryCacheStore(params) -{ } +S3BinaryCacheStore::Config::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , BinaryCacheStore::Config::Descriptions{BinaryCacheStore::Config::descriptions} + , S3BinaryCacheStoreConfigT{ + .profile{ + .name = "profile", + .description = R"( + The name of the AWS configuration profile to use. By default + Nix will use the `default` profile. + )", + }, + .region{ + .name = "region", + .description = R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )", + }, + .scheme{ + .name = "scheme", + .description = R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )", + }, + .endpoint{ + .name = "endpoint", + .description = R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and will use path-based + > addressing instead of virtual host based addressing. + )", + }, + .narinfoCompression{ + .name = "narinfo-compression", + .description = "Compression method for `.narinfo` files.", + }, + .lsCompression{ + .name = "ls-compression", + .description = "Compression method for `.ls` files.", + }, + .logCompression{ + .name = "log-compression", + .description = R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )", + }, + .multipartUpload{ + .name = "multipart-upload", + .description = "Whether to use multi-part uploads.", + }, + .bufferSize{ + .name = "buffer-size", + .description = "Size (in bytes) of each part in multi-part uploads.", + }, + } +{} + +const S3BinaryCacheStore::Config::Descriptions S3BinaryCacheStore::Config::descriptions{}; + +decltype(S3BinaryCacheStore::Config::defaults) S3BinaryCacheStore::Config::defaults = { + .profile{""}, + .region{Aws::Region::US_EAST_1}, + .scheme{""}, + .endpoint{""}, + .narinfoCompression{""}, + .lsCompression{""}, + .logCompression{""}, + .multipartUpload{false}, + .bufferSize{5 * 1024 * 1024}, +}; -S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( - std::string_view uriScheme, - std::string_view bucketName, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , bucketName(bucketName) +S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig( + std::string_view scheme, + std::string_view authority, + const StoreReference::Params & params) + : Store::Config(params) + , BinaryCacheStore::Config(params) + , S3BinaryCacheStoreConfigT{ + CONFIG_ROW(profile), + CONFIG_ROW(region), + CONFIG_ROW(scheme), + CONFIG_ROW(endpoint), + CONFIG_ROW(narinfoCompression), + CONFIG_ROW(lsCompression), + CONFIG_ROW(logCompression), + CONFIG_ROW(multipartUpload), + CONFIG_ROW(bufferSize), + } { if (bucketName.empty()) - throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme); + throw UsageError("`%s` store requires a bucket name in its Store URI", scheme); } + +S3BinaryCacheStore::S3BinaryCacheStore(const Config & config) + : BinaryCacheStore(config) +{ } + std::string S3BinaryCacheStoreConfig::doc() { return @@ -282,22 +310,17 @@ std::string S3BinaryCacheStoreConfig::doc() struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore { - std::string bucketName; - Stats stats; S3Helper s3Helper; - S3BinaryCacheStoreImpl( - std::string_view uriScheme, - std::string_view bucketName, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , S3BinaryCacheStoreConfig(uriScheme, bucketName, params) - , Store(params) - , BinaryCacheStore(params) - , S3BinaryCacheStore(params) + S3BinaryCacheStoreImpl(const Config & config) + : Store::Config{config} + , BinaryCacheStore::Config{config} + , S3BinaryCacheStore::Config{config} + , Store{static_cast(*this)} + , BinaryCacheStore{static_cast(*this)} + , S3BinaryCacheStore{static_cast(*this)} , s3Helper(profile, region, scheme, endpoint) { diskCache = getNarInfoDiskCache(); @@ -539,7 +562,12 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual } }; -static RegisterStoreImplementation regS3BinaryCacheStore; +ref S3BinaryCacheStoreImpl::Config::openStore() const +{ + return make_ref(*this); +} + +static RegisterStoreImplementation regS3BinaryCacheStore; } diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index c28e9320378..bff26f6b3d0 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -53,6 +53,8 @@ struct S3BinaryCacheStoreConfig : } std::string doc() override; + + ref openStore() const override; }; struct S3BinaryCacheStore : virtual BinaryCacheStore diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6571fd37c10..56fef0f1a7b 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -154,7 +154,7 @@ struct StoreConfig : * Open a store of the type corresponding to this configuration * type. */ - virtual std::shared_ptr openStore() const = 0; + virtual ref openStore() const = 0; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index f968bbc5b7f..681ac032150 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -1254,29 +1254,46 @@ bool LocalDerivationGoal::isAllowed(const DerivedPath & req) return this->isAllowed(pathPartOfReq(req)); } - struct RestrictedStoreConfig : virtual LocalFSStoreConfig { - using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } -}; + const std::string name() override { return "Restricted Store"; } + + ref openStore() const override; -/* A wrapper around LocalStore that only allows building/querying of - paths that are in the input closures of the build or were added via - recursive Nix calls. */ -struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual IndirectRootStore, public virtual GcStore -{ ref next; LocalDerivationGoal & goal; - RestrictedStore(const Params & params, ref next, LocalDerivationGoal & goal) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RestrictedStoreConfig(params) - , Store(params) - , LocalFSStore(params) - , next(next), goal(goal) + RestrictedStoreConfig( + const StoreReference::Params & params, + ref next, + LocalDerivationGoal & goal) + : Store::Config{params} + , LocalFSStore::Config{params} + , next{next} + , goal{goal} + { + } +}; + +/** + * A wrapper around LocalStore that only allows building/querying of + * paths that are in the input closures of the build or were added via + * recursive Nix calls. + */ +struct RestrictedStore : + virtual RestrictedStoreConfig, + virtual IndirectRootStore, + virtual GcStore +{ + using Config = RestrictedStoreConfig; + + RestrictedStore(const RestrictedStoreConfig & config) + : Store::Config(config) + , LocalFSStore::Config(config) + , RestrictedStore::Config(config) + , Store(static_cast(*this)) + , LocalFSStore(static_cast(*this)) { } Path getRealStoreDir() override @@ -1480,20 +1497,28 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In }; +ref RestrictedStore::Config::openStore() const +{ + return make_ref(*this); +} + + void LocalDerivationGoal::startDaemon() { experimentalFeatureSettings.require(Xp::RecursiveNix); - Store::Params params; + StoreReference::Params params; params["path-info-cache-size"] = "0"; params["store"] = worker.store.storeDir; if (auto & optRoot = getLocalStore().rootDir.get()) params["root"] = *optRoot; params["state"] = "/no-such-path"; params["log"] = "/no-such-path"; - auto store = make_ref(params, + auto store = RestrictedStore::Config{ + params, ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), - *this); + *this, + }.openStore(); addedPaths.clear();