Merge pull request #5048 from tweag/flox-eval-store

--eval-store and faster closure copying
NixOS · Jul 27, 2021 · c000cec · c000cec
2 parents f52fa47 + 29e4913
commit c000cec
Show file tree

Hide file tree

Showing 42 changed files with 472 additions and 203 deletions.
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
@@ -270,7 +270,7 @@ static int main_build_remote(int argc, char * * argv)
 
         {
             Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri));
-            copyPaths(store, ref<Store>(sshStore), store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute);
+            copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute);
         }
 
         uploadLock = -1;
@@ -321,7 +321,7 @@ static int main_build_remote(int argc, char * * argv)
             if (auto localStore = store.dynamic_pointer_cast<LocalStore>())
                 for (auto & path : missingPaths)
                     localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */
-            copyPaths(ref<Store>(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute);
+            copyPaths(*sshStore, *store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute);
         }
         // XXX: Should be done as part of `copyPaths`
         for (auto & realisation : missingRealisations) {

diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc
@@ -54,6 +54,30 @@ void StoreCommand::run()
     run(getStore());
 }
 
+EvalCommand::EvalCommand()
+{
+}
+
+EvalCommand::~EvalCommand()
+{
+    if (evalState)
+        evalState->printStats();
+}
+
+ref<Store> EvalCommand::getEvalStore()
+{
+    if (!evalStore)
+        evalStore = evalStoreUrl ? openStore(*evalStoreUrl) : getStore();
+    return ref<Store>(evalStore);
+}
+
+ref<EvalState> EvalCommand::getEvalState()
+{
+    if (!evalState)
+        evalState = std::make_shared<EvalState>(searchPath, getEvalStore(), getStore());
+    return ref<EvalState>(evalState);
+}
+
 BuiltPathsCommand::BuiltPathsCommand(bool recursive)
     : recursive(recursive)
 {
@@ -91,7 +115,7 @@ void BuiltPathsCommand::run(ref<Store> store)
         for (auto & p : store->queryAllValidPaths())
             paths.push_back(BuiltPath::Opaque{p});
     } else {
-        paths = toBuiltPaths(store, realiseMode, operateOn, installables);
+        paths = toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
         if (recursive) {
             // XXX: This only computes the store path closure, ignoring
             // intermediate realisations

diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh
@@ -45,11 +45,18 @@ private:
 
 struct EvalCommand : virtual StoreCommand, MixEvalArgs
 {
+    EvalCommand();
+
+    ~EvalCommand();
+
+    ref<Store> getEvalStore();
+
     ref<EvalState> getEvalState();
 
-    std::shared_ptr<EvalState> evalState;
+private:
+    std::shared_ptr<Store> evalStore;
 
-    ~EvalCommand();
+    std::shared_ptr<EvalState> evalState;
 };
 
 struct MixFlakeOptions : virtual Args, EvalCommand
@@ -216,22 +223,29 @@ static RegisterCommand registerCommand2(std::vector<std::string> && name)
     return RegisterCommand(std::move(name), [](){ return make_ref<T>(); });
 }
 
-BuiltPaths build(ref<Store> store, Realise mode,
+BuiltPaths build(ref<Store> evalStore, ref<Store> store, Realise mode,
     std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode = bmNormal);
 
-std::set<StorePath> toStorePaths(ref<Store> store,
-    Realise mode, OperateOn operateOn,
+std::set<StorePath> toStorePaths(
+    ref<Store> evalStore,
+    ref<Store> store,
+    Realise mode,
+    OperateOn operateOn,
     std::vector<std::shared_ptr<Installable>> installables);
 
-StorePath toStorePath(ref<Store> store,
-    Realise mode, OperateOn operateOn,
+StorePath toStorePath(
+    ref<Store> evalStore,
+    ref<Store> store,
+    Realise mode,
+    OperateOn operateOn,
     std::shared_ptr<Installable> installable);
 
 std::set<StorePath> toDerivations(ref<Store> store,
     std::vector<std::shared_ptr<Installable>> installables,
     bool useDeriver = false);
 
 BuiltPaths toBuiltPaths(
+    ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
     OperateOn operateOn,

diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
@@ -289,19 +289,6 @@ void completeFlakeRefWithFragment(
     completeFlakeRef(evalState->store, prefix);
 }
 
-ref<EvalState> EvalCommand::getEvalState()
-{
-    if (!evalState)
-        evalState = std::make_shared<EvalState>(searchPath, getStore());
-    return ref<EvalState>(evalState);
-}
-
-EvalCommand::~EvalCommand()
-{
-    if (evalState)
-        evalState->printStats();
-}
-
 void completeFlakeRef(ref<Store> store, std::string_view prefix)
 {
     if (prefix == "")
@@ -391,13 +378,15 @@ DerivedPaths InstallableValue::toDerivedPaths()
     DerivedPaths res;
 
     std::map<StorePath, std::set<std::string>> drvsToOutputs;
+    RealisedPath::Set drvsToCopy;
 
     // Group by derivation, helps with .all in particular
     for (auto & drv : toDerivations()) {
         auto outputName = drv.outputName;
         if (outputName == "")
             throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath));
         drvsToOutputs[drv.drvPath].insert(outputName);
+        drvsToCopy.insert(drv.drvPath);
     }
 
     for (auto & i : drvsToOutputs)
@@ -712,25 +701,24 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
     return installables.front();
 }
 
-BuiltPaths getBuiltPaths(ref<Store> store, DerivedPaths hopefullyBuiltPaths)
+BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPaths & hopefullyBuiltPaths)
 {
     BuiltPaths res;
-    for (auto& b : hopefullyBuiltPaths)
+    for (auto & b : hopefullyBuiltPaths)
         std::visit(
             overloaded{
                 [&](DerivedPath::Opaque bo) {
                     res.push_back(BuiltPath::Opaque{bo.path});
                 },
                 [&](DerivedPath::Built bfd) {
                     OutputPathMap outputs;
-                    auto drv = store->readDerivation(bfd.drvPath);
-                    auto outputHashes = staticOutputHashes(*store, drv);
+                    auto drv = evalStore->readDerivation(bfd.drvPath);
+                    auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
                     auto drvOutputs = drv.outputsAndOptPaths(*store);
-                    for (auto& output : bfd.outputs) {
+                    for (auto & output : bfd.outputs) {
                         if (!outputHashes.count(output))
                             throw Error(
-                                "the derivation '%s' doesn't have an output "
-                                "named '%s'",
+                                "the derivation '%s' doesn't have an output named '%s'",
                                 store->printStorePath(bfd.drvPath), output);
                         if (settings.isExperimentalFeatureEnabled(
                                 "ca-derivations")) {
@@ -762,7 +750,7 @@ BuiltPaths getBuiltPaths(ref<Store> store, DerivedPaths hopefullyBuiltPaths)
     return res;
 }
 
-BuiltPaths build(ref<Store> store, Realise mode,
+BuiltPaths build(ref<Store> evalStore, ref<Store> store, Realise mode,
     std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode)
 {
     if (mode == Realise::Nothing)
@@ -778,20 +766,21 @@ BuiltPaths build(ref<Store> store, Realise mode,
     if (mode == Realise::Nothing)
         printMissing(store, pathsToBuild, lvlError);
     else if (mode == Realise::Outputs)
-        store->buildPaths(pathsToBuild, bMode);
+        store->buildPaths(pathsToBuild, bMode, evalStore);
 
-    return getBuiltPaths(store, pathsToBuild);
+    return getBuiltPaths(evalStore, store, pathsToBuild);
 }
 
 BuiltPaths toBuiltPaths(
+    ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
     OperateOn operateOn,
     std::vector<std::shared_ptr<Installable>> installables)
 {
-    if (operateOn == OperateOn::Output) {
-        return build(store, mode, installables);
-    } else {
+    if (operateOn == OperateOn::Output)
+        return build(evalStore, store, mode, installables);
+    else {
         if (mode == Realise::Nothing)
             settings.readOnlyMode = true;
 
@@ -802,23 +791,27 @@ BuiltPaths toBuiltPaths(
     }
 }
 
-StorePathSet toStorePaths(ref<Store> store,
+StorePathSet toStorePaths(
+    ref<Store> evalStore,
+    ref<Store> store,
     Realise mode, OperateOn operateOn,
     std::vector<std::shared_ptr<Installable>> installables)
 {
     StorePathSet outPaths;
-    for (auto & path : toBuiltPaths(store, mode, operateOn, installables)) {
+    for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) {
         auto thisOutPaths = path.outPaths();
         outPaths.insert(thisOutPaths.begin(), thisOutPaths.end());
     }
     return outPaths;
 }
 
-StorePath toStorePath(ref<Store> store,
+StorePath toStorePath(
+    ref<Store> evalStore,
+    ref<Store> store,
     Realise mode, OperateOn operateOn,
     std::shared_ptr<Installable> installable)
 {
-    auto paths = toStorePaths(store, mode, operateOn, {installable});
+    auto paths = toStorePaths(evalStore, store, mode, operateOn, {installable});
 
     if (paths.size() != 1)
         throw Error("argument '%s' should evaluate to one store path", installable->what());

diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh
@@ -26,7 +26,7 @@ struct App
 struct UnresolvedApp
 {
     App unresolved;
-    App resolve(ref<Store>);
+    App resolve(ref<Store> evalStore, ref<Store> store);
 };
 
 struct Installable

diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc
@@ -61,6 +61,14 @@ MixEvalArgs::MixEvalArgs()
             fetchers::overrideRegistry(from.input, to.input, extraAttrs);
         }}
     });
+
+    addFlag({
+        .longName = "eval-store",
+        .description = "The Nix store to use for evaluations.",
+        .category = category,
+        .labels = {"store-url"},
+        .handler = {&evalStoreUrl},
+    });
 }
 
 Bindings * MixEvalArgs::getAutoArgs(EvalState & state)

diff --git a/src/libexpr/common-eval-args.hh b/src/libexpr/common-eval-args.hh
@@ -16,8 +16,9 @@ struct MixEvalArgs : virtual Args
 
     Strings searchPath;
 
-private:
+    std::optional<std::string> evalStoreUrl;
 
+private:
     std::map<std::string, std::string> autoArgs;
 };
 

diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
@@ -378,7 +378,10 @@ static Strings parseNixPath(const string & s)
 }
 
 
-EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
+EvalState::EvalState(
+    const Strings & _searchPath,
+    ref<Store> store,
+    std::shared_ptr<Store> buildStore)
     : sWith(symbols.create("<with>"))
     , sOutPath(symbols.create("outPath"))
     , sDrvPath(symbols.create("drvPath"))
@@ -411,6 +414,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
     , sEpsilon(symbols.create(""))
     , repair(NoRepair)
     , store(store)
+    , buildStore(buildStore ? buildStore : store)
     , regexCache(makeRegexCache())
     , baseEnv(allocEnv(128))
     , staticBaseEnv(false, 0)

diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
@@ -94,8 +94,12 @@ public:
 
     Value vEmptySet;
 
+    /* Store used to materialise .drv files. */
     const ref<Store> store;
 
+    /* Store used to build stuff. */
+    const ref<Store> buildStore;
+
 
 private:
     SrcToStore srcToStore;
@@ -128,7 +132,10 @@ private:
 
 public:
 
-    EvalState(const Strings & _searchPath, ref<Store> store);
+    EvalState(
+        const Strings & _searchPath,
+        ref<Store> store,
+        std::shared_ptr<Store> buildStore = nullptr);
     ~EvalState();
 
     void addToSearchPath(const string & s);

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
@@ -165,7 +165,7 @@ void DerivationGoal::getDerivation()
     /* The first thing to do is to make sure that the derivation
        exists.  If it doesn't, it may be created through a
        substitute. */
-    if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) {
+    if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) {
         loadDerivation();
         return;
     }
@@ -188,12 +188,12 @@ void DerivationGoal::loadDerivation()
     /* `drvPath' should already be a root, but let's be on the safe
        side: if the user forgot to make it a root, we wouldn't want
        things being garbage collected while we're busy. */
-    worker.store.addTempRoot(drvPath);
+    worker.evalStore.addTempRoot(drvPath);
 
-    assert(worker.store.isValidPath(drvPath));
+    assert(worker.evalStore.isValidPath(drvPath));
 
     /* Get the derivation. */
-    drv = std::make_unique<Derivation>(worker.store.derivationFromPath(drvPath));
+    drv = std::make_unique<Derivation>(worker.evalStore.derivationFromPath(drvPath));
 
     haveDerivation();
 }
@@ -212,8 +212,8 @@ void DerivationGoal::haveDerivation()
         if (i.second.second)
             worker.store.addTempRoot(*i.second.second);
 
-    auto outputHashes = staticOutputHashes(worker.store, *drv);
-    for (auto &[outputName, outputHash] : outputHashes)
+    auto outputHashes = staticOutputHashes(worker.evalStore, *drv);
+    for (auto & [outputName, outputHash] : outputHashes)
       initialOutputs.insert({
             outputName,
             InitialOutput{
@@ -337,6 +337,15 @@ void DerivationGoal::gaveUpOnSubstitution()
         for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
             addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
 
+    /* Copy the input sources from the eval store to the build
+       store. */
+    if (&worker.evalStore != &worker.store) {
+        RealisedPath::Set inputSrcs;
+        for (auto & i : drv->inputSrcs)
+            inputSrcs.insert(i);
+        copyClosure(worker.evalStore, worker.store, inputSrcs);
+    }
+
     for (auto & i : drv->inputSrcs) {
         if (worker.store.isValidPath(i)) continue;
         if (!settings.useSubstitutes)
@@ -478,8 +487,8 @@ void DerivationGoal::inputsRealised()
             /* Add the relevant output closures of the input derivation
                `i' as input paths.  Only add the closures of output paths
                that are specified as inputs. */
-            assert(worker.store.isValidPath(drvPath));
-            auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath);
+            assert(worker.evalStore.isValidPath(drvPath));
+            auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath);
             for (auto & j : wantedDepOutputs) {
                 if (outputs.count(j) > 0) {
                     auto optRealizedInput = outputs.at(j);