From 2f5d3da8062ae58242a8de2bad470a66478edea4 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Fri, 25 Aug 2023 09:53:12 -0400
Subject: [PATCH 1/4] Introduce `OutputName` and `OutputNameView` type aliases
Hopefully they make the code easier to understand!
---
src/libexpr/primops.cc | 2 +-
src/libstore/build/local-derivation-goal.cc | 2 +-
src/libstore/build/local-derivation-goal.hh | 2 +-
src/libstore/derivations.cc | 12 +++++-----
src/libstore/derivations.hh | 12 +++++-----
src/libstore/derived-path.cc | 4 ++--
src/libstore/derived-path.hh | 4 ++--
src/libstore/downstream-placeholder.cc | 4 ++--
src/libstore/downstream-placeholder.hh | 4 ++--
src/libstore/outputs-spec.hh | 26 +++++++++++++++------
src/libstore/realisation.hh | 6 ++---
11 files changed, 45 insertions(+), 33 deletions(-)
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 5067da449b6..6b99b91e4ef 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -1843,7 +1843,7 @@ static void prim_outputOf(EvalState & state, const PosIdx pos, Value * * args, V
{
SingleDerivedPath drvPath = state.coerceToSingleDerivedPath(pos, *args[0], "while evaluating the first argument to builtins.outputOf");
- std::string_view outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf");
+ OutputNameView outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf");
state.mkSingleDerivedPathString(
SingleDerivedPath::Built {
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 78f943d1fd8..64b55ca6ac2 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -2955,7 +2955,7 @@ bool LocalDerivationGoal::isReadDesc(int fd)
}
-StorePath LocalDerivationGoal::makeFallbackPath(std::string_view outputName)
+StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName)
{
return worker.store.makeStorePath(
"rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh
index 8827bfca3b2..0a05081c78a 100644
--- a/src/libstore/build/local-derivation-goal.hh
+++ b/src/libstore/build/local-derivation-goal.hh
@@ -297,7 +297,7 @@ struct LocalDerivationGoal : public DerivationGoal
* @todo Add option to randomize, so we can audit whether our
* rewrites caught everything
*/
- StorePath makeFallbackPath(std::string_view outputName);
+ StorePath makeFallbackPath(OutputNameView outputName);
};
}
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 0b8bdaf1c1e..dc32c3847de 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -12,7 +12,7 @@
namespace nix {
-std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const
+std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, OutputNameView outputName) const
{
return std::visit(overloaded {
[](const DerivationOutput::InputAddressed & doi) -> std::optional<StorePath> {
@@ -36,7 +36,7 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
}
-StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
+StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, OutputNameView outputName) const
{
return store.makeFixedOutputPathFromCA(
outputPathName(drvName, outputName),
@@ -466,7 +466,7 @@ bool isDerivation(std::string_view fileName)
}
-std::string outputPathName(std::string_view drvName, std::string_view outputName) {
+std::string outputPathName(std::string_view drvName, OutputNameView outputName) {
std::string res { drvName };
if (outputName != "out") {
res += "-";
@@ -810,7 +810,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
}
-std::string hashPlaceholder(const std::string_view outputName)
+std::string hashPlaceholder(const OutputNameView outputName)
{
// FIXME: memoize?
return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false);
@@ -963,7 +963,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
const Hash impureOutputHash = hashString(htSHA256, "impure");
nlohmann::json DerivationOutput::toJSON(
- const Store & store, std::string_view drvName, std::string_view outputName) const
+ const Store & store, std::string_view drvName, OutputNameView outputName) const
{
nlohmann::json res = nlohmann::json::object();
std::visit(overloaded {
@@ -990,7 +990,7 @@ nlohmann::json DerivationOutput::toJSON(
DerivationOutput DerivationOutput::fromJSON(
- const Store & store, std::string_view drvName, std::string_view outputName,
+ const Store & store, std::string_view drvName, OutputNameView outputName,
const nlohmann::json & _json,
const ExperimentalFeatureSettings & xpSettings)
{
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index a92082089dd..106056f2dde 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -55,7 +55,7 @@ struct DerivationOutput
* @param drvName The name of the derivation this is an output of, without the `.drv`.
* @param outputName The name of this output.
*/
- StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
+ StorePath path(const Store & store, std::string_view drvName, OutputNameView outputName) const;
GENERATE_CMP(CAFixed, me->ca);
};
@@ -132,19 +132,19 @@ struct DerivationOutput
* the safer interface provided by
* BasicDerivation::outputsAndOptPaths
*/
- std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const;
+ std::optional<StorePath> path(const Store & store, std::string_view drvName, OutputNameView outputName) const;
nlohmann::json toJSON(
const Store & store,
std::string_view drvName,
- std::string_view outputName) const;
+ OutputNameView outputName) const;
/**
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static DerivationOutput fromJSON(
const Store & store,
std::string_view drvName,
- std::string_view outputName,
+ OutputNameView outputName,
const nlohmann::json & json,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
};
@@ -405,7 +405,7 @@ bool isDerivation(std::string_view fileName);
* This is usually <drv-name>-<output-name>, but is just <drv-name> when
* the output name is "out".
*/
-std::string outputPathName(std::string_view drvName, std::string_view outputName);
+std::string outputPathName(std::string_view drvName, OutputNameView outputName);
/**
@@ -499,7 +499,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
* own outputs without needing to use the hash of a derivation in
* itself, making the hash near-impossible to calculate.
*/
-std::string hashPlaceholder(const std::string_view outputName);
+std::string hashPlaceholder(const OutputNameView outputName);
extern const Hash impureOutputHash;
diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc
index 3594b757053..47d784deb37 100644
--- a/src/libstore/derived-path.cc
+++ b/src/libstore/derived-path.cc
@@ -167,7 +167,7 @@ void drvRequireExperiment(
SingleDerivedPath::Built SingleDerivedPath::Built::parse(
const Store & store, ref<SingleDerivedPath> drv,
- std::string_view output,
+ OutputNameView output,
const ExperimentalFeatureSettings & xpSettings)
{
drvRequireExperiment(*drv, xpSettings);
@@ -179,7 +179,7 @@ SingleDerivedPath::Built SingleDerivedPath::Built::parse(
DerivedPath::Built DerivedPath::Built::parse(
const Store & store, ref<SingleDerivedPath> drv,
- std::string_view outputsS,
+ OutputNameView outputsS,
const ExperimentalFeatureSettings & xpSettings)
{
drvRequireExperiment(*drv, xpSettings);
diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh
index ec30dac611d..4d7033df2f1 100644
--- a/src/libstore/derived-path.hh
+++ b/src/libstore/derived-path.hh
@@ -42,7 +42,7 @@ struct SingleDerivedPath;
*/
struct SingleDerivedPathBuilt {
ref<SingleDerivedPath> drvPath;
- std::string output;
+ OutputName output;
/**
* Get the store path this is ultimately derived from (by realising
@@ -71,7 +71,7 @@ struct SingleDerivedPathBuilt {
*/
static SingleDerivedPathBuilt parse(
const Store & store, ref<SingleDerivedPath> drvPath,
- std::string_view outputs,
+ OutputNameView outputs,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
nlohmann::json toJSON(Store & store) const;
diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc
index d951b7b7de5..7e3f7548d99 100644
--- a/src/libstore/downstream-placeholder.cc
+++ b/src/libstore/downstream-placeholder.cc
@@ -11,7 +11,7 @@ std::string DownstreamPlaceholder::render() const
DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput(
const StorePath & drvPath,
- std::string_view outputName,
+ OutputNameView outputName,
const ExperimentalFeatureSettings & xpSettings)
{
xpSettings.require(Xp::CaDerivations);
@@ -25,7 +25,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput(
DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
const DownstreamPlaceholder & placeholder,
- std::string_view outputName,
+ OutputNameView outputName,
const ExperimentalFeatureSettings & xpSettings)
{
xpSettings.require(Xp::DynamicDerivations);
diff --git a/src/libstore/downstream-placeholder.hh b/src/libstore/downstream-placeholder.hh
index d58a2ac1442..c911ecea2ed 100644
--- a/src/libstore/downstream-placeholder.hh
+++ b/src/libstore/downstream-placeholder.hh
@@ -58,7 +58,7 @@ public:
*/
static DownstreamPlaceholder unknownCaOutput(
const StorePath & drvPath,
- std::string_view outputName,
+ OutputNameView outputName,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
/**
@@ -72,7 +72,7 @@ public:
*/
static DownstreamPlaceholder unknownDerivation(
const DownstreamPlaceholder & drvPlaceholder,
- std::string_view outputName,
+ OutputNameView outputName,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
/**
diff --git a/src/libstore/outputs-spec.hh b/src/libstore/outputs-spec.hh
index ae19f1040be..1ef99a5fc67 100644
--- a/src/libstore/outputs-spec.hh
+++ b/src/libstore/outputs-spec.hh
@@ -13,24 +13,36 @@
namespace nix {
+/**
+ * An (owned) output name. Just a type alias used to make code more
+ * readible.
+ */
+typedef std::string OutputName;
+
+/**
+ * A borrowed output name. Just a type alias used to make code more
+ * readible.
+ */
+typedef std::string_view OutputNameView;
+
struct OutputsSpec {
/**
* A non-empty set of outputs, specified by name
*/
- struct Names : std::set<std::string> {
- using std::set<std::string>::set;
+ struct Names : std::set<OutputName> {
+ using std::set<OutputName>::set;
/* These need to be "inherited manually" */
- Names(const std::set<std::string> & s)
- : std::set<std::string>(s)
+ Names(const std::set<OutputName> & s)
+ : std::set<OutputName>(s)
{ assert(!empty()); }
/**
* Needs to be "inherited manually"
*/
- Names(std::set<std::string> && s)
- : std::set<std::string>(s)
+ Names(std::set<OutputName> && s)
+ : std::set<OutputName>(s)
{ assert(!empty()); }
/* This set should always be non-empty, so we delete this
@@ -57,7 +69,7 @@ struct OutputsSpec {
*/
OutputsSpec() = delete;
- bool contains(const std::string & output) const;
+ bool contains(const OutputName & output) const;
/**
* Create a new OutputsSpec which is the union of this and that.
diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh
index 0548b30c16d..559483ce3b4 100644
--- a/src/libstore/realisation.hh
+++ b/src/libstore/realisation.hh
@@ -34,7 +34,7 @@ struct DrvOutput {
/**
* The name of the output.
*/
- std::string outputName;
+ OutputName outputName;
std::string to_string() const;
@@ -84,7 +84,7 @@ struct Realisation {
* Since these are the outputs of a single derivation, we know the
* output names are unique so we can use them as the map key.
*/
-typedef std::map<std::string, Realisation> SingleDrvOutputs;
+typedef std::map<OutputName, Realisation> SingleDrvOutputs;
/**
* Collection type for multiple derivations' outputs' `Realisation`s.
@@ -146,7 +146,7 @@ public:
MissingRealisation(DrvOutput & outputId)
: MissingRealisation(outputId.outputName, outputId.strHash())
{}
- MissingRealisation(std::string_view drv, std::string outputName)
+ MissingRealisation(std::string_view drv, OutputName outputName)
: Error( "cannot operate on output '%s' of the "
"unbuilt derivation '%s'",
outputName,
From 1c4caef14b51dbb4f749c45311c8e2c9acb75a60 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Wed, 16 Aug 2023 00:05:35 -0400
Subject: [PATCH 2/4] Throw `MissingRealisation` not plain `Error` in both
`resolveDerivedPath`
Now we are consistent with the other `resolveDerivedPath`, and other
such functions.
---
src/libstore/misc.cc | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 63121330699..c043b9b9305 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -399,8 +399,7 @@ StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store
store.printStorePath(drvPath), bfd.output);
auto & optPath = outputPaths.at(bfd.output);
if (!optPath)
- throw Error("'%s' does not yet map to a known concrete store path",
- bfd.to_string(store));
+ throw MissingRealisation(bfd.drvPath->to_string(store), bfd.output);
return *optPath;
},
}, req.raw());
From 692074f7142fcf8ede1266b6d8cbbd5feaf3221f Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Sun, 15 Jan 2023 17:47:24 -0500
Subject: [PATCH 3/4] Use `Worker::makeDerivationGoal` less
We're about to split up `DerivationGoal` a bit. At that point
`makeDerivationGoal` will mean something more specific than it does
today. (Perhaps a future rename will make this clearer.)
On the other hand, the more public `Worker::makeGoal` function will
continue to work exactly as before. So by moving some call sites to use
that instead, we preemptively avoid issues in the next step.
---
src/libstore/build/derivation-goal.cc | 14 ++++++++++++--
src/libstore/build/entry-points.cc | 7 +++++--
2 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 84da7f2e117..ea56c028829 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -380,7 +380,12 @@ void DerivationGoal::gaveUpOnSubstitution()
worker.store.printStorePath(i.first));
}
- addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
+ addWaitee(worker.makeGoal(
+ DerivedPath::Built {
+ .drvPath = makeConstantStorePathRef(i.first),
+ .outputs = i.second,
+ },
+ buildMode == bmRepair ? bmRepair : bmNormal));
}
/* Copy the input sources from the eval store to the build
@@ -452,7 +457,12 @@ void DerivationGoal::repairClosure()
if (drvPath2 == outputsToDrv.end())
addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
else
- addWaitee(worker.makeDerivationGoal(drvPath2->second, OutputsSpec::All(), bmRepair));
+ addWaitee(worker.makeGoal(
+ DerivedPath::Built {
+ .drvPath = makeConstantStorePathRef(drvPath2->second),
+ .outputs = OutputsSpec::All { },
+ },
+ bmRepair));
}
if (waitees.empty()) {
diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index e941b4e6533..f71fb35a611 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -124,8 +124,11 @@ void Store::repairPath(const StorePath & path)
auto info = queryPathInfo(path);
if (info->deriver && isValidPath(*info->deriver)) {
goals.clear();
- // FIXME: Should just build the specific output we need.
- goals.insert(worker.makeDerivationGoal(*info->deriver, OutputsSpec::All { }, bmRepair));
+ goals.insert(worker.makeGoal(DerivedPath::Built {
+ .drvPath = makeConstantStorePathRef(*info->deriver),
+ // FIXME: Should just build the specific output we need.
+ .outputs = OutputsSpec::All { },
+ }, bmRepair));
worker.run(goals);
} else
throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path));
From 5e3986f59cb58f48186a49dcec7aa317b4787522 Mon Sep 17 00:00:00 2001
From: John Ericson <John.Ericson@Obsidian.Systems>
Date: Mon, 8 Mar 2021 16:24:49 -0500
Subject: [PATCH 4/4] Adapt scheduler to work with dynamic derivations
To avoid dealing with an optional `drvPath` (because we might not know
it yet) everywhere, make an `CreateDerivationAndRealiseGoal`. This goal
just builds/substitutes the derivation file, and then kicks of a build
for that obtained derivation; in other words it does the chaining of
goals when the drv file is missing (as can already be the case) or
computed (new case).
This also means the `getDerivation` state can be removed from
`DerivationGoal`, which makes the `BasicDerivation` / in memory case and
`Derivation` / drv file file case closer together.
The map type is factored out for clarity, and because we will soon hvae
a second use for it (`Derivation` itself).
Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
.../create-derivation-and-realise-goal.cc | 157 ++++++++++++++++++
.../create-derivation-and-realise-goal.hh | 96 +++++++++++
src/libstore/build/derivation-goal.cc | 22 +--
src/libstore/build/derivation-goal.hh | 15 +-
.../build/drv-output-substitution-goal.hh | 4 +-
src/libstore/build/entry-points.cc | 11 +-
src/libstore/build/goal.cc | 2 +-
src/libstore/build/goal.hh | 22 ++-
src/libstore/build/substitution-goal.hh | 4 +-
src/libstore/build/worker.cc | 114 ++++++++++---
src/libstore/build/worker.hh | 22 +++
src/libstore/derived-path-map.cc | 33 ++++
src/libstore/derived-path-map.hh | 73 ++++++++
tests/dyn-drv/build-built-drv.sh | 4 +-
14 files changed, 523 insertions(+), 56 deletions(-)
create mode 100644 src/libstore/build/create-derivation-and-realise-goal.cc
create mode 100644 src/libstore/build/create-derivation-and-realise-goal.hh
create mode 100644 src/libstore/derived-path-map.cc
create mode 100644 src/libstore/derived-path-map.hh
diff --git a/src/libstore/build/create-derivation-and-realise-goal.cc b/src/libstore/build/create-derivation-and-realise-goal.cc
new file mode 100644
index 00000000000..b01042f0032
--- /dev/null
+++ b/src/libstore/build/create-derivation-and-realise-goal.cc
@@ -0,0 +1,157 @@
+#include "create-derivation-and-realise-goal.hh"
+#include "worker.hh"
+
+namespace nix {
+
+CreateDerivationAndRealiseGoal::CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
+ const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
+ : Goal(worker, DerivedPath::Built { .drvPath = drvReq, .outputs = wantedOutputs })
+ , drvReq(drvReq)
+ , wantedOutputs(wantedOutputs)
+ , buildMode(buildMode)
+{
+ state = &CreateDerivationAndRealiseGoal::getDerivation;
+ name = fmt(
+ "outer obtaining drv from '%s' and then building outputs %s",
+ drvReq->to_string(worker.store),
+ std::visit(overloaded {
+ [&](const OutputsSpec::All) -> std::string {
+ return "* (all of them)";
+ },
+ [&](const OutputsSpec::Names os) {
+ return concatStringsSep(", ", quoteStrings(os));
+ },
+ }, wantedOutputs.raw));
+ trace("created outer");
+
+ worker.updateProgress();
+}
+
+
+CreateDerivationAndRealiseGoal::~CreateDerivationAndRealiseGoal()
+{
+}
+
+
+static StorePath pathPartOfReq(const SingleDerivedPath & req)
+{
+ return std::visit(overloaded {
+ [&](const SingleDerivedPath::Opaque & bo) {
+ return bo.path;
+ },
+ [&](const SingleDerivedPath::Built & bfd) {
+ return pathPartOfReq(*bfd.drvPath);
+ },
+ }, req.raw());
+}
+
+
+std::string CreateDerivationAndRealiseGoal::key()
+{
+ /* Ensure that derivations get built in order of their name,
+ i.e. a derivation named "aardvark" always comes before "baboon". And
+ substitution goals and inner derivation goals always happen before
+ derivation goals (due to "b$"). */
+ return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store);
+}
+
+
+void CreateDerivationAndRealiseGoal::timedOut(Error && ex)
+{
+}
+
+
+void CreateDerivationAndRealiseGoal::work()
+{
+ (this->*state)();
+}
+
+
+void CreateDerivationAndRealiseGoal::addWantedOutputs(const OutputsSpec & outputs)
+{
+ /* If we already want all outputs, there is nothing to do. */
+ auto newWanted = wantedOutputs.union_(outputs);
+ bool needRestart = !newWanted.isSubsetOf(wantedOutputs);
+ wantedOutputs = newWanted;
+
+ if (!needRestart) return;
+
+ if (!optDrvPath)
+ // haven't started steps where the outputs matter yet
+ return;
+ worker.makeDerivationGoal(*optDrvPath, outputs, buildMode);
+}
+
+
+void CreateDerivationAndRealiseGoal::getDerivation()
+{
+ trace("outer init");
+
+ /* The first thing to do is to make sure that the derivation
+ exists. If it doesn't, it may be created through a
+ substitute. */
+ if (auto optDrvPath = [this]() -> std::optional<StorePath> {
+ if (buildMode != bmNormal) return std::nullopt;
+
+ auto drvPath = StorePath::dummy;
+ try {
+ drvPath = resolveDerivedPath(worker.store, *drvReq);
+ } catch (MissingRealisation) {
+ return std::nullopt;
+ }
+ return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath)
+ ? std::optional { drvPath }
+ : std::nullopt;
+ }()) {
+ trace(fmt("already have drv '%s' for '%s', can go straight to building",
+ worker.store.printStorePath(*optDrvPath),
+ drvReq->to_string(worker.store)));
+
+ loadAndBuildDerivation();
+ } else {
+ trace("need to obtain drv we want to build");
+
+ addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq)));
+
+ state = &CreateDerivationAndRealiseGoal::loadAndBuildDerivation;
+ if (waitees.empty()) work();
+ }
+}
+
+
+void CreateDerivationAndRealiseGoal::loadAndBuildDerivation()
+{
+ trace("outer load and build derivation");
+
+ if (nrFailed != 0) {
+ amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store)));
+ return;
+ }
+
+ StorePath drvPath = resolveDerivedPath(worker.store, *drvReq);
+ /* Build this step! */
+ concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode);
+ addWaitee(upcast_goal(concreteDrvGoal));
+ state = &CreateDerivationAndRealiseGoal::buildDone;
+ optDrvPath = std::move(drvPath);
+ if (waitees.empty()) work();
+}
+
+
+void CreateDerivationAndRealiseGoal::buildDone()
+{
+ trace("outer build done");
+
+ buildResult = upcast_goal(concreteDrvGoal)->getBuildResult(DerivedPath::Built {
+ .drvPath = drvReq,
+ .outputs = wantedOutputs,
+ });
+
+ if (buildResult.success())
+ amDone(ecSuccess);
+ else
+ amDone(ecFailed, Error("building '%s' failed", drvReq->to_string(worker.store)));
+}
+
+
+}
diff --git a/src/libstore/build/create-derivation-and-realise-goal.hh b/src/libstore/build/create-derivation-and-realise-goal.hh
new file mode 100644
index 00000000000..ca936fc95de
--- /dev/null
+++ b/src/libstore/build/create-derivation-and-realise-goal.hh
@@ -0,0 +1,96 @@
+#pragma once
+
+#include "parsed-derivations.hh"
+#include "lock.hh"
+#include "store-api.hh"
+#include "pathlocks.hh"
+#include "goal.hh"
+
+namespace nix {
+
+struct DerivationGoal;
+
+/**
+ * This goal type is essentially the serial composition (like function
+ * composition) of a goal for getting a derivation, and then a
+ * `DerivationGoal` using the newly-obtained derivation.
+ *
+ * In the (currently experimental) general inductive case of derivations
+ * that are themselves build outputs, that first goal will be *another*
+ * `CreateDerivationAndRealiseGoal`. In the (much more common) base-case
+ * where the derivation has no provence and is just referred to by
+ * (content-addressed) store path, that first goal is a
+ * `SubstitutionGoal`.
+ *
+ * If we already have the derivation (e.g. if the evalutator has created
+ * the derivation locally and then instructured the store to build it),
+ * we can skip the first goal entirely as a small optimization.
+ */
+struct CreateDerivationAndRealiseGoal : public Goal
+{
+ /**
+ * How to obtain a store path of the derivation to build.
+ */
+ ref<SingleDerivedPath> drvReq;
+
+ /**
+ * The path of the derivation, once obtained.
+ **/
+ std::optional<StorePath> optDrvPath;
+
+ /**
+ * The goal for the corresponding concrete derivation.
+ **/
+ std::shared_ptr<DerivationGoal> concreteDrvGoal;
+
+ /**
+ * The specific outputs that we need to build.
+ */
+ OutputsSpec wantedOutputs;
+
+ typedef void (CreateDerivationAndRealiseGoal::*GoalState)();
+ GoalState state;
+
+ /**
+ * The final output paths of the build.
+ *
+ * - For input-addressed derivations, always the precomputed paths
+ *
+ * - For content-addressed derivations, calcuated from whatever the
+ * hash ends up being. (Note that fixed outputs derivations that
+ * produce the "wrong" output still install that data under its
+ * true content-address.)
+ */
+ OutputPathMap finalOutputs;
+
+ BuildMode buildMode;
+
+ CreateDerivationAndRealiseGoal(ref<SingleDerivedPath> drvReq,
+ const OutputsSpec & wantedOutputs, Worker & worker,
+ BuildMode buildMode = bmNormal);
+ virtual ~CreateDerivationAndRealiseGoal();
+
+ void timedOut(Error && ex) override;
+
+ std::string key() override;
+
+ void work() override;
+
+ /**
+ * Add wanted outputs to an already existing derivation goal.
+ */
+ void addWantedOutputs(const OutputsSpec & outputs);
+
+ /**
+ * The states.
+ */
+ void getDerivation();
+ void loadAndBuildDerivation();
+ void buildDone();
+
+ JobCategory jobCategory() const override {
+ return JobCategory::Administration;
+ };
+};
+
+}
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index ea56c028829..bec0bc538ab 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -71,7 +71,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
, wantedOutputs(wantedOutputs)
, buildMode(buildMode)
{
- state = &DerivationGoal::getDerivation;
+ state = &DerivationGoal::loadDerivation;
name = fmt(
"building of '%s' from .drv file",
DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store));
@@ -164,24 +164,6 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs)
}
-void DerivationGoal::getDerivation()
-{
- trace("init");
-
- /* The first thing to do is to make sure that the derivation
- exists. If it doesn't, it may be created through a
- substitute. */
- if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) {
- loadDerivation();
- return;
- }
-
- addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
-
- state = &DerivationGoal::loadDerivation;
-}
-
-
void DerivationGoal::loadDerivation()
{
trace("loading derivation");
@@ -1493,7 +1475,7 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
if (!useDerivation) return;
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
- auto * dg = dynamic_cast<DerivationGoal *>(&*waitee);
+ auto * dg = tryGetConcreteDrvGoal(waitee);
if (!dg) return;
auto outputs = fullDrv.inputDrvs.find(dg->drvPath);
diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh
index 9d6fe1c0fa7..62b122c27e6 100644
--- a/src/libstore/build/derivation-goal.hh
+++ b/src/libstore/build/derivation-goal.hh
@@ -50,6 +50,13 @@ struct InitialOutput {
std::optional<InitialOutputStatus> known;
};
+/**
+ * A goal for building some or all of the outputs of a derivation.
+ *
+ * The derivation must already be present, either in the store in a drv
+ * or in memory. If the derivation itself needs to be gotten first, a
+ * `CreateDerivationAndRealiseGoal` goal must be used instead.
+ */
struct DerivationGoal : public Goal
{
/**
@@ -66,8 +73,7 @@ struct DerivationGoal : public Goal
std::shared_ptr<DerivationGoal> resolvedDrvGoal;
/**
- * The specific outputs that we need to build. Empty means all of
- * them.
+ * The specific outputs that we need to build.
*/
OutputsSpec wantedOutputs;
@@ -229,7 +235,6 @@ struct DerivationGoal : public Goal
/**
* The states.
*/
- void getDerivation();
void loadDerivation();
void haveDerivation();
void outputsSubstitutionTried();
@@ -334,7 +339,9 @@ struct DerivationGoal : public Goal
StorePathSet exportReferences(const StorePathSet & storePaths);
- JobCategory jobCategory() override { return JobCategory::Build; };
+ JobCategory jobCategory() const override {
+ return JobCategory::Build;
+ };
};
MakeError(NotDeterministic, BuildError);
diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh
index 5d1253a71ef..da2426e5e86 100644
--- a/src/libstore/build/drv-output-substitution-goal.hh
+++ b/src/libstore/build/drv-output-substitution-goal.hh
@@ -73,7 +73,9 @@ public:
void work() override;
void handleEOF(int fd) override;
- JobCategory jobCategory() override { return JobCategory::Substitution; };
+ JobCategory jobCategory() const override {
+ return JobCategory::Substitution;
+ };
};
}
diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index f71fb35a611..f0f0e551935 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -1,5 +1,6 @@
#include "worker.hh"
#include "substitution-goal.hh"
+#include "create-derivation-and-realise-goal.hh"
#include "derivation-goal.hh"
#include "local-store.hh"
@@ -15,7 +16,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
worker.run(goals);
- StorePathSet failed;
+ StringSet failed;
std::optional<Error> ex;
for (auto & i : goals) {
if (i->ex) {
@@ -25,8 +26,10 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
ex = std::move(i->ex);
}
if (i->exitCode != Goal::ecSuccess) {
- if (auto i2 = dynamic_cast<DerivationGoal *>(i.get())) failed.insert(i2->drvPath);
- else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get())) failed.insert(i2->storePath);
+ if (auto i2 = dynamic_cast<CreateDerivationAndRealiseGoal *>(i.get()))
+ failed.insert(i2->drvReq->to_string(*this));
+ else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get()))
+ failed.insert(printStorePath(i2->storePath));
}
}
@@ -35,7 +38,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
throw std::move(*ex);
} else if (!failed.empty()) {
if (ex) logError(ex->info());
- throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed));
+ throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed)));
}
}
diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc
index ca7097a68df..f8db9828076 100644
--- a/src/libstore/build/goal.cc
+++ b/src/libstore/build/goal.cc
@@ -11,7 +11,7 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
}
-BuildResult Goal::getBuildResult(const DerivedPath & req) {
+BuildResult Goal::getBuildResult(const DerivedPath & req) const {
BuildResult res { buildResult };
if (auto pbp = std::get_if<DerivedPath::Built>(&req)) {
diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh
index d3127caeaa5..01d3c3491a5 100644
--- a/src/libstore/build/goal.hh
+++ b/src/libstore/build/goal.hh
@@ -41,8 +41,24 @@ typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
* of each category in parallel.
*/
enum struct JobCategory {
+ /**
+ * A build of a derivation; it will use CPU and disk resources.
+ */
Build,
+ /**
+ * A substitution an arbitrary store object; it will use network resources.
+ */
Substitution,
+ /**
+ * A goal that does no "real" work by itself, and just exists to depend on
+ * other goals which *do* do real work. These goals therefore are not
+ * limited.
+ *
+ * These goals cannot infinitely create themselves, so there is no risk of
+ * a "fork bomb" type situation (which would be a problem even though the
+ * goal do no real work) either.
+ */
+ Administration,
};
struct Goal : public std::enable_shared_from_this<Goal>
@@ -110,7 +126,7 @@ public:
* sake of both privacy and determinism, and this "safe accessor"
* ensures we don't.
*/
- BuildResult getBuildResult(const DerivedPath &);
+ BuildResult getBuildResult(const DerivedPath &) const;
/**
* Exception containing an error message, if any.
@@ -144,7 +160,7 @@ public:
void trace(std::string_view s);
- std::string getName()
+ std::string getName() const
{
return name;
}
@@ -166,7 +182,7 @@ public:
* @brief Hint for the scheduler, which concurrency limit applies.
* @see JobCategory
*/
- virtual JobCategory jobCategory() = 0;
+ virtual JobCategory jobCategory() const = 0;
};
void addToWeakGoals(WeakGoals & goals, GoalPtr p);
diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh
index 1b693baa1b3..1d389d328ff 100644
--- a/src/libstore/build/substitution-goal.hh
+++ b/src/libstore/build/substitution-goal.hh
@@ -117,7 +117,9 @@ public:
/* Called by destructor, can't be overridden */
void cleanup() override final;
- JobCategory jobCategory() override { return JobCategory::Substitution; };
+ JobCategory jobCategory() const override {
+ return JobCategory::Substitution;
+ };
};
}
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index b58fc5c1c0c..f65f63b9947 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -2,6 +2,7 @@
#include "worker.hh"
#include "substitution-goal.hh"
#include "drv-output-substitution-goal.hh"
+#include "create-derivation-and-realise-goal.hh"
#include "local-derivation-goal.hh"
#include "hook-instance.hh"
@@ -41,6 +42,24 @@ Worker::~Worker()
}
+std::shared_ptr<CreateDerivationAndRealiseGoal> Worker::makeCreateDerivationAndRealiseGoal(
+ ref<SingleDerivedPath> drvReq,
+ const OutputsSpec & wantedOutputs,
+ BuildMode buildMode)
+{
+ std::weak_ptr<CreateDerivationAndRealiseGoal> & goal_weak = outerDerivationGoals.ensureSlot(*drvReq).value;
+ std::shared_ptr<CreateDerivationAndRealiseGoal> goal = goal_weak.lock();
+ if (!goal) {
+ goal = std::make_shared<CreateDerivationAndRealiseGoal>(drvReq, wantedOutputs, *this, buildMode);
+ goal_weak = goal;
+ wakeUp(goal);
+ } else {
+ goal->addWantedOutputs(wantedOutputs);
+ }
+ return goal;
+}
+
+
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
const StorePath & drvPath,
const OutputsSpec & wantedOutputs,
@@ -111,10 +130,7 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
{
return std::visit(overloaded {
[&](const DerivedPath::Built & bfd) -> GoalPtr {
- if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath))
- return makeDerivationGoal(bop->path, bfd.outputs, buildMode);
- else
- throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented.");
+ return makeCreateDerivationAndRealiseGoal(bfd.drvPath, bfd.outputs, buildMode);
},
[&](const DerivedPath::Opaque & bo) -> GoalPtr {
return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
@@ -123,24 +139,46 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
}
+template<typename K, typename V, typename F>
+static void cullMap(std::map<K, V> & goalMap, F f)
+{
+ for (auto i = goalMap.begin(); i != goalMap.end();)
+ if (!f(i->second))
+ i = goalMap.erase(i);
+ else ++i;
+}
+
+
template<typename K, typename G>
static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
{
/* !!! inefficient */
- for (auto i = goalMap.begin();
- i != goalMap.end(); )
- if (i->second.lock() == goal) {
- auto j = i; ++j;
- goalMap.erase(i);
- i = j;
- }
- else ++i;
+ cullMap(goalMap, [&](const std::weak_ptr<G> & gp) -> bool {
+ return gp.lock() != goal;
+ });
+}
+
+template<typename K>
+static void removeGoal(std::shared_ptr<CreateDerivationAndRealiseGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode> & goalMap);
+
+template<typename K>
+static void removeGoal(std::shared_ptr<CreateDerivationAndRealiseGoal> goal, std::map<K, DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode> & goalMap)
+{
+ /* !!! inefficient */
+ cullMap(goalMap, [&](DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>::ChildNode & node) -> bool {
+ if (node.value.lock() == goal)
+ node.value.reset();
+ removeGoal(goal, node.childMap);
+ return !node.value.expired() || !node.childMap.empty();
+ });
}
void Worker::removeGoal(GoalPtr goal)
{
- if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
+ if (auto drvGoal = std::dynamic_pointer_cast<CreateDerivationAndRealiseGoal>(goal))
+ nix::removeGoal(drvGoal, outerDerivationGoals.map);
+ else if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
nix::removeGoal(drvGoal, derivationGoals);
else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
nix::removeGoal(subGoal, substitutionGoals);
@@ -198,8 +236,19 @@ void Worker::childStarted(GoalPtr goal, const std::set<int> & fds,
child.respectTimeouts = respectTimeouts;
children.emplace_back(child);
if (inBuildSlot) {
- if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++;
- else nrLocalBuilds++;
+ switch (goal->jobCategory()) {
+ case JobCategory::Substitution:
+ nrSubstitutions++;
+ break;
+ case JobCategory::Build:
+ nrLocalBuilds++;
+ break;
+ case JobCategory::Administration:
+ /* Intentionally not limited, see docs */
+ break;
+ default:
+ abort();
+ }
}
}
@@ -211,12 +260,20 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers)
if (i == children.end()) return;
if (i->inBuildSlot) {
- if (goal->jobCategory() == JobCategory::Substitution) {
+ switch (goal->jobCategory()) {
+ case JobCategory::Substitution:
assert(nrSubstitutions > 0);
nrSubstitutions--;
- } else {
+ break;
+ case JobCategory::Build:
assert(nrLocalBuilds > 0);
nrLocalBuilds--;
+ break;
+ case JobCategory::Administration:
+ /* Intentionally not limited, see docs */
+ break;
+ default:
+ abort();
}
}
@@ -267,9 +324,9 @@ void Worker::run(const Goals & _topGoals)
for (auto & i : _topGoals) {
topGoals.insert(i);
- if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
+ if (auto goal = dynamic_cast<CreateDerivationAndRealiseGoal *>(i.get())) {
topPaths.push_back(DerivedPath::Built {
- .drvPath = makeConstantStorePathRef(goal->drvPath),
+ .drvPath = goal->drvReq,
.outputs = goal->wantedOutputs,
});
} else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
@@ -522,11 +579,26 @@ void Worker::markContentsGood(const StorePath & path)
}
-GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal) {
+GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal)
+{
+ return subGoal;
+}
+
+GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal)
+{
return subGoal;
}
-GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal) {
+
+GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal)
+{
return subGoal;
}
+const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee)
+{
+ auto * odg = dynamic_cast<CreateDerivationAndRealiseGoal *>(&*waitee);
+ if (!odg) return nullptr;
+ return &*odg->concreteDrvGoal;
+}
+
}
diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh
index 5abceca0d71..a778e311c18 100644
--- a/src/libstore/build/worker.hh
+++ b/src/libstore/build/worker.hh
@@ -4,6 +4,7 @@
#include "types.hh"
#include "lock.hh"
#include "store-api.hh"
+#include "derived-path-map.hh"
#include "goal.hh"
#include "realisation.hh"
@@ -13,6 +14,7 @@
namespace nix {
/* Forward definition. */
+struct CreateDerivationAndRealiseGoal;
struct DerivationGoal;
struct PathSubstitutionGoal;
class DrvOutputSubstitutionGoal;
@@ -31,9 +33,23 @@ class DrvOutputSubstitutionGoal;
*/
GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
+GoalPtr upcast_goal(std::shared_ptr<DerivationGoal> subGoal);
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
+/**
+ * The current implementation of impure derivations has
+ * `DerivationGoal`s accumulate realisations from their waitees.
+ * Unfortunately, `DerivationGoal`s don't directly depend on other
+ * goals, but instead depend on `CreateDerivationAndRealiseGoal`s.
+ *
+ * We try not to share any of the details of any goal type with any
+ * other, for sake of modularity and quicker rebuilds. This means we
+ * cannot "just" downcast and fish out the field. So as an escape hatch,
+ * we have made the function, written in `worker.cc` where all the goal
+ * types are visible, and use it instead.
+ */
+const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee);
/**
* A mapping used to remember for each child process to what goal it
@@ -102,6 +118,9 @@ private:
* Maps used to prevent multiple instantiations of a goal for the
* same derivation / path.
*/
+
+ DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>> outerDerivationGoals;
+
std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
@@ -189,6 +208,9 @@ public:
* @ref DerivationGoal "derivation goal"
*/
private:
+ std::shared_ptr<CreateDerivationAndRealiseGoal> makeCreateDerivationAndRealiseGoal(
+ ref<SingleDerivedPath> drvPath,
+ const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal);
std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
const StorePath & drvPath, const OutputsSpec & wantedOutputs,
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc
new file mode 100644
index 00000000000..5c8c7a4f212
--- /dev/null
+++ b/src/libstore/derived-path-map.cc
@@ -0,0 +1,33 @@
+#include "derived-path-map.hh"
+
+namespace nix {
+
+template<typename V>
+typename DerivedPathMap<V>::ChildNode & DerivedPathMap<V>::ensureSlot(const SingleDerivedPath & k)
+{
+ std::function<ChildNode &(const SingleDerivedPath & )> initIter;
+ initIter = [&](const auto & k) -> auto & {
+ return std::visit(overloaded {
+ [&](const SingleDerivedPath::Opaque & bo) -> auto & {
+ // will not overwrite if already there
+ return map[bo.path];
+ },
+ [&](const SingleDerivedPath::Built & bfd) -> auto & {
+ auto & n = initIter(*bfd.drvPath);
+ return n.childMap[bfd.output];
+ },
+ }, k.raw());
+ };
+ return initIter(k);
+}
+
+}
+
+// instantiations
+
+#include "create-derivation-and-realise-goal.hh"
+namespace nix {
+
+template struct DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>;
+
+}
diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh
new file mode 100644
index 00000000000..9ce58206eee
--- /dev/null
+++ b/src/libstore/derived-path-map.hh
@@ -0,0 +1,73 @@
+#pragma once
+
+#include "types.hh"
+#include "derived-path.hh"
+
+namespace nix {
+
+/**
+ * A simple Trie, of sorts. Conceptually a map of `SingleDerivedPath` to
+ * values.
+ *
+ * Concretely, an n-ary tree, as described below. A
+ * `SingleDerivedPath::Opaque` maps to the value of an immediate child
+ * of the root node. A `SingleDerivedPath::Built` maps to a deeper child
+ * node: the `SingleDerivedPath::Built::drvPath` is first mapped to a a
+ * child node (inductively), and then the
+ * `SingleDerivedPath::Built::output` is used to look up that child's
+ * child via its map. In this manner, every `SingleDerivedPath` is
+ * mapped to a child node.
+ *
+ * @param V A type to instantiate for each output. It should probably
+ * should be an "optional" type so not every interior node has to have a
+ * value. For example, the scheduler uses
+ * `DerivedPathMap<std::weak_ptr<CreateDerivationAndRealiseGoal>>` to
+ * remember which goals correspond to which outputs. `* const Something`
+ * or `std::optional<Something>` would also be good choices for
+ * "optional" types.
+ */
+template<typename V>
+struct DerivedPathMap {
+ /**
+ * A child node (non-root node).
+ */
+ struct ChildNode {
+ /**
+ * Value of this child node.
+ *
+ * @see DerivedPathMap for what `V` should be.
+ */
+ V value;
+
+ /**
+ * The map type for the root node.
+ */
+ using Map = std::map<OutputName, ChildNode>;
+
+ /**
+ * The map of the root node.
+ */
+ Map childMap;
+ };
+
+ /**
+ * The map type for the root node.
+ */
+ using Map = std::map<StorePath, ChildNode>;
+
+ /**
+ * The map of root node.
+ */
+ Map map;
+
+ /**
+ * Find the node for `k`, creating it if needed.
+ *
+ * The node is referred to as a "slot" on the assumption that `V` is
+ * some sort of optional type, so the given key can be set or unset
+ * by changing this node.
+ */
+ ChildNode & ensureSlot(const SingleDerivedPath & k);
+};
+
+}
diff --git a/tests/dyn-drv/build-built-drv.sh b/tests/dyn-drv/build-built-drv.sh
index 647be945716..94f3550bdc3 100644
--- a/tests/dyn-drv/build-built-drv.sh
+++ b/tests/dyn-drv/build-built-drv.sh
@@ -18,4 +18,6 @@ clearStore
drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv)
-expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented"
+out2=$(nix build "${drvDep}^out^out" --no-link)
+
+test $out1 == $out2