Parse string context elements properly

Prior to this change, we had a bunch of ad-hoc string manipulation code
scattered around. This made it hard to figure out what data model for
string contexts is.

Now, we still store string contexts most of the time as encoded strings
--- I was wary of the performance implications of changing that --- but
whenever we parse them we do so only through the
`NixStringContextElem::parse` method, which handles all cases. This
creates a data type that is very similar to `DerivedPath` but:

 - Represents the funky `=<drvpath>` case as properly distinct from the
   others.

 - Only encodes a single output, no wildcards and no set, for the
   "built" case.

(I would like to deprecate `=<path>`, after which we are in spitting
distance of `DerivedPath` and could maybe get away with fewer types, but
that is another topic for another day.)

Additionally, `App` now uses `DerivedPath` not `StorePathWithOutputs`
for its (simplified) context. This is just better as
`StorePathWithOutputs` is the deprecated old type just for the old CLI.

src/libcmd/installables.hh

@@ -2,7 +2,6 @@
 
 #include "util.hh"
 #include "path.hh"
-#include "path-with-outputs.hh"
 #include "derived-path.hh"
 #include "eval.hh"
 #include "store-api.hh"
@@ -20,7 +19,7 @@ namespace eval_cache { class EvalCache; class AttrCursor; }
 
 struct App
 {
-    std::vector<StorePathWithOutputs> context;
+    std::vector<DerivedPath> context;
     Path program;
     // FIXME: add args, sandbox settings, metadata, ...
 };

src/libexpr/eval-cache.cc

@@ -300,7 +300,7 @@ struct AttrDb
                 NixStringContext context;
                 if (!queryAttribute.isNull(3))
                     for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
-                        context.push_back(decodeContext(cfg, s));
+                        context.push_back(NixStringContextElem::parse(cfg, s));
                 return {{rowId, string_t{queryAttribute.getStr(2), context}}};
             }
             case AttrType::Bool:
@@ -592,7 +592,18 @@ string_t AttrCursor::getStringWithContext()
             if (auto s = std::get_if<string_t>(&cachedValue->second)) {
                 bool valid = true;
                 for (auto & c : s->second) {
-                    if (!root->state.store->isValidPath(c.first)) {
+                    const StorePath & path = std::visit(overloaded {
+                        [&](const NixStringContextElem::DrvDeep & d) -> const StorePath & {
+                            return d.drvPath;
+                        },
+                        [&](const NixStringContextElem::Built & b) -> const StorePath & {
+                            return b.drvPath;
+                        },
+                        [&](const NixStringContextElem::Opaque & o) -> const StorePath & {
+                            return o.path;
+                        },
+                    }, c.raw());
+                    if (!root->state.store->isValidPath(path)) {
                         valid = false;
                         break;
                     }

src/libexpr/eval.cc

@@ -2068,27 +2068,6 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string
 }
 
 
-/* Decode a context string ‘!<name>!<path>’ into a pair <path,
-   name>. */
-NixStringContextElem decodeContext(const Store & store, std::string_view s)
-{
-    if (s.at(0) == '!') {
-        size_t index = s.find("!", 1);
-        return {
-            store.parseStorePath(s.substr(index + 1)),
-            std::string(s.substr(1, index - 1)),
-        };
-    } else
-        return {
-            store.parseStorePath(
-                s.at(0) == '/'
-                ? s
-                : s.substr(1)),
-            "",
-        };
-}
-
-
 void copyContext(const Value & v, PathSet & context)
 {
     if (v.string.context)
@@ -2103,7 +2082,7 @@ NixStringContext Value::getContext(const Store & store)
     assert(internalType == tString);
     if (string.context)
         for (const char * * p = string.context; *p; ++p)
-            res.push_back(decodeContext(store, *p));
+            res.push_back(NixStringContextElem::parse(store, *p));
     return res;
 }
 

src/libexpr/eval.hh

@@ -551,10 +551,6 @@ struct DebugTraceStacker {
 std::string_view showType(ValueType type);
 std::string showType(const Value & v);
 
-/* Decode a context string ‘!<name>!<path>’ into a pair <path,
-   name>. */
-NixStringContextElem decodeContext(const Store & store, std::string_view s);
-
 /* If `path' refers to a directory, then append "/default.nix". */
 Path resolveExprPath(Path path);
 

src/libexpr/local.mk

@@ -6,6 +6,7 @@ libexpr_DIR := $(d)
 
 libexpr_SOURCES := \
   $(wildcard $(d)/*.cc) \
+  $(wildcard $(d)/value/*.cc) \
   $(wildcard $(d)/primops/*.cc) \
   $(wildcard $(d)/flake/*.cc) \
   $(d)/lexer-tab.cc \
@@ -37,6 +38,8 @@ clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexe
 
 $(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644))
 
+$(foreach i, $(wildcard src/libexpr/value/*.hh), \
+  $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644)))
 $(foreach i, $(wildcard src/libexpr/flake/*.hh), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
 

src/libexpr/primops.cc

@@ -43,16 +43,32 @@ StringMap EvalState::realiseContext(const PathSet & context)
     std::vector<DerivedPath::Built> drvs;
     StringMap res;
 
-    for (auto & i : context) {
-        auto [ctx, outputName] = decodeContext(*store, i);
-        auto ctxS = store->printStorePath(ctx);
-        if (!store->isValidPath(ctx))
-            debugThrowLastTrace(InvalidPathError(store->printStorePath(ctx)));
-        if (!outputName.empty() && ctx.isDerivation()) {
-            drvs.push_back({ctx, {outputName}});
-        } else {
-            res.insert_or_assign(ctxS, ctxS);
-        }
+    for (auto & c_ : context) {
+        auto ensureValid = [&](const StorePath & p, const std::string & pstr) {
+            if (!store->isValidPath(p))
+                debugThrowLastTrace(InvalidPathError(pstr));
+        };
+        auto c = NixStringContextElem::parse(*store, c_);
+        std::visit(overloaded {
+            [&](const NixStringContextElem::Built & b) {
+                drvs.push_back(DerivedPath::Built {
+                    .drvPath = b.drvPath,
+                    .outputs = std::set { b.output },
+                });
+                ensureValid(b.drvPath, store->printStorePath(b.drvPath));
+            },
+            [&](const NixStringContextElem::Opaque & o) {
+                auto ctxS = store->printStorePath(o.path);
+                res.insert_or_assign(ctxS, ctxS);
+                ensureValid(o.path, ctxS);
+            },
+            [&](const NixStringContextElem::DrvDeep & d) {
+                /* Treat same as Opaque */
+                auto ctxS = store->printStorePath(d.drvPath);
+                res.insert_or_assign(ctxS, ctxS);
+                ensureValid(d.drvPath, ctxS);
+            },
+        }, c.raw());
     }
 
     if (drvs.empty()) return {};
@@ -1179,35 +1195,31 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * *
     /* Everything in the context of the strings in the derivation
        attributes should be added as dependencies of the resulting
        derivation. */
-    for (auto & path : context) {
-
-        /* Paths marked with `=' denote that the path of a derivation
-           is explicitly passed to the builder.  Since that allows the
-           builder to gain access to every path in the dependency
-           graph of the derivation (including all outputs), all paths
-           in the graph must be added to this derivation's list of
-           inputs to ensure that they are available when the builder
-           runs. */
-        if (path.at(0) == '=') {
-            /* !!! This doesn't work if readOnlyMode is set. */
-            StorePathSet refs;
-            state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs);
-            for (auto & j : refs) {
-                drv.inputSrcs.insert(j);
-                if (j.isDerivation())
-                    drv.inputDrvs[j] = state.store->readDerivation(j).outputNames();
-            }
-        }
-
-        /* Handle derivation outputs of the form ‘!<name>!<path>’. */
-        else if (path.at(0) == '!') {
-            auto ctx = decodeContext(*state.store, path);
-            drv.inputDrvs[ctx.first].insert(ctx.second);
-        }
-
-        /* Otherwise it's a source file. */
-        else
-            drv.inputSrcs.insert(state.store->parseStorePath(path));
+    for (auto & c_ : context) {
+        auto c = NixStringContextElem::parse(*state.store, c_);
+        std::visit(overloaded {
+            /* Since this allows the builder to gain access to every
+               path in the dependency graph of the derivation (including
+               all outputs), all paths in the graph must be added to
+               this derivation's list of inputs to ensure that they are
+               available when the builder runs. */
+            [&](const NixStringContextElem::DrvDeep & d) {
+                /* !!! This doesn't work if readOnlyMode is set. */
+                StorePathSet refs;
+                state.store->computeFSClosure(d.drvPath, refs);
+                for (auto & j : refs) {
+                    drv.inputSrcs.insert(j);
+                    if (j.isDerivation())
+                        drv.inputDrvs[j] = state.store->readDerivation(j).outputNames();
+                }
+            },
+            [&](const NixStringContextElem::Built & b) {
+                drv.inputDrvs[b.drvPath].insert(b.output);
+            },
+            [&](const NixStringContextElem::Opaque & o) {
+                drv.inputSrcs.insert(o.path);
+            },
+        }, c.raw());
     }
 
     /* Do we have all required attributes? */

src/libexpr/primops/context.cc

@@ -37,8 +37,15 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p
     auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardOutputDependency");
 
     PathSet context2;
-    for (auto & p : context)
-        context2.insert(p.at(0) == '=' ? std::string(p, 1) : p);
+    for (auto && p : context) {
+        auto c = NixStringContextElem::parse(*state.store, p);
+        if (auto * ptr = std::get_if<NixStringContextElem::DrvDeep>(&c)) {
+            context2.emplace(state.store->printStorePath(ptr->drvPath));
+        } else {
+            /* Can reuse original item */
+            context2.emplace(std::move(p));
+        }
+    }
 
     v.mkString(*s, context2);
 }
@@ -74,34 +81,22 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
     };
     PathSet context;
     state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.getContext");
-    auto contextInfos = std::map<Path, ContextInfo>();
+    auto contextInfos = std::map<StorePath, ContextInfo>();
     for (const auto & p : context) {
         Path drv;
         std::string output;
-        const Path * path = &p;
-        if (p.at(0) == '=') {
-            drv = std::string(p, 1);
-            path = &drv;
-        } else if (p.at(0) == '!') {
-            NixStringContextElem ctx = decodeContext(*state.store, p);
-            drv = state.store->printStorePath(ctx.first);
-            output = ctx.second;
-            path = &drv;
-        }
-        auto isPath = drv.empty();
-        auto isAllOutputs = (!drv.empty()) && output.empty();
-
-        auto iter = contextInfos.find(*path);
-        if (iter == contextInfos.end()) {
-            contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}});
-        } else {
-            if (isPath)
-                iter->second.path = true;
-            else if (isAllOutputs)
-                iter->second.allOutputs = true;
-            else
-                iter->second.outputs.emplace_back(std::move(output));
-        }
+        NixStringContextElem ctx = NixStringContextElem::parse(*state.store, p);
+        std::visit(overloaded {
+            [&](NixStringContextElem::DrvDeep & d) {
+                contextInfos[d.drvPath].allOutputs = true;
+            },
+            [&](NixStringContextElem::Built & b) {
+                contextInfos[b.drvPath].outputs.emplace_back(std::move(output));
+            },
+            [&](NixStringContextElem::Opaque & o) {
+                contextInfos[o.path].path = true;
+            },
+        }, ctx.raw());
     }
 
     auto attrs = state.buildBindings(contextInfos.size());
@@ -120,7 +115,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
             for (const auto & [i, output] : enumerate(info.second.outputs))
                 (outputsVal.listElems()[i] = state.allocValue())->mkString(output);
         }
-        attrs.alloc(info.first).mkAttrs(infoAttrs);
+        attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs);
     }
 
     v.mkAttrs(attrs);

src/libexpr/value.hh

@@ -3,6 +3,7 @@
 #include <cassert>
 
 #include "symbol-table.hh"
+#include "value/context.hh"
 
 #if HAVE_BOEHMGC
 #include <gc/gc_allocator.h>
@@ -67,8 +68,6 @@ class XMLWriter;
 
 typedef int64_t NixInt;
 typedef double NixFloat;
-typedef std::pair<StorePath, std::string> NixStringContextElem;
-typedef std::vector<NixStringContextElem> NixStringContext;
 
 /* External values must descend from ExternalValueBase, so that
  * type-agnostic nix functions (e.g. showType) can be implemented

src/libexpr/value/context.cc

@@ -0,0 +1,53 @@
+#include "value/context.hh"
+#include "store-api.hh"
+
+#include <optional>
+
+namespace nix {
+
+NixStringContextElem NixStringContextElem::parse(const Store & store, std::string_view s)
+{
+    switch (s.at(0)) {
+    case '!': {
+        size_t index = s.find("!", 1);
+        return NixStringContextElem::Built {
+            .drvPath = store.parseStorePath(s.substr(index + 1)),
+            .output = std::string(s.substr(1, index - 1)),
+        };
+    }
+    case '=': {
+        return NixStringContextElem::DrvDeep {
+            .drvPath = store.parseStorePath(s.substr(1)),
+        };
+    }
+    default: {
+        return NixStringContextElem::Opaque {
+            .path = store.parseStorePath(s),
+        };
+    }
+    }
+}
+
+std::string NixStringContextElem::render(const Store & store) const {
+    return std::visit(overloaded {
+        [&](const NixStringContextElem::Built & b) {
+            std::string res;
+            res += '!';
+            res += store.printStorePath(b.drvPath);
+            res += '!';
+            res += b.output;
+            return res;
+        },
+        [&](const NixStringContextElem::DrvDeep & d) {
+            std::string res;
+            res += '=';
+            res += store.printStorePath(d.drvPath);
+            return res;
+        },
+        [&](const NixStringContextElem::Opaque & o) {
+            return store.printStorePath(o.path);
+        },
+    }, raw());
+}
+
+}