Mounted SSH Store

doc/manual/src/release-notes/rl-next.md

@@ -1,3 +1,6 @@
 # Release X.Y (202?-??-??)
 
-- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
+- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set.
+
+- Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md).
+  This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem.

src/libstore/daemon.cc

@@ -657,6 +657,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
         break;
     }
 
+    case WorkerProto::Op::AddPermRoot: {
+        if (!trusted)
+            throw Error(
+                "you are not privileged to create perm roots\n\n"
+                "hint: you can just do this client-side without special privileges, and probably want to do that instead.");
+        auto storePath = WorkerProto::Serialise<StorePath>::read(*store, rconn);
+        Path gcRoot = absPath(readString(from));
+        logger->startWork();
+        auto & localFSStore = require<LocalFSStore>(*store);
+        localFSStore.addPermRoot(storePath, gcRoot);
+        logger->stopWork();
+        to << gcRoot;
+        break;
+    }
+
     case WorkerProto::Op::AddIndirectRoot: {
         Path path = absPath(readString(from));
 

src/libstore/indirect-root-store.hh

@@ -11,6 +11,30 @@ namespace nix {
  * reference.
  *
  * See methods for details on the operations it represents.
+ *
+ * @note
+ * To understand the purpose of this class, it might help to do some
+ * "closed-world" rather than "open-world" reasoning, and consider the
+ * problem it solved for us. This class was factored out from
+ * `LocalFSStore` in order to support the following table, which
+ * contains 4 concrete store types (non-abstract classes, exposed to the
+ * user), and how they implemented the two GC root methods:
+ *
+ * @note
+ * |                   | `addPermRoot()` | `addIndirectRoot()` |
+ * |-------------------|-----------------|---------------------|
+ * | `LocalStore`      | local           | local               |
+ * | `UDSRemoteStore`  | local           | remote              |
+ * | `SSHStore`        | doesn't have    | doesn't have        |
+ * | `MountedSSHStore` | remote          | doesn't have        |
+ *
+ * @note
+ * Note how only the local implementations of `addPermRoot()` need
+ * `addIndirectRoot()`; that is what this class enforces. Without it,
+ * and with `addPermRoot()` and `addIndirectRoot()` both `virtual`, we
+ * would accidentally be allowing for a combinatorial explosion of
+ * possible implementations many of which make no sense. Having this and
+ * that invariant enforced cuts down that space.
  */
 struct IndirectRootStore : public virtual LocalFSStore
 {

src/libstore/mounted-ssh-store.md

@@ -0,0 +1,18 @@
+R"(
+
+**Store URL format**: `mounted-ssh-ng://[username@]hostname`
+
+Experimental store type that allows full access to a Nix store on a remote machine,
+and additionally requires that store be mounted in the local file system.
+
+The mounting of that store is not managed by Nix, and must by managed manually.
+It could be accomplished with SSHFS or NFS, for example.
+
+The local file system is used to optimize certain operations.
+For example, rather than serializing Nix archives and sending over the Nix channel,
+we can directly access the file system data via the mount-point.
+
+The local file system is also used to make certain operations possible that wouldn't otherwise be.
+For example, persistent GC roots can be created if they reside on the same file system as the remote store:
+the remote side will create the symlinks necessary to avoid race conditions.
+)"

src/libstore/ssh-store.cc

@@ -3,9 +3,10 @@
 #include "local-fs-store.hh"
 #include "remote-store.hh"
 #include "remote-store-connection.hh"
-#include "remote-fs-accessor.hh"
+#include "source-accessor.hh"
 #include "archive.hh"
 #include "worker-protocol.hh"
+#include "worker-protocol-impl.hh"
 #include "pool.hh"
 #include "ssh.hh"
 
@@ -78,6 +79,8 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore
 
     std::string host;
 
+    std::vector<std::string> extraRemoteProgramArgs;
+
     SSHMaster master;
 
     void setOptions(RemoteStore::Connection & conn) override
@@ -91,13 +94,130 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore
     };
 };
 
+struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig
+{
+    using SSHStoreConfig::SSHStoreConfig;
+    using LocalFSStoreConfig::LocalFSStoreConfig;
+
+    MountedSSHStoreConfig(StringMap params)
+        : StoreConfig(params)
+        , RemoteStoreConfig(params)
+        , CommonSSHStoreConfig(params)
+        , SSHStoreConfig(params)
+        , LocalFSStoreConfig(params)
+    {
+    }
+
+    const std::string name() override { return "Experimental SSH Store with filesytem mounted"; }
+
+    std::string doc() override
+    {
+        return
+          #include "mounted-ssh-store.md"
+          ;
+    }
+
+    std::optional<ExperimentalFeature> experimentalFeature() const override
+    {
+        return ExperimentalFeature::MountedSSHStore;
+    }
+};
+
+/**
+ * The mounted ssh store assumes that filesystems on the remote host are
+ * shared with the local host. This means that the remote nix store is
+ * available locally and is therefore treated as a local filesystem
+ * store.
+ *
+ * MountedSSHStore is very similar to UDSRemoteStore --- ignoring the
+ * superficial differnce of SSH vs Unix domain sockets, they both are
+ * accessing remote stores, and they both assume the store will be
+ * mounted in the local filesystem.
+ *
+ * The difference lies in how they manage GC roots. See addPermRoot
+ * below for details.
+ */
+class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSHStore, public virtual LocalFSStore
+{
+public:
+
+    MountedSSHStore(const std::string & scheme, const std::string & host, const Params & params)
+        : StoreConfig(params)
+        , RemoteStoreConfig(params)
+        , CommonSSHStoreConfig(params)
+        , SSHStoreConfig(params)
+        , LocalFSStoreConfig(params)
+        , MountedSSHStoreConfig(params)
+        , Store(params)
+        , RemoteStore(params)
+        , SSHStore(scheme, host, params)
+        , LocalFSStore(params)
+    {
+        extraRemoteProgramArgs = {
+            "--process-ops",
+        };
+    }
+
+    static std::set<std::string> uriSchemes()
+    {
+        return {"mounted-ssh-ng"};
+    }
+
+    std::string getUri() override
+    {
+        return *uriSchemes().begin() + "://" + host;
+    }
+
+    void narFromPath(const StorePath & path, Sink & sink) override
+    {
+        return LocalFSStore::narFromPath(path, sink);
+    }
+
+    ref<SourceAccessor> getFSAccessor(bool requireValidPath) override
+    {
+        return LocalFSStore::getFSAccessor(requireValidPath);
+    }
+
+    std::optional<std::string> getBuildLogExact(const StorePath & path) override
+    {
+        return LocalFSStore::getBuildLogExact(path);
+    }
+
+    /**
+     * This is the key difference from UDSRemoteStore: UDSRemote store
+     * has the client create the direct root, and the remote side create
+     * the indirect root.
+     *
+     * We could also do that, but the race conditions (will the remote
+     * side see the direct root the client made?) seems bigger.
+     *
+     * In addition, the remote-side will have a process associated with
+     * the authenticating user handling the connection (even if there
+     * is a system-wide daemon or similar). This process can safely make
+     * the direct and indirect roots without there being such a risk of
+     * privilege escalation / symlinks in directories owned by the
+     * originating requester that they cannot delete.
+     */
+    Path addPermRoot(const StorePath & path, const Path & gcRoot) override
+    {
+        auto conn(getConnection());
+        conn->to << WorkerProto::Op::AddPermRoot;
+        WorkerProto::write(*this, *conn, path);
+        WorkerProto::write(*this, *conn, gcRoot);
+        conn.processStderr();
+        return readString(conn->from);
+    }
+};
+
 ref<RemoteStore::Connection> SSHStore::openConnection()
 {
     auto conn = make_ref<Connection>();
 
     std::string command = remoteProgram + " --stdio";
     if (remoteStore.get() != "")
         command += " --store " + shellEscape(remoteStore.get());
+    for (auto & arg : extraRemoteProgramArgs)
+        command += " " + shellEscape(arg);
 
     conn->sshConn = master.startCommand(command);
     conn->to = FdSink(conn->sshConn->in.get());
@@ -106,5 +226,6 @@ ref<RemoteStore::Connection> SSHStore::openConnection()
 }
 
 static RegisterStoreImplementation<SSHStore, SSHStoreConfig> regSSHStore;
+static RegisterStoreImplementation<MountedSSHStore, MountedSSHStoreConfig> regMountedSSHStore;
 
 }

src/libstore/worker-protocol.hh

@@ -9,7 +9,7 @@ namespace nix {
 #define WORKER_MAGIC_1 0x6e697863
 #define WORKER_MAGIC_2 0x6478696f
 
-#define PROTOCOL_VERSION (1 << 8 | 35)
+#define PROTOCOL_VERSION (1 << 8 | 36)
 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
 #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
 
@@ -161,6 +161,7 @@ enum struct WorkerProto::Op : uint64_t
     AddMultipleToStore = 44,
     AddBuildLog = 45,
     BuildPathsWithResults = 46,
+    AddPermRoot = 47,
 };
 
 /**

src/libutil/experimental-features.cc

@@ -262,6 +262,13 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails
             Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting.
         )",
     },
+    {
+        .tag = Xp::MountedSSHStore,
+        .name = "mounted-ssh-store",
+        .description = R"(
+            Allow the use of the [`mounted SSH store`](@docroot@/command-ref/new-cli/nix3-help-stores.html#experimental-ssh-store-with-filesytem-mounted).
+        )",
+    },
     {
         .tag = Xp::VerifiedFetches,
         .name = "verified-fetches",

src/libutil/experimental-features.hh

@@ -34,6 +34,7 @@ enum struct ExperimentalFeature
     ParseTomlTimestamps,
     ReadOnlyLocalStore,
     ConfigurableImpureEnv,
+    MountedSSHStore,
     VerifiedFetches,
 };
 

src/nix/daemon.cc

@@ -443,16 +443,23 @@ static void processStdioConnection(ref<Store> store, TrustedFlag trustClient)
  *
  * @param forceTrustClientOpt See `daemonLoop()` and the parameter with
  * the same name over there for details.
+ *
+ * @param procesOps Whether to force processing ops even if the next
+ * store also is a remote store and could process it directly.
  */
-static void runDaemon(bool stdio, std::optional<TrustedFlag> forceTrustClientOpt)
+static void runDaemon(bool stdio, std::optional<TrustedFlag> forceTrustClientOpt, bool processOps)
 {
     if (stdio) {
         auto store = openUncachedStore();
 
+        std::shared_ptr<RemoteStore> remoteStore;
+
         // If --force-untrusted is passed, we cannot forward the connection and
         // must process it ourselves (before delegating to the next store) to
         // force untrusting the client.
-        if (auto remoteStore = store.dynamic_pointer_cast<RemoteStore>(); remoteStore && (!forceTrustClientOpt || *forceTrustClientOpt != NotTrusted))
+        processOps |= !forceTrustClientOpt || *forceTrustClientOpt != NotTrusted;
+
+        if (!processOps && (remoteStore = store.dynamic_pointer_cast<RemoteStore>()))
             forwardStdioConnection(*remoteStore);
         else
             // `Trusted` is passed in the auto (no override case) because we
@@ -468,6 +475,7 @@ static int main_nix_daemon(int argc, char * * argv)
     {
         auto stdio = false;
         std::optional<TrustedFlag> isTrustedOpt = std::nullopt;
+        auto processOps = false;
 
         parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) {
             if (*arg == "--daemon")
@@ -487,11 +495,14 @@ static int main_nix_daemon(int argc, char * * argv)
             } else if (*arg == "--default-trust") {
                 experimentalFeatureSettings.require(Xp::DaemonTrustOverride);
                 isTrustedOpt = std::nullopt;
+            } else if (*arg == "--process-ops") {
+                experimentalFeatureSettings.require(Xp::MountedSSHStore);
+                processOps = true;
             } else return false;
             return true;
         });
 
-        runDaemon(stdio, isTrustedOpt);
+        runDaemon(stdio, isTrustedOpt, processOps);
 
         return 0;
     }
@@ -503,6 +514,7 @@ struct CmdDaemon : StoreCommand
 {
     bool stdio = false;
     std::optional<TrustedFlag> isTrustedOpt = std::nullopt;
+    bool processOps = false;
 
     CmdDaemon()
     {
@@ -538,6 +550,19 @@ struct CmdDaemon : StoreCommand
             }},
             .experimentalFeature = Xp::DaemonTrustOverride,
         });
+
+        addFlag({
+            .longName = "process-ops",
+            .description = R"(
+              Forces the daemon to process received commands itself rather than forwarding the commands straight to the remote store.
+
+              This is useful for the `mounted-ssh://` store where some actions need to be performed on the remote end but as connected user, and not as the user of the underlying daemon on the remote end.
+            )",
+            .handler = {[&]() {
+                processOps = true;
+            }},
+            .experimentalFeature = Xp::MountedSSHStore,
+        });
     }
 
     std::string description() override
@@ -556,7 +581,7 @@ struct CmdDaemon : StoreCommand
 
     void run(ref<Store> store) override
     {
-        runDaemon(stdio, isTrustedOpt);
+        runDaemon(stdio, isTrustedOpt, processOps);
     }
 };
 

tests/functional/build-remote-with-mounted-ssh-ng.sh

@@ -0,0 +1,22 @@
+source common.sh
+
+requireSandboxSupport
+[[ $busybox =~ busybox ]] || skipTest "no busybox"
+
+enableFeatures mounted-ssh-store
+
+nix build -Lvf simple.nix \
+  --arg busybox $busybox \
+  --out-link $TEST_ROOT/result-from-remote \
+  --store mounted-ssh-ng://localhost
+
+nix build -Lvf simple.nix \
+  --arg busybox $busybox \
+  --out-link $TEST_ROOT/result-from-remote-new-cli \
+  --store 'mounted-ssh-ng://localhost?remote-program=nix daemon'
+
+# This verifies that the out link was actually created and valid. The ability
+# to create out links (permanent gc roots) is the distinguishing feature of
+# the mounted-ssh-ng store.
+cat $TEST_ROOT/result-from-remote/hello | grepQuiet 'Hello World!'
+cat $TEST_ROOT/result-from-remote-new-cli/hello | grepQuiet 'Hello World!'

tests/functional/local.mk

@@ -69,6 +69,7 @@ nix_tests = \
   build-remote-trustless-should-pass-2.sh \
   build-remote-trustless-should-pass-3.sh \
   build-remote-trustless-should-fail-0.sh \
+  build-remote-with-mounted-ssh-ng.sh \
   nar-access.sh \
   pure-eval.sh \
   eval.sh \