diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 08adbe0c99d..dce6ee87fc3 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -100,7 +100,7 @@ std::pair parseFlakeRefWithFragment( .url = url, .base = "flake:" + match.str(1), .scheme = "flake", - .authority = "", + .authority = ParsedURLAuthority{}, .path = match[1], }; @@ -161,7 +161,7 @@ std::pair parseFlakeRefWithFragment( .url = base, // FIXME .base = base, .scheme = "git+file", - .authority = "", + .authority = {{}}, .path = flakeRoot, .query = decodeQuery(match[2]), }; diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 1d23ef53b9d..6fc2939277f 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -71,7 +71,7 @@ std::string fixURI(std::string uri, EvalState & state, const std::string & defau if (uri.find("://") == std::string::npos) { const auto p = ParsedURL { .scheme = defaultScheme, - .authority = "", + .authority = {{}}, .path = uri }; return p.to_string(); diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 61541e69dfb..008f1a79cad 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -10,8 +10,8 @@ struct PathInputScheme : InputScheme { if (url.scheme != "path") return {}; - if (url.authority && *url.authority != "") - throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); + if (url.authority && *url.authority != ParsedURLAuthority {}) + throw Error("path URL '%s' should not have an authority ('%s')", url.url, url.authority->to_string()); Input input; input.attrs.insert_or_assign("type", "path"); diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 74d6ed3b518..d1613634020 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -18,7 +18,7 @@ struct DummyStoreConfig : virtual StoreConfig { struct DummyStore : public virtual DummyStoreConfig, public virtual Store { - DummyStore(const std::string scheme, const std::string uri, const Params & params) + DummyStore(const std::string scheme, const std::string uri, std::optional port, const Params & params) : DummyStore(params) { } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 85c5eed4c01..aa65adbdfe7 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -41,13 +41,14 @@ class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public v HttpBinaryCacheStore( const std::string & scheme, const Path & _cacheUri, + std::optional port, const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) , HttpBinaryCacheStoreConfig(params) , Store(params) , BinaryCacheStore(params) - , cacheUri(scheme + "://" + _cacheUri) + , cacheUri(scheme + "://" + _cacheUri + (port ? fmt(":%d", *port) : "")) { if (cacheUri.back() == '/') cacheUri.pop_back(); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index fa17d606d43..05452fe1b08 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -94,7 +94,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor static std::set uriSchemes() { return {"ssh"}; } - LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params) + LegacySSHStore(const std::string & scheme, const std::string & host, std::optional port, const Params & params) : StoreConfig(params) , CommonSSHStoreConfig(params) , LegacySSHStoreConfig(params) @@ -107,6 +107,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor )) , master( host, + port, sshKey, sshPublicHostKey, // Use SSH master only if using more than 1 connection. diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 5481dd762e2..8f333209882 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -31,6 +31,7 @@ class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public LocalBinaryCacheStore( const std::string scheme, const Path & binaryCacheDir, + std::optional port, const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) @@ -39,6 +40,8 @@ class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public , BinaryCacheStore(params) , binaryCacheDir(binaryCacheDir) { + if (port) + throw Error("file:// store does not accept a port number"); } void init() override; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e69460e6cd3..aa5462a9f94 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -440,7 +440,7 @@ LocalStore::LocalStore(const Params & params) } -LocalStore::LocalStore(std::string scheme, std::string path, const Params & params) +LocalStore::LocalStore(std::string scheme, std::string path, std::optional port, const Params & params) : LocalStore(params) { throw UnimplementedError("LocalStore"); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 8a3b0b43fc1..0047429c2fa 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -138,7 +138,7 @@ public: * necessary. */ LocalStore(const Params & params); - LocalStore(std::string scheme, std::string path, const Params & params); + LocalStore(std::string scheme, std::string path, std::optional port, const Params & params); ~LocalStore(); diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index d2fc6abafe4..af767ff8d15 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -271,6 +271,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStoreImpl( const std::string & uriScheme, const std::string & bucketName, + std::optional port, const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) @@ -281,6 +282,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual , bucketName(bucketName) , s3Helper(profile, region, scheme, endpoint) { + if (port) + throw Error("s3:// store does not accept a port number"); diskCache = getNarInfoDiskCache(); } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 0200076c08b..fee31518888 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -32,7 +32,7 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore { public: - SSHStore(const std::string & scheme, const std::string & host, const Params & params) + SSHStore(const std::string & scheme, const std::string & host, std::optional port, const Params & params) : StoreConfig(params) , RemoteStoreConfig(params) , CommonSSHStoreConfig(params) @@ -42,6 +42,7 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore , host(host) , master( host, + port, sshKey, sshPublicHostKey, // Use SSH master only if using more than 1 connection. diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index fae99d75b92..3933488d77f 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -3,9 +3,10 @@ namespace nix { -SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD) +SSHMaster::SSHMaster(const std::string & host, std::optional port, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD) : host(host) - , fakeSSH(host == "localhost") + , port(port) + , fakeSSH(host == "localhost" && !port) , keyFile(keyFile) , sshPublicHostKey(sshPublicHostKey) , useMaster(useMaster && !fakeSSH) @@ -36,6 +37,8 @@ void SSHMaster::addCommonSSHOpts(Strings & args) } if (compress) args.push_back("-C"); + if (port) + args.insert(args.end(), {"-p", std::to_string(*port)}); args.push_back("-oPermitLocalCommand=yes"); args.push_back("-oLocalCommand=echo started"); diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index 94b952af9c0..16ebfb6b752 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -11,6 +11,7 @@ class SSHMaster private: const std::string host; + const std::optional port; bool fakeSSH; const std::string keyFile; const std::string sshPublicHostKey; @@ -32,7 +33,7 @@ private: public: - SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD = -1); + SSHMaster(const std::string & host, std::optional port, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD = -1); struct Connection { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5bee1af9fd1..9c60e14704d 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1433,26 +1433,25 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para // // This function now ensures that a usable connection string is available: // * If the store to be opened is not an SSH store, nothing will be done. -// * If the URL looks like `root@[::1]` (which is allowed by the URL parser and probably +// * If the URL host looks like `[::1]` (which is allowed by the URL parser and probably // needed to pass further flags), it -// will be transformed into `root@::1` for SSH (same for `[::1]` -> `::1`). -// * If the URL looks like `root@::1` it will be left as-is. +// will be transformed into `::1` for SSH (same for `[::1]` -> `::1`). +// * If the URL host looks like `::1` it will be left as-is. // * In any other case, the string will be left as-is. -static std::string extractConnStr(const std::string &proto, const std::string &connStr) +static std::string extractConnStr( + const std::string & proto, + const std::string & host) { if (proto.rfind("ssh") != std::string::npos) { std::smatch result; - std::regex v6AddrRegex("^((.*)@)?\\[(.*)\\]$"); + std::regex v6AddrRegex("^\\[(.*)\\]$"); - if (std::regex_match(connStr, result, v6AddrRegex)) { - if (result[1].matched) { - return result.str(1) + result.str(3); - } - return result.str(3); + if (std::regex_match(host, result, v6AddrRegex)) { + return result.str(1); } } - return connStr; + return host; } ref openStore(const std::string & uri_, @@ -1463,14 +1462,13 @@ ref openStore(const std::string & uri_, auto parsedUri = parseURL(uri_); params.insert(parsedUri.query.begin(), parsedUri.query.end()); - auto baseURI = extractConnStr( - parsedUri.scheme, - parsedUri.authority.value_or("") + parsedUri.path - ); + auto authHack = parsedUri.authority.value_or(ParsedURLAuthority {}); + authHack.host = extractConnStr(parsedUri.scheme, authHack.host); + authHack.host += parsedUri.path; for (auto implem : *Implementations::registered) { if (implem.uriSchemes.count(parsedUri.scheme)) { - auto store = implem.create(parsedUri.scheme, baseURI, params); + auto store = implem.create(parsedUri.scheme, authHack.host, authHack.port, params); if (store) { store->init(); store->warnUnknownSettings(); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 14a862eefca..9f8a3d25abe 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -967,7 +967,7 @@ std::list> getDefaultSubstituters(); struct StoreFactory { std::set uriSchemes; - std::function (const std::string & scheme, const std::string & uri, const Store::Params & params)> create; + std::function (const std::string & scheme, const std::string & host, std::optional optPort, const Store::Params & params)> create; std::function ()> getConfig; }; @@ -982,9 +982,9 @@ struct Implementations StoreFactory factory{ .uriSchemes = T::uriSchemes(), .create = - ([](const std::string & scheme, const std::string & uri, const Store::Params & params) + ([](const std::string & scheme, const std::string & host, std::optional optPort, const Store::Params & params) -> std::shared_ptr - { return std::make_shared(scheme, uri, params); }), + { return std::make_shared(scheme, host, optPort, params); }), .getConfig = ([]() -> std::shared_ptr diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 69dae2da560..7d2b03d3023 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -36,9 +36,12 @@ UDSRemoteStore::UDSRemoteStore(const Params & params) UDSRemoteStore::UDSRemoteStore( const std::string scheme, std::string socket_path, + std::optional port, const Params & params) : UDSRemoteStore(params) { + if (port) + throw Error("unix:// store does not accept a port number"); path.emplace(socket_path); } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 2bd6517fa2c..255ccc71680 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -26,7 +26,7 @@ class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual Local public: UDSRemoteStore(const Params & params); - UDSRemoteStore(const std::string scheme, std::string path, const Params & params); + UDSRemoteStore(const std::string scheme, std::string path, std::optional port, const Params & params); std::string getUri() override; diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc index a908631e641..a8b951e71eb 100644 --- a/src/libutil/tests/url.cc +++ b/src/libutil/tests/url.cc @@ -25,7 +25,8 @@ namespace nix { << "url: " << p.url << "\n" << "base: " << p.base << "\n" << "scheme: " << p.scheme << "\n" - << "authority: " << p.authority.value() << "\n" + << "host: " << p.authority.value().host << "\n" + << "port: " << p.authority.value().port.value() << "\n" << "path: " << p.path << "\n" << "query: " << print_map(p.query) << "\n" << "fragment: " << p.fragment << "\n"; @@ -39,7 +40,9 @@ namespace nix { .url = "http://www.example.org/file.tar.gz", .base = "http://www.example.org/file.tar.gz", .scheme = "http", - .authority = "www.example.org", + .authority = {{ + .host = "www.example.org", + }}, .path = "/file.tar.gz", .query = (StringMap) { }, .fragment = "", @@ -56,7 +59,9 @@ namespace nix { .url = "https://www.example.org/file.tar.gz", .base = "https://www.example.org/file.tar.gz", .scheme = "https", - .authority = "www.example.org", + .authority = {{ + .host = "www.example.org", + }}, .path = "/file.tar.gz", .query = (StringMap) { }, .fragment = "", @@ -73,7 +78,9 @@ namespace nix { .url = "https://www.example.org/file.tar.gz", .base = "https://www.example.org/file.tar.gz", .scheme = "https", - .authority = "www.example.org", + .authority = {{ + .host = "www.example.org", + }}, .path = "/file.tar.gz", .query = (StringMap) { { "download", "fast" }, { "when", "now" } }, .fragment = "hello", @@ -90,7 +97,9 @@ namespace nix { .url = "http://www.example.org/file.tar.gz", .base = "http://www.example.org/file.tar.gz", .scheme = "http", - .authority = "www.example.org", + .authority = {{ + .host = "www.example.org", + }}, .path = "/file.tar.gz", .query = (StringMap) { { "field", "value" } }, .fragment = "?foo=bar#", @@ -107,7 +116,9 @@ namespace nix { .url = "file+https://www.example.org/video.mp4", .base = "https://www.example.org/video.mp4", .scheme = "file+https", - .authority = "www.example.org", + .authority = {{ + .host = "www.example.org", + }}, .path = "/video.mp4", .query = (StringMap) { }, .fragment = "", @@ -129,7 +140,10 @@ namespace nix { .url = "http://127.0.0.1:8080/file.tar.gz", .base = "https://127.0.0.1:8080/file.tar.gz", .scheme = "http", - .authority = "127.0.0.1:8080", + .authority = {{ + .host = "127.0.0.1", + .port = 8080, + }}, .path = "/file.tar.gz", .query = (StringMap) { { "download", "fast" }, { "when", "now" } }, .fragment = "hello", @@ -146,7 +160,10 @@ namespace nix { .url = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080", .base = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080", .scheme = "http", - .authority = "[fe80::818c:da4d:8975:415c\%enp0s25]:8080", + .authority = {{ + .host = "[fe80::818c:da4d:8975:415c\%enp0s25]", + .port = 8080, + }}, .path = "", .query = (StringMap) { }, .fragment = "", @@ -164,7 +181,10 @@ namespace nix { .url = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", .base = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", .scheme = "http", - .authority = "[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", + .authority = {{ + .host = "[2a02:8071:8192:c100:311d:192d:81ac:11ea]", + .port = 8080, + }}, .path = "", .query = (StringMap) { }, .fragment = "", @@ -188,7 +208,11 @@ namespace nix { .url = "http://user:pass@www.example.org/file.tar.gz", .base = "http://user:pass@www.example.org/file.tar.gz", .scheme = "http", - .authority = "user:pass@www.example.org:8080", + .authority = {{ + .user = "user:pass", + .host = "www.example.org", + .port = 8080, + }}, .path = "/file.tar.gz", .query = (StringMap) { }, .fragment = "", @@ -206,7 +230,7 @@ namespace nix { .url = "", .base = "", .scheme = "file", - .authority = "", + .authority = {{}}, .path = "/none/of//your/business", .query = (StringMap) { }, .fragment = "", @@ -231,7 +255,9 @@ namespace nix { .url = "ftp://ftp.nixos.org/downloads/nixos.iso", .base = "ftp://ftp.nixos.org/downloads/nixos.iso", .scheme = "ftp", - .authority = "ftp.nixos.org", + .authority = {{ + .host = "ftp.nixos.org", + }}, .path = "/downloads/nixos.iso", .query = (StringMap) { }, .fragment = "", diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 98162b0f78a..b18e63882cf 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -16,7 +16,8 @@ const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)"; const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")"; const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)"; -const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; +const static std::string portRegex = "(?:[0-9]+)"; +const static std::string authorityRegex = "(?:(" + userRegex + ")@)?(" + hostRegex + ")(?::(" + portRegex + "))?"; const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; const static std::string segmentRegex = "(?:" + pcharRegex + "*)"; diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 9e44241ac25..85164eab6b4 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -14,7 +14,7 @@ ParsedURL parseURL(const std::string & url) { static std::regex uriRegex( "((" + schemeRegex + "):" - + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))" + + "(?:(?://" + authorityRegex + "(" + absPathRegex + "))|(/?" + pathRegex + ")))" + "(?:\\?(" + queryRegex + "))?" + "(?:#(" + queryRegex + "))?", std::regex::ECMAScript); @@ -24,17 +24,33 @@ ParsedURL parseURL(const std::string & url) if (std::regex_match(url, match, uriRegex)) { auto & base = match[1]; std::string scheme = match[2]; - auto authority = match[3].matched - ? std::optional(match[3]) : std::nullopt; - std::string path = match[4].matched ? match[4] : match[5]; - auto & query = match[6]; - auto & fragment = match[7]; + auto authority = match[4].matched + ? std::optional { ParsedURLAuthority { + .user = match[3].matched + ? std::optional { match[3] } + : std::nullopt, + .host = std::string { match[4] }, + .port = match[5].matched + ? std::optional { ({ + size_t p = std::stoul(match[5]); + if (p == 0) + throw BadURL("URL '%s', has port number '%d' which cannot be zero"); + if (p > std::numeric_limits::max()) + throw BadURL("URL '%s', has port number '%d' which is too large"); + p; + }) } + : std::nullopt, + }} + : std::nullopt; + std::string path = match[6].matched ? match[6] : match[7]; + auto & query = match[8]; + auto & fragment = match[9]; auto transportIsFile = parseUrlScheme(scheme).transport == "file"; - if (authority && *authority != "" && transportIsFile) + if (authority && *authority != ParsedURLAuthority {} && transportIsFile) throw BadURL("file:// URL '%s' has unexpected authority '%s'", - url, *authority); + url, authority->to_string()); if (transportIsFile && path.empty()) path = "/"; @@ -121,12 +137,28 @@ std::string encodeQuery(const std::map & ss) return res; } +std::string ParsedURLAuthority::to_string() const +{ + return + (user ? *user + "@" : "") + + host + + (port ? fmt(":%d", *port) : ""); +} + +bool ParsedURLAuthority::operator ==(const ParsedURLAuthority & other) const +{ + return + user == other.user + && host == other.host + && port == other.port; +} + std::string ParsedURL::to_string() const { return scheme + ":" - + (authority ? "//" + *authority : "") + + (authority ? "//" + authority->to_string() : "") + percentEncode(path, allowedInPath) + (query.empty() ? "" : "?" + encodeQuery(query)) + (fragment.empty() ? "" : "#" + percentEncode(fragment)); diff --git a/src/libutil/url.hh b/src/libutil/url.hh index d2413ec0efc..8e805a437bc 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -5,13 +5,24 @@ namespace nix { +struct ParsedURLAuthority +{ + std::optional user; + std::string host; + std::optional port; + + std::string to_string() const; + + bool operator ==(const ParsedURLAuthority & other) const; +}; + struct ParsedURL { std::string url; /// URL without query/fragment std::string base; std::string scheme; - std::optional authority; + std::optional authority; std::string path; std::map query; std::string fragment;