Skip to content

Commit

Permalink
Git fetcher: Handle submodules for workdirs
Browse files Browse the repository at this point in the history
  • Loading branch information
@edolstra
edolstra committed Oct 31, 2023
1 parent 669b074 commit 0c5eac9
Show file tree
Hide file tree
Showing 4 changed files with 119 additions and 54 deletions.
83 changes: 49 additions & 34 deletions src/libfetchers/git-utils.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,43 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
return toHash(*oid);
}

std::vector<Submodule> parseSubmodules(const CanonPath & configFile)
{
GitConfig config;
if (git_config_open_ondisk(Setter(config), configFile.abs().c_str()))
throw Error("parsing .gitmodules file: %s", git_error_last()->message);

ConfigIterator it;
if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$"))
throw Error("iterating over .gitmodules: %s", git_error_last()->message);

std::map<std::string, std::string> entries;

while (true) {
git_config_entry * entry = nullptr;
if (auto err = git_config_next(&entry, it.get())) {
if (err == GIT_ITEROVER) break;
throw Error("iterating over .gitmodules: %s", git_error_last()->message);
}
entries.emplace(entry->name + 10, entry->value);
}

std::vector<Submodule> result;

for (auto & [key, value] : entries) {
if (!hasSuffix(key, ".path")) continue;
std::string key2(key, 0, key.size() - 5);
auto path = CanonPath(value);
result.push_back(Submodule {
.path = path,
.url = entries[key2 + ".url"],
.branch = entries[key2 + ".branch"],
});
}

return result;
}

WorkdirInfo getWorkdirInfo() override
{
WorkdirInfo info;
Expand Down Expand Up @@ -246,6 +283,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback))
throw Error("getting working directory status: %s", git_error_last()->message);

/* Get submodule info. */
auto modulesFile = path + ".gitmodules";
if (pathExists(modulesFile.abs()))
info.submodules = parseSubmodules(modulesFile);

return info;
}

Expand All @@ -261,7 +303,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
return std::nullopt;
}

std::vector<Submodule> getSubmodules(const Hash & rev) override;
std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev) override;

std::string resolveSubmoduleUrl(const std::string & url) override
{
Expand Down Expand Up @@ -521,52 +563,25 @@ ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev)
return make_ref<GitInputAccessor>(ref<GitRepoImpl>(shared_from_this()), rev);
}

std::vector<GitRepoImpl::Submodule> GitRepoImpl::getSubmodules(const Hash & rev)
std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules(const Hash & rev)
{
/* Read the .gitmodules files from this revision. */
CanonPath modulesFile(".gitmodules");

auto accessor = getAccessor(rev);
if (!accessor->pathExists(modulesFile)) return {};

/* Parse it. */
/* Parse it and get the revision of each submodule. */
auto configS = accessor->readFile(modulesFile);

auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules");
writeFull(fdTemp.get(), configS);

GitConfig config;
if (git_config_open_ondisk(Setter(config), pathTemp.c_str()))
throw Error("parsing .gitmodules file: %s", git_error_last()->message);

ConfigIterator it;
if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$"))
throw Error("iterating over .gitmodules: %s", git_error_last()->message);

std::map<std::string, std::string> entries;

while (true) {
git_config_entry * entry = nullptr;
if (auto err = git_config_next(&entry, it.get())) {
if (err == GIT_ITEROVER) break;
throw Error("iterating over .gitmodules: %s", git_error_last()->message);
}
entries.emplace(entry->name + 10, entry->value);
}
std::vector<std::tuple<Submodule, Hash>> result;

std::vector<Submodule> result;

for (auto & [key, value] : entries) {
if (!hasSuffix(key, ".path")) continue;
std::string key2(key, 0, key.size() - 5);
auto path = CanonPath(value);
auto rev = accessor.dynamic_pointer_cast<GitInputAccessor>()->getSubmoduleRev(path);
result.push_back(Submodule {
.path = path,
.url = entries[key2 + ".url"],
.branch = entries[key2 + ".branch"],
.rev = rev,
});
for (auto & submodule : parseSubmodules(CanonPath(pathTemp))) {
auto rev = accessor.dynamic_pointer_cast<GitInputAccessor>()->getSubmoduleRev(submodule.path);
result.push_back({std::move(submodule), rev});
}

return result;
Expand Down
27 changes: 18 additions & 9 deletions src/libfetchers/git-utils.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,16 @@ struct GitRepo
/* Return the commit hash to which a ref points. */
virtual Hash resolveRef(std::string ref) = 0;

/**
* Info about a submodule.
*/
struct Submodule
{
CanonPath path;
std::string url;
std::string branch;
};

struct WorkdirInfo
{
bool isDirty = false;
Expand All @@ -31,22 +41,21 @@ struct GitRepo
/* All files in the working directory that are unchanged,
modified or added, but excluding deleted files. */
std::set<CanonPath> files;

/* The submodules listed in .gitmodules of this workdir. */
std::vector<Submodule> submodules;
};

virtual WorkdirInfo getWorkdirInfo() = 0;

/* Get the ref that HEAD points to. */
virtual std::optional<std::string> getWorkdirRef() = 0;

struct Submodule
{
CanonPath path;
std::string url;
std::string branch;
Hash rev;
};

virtual std::vector<Submodule> getSubmodules(const Hash & rev) = 0;
/**
* Return the submodules of this repo at the indicated revision,
* along with the revision of each submodule.
*/
virtual std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev) = 0;

virtual std::string resolveSubmoduleUrl(const std::string & url) = 0;

Expand Down
49 changes: 41 additions & 8 deletions src/libfetchers/git.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -525,16 +525,16 @@ struct GitInputScheme : InputScheme
if (repoInfo.submodules) {
std::map<CanonPath, nix::ref<InputAccessor>> mounts;

for (auto & submodule : repo->getSubmodules(rev)) {
for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) {
auto resolved = repo->resolveSubmoduleUrl(submodule.url);
debug("Git submodule %s: %s %s %s -> %s",
submodule.path, submodule.url, submodule.branch, submodule.rev.gitRev(), resolved);
submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved);
fetchers::Attrs attrs;
attrs.insert_or_assign("type", "git");
attrs.insert_or_assign("url", resolved);
if (submodule.branch != "")
attrs.insert_or_assign("ref", submodule.branch);
attrs.insert_or_assign("rev", submodule.rev.gitRev());
attrs.insert_or_assign("rev", submoduleRev.gitRev());
auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
auto [submoduleAccessor, submoduleInput2] =
submoduleInput.scheme->getAccessor(store, submoduleInput);
Expand All @@ -557,9 +557,45 @@ struct GitInputScheme : InputScheme
}

std::pair<ref<InputAccessor>, Input> getAccessorFromWorkdir(
ref<Store> store,
RepoInfo & repoInfo,
Input && input) const
{
if (repoInfo.submodules)
/* Create mountpoints for the submodules. */
for (auto & submodule : repoInfo.workdirInfo.submodules)
repoInfo.workdirInfo.files.insert(submodule.path);

ref<InputAccessor> accessor =
makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url));

/* If the repo has submodules, return a union input accessor
consisting of the accessor for the top-level repo and the
accessors for the submodule workdirs. */
if (repoInfo.submodules && !repoInfo.workdirInfo.submodules.empty()) {
std::map<CanonPath, nix::ref<InputAccessor>> mounts;

for (auto & submodule : repoInfo.workdirInfo.submodules) {
auto submodulePath = CanonPath(repoInfo.url) + submodule.path;
fetchers::Attrs attrs;
attrs.insert_or_assign("type", "git");
attrs.insert_or_assign("url", submodulePath.abs());
auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
auto [submoduleAccessor, submoduleInput2] =
submoduleInput.scheme->getAccessor(store, submoduleInput);

/* If the submodule is dirty, mark this repo dirty as
well. */
if (!submoduleInput2.getRev())
repoInfo.workdirInfo.isDirty = true;

mounts.insert_or_assign(submodule.path, submoduleAccessor);
}

mounts.insert_or_assign(CanonPath::root, accessor);
accessor = makeUnionInputAccessor(std::move(mounts));
}

if (!repoInfo.workdirInfo.isDirty) {
if (auto ref = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef())
input.attrs.insert_or_assign("ref", *ref);
Expand Down Expand Up @@ -588,10 +624,7 @@ struct GitInputScheme : InputScheme

input.locked = true; // FIXME

return {
makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url)),
std::move(input)
};
return {accessor, std::move(input)};
}

std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
Expand All @@ -603,7 +636,7 @@ struct GitInputScheme : InputScheme
if (input.getRef() || input.getRev() || !repoInfo.isLocal)
return getAccessorFromCommit(store, repoInfo, std::move(input));
else
return getAccessorFromWorkdir(repoInfo, std::move(input));
return getAccessorFromWorkdir(store, repoInfo, std::move(input));
}
};

Expand Down
14 changes: 11 additions & 3 deletions tests/functional/flakes/flake-in-submodule.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,16 @@ echo '"expression in root repo"' > $rootRepo/root.nix
git -C $rootRepo add root.nix
git -C $rootRepo commit -m "Add root.nix"

# FIXME
flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule

# Flake can live inside a submodule and can be accessed via ?dir=submodule
#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#sub ) = '"expression in submodule"' ]]
[[ $(nix eval --json $flakeref#sub ) = '"expression in submodule"' ]]

# The flake can access content outside of the submodule
#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#root ) = '"expression in root repo"' ]]
[[ $(nix eval --json $flakeref#root ) = '"expression in root repo"' ]]

# Check that dirtying a submodule makes the entire thing dirty.
[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) != null ]]
echo '"foo"' > $rootRepo/submodule/sub.nix
[[ $(nix eval --json $flakeref#sub ) = '"foo"' ]]
[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) = null ]]

0 comments on commit 0c5eac9

Please sign in to comment.