diff --git a/ChangeLog.md b/ChangeLog.md
index 6bf7a3554e..da19d02729 100644
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -11,6 +11,8 @@ Major changes:
 
 Behavior changes:
 
+* cloning git repositories isn't per sub-directory anymore, see [#5411](https://github.com/commercialhaskell/stack/issues/5411)
+
 Other enhancements:
 
 * `stack setup` supports installing GHC for macOS aarch64 (M1)
diff --git a/package.yaml b/package.yaml
index e4188725b6..a70d058dce 100644
--- a/package.yaml
+++ b/package.yaml
@@ -92,7 +92,7 @@ dependencies:
 - network-uri
 - open-browser
 - optparse-applicative >= 0.14.3.0
-- pantry >= 0.5.2
+- pantry >= 0.5.3
 - casa-client
 - casa-types
 - path
diff --git a/src/Stack/Config.hs b/src/Stack/Config.hs
index f822b016ca..b013fd29ed 100644
--- a/src/Stack/Config.hs
+++ b/src/Stack/Config.hs
@@ -6,6 +6,7 @@
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE RecordWildCards #-}
 {-# LANGUAGE TypeFamilies #-}
+{-# LANGUAGE LambdaCase #-}
 
 -- | The general Stack configuration that starts everything off. This should
 -- be smart to falback if there is no stack.yaml, instead relying on
@@ -652,6 +653,14 @@ fillProjectWanted stackYamlFP config project locCache snapCompiler snapPackages
       pp <- mkProjectPackage YesPrintWarnings resolved (boptsHaddock bopts)
       pure (cpName $ ppCommon pp, pp)
 
+    -- prefetch git repos to avoid cloning per subdirectory
+    -- see https://github.com/commercialhaskell/stack/issues/5411
+    let gitRepos = mapMaybe (\case
+                              (RPLImmutable (RPLIRepo repo rpm)) -> Just (repo, rpm)
+                              _ -> Nothing) (projectDependencies project)
+    logDebug ("Prefetching git repos: " <> display (T.pack (show gitRepos)))
+    fetchReposRaw gitRepos
+
     (deps0, mcompleted) <- fmap unzip . forM (projectDependencies project) $ \rpl -> do
       (pl, mCompleted) <- case rpl of
          RPLImmutable rpli -> do
diff --git a/src/Stack/Setup.hs b/src/Stack/Setup.hs
index 95f2f1c376..f186b6ecfd 100644
--- a/src/Stack/Setup.hs
+++ b/src/Stack/Setup.hs
@@ -836,15 +836,8 @@ buildGhcFromSource getSetupInfo' installed (CompilerRepository url) commitId fla
    if compilerTool `elem` installed
      then return (compilerTool,CompilerBuildStandard)
      else do
-       let repo = Repo
-            { repoCommit = commitId
-            , repoUrl    = url
-            , repoType   = RepoGit
-            , repoSubdir = mempty
-            }
-
        -- clone the repository and execute the given commands
-       Pantry.withRepo repo $ do
+       Pantry.withRepo (Pantry.SimpleRepo url commitId RepoGit) $ do
          -- withRepo is guaranteed to set workingDirL, so let's get it
          mcwd <- traverse parseAbsDir =<< view workingDirL
          let cwd = fromMaybe (error "Invalid working directory") mcwd
diff --git a/stack-ghc-88.yaml b/stack-ghc-88.yaml
index 7063887dea..8a0f28891d 100644
--- a/stack-ghc-88.yaml
+++ b/stack-ghc-88.yaml
@@ -32,6 +32,7 @@ extra-deps:
 - casa-types-0.0.1@rev:0
 - rio-0.1.21.0@rev:0
 - rio-prettyprint-0.1.1.0@rev:0
+- pantry-0.5.3@rev:0
 
 drop-packages:
 # See https://github.com/commercialhaskell/stack/pull/4712
diff --git a/stack-nightly.yaml b/stack-nightly.yaml
index 653c94964e..cf38dd33e9 100644
--- a/stack-nightly.yaml
+++ b/stack-nightly.yaml
@@ -20,7 +20,7 @@ ghc-options:
    "$locals": -fhide-source-paths
 
 extra-deps:
-- pantry-0.5.2@rev:0
+- pantry-0.5.3@rev:0
 - rio-0.1.21.0@rev:0
 
 drop-packages:
diff --git a/stack.cabal b/stack.cabal
index 13801cdd69..423971375b 100644
--- a/stack.cabal
+++ b/stack.cabal
@@ -272,7 +272,7 @@ library
     , network-uri
     , open-browser
     , optparse-applicative >=0.14.3.0
-    , pantry >=0.5.2
+    , pantry >=0.5.3
     , path
     , path-io
     , persistent
@@ -395,7 +395,7 @@ executable stack
     , network-uri
     , open-browser
     , optparse-applicative >=0.14.3.0
-    , pantry >=0.5.2
+    , pantry >=0.5.3
     , path
     , path-io
     , persistent
@@ -517,7 +517,7 @@ executable stack-integration-test
     , open-browser
     , optparse-applicative >=0.14.3.0
     , optparse-generic
-    , pantry >=0.5.2
+    , pantry >=0.5.3
     , path
     , path-io
     , persistent
@@ -644,7 +644,7 @@ test-suite stack-test
     , network-uri
     , open-browser
     , optparse-applicative >=0.14.3.0
-    , pantry >=0.5.2
+    , pantry >=0.5.3
     , path
     , path-io
     , persistent
diff --git a/stack.yaml b/stack.yaml
index e2022bb73f..d1f4502f4a 100644
--- a/stack.yaml
+++ b/stack.yaml
@@ -5,6 +5,8 @@ packages:
 
 extra-deps:
 - rio-0.1.21.0@rev:0
+- pantry-0.5.3@rev:0
+
 
 docker:
   enable: false