From 97ff4ad34612eed56f1dc6c6aaee19617e45e2be Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 20 Jun 2018 12:19:38 -0400 Subject: [PATCH] cmd/go/internal/modfetch: fix Lookup, Import; add ImportRepoRev - Explain Lookup and Import, module paths vs import paths. - Add ImportRepoRev, because neither Lookup nor Import is appropriate for translating legacy versioning system configs. There's more to do in ImportRepoRev, but it's already more accurate than before. - Drop the old ad-hoc logic about github/bitbucket/etc in favor of resolving source code hosts using the "go get" logic. This enables paths like git.apache.org/thrift.git/lib/go/thrift again. - Structure $GOPATH/src/mod/cache a little more: make separate top-level directories mod/cache/download and mod/cache/vcs. Fixes golang/go#23983. Fixes golang/go#24687. Fixes golang/go#25590. Fixes golang/go#25654. Change-Id: I0de2d75c5846a1c054079e1c67b9a1100ed161c8 Reviewed-on: https://go-review.googlesource.com/120042 Run-TryBot: Russ Cox TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- .../go/internal/modfetch/bitbucket/fetch.go | 23 -- vendor/cmd/go/internal/modfetch/coderepo.go | 2 +- .../cmd/go/internal/modfetch/coderepo_test.go | 35 +-- vendor/cmd/go/internal/modfetch/convert.go | 23 +- vendor/cmd/go/internal/modfetch/domain.go | 178 -------------- .../cmd/go/internal/modfetch/github/fetch.go | 25 -- vendor/cmd/go/internal/modfetch/gopkgin.go | 23 -- vendor/cmd/go/internal/modfetch/repo.go | 221 ++++++++++++++++-- vendor/cmd/go/internal/vgo/fetch.go | 12 +- vendor/cmd/go/internal/vgo/init.go | 4 +- vendor/cmd/go/vgo_test.go | 8 +- 11 files changed, 238 insertions(+), 316 deletions(-) delete mode 100644 vendor/cmd/go/internal/modfetch/bitbucket/fetch.go delete mode 100644 vendor/cmd/go/internal/modfetch/domain.go delete mode 100644 vendor/cmd/go/internal/modfetch/github/fetch.go delete mode 100644 vendor/cmd/go/internal/modfetch/gopkgin.go diff --git a/vendor/cmd/go/internal/modfetch/bitbucket/fetch.go b/vendor/cmd/go/internal/modfetch/bitbucket/fetch.go deleted file mode 100644 index 58e74a5..0000000 --- a/vendor/cmd/go/internal/modfetch/bitbucket/fetch.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package bitbucket - -import ( - "fmt" - "strings" - - "cmd/go/internal/modfetch/codehost" - "cmd/go/internal/modfetch/gitrepo" -) - -func Lookup(path string) (codehost.Repo, string, error) { - f := strings.Split(path, "/") - if len(f) < 3 || f[0] != "bitbucket.org" { - return nil, "", fmt.Errorf("bitbucket repo must be bitbucket.org/org/project") - } - path = f[0] + "/" + f[1] + "/" + f[2] - repo, err := gitrepo.Repo("https://" + path) - return repo, path, err -} diff --git a/vendor/cmd/go/internal/modfetch/coderepo.go b/vendor/cmd/go/internal/modfetch/coderepo.go index 17c8b92..dd0814b 100644 --- a/vendor/cmd/go/internal/modfetch/coderepo.go +++ b/vendor/cmd/go/internal/modfetch/coderepo.go @@ -293,7 +293,7 @@ func (r *codeRepo) legacyGoMod(rev, dir string) []byte { if convert == nil { continue } - if err := ConvertLegacyConfig(mf, file, data); err != nil { + if err := ConvertLegacyConfig(mf, path.Join(r.codeRoot+"@"+rev, dir, file), data); err != nil { continue } break diff --git a/vendor/cmd/go/internal/modfetch/coderepo_test.go b/vendor/cmd/go/internal/modfetch/coderepo_test.go index c8233a5..91e9a27 100644 --- a/vendor/cmd/go/internal/modfetch/coderepo_test.go +++ b/vendor/cmd/go/internal/modfetch/coderepo_test.go @@ -243,8 +243,10 @@ var codeRepoTests = []struct { }, { // package in subdirectory - custom domain - path: "golang.org/x/net/context", - lookerr: "module root is \"golang.org/x/net\"", + // In general we can't reject these definitively in Lookup, + // but gopkg.in is special. + path: "gopkg.in/yaml.v2/abc", + lookerr: "invalid module path \"gopkg.in/yaml.v2/abc\"", }, { // package in subdirectory - github @@ -329,13 +331,13 @@ func TestCodeRepo(t *testing.T) { for _, tt := range codeRepoTests { t.Run(strings.Replace(tt.path, "/", "_", -1)+"/"+tt.rev, func(t *testing.T) { repo, err := Lookup(tt.path) - if err != nil { - if tt.lookerr != "" { - if err.Error() == tt.lookerr { - return - } - t.Errorf("Lookup(%q): %v, want error %q", tt.path, err, tt.lookerr) + if tt.lookerr != "" { + if err != nil && err.Error() == tt.lookerr { + return } + t.Errorf("Lookup(%q): %v, want error %q", tt.path, err, tt.lookerr) + } + if err != nil { t.Fatalf("Lookup(%q): %v", tt.path, err) } if tt.mpath == "" { @@ -442,7 +444,8 @@ var importTests = []struct { }, { path: "golang.org/x/foo/bar", - err: "unknown module golang.org/x/foo/bar: no go-import tags", + // TODO(rsc): This error comes from old go get and is terrible. Fix. + err: `unrecognized import path "golang.org/x/foo/bar" (parse https://golang.org/x/foo/bar?go-get=1: no go-import meta tags ())`, }, } @@ -452,14 +455,14 @@ func TestImport(t *testing.T) { for _, tt := range importTests { t.Run(strings.Replace(tt.path, "/", "_", -1), func(t *testing.T) { repo, info, err := Import(tt.path, nil) - if err != nil { - if tt.err != "" { - if err.Error() == tt.err { - return - } - t.Errorf("Import(%q): %v, want error %q", tt.path, err, tt.err) + if tt.err != "" { + if err != nil && err.Error() == tt.err { + return } - t.Fatalf("Lookup(%q): %v", tt.path, err) + t.Fatalf("Import(%q): %v, want error %q", tt.path, err, tt.err) + } + if err != nil { + t.Fatalf("Import(%q): %v", tt.path, err) } if mpath := repo.ModulePath(); mpath != tt.mpath { t.Errorf("repo.ModulePath() = %q (%v), want %q", mpath, info.Version, tt.mpath) diff --git a/vendor/cmd/go/internal/modfetch/convert.go b/vendor/cmd/go/internal/modfetch/convert.go index bebb6ee..e12d200 100644 --- a/vendor/cmd/go/internal/modfetch/convert.go +++ b/vendor/cmd/go/internal/modfetch/convert.go @@ -33,7 +33,7 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error { if convert == nil { return fmt.Errorf("unknown legacy config file %s", file) } - result, err := convert(file, data) + mf, err := convert(file, data) if err != nil { return fmt.Errorf("parsing %s: %v", file, err) } @@ -41,20 +41,12 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error { // Convert requirements block, which may use raw SHA1 hashes as versions, // to valid semver requirement list, respecting major versions. var work par.Work - for _, r := range result.Require { + for _, r := range mf.Require { m := r.Mod if m.Path == "" { continue } - - // TODO: Something better here. - if strings.HasPrefix(m.Path, "github.com/") || strings.HasPrefix(m.Path, "golang.org/x/") { - f := strings.Split(m.Path, "/") - if len(f) > 3 { - m.Path = strings.Join(f[:3], "/") - } - } - work.Add(m) + work.Add(r.Mod) } var ( @@ -63,13 +55,14 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error { ) work.Do(10, func(item interface{}) { r := item.(module.Version) - info, err := Stat(r.Path, r.Version) + repo, info, err := ImportRepoRev(r.Path, r.Version) if err != nil { - fmt.Fprintf(os.Stderr, "vgo: stat %s@%s: %v\n", r.Path, r.Version, err) + fmt.Fprintf(os.Stderr, "vgo: converting %s: stat %s@%s: %v\n", file, r.Path, r.Version, err) return } mu.Lock() - need[r.Path] = semver.Max(need[r.Path], info.Version) + path := repo.ModulePath() + need[path] = semver.Max(need[path], info.Version) mu.Unlock() }) @@ -82,7 +75,7 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error { f.AddNewRequire(path, need[path]) } - for _, r := range result.Replace { + for _, r := range mf.Replace { err := f.AddReplace(r.Old.Path, r.Old.Version, r.New.Path, r.New.Version) if err != nil { return fmt.Errorf("add replace: %v", err) diff --git a/vendor/cmd/go/internal/modfetch/domain.go b/vendor/cmd/go/internal/modfetch/domain.go deleted file mode 100644 index c1e6813..0000000 --- a/vendor/cmd/go/internal/modfetch/domain.go +++ /dev/null @@ -1,178 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Support for custom domains. - -package modfetch - -import ( - "encoding/xml" - "fmt" - "io" - "net/url" - "os" - "strings" - - "cmd/go/internal/modfetch/codehost" - "cmd/go/internal/modfetch/gitrepo" -) - -// metaImport represents the parsed tags from HTML files. -type metaImport struct { - Prefix, VCS, RepoRoot string -} - -func lookupCustomDomain(path string) (Repo, error) { - dom := path - if i := strings.Index(dom, "/"); i >= 0 { - dom = dom[:i] - } - if !strings.Contains(dom, ".") { - return nil, fmt.Errorf("unknown module %s: not a domain name", path) - } - var body io.ReadCloser - err := webGetGoGet("https://"+path+"?go-get=1", &body) - if body != nil { - defer body.Close() - } - if err != nil { - fmt.Fprintf(os.Stderr, "FindRepo: %v\n", err) - return nil, err - } - // Note: accepting a non-200 OK here, so people can serve a - // meta import in their http 404 page. - imports, err := parseMetaGoImports(body) - if err != nil { - fmt.Fprintf(os.Stderr, "findRepo: %v\n", err) - return nil, err - } - if len(imports) == 0 { - return nil, fmt.Errorf("unknown module %s: no go-import tags", path) - } - - // First look for new module definition. - for _, imp := range imports { - if path == imp.Prefix || strings.HasPrefix(path, imp.Prefix+"/") { - if imp.VCS == "mod" { - u, err := url.Parse(imp.RepoRoot) - if err != nil { - return nil, fmt.Errorf("invalid module URL %q", imp.RepoRoot) - } else if u.Scheme != "https" { - // TODO: Allow -insecure flag as a build flag? - return nil, fmt.Errorf("invalid module URL %q: must be HTTPS", imp.RepoRoot) - } - return newProxyRepo(imp.RepoRoot, imp.Prefix), nil - } - } - } - - // Fall back to redirections to known version control systems. - for _, imp := range imports { - if path == imp.Prefix { - if !strings.HasPrefix(imp.RepoRoot, "https://") { - // TODO: Allow -insecure flag as a build flag? - return nil, fmt.Errorf("invalid server URL %q: must be HTTPS", imp.RepoRoot) - } - if imp.VCS == "git" { - code, err := gitrepo.Repo(imp.RepoRoot) - if err != nil { - return nil, err - } - return newCodeRepo(code, imp.Prefix, path) - } - return nil, fmt.Errorf("unknown VCS, Repo: %s, %s", imp.VCS, imp.RepoRoot) - } - } - - // Check for redirect to repo root. - for _, imp := range imports { - if strings.HasPrefix(path, imp.Prefix+"/") { - return nil, &ModuleSubdirError{imp.Prefix} - } - } - - return nil, fmt.Errorf("unknown module %s: no matching go-import tags", path) -} - -type ModuleSubdirError struct { - ModulePath string -} - -func (e *ModuleSubdirError) Error() string { - return fmt.Sprintf("module root is %q", e.ModulePath) -} - -type customPrefix struct { - codehost.Repo - root string -} - -func (c *customPrefix) Root() string { - return c.root -} - -// parseMetaGoImports returns meta imports from the HTML in r. -// Parsing ends at the end of the section or the beginning of the . -func parseMetaGoImports(r io.Reader) (imports []metaImport, err error) { - d := xml.NewDecoder(r) - d.CharsetReader = charsetReader - d.Strict = false - var t xml.Token - for { - t, err = d.RawToken() - if err != nil { - if err == io.EOF || len(imports) > 0 { - err = nil - } - return - } - if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") { - return - } - if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") { - return - } - e, ok := t.(xml.StartElement) - if !ok || !strings.EqualFold(e.Name.Local, "meta") { - continue - } - if attrValue(e.Attr, "name") != "go-import" { - continue - } - if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 { - imports = append(imports, metaImport{ - Prefix: f[0], - VCS: f[1], - RepoRoot: f[2], - }) - } - } -} - -// attrValue returns the attribute value for the case-insensitive key -// `name', or the empty string if nothing is found. -func attrValue(attrs []xml.Attr, name string) string { - for _, a := range attrs { - if strings.EqualFold(a.Name.Local, name) { - return a.Value - } - } - return "" -} - -// charsetReader returns a reader for the given charset. Currently -// it only supports UTF-8 and ASCII. Otherwise, it returns a meaningful -// error which is printed by go get, so the user can find why the package -// wasn't downloaded if the encoding is not supported. Note that, in -// order to reduce potential errors, ASCII is treated as UTF-8 (i.e. characters -// greater than 0x7f are not rejected). -func charsetReader(charset string, input io.Reader) (io.Reader, error) { - switch strings.ToLower(charset) { - case "ascii": - return input, nil - default: - return nil, fmt.Errorf("can't decode XML document using charset %q", charset) - } -} diff --git a/vendor/cmd/go/internal/modfetch/github/fetch.go b/vendor/cmd/go/internal/modfetch/github/fetch.go deleted file mode 100644 index bf5c720..0000000 --- a/vendor/cmd/go/internal/modfetch/github/fetch.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package github - -import ( - "fmt" - "strings" - - "cmd/go/internal/modfetch/codehost" - "cmd/go/internal/modfetch/gitrepo" -) - -// Lookup returns the code repository enclosing the given module path, -// which must begin with github.com/. -func Lookup(path string) (codehost.Repo, string, error) { - f := strings.Split(path, "/") - if len(f) < 3 || f[0] != "github.com" { - return nil, "", fmt.Errorf("github repo must be github.com/org/project") - } - path = f[0] + "/" + f[1] + "/" + f[2] - repo, err := gitrepo.Repo("https://" + path) - return repo, path, err -} diff --git a/vendor/cmd/go/internal/modfetch/gopkgin.go b/vendor/cmd/go/internal/modfetch/gopkgin.go deleted file mode 100644 index f79edc0..0000000 --- a/vendor/cmd/go/internal/modfetch/gopkgin.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// TODO: Figure out what gopkg.in should do. - -package modfetch - -import ( - "cmd/go/internal/modfetch/codehost" - "cmd/go/internal/modfetch/gitrepo" - "cmd/go/internal/modfile" - "fmt" -) - -func gopkginLookup(path string) (codehost.Repo, string, error) { - root, _, _, _, ok := modfile.ParseGopkgIn(path) - if !ok { - return nil, "", fmt.Errorf("invalid gopkg.in/ path: %q", path) - } - repo, err := gitrepo.Repo("https://" + root) - return repo, root, err -} diff --git a/vendor/cmd/go/internal/modfetch/repo.go b/vendor/cmd/go/internal/modfetch/repo.go index 3542dcb..2de9225 100644 --- a/vendor/cmd/go/internal/modfetch/repo.go +++ b/vendor/cmd/go/internal/modfetch/repo.go @@ -5,22 +5,20 @@ package modfetch import ( - "errors" "fmt" "os" pathpkg "path" "sort" - "strings" "time" "cmd/go/internal/cfg" - "cmd/go/internal/modfetch/bitbucket" + "cmd/go/internal/get" "cmd/go/internal/modfetch/codehost" - "cmd/go/internal/modfetch/github" - "cmd/go/internal/modfetch/googlesource" + "cmd/go/internal/modfetch/gitrepo" "cmd/go/internal/module" "cmd/go/internal/par" "cmd/go/internal/semver" + web "cmd/go/internal/web" ) const traceRepo = false // trace all repo actions, for debugging @@ -65,9 +63,124 @@ type RevInfo struct { Time time.Time // commit time } +// Re: module paths, import paths, repository roots, and lookups +// +// A module is a collection of Go packages stored in a file tree +// with a go.mod file at the root of the tree. +// The go.mod defines the module path, which is the import path +// corresponding to the root of the file tree. +// The import path of a directory within that file tree is the module path +// joined with the name of the subdirectory relative to the root. +// +// For example, the module with path rsc.io/qr corresponds to the +// file tree in the repository https://github.com/rsc/qr. +// That file tree has a go.mod that says "module rsc.io/qr". +// The package in the root directory has import path "rsc.io/qr". +// The package in the gf256 subdirectory has import path "rsc.io/qr/gf256". +// In this example, "rsc.io/qr" is both a module path and an import path. +// But "rsc.io/qr/gf256" is only an import path, not a module path: +// it names an importable package, but not a module. +// +// As a special case to incorporate code written before modules were +// introduced, if a path p resolves using the pre-module "go get" lookup +// to the root of a source code repository without a go.mod file, +// that repository is treated as if it had a go.mod in its root directory +// declaring module path p. (The go.mod is further considered to +// contain requirements corresponding to any legacy version +// tracking format such as Gopkg.lock, vendor/vendor.conf, and so on.) +// +// The presentation so far ignores the fact that a source code repository +// has many different versions of a file tree, and those versions may +// differ in whether a particular go.mod exists and what it contains. +// In fact there is a well-defined mapping only from a module path, version +// pair - often written path@version - to a particular file tree. +// For example rsc.io/qr@v0.1.0 depends on the "implicit go.mod at root of +// repository" rule, while rsc.io/qr@v0.2.0 has an explicit go.mod. +// Because the "go get" import paths rsc.io/qr and github.com/rsc/qr +// both redirect to the Git repository https://github.com/rsc/qr, +// github.com/rsc/qr@v0.1.0 is the same file tree as rsc.io/qr@v0.1.0 +// but a different module (a different name). In contrast, since v0.2.0 +// of that repository has an explicit go.mod that declares path rsc.io/qr, +// github.com/rsc/qr@v0.2.0 is an invalid module path, version pair. +// Before modules, import comments would have had the same effect. +// +// The set of import paths associated with a given module path is +// clearly not fixed: at the least, new directories with new import paths +// can always be added. But another potential operation is to split a +// subtree out of a module into its own module. If done carefully, +// this operation can be done while preserving compatibility for clients. +// For example, suppose that we want to split rsc.io/qr/gf256 into its +// own module, so that there would be two modules rsc.io/qr and rsc.io/qr/gf256. +// Then we can simultaneously issue rsc.io/qr v0.3.0 (dropping the gf256 subdirectory) +// and rsc.io/qr/gf256 v0.1.0, including in their respective go.mod +// cyclic requirements pointing at each other: rsc.io/qr v0.3.0 requires +// rsc.io/qr/gf256 v0.1.0 and vice versa. Then a build can be +// using an older rsc.io/qr module that includes the gf256 package, but if +// it adds a requirement on either the newer rsc.io/qr or the newer +// rsc.io/qr/gf256 module, it will automatically add the requirement +// on the complementary half, ensuring both that rsc.io/qr/gf256 is +// available for importing by the build and also that it is only defined +// by a single module. The gf256 package could move back into the +// original by another simultaneous release of rsc.io/qr v0.4.0 including +// the gf256 subdirectory and an rsc.io/qr/gf256 v0.2.0 with no code +// in its root directory, along with a new requirement cycle. +// The ability to shift module boundaries in this way is expected to be +// important in large-scale program refactorings, similar to the ones +// described in https://talks.golang.org/2016/refactor.article. +// +// The possibility of shifting module boundaries reemphasizes +// that you must know both the module path and its version +// to determine the set of packages provided directly by that module. +// +// On top of all this, it is possible for a single code repository +// to contain multiple modules, either in branches or subdirectories, +// as a limited kind of monorepo. For example rsc.io/qr/v2, +// the v2.x.x continuation of rsc.io/qr, is expected to be found +// in v2-tagged commits in https://github.com/rsc/qr, either +// in the root or in a v2 subdirectory, disambiguated by go.mod. +// Again the precise file tree corresponding to a module +// depends on which version we are considering. +// +// It is also possible for the underlying repository to change over time, +// without changing the module path. If I copy the github repo over +// to https://bitbucket.org/rsc/qr and update https://rsc.io/qr?go-get=1, +// then clients of all versions should start fetching from bitbucket +// instead of github. That is, in contrast to the exact file tree, +// the location of the source code repository associated with a module path +// does not depend on the module version. (This is by design, as the whole +// point of these redirects is to allow package authors to establish a stable +// name that can be updated as code moves from one service to another.) +// +// All of this is important background for the lookup APIs defined in this +// file. +// +// The Lookup function takes a module path and returns a Repo representing +// that module path. Lookup can do only a little with the path alone. +// It can check that the path is well-formed (see semver.CheckPath) +// and it can check that the path can be resolved to a target repository. +// To avoid version control access except when absolutely necessary, +// Lookup does not attempt to connect to the repository itself. +// +// The Import function takes an import path found in source code and +// determines which module to add to the requirement list to satisfy +// that import. It checks successive truncations of the import path +// to determine possible modules and stops when it finds a module +// in which the latest version satisfies the import path. +// +// The ImportRepoRev function is a variant of Import which is limited +// to code in a source code repository at a particular revision identifier +// (usually a commit hash or source code repository tag, not necessarily +// a module version). +// ImportRepoRev is used when converting legacy dependency requirements +// from older systems into go.mod files. Those older systems worked +// at either package or repository granularity, and most of the time they +// recorded commit hashes, not tagged versions. + var lookupCache par.Cache // Lookup returns the module with the given module path. +// A successful return does not guarantee that the module +// has any defined versions. func Lookup(path string) (Repo, error) { if traceRepo { defer logCall("Lookup(%q)", path)() @@ -99,19 +212,49 @@ func lookup(path string) (r Repo, err error) { if proxyURL != "" { return lookupProxy(path) } - if code, root, err := lookupCodeHost(path, false); err != errNotHosted { - if err != nil { - return nil, err - } - return newCodeRepo(code, root, path) + + rr, err := get.RepoRootForImportPath(path, get.PreferMod, web.Secure) + if err != nil { + // We don't know where to find code for a module with this path. + return nil, err + } + + if rr.VCS == "mod" { + // Fetch module from proxy with base URL rr.Repo. + return newProxyRepo(rr.Repo, path), nil } - return lookupCustomDomain(path) + + code, err := lookupCodeRepo(rr) + if err != nil { + return nil, err + } + return newCodeRepo(code, rr.Root, path) } +func lookupCodeRepo(rr *get.RepoRoot) (codehost.Repo, error) { + switch rr.VCS { + default: + return nil, fmt.Errorf("lookup %s: unknown VCS %s %s", rr.Root, rr.VCS, rr.Repo) + case "git": + return gitrepo.Repo(rr.Repo) + // TODO: "hg", "svn", "bzr", "fossil" + } +} + +// Import returns the module repo and version to use to satisfy the given import path. +// It considers a sequence of module paths starting with the import path and +// removing successive path elements from the end. It stops when it finds a module +// path for which the latest version of the module provides the expected package. +// If non-nil, the allowed function is used to filter known versions of a given module +// before determining which one is "latest". func Import(path string, allowed func(module.Version) bool) (Repo, *RevInfo, error) { + if cfg.BuildGetmode != "" { + return nil, nil, fmt.Errorf("import resolution disabled by -getmode=%s", cfg.BuildGetmode) + } if traceRepo { defer logCall("Import(%q, ...)", path)() } + try := func(path string) (Repo, *RevInfo, error) { r, err := Lookup(path) if err != nil { @@ -125,9 +268,14 @@ func Import(path string, allowed func(module.Version) bool) (Repo, *RevInfo, err if err != nil { return nil, nil, err } + // TODO(rsc): Do what the docs promise: download the module + // source code and check that it actually contains code for the + // target import path. To do that efficiently we will need to move + // the unzipped code cache out of ../vgo into this package. return r, info, nil } + // Find enclosing module by walking up path element by element. var firstErr error for { r, info, err := try(path) @@ -146,20 +294,47 @@ func Import(path string, allowed func(module.Version) bool) (Repo, *RevInfo, err return nil, nil, firstErr } -var errNotHosted = errors.New("not hosted") +// ImportRepoRev returns the module and version to use to access +// the given import path loaded from the source code repository that +// the original "go get" would have used, at the specific repository revision +// (typically a commit hash, but possibly also a source control tag). +func ImportRepoRev(path, rev string) (Repo, *RevInfo, error) { + if cfg.BuildGetmode != "" { + return nil, nil, fmt.Errorf("repo version lookup disabled by -getmode=%s", cfg.BuildGetmode) + } + + // Note: Because we are converting a code reference from a legacy + // version control system, we ignore meta tags about modules + // and use only direct source control entries (get.IgnoreMod). + rr, err := get.RepoRootForImportPath(path, get.IgnoreMod, web.Secure) + if err != nil { + return nil, nil, err + } + + code, err := lookupCodeRepo(rr) + if err != nil { + return nil, nil, err + } + + revInfo, err := code.Stat(rev) + if err != nil { + return nil, nil, err + } + + // TODO: Look in repo to find path, check for go.mod files. + // For now we're just assuming rr.Root is the module path, + // which is true in the absence of go.mod files. + + repo, err := newCodeRepo(code, rr.Root, rr.Root) + if err != nil { + return nil, nil, err + } -func lookupCodeHost(path string, customDomain bool) (codehost.Repo, string, error) { - switch { - case strings.HasPrefix(path, "github.com/"): - return github.Lookup(path) - case strings.HasPrefix(path, "bitbucket.org/"): - return bitbucket.Lookup(path) - case customDomain && strings.HasSuffix(path[:strings.Index(path, "/")+1], ".googlesource.com/"): - return googlesource.Lookup(path) - case strings.HasPrefix(path, "gopkg.in/"): - return gopkginLookup(path) + info, err := repo.(*codeRepo).convert(revInfo) + if err != nil { + return nil, nil, err } - return nil, "", errNotHosted + return repo, info, nil } func SortVersions(list []string) { diff --git a/vendor/cmd/go/internal/vgo/fetch.go b/vendor/cmd/go/internal/vgo/fetch.go index 7b91908..0723dfa 100644 --- a/vendor/cmd/go/internal/vgo/fetch.go +++ b/vendor/cmd/go/internal/vgo/fetch.go @@ -40,14 +40,14 @@ func fetch(mod module.Version) (dir string, err error) { modpath := mod.Path + "@" + mod.Version dir = filepath.Join(SrcMod, modpath) if files, _ := ioutil.ReadDir(dir); len(files) == 0 { - zipfile := filepath.Join(SrcMod, "cache", mod.Path, "@v", mod.Version+".zip") + zipfile := filepath.Join(SrcMod, "cache/download", mod.Path, "@v", mod.Version+".zip") if _, err := os.Stat(zipfile); err == nil { // Use it. - // This should only happen if the v/cache directory is preinitialized - // or if src/v/modpath was removed but not src/v/cache. + // This should only happen if the mod/cache directory is preinitialized + // or if src/mod/path was removed but not src/mod/cache/download. fmt.Fprintf(os.Stderr, "vgo: extracting %s %s\n", mod.Path, mod.Version) } else { - if err := os.MkdirAll(filepath.Join(SrcMod, "cache", mod.Path, "@v"), 0777); err != nil { + if err := os.MkdirAll(filepath.Join(SrcMod, "cache/download", mod.Path, "@v"), 0777); err != nil { return "", err } fmt.Fprintf(os.Stderr, "vgo: downloading %s %s\n", mod.Path, mod.Version) @@ -159,7 +159,7 @@ func checkModHash(mod module.Version) { return } - data, err := ioutil.ReadFile(filepath.Join(SrcMod, "cache", mod.Path, "@v", mod.Version+".ziphash")) + data, err := ioutil.ReadFile(filepath.Join(SrcMod, "cache/download", mod.Path, "@v", mod.Version+".ziphash")) if err != nil { base.Fatalf("vgo: verifying %s %s: %v", mod.Path, mod.Version, err) } @@ -183,7 +183,7 @@ func checkModHash(mod module.Version) { } func findModHash(mod module.Version) string { - data, err := ioutil.ReadFile(filepath.Join(SrcMod, "cache", mod.Path, "@v", mod.Version+".ziphash")) + data, err := ioutil.ReadFile(filepath.Join(SrcMod, "cache/download", mod.Path, "@v", mod.Version+".ziphash")) if err != nil { return "" } diff --git a/vendor/cmd/go/internal/vgo/init.go b/vendor/cmd/go/internal/vgo/init.go index 94741bb..e66ce72 100644 --- a/vendor/cmd/go/internal/vgo/init.go +++ b/vendor/cmd/go/internal/vgo/init.go @@ -188,8 +188,8 @@ func InitMod() { os.Rename(srcV, SrcMod) } - modfetch.CacheRoot = filepath.Join(SrcMod, "cache") - codehost.WorkRoot = filepath.Join(SrcMod, "cache/vcswork") + modfetch.CacheRoot = filepath.Join(SrcMod, "cache/download") + codehost.WorkRoot = filepath.Join(SrcMod, "cache/vcs") if CmdModInit { // Running go mod -init: do legacy module conversion diff --git a/vendor/cmd/go/vgo_test.go b/vendor/cmd/go/vgo_test.go index ef911bc..dcd90be 100644 --- a/vendor/cmd/go/vgo_test.go +++ b/vendor/cmd/go/vgo_test.go @@ -354,9 +354,9 @@ func TestVgoBadDomain(t *testing.T) { tg.cd(filepath.Join(wd, "testdata/badmod")) tg.runFail("-vgo", "get", "appengine") - tg.grepStderr("unknown module appengine: not a domain name", "expected domain error") + tg.grepStderr(`unrecognized import path \"appengine\"`, "expected appengine error ") tg.runFail("-vgo", "get", "x/y.z") - tg.grepStderr("unknown module x/y.z: not a domain name", "expected domain error") + tg.grepStderr(`unrecognized import path \"x/y.z\" \(import path does not begin with hostname\)`, "expected domain error") tg.runFail("-vgo", "build") tg.grepStderrNot("unknown module appengine: not a domain name", "expected nothing about appengine") @@ -485,7 +485,7 @@ func TestVgoVendor(t *testing.T) { tg.grepStdout(`vendormod[/\\]w`, "expected w in vendormod/w") tg.runFail("-vgo", "list", "-getmode=local", "-f={{.Dir}}", "newpkg") - tg.grepStderr(`module lookup disabled by -getmode=local`, "expected -getmode=local to avoid network") + tg.grepStderr(`disabled by -getmode=local`, "expected -getmode=local to avoid network") if !testing.Short() { tg.run("-vgo", "build") @@ -625,7 +625,7 @@ func TestConvertLegacyConfig(t *testing.T) { // it would choose a newer version (like v0.8.0 or maybe // something even newer). Check for the older version to // make sure Gopkg.lock was properly used. - tg.grepStderr("v0.6.0", "expected github.com/pkg/errors at v0.6.0") + tg.grepStdout("v0.6.0", "expected github.com/pkg/errors at v0.6.0") } func TestVerifyNotDownloaded(t *testing.T) {