Make Beats buildable outside GOPATH (#16329)

## What does this PR do?

This PR adds support for building Beats outside GOPATH by separating repo information collection depending on the root path of the project.

## Why is it important?

This lets users compile and package Beats if they have checked out the code to a non-GOPATH folder.

dev-tools/mage/crossbuild.go

@@ -215,7 +215,7 @@ func (b GolangCrossBuilder) Build() error {
 		return errors.Wrap(err, "failed to determine repo root and package sub dir")
 	}
 
-	mountPoint := filepath.ToSlash(filepath.Join("/go", "src", repoInfo.RootImportPath))
+	mountPoint := filepath.ToSlash(filepath.Join("/go", "src", repoInfo.CanonicalRootImportPath))
 	// use custom dir for build if given, subdir if not:
 	cwd := repoInfo.SubDir
 	if b.InDir != "" {

dev-tools/mage/gotool/go.go

@@ -18,6 +18,7 @@
 package gotool
 
 import (
+	"fmt"
 	"os"
 	"strings"
 
@@ -55,6 +56,18 @@ type goTest func(opts ...ArgOpt) error
 // Test runs `go test` and provides optionals for adding command line arguments.
 var Test goTest = runGoTest
 
+// GetModuleName returns the name of the module.
+func GetModuleName() (string, error) {
+	lines, err := getLines(callGo(nil, "list", "-m"))
+	if err != nil {
+		return "", err
+	}
+	if len(lines) != 1 {
+		return "", fmt.Errorf("unexpected number of lines")
+	}
+	return lines[0], nil
+}
+
 // ListProjectPackages lists all packages in the current project
 func ListProjectPackages() ([]string, error) {
 	return ListPackages("./...")

dev-tools/mage/settings.go

@@ -33,6 +33,8 @@ import (
 	"github.com/magefile/mage/sh"
 	"github.com/pkg/errors"
 	"golang.org/x/tools/go/vcs"
+
+	"github.com/elastic/beats/dev-tools/mage/gotool"
 )
 
 const (
@@ -189,16 +191,17 @@ VersionQualifier = {{.Qualifier}}
 
 ## Functions
 
-beat_doc_branch     = {{ beat_doc_branch }}
-beat_version        = {{ beat_version }}
-commit              = {{ commit }}
-date                = {{ date }}
-elastic_beats_dir   = {{ elastic_beats_dir }}
-go_version          = {{ go_version }}
-repo.RootImportPath = {{ repo.RootImportPath }}
-repo.RootDir        = {{ repo.RootDir }}
-repo.ImportPath     = {{ repo.ImportPath }}
-repo.SubDir         = {{ repo.SubDir }}
+beat_doc_branch              = {{ beat_doc_branch }}
+beat_version                 = {{ beat_version }}
+commit                       = {{ commit }}
+date                         = {{ date }}
+elastic_beats_dir            = {{ elastic_beats_dir }}
+go_version                   = {{ go_version }}
+repo.RootImportPath          = {{ repo.RootImportPath }}
+repo.CanonicalRootImportPath = {{ repo.CanonicalRootImportPath }}
+repo.RootDir                 = {{ repo.RootDir }}
+repo.ImportPath              = {{ repo.ImportPath }}
+repo.SubDir                  = {{ repo.SubDir }}
 `
 
 	return Expand(dumpTemplate)
@@ -539,16 +542,17 @@ func parseDocBranch(data []byte) (string, error) {
 
 // ProjectRepoInfo contains information about the project's repo.
 type ProjectRepoInfo struct {
-	RootImportPath string // Import path at the project root.
-	RootDir        string // Root directory of the project.
-	ImportPath     string // Import path of the current directory.
-	SubDir         string // Relative path from the root dir to the current dir.
+	RootImportPath          string // Import path at the project root.
+	CanonicalRootImportPath string // Pre-modules root import path (does not contain semantic import version identifier).
+	RootDir                 string // Root directory of the project.
+	ImportPath              string // Import path of the current directory.
+	SubDir                  string // Relative path from the root dir to the current dir.
 }
 
 // IsElasticBeats returns true if the current project is
 // github.com/elastic/beats.
 func (r *ProjectRepoInfo) IsElasticBeats() bool {
-	return r.RootImportPath == elasticBeatsImportPath
+	return strings.HasPrefix(r.RootImportPath, elasticBeatsImportPath)
 }
 
 var (
@@ -561,61 +565,177 @@ var (
 // import path and the current directory's import path.
 func GetProjectRepoInfo() (*ProjectRepoInfo, error) {
 	repoInfoOnce.Do(func() {
-		repoInfoValue, repoInfoErr = getProjectRepoInfo()
+		if isUnderGOPATH() {
+			repoInfoValue, repoInfoErr = getProjectRepoInfoUnderGopath()
+		} else {
+			repoInfoValue, repoInfoErr = getProjectRepoInfoWithModules()
+		}
 	})
 
 	return repoInfoValue, repoInfoErr
 }
 
-func getProjectRepoInfo() (*ProjectRepoInfo, error) {
+func isUnderGOPATH() bool {
+	underGOPATH := false
+	srcDirs, err := listSrcGOPATHs()
+	if err != nil {
+		return false
+	}
+	for _, srcDir := range srcDirs {
+		rel, err := filepath.Rel(srcDir, CWD())
+		if err != nil {
+			continue
+		}
+
+		if !strings.Contains(rel, "..") {
+			underGOPATH = true
+		}
+	}
+
+	return underGOPATH
+}
+
+func getProjectRepoInfoWithModules() (*ProjectRepoInfo, error) {
 	var (
-		cwd            = CWD()
-		rootImportPath string
-		srcDir         string
+		cwd     = CWD()
+		rootDir string
+		subDir  string
 	)
 
-	// Search upward from the CWD to determine the project root based on VCS.
+	possibleRoot := cwd
 	var errs []string
-	for _, gopath := range filepath.SplitList(build.Default.GOPATH) {
-		gopath = filepath.Clean(gopath)
+	for {
+		isRoot, err := isGoModRoot(possibleRoot)
+		if err != nil {
+			errs = append(errs, err.Error())
+		}
 
-		if !strings.HasPrefix(cwd, gopath) {
-			// Fixes an issue on macOS when /var is actually /private/var.
-			var err error
-			gopath, err = filepath.EvalSymlinks(gopath)
-			if err != nil {
-				errs = append(errs, err.Error())
-				continue
-			}
+		if isRoot {
+			rootDir = possibleRoot
+			break
+		}
+
+		subDir, err = filepath.Rel(possibleRoot, cwd)
+		if err != nil {
+			errs = append(errs, err.Error())
 		}
+		possibleRoot = filepath.Dir(possibleRoot)
+	}
+
+	if rootDir == "" {
+		return nil, errors.Errorf("failed to find root dir of module file: %v", errs)
+	}
+
+	rootImportPath, err := gotool.GetModuleName()
+	if err != nil {
+		return nil, err
+	}
+
+	return &ProjectRepoInfo{
+		RootImportPath:          rootImportPath,
+		CanonicalRootImportPath: filepath.ToSlash(extractCanonicalRootImportPath(rootImportPath)),
+		RootDir:                 rootDir,
+		SubDir:                  subDir,
+		ImportPath:              filepath.ToSlash(filepath.Join(rootImportPath, subDir)),
+	}, nil
+}
 
-		srcDir = filepath.Join(gopath, "src")
+func isGoModRoot(path string) (bool, error) {
+	gomodPath := filepath.Join(path, "go.mod")
+	_, err := os.Stat(gomodPath)
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	if err != nil {
+		return false, err
+	}
+
+	return true, nil
+}
+
+func getProjectRepoInfoUnderGopath() (*ProjectRepoInfo, error) {
+	var (
+		cwd     = CWD()
+		errs    []string
+		rootDir string
+	)
+
+	srcDirs, err := listSrcGOPATHs()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, srcDir := range srcDirs {
 		_, root, err := vcs.FromDir(cwd, srcDir)
 		if err != nil {
 			// Try the next gopath.
 			errs = append(errs, err.Error())
 			continue
 		}
-		rootImportPath = root
+		rootDir = filepath.Join(srcDir, root)
 		break
 	}
-	if rootImportPath == "" {
-		return nil, errors.Errorf("failed to determine root import path (Did "+
-			"you git init?, Is the project in the GOPATH? GOPATH=%v, CWD=%v?): %v",
-			build.Default.GOPATH, cwd, errs)
+
+	if rootDir == "" {
+		return nil, errors.Errorf("error while determining root directory: %v", errs)
 	}
 
-	rootDir := filepath.Join(srcDir, rootImportPath)
 	subDir, err := filepath.Rel(rootDir, cwd)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get relative path to repo root")
 	}
-	importPath := filepath.ToSlash(filepath.Join(rootImportPath, subDir))
+
+	rootImportPath, err := gotool.GetModuleName()
+	if err != nil {
+		return nil, err
+	}
 
 	return &ProjectRepoInfo{
-		RootImportPath: rootImportPath,
-		RootDir:        rootDir,
-		SubDir:         subDir,
-		ImportPath:     importPath,
+		RootImportPath:          rootImportPath,
+		CanonicalRootImportPath: filepath.ToSlash(extractCanonicalRootImportPath(rootImportPath)),
+		RootDir:                 rootDir,
+		SubDir:                  subDir,
+		ImportPath:              filepath.ToSlash(filepath.Join(rootImportPath, subDir)),
 	}, nil
 }
+
+func extractCanonicalRootImportPath(rootImportPath string) string {
+	// In order to be compatible with go modules, the root import
+	// path of any module at major version v2 or higher must include
+	// the major version.
+	// Ref: https://github.com/golang/go/wiki/Modules#semantic-import-versioning
+	//
+	// Thus, Beats has to include the major version as well.
+	// This regex removes the major version from the import path.
+	re := regexp.MustCompile(`(/v[1-9][0-9]*)$`)
+	return re.ReplaceAllString(rootImportPath, "")
+}
+
+func listSrcGOPATHs() ([]string, error) {
+	var (
+		cwd     = CWD()
+		errs    []string
+		srcDirs []string
+	)
+	for _, gopath := range filepath.SplitList(build.Default.GOPATH) {
+		gopath = filepath.Clean(gopath)
+
+		if !strings.HasPrefix(cwd, gopath) {
+			// Fixes an issue on macOS when /var is actually /private/var.
+			var err error
+			gopath, err = filepath.EvalSymlinks(gopath)
+			if err != nil {
+				errs = append(errs, err.Error())
+				continue
+			}
+		}
+
+		srcDirs = append(srcDirs, filepath.Join(gopath, "src"))
+	}
+
+	if len(srcDirs) == 0 {
+		return srcDirs, errors.Errorf("failed to find any GOPATH %v", errs)
+	}
+
+	return srcDirs, nil
+}

docs/devguide/contributing.asciidoc

@@ -55,10 +55,6 @@ After https://golang.org/doc/install[installing Go], set the
 https://golang.org/doc/code.html#GOPATH[GOPATH] environment variable to point to
 your workspace location, and make sure `$GOPATH/bin` is in your PATH.
 
-The location where you clone is important. Make a directory structure under
-`GOPATH` that matches the URL used for Elastic repositories, then clone the
-beats repository under the new directory:
-
 [source,shell]
 ----------------------------------------------------------------------
 mkdir -p ${GOPATH}/src/github.com/elastic
@@ -178,11 +174,9 @@ external tools.
 [float]
 ==== Golang
 
-To manage the `vendor/` folder we use
-https://github.com/kardianos/govendor[govendor]. Please see
-the govendor documentation on how to add or update vendored dependencies.
+To manage the `vendor/` folder we use go modules.
 
-In most cases `govendor fetch your/dependency@version +out` will get the job done.
+To update the contents of `vendor/`, run `mage vendor`.
 
 [float]
 ==== Other dependencies