mage: optionally mount module cache for crossbuild

dev-tools/mage/common.go

@@ -49,6 +49,8 @@ import (
 	"github.com/magefile/mage/sh"
 	"github.com/magefile/mage/target"
 	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
 )
 
 // Expand expands the given Go text/template string.
@@ -791,3 +793,13 @@ func ParseVersion(version string) (major, minor, patch int, err error) {
 	patch, _ = strconv.Atoi(data["patch"])
 	return
 }
+
+// listModuleDir calls gotool.ListModuleVendorDir or
+// gotool.ListModuleCacheDir, depending on the value of
+// UseVendor.
+func listModuleDir(modpath string) (string, error) {
+	if UseVendor {
+		return gotool.ListModuleVendorDir(modpath)
+	}
+	return gotool.ListModuleCacheDir(modpath)
+}

dev-tools/mage/config.go

@@ -38,21 +38,6 @@ var (
 	shortTemplate     = filepath.Join("build", BeatName+".yml.tmpl")
 	referenceTemplate = filepath.Join("build", BeatName+".reference.yml.tmpl")
 	dockerTemplate    = filepath.Join("build", BeatName+".docker.yml.tmpl")
-
-	defaultConfigFileParams = ConfigFileParams{
-		ShortParts: []string{
-			OSSBeatDir("_meta/beat.yml"),
-			LibbeatDir("_meta/config.yml.tmpl"),
-		},
-		ReferenceParts: []string{
-			OSSBeatDir("_meta/beat.reference.yml"),
-			LibbeatDir("_meta/config.reference.yml.tmpl"),
-		},
-		DockerParts: []string{
-			OSSBeatDir("_meta/beat.docker.yml"),
-			LibbeatDir("_meta/config.docker.yml"),
-		},
-	}
 )
 
 // ConfigFileType is a bitset that indicates what types of config files to
@@ -97,7 +82,20 @@ func (c ConfigFileParams) Empty() bool {
 // host for the generated configs. Defaults to linux/amd64.
 func Config(types ConfigFileType, args ConfigFileParams, targetDir string) error {
 	if args.Empty() {
-		args = defaultConfigFileParams
+		args = ConfigFileParams{
+			ShortParts: []string{
+				OSSBeatDir("_meta/beat.yml"),
+				LibbeatDir("_meta/config.yml.tmpl"),
+			},
+			ReferenceParts: []string{
+				OSSBeatDir("_meta/beat.reference.yml"),
+				LibbeatDir("_meta/config.reference.yml.tmpl"),
+			},
+			DockerParts: []string{
+				OSSBeatDir("_meta/beat.docker.yml"),
+				LibbeatDir("_meta/config.docker.yml"),
+			},
+		}
 	}
 
 	if err := makeConfigTemplates(types, args); err != nil {

dev-tools/mage/crossbuild.go

@@ -19,6 +19,7 @@ package mage
 
 import (
 	"fmt"
+	"go/build"
 	"log"
 	"os"
 	"path/filepath"
@@ -31,6 +32,7 @@ import (
 	"github.com/magefile/mage/sh"
 	"github.com/pkg/errors"
 
+	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
 	"github.com/elastic/beats/v7/libbeat/common/file"
 )
 
@@ -127,6 +129,12 @@ func CrossBuild(options ...CrossBuildOption) error {
 		return nil
 	}
 
+	if CrossBuildMountModcache {
+		// Make sure the module dependencies are downloaded on the host,
+		// as they will be mounted into the container read-only.
+		mg.Deps(func() error { return gotool.Mod.Download() })
+	}
+
 	// Build the magefile for Linux so we can run it inside the container.
 	mg.Deps(buildMage)
 
@@ -250,6 +258,11 @@ func (b GolangCrossBuilder) Build() error {
 	if UseVendor {
 		args = append(args, "--env", "GOFLAGS=-mod=vendor")
 	}
+	if CrossBuildMountModcache {
+		// Mount $GOPATH/pkg/mod into the container, read-only.
+		hostDir := filepath.Join(build.Default.GOPATH, "pkg", "mod")
+		args = append(args, "-v", hostDir+":/go/pkg/mod:ro")
+	}
 
 	args = append(args,
 		"--rm",

dev-tools/mage/godaemon.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"log"
 	"os"
+	"path/filepath"
 )
 
 var (
@@ -42,8 +43,12 @@ func BuildGoDaemon() error {
 	}
 
 	// Test if binaries are up-to-date.
+	godaemonDir, err := listModuleDir("github.com/tsg/go-daemon")
+	if err != nil {
+		return err
+	}
+	input := filepath.Join(godaemonDir, "src", "god.c")
 	output := MustExpand("build/golang-crossbuild/god-{{.Platform.GOOS}}-{{.Platform.Arch}}")
-	input := MustExpand("{{ elastic_beats_dir }}/vendor/github.com/tsg/go-daemon/src/god.c")
 	if IsUpToDate(output, input) {
 		log.Println(">>> buildGoDaemon is up-to-date for", Platform.Name)
 		return nil

dev-tools/mage/gomod.go

@@ -75,7 +75,7 @@ func Vendor() error {
 
 	// copy packages which require the whole tree
 	for _, p := range copyAll {
-		path, err := gotool.ListModulePath(p.name)
+		path, err := gotool.ListModuleVendorDir(p.name)
 		if err != nil {
 			return err
 		}

dev-tools/mage/gotool/go.go

@@ -93,14 +93,31 @@ func ListTestFiles(pkg string) ([]string, error) {
 	return getLines(callGo(nil, "list", "-f", tmpl, pkg))
 }
 
-// ListModulePath returns the path to the module in the cache.
-func ListModulePath(pkg string) (string, error) {
-	const tmpl = `{{.Dir}}`
+// ListModuleCacheDir returns the module cache directory containing
+// the specified module. If the module does not exist in the cache,
+// an error will be returned.
+func ListModuleCacheDir(pkg string) (string, error) {
+	return listModuleDir(pkg, false)
+}
+
+// ListModuleVendorDir returns the vendor directory containing the
+// specified module. If the module has not been vendored, an error
+// will be returned.
+func ListModuleVendorDir(pkg string) (string, error) {
+	return listModuleDir(pkg, true)
+}
+
+func listModuleDir(pkg string, vendor bool) (string, error) {
 	env := map[string]string{
-		// make sure to look in the module cache
+		// Make sure GOFLAGS does not influence behaviour.
 		"GOFLAGS": "",
 	}
-	lines, err := getLines(callGo(env, "list", "-m", "-f", tmpl, pkg))
+	args := []string{"-m", "-f", "{{.Dir}}"}
+	if vendor {
+		args = append(args, "-mod=vendor")
+	}
+	args = append(args, pkg)
+	lines, err := getLines(callGo(env, "list", args...))
 	if err != nil {
 		return "", err
 	}

dev-tools/mage/gotool/modules.go

@@ -19,10 +19,11 @@ package gotool
 
 // Mod is the command go mod.
 var Mod = goMod{
-	Init:   modCommand{"init"}.run,
-	Tidy:   modCommand{"tidy"}.run,
-	Verify: modCommand{"verify"}.run,
-	Vendor: modCommand{"vendor"}.run,
+	Download: modCommand{"download"}.run,
+	Init:     modCommand{"init"}.run,
+	Tidy:     modCommand{"tidy"}.run,
+	Verify:   modCommand{"verify"}.run,
+	Vendor:   modCommand{"vendor"}.run,
 }
 
 type modCommand struct {
@@ -40,12 +41,16 @@ func (cmd modCommand) run(opts ...ArgOpt) error {
 }
 
 type goMod struct {
-	Init   modInit
-	Tidy   modTidy
-	Verify modVerify
-	Vendor modVendor
+	Download modDownload
+	Init     modInit
+	Tidy     modTidy
+	Verify   modVerify
+	Vendor   modVendor
 }
 
+// modDownload cleans the go.mod file
+type modDownload func(opts ...ArgOpt) error
+
 // modInit initializes a new go module in folder.
 type modInit func(opts ...ArgOpt) error
 

dev-tools/mage/settings.go

@@ -60,6 +60,10 @@ var (
 	TestCoverage = false
 	UseVendor    = true
 
+	// CrossBuildMountModcache, if true, mounts $GOPATH/pkg/mod into
+	// the crossbuild images at /go/pkg/mod, read-only.
+	CrossBuildMountModcache = false
+
 	BeatName        = EnvOr("BEAT_NAME", filepath.Base(CWD()))
 	BeatServiceName = EnvOr("BEAT_SERVICE_NAME", BeatName)
 	BeatIndexPrefix = EnvOr("BEAT_INDEX_PREFIX", BeatName)
@@ -280,8 +284,7 @@ func findElasticBeatsDir() (string, error) {
 	if repo.IsElasticBeats() {
 		return repo.RootDir, nil
 	}
-
-	return gotool.ListModulePath("github.com/elastic/beats/v7")
+	return listModuleDir("github.com/elastic/beats/v7")
 }
 
 var (

generator/common/beatgen/beatgen.go

@@ -101,6 +101,12 @@ func Generate() error {
 		return err
 	}
 
+	// Make sure the ElasticBeatsDir value is cached
+	// before changing directory below.
+	if _, err := devtools.ElasticBeatsDir(); err != nil {
+		return err
+	}
+
 	err = setup.GenNewBeat(cfg)
 	if err != nil {
 		return errors.Wrap(err, "error generating new beat")

generator/common/beatgen/setup/setup.go

@@ -112,7 +112,7 @@ func CopyVendor() error {
 		return err
 	}
 
-	path, err := gotool.ListModulePath("github.com/elastic/beats/v7")
+	path, err := gotool.ListModuleCacheDir("github.com/elastic/beats/v7")
 	if err != nil {
 		return err
 	}