Merge pull request #47 from schibsted/dont-track-go-deps

By default, don't track Go dependencies.

compile.sh

@@ -13,7 +13,7 @@ elif ! command -v seb > /dev/null; then
 	export PATH="$HOME/go/bin:$PATH"
 fi
 
-seb
+seb "$@"
 [ -z "$RUNTESTS" ] && exit 0
 
 ninja -f "$BUILDPATH"/build.ninja "$BUILDPATH"/dev/gotest/buildbuild

docs/descriptors/config.md

@@ -205,6 +205,18 @@ is enough to rebuild all Go targets when go.mod changes.
 
 Empty list by default.
 
+## go_track_deps
+If set to non-empty, enables tracking of Go depencies just like other
+languages. Unfortunately, using `go list` to find the dependencies is sometimes
+slower than the actual compilation. Especially since Go has a built-in build
+cache that makes it quick to recompile unchanged programs. Thus the dependency
+tracking is disabled by default for Go programs, they're instead compiled every
+time sebuild invokes ninja, but usually cached.
+
+Enable this if you find it faster or if you need to have proper dependency
+tracking. Without this flag set, ninja will never report "nothing to do" if
+there are go targets present, since they're always re-run.
+
 ## extensions
 A list of plugins to load. Plugins are go modules that can customize the
 desciptors and other parts of sebuild. See the

docs/descriptors/gomodule.md

@@ -18,3 +18,7 @@ arguments. Special conditions and ninja variables are described on that
 page as well. Since Go doesn't support compiling these modules without
 cgo the `nocgo` argument is not available. Other settings disabling cgo
 are also ignored.
+
+By default, dependency tracking is disabled for Go modules, since it can be
+quite slow. See the [go_track_deps](gonfig.md#go_track_deps) CONFIG argument
+for more details.

docs/descriptors/goprog.md

@@ -16,6 +16,10 @@ argument](../arguments/extravars.md).
 Additionally, setting the `nocgo` [condition](../conditions.md) disables cgo
 for all programs.
 
+By default, dependency tracking is disabled for Go programs, since it can be
+quite slow. See the [go_track_deps](gonfig.md#go_track_deps) CONFIG argument
+for more details.
+
 ## Arguments
 
 ### nocgo

docs/descriptors/gotest.md

@@ -45,3 +45,7 @@ To easily generate all go coverage report, the target
     build/<flavor>/gocover
 
 maps to the list of reports and can be used to generate all of them.
+
+By default, dependency tracking is disabled for Go tests, since it can be
+quite slow. See the [go_track_deps](gonfig.md#go_track_deps) CONFIG argument
+for more details.

internal/cmd/gobuild/gobuild.go

@@ -62,6 +62,8 @@ func Main(args ...string) {
 		flagset.Usage()
 		os.Exit(2)
 	}
+	skipDepfile := strings.HasSuffix(outpath, ".phony")
+	outpath = strings.TrimSuffix(outpath, ".phony")
 	if depfile == "" && needDepfile(*mode) {
 		flagset.Usage()
 		os.Exit(2)
@@ -79,7 +81,7 @@ func Main(args ...string) {
 		fmt.Printf("gobuild: Entering directory `%s'\n", absin)
 	}
 
-	if depfile != "" {
+	if !skipDepfile && depfile != "" {
 		depf, err := os.OpenFile(absdep, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
 		if err != nil {
 			fmt.Fprintln(os.Stderr, err)

internal/pkg/assets/rules_ninja.go

@@ -122,6 +122,8 @@ pool gobuilds_piclib
 # variables if not set there however.  This is for dependencies to work more
 # properly as configvars script changes retrigger builds but environment
 # variables do not.
+# Note that for gobuild the depfile is only used if enabled. By default
+# the commands are always run and instead use the Go build cache.
 gobuild_tool=GOBUILD_FLAGS=$gobuild_flags GOBUILD_TEST_FLAGS=$gobuild_test_flags CGO_ENABLED=$cgo_enabled seb -tool gobuild
 
 rule gobuild

pkg/buildbuild/config.go

@@ -96,6 +96,8 @@ type Config struct {
 	BuiltinDefaultsNinja string
 	BuiltinRulesNinja    string
 	BuiltinStaticNinja   string
+
+	GoTrackDeps string
 }
 
 type FlavorConfig struct {
@@ -235,6 +237,7 @@ func (ops *GlobalOps) ParseConfig(srcdir string, s *Scanner, flavors []string) P
 		{"builtin_defaults_ninja", &ops.Config.BuiltinDefaultsNinja},
 		{"builtin_rules_ninja", &ops.Config.BuiltinRulesNinja},
 		{"builtin_static_ninja", &ops.Config.BuiltinStaticNinja},
+		{"go_track_deps", &ops.Config.GoTrackDeps},
 	} {
 		if args.Unflavored[conf.key] != nil {
 			*conf.conf = strings.Join(args.Unflavored[conf.key], " ")

pkg/buildbuild/go.go

@@ -85,7 +85,8 @@ func (g *GoProgDesc) Finalize(ops *GlobalOps) {
 		// Buildmode plugin currently does not support cgo disabled.
 		eas = append(eas, "cgo_enabled=1")
 	}
-	target := g.AddTarget(tname, "gobuild", []string{g.Srcdir}, g.Destdir, "", eas, g.TargetOptions)
+	tgopts := filterGoTargetOptions(g.TargetOptions, ops)
+	target := g.AddTarget(tname, "gobuild", []string{g.Srcdir}, g.Destdir, "", eas, tgopts)
 	AddGodeps(target, ops)
 	g.GeneralDesc.Finalize(ops)
 }
@@ -112,7 +113,8 @@ func (g *GoTestDesc) Finalize(ops *GlobalOps) {
 	}
 
 	eas = append(eas, "gomode=test-prog")
-	target := g.AddTarget(name+".test", "gobuild", []string{g.Srcdir}, g.Destdir, "", eas, g.TargetOptions)
+	tgopts := filterGoTargetOptions(g.TargetOptions, ops)
+	target := g.AddTarget(name+".test", "gobuild", []string{g.Srcdir}, g.Destdir, "", eas, tgopts)
 	eas = eas[:len(eas)-1]
 	AddGodeps(target, ops)
 
@@ -141,12 +143,23 @@ func (g *GoTestDesc) Finalize(ops *GlobalOps) {
 	g.GeneralDesc.Finalize(ops)
 }
 
+func filterGoTargetOptions(in map[string]bool, ops *GlobalOps) map[string]bool {
+	tgopts := make(map[string]bool, len(in))
+	for k, v := range in {
+		if k == "always-all" && ops.Config.GoTrackDeps != "" {
+			continue
+		}
+		tgopts[k] = v
+	}
+	return tgopts
+}
+
 var GoprogTemplate = GoProgDesc{
 	Mode: "prog",
 	LinkDesc: LinkDesc{
 		GeneralDesc: GeneralDesc{
 			Destdir:       "dest_bin",
-			TargetOptions: map[string]bool{"all": true, "incdeps": true, "libdeps": true},
+			TargetOptions: map[string]bool{"always-all": true, "all": true, "incdeps": true, "libdeps": true},
 		},
 	},
 }
@@ -156,7 +169,7 @@ var GomoduleTemplate = GoProgDesc{
 	LinkDesc: LinkDesc{
 		GeneralDesc: GeneralDesc{
 			Destdir:       "dest_mod",
-			TargetOptions: map[string]bool{"all": true, "incdeps": true, "libdeps": true},
+			TargetOptions: map[string]bool{"always-all": true, "all": true, "incdeps": true, "libdeps": true},
 		},
 	},
 }
@@ -165,7 +178,7 @@ var GotestTemplate = GoTestDesc{
 	LinkDesc: LinkDesc{
 		GeneralDesc: GeneralDesc{
 			Destdir:       "gotest",
-			TargetOptions: map[string]bool{"all": true, "incdeps": true, "libdeps": true},
+			TargetOptions: map[string]bool{"always-all": true, "all": true, "incdeps": true, "libdeps": true},
 		},
 	},
 }

pkg/buildbuild/output.go

@@ -322,7 +322,12 @@ func (ops *GlobalOps) OutputDescriptor(desc Descriptor, builddir, objdir string)
 		orderDeps := desc.ResolveOrderDeps(target)
 		srcs := desc.ResolveSrcs(ops, tname, target.Sources...)
 
-		fmt.Fprintf(w, "build %s: %s ", dest, rule)
+		if target.Options["always-all"] {
+			fmt.Fprintf(w, "build %s: phony %s.phony\n", dest, dest)
+			fmt.Fprintf(w, "build %s.phony: %s ", dest, rule)
+		} else {
+			fmt.Fprintf(w, "build %s: %s ", dest, rule)
+		}
 		fmt.Fprint(w, strings.Join(srcs, " "))
 
 		if len(deps) > 0 {
@@ -344,7 +349,10 @@ func (ops *GlobalOps) OutputDescriptor(desc Descriptor, builddir, objdir string)
 		if len(target.Srcopts) > 0 {
 			fmt.Fprint(w, "    srcopts=", strings.Join(target.Srcopts, " "), "\n")
 		}
-		if target.Options["all"] {
+		if target.Options["always-all"] {
+			fmt.Fprintf(w, "default %s.phony\n", dest)
+			defaults = append(defaults, dest+".phony")
+		} else if target.Options["all"] {
 			fmt.Fprintf(w, "default %s\n", dest)
 			defaults = append(defaults, dest)
 		}