diff --git a/cmd/tool/matrix/matrix.go b/cmd/tool/matrix/matrix.go new file mode 100644 index 00000000..b5db7972 --- /dev/null +++ b/cmd/tool/matrix/matrix.go @@ -0,0 +1,274 @@ +package matrix + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "math" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/dnephin/pflag" + "gotest.tools/gotestsum/internal/log" + "gotest.tools/gotestsum/testjson" +) + +func Run(name string, args []string) error { + flags, opts := setupFlags(name) + switch err := flags.Parse(args); { + case err == pflag.ErrHelp: + return nil + case err != nil: + usage(os.Stderr, name, flags) + return err + } + opts.stdin = os.Stdin + opts.stdout = os.Stdout + return run(*opts) +} + +type options struct { + numPartitions uint + timingFilesPattern string + debug bool + + // shims for testing + stdin io.Reader + stdout io.Writer +} + +func setupFlags(name string) (*pflag.FlagSet, *options) { + opts := &options{} + flags := pflag.NewFlagSet(name, pflag.ContinueOnError) + flags.SetInterspersed(false) + flags.Usage = func() { + usage(os.Stdout, name, flags) + } + flags.UintVar(&opts.numPartitions, "partitions", 0, + "number of parallel partitions to create in the test matrix") + flags.StringVar(&opts.timingFilesPattern, "timing-files", "", + "glob pattern to match files that contain test2json events, ex: ./logs/*.log") + flags.BoolVar(&opts.debug, "debug", false, + "enable debug logging") + return flags, opts +} + +func usage(out io.Writer, name string, flags *pflag.FlagSet) { + fmt.Fprintf(out, `Usage: + %[1]s [flags] + +Read a list of packages from stdin and output a GitHub Actions matrix strategy +that splits the packages by previous run times to minimize overall CI runtime. + + echo -n "matrix=" >> $GITHUB_OUTPUT + go list ./... | %[1]s --timing-files ./*.log --partitions 4 >> $GITHUB_OUTPUT + +The output of the command is a JSON object that can be used as the matrix +strategy for a test job. + + +Flags: +`, name) + flags.SetOutput(out) + flags.PrintDefaults() +} + +func run(opts options) error { + log.SetLevel(log.InfoLevel) + if opts.debug { + log.SetLevel(log.DebugLevel) + } + if opts.numPartitions < 2 { + return fmt.Errorf("--partitions must be atleast 2") + } + if opts.timingFilesPattern == "" { + return fmt.Errorf("--timing-files is required") + } + + pkgs, err := readPackages(opts.stdin) + if err != nil { + return fmt.Errorf("failed to read packages from stdin: %v", err) + } + + files, err := readTimingReports(opts) + if err != nil { + return fmt.Errorf("failed to read or delete timing files: %v", err) + } + defer closeFiles(files) + + pkgTiming, err := packageTiming(files) + if err != nil { + return err + } + + buckets := bucketPackages(packagePercentile(pkgTiming), pkgs, opts.numPartitions) + return writeMatrix(opts.stdout, buckets) +} + +func readPackages(stdin io.Reader) ([]string, error) { + var packages []string + scan := bufio.NewScanner(stdin) + for scan.Scan() { + packages = append(packages, scan.Text()) + } + return packages, scan.Err() +} + +func readTimingReports(opts options) ([]*os.File, error) { + fileNames, err := filepath.Glob(opts.timingFilesPattern) + if err != nil { + return nil, err + } + + files := make([]*os.File, 0, len(fileNames)) + for _, fileName := range fileNames { + fh, err := os.Open(fileName) + if err != nil { + return nil, err + } + files = append(files, fh) + } + + log.Infof("Found %v timing files in %v", len(files), opts.timingFilesPattern) + return files, nil +} + +func parseEvent(reader io.Reader) (testjson.TestEvent, error) { + event := testjson.TestEvent{} + err := json.NewDecoder(reader).Decode(&event) + return event, err +} + +func packageTiming(files []*os.File) (map[string][]time.Duration, error) { + timing := make(map[string][]time.Duration) + for _, fh := range files { + exec, err := testjson.ScanTestOutput(testjson.ScanConfig{Stdout: fh}) + if err != nil { + return nil, fmt.Errorf("failed to read events from %v: %v", fh.Name(), err) + } + + for _, pkg := range exec.Packages() { + timing[pkg] = append(timing[pkg], exec.Package(pkg).Elapsed()) + } + } + return timing, nil +} + +func packagePercentile(timing map[string][]time.Duration) map[string]time.Duration { + result := make(map[string]time.Duration) + for pkg, times := range timing { + lenTimes := len(times) + if lenTimes == 0 { + result[pkg] = 0 + continue + } + + sort.Slice(times, func(i, j int) bool { + return times[i] < times[j] + }) + + r := int(math.Ceil(0.85 * float64(lenTimes))) + if r == 0 { + result[pkg] = times[0] + continue + } + result[pkg] = times[r-1] + } + return result +} + +func closeFiles(files []*os.File) { + for _, fh := range files { + _ = fh.Close() + } +} + +func bucketPackages(timing map[string]time.Duration, packages []string, n uint) []bucket { + sort.SliceStable(packages, func(i, j int) bool { + return timing[packages[i]] >= timing[packages[j]] + }) + + buckets := make([]bucket, n) + for _, pkg := range packages { + i := minBucket(buckets) + buckets[i].Total += timing[pkg] + buckets[i].Packages = append(buckets[i].Packages, pkg) + log.Debugf("adding %v (%v) to bucket %v with total %v", + pkg, timing[pkg], i, buckets[i].Total) + } + return buckets +} + +func minBucket(buckets []bucket) int { + var n int + var min time.Duration = -1 + for i, b := range buckets { + switch { + case min < 0 || b.Total < min: + min = b.Total + n = i + case b.Total == min && len(buckets[i].Packages) < len(buckets[n].Packages): + n = i + } + } + return n +} + +type bucket struct { + Total time.Duration + Packages []string +} + +type matrix struct { + Include []Partition `json:"include"` +} + +type Partition struct { + ID int `json:"id"` + EstimatedRuntime string `json:"estimatedRuntime"` + Packages string `json:"packages"` + Description string `json:"description"` +} + +func writeMatrix(out io.Writer, buckets []bucket) error { + m := matrix{Include: make([]Partition, len(buckets))} + for i, bucket := range buckets { + p := Partition{ + ID: i, + EstimatedRuntime: bucket.Total.String(), + Packages: strings.Join(bucket.Packages, " "), + } + if len(bucket.Packages) > 0 { + var extra string + if len(bucket.Packages) > 1 { + extra = fmt.Sprintf(" and %d others", len(bucket.Packages)-1) + } + p.Description = fmt.Sprintf("partition %d - package %v%v", + p.ID, testjson.RelativePackagePath(bucket.Packages[0]), extra) + } + + m.Include[i] = p + } + + log.Debugf("%v\n", debugMatrix(m)) + + err := json.NewEncoder(out).Encode(m) + if err != nil { + return fmt.Errorf("failed to json encode output: %v", err) + } + return nil +} + +type debugMatrix matrix + +func (d debugMatrix) String() string { + raw, err := json.MarshalIndent(d, "", " ") + if err != nil { + return fmt.Sprintf("failed to marshal: %v", err.Error()) + } + return string(raw) +} diff --git a/cmd/tool/matrix/matrix_test.go b/cmd/tool/matrix/matrix_test.go new file mode 100644 index 00000000..b537318d --- /dev/null +++ b/cmd/tool/matrix/matrix_test.go @@ -0,0 +1,258 @@ +package matrix + +import ( + "bytes" + "encoding/json" + "io" + "os" + "strconv" + "strings" + "testing" + "time" + + "gotest.tools/gotestsum/testjson" + "gotest.tools/v3/assert" + "gotest.tools/v3/fs" +) + +func TestPackagePercentile(t *testing.T) { + ms := time.Millisecond + timing := map[string][]time.Duration{ + "none": {}, + "one": {time.Second}, + "two": {4 * ms, 2 * ms}, + "three": {2 * ms, 3 * ms, 5 * ms}, + "four": {4 * ms, 3 * ms, ms, 2 * ms}, + "five": {6 * ms, 2 * ms, 3 * ms, 4 * ms, 9 * ms}, + "nine": {6 * ms, 2 * ms, 3 * ms, 4 * ms, 9 * ms, 1 * ms, 5 * ms, 7 * ms, 8 * ms}, + "ten": {6 * ms, 2 * ms, 3 * ms, 4 * ms, 9 * ms, 5 * ms, 7 * ms, 8 * ms, ms, ms}, + "twenty": { + 6 * ms, 2 * ms, 3 * ms, 4 * ms, 9 * ms, 5 * ms, 7 * ms, 8 * ms, ms, ms, + 100, 200, 600, 700, 800, 900, 200, 300, 400, 500, + }, + } + + out := packagePercentile(timing) + expected := map[string]time.Duration{ + "none": 0, + "one": time.Second, + "two": 4 * ms, + "three": 5 * ms, + "four": 4 * ms, + "five": 9 * ms, + "nine": 8 * ms, + "ten": 8 * ms, + "twenty": 6 * ms, + } + assert.DeepEqual(t, out, expected) +} + +func TestBucketPackages(t *testing.T) { + ms := time.Millisecond + timing := map[string]time.Duration{ + "one": 190 * ms, + "two": 200 * ms, + "three": 3800 * ms, + "four": 4000 * ms, + "five": 50 * ms, + "six": 606 * ms, + "rm1": time.Second, + "rm2": time.Second, + } + packages := []string{"new1", "new2", "one", "two", "three", "four", "five", "six"} + + type testCase struct { + n uint + expected []bucket + } + + run := func(t *testing.T, tc testCase) { + buckets := bucketPackages(timing, packages, tc.n) + assert.DeepEqual(t, buckets, tc.expected) + } + + testCases := []testCase{ + { + n: 2, + expected: []bucket{ + 0: {Total: 4440 * ms, Packages: []string{"four", "two", "one", "five"}}, + 1: {Total: 4406 * ms, Packages: []string{"three", "six", "new2", "new1"}}, + }, + }, + { + n: 3, + expected: []bucket{ + 0: {Total: 4000 * ms, Packages: []string{"four"}}, + 1: {Total: 3800 * ms, Packages: []string{"three"}}, + 2: {Total: 1046 * ms, Packages: []string{"six", "two", "one", "five", "new1", "new2"}}, + }, + }, + { + n: 4, + expected: []bucket{ + 0: {Total: 4000 * ms, Packages: []string{"four"}}, + 1: {Total: 3800 * ms, Packages: []string{"three"}}, + 2: {Total: 606 * ms, Packages: []string{"six"}}, + 3: {Total: 440 * ms, Packages: []string{"two", "one", "five", "new2", "new1"}}, + }, + }, + { + n: 8, + expected: []bucket{ + 0: {Total: 4000 * ms, Packages: []string{"four"}}, + 1: {Total: 3800 * ms, Packages: []string{"three"}}, + 2: {Total: 606 * ms, Packages: []string{"six"}}, + 3: {Total: 200 * ms, Packages: []string{"two"}}, + 4: {Total: 190 * ms, Packages: []string{"one"}}, + 5: {Total: 50 * ms, Packages: []string{"five"}}, + 6: {Packages: []string{"new1"}}, + 7: {Packages: []string{"new2"}}, + }, + }, + } + + for _, tc := range testCases { + t.Run(strconv.FormatUint(uint64(tc.n), 10), func(t *testing.T) { + run(t, tc) + }) + } +} + +func TestReadTimingReports(t *testing.T) { + events := func(t *testing.T, start time.Time) string { + t.Helper() + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + for _, i := range []int{0, 1, 2} { + assert.NilError(t, encoder.Encode(testjson.TestEvent{ + Time: start.Add(time.Duration(i) * time.Second), + Action: testjson.ActionRun, + Package: "pkg" + strconv.Itoa(i), + })) + } + return buf.String() + } + + now := time.Now() + dir := fs.NewDir(t, "timing-files", + fs.WithFile("report1.log", events(t, now.Add(-time.Hour))), + fs.WithFile("report2.log", events(t, now.Add(-47*time.Hour))), + fs.WithFile("report3.log", events(t, now.Add(-49*time.Hour))), + fs.WithFile("report4.log", events(t, now.Add(-101*time.Hour)))) + + t.Run("match files", func(t *testing.T) { + opts := options{ + timingFilesPattern: dir.Join("*.log"), + } + + files, err := readTimingReports(opts) + assert.NilError(t, err) + defer closeFiles(files) + assert.Equal(t, len(files), 4) + + for _, fh := range files { + // check the files are properly seeked to 0 + event, err := parseEvent(fh) + assert.NilError(t, err) + assert.Equal(t, event.Package, "pkg0") + } + + actual, err := os.ReadDir(dir.Path()) + assert.NilError(t, err) + assert.Equal(t, len(actual), 4) + }) + + t.Run("no glob match, func", func(t *testing.T) { + opts := options{ + timingFilesPattern: dir.Join("*.json"), + } + + files, err := readTimingReports(opts) + assert.NilError(t, err) + assert.Equal(t, len(files), 0) + }) +} + +func TestRun(t *testing.T) { + events := func(t *testing.T) string { + t.Helper() + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + for _, i := range []int{0, 1, 2} { + elapsed := time.Duration(i+1) * 2 * time.Second + end := time.Now().Add(-5 * time.Second) + start := end.Add(-elapsed) + + assert.NilError(t, encoder.Encode(testjson.TestEvent{ + Time: start, + Action: testjson.ActionRun, + Package: "pkg" + strconv.Itoa(i), + })) + assert.NilError(t, encoder.Encode(testjson.TestEvent{ + Time: end, + Action: testjson.ActionPass, + Package: "pkg" + strconv.Itoa(i), + Elapsed: elapsed.Seconds(), + })) + } + return buf.String() + } + + dir := fs.NewDir(t, "timing-files", + fs.WithFile("report1.log", events(t)), + fs.WithFile("report2.log", events(t)), + fs.WithFile("report3.log", events(t)), + fs.WithFile("report4.log", events(t)), + fs.WithFile("report5.log", events(t))) + + stdout := new(bytes.Buffer) + opts := options{ + numPartitions: 3, + timingFilesPattern: dir.Join("*.log"), + debug: true, + stdout: stdout, + stdin: strings.NewReader("pkg0\npkg1\npkg2\nother"), + } + err := run(opts) + assert.NilError(t, err) + assert.Equal(t, strings.Count(stdout.String(), "\n"), 1, + "the output should be a single line") + + assert.Equal(t, formatJSON(t, stdout), expectedMatrix) +} + +// expectedMatrix can be automatically updated by running tests with -update +// nolint:lll +var expectedMatrix = `{ + "include": [ + { + "description": "partition 0 - package pkg2", + "estimatedRuntime": "6s", + "id": 0, + "packages": "pkg2" + }, + { + "description": "partition 1 - package pkg1", + "estimatedRuntime": "4s", + "id": 1, + "packages": "pkg1" + }, + { + "description": "partition 2 - package pkg0 and 1 others", + "estimatedRuntime": "2s", + "id": 2, + "packages": "pkg0 other" + } + ] +}` + +func formatJSON(t *testing.T, v io.Reader) string { + t.Helper() + raw := map[string]interface{}{} + err := json.NewDecoder(v).Decode(&raw) + assert.NilError(t, err) + + formatted, err := json.MarshalIndent(raw, "", " ") + assert.NilError(t, err) + return string(formatted) +} diff --git a/internal/log/log.go b/internal/log/log.go index 9a3e85f3..d68955bf 100644 --- a/internal/log/log.go +++ b/internal/log/log.go @@ -11,6 +11,7 @@ type Level uint8 const ( ErrorLevel Level = iota WarnLevel + InfoLevel DebugLevel ) @@ -43,6 +44,15 @@ func Debugf(format string, args ...interface{}) { fmt.Fprint(out, "\n") } +// Infof prints the message to stderr, with no prefix. +func Infof(format string, args ...interface{}) { + if level < InfoLevel { + return + } + fmt.Fprintf(out, format, args...) + fmt.Fprint(out, "\n") +} + // Errorf prints the message to stderr, with a red ERROR prefix. func Errorf(format string, args ...interface{}) { if level < ErrorLevel { diff --git a/main.go b/main.go index d2ed49f7..f8af75e5 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( "os" "gotest.tools/gotestsum/cmd" + "gotest.tools/gotestsum/cmd/tool/matrix" "gotest.tools/gotestsum/cmd/tool/slowest" "gotest.tools/gotestsum/internal/log" ) @@ -55,6 +56,7 @@ func toolRun(name string, args []string) error { Commands: %[1]s slowest find or skip the slowest tests + %[1]s ci-matrix use previous test runtime to place packages into optimal buckets Use '%[1]s COMMAND --help' for command specific help. `, name) @@ -67,6 +69,8 @@ Use '%[1]s COMMAND --help' for command specific help. return nil case "slowest": return slowest.Run(name+" "+next, rest) + case "ci-matrix": + return matrix.Run(name+" "+next, rest) default: fmt.Fprintln(os.Stderr, usage(name)) return fmt.Errorf("invalid command: %v %v", name, next)