-
Notifications
You must be signed in to change notification settings - Fork 120
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Benchmark: run XcodeBenchmark with different disk settings (#1000)
* Benchmark: run XcodeBenchmark with different disk settings * Add Xcode benchmark results
- Loading branch information
Showing
8 changed files
with
306 additions
and
48 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| package xcode | ||
|
|
||
| type Benchmark struct { | ||
| Name string | ||
| Command string | ||
| } | ||
|
|
||
| var benchmarks = []Benchmark{ | ||
| { | ||
| Name: "XcodeBenchmark (d869315)", | ||
| Command: "git clone https://github.com/devMEremenko/XcodeBenchmark.git && cd XcodeBenchmark && git reset --hard d86931529ada1df2a1c6646dd85958c360954065 && sh benchmark.sh", | ||
| }, | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| package xcode | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "regexp" | ||
| "time" | ||
| ) | ||
|
|
||
| type Output struct { | ||
| Started time.Time | ||
| Ended time.Time | ||
| } | ||
|
|
||
| func ParseOutput(s string) (*Output, error) { | ||
| // Ensure that the build has succeeded | ||
| matched, err := regexp.MatchString("(?m)^\\*\\* BUILD SUCCEEDED \\*\\*.*$", s) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to parse output: regexp failed: %v", err) | ||
| } | ||
| if !matched { | ||
| return nil, fmt.Errorf("failed to parse output: \"** BUILD SUCCEEDED **\" string " + | ||
| "not found on a separate line, make sure you have Xcode installed") | ||
| } | ||
|
|
||
| re := regexp.MustCompile("Started\\s+(?P<started>.*)\\n.*Ended\\s+(?P<ended>.*)\\n") | ||
|
|
||
| matches := re.FindStringSubmatch(s) | ||
|
|
||
| if len(matches) != re.NumSubexp()+1 { | ||
| return nil, fmt.Errorf("failed to parse output: cannot find Started and Ended times") | ||
| } | ||
|
|
||
| startedRaw := matches[re.SubexpIndex("started")] | ||
| started, err := time.Parse(time.TimeOnly, startedRaw) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to parse started time %q: unsupported format", startedRaw) | ||
| } | ||
|
|
||
| endedRaw := matches[re.SubexpIndex("ended")] | ||
| ended, err := time.Parse(time.TimeOnly, endedRaw) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to parse ended time %q: unsupported format", startedRaw) | ||
| } | ||
|
|
||
| return &Output{ | ||
| Started: started, | ||
| Ended: ended, | ||
| }, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| package xcode_test | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "github.com/cirruslabs/tart/benchmark/internal/command/xcode" | ||
| "github.com/stretchr/testify/require" | ||
| "testing" | ||
| "time" | ||
| ) | ||
|
|
||
| func TestParseOutput(t *testing.T) { | ||
| result, err := xcode.ParseOutput(`** BUILD SUCCEEDED ** [219.713 sec] | ||
| System Version: 14.6 | ||
| Xcode 15.4 | ||
| Hardware Overview | ||
| Model Name: Apple Virtual Machine 1 | ||
| Model Identifier: VirtualMac2,1 | ||
| Total Number of Cores: 4 | ||
| Memory: 8 GB | ||
| ✅ XcodeBenchmark has completed | ||
| 1️⃣ Take a screenshot of this window (Cmd + Shift + 4 + Space) and resize to include: | ||
| - Build Time (See ** BUILD SUCCEEDED ** [XYZ sec]) | ||
| - System Version | ||
| - Xcode Version | ||
| - Hardware Overview | ||
| - Started 13:46:20 | ||
| - Ended 13:50:02 | ||
| - Date Thu Jan 16 13:50:02 UTC 2025 | ||
| 2️⃣ Share your results at https://github.com/devMEremenko/XcodeBenchmark | ||
| `) | ||
| require.NoError(t, err) | ||
| fmt.Println(result) | ||
| require.Equal(t, 222*time.Second, result.Ended.Sub(result.Started)) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,108 @@ | ||
| package xcode | ||
|
|
||
| import ( | ||
| "fmt" | ||
| executorpkg "github.com/cirruslabs/tart/benchmark/internal/executor" | ||
| "github.com/gosuri/uitable" | ||
| "github.com/spf13/cobra" | ||
| "go.uber.org/zap" | ||
| "go.uber.org/zap/zapio" | ||
| "os" | ||
| "os/exec" | ||
| ) | ||
|
|
||
| var debug bool | ||
| var image string | ||
| var prepare string | ||
|
|
||
| func NewCommand() *cobra.Command { | ||
| cmd := &cobra.Command{ | ||
| Use: "xcode", | ||
| Short: "run XCode benchmarks", | ||
| RunE: run, | ||
| } | ||
|
|
||
| cmd.Flags().BoolVar(&debug, "debug", false, "enable debug logging") | ||
| cmd.Flags().StringVar(&image, "image", "ghcr.io/cirruslabs/macos-sonoma-xcode:latest", "image to use for testing") | ||
| cmd.Flags().StringVar(&prepare, "prepare", "", "command to run before running each benchmark") | ||
|
|
||
| return cmd | ||
| } | ||
|
|
||
| func run(cmd *cobra.Command, args []string) error { | ||
| config := zap.NewProductionConfig() | ||
| if debug { | ||
| config.Level = zap.NewAtomicLevelAt(zap.DebugLevel) | ||
| } | ||
| logger, err := config.Build() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer func() { | ||
| _ = logger.Sync() | ||
| }() | ||
|
|
||
| table := uitable.New() | ||
| table.AddRow("Name", "Executor", "Time") | ||
|
|
||
| for _, benchmark := range benchmarks { | ||
| for _, executorInitializer := range executorpkg.DefaultInitializers(cmd.Context(), image, logger) { | ||
| if prepare != "" { | ||
| shell := "/bin/sh" | ||
|
|
||
| if shellFromEnv, ok := os.LookupEnv("SHELL"); ok { | ||
| shell = shellFromEnv | ||
| } | ||
|
|
||
| logger.Sugar().Infof("running prepare command %q using shell %q", | ||
| prepare, shell) | ||
|
|
||
| cmd := exec.CommandContext(cmd.Context(), shell, "-c", prepare) | ||
|
|
||
| loggerWriter := &zapio.Writer{Log: logger, Level: zap.DebugLevel} | ||
|
|
||
| cmd.Stdout = loggerWriter | ||
| cmd.Stderr = loggerWriter | ||
|
|
||
| if err := cmd.Run(); err != nil { | ||
| return fmt.Errorf("failed to run prepare command %q: %v", prepare, err) | ||
| } | ||
| } | ||
|
|
||
| logger.Sugar().Infof("initializing executor %s", executorInitializer.Name) | ||
|
|
||
| executor, err := executorInitializer.Fn() | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| logger.Sugar().Infof("running benchmark %q on %s executor", benchmark.Name, | ||
| executorInitializer.Name) | ||
|
|
||
| stdout, err := executor.Run(cmd.Context(), benchmark.Command) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| output, err := ParseOutput(string(stdout)) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| duration := output.Ended.Sub(output.Started) | ||
|
|
||
| logger.Sugar().Infof("Xcode benchmark duration: %s", duration) | ||
|
|
||
| table.AddRow(benchmark.Name, executorInitializer.Name, duration) | ||
|
|
||
| if err := executor.Close(); err != nil { | ||
| return fmt.Errorf("failed to close executor %s: %w", | ||
| executorInitializer.Name, err) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| fmt.Println(table.String()) | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| package executor | ||
|
|
||
| import ( | ||
| "context" | ||
| "github.com/cirruslabs/tart/benchmark/internal/executor/local" | ||
| "github.com/cirruslabs/tart/benchmark/internal/executor/tart" | ||
| "go.uber.org/zap" | ||
| ) | ||
|
|
||
| type Initializer struct { | ||
| Name string | ||
| Fn func() (Executor, error) | ||
| } | ||
|
|
||
| func DefaultInitializers(ctx context.Context, image string, logger *zap.Logger) []Initializer { | ||
| return []Initializer{ | ||
| { | ||
| Name: "local", | ||
| Fn: func() (Executor, error) { | ||
| return local.New(logger) | ||
| }, | ||
| }, | ||
| { | ||
| Name: "Tart", | ||
| Fn: func() (Executor, error) { | ||
| return tart.New(ctx, image, nil, logger) | ||
| }, | ||
| }, | ||
| { | ||
| Name: "Tart (--root-disk-opts=\"sync=none\")", | ||
| Fn: func() (Executor, error) { | ||
| return tart.New(ctx, image, []string{ | ||
| "--root-disk-opts", | ||
| "sync=none", | ||
| }, logger) | ||
| }, | ||
| }, | ||
| { | ||
| Name: "Tart (--root-disk-opts=\"caching=cached\")", | ||
| Fn: func() (Executor, error) { | ||
| return tart.New(ctx, image, []string{ | ||
| "--root-disk-opts", | ||
| "caching=cached", | ||
| }, logger) | ||
| }, | ||
| }, | ||
| { | ||
| Name: "Tart (--root-disk-opts=\"sync=none,caching=cached\")", | ||
| Fn: func() (Executor, error) { | ||
| return tart.New(ctx, image, []string{ | ||
| "--root-disk-opts", | ||
| "sync=none,caching=cached", | ||
| }, logger) | ||
| }, | ||
| }, | ||
| } | ||
| } |