Optimize OCI archive imports, fix up Bass's build

.gitignore

@@ -1,6 +1,7 @@
 # binaries are the path to darkness
 *.tgz
 *.zip
+*.tar
 
 # git worktree with wip branch
 /wip

Makefile

@@ -6,14 +6,11 @@ VERSION ?= dev
 arches=amd64 arm arm64
 shims=$(foreach arch,$(arches),pkg/runtimes/bin/exe.$(arch))
 
-all: cmd/bass/bass
+all: dist
 
 pkg/runtimes/bin/exe.%: pkg/runtimes/shim/*.go
 	env GOOS=linux GOARCH=$* CGO_ENABLED=0 go build -ldflags "-s -w" -o $@ ./pkg/runtimes/shim
 
-cmd/bass/bass: shims
-	env GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=0 go build -trimpath -ldflags "-X main.version=$(VERSION)" -o ./cmd/bass/bass ./cmd/bass
-
 pkg/proto/%.pb.go: proto/%.proto
 	protoc -I=./proto --go_out=. --go-grpc_out=. proto/$*.proto
 
@@ -25,14 +22,19 @@ shims: $(shims)
 	which upx # required for compressing shims
 	upx $(shims) || true # swallow AlreadyPackedException :/
 
+.PHONY: dist
+dist:
+	mkdir -p ./dist/
+	env GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=0 go build -trimpath -ldflags "-X main.version=$(VERSION)" -o ./dist/bass ./cmd/bass
+
 .PHONY: install
-install: cmd/bass/bass
+install: shims dist
 	mkdir -p $(DESTDIR)
-	cp $< $(DESTDIR)
+	cp ./dist/bass $(DESTDIR)
 
 .PHONY: proto
 proto: pkg/proto/bass.pb.go pkg/proto/runtime.pb.go pkg/proto/progress.pb.go pkg/proto/memo.pb.go
 
 .PHONY: clean
 clean:
-	rm -f cmd/bass/bass $(shims)
+	rm -f ./dist/ $(shims)

bass/bass.bass

@@ -39,16 +39,12 @@
 (provide [deps deps+go]
   ; monolithic image containing dependencies for building and testing
   (defn deps [src]
-    (let [only-nix (subgit src ./nix/ ./flake.nix ./flake.lock ./default.nix)]
-      {:file (nix:result
-               (-> ($ nix build ".#depsOci")
-                   (with-mount only-nix/nix/ ./nix/)
-                   (with-mount only-nix/flake.nix ./flake.nix)
-                   (with-mount only-nix/flake.lock ./flake.lock)
-                   (with-mount only-nix/default.nix ./default.nix))
-               ./image.tar)
-       :platform {:os "linux"}
-       :tag "latest"}))
+    (oci-load
+      (nix:result
+        (cd (subgit src ./nix/ ./flake.nix ./flake.lock ./default.nix)
+          ($ nix build ".#depsImage"))
+        ./deps-image.tar)
+      {:os "linux"}))
 
   ; deps with Go dependencies pre-fetched
   (defn deps+go [src]
@@ -67,8 +63,7 @@
            (str "VERSION=" version)
            (str "GOOS=" os)
            (str "GOARCH=" arch)
-           "DESTDIR=./dist/"
-           install)
+           dist)
         (with-deps-and-shims src)
         (subpath ./dist/)))
 
@@ -77,21 +72,20 @@
     (archive src (dist src version os arch) os arch))
 
   (defn with-deps-and-shims [thunk src]
-    (let [submodules (git-submodules src)
-          only-shim (subgit src ./pkg/runtimes/shim/ ./Makefile ./go.mod ./go.sum & submodules)
-          built-shims (make-shims only-shim)]
-      (-> thunk
-          (with-mount src ./)
-          (with-mount built-shims/pkg/runtimes/bin/ ./pkg/runtimes/bin/)
-          with-go-cache
-          (with-image (deps+go src)))))
+    (-> thunk
+        (with-mount src ./)
+        (with-mount (make-shims src) ./pkg/runtimes/bin/)
+        with-go-cache
+        (with-image (deps+go src))))
 
   ; returns a thunk with the make targets built into the output directory, as
   ; an overlay of src
   (defn make-shims [src]
-    (-> ($ make -j shims)
-        (with-mount src ./)
-        (with-image (deps+go src))))
+    (let [submodules (git-submodules src)]
+      (-> (from (deps+go src)
+            (cd (subgit src ./pkg/runtimes/shim/ ./Makefile ./go.mod ./go.sum & submodules)
+              ($ make -j shims)))
+          (subpath ./pkg/runtimes/bin/))))
 
   ; creates an archive appropriate for the given platform
   (defn archive [src out os arch]
@@ -234,5 +228,7 @@
 
 (defn with-go-cache [thunk]
   (-> thunk
-      (with-mount (cache-dir "bass go mod") /go/pkg/mod/)
-      (with-mount (cache-dir "bass go cache") /root/.cache/go-build)))
+      (with-env {:GOCACHE /go/cache/build/
+                 :GOMODCACHE /go/cache/mod/})
+      (with-mount (cache-dir "bass go mod") /go/cache/mod/)
+      (with-mount (cache-dir "bass go cache") /go/cache/build/)))

bass/bass.lock

@@ -456,6 +456,62 @@ memos: {
         }
       }
     }
+    results: {
+      input: {
+        array: {
+          values: {
+            object: {
+              bindings: {
+                symbol: "platform"
+                value: {
+                  object: {
+                    bindings: {
+                      symbol: "os"
+                      value: {
+                        string: {
+                          value: "linux"
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+              bindings: {
+                symbol: "repository"
+                value: {
+                  string: {
+                    value: "golang"
+                  }
+                }
+              }
+              bindings: {
+                symbol: "tag"
+                value: {
+                  string: {
+                    value: "latest"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+      output: {
+        thunk: {
+          image: {
+            ref: {
+              platform: {
+                os: "linux"
+                arch: "amd64"
+              }
+              repository: "golang"
+              tag: "latest"
+              digest: "sha256:403f48633fb5ebd49f9a2b6ad6719f912df23dae44974a0c9445be331e72ff5e"
+            }
+          }
+        }
+      }
+    }
   }
 }
 memos: {
@@ -525,5 +581,26 @@ memos: {
         }
       }
     }
+    results: {
+      input: {
+        array: {
+          values: {
+            string: {
+              value: "https://github.com/containers/dnsname"
+            }
+          }
+          values: {
+            string: {
+              value: "v1.3.1"
+            }
+          }
+        }
+      }
+      output: {
+        string: {
+          value: "512aea91bf1095fbcad0abceec38798df5c40c14"
+        }
+      }
+    }
   }
 }

bass/build

@@ -10,4 +10,4 @@
     (log "building + smoke-testing" :version version :os os :arch arch)
     (let [dist (bass:build src version os arch)]
       (bass:smoke-test dist)
-      (emit dist *stdout*))))
+      (write dist *dir*/dist.tar))))

bass/buildkit.bass

@@ -1,3 +1,5 @@
+(def *memos* *dir*/bass.lock)
+
 (def *buildkit-repository* "moby/buildkit")
 
 ; bumped by hack/bump-buildkit

cmd/bass/bump.go

@@ -13,6 +13,11 @@ import (
 )
 
 func bump(ctx context.Context) error {
+	ctx, _, err := setupPool(ctx, true)
+	if err != nil {
+		return err
+	}
+
 	return cli.Task(ctx, cmdline, func(ctx context.Context, vertex *progrock.VertexRecorder) error {
 		for _, bumpLock := range flags.Args() {
 			if err := bumpLockfile(ctx, bumpLock); err != nil {

cmd/bass/export.go

@@ -18,6 +18,11 @@ import (
 )
 
 func export(ctx context.Context) error {
+	ctx, _, err := setupPool(ctx, true)
+	if err != nil {
+		return err
+	}
+
 	return cli.Task(ctx, cmdline, func(ctx context.Context, vertex *progrock.VertexRecorder) error {
 		dec := bass.NewRawDecoder(os.Stdin)
 

cmd/bass/main.go

@@ -105,26 +105,6 @@ func main() {
 	}
 }
 
-var DefaultConfig = bass.Config{
-	Runtimes: []bass.RuntimeConfig{},
-}
-
-func init() {
-	var runtime string
-	if os.Getenv("DAGGER_SESSION_PORT") != "" || os.Getenv("_EXPERIMENTAL_DAGGER_CLI_BIN") != "" {
-		runtime = runtimes.DaggerName
-	} else {
-		runtime = runtimes.BuildkitName
-	}
-
-	DefaultConfig.Runtimes = []bass.RuntimeConfig{
-		{
-			Platform: bass.LinuxPlatform,
-			Runtime:  runtime,
-		},
-	}
-}
-
 func root(ctx context.Context) error {
 	if showVersion {
 		printVersion(ctx)
@@ -165,20 +145,6 @@ func root(ctx context.Context) error {
 		return frontend(ctx)
 	}
 
-	config, err := bass.LoadConfig(DefaultConfig)
-	if err != nil {
-		cli.WriteError(ctx, err)
-		return err
-	}
-
-	pool, err := runtimes.NewPool(ctx, config)
-	if err != nil {
-		cli.WriteError(ctx, err)
-		return err
-	}
-
-	ctx = bass.WithRuntimePool(ctx, pool)
-
 	if runnerAddr != "" {
 		client, err := runnerClient(ctx, runnerAddr)
 		if err != nil {
@@ -187,7 +153,7 @@ func root(ctx context.Context) error {
 		}
 
 		return cli.WithProgress(ctx, func(ctx context.Context) error {
-			return runnerLoop(ctx, client, pool.Runtimes)
+			return runnerLoop(ctx, client)
 		})
 	}
 
@@ -218,13 +184,39 @@ func root(ctx context.Context) error {
 	return cli.WithProgress(ctx, run)
 }
 
-func repl(ctx context.Context) error {
-	scope := bass.NewRunScope(bass.Ground, bass.RunState{
-		Dir:    bass.NewHostDir("."),
-		Stdin:  bass.Stdin,
-		Stdout: bass.Stdout,
-		Env:    bass.ImportSystemEnv(),
-	})
+func setupPool(ctx context.Context, oneShot bool) (context.Context, *runtimes.Pool, error) {
+	defaultConfig := bass.Config{
+		Runtimes: []bass.RuntimeConfig{},
+	}
+
+	var runtime string
+	if os.Getenv("DAGGER_SESSION_PORT") != "" || os.Getenv("_EXPERIMENTAL_DAGGER_CLI_BIN") != "" {
+		runtime = runtimes.DaggerName
+	} else {
+		runtime = runtimes.BuildkitName
+	}
+
+	defaultConfig.Runtimes = []bass.RuntimeConfig{
+		{
+			Platform: bass.LinuxPlatform,
+			Runtime:  runtime,
+			Config: bass.Bindings{
+				"oneshot": bass.Bool(oneShot),
+			}.Scope(),
+		},
+	}
+
+	config, err := bass.LoadConfig(defaultConfig)
+	if err != nil {
+		cli.WriteError(ctx, err)
+		return nil, nil, err
+	}
+
+	pool, err := runtimes.NewPool(ctx, config)
+	if err != nil {
+		cli.WriteError(ctx, err)
+		return nil, nil, err
+	}
 
-	return cli.Repl(ctx, scope)
+	return bass.WithRuntimePool(ctx, pool), pool, nil
 }

cmd/bass/prune.go

@@ -10,6 +10,11 @@ import (
 )
 
 func prune(ctx context.Context) error {
+	ctx, _, err := setupPool(ctx, true)
+	if err != nil {
+		return err
+	}
+
 	return cli.Task(ctx, cmdline, func(ctx context.Context, vertex *progrock.VertexRecorder) error {
 		pool, err := bass.RuntimePoolFromContext(ctx)
 		if err != nil {

cmd/bass/repl.go

@@ -0,0 +1,24 @@
+package main
+
+import (
+	"context"
+
+	"github.com/vito/bass/pkg/bass"
+	"github.com/vito/bass/pkg/cli"
+)
+
+func repl(ctx context.Context) error {
+	ctx, _, err := setupPool(ctx, false)
+	if err != nil {
+		return err
+	}
+
+	scope := bass.NewRunScope(bass.Ground, bass.RunState{
+		Dir:    bass.NewHostDir("."),
+		Stdin:  bass.Stdin,
+		Stdout: bass.Stdout,
+		Env:    bass.ImportSystemEnv(),
+	})
+
+	return cli.Repl(ctx, scope)
+}