From b9e7312f06f4c852c0a15a21adb1a300e1010bae Mon Sep 17 00:00:00 2001 From: Alex Suraci Date: Sun, 9 Apr 2023 22:05:16 -0400 Subject: [PATCH 1/9] add Bass Buildkit frontend Bass can now be used anywhere that accepts a Dockerfile. (Assuming they're using buildx/Buildkit.) --- .bassignore | 3 + Dockerfile | 10 + Dockerfile.bass | 3 + bass/bass.lock | 2 +- bass/bump-frontend | 24 + cmd/bass/frontend.go | 244 +++++++ cmd/bass/main.go | 13 +- go.sum | 2 - nix/vendorSha256.txt | 2 +- pkg/bass/ground.go | 2 +- pkg/bass/host_path.go | 17 +- pkg/bass/reader.go | 2 + pkg/basstls/basstls.go | 2 + pkg/runtimes/buildkit.go | 935 +++++++++++++++--------- pkg/runtimes/shim/run.go | 2 +- pkg/runtimes/shim/tls.go | 17 +- pkg/runtimes/testdata/docker-build.bass | 13 +- pkg/runtimes/util/reffs.go | 120 +++ 18 files changed, 1041 insertions(+), 372 deletions(-) create mode 100644 Dockerfile create mode 100644 Dockerfile.bass create mode 100755 bass/bump-frontend create mode 100644 cmd/bass/frontend.go create mode 100644 pkg/runtimes/util/reffs.go diff --git a/.bassignore b/.bassignore index 9ac43290..21004ad8 100644 --- a/.bassignore +++ b/.bassignore @@ -1,3 +1,6 @@ Session.vim go.work go.work.sum +.direnv +.bassignore +.git/index diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..4f90153a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +# syntax = basslang/frontend:dev + +(use (*dir*/bass/bass.bass)) + +(def dist + (bass:dist *context* "dev" "linux" "amd64")) + +(-> (from (linux/alpine) + ($ cp dist/bass /usr/local/bin/bass)) + (with-entrypoint ["bass"])) diff --git a/Dockerfile.bass b/Dockerfile.bass new file mode 100644 index 00000000..859136be --- /dev/null +++ b/Dockerfile.bass @@ -0,0 +1,3 @@ +(run (from (docker-build *dir* {:os "linux"}) + ($ bass --version) + ($$ --version))) diff --git a/bass/bass.lock b/bass/bass.lock index 852c360b..9f6e0b44 100644 --- a/bass/bass.lock +++ b/bass/bass.lock @@ -521,7 +521,7 @@ memos: { } output: { string: { - value: "0fd154b53db44c0dced9b61092db79953d69bd13" + value: "a6c8ee8f81531c08c5136feca23cc6cca0be3b24" } } } diff --git a/bass/bump-frontend b/bass/bump-frontend new file mode 100755 index 00000000..5dbf1d12 --- /dev/null +++ b/bass/bump-frontend @@ -0,0 +1,24 @@ +#!/usr/bin/env bass + +(use (*dir*/bass.bass)) + +(def {:src src + (:version "dev") version + (:os "linux") os + (:arch "amd64") arch} + (next *stdin*)) + +(def dist + (bass:dist src version os arch)) + +(def thunk + (-> (from (linux/alpine) + ($ cp dist/bass /usr/local/bin/bass)) + (with-entrypoint ["bass" "--frontend"]) + (with-label :moby.buildkit.frontend.network.none "true") + (with-label :moby.buildkit.frontend.caps + "moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests,moby.buildkit.frontend.contexts"))) + +(let [ref (str "basslang/frontend:" version) + published (publish thunk ref)] + (log "published" :ref published)) diff --git a/cmd/bass/frontend.go b/cmd/bass/frontend.go new file mode 100644 index 00000000..7689e38a --- /dev/null +++ b/cmd/bass/frontend.go @@ -0,0 +1,244 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "io/fs" + "os" + "path" + "runtime" + + "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + gwclient "github.com/moby/buildkit/frontend/gateway/client" + "github.com/moby/buildkit/frontend/gateway/grpcclient" + "github.com/moby/buildkit/util/apicaps" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/vito/bass/pkg/bass" + "github.com/vito/bass/pkg/basstls" + "github.com/vito/bass/pkg/cli" + "github.com/vito/bass/pkg/runtimes" + "github.com/vito/bass/pkg/runtimes/util" +) + +func frontend(ctx context.Context) error { + err := grpcclient.RunFromEnvironment(ctx, frontendBuild) + if err != nil { + cli.WriteError(ctx, err) + } + + return err +} + +// mimic dockerfile.v1 frontend +const ( + localNameContext = "context" + localNameDockerfile = "dockerfile" + localNameBassTLS = "bass-tls" + keyFilename = "filename" +) + +type InputsFilesystem struct { + ctx context.Context + gw gwclient.Client + caps apicaps.CapSet + inputs map[string]llb.State +} + +var _ bass.Filesystem = &InputsFilesystem{} + +func (fs *InputsFilesystem) FS(contextDir string) (fs.FS, error) { + input, found := fs.inputs[contextDir] + if !found { + return nil, fmt.Errorf("unknown input: %s", contextDir) + } + + return util.OpenRefFS(fs.ctx, fs.gw, input, llb.WithCaps(fs.caps)) +} + +func (fs *InputsFilesystem) Write(path string, r io.Reader) error { + return nil +} + +func frontendBuild(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { + caps := gw.BuildOpts().Caps + + scriptFn := gw.BuildOpts().Opts[keyFilename] + if scriptFn == "" { + scriptFn = "Dockerfile" + } + + inputs, err := gw.Inputs(ctx) + if err != nil { + return nil, err + } + + _, found := inputs[localNameContext] + if !found { + // running from 'docker build', which doesn't set inputs + inputs[localNameContext] = llb.Local(localNameContext, + llb.SessionID(gw.BuildOpts().SessionID), + llb.WithCustomName("[internal] local bass workdir"), + ) + } + + scriptInput, found := inputs[localNameDockerfile] + if !found { + // running from 'docker build', which doesn't set inputs + scriptInput = llb.Local(localNameDockerfile, + llb.SessionID(gw.BuildOpts().SessionID), + llb.WithCustomName("[internal] local bass script"), + ) + + inputs[localNameDockerfile] = scriptInput + } + + // Override real filesystem with one that knows how to read directly from the + // given inputs, and discard writes. + bass.FS = &InputsFilesystem{ + ctx: ctx, + gw: gw, + caps: caps, + inputs: inputs, + } + + var certsDir string + certsInput, found := inputs[localNameBassTLS] + if found { + certFS, err := util.OpenRefFS(ctx, gw, certsInput, llb.WithCaps(caps)) + if err != nil { + return nil, err + } + + certsDir = basstls.DefaultDir + if err := os.MkdirAll(certsDir, 0700); err != nil { + return nil, err + } + + for _, name := range basstls.CAFiles { + source, err := certFS.Open(name) + if err != nil { + return nil, err + } + + fi, err := source.Stat() + if err != nil { + return nil, err + } + + target, err := os.OpenFile(path.Join(certsDir, name), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, fi.Mode()) + if err != nil { + return nil, err + } + + if _, err := io.Copy(target, source); err != nil { + return nil, err + } + + if err := target.Close(); err != nil { + return nil, err + } + + if err := source.Close(); err != nil { + return nil, err + } + } + } + + scriptFs, err := util.OpenRefFS(ctx, gw, scriptInput, llb.WithCaps(caps)) + if err != nil { + return nil, err + } + + // contextFs, err := newRefFS(ctx, gw, contextInput, llb.WithCaps(caps)) + // if err != nil { + // return nil, err + // } + + pool := &runtimes.Pool{} + + kitdruntime, err := runtimes.NewBuildkitFrontend(gw, inputs, runtimes.BuildkitConfig{ + CertsDir: certsDir, + }) + if err != nil { + return nil, err + } + + pool.Runtimes = append(pool.Runtimes, runtimes.Assoc{ + Runtime: kitdruntime, + Platform: bass.LinuxPlatform, + }) + + ctx = bass.WithRuntimePool(ctx, pool) + + runSt := bass.RunState{ + Env: bass.NewEmptyScope(), // TODO: build args + Dir: bass.NewFSPath(scriptFs, bass.ParseFileOrDirPath(path.Dir(scriptFn))), + Stdin: bass.Stdin, + Stdout: bass.Stdout, + } + + module := bass.NewRunScope(bass.Ground, runSt) + // directly pass the context by local name, masquerading it as the host path + module.Set("*context*", bass.NewHostDir(localNameContext)) + + val, err := bass.EvalFSFile(ctx, module, bass.NewFSPath(scriptFs, bass.ParseFileOrDirPath(scriptFn))) + if err != nil { + return nil, err + } + + var thunk bass.Thunk + if err := val.Decode(&thunk); err != nil { + return nil, err + } + + platform := ocispecs.Platform{ + OS: runtime.GOOS, + Architecture: runtime.GOARCH, + } + + builder := kitdruntime.NewBuilder(ctx, gw) + + ib, err := builder.Build( + ctx, + thunk, + func(st llb.ExecState, sourcePath string) llb.State { + return st.Root() + }, + false, // don't run any entrypoint + ) + if err != nil { + return nil, err + } + + def, err := ib.FS.Marshal(ctx, llb.WithCaps(caps)) + if err != nil { + return nil, err + } + + outRes, err := gw.Solve(ctx, gwclient.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, err + } + + if _, hasConfig := outRes.Metadata[exptypes.ExporterImageConfigKey]; !hasConfig { + configBytes, err := json.Marshal(ocispecs.Image{ + Architecture: platform.Architecture, + OS: platform.OS, + OSVersion: platform.OSVersion, + OSFeatures: platform.OSFeatures, + Config: ib.Config, + }) + if err != nil { + return nil, err + } + + outRes.AddMeta(exptypes.ExporterImageConfigKey, configBytes) + } + + return outRes, nil +} diff --git a/cmd/bass/main.go b/cmd/bass/main.go index ea8e75a8..49a6bc57 100644 --- a/cmd/bass/main.go +++ b/cmd/bass/main.go @@ -10,6 +10,7 @@ import ( "runtime/pprof" "strings" + "github.com/moby/buildkit/util/appcontext" flag "github.com/spf13/pflag" "github.com/vito/bass/pkg/bass" "github.com/vito/bass/pkg/cli" @@ -34,6 +35,8 @@ var runnerAddr string var runLSP bool var lspLogs string +var runFrontend bool + var profPort int var profFilePath string @@ -58,6 +61,8 @@ func init() { flags.BoolVar(&runLSP, "lsp", false, "run the bass language server") flags.StringVar(&lspLogs, "lsp-log-file", "", "write language server logs to this file") + flags.BoolVar(&runFrontend, "frontend", false, "run the bass buildkit frontend") + flags.IntVar(&profPort, "profile", 0, "port number to bind for Go HTTP profiling") flags.StringVar(&profFilePath, "cpu-profile", "", "take a CPU profile and save it to this path") @@ -76,7 +81,9 @@ func logLevel() zapcore.LevelEnabler { } func main() { - ctx := context.Background() + // reusing for convenience; originally for frontend + ctx := appcontext.Context() + ctx = bass.WithTrace(ctx, &bass.Trace{}) ctx = ioctx.StderrToContext(ctx, os.Stderr) @@ -154,6 +161,10 @@ func root(ctx context.Context) error { defer pprof.StopCPUProfile() } + if runFrontend { + return frontend(ctx) + } + config, err := bass.LoadConfig(DefaultConfig) if err != nil { cli.WriteError(ctx, err) diff --git a/go.sum b/go.sum index bb5fffc6..f05a154e 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,6 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dagger.io/dagger v0.5.2 h1:xNMUnLWqcsb5UrvqOgJx4MJVfe7xDb50kBMXdC63Cjs= -dagger.io/dagger v0.5.2/go.mod h1:1nbGnLdIfoBV2ahbQjheI//SNGz+b5q1jqf0A+pJ+Oc= dagger.io/dagger v0.6.0 h1:3cN0QxS/re2RKyHW3BGyk/Hz7Ux46EfvB4zbZLA/X6o= dagger.io/dagger v0.6.0/go.mod h1:/sSGPh+1LInVuHzTkkr1pZ5N0BAEDoqJ94eM2Xoh/iE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= diff --git a/nix/vendorSha256.txt b/nix/vendorSha256.txt index a32ec7ec..4c1b7db5 100644 --- a/nix/vendorSha256.txt +++ b/nix/vendorSha256.txt @@ -1 +1 @@ -sha256-dzdfzE4r36vx2HoWPGQyM/wnSAH2cnTVfoePFRLekD0= +sha256-tUuwQbJZwz5yaKj559CsIcDTOpRojdR6EV7jVI6Elz8= diff --git a/pkg/bass/ground.go b/pkg/bass/ground.go index a39bb703..efdc284d 100644 --- a/pkg/bass/ground.go +++ b/pkg/bass/ground.go @@ -1132,5 +1132,5 @@ func zapField(k Symbol, v Value) (zap.Field, error) { return zap.Object(name, om), nil } - return zap.Field{}, EncodeError{v} + return zap.String(name, v.String()), nil } diff --git a/pkg/bass/host_path.go b/pkg/bass/host_path.go index fc90416d..561ba093 100644 --- a/pkg/bass/host_path.go +++ b/pkg/bass/host_path.go @@ -45,7 +45,7 @@ func ParseHostPath(path string) HostPath { } type Filesystem interface { - FS(root string) fs.FS + FS(root string) (fs.FS, error) Write(path string, r io.Reader) error } @@ -55,11 +55,13 @@ var FS Filesystem = HostFilesystem{} type HostFilesystem struct{} +var _ Filesystem = HostFilesystem{} + // AtomicSuffix is appended to the Write destination to form a path used for // atomic writes. const AtomicSuffix = ".new" -func (HostFilesystem) FS(root string) fs.FS { return os.DirFS(root) } +func (HostFilesystem) FS(root string) (fs.FS, error) { return os.DirFS(root), nil } func (HostFilesystem) Write(path string, r io.Reader) error { atomic := path + AtomicSuffix @@ -84,7 +86,9 @@ func (HostFilesystem) Write(path string, r io.Reader) error { type DiscardFilesystem struct{} -func (DiscardFilesystem) FS(root string) fs.FS { return os.DirFS(root) } +var _ Filesystem = DiscardFilesystem{} + +func (DiscardFilesystem) FS(root string) (fs.FS, error) { return os.DirFS(root), nil } func (DiscardFilesystem) Write(path string, r io.Reader) error { return nil } @@ -211,7 +215,12 @@ func (path HostPath) Open(context.Context) (io.ReadCloser, error) { return nil, err } - return FS.FS(path.ContextDir).Open(rel) + fs, err := FS.FS(path.ContextDir) + if err != nil { + return nil, err + } + + return fs.Open(rel) } func (path HostPath) Write(ctx context.Context, src io.Reader) error { diff --git a/pkg/bass/reader.go b/pkg/bass/reader.go index 1dc4dc01..ec26c90e 100644 --- a/pkg/bass/reader.go +++ b/pkg/bass/reader.go @@ -74,6 +74,8 @@ func NewReader(src io.Reader, file Readable) *Reader { r.SetMacro(';', false, reader.readCommented) r.SetMacro('^', false, reader.readMeta) r.SetMacro('!', true, readShebang) + // skip '# ' as a comment too for e.g. Dockerfile frontends + r.SetMacro(' ', true, readShebang) r.SetMacro('\'', false, nil) r.SetMacro('~', false, nil) r.SetMacro('`', false, nil) diff --git a/pkg/basstls/basstls.go b/pkg/basstls/basstls.go index fdb4a28d..7d1b612d 100644 --- a/pkg/basstls/basstls.go +++ b/pkg/basstls/basstls.go @@ -34,6 +34,8 @@ var ( DefaultDir = filepath.Join(xdg.ConfigHome, "bass", "tls") ) +var CAFiles = []string{CAName + ".crt", CAName + ".key"} + // CACert returns the path to the CA certificate in the given dir. func CACert(dir string) string { return filepath.Join(dir, CAName+".crt") diff --git a/pkg/runtimes/buildkit.go b/pkg/runtimes/buildkit.go index ed50712c..d38d692a 100644 --- a/pkg/runtimes/buildkit.go +++ b/pkg/runtimes/buildkit.go @@ -1,6 +1,7 @@ package runtimes import ( + "archive/tar" "context" "embed" "encoding/json" @@ -10,6 +11,7 @@ import ( "os" "path" "path/filepath" + "runtime" "strings" "sync" "syscall" @@ -25,7 +27,7 @@ import ( dockerconfig "github.com/docker/cli/cli/config" "github.com/docker/distribution/reference" "github.com/hashicorp/go-multierror" - kitdclient "github.com/moby/buildkit/client" + bkclient "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/frontend/dockerfile/dockerignore" @@ -33,11 +35,14 @@ import ( gwclient "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/auth/authprovider" + "github.com/moby/buildkit/session/filesync" "github.com/moby/buildkit/session/secrets/secretsprovider" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/entitlements" "github.com/morikuni/aec" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/tonistiigi/fsutil" + fstypes "github.com/tonistiigi/fsutil/types" "github.com/tonistiigi/units" "github.com/vito/progrock" "github.com/vito/progrock/graph" @@ -47,6 +52,7 @@ import ( "github.com/vito/bass/pkg/basstls" "github.com/vito/bass/pkg/cli" "github.com/vito/bass/pkg/ioctx" + "github.com/vito/bass/pkg/runtimes/util" "github.com/vito/bass/pkg/runtimes/util/buildkitd" "github.com/vito/bass/pkg/zapctx" ) @@ -102,8 +108,11 @@ func init() { type Buildkit struct { Config BuildkitConfig - Client *kitdclient.Client Platform ocispecs.Platform + Inputs map[string]llb.State + + client *bkclient.Client + gateway gwclient.Client authp session.Attachable @@ -112,6 +121,39 @@ type Buildkit struct { const DefaultBuildkitInstallation = "bass-buildkitd" +func (buildkit *Buildkit) Client() (*bkclient.Client, error) { + if buildkit.client == nil { + return nil, fmt.Errorf("buildkit client unavailable") + } + + return buildkit.client, nil +} + +func (runtime *Buildkit) WithGateway(ctx context.Context, doBuild func(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error)) error { + if runtime.gateway != nil { + _, err := doBuild(ctx, runtime.gateway) + return err + } + + statusProxy := forwardStatus(progrock.RecorderFromContext(ctx)) + defer statusProxy.Wait() + + _, err := runtime.client.Build(ctx, bkclient.SolveOpt{ + Session: []session.Attachable{ + runtime.authp, + filesync.NewFSSyncProvider(AnyDirSource{}), + }, + OCIStores: map[string]content.Store{ + ociStoreName: runtime.ociStore, + }, + }, buildkitProduct, doBuild, statusProxy.Writer()) + if err != nil { + return statusProxy.NiceError("resolve failed", err) + } + + return nil +} + func NewBuildkit(ctx context.Context, _ bass.RuntimePool, cfg *bass.Scope) (bass.Runtime, error) { var config BuildkitConfig if cfg != nil { @@ -128,13 +170,20 @@ func NewBuildkit(ctx context.Context, _ bass.RuntimePool, cfg *bass.Scope) (bass config.Installation = DefaultBuildkitInstallation } - err := basstls.Init(config.CertsDir) + if config.OCIStoreDir == "" { + config.OCIStoreDir = filepath.Join(xdg.DataHome, "bass", "oci") + } + + store, err := local.NewStore(config.OCIStoreDir) if err != nil { - return nil, fmt.Errorf("init tls depot: %w", err) + return nil, fmt.Errorf("create oci store: %w", err) } - if config.OCIStoreDir == "" { - config.OCIStoreDir = filepath.Join(xdg.DataHome, "bass", "oci") + if config.CertsDir != "" { + err := basstls.Init(config.CertsDir) + if err != nil { + return nil, fmt.Errorf("init tls depot: %w", err) + } } client, err := dialBuildkit(ctx, config.Addr, config.Installation, config.CertsDir) @@ -147,8 +196,8 @@ func NewBuildkit(ctx context.Context, _ bass.RuntimePool, cfg *bass.Scope) (bass return nil, fmt.Errorf("list buildkit workers: %w", err) } - var platform ocispecs.Platform var checkSame platforms.Matcher + var platform ocispecs.Platform for _, w := range workers { if checkSame != nil && !checkSame.Match(w.Platforms[0]) { return nil, fmt.Errorf("TODO: workers have different platforms: %s != %s", w.Platforms[0], platform) @@ -158,57 +207,45 @@ func NewBuildkit(ctx context.Context, _ bass.RuntimePool, cfg *bass.Scope) (bass checkSame = platforms.Only(platform) } - store, err := local.NewStore(config.OCIStoreDir) - if err != nil { - return nil, fmt.Errorf("create oci store: %w", err) - } - return &Buildkit{ - Config: config, - Client: client, + Config: config, + + // TODO: report all supported platforms by workers instead Platform: platform, + client: client, + gateway: nil, + authp: authprovider.NewDockerAuthProvider(dockerconfig.LoadDefaultConfigFile(os.Stderr)), ociStore: store, }, nil } -func dialBuildkit(ctx context.Context, addr string, installation string, certsDir string) (*kitdclient.Client, error) { - if addr == "" { - addr = os.Getenv("BUILDKIT_HOST") +func NewBuildkitFrontend(gw gwclient.Client, inputs map[string]llb.State, config BuildkitConfig) (*Buildkit, error) { + if config.OCIStoreDir == "" { + config.OCIStoreDir = filepath.Join(xdg.DataHome, "bass", "oci") } - if addr == "" { - sockPath, err := xdg.SearchConfigFile("bass/buildkitd.sock") - if err == nil { - // support respecting XDG_RUNTIME_DIR instead of assuming /run/ - addr = "unix://" + sockPath - } - - sockPath, err = xdg.SearchRuntimeFile("buildkit/buildkitd.sock") - if err == nil { - // support respecting XDG_RUNTIME_DIR instead of assuming /run/ - addr = "unix://" + sockPath - } + ociStore, err := local.NewStore(config.OCIStoreDir) + if err != nil { + return nil, fmt.Errorf("create oci store: %w", err) } - var errs error - if addr == "" { - var startErr error - addr, startErr = buildkitd.Start(ctx, installation, certsDir) - if startErr != nil { - errs = multierror.Append(startErr) - } - } + return &Buildkit{ + Config: config, + Platform: ocispecs.Platform{ + OS: runtime.GOOS, + Architecture: runtime.GOARCH, + }, + Inputs: inputs, - client, err := kitdclient.New(context.TODO(), addr) - if err != nil { - errs = multierror.Append(errs, err) - return nil, errs - } + gateway: gw, - return client, nil + // NB: same as above, just assigning anyway + authp: authprovider.NewDockerAuthProvider(dockerconfig.LoadDefaultConfigFile(os.Stderr)), + ociStore: ociStore, + }, nil } func (runtime *Buildkit) Resolve(ctx context.Context, imageRef bass.ImageRef) (bass.Thunk, error) { @@ -216,7 +253,7 @@ func (runtime *Buildkit) Resolve(ctx context.Context, imageRef bass.ImageRef) (b ctx, svcs := bass.TrackRuns(ctx) defer svcs.StopAndWait() - ref, err := runtime.ref(ctx, imageRef) + ref, err := ref(ctx, runtime, imageRef) if err != nil { // TODO: it might make sense to resolve an OCI archive ref to a digest too return bass.Thunk{}, fmt.Errorf("resolve ref %v: %w", imageRef, err) @@ -228,10 +265,7 @@ func (runtime *Buildkit) Resolve(ctx context.Context, imageRef bass.ImageRef) (b return bass.Thunk{}, fmt.Errorf("normalize ref: %w", err) } - statusProxy := forwardStatus(progrock.RecorderFromContext(ctx)) - defer statusProxy.Wait() - - doBuild := func(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { + err = runtime.WithGateway(ctx, func(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { digest, _, err := gw.ResolveImageConfig(ctx, normalized.String(), llb.ResolveImageConfigOpt{ Platform: &runtime.Platform, }) @@ -242,15 +276,9 @@ func (runtime *Buildkit) Resolve(ctx context.Context, imageRef bass.ImageRef) (b imageRef.Digest = digest.String() return &gwclient.Result{}, nil - } - - _, err = runtime.Client.Build(ctx, kitdclient.SolveOpt{ - Session: []session.Attachable{ - runtime.authp, - }, - }, buildkitProduct, doBuild, statusProxy.Writer()) + }) if err != nil { - return bass.Thunk{}, statusProxy.NiceError("resolve failed", err) + return bass.Thunk{}, fmt.Errorf("resolve image config: %w", err) } return imageRef.Thunk(), nil @@ -266,37 +294,68 @@ func (runtime *Buildkit) Run(ctx context.Context, thunk bass.Thunk) error { return st.GetMount(ioDir) }, nil, // exports + nil, // callback true, // inherit entrypoint/cmd ) return err } func (runtime *Buildkit) Start(ctx context.Context, thunk bass.Thunk) (StartResult, error) { - ctx, stop := context.WithCancel(ctx) + var res StartResult + + err := runtime.WithGateway(ctx, func(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { + builder := runtime.NewBuilder(ctx, gw) + + var err error + res, err = builder.Start(ctx, thunk) + if err != nil { + return nil, err + } + + return gwclient.NewResult(), nil + }) + if err != nil { + return res, fmt.Errorf("start: %w", err) + } + return res, err +} + +func (b *buildkitBuilder) Start(ctx context.Context, thunk bass.Thunk) (StartResult, error) { host := thunk.Name() - health := runtime.newHealth(host, thunk.Ports) + health := newHealth(b.gw, b.platform, host, thunk.Ports) + + ib, err := b.Build(ctx, thunk, getWorkdir, true) + if err != nil { + return StartResult{}, err + } + + def, err := ib.FS.Marshal(ctx, llb.Platform(b.platform)) + if err != nil { + return StartResult{}, err + } + + ctx, stop := context.WithCancel(ctx) runs := bass.RunsFromContext(ctx) checked := make(chan error, 1) runs.Go(stop, func() error { - checked <- health.Check(ctx) + res := health.Check(ctx) + checked <- res return nil }) exited := make(chan error, 1) runs.Go(stop, func() error { - _, err := runtime.build( - ctx, - thunk, - func(st llb.ExecState, _ string) llb.State { - return st.GetMount(ioDir) - }, - nil, // exports - true, // inherit entrypoint/cmd - ) + _, err := b.gw.Solve(ctx, gwclient.SolveRequest{ + Evaluate: true, + Definition: def.ToPB(), + }) + if err != nil { + return err + } exited <- err @@ -304,7 +363,11 @@ func (runtime *Buildkit) Start(ctx context.Context, thunk bass.Thunk) (StartResu }) select { - case <-checked: + case err := <-checked: + if err != nil { + return StartResult{}, fmt.Errorf("check error: %w", err) + } + result := StartResult{ Ports: PortInfos{}, } @@ -332,48 +395,39 @@ func (runtime *Buildkit) Read(ctx context.Context, w io.Writer, thunk bass.Thunk ctx, svcs := bass.TrackRuns(ctx) defer svcs.StopAndWait() - hash, err := thunk.Hash() - if err != nil { - return err - } - - tmp, err := os.MkdirTemp("", "thunk-"+hash) - if err != nil { - return err - } - - defer os.RemoveAll(tmp) - - _, err = runtime.build( + _, err := runtime.build( ctx, thunk, func(st llb.ExecState, _ string) llb.State { return st.GetMount(ioDir) }, - []kitdclient.ExportEntry{ - { - Type: kitdclient.ExporterLocal, - OutputDir: tmp, - }, - }, - true, // inherit entrypoint/cmd - llb.AddEnv("_BASS_OUTPUT", outputFile), - ) - if err != nil { - return err - } + nil, + func(ctx context.Context, gw gwclient.Client, res *gwclient.Result) (*gwclient.Result, error) { + ref, err := res.SingleRef() + if err != nil { + return nil, err + } - response, err := os.Open(filepath.Join(tmp, filepath.Base(outputFile))) - if err == nil { - defer response.Close() + fs := util.NewRefFS(ctx, ref) - _, err = io.Copy(w, response) - if err != nil { - return fmt.Errorf("read response: %w", err) - } - } + f, err := fs.Open(path.Base(outputFile)) + if err != nil { + return nil, err + } - return nil + if _, err := io.Copy(w, f); err != nil { + return nil, err + } + + if err := f.Close(); err != nil { + return nil, err + } + + return res, nil + }, + true, // inherit entrypoint/cmd + ) + return err } type marshalable interface { @@ -389,14 +443,15 @@ func (runtime *Buildkit) Export(ctx context.Context, w io.Writer, thunk bass.Thu func(st llb.ExecState, _ string) llb.State { return st.Root() }, - []kitdclient.ExportEntry{ + []bkclient.ExportEntry{ { - Type: kitdclient.ExporterOCI, + Type: bkclient.ExporterOCI, Output: func(map[string]string) (io.WriteCloser, error) { return nopCloser{w}, nil }, }, }, + nil, // callback false, // do not inherit entrypoint/cmd ) return err @@ -417,15 +472,16 @@ func (runtime *Buildkit) Publish(ctx context.Context, ref bass.ImageRef, thunk b func(st llb.ExecState, _ string) llb.State { return st.Root() }, - []kitdclient.ExportEntry{ + []bkclient.ExportEntry{ { - Type: kitdclient.ExporterImage, + Type: bkclient.ExporterImage, Attrs: map[string]string{ "name": addr, "push": "true", }, }, }, + nil, // callback false, // do not inherit entrypoint/cmd ) if err != nil { @@ -447,30 +503,102 @@ func (runtime *Buildkit) ExportPath(ctx context.Context, w io.Writer, tp bass.Th thunk := tp.Thunk path := tp.Path - _, err := runtime.build( - ctx, - thunk, - func(st llb.ExecState, sp string) llb.State { - copyOpt := &llb.CopyInfo{} - if path.FilesystemPath().IsDir() { - copyOpt.CopyDirContentsOnly = true - } + var err error + if path.FilesystemPath().IsDir() || runtime.client != nil { + _, err = runtime.build( + ctx, + thunk, + func(st llb.ExecState, sp string) llb.State { + copyOpt := &llb.CopyInfo{} + if path.FilesystemPath().IsDir() { + copyOpt.CopyDirContentsOnly = true + } - return llb.Scratch().File( - llb.Copy(st.GetMount(workDir), filepath.Join(sp, path.FilesystemPath().FromSlash()), ".", copyOpt), - llb.WithCustomNamef("[hide] copy %s", path.Slash()), - ) - }, - []kitdclient.ExportEntry{ - { - Type: kitdclient.ExporterTar, - Output: func(map[string]string) (io.WriteCloser, error) { - return nopCloser{w}, nil + return llb.Scratch().File( + llb.Copy(st.GetMount(workDir), filepath.Join(sp, path.FilesystemPath().FromSlash()), ".", copyOpt), + llb.WithCustomNamef("[hide] copy %s", path.Slash()), + ) + }, + []bkclient.ExportEntry{ + { + Type: bkclient.ExporterTar, + Output: func(map[string]string) (io.WriteCloser, error) { + return nopCloser{w}, nil + }, }, }, - }, - true, // inherit entryopint/cmd - ) + nil, // callback? TODO + true, // inherit entryopint/cmd + ) + } else { + tw := tar.NewWriter(w) + _, err = runtime.build( + ctx, + thunk, + func(st llb.ExecState, sp string) llb.State { + copyOpt := &llb.CopyInfo{} + if path.FilesystemPath().IsDir() { + copyOpt.CopyDirContentsOnly = true + } + + return llb.Scratch().File( + llb.Copy(st.GetMount(workDir), filepath.Join(sp, path.FilesystemPath().FromSlash()), ".", copyOpt), + llb.WithCustomNamef("[hide] copy %s", path.Slash()), + ) + }, + nil, + func(ctx context.Context, gw gwclient.Client, res *gwclient.Result) (*gwclient.Result, error) { + ref, err := res.SingleRef() + if err != nil { + return nil, err + } + + stat, err := ref.StatFile(ctx, gwclient.StatRequest{ + Path: path.FilesystemPath().Name(), + }) + if err != nil { + return nil, err + } + + err = tw.WriteHeader(&tar.Header{ + Name: path.FilesystemPath().FromSlash(), + Mode: int64(stat.Mode), + Uid: int(stat.Uid), + Gid: int(stat.Gid), + Size: int64(stat.Size_), + ModTime: time.Unix(stat.ModTime/int64(time.Second), stat.ModTime%int64(time.Second)), + Linkname: stat.Linkname, + Devmajor: stat.Devmajor, + Devminor: stat.Devmajor, + }) + if err != nil { + return nil, fmt.Errorf("write tar header: %w", err) + } + + fs := util.NewRefFS(ctx, ref) + + f, err := fs.Open(path.FilesystemPath().Name()) + if err != nil { + return nil, err + } + + if _, err := io.Copy(w, f); err != nil { + return nil, err + } + + if err := f.Close(); err != nil { + return nil, err + } + + if err := tw.Close(); err != nil { + return nil, err + } + + return res, nil + }, + true, + ) + } return err } @@ -478,7 +606,7 @@ func (runtime *Buildkit) Prune(ctx context.Context, opts bass.PruneOpts) error { stderr := ioctx.StderrFromContext(ctx) tw := tabwriter.NewWriter(stderr, 2, 8, 2, ' ', 0) - ch := make(chan kitdclient.UsageInfo) + ch := make(chan bkclient.UsageInfo) printed := make(chan struct{}) total := int64(0) @@ -501,15 +629,20 @@ func (runtime *Buildkit) Prune(ctx context.Context, opts bass.PruneOpts) error { } }() - kitdOpts := []kitdclient.PruneOption{ - kitdclient.WithKeepOpt(opts.KeepDuration, opts.KeepBytes), + kitdOpts := []bkclient.PruneOption{ + bkclient.WithKeepOpt(opts.KeepDuration, opts.KeepBytes), } if opts.All { - kitdOpts = append(kitdOpts, kitdclient.PruneAll) + kitdOpts = append(kitdOpts, bkclient.PruneAll) + } + + client, err := runtime.Client() + if err != nil { + return err } - err := runtime.Client.Prune(ctx, ch, kitdOpts...) + err = client.Prune(ctx, ch, kitdOpts...) close(ch) <-printed if err != nil { @@ -522,20 +655,24 @@ func (runtime *Buildkit) Prune(ctx context.Context, opts bass.PruneOpts) error { } func (runtime *Buildkit) Close() error { - return runtime.Client.Close() + if runtime.client != nil { + return runtime.client.Close() + } + return nil } func (runtime *Buildkit) build( ctx context.Context, thunk bass.Thunk, transform func(llb.ExecState, string) llb.State, - exports []kitdclient.ExportEntry, + exports []bkclient.ExportEntry, + cb func(context.Context, gwclient.Client, *gwclient.Result) (*gwclient.Result, error), forceExec bool, runOpts ...llb.RunOption, -) (*kitdclient.SolveResponse, error) { +) (*bkclient.SolveResponse, error) { var def *llb.Definition var secrets map[string][]byte - var localDirs map[string]string + // var localDirs map[string]string var imageConfig ocispecs.ImageConfig var allowed []entitlements.Entitlement @@ -543,47 +680,34 @@ func (runtime *Buildkit) build( defer statusProxy.Wait() // build llb definition using the remote gateway for image resolution - _, err := runtime.Client.Build(ctx, kitdclient.SolveOpt{ - Session: []session.Attachable{runtime.authp}, - }, buildkitProduct, func(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { - b := runtime.newBuilder(ctx, gw) + err := runtime.WithGateway(ctx, func(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { + b := runtime.NewBuilder(ctx, gw) - ib, err := b.llb(ctx, thunk, transform, forceExec, runOpts...) + ib, err := b.Build(ctx, thunk, transform, forceExec, runOpts...) if err != nil { return nil, err } - if ib.needsInsecure { + if ib.NeedsInsecure { allowed = append(allowed, entitlements.EntitlementSecurityInsecure) } - localDirs = b.localDirs + // localDirs = b.localDirs secrets = b.secrets - imageConfig = ib.config + imageConfig = ib.Config - def, err = ib.output.Marshal(ctx) + def, err = ib.Output.Marshal(ctx) if err != nil { return nil, err } return &gwclient.Result{}, nil - }, statusProxy.Writer()) + }) if err != nil { - return nil, statusProxy.NiceError("llb build failed", err) + return nil, err } - res, err := runtime.Client.Build(ctx, kitdclient.SolveOpt{ - LocalDirs: localDirs, - AllowedEntitlements: allowed, - Session: []session.Attachable{ - runtime.authp, - secretsprovider.FromMap(secrets), - }, - Exports: exports, - OCIStores: map[string]content.Store{ - ociStoreName: runtime.ociStore, - }, - }, buildkitProduct, func(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { + doBuild := func(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { res, err := gw.Solve(ctx, gwclient.SolveRequest{ Evaluate: true, Definition: def.ToPB(), @@ -604,13 +728,41 @@ func (runtime *Buildkit) build( } res.AddMeta(exptypes.ExporterImageConfigKey, cfgBytes) - return res, nil - }, statusProxy.Writer()) + if cb != nil { + res, err = cb(ctx, gw, res) + } + + return res, err + } + + solveOpt := bkclient.SolveOpt{ + // LocalDirs: localDirs, + AllowedEntitlements: allowed, + Session: []session.Attachable{ + runtime.authp, + secretsprovider.FromMap(secrets), + filesync.NewFSSyncProvider(AnyDirSource{}), + }, + Exports: exports, + OCIStores: map[string]content.Store{ + ociStoreName: runtime.ociStore, + }, + } + + if client, err := runtime.Client(); err == nil { + return client.Build(ctx, solveOpt, buildkitProduct, doBuild, statusProxy.Writer()) + } + + if len(exports) > 0 { + return nil, fmt.Errorf("gateway client does not support exporting") + } + + err = runtime.WithGateway(ctx, doBuild) if err != nil { - return nil, statusProxy.NiceError("build failed", err) + return nil, err } - return res, nil + return &bkclient.SolveResponse{}, nil } func result(ctx context.Context, gw gwclient.Client, st marshalable) (*gwclient.Result, error) { @@ -625,47 +777,39 @@ func result(ctx context.Context, gw gwclient.Client, st marshalable) (*gwclient. } type portHealthChecker struct { - runtime *Buildkit + gw gwclient.Client + platform ocispecs.Platform host string ports []bass.ThunkPort } -func (runtime *Buildkit) newHealth(host string, ports []bass.ThunkPort) *portHealthChecker { +func newHealth(gw gwclient.Client, platform ocispecs.Platform, host string, ports []bass.ThunkPort) *portHealthChecker { return &portHealthChecker{ - runtime: runtime, - - host: host, - ports: ports, + gw: gw, + platform: platform, + host: host, + ports: ports, } } func (d *portHealthChecker) Check(ctx context.Context) error { - _, err := d.runtime.Client.Build(ctx, kitdclient.SolveOpt{ - Session: []session.Attachable{ - d.runtime.authp, - }, - }, buildkitProduct, d.doBuild, nil) - return err -} - -func (d *portHealthChecker) doBuild(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { - shimExe, err := d.runtime.shim() + shimExe, err := shim(d.platform.Architecture) if err != nil { - return nil, err + return err } - shimRes, err := result(ctx, gw, shimExe) + shimRes, err := result(ctx, d.gw, shimExe) if err != nil { - return nil, err + return err } - scratchRes, err := result(ctx, gw, llb.Scratch()) + scratchRes, err := result(ctx, d.gw, llb.Scratch()) if err != nil { - return nil, err + return err } - container, err := gw.NewContainer(ctx, gwclient.NewContainerRequest{ + container, err := d.gw.NewContainer(ctx, gwclient.NewContainerRequest{ Mounts: []gwclient.Mount{ { Dest: "/", @@ -681,7 +825,7 @@ func (d *portHealthChecker) doBuild(ctx context.Context, gw gwclient.Client) (*g }, }) if err != nil { - return nil, err + return err } // NB: use a different ctx than the one that'll be interrupted for anything @@ -701,7 +845,7 @@ func (d *portHealthChecker) doBuild(ctx context.Context, gw gwclient.Client) (*g Stderr: nopCloser{ioctx.StderrFromContext(ctx)}, }) if err != nil { - return nil, err + return err } exited := make(chan error, 1) @@ -712,47 +856,76 @@ func (d *portHealthChecker) doBuild(ctx context.Context, gw gwclient.Client) (*g select { case err := <-exited: if err != nil { - return nil, err + return err } - return &gwclient.Result{}, nil + return nil case <-ctx.Done(): err := proc.Signal(cleanupCtx, syscall.SIGKILL) if err != nil { - return nil, fmt.Errorf("interrupt check: %w", err) + return fmt.Errorf("interrupt check: %w", err) } <-exited - return nil, ctx.Err() + return ctx.Err() } } type buildkitBuilder struct { - runtime *Buildkit - resolver llb.ImageMetaResolver + gw gwclient.Client + platform ocispecs.Platform + inputs map[string]llb.State + certsDir string + ociStore content.Store + debug bool + disableCache bool secrets map[string][]byte localDirs map[string]string } -func (runtime *Buildkit) newBuilder(ctx context.Context, resolver llb.ImageMetaResolver) *buildkitBuilder { +func (runtime *Buildkit) NewBuilder(ctx context.Context, client gwclient.Client) *buildkitBuilder { + return NewBuilder( + client, + runtime.Platform, + runtime.Inputs, + runtime.Config.CertsDir, + runtime.ociStore, + runtime.Config.Debug, + runtime.Config.DisableCache, + ) +} + +func NewBuilder( + client gwclient.Client, + platform ocispecs.Platform, + inputs map[string]llb.State, + certsDir string, + ociStore content.Store, + debug, disableCache bool, +) *buildkitBuilder { return &buildkitBuilder{ - runtime: runtime, - resolver: resolver, + gw: client, + platform: platform, + inputs: inputs, + certsDir: certsDir, + ociStore: ociStore, + debug: debug, + disableCache: disableCache, secrets: map[string][]byte{}, localDirs: map[string]string{}, } } -func (b *buildkitBuilder) llb( +func (b *buildkitBuilder) Build( ctx context.Context, thunk bass.Thunk, transform func(llb.ExecState, string) llb.State, forceExec bool, extraOpts ...llb.RunOption, -) (intermediateBuild, error) { +) (IntermediateBuild, error) { ib, err := b.image(ctx, thunk.Image) if err != nil { return ib, err @@ -763,30 +936,30 @@ func (b *buildkitBuilder) llb( return ib, err } - cmd, err := NewCommand(ctx, b.runtime, thunk) + cmd, err := NewCommand(ctx, b, thunk) if err != nil { return ib, err } // propagate thunk's entrypoint to the child if len(thunk.Entrypoint) > 0 || thunk.ClearEntrypoint { - ib.config.Entrypoint = thunk.Entrypoint + ib.Config.Entrypoint = thunk.Entrypoint } // propagate thunk's default command if len(thunk.DefaultArgs) > 0 || thunk.ClearDefaultArgs { - ib.config.Cmd = thunk.DefaultArgs + ib.Config.Cmd = thunk.DefaultArgs } if thunk.Labels != nil { - ib.config.Labels = map[string]string{} + ib.Config.Labels = map[string]string{} err := thunk.Labels.Each(func(k bass.Symbol, v bass.Value) error { var str string if err := v.Decode(&str); err != nil { return err } - ib.config.Labels[k.String()] = str + ib.Config.Labels[k.String()] = str return nil }) if err != nil { @@ -795,18 +968,18 @@ func (b *buildkitBuilder) llb( } if len(thunk.Ports) > 0 { - if ib.config.ExposedPorts == nil { - ib.config.ExposedPorts = map[string]struct{}{} + if ib.Config.ExposedPorts == nil { + ib.Config.ExposedPorts = map[string]struct{}{} } for _, port := range thunk.Ports { - ib.config.ExposedPorts[fmt.Sprintf("%d/tcp", port.Port)] = struct{}{} + ib.Config.ExposedPorts[fmt.Sprintf("%d/tcp", port.Port)] = struct{}{} } } useEntrypoint := thunk.UseEntrypoint if len(cmd.Args) == 0 { if forceExec { - cmd.Args = ib.config.Cmd + cmd.Args = ib.Config.Cmd useEntrypoint = true } else { // no command; just overriding config @@ -815,7 +988,7 @@ func (b *buildkitBuilder) llb( } if useEntrypoint { - cmd.Args = append(ib.config.Entrypoint, cmd.Args...) + cmd.Args = append(ib.Config.Entrypoint, cmd.Args...) } if len(cmd.Args) == 0 { @@ -827,12 +1000,7 @@ func (b *buildkitBuilder) llb( return ib, err } - shimExe, err := b.runtime.shim() - if err != nil { - return ib, err - } - - rootCA, err := os.ReadFile(basstls.CACert(b.runtime.Config.CertsDir)) + shimExe, err := shim(b.platform.Architecture) if err != nil { return ib, err } @@ -848,16 +1016,30 @@ func (b *buildkitBuilder) llb( llb.WithCustomName("[hide] mount command json"), )), llb.AddMount(shimExePath, shimExe, llb.SourcePath("run")), - llb.AddMount(caFile, llb.Scratch().File( - llb.Mkfile("ca.crt", 0600, rootCA), - llb.WithCustomName("[hide] mount bass ca"), - ), llb.SourcePath("ca.crt")), llb.With(llb.Dir(workDir)), + llb.AddEnv("_BASS_OUTPUT", outputFile), llb.Args([]string{shimExePath, "run", inputFile}), } + if b.certsDir != "" { + rootCA, err := os.ReadFile(basstls.CACert(b.certsDir)) + if err != nil { + return ib, err + } + + runOpt = append(runOpt, + llb.AddMount(caFile, llb.Scratch().File( + llb.Mkfile("ca.crt", 0600, rootCA), + llb.WithCustomName("[hide] mount bass ca"), + ), llb.SourcePath("ca.crt"))) + } + if thunk.TLS != nil { - crt, key, err := basstls.Generate(b.runtime.Config.CertsDir, thunkName) + if b.certsDir == "" { + return ib, fmt.Errorf("TLS not configured") + } + + crt, key, err := basstls.Generate(b.certsDir, thunkName) if err != nil { return ib, fmt.Errorf("tls: generate: %w", err) } @@ -892,7 +1074,7 @@ func (b *buildkitBuilder) llb( ) } - if b.runtime.Config.Debug { + if b.debug { runOpt = append(runOpt, llb.AddEnv("_BASS_DEBUG", "1")) } @@ -903,7 +1085,7 @@ func (b *buildkitBuilder) llb( } if thunk.Insecure { - ib.needsInsecure = true + ib.NeedsInsecure = true runOpt = append(runOpt, llb.WithCgroupParent(thunkName), @@ -926,43 +1108,43 @@ func (b *buildkitBuilder) llb( if targetPath == workDir { remountedWorkdir = true - ib.sourcePath = sp + ib.SourcePath = sp } if ni { - ib.needsInsecure = true + ib.NeedsInsecure = true } runOpt = append(runOpt, mountOpt) } if !remountedWorkdir { - if ib.sourcePath != "" { + if ib.SourcePath != "" { // NB: could just call SourcePath with "", but this is to ensure there's // code coverage - runOpt = append(runOpt, llb.AddMount(workDir, ib.output, llb.SourcePath(ib.sourcePath))) + runOpt = append(runOpt, llb.AddMount(workDir, ib.Output, llb.SourcePath(ib.SourcePath))) } else { - runOpt = append(runOpt, llb.AddMount(workDir, ib.output)) + runOpt = append(runOpt, llb.AddMount(workDir, ib.Output)) } } - if b.runtime.Config.DisableCache { + if b.disableCache { runOpt = append(runOpt, llb.IgnoreCache) } runOpt = append(runOpt, extraOpts...) - execSt := ib.fs.Run(runOpt...) - ib.output = transform(execSt, ib.sourcePath) - ib.fs = execSt.State + execSt := ib.FS.Run(runOpt...) + ib.Output = transform(execSt, ib.SourcePath) + ib.FS = execSt.State return ib, nil } -func (runtime *Buildkit) shim() (llb.State, error) { - shimExe, found := allShims["exe."+runtime.Platform.Architecture] +func shim(arch string) (llb.State, error) { + shimExe, found := allShims["exe."+arch] if !found { - return llb.State{}, fmt.Errorf("no shim found for %s", runtime.Platform.Architecture) + return llb.State{}, fmt.Errorf("no shim found for %s", arch) } return llb.Scratch().File( @@ -971,11 +1153,11 @@ func (runtime *Buildkit) shim() (llb.State, error) { ), nil } -func (r *Buildkit) ref(ctx context.Context, imageRef bass.ImageRef) (string, error) { +func ref(ctx context.Context, starter Starter, imageRef bass.ImageRef) (string, error) { if imageRef.Repository.Addr != nil { addr := imageRef.Repository.Addr - result, err := r.Start(ctx, addr.Thunk) + result, err := starter.Start(ctx, addr.Thunk) if err != nil { return "", err } @@ -999,27 +1181,57 @@ func (r *Buildkit) ref(ctx context.Context, imageRef bass.ImageRef) (string, err return imageRef.Ref() } -type intermediateBuild struct { - fs llb.State +type IntermediateBuild struct { + FS llb.State + Output llb.State + + SourcePath string + NeedsInsecure bool + + Config ocispecs.ImageConfig +} - output llb.State +func (ib IntermediateBuild) WithImageConfig(config []byte) (IntermediateBuild, error) { + var img ocispecs.Image + if err := json.Unmarshal(config, &img); err != nil { + return ib, err + } - sourcePath string - needsInsecure bool + ib.Config = img.Config - config ocispecs.ImageConfig + for _, env := range img.Config.Env { + parts := strings.SplitN(env, "=", 2) + if len(parts[0]) > 0 { + var v string + if len(parts) > 1 { + v = parts[1] + } + ib.FS = ib.FS.AddEnv(parts[0], v) + } + } + + ib.FS = ib.FS.Dir(img.Config.WorkingDir) + if img.Architecture != "" && img.OS != "" { + ib.FS = ib.FS.Platform(ocispecs.Platform{ + OS: img.OS, + Architecture: img.Architecture, + Variant: img.Variant, + }) + } + + return ib, nil } -func (b *buildkitBuilder) image(ctx context.Context, image *bass.ThunkImage) (ib intermediateBuild, err error) { +func (b *buildkitBuilder) image(ctx context.Context, image *bass.ThunkImage) (ib IntermediateBuild, err error) { switch { case image == nil: // TODO: test; how is this possible? - ib.fs = llb.Scratch() - ib.output = llb.Scratch() + ib.FS = llb.Scratch() + ib.Output = llb.Scratch() return ib, nil case image.Ref != nil: - ref, err := b.runtime.ref(ctx, *image.Ref) + ref, err := ref(ctx, b, *image.Ref) if err != nil { return ib, err } @@ -1030,9 +1242,9 @@ func (b *buildkitBuilder) image(ctx context.Context, image *bass.ThunkImage) (ib ref = r.String() } - dgst, config, err := b.resolver.ResolveImageConfig(ctx, ref, llb.ResolveImageConfigOpt{ + dgst, config, err := b.gw.ResolveImageConfig(ctx, ref, llb.ResolveImageConfigOpt{ ResolverType: llb.ResolverTypeRegistry, - Platform: &b.runtime.Platform, + Platform: &b.platform, ResolveMode: llb.ResolveModeDefault.String(), }) if err != nil { @@ -1047,38 +1259,18 @@ func (b *buildkitBuilder) image(ctx context.Context, image *bass.ThunkImage) (ib ref = r.String() } - st := llb.Image(ref, llb.Platform(b.runtime.Platform)) + ib.FS = llb.Image(ref, llb.Platform(b.platform)) - var img ocispecs.Image - if err := json.Unmarshal(config, &img); err != nil { - return ib, err - } - for _, env := range img.Config.Env { - parts := strings.SplitN(env, "=", 2) - if len(parts[0]) > 0 { - var v string - if len(parts) > 1 { - v = parts[1] - } - st = st.AddEnv(parts[0], v) - } - } - st = st.Dir(img.Config.WorkingDir) - if img.Architecture != "" && img.OS != "" { - st = st.Platform(ocispecs.Platform{ - OS: img.OS, - Architecture: img.Architecture, - Variant: img.Variant, - }) + ib, err = ib.WithImageConfig(config) + if err != nil { + return ib, fmt.Errorf("image archive with image config: %w", err) } - ib.fs = st - ib.output = llb.Scratch() - ib.config = img.Config + ib.Output = llb.Scratch() return ib, nil case image.Thunk != nil: - return b.llb(ctx, *image.Thunk, getWorkdir, false) + return b.Build(ctx, *image.Thunk, getWorkdir, false) case image.Archive != nil: file, err := image.Archive.File.Open(ctx) @@ -1090,17 +1282,17 @@ func (b *buildkitBuilder) image(ctx context.Context, image *bass.ThunkImage) (ib stream := archive.NewImageImportStream(file, "") - desc, err := stream.Import(ctx, b.runtime.ociStore) + desc, err := stream.Import(ctx, b.ociStore) if err != nil { return ib, fmt.Errorf("image archive import: %w", err) } - manifestDesc, err := resolveIndex(ctx, b.runtime.ociStore, desc, b.runtime.Platform, image.Archive.Tag) + manifestDesc, err := resolveIndex(ctx, b.ociStore, desc, b.platform, image.Archive.Tag) if err != nil { return ib, fmt.Errorf("image archive resolve index: %w", err) } - manifestBlob, err := content.ReadBlob(ctx, b.runtime.ociStore, *manifestDesc) + manifestBlob, err := content.ReadBlob(ctx, b.ociStore, *manifestDesc) if err != nil { return ib, fmt.Errorf("image archive read manifest blob: %w", err) } @@ -1112,7 +1304,7 @@ func (b *buildkitBuilder) image(ctx context.Context, image *bass.ThunkImage) (ib st := llb.OCILayout( fmt.Sprintf("%s@%s", dummyRepo, manifestDesc.Digest), llb.OCIStore("", ociStoreName), - llb.Platform(b.runtime.Platform), + llb.Platform(b.platform), ) var m ocispecs.Manifest @@ -1121,19 +1313,20 @@ func (b *buildkitBuilder) image(ctx context.Context, image *bass.ThunkImage) (ib return ib, fmt.Errorf("image archive unmarshal manifest: %w", err) } - configBlob, err := content.ReadBlob(ctx, b.runtime.ociStore, m.Config) + configBlob, err := content.ReadBlob(ctx, b.ociStore, m.Config) if err != nil { return ib, fmt.Errorf("image archive read blob: %w", err) } - st, err = st.WithImageConfig(configBlob) + ib.FS = st + + ib, err = ib.WithImageConfig(configBlob) if err != nil { - return ib, fmt.Errorf("image archive unmarshal index: %w", err) + return ib, fmt.Errorf("image archive with image config: %w", err) } - ib.fs = st - ib.output = llb.Scratch() - // ib.config = img.Config + ib.Output = llb.Scratch() + return ib, nil } @@ -1186,76 +1379,63 @@ func (b *buildkitBuilder) image(ctx context.Context, image *bass.ThunkImage) (ib return ib, fmt.Errorf("docker build marshal: %w", err) } - var allowed []entitlements.Entitlement - if needsInsecure { - allowed = append(allowed, entitlements.EntitlementSecurityInsecure) - } - inputs := map[string]*pb.Definition{ dockerui.DefaultLocalNameContext: ctxDef.ToPB(), dockerui.DefaultLocalNameDockerfile: ctxDef.ToPB(), } - statusProxy := forwardStatus(progrock.RecorderFromContext(ctx)) - defer statusProxy.Wait() - - var st llb.State - doBuild := func(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { - res, err := gw.Solve(ctx, gwclient.SolveRequest{ - Frontend: "dockerfile.v0", - FrontendOpt: opts, - FrontendInputs: inputs, - // Evaluate: true, // TODO: maybe? - }) + if b.certsDir != "" { + certDef, err := llb.Local(b.certsDir, + llb.IncludePatterns(basstls.CAFiles), + llb.Differ(llb.DiffMetadata, false)). + Marshal(ctx, llb.Platform(b.platform)) if err != nil { - return nil, err + return ib, fmt.Errorf("bass tls def: %w", err) } - bkref, err := res.SingleRef() - if err != nil { - return nil, statusProxy.NiceError("build failed", err) - } - - if bkref == nil { - st = llb.Scratch() - } else { - st, err = bkref.ToState() - if err != nil { - return nil, err - } - } + inputs["bass-tls"] = certDef.ToPB() + } - cfgBytes, found := res.Metadata[exptypes.ExporterImageConfigKey] - if found { - st, err = st.WithImageConfig(cfgBytes) - if err != nil { - return nil, fmt.Errorf("with image config: %w", err) - } - } + statusProxy := forwardStatus(progrock.RecorderFromContext(ctx)) + defer statusProxy.Wait() - return res, nil + res, err := b.gw.Solve(ctx, gwclient.SolveRequest{ + Frontend: "dockerfile.v0", + FrontendOpt: opts, + FrontendInputs: inputs, + }) + if err != nil { + return ib, err } - _, err = b.runtime.Client.Build(ctx, kitdclient.SolveOpt{ - Session: []session.Attachable{ - b.runtime.authp, - }, - AllowedEntitlements: allowed, - LocalDirs: b.localDirs, - }, buildkitProduct, doBuild, statusProxy.Writer()) + bkref, err := res.SingleRef() if err != nil { - return ib, statusProxy.NiceError("build failed", err) + return ib, fmt.Errorf("single ref: %w", err) } - wd, err := st.GetDir(ctx) - if err != nil { - return ib, fmt.Errorf("get dir: %w", err) + var st llb.State + if bkref == nil { + st = llb.Scratch() + } else { + st, err = bkref.ToState() + if err != nil { + return ib, err + } } - ib.fs = st - ib.output = st - ib.sourcePath = wd - ib.needsInsecure = needsInsecure + ib.FS = st + + cfgBytes, found := res.Metadata[exptypes.ExporterImageConfigKey] + if found { + ib, err = ib.WithImageConfig(cfgBytes) + if err != nil { + return ib, fmt.Errorf("with image config: %w", err) + } + } + + ib.Output = ib.FS + ib.SourcePath = ib.Config.WorkingDir + ib.NeedsInsecure = needsInsecure return ib, nil } @@ -1340,23 +1520,31 @@ func (b *buildkitBuilder) initializeMount(ctx context.Context, source bass.Thunk } func (b *buildkitBuilder) thunkPathSt(ctx context.Context, source bass.ThunkPath) (llb.State, string, bool, error) { - ib, err := b.llb(ctx, source.Thunk, getWorkdir, true) + ib, err := b.Build(ctx, source.Thunk, getWorkdir, true) if err != nil { return llb.State{}, "", false, fmt.Errorf("thunk llb: %w", err) } - return ib.output, - filepath.Join(ib.sourcePath, source.Path.FilesystemPath().FromSlash()), - ib.needsInsecure, + return ib.Output, + filepath.Join(ib.SourcePath, source.Path.FilesystemPath().FromSlash()), + ib.NeedsInsecure, nil } func (b *buildkitBuilder) hostPathSt(ctx context.Context, source bass.HostPath) (llb.State, string, error) { - contextDir := source.ContextDir - b.localDirs[contextDir] = source.ContextDir + // TODO: can we restrict this to a more fine grained path? + localName := source.ContextDir + + sourcePath := source.Path.FilesystemPath().FromSlash() + if input, found := b.inputs[localName]; found { + return input, sourcePath, nil + } + + fullPath := source.FromSlash() + b.localDirs[fullPath] = fullPath var excludes []string - ignorePath := filepath.Join(contextDir, ".bassignore") + ignorePath := filepath.Join(localName, ".bassignore") ignore, err := os.Open(ignorePath) if err == nil { excludes, err = dockerignore.ReadAll(ignore) @@ -1365,10 +1553,9 @@ func (b *buildkitBuilder) hostPathSt(ctx context.Context, source bass.HostPath) } } - sourcePath := source.Path.FilesystemPath().FromSlash() st := llb.Scratch().File(llb.Copy( llb.Local( - contextDir, + localName, llb.ExcludePatterns(excludes), llb.Differ(llb.DiffMetadata, false), ), @@ -1463,7 +1650,7 @@ type statusProxy struct { prog *cli.Progress } -func (proxy *statusProxy) proxy(rec *progrock.Recorder, statuses chan *kitdclient.SolveStatus) { +func (proxy *statusProxy) proxy(rec *progrock.Recorder, statuses chan *bkclient.SolveStatus) { for { s, ok := <-statuses if !ok { @@ -1506,8 +1693,8 @@ func (proxy *statusProxy) proxy(rec *progrock.Recorder, statuses chan *kitdclien } } -func (proxy *statusProxy) Writer() chan *kitdclient.SolveStatus { - statuses := make(chan *kitdclient.SolveStatus) +func (proxy *statusProxy) Writer() chan *bkclient.SolveStatus { + statuses := make(chan *bkclient.SolveStatus) proxy.wg.Add(1) go func() { @@ -1530,6 +1717,56 @@ func getWorkdir(st llb.ExecState, _ string) llb.State { return st.GetMount(workDir) } +func dialBuildkit(ctx context.Context, addr string, installation string, certsDir string) (*bkclient.Client, error) { + if addr == "" { + addr = os.Getenv("BUILDKIT_HOST") + } + + if addr == "" { + sockPath, err := xdg.SearchConfigFile("bass/buildkitd.sock") + if err == nil { + // support respecting XDG_RUNTIME_DIR instead of assuming /run/ + addr = "unix://" + sockPath + } + + sockPath, err = xdg.SearchRuntimeFile("buildkit/buildkitd.sock") + if err == nil { + // support respecting XDG_RUNTIME_DIR instead of assuming /run/ + addr = "unix://" + sockPath + } + } + + var errs error + if addr == "" { + var startErr error + addr, startErr = buildkitd.Start(ctx, installation, certsDir) + if startErr != nil { + errs = multierror.Append(startErr) + } + } + + client, err := bkclient.New(context.TODO(), addr) + if err != nil { + errs = multierror.Append(errs, err) + return nil, errs + } + + return client, nil +} + +type AnyDirSource struct{} + +func (AnyDirSource) LookupDir(name string) (filesync.SyncedDir, bool) { + return filesync.SyncedDir{ + Dir: name, + Map: func(p string, st *fstypes.Stat) fsutil.MapResult { + st.Uid = 0 + st.Gid = 0 + return fsutil.MapResultKeep + }, + }, true +} + func resolveIndex(ctx context.Context, store content.Store, desc ocispecs.Descriptor, platform ocispecs.Platform, tag string) (*ocispecs.Descriptor, error) { if desc.MediaType != ocispecs.MediaTypeImageIndex { return nil, fmt.Errorf("expected index, got %s", desc.MediaType) diff --git a/pkg/runtimes/shim/run.go b/pkg/runtimes/shim/run.go index 564b92d7..debedc9b 100644 --- a/pkg/runtimes/shim/run.go +++ b/pkg/runtimes/shim/run.go @@ -83,7 +83,7 @@ func run(args []string) error { defer response.Close() - stdout = response + stdout = io.MultiWriter(stdout, response) } for _, e := range cmd.Env { diff --git a/pkg/runtimes/shim/tls.go b/pkg/runtimes/shim/tls.go index b81bdc85..8f1d8c3b 100644 --- a/pkg/runtimes/shim/tls.go +++ b/pkg/runtimes/shim/tls.go @@ -45,6 +45,16 @@ func binaryExists(name string) bool { } func installCert() error { + cert, err := os.ReadFile(BassCAFile) + if err != nil { + if os.IsNotExist(err) { + // NB: certs might not be installed, intentionally + return nil + } + + return fmt.Errorf("read bass CA: %w", err) + } + // first try to install gracefully to the system trust for pathTemplate, cmd := range trusts { if _, err := os.Stat(filepath.Dir(pathTemplate)); err != nil { @@ -52,12 +62,6 @@ func installCert() error { } trustPath := fmt.Sprintf(pathTemplate, "bass") - - cert, err := os.ReadFile(BassCAFile) - if err != nil { - return fmt.Errorf("read bass CA: %w", err) - } - if err := os.WriteFile(trustPath, cert, 0600); err != nil { return fmt.Errorf("write bass CA to system trust: %w", err) } @@ -75,6 +79,7 @@ func installCert() error { return nil } + // if that doesn't work, try to inject into the bundle return injectCert() } diff --git a/pkg/runtimes/testdata/docker-build.bass b/pkg/runtimes/testdata/docker-build.bass index 4e4bfa89..e1fe7d90 100644 --- a/pkg/runtimes/testdata/docker-build.bass +++ b/pkg/runtimes/testdata/docker-build.bass @@ -1,33 +1,34 @@ -(defn out [thunk] +(defn read-all [thunk] (-> thunk (read :raw) next)) (assert = "hello from Dockerfile\n" - (out + (read-all (from (docker-build *dir*/docker-build/ {:os "linux"}) + ($ pwd) ($ cat ./wd-file)))) (assert = "hello from Dockerfile.alt\n" - (out + (read-all (from (docker-build *dir*/docker-build/ {:os "linux"} :dockerfile ./Dockerfile.alt) ($ cat ./wd-file)))) (assert = "hello from alt stage in Dockerfile\n" - (out + (read-all (from (docker-build *dir*/docker-build/ {:os "linux"} :target "alt") ($ cat ./wd-file)))) (assert = "hello from Dockerfile with message sup\n" - (out + (read-all (from (docker-build *dir*/docker-build/ {:os "linux"} :target "arg" :args {:MESSAGE "sup"}) ($ cat ./wd-file)))) (assert = "hello from Dockerfile with env bar\nbar\n" - (out + (read-all (from (docker-build *dir*/docker-build/ {:os "linux"} :target "env") ($ sh -c "cat ./wd-file; echo $FOO")))) diff --git a/pkg/runtimes/util/reffs.go b/pkg/runtimes/util/reffs.go new file mode 100644 index 00000000..712883e2 --- /dev/null +++ b/pkg/runtimes/util/reffs.go @@ -0,0 +1,120 @@ +package util + +import ( + "context" + "fmt" + "io" + "io/fs" + "time" + + "github.com/moby/buildkit/client/llb" + gwclient "github.com/moby/buildkit/frontend/gateway/client" + fstypes "github.com/tonistiigi/fsutil/types" +) + +type refFS struct { + ctx context.Context + ref gwclient.Reference +} + +func NewRefFS(ctx context.Context, ref gwclient.Reference) fs.FS { + return &refFS{ctx: ctx, ref: ref} +} + +func OpenRefFS(ctx context.Context, gw gwclient.Client, st llb.State, opts ...llb.ConstraintsOpt) (fs.FS, error) { + def, err := st.Marshal(ctx, opts...) + if err != nil { + return nil, err + } + + res, err := gw.Solve(ctx, gwclient.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, err + } + + ref, err := res.SingleRef() + if err != nil { + return nil, err + } + + if ref == nil { + return nil, fmt.Errorf("no ref returned") + } + + return &refFS{ctx: ctx, ref: ref}, nil +} + +func (fs *refFS) Open(name string) (fs.File, error) { + stat, err := fs.ref.StatFile(fs.ctx, gwclient.StatRequest{Path: name}) + if err != nil { + return nil, err + } + + return &refFile{ctx: fs.ctx, ref: fs.ref, stat: stat, name: name}, nil +} + +type refFile struct { + ctx context.Context + ref gwclient.Reference + name string + stat *fstypes.Stat + offset int64 +} + +func (f *refFile) Stat() (fs.FileInfo, error) { + return &refFileInfo{stat: f.stat}, nil +} + +func (f *refFile) Read(p []byte) (int, error) { + if f.offset >= f.stat.Size_ { + return 0, io.EOF + } + + content, err := f.ref.ReadFile(f.ctx, gwclient.ReadRequest{ + Filename: f.name, + Range: &gwclient.FileRange{ + Offset: int(f.offset), + Length: len(p), + }, + }) + if err != nil { + return 0, err + } + n := copy(p, content) + f.offset += int64(n) + return n, nil +} + +func (fi *refFile) Close() error { + return nil +} + +type refFileInfo struct { + stat *fstypes.Stat +} + +func (fi *refFileInfo) Name() string { + return fi.stat.Path +} + +func (fi *refFileInfo) Size() int64 { + return int64(fi.stat.Size_) // NB: *NOT* Size()! +} + +func (fi *refFileInfo) Mode() fs.FileMode { + return fs.FileMode(fi.stat.Mode) +} + +func (fi *refFileInfo) ModTime() time.Time { + return time.Unix(fi.stat.ModTime/int64(time.Second), fi.stat.ModTime%int64(time.Second)) +} + +func (fi *refFileInfo) IsDir() bool { + return fi.stat.IsDir() +} + +func (fi *refFileInfo) Sys() interface{} { + return nil +} From 31199fc97800c9707fdfd8b15220320b789546ee Mon Sep 17 00:00:00 2001 From: Alex Suraci Date: Mon, 10 Apr 2023 23:33:42 -0400 Subject: [PATCH 2/9] refactor, fix empty image export/publishes This is sorely lacking test coverage. Probably need a test that exports and imports again. But it's not possible to import a host path yet. --- bass/bass.bass | 2 +- cmd/bass/frontend.go | 3 - pkg/runtimes/buildkit.go | 384 ++++++++++-------- pkg/runtimes/suite.go | 6 - .../testdata/remount-workdir-subdir.bass | 15 +- 5 files changed, 230 insertions(+), 180 deletions(-) diff --git a/bass/bass.bass b/bass/bass.bass index c151915d..456bf80e 100644 --- a/bass/bass.bass +++ b/bass/bass.bass @@ -59,7 +59,7 @@ ($ cp -a & $submodule-cp-args) ($ go mod download))))) -(provide [build smoke-test tests docs coverage] +(provide [build dist smoke-test tests docs coverage] (use (*dir*/buildkit.bass)) (defn dist [src version os arch] diff --git a/cmd/bass/frontend.go b/cmd/bass/frontend.go index 7689e38a..c2cfbc40 100644 --- a/cmd/bass/frontend.go +++ b/cmd/bass/frontend.go @@ -204,9 +204,6 @@ func frontendBuild(ctx context.Context, gw gwclient.Client) (*gwclient.Result, e ib, err := builder.Build( ctx, thunk, - func(st llb.ExecState, sourcePath string) llb.State { - return st.Root() - }, false, // don't run any entrypoint ) if err != nil { diff --git a/pkg/runtimes/buildkit.go b/pkg/runtimes/buildkit.go index d38d692a..455168fe 100644 --- a/pkg/runtimes/buildkit.go +++ b/pkg/runtimes/buildkit.go @@ -290,11 +290,10 @@ func (runtime *Buildkit) Run(ctx context.Context, thunk bass.Thunk) error { _, err := runtime.build( ctx, thunk, - func(st llb.ExecState, _ string) llb.State { - return st.GetMount(ioDir) + nil, // exports + func(ctx context.Context, gw gwclient.Client, ib IntermediateBuild) (*gwclient.Result, error) { + return ib.ForRun(ctx, gw) }, - nil, // exports - nil, // callback true, // inherit entrypoint/cmd ) return err @@ -326,7 +325,7 @@ func (b *buildkitBuilder) Start(ctx context.Context, thunk bass.Thunk) (StartRes health := newHealth(b.gw, b.platform, host, thunk.Ports) - ib, err := b.Build(ctx, thunk, getWorkdir, true) + ib, err := b.Build(ctx, thunk, true) if err != nil { return StartResult{}, err } @@ -398,32 +397,9 @@ func (runtime *Buildkit) Read(ctx context.Context, w io.Writer, thunk bass.Thunk _, err := runtime.build( ctx, thunk, - func(st llb.ExecState, _ string) llb.State { - return st.GetMount(ioDir) - }, nil, - func(ctx context.Context, gw gwclient.Client, res *gwclient.Result) (*gwclient.Result, error) { - ref, err := res.SingleRef() - if err != nil { - return nil, err - } - - fs := util.NewRefFS(ctx, ref) - - f, err := fs.Open(path.Base(outputFile)) - if err != nil { - return nil, err - } - - if _, err := io.Copy(w, f); err != nil { - return nil, err - } - - if err := f.Close(); err != nil { - return nil, err - } - - return res, nil + func(ctx context.Context, gw gwclient.Client, ib IntermediateBuild) (*gwclient.Result, error) { + return ib.ReadStdout(ctx, gw, w) }, true, // inherit entrypoint/cmd ) @@ -440,9 +416,6 @@ func (runtime *Buildkit) Export(ctx context.Context, w io.Writer, thunk bass.Thu _, err := runtime.build( ctx, thunk, - func(st llb.ExecState, _ string) llb.State { - return st.Root() - }, []bkclient.ExportEntry{ { Type: bkclient.ExporterOCI, @@ -451,7 +424,9 @@ func (runtime *Buildkit) Export(ctx context.Context, w io.Writer, thunk bass.Thu }, }, }, - nil, // callback + func(ctx context.Context, gw gwclient.Client, ib IntermediateBuild) (*gwclient.Result, error) { + return ib.ForPublish(ctx, gw) + }, false, // do not inherit entrypoint/cmd ) return err @@ -469,9 +444,6 @@ func (runtime *Buildkit) Publish(ctx context.Context, ref bass.ImageRef, thunk b res, err := runtime.build( ctx, thunk, - func(st llb.ExecState, _ string) llb.State { - return st.Root() - }, []bkclient.ExportEntry{ { Type: bkclient.ExporterImage, @@ -481,7 +453,9 @@ func (runtime *Buildkit) Publish(ctx context.Context, ref bass.ImageRef, thunk b }, }, }, - nil, // callback + func(ctx context.Context, gw gwclient.Client, ib IntermediateBuild) (*gwclient.Result, error) { + return ib.ForPublish(ctx, gw) + }, false, // do not inherit entrypoint/cmd ) if err != nil { @@ -504,21 +478,10 @@ func (runtime *Buildkit) ExportPath(ctx context.Context, w io.Writer, tp bass.Th path := tp.Path var err error - if path.FilesystemPath().IsDir() || runtime.client != nil { + if path.FilesystemPath().IsDir() { _, err = runtime.build( ctx, thunk, - func(st llb.ExecState, sp string) llb.State { - copyOpt := &llb.CopyInfo{} - if path.FilesystemPath().IsDir() { - copyOpt.CopyDirContentsOnly = true - } - - return llb.Scratch().File( - llb.Copy(st.GetMount(workDir), filepath.Join(sp, path.FilesystemPath().FromSlash()), ".", copyOpt), - llb.WithCustomNamef("[hide] copy %s", path.Slash()), - ) - }, []bkclient.ExportEntry{ { Type: bkclient.ExporterTar, @@ -527,7 +490,9 @@ func (runtime *Buildkit) ExportPath(ctx context.Context, w io.Writer, tp bass.Th }, }, }, - nil, // callback? TODO + func(ctx context.Context, gw gwclient.Client, ib IntermediateBuild) (*gwclient.Result, error) { + return ib.ForExportDir(ctx, gw, *path.Dir) + }, true, // inherit entryopint/cmd ) } else { @@ -535,66 +500,9 @@ func (runtime *Buildkit) ExportPath(ctx context.Context, w io.Writer, tp bass.Th _, err = runtime.build( ctx, thunk, - func(st llb.ExecState, sp string) llb.State { - copyOpt := &llb.CopyInfo{} - if path.FilesystemPath().IsDir() { - copyOpt.CopyDirContentsOnly = true - } - - return llb.Scratch().File( - llb.Copy(st.GetMount(workDir), filepath.Join(sp, path.FilesystemPath().FromSlash()), ".", copyOpt), - llb.WithCustomNamef("[hide] copy %s", path.Slash()), - ) - }, nil, - func(ctx context.Context, gw gwclient.Client, res *gwclient.Result) (*gwclient.Result, error) { - ref, err := res.SingleRef() - if err != nil { - return nil, err - } - - stat, err := ref.StatFile(ctx, gwclient.StatRequest{ - Path: path.FilesystemPath().Name(), - }) - if err != nil { - return nil, err - } - - err = tw.WriteHeader(&tar.Header{ - Name: path.FilesystemPath().FromSlash(), - Mode: int64(stat.Mode), - Uid: int(stat.Uid), - Gid: int(stat.Gid), - Size: int64(stat.Size_), - ModTime: time.Unix(stat.ModTime/int64(time.Second), stat.ModTime%int64(time.Second)), - Linkname: stat.Linkname, - Devmajor: stat.Devmajor, - Devminor: stat.Devmajor, - }) - if err != nil { - return nil, fmt.Errorf("write tar header: %w", err) - } - - fs := util.NewRefFS(ctx, ref) - - f, err := fs.Open(path.FilesystemPath().Name()) - if err != nil { - return nil, err - } - - if _, err := io.Copy(w, f); err != nil { - return nil, err - } - - if err := f.Close(); err != nil { - return nil, err - } - - if err := tw.Close(); err != nil { - return nil, err - } - - return res, nil + func(ctx context.Context, gw gwclient.Client, ib IntermediateBuild) (*gwclient.Result, error) { + return ib.ExportFile(ctx, gw, tw, *path.File) }, true, ) @@ -664,26 +572,22 @@ func (runtime *Buildkit) Close() error { func (runtime *Buildkit) build( ctx context.Context, thunk bass.Thunk, - transform func(llb.ExecState, string) llb.State, exports []bkclient.ExportEntry, - cb func(context.Context, gwclient.Client, *gwclient.Result) (*gwclient.Result, error), + cb func(context.Context, gwclient.Client, IntermediateBuild) (*gwclient.Result, error), forceExec bool, runOpts ...llb.RunOption, ) (*bkclient.SolveResponse, error) { - var def *llb.Definition var secrets map[string][]byte // var localDirs map[string]string - var imageConfig ocispecs.ImageConfig var allowed []entitlements.Entitlement - statusProxy := forwardStatus(progrock.RecorderFromContext(ctx)) - defer statusProxy.Wait() - // build llb definition using the remote gateway for image resolution + var ib IntermediateBuild err := runtime.WithGateway(ctx, func(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { b := runtime.NewBuilder(ctx, gw) - ib, err := b.Build(ctx, thunk, transform, forceExec, runOpts...) + var err error + ib, err = b.Build(ctx, thunk, forceExec, runOpts...) if err != nil { return nil, err } @@ -694,12 +598,6 @@ func (runtime *Buildkit) build( // localDirs = b.localDirs secrets = b.secrets - imageConfig = ib.Config - - def, err = ib.Output.Marshal(ctx) - if err != nil { - return nil, err - } return &gwclient.Result{}, nil }) @@ -707,34 +605,6 @@ func (runtime *Buildkit) build( return nil, err } - doBuild := func(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { - res, err := gw.Solve(ctx, gwclient.SolveRequest{ - Evaluate: true, - Definition: def.ToPB(), - }) - if err != nil { - return nil, err - } - - cfgBytes, err := json.Marshal(ocispecs.Image{ - Architecture: runtime.Platform.Architecture, - OS: runtime.Platform.OS, - OSVersion: runtime.Platform.OSVersion, - OSFeatures: runtime.Platform.OSFeatures, - Config: imageConfig, - }) - if err != nil { - return nil, err - } - res.AddMeta(exptypes.ExporterImageConfigKey, cfgBytes) - - if cb != nil { - res, err = cb(ctx, gw, res) - } - - return res, err - } - solveOpt := bkclient.SolveOpt{ // LocalDirs: localDirs, AllowedEntitlements: allowed, @@ -749,7 +619,13 @@ func (runtime *Buildkit) build( }, } + doBuild := func(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { + return cb(ctx, gw, ib) + } + if client, err := runtime.Client(); err == nil { + statusProxy := forwardStatus(progrock.RecorderFromContext(ctx)) + defer statusProxy.Wait() return client.Build(ctx, solveOpt, buildkitProduct, doBuild, statusProxy.Writer()) } @@ -922,7 +798,6 @@ func NewBuilder( func (b *buildkitBuilder) Build( ctx context.Context, thunk bass.Thunk, - transform func(llb.ExecState, string) llb.State, forceExec bool, extraOpts ...llb.RunOption, ) (IntermediateBuild, error) { @@ -1108,7 +983,7 @@ func (b *buildkitBuilder) Build( if targetPath == workDir { remountedWorkdir = true - ib.SourcePath = sp + ib.OutputSourcePath = sp } if ni { @@ -1119,10 +994,10 @@ func (b *buildkitBuilder) Build( } if !remountedWorkdir { - if ib.SourcePath != "" { + if ib.OutputSourcePath != "" { // NB: could just call SourcePath with "", but this is to ensure there's // code coverage - runOpt = append(runOpt, llb.AddMount(workDir, ib.Output, llb.SourcePath(ib.SourcePath))) + runOpt = append(runOpt, llb.AddMount(workDir, ib.Output, llb.SourcePath(ib.OutputSourcePath))) } else { runOpt = append(runOpt, llb.AddMount(workDir, ib.Output)) } @@ -1135,8 +1010,9 @@ func (b *buildkitBuilder) Build( runOpt = append(runOpt, extraOpts...) execSt := ib.FS.Run(runOpt...) - ib.Output = transform(execSt, ib.SourcePath) - ib.FS = execSt.State + ib.Exec = execSt + ib.Output = execSt.GetMount(workDir) + ib.FS = execSt.Root() return ib, nil } @@ -1183,12 +1059,14 @@ func ref(ctx context.Context, starter Starter, imageRef bass.ImageRef) (string, type IntermediateBuild struct { FS llb.State + Exec llb.ExecState Output llb.State - SourcePath string - NeedsInsecure bool + OutputSourcePath string + NeedsInsecure bool - Config ocispecs.ImageConfig + Platform ocispecs.Platform + Config ocispecs.ImageConfig } func (ib IntermediateBuild) WithImageConfig(config []byte) (IntermediateBuild, error) { @@ -1222,7 +1100,181 @@ func (ib IntermediateBuild) WithImageConfig(config []byte) (IntermediateBuild, e return ib, nil } -func (b *buildkitBuilder) image(ctx context.Context, image *bass.ThunkImage) (ib IntermediateBuild, err error) { +func (ib IntermediateBuild) ForPublish(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { + def, err := ib.FS.Marshal(ctx) + if err != nil { + return nil, err + } + + res, err := gw.Solve(ctx, gwclient.SolveRequest{ + Evaluate: true, + Definition: def.ToPB(), + }) + if err != nil { + return nil, err + } + + cfgBytes, err := json.Marshal(ocispecs.Image{ + Architecture: ib.Platform.Architecture, + OS: ib.Platform.OS, + OSVersion: ib.Platform.OSVersion, + OSFeatures: ib.Platform.OSFeatures, + Config: ib.Config, + }) + if err != nil { + return nil, err + } + res.AddMeta(exptypes.ExporterImageConfigKey, cfgBytes) + + return res, nil +} + +func (ib IntermediateBuild) ForRun(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { + def, err := ib.Exec.Marshal(ctx) + if err != nil { + return nil, err + } + + return gw.Solve(ctx, gwclient.SolveRequest{ + Evaluate: true, + Definition: def.ToPB(), + }) +} + +func (ib IntermediateBuild) ReadStdout(ctx context.Context, gw gwclient.Client, w io.Writer) (*gwclient.Result, error) { + def, err := ib.Exec.GetMount(ioDir).Marshal(ctx) + if err != nil { + return nil, err + } + + res, err := gw.Solve(ctx, gwclient.SolveRequest{ + Evaluate: true, + Definition: def.ToPB(), + }) + if err != nil { + return nil, err + } + + ref, err := res.SingleRef() + if err != nil { + return nil, err + } + + fs := util.NewRefFS(ctx, ref) + + f, err := fs.Open(path.Base(outputFile)) + if err != nil { + return nil, err + } + + if _, err := io.Copy(w, f); err != nil { + return nil, err + } + + if err := f.Close(); err != nil { + return nil, err + } + + return res, nil +} + +func (ib IntermediateBuild) ForExportDir(ctx context.Context, gw gwclient.Client, fsp bass.DirPath) (*gwclient.Result, error) { + copyOpt := &llb.CopyInfo{} + if fsp.IsDir() { + copyOpt.CopyDirContentsOnly = true + } + + st := llb.Scratch().File( + llb.Copy( + ib.Output, + filepath.Join(ib.OutputSourcePath, fsp.FromSlash()), + ".", + copyOpt, + ), + llb.WithCustomNamef("[hide] copy %s", fsp.Slash()), + ) + + def, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + + return gw.Solve(ctx, gwclient.SolveRequest{ + Evaluate: true, + Definition: def.ToPB(), + }) +} + +func (ib IntermediateBuild) ExportFile(ctx context.Context, gw gwclient.Client, tw *tar.Writer, fsp bass.FilePath) (*gwclient.Result, error) { + def, err := ib.Output.Marshal(ctx) + if err != nil { + return nil, err + } + + res, err := gw.Solve(ctx, gwclient.SolveRequest{ + Evaluate: true, + Definition: def.ToPB(), + }) + if err != nil { + return nil, err + } + + ref, err := res.SingleRef() + if err != nil { + return nil, err + } + + filePath := filepath.Join(ib.OutputSourcePath, fsp.FromSlash()) + + stat, err := ref.StatFile(ctx, gwclient.StatRequest{ + Path: filePath, + }) + if err != nil { + return nil, err + } + + err = tw.WriteHeader(&tar.Header{ + Name: fsp.FromSlash(), + Mode: int64(stat.Mode), + Uid: int(stat.Uid), + Gid: int(stat.Gid), + Size: int64(stat.Size_), + ModTime: time.Unix(stat.ModTime/int64(time.Second), stat.ModTime%int64(time.Second)), + Linkname: stat.Linkname, + Devmajor: stat.Devmajor, + Devminor: stat.Devmajor, + }) + if err != nil { + return nil, fmt.Errorf("write tar header: %w", err) + } + + fs := util.NewRefFS(ctx, ref) + + f, err := fs.Open(filePath) + if err != nil { + return nil, err + } + + if _, err := io.Copy(tw, f); err != nil { + return nil, err + } + + if err := f.Close(); err != nil { + return nil, err + } + + if err := tw.Close(); err != nil { + return nil, err + } + + return res, nil +} + +func (b *buildkitBuilder) image(ctx context.Context, image *bass.ThunkImage) (IntermediateBuild, error) { + ib := IntermediateBuild{ + Platform: b.platform, + } + switch { case image == nil: // TODO: test; how is this possible? @@ -1270,7 +1322,7 @@ func (b *buildkitBuilder) image(ctx context.Context, image *bass.ThunkImage) (ib return ib, nil case image.Thunk != nil: - return b.Build(ctx, *image.Thunk, getWorkdir, false) + return b.Build(ctx, *image.Thunk, false) case image.Archive != nil: file, err := image.Archive.File.Open(ctx) @@ -1434,7 +1486,7 @@ func (b *buildkitBuilder) image(ctx context.Context, image *bass.ThunkImage) (ib } ib.Output = ib.FS - ib.SourcePath = ib.Config.WorkingDir + ib.OutputSourcePath = ib.Config.WorkingDir ib.NeedsInsecure = needsInsecure return ib, nil } @@ -1520,13 +1572,13 @@ func (b *buildkitBuilder) initializeMount(ctx context.Context, source bass.Thunk } func (b *buildkitBuilder) thunkPathSt(ctx context.Context, source bass.ThunkPath) (llb.State, string, bool, error) { - ib, err := b.Build(ctx, source.Thunk, getWorkdir, true) + ib, err := b.Build(ctx, source.Thunk, true) if err != nil { return llb.State{}, "", false, fmt.Errorf("thunk llb: %w", err) } return ib.Output, - filepath.Join(ib.SourcePath, source.Path.FilesystemPath().FromSlash()), + filepath.Join(ib.OutputSourcePath, source.Path.FilesystemPath().FromSlash()), ib.NeedsInsecure, nil } diff --git a/pkg/runtimes/suite.go b/pkg/runtimes/suite.go index 5a15986b..b80966c6 100644 --- a/pkg/runtimes/suite.go +++ b/pkg/runtimes/suite.go @@ -206,12 +206,6 @@ func Suite(t *testing.T, runtimeConfig bass.RuntimeConfig, opts ...SuiteOpt) { }, { File: "remount-workdir-subdir.bass", - Result: bass.NewList( - bass.String("bar\nbaz\nfoo\n"), - bass.String("foo\n"), - bass.String("bar\n"), - bass.String("baz\n"), - ), }, { File: "timestamps.bass", diff --git a/pkg/runtimes/testdata/remount-workdir-subdir.bass b/pkg/runtimes/testdata/remount-workdir-subdir.bass index 3f045faa..25766694 100644 --- a/pkg/runtimes/testdata/remount-workdir-subdir.bass +++ b/pkg/runtimes/testdata/remount-workdir-subdir.bass @@ -14,7 +14,14 @@ (from remount ($ sh -c "echo baz > ./baz"))) -[(next (read (from from-remount ($ ls)) :raw)) - (next (read from-remount/foo :raw)) - (next (read from-remount/bar :raw)) - (next (read from-remount/baz :raw))] +(assert = "bar\nbaz\nfoo\n" + (next (read (from from-remount ($ ls)) :raw))) + +(assert = "foo\n" + (next (read from-remount/foo :raw))) + +(assert = "bar\n" + (next (read from-remount/bar :raw))) + +(assert = "baz\n" + (next (read from-remount/baz :raw))) From 5f4b59e55fe3e1a30c1c87718746e69e76c1e97f Mon Sep 17 00:00:00 2001 From: Alex Suraci Date: Wed, 12 Apr 2023 22:53:44 -0400 Subject: [PATCH 3/9] simplify SolveOpt setup, fix insecure builds * use a global content-addressed secret store * always request insecure entitlement, since it would just automatically do it anyway before. would make sense to respect worker-side config like Dagger later. --- pkg/runtimes/buildkit.go | 216 +++++++++++++++++++++++---------------- 1 file changed, 130 insertions(+), 86 deletions(-) diff --git a/pkg/runtimes/buildkit.go b/pkg/runtimes/buildkit.go index 455168fe..701aadc3 100644 --- a/pkg/runtimes/buildkit.go +++ b/pkg/runtimes/buildkit.go @@ -3,7 +3,9 @@ package runtimes import ( "archive/tar" "context" + "crypto/sha256" "embed" + "encoding/hex" "encoding/json" "fmt" "io" @@ -19,6 +21,7 @@ import ( "time" "github.com/adrg/xdg" + "github.com/pkg/errors" "github.com/containerd/containerd/content" "github.com/containerd/containerd/content/local" @@ -36,6 +39,7 @@ import ( "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/auth/authprovider" "github.com/moby/buildkit/session/filesync" + "github.com/moby/buildkit/session/secrets" "github.com/moby/buildkit/session/secrets/secretsprovider" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/entitlements" @@ -116,44 +120,14 @@ type Buildkit struct { authp session.Attachable + solveOpt bkclient.SolveOpt + + secrets *secretStore ociStore content.Store } const DefaultBuildkitInstallation = "bass-buildkitd" -func (buildkit *Buildkit) Client() (*bkclient.Client, error) { - if buildkit.client == nil { - return nil, fmt.Errorf("buildkit client unavailable") - } - - return buildkit.client, nil -} - -func (runtime *Buildkit) WithGateway(ctx context.Context, doBuild func(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error)) error { - if runtime.gateway != nil { - _, err := doBuild(ctx, runtime.gateway) - return err - } - - statusProxy := forwardStatus(progrock.RecorderFromContext(ctx)) - defer statusProxy.Wait() - - _, err := runtime.client.Build(ctx, bkclient.SolveOpt{ - Session: []session.Attachable{ - runtime.authp, - filesync.NewFSSyncProvider(AnyDirSource{}), - }, - OCIStores: map[string]content.Store{ - ociStoreName: runtime.ociStore, - }, - }, buildkitProduct, doBuild, statusProxy.Writer()) - if err != nil { - return statusProxy.NiceError("resolve failed", err) - } - - return nil -} - func NewBuildkit(ctx context.Context, _ bass.RuntimePool, cfg *bass.Scope) (bass.Runtime, error) { var config BuildkitConfig if cfg != nil { @@ -174,11 +148,6 @@ func NewBuildkit(ctx context.Context, _ bass.RuntimePool, cfg *bass.Scope) (bass config.OCIStoreDir = filepath.Join(xdg.DataHome, "bass", "oci") } - store, err := local.NewStore(config.OCIStoreDir) - if err != nil { - return nil, fmt.Errorf("create oci store: %w", err) - } - if config.CertsDir != "" { err := basstls.Init(config.CertsDir) if err != nil { @@ -207,6 +176,19 @@ func NewBuildkit(ctx context.Context, _ bass.RuntimePool, cfg *bass.Scope) (bass checkSame = platforms.Only(platform) } + authp := authprovider.NewDockerAuthProvider( + dockerconfig.LoadDefaultConfigFile(os.Stderr), + ) + + secrets := newSecretStore() + + ociStore, err := local.NewStore(config.OCIStoreDir) + if err != nil { + return nil, fmt.Errorf("create oci store: %w", err) + } + + solveOpt := newSolveOpt(authp, secrets, ociStore) + return &Buildkit{ Config: config, @@ -216,22 +198,51 @@ func NewBuildkit(ctx context.Context, _ bass.RuntimePool, cfg *bass.Scope) (bass client: client, gateway: nil, - authp: authprovider.NewDockerAuthProvider(dockerconfig.LoadDefaultConfigFile(os.Stderr)), - - ociStore: store, + authp: authp, + secrets: secrets, + ociStore: ociStore, + solveOpt: solveOpt, }, nil } +func newSolveOpt( + authp session.Attachable, + secrets *secretStore, + ociStore content.Store, +) bkclient.SolveOpt { + return bkclient.SolveOpt{ + AllowedEntitlements: []entitlements.Entitlement{ + entitlements.EntitlementSecurityInsecure, + }, + Session: []session.Attachable{ + authp, + secretsprovider.NewSecretProvider(secrets), + filesync.NewFSSyncProvider(AnyDirSource{}), + }, + OCIStores: map[string]content.Store{ + ociStoreName: ociStore, + }, + } +} + func NewBuildkitFrontend(gw gwclient.Client, inputs map[string]llb.State, config BuildkitConfig) (*Buildkit, error) { if config.OCIStoreDir == "" { config.OCIStoreDir = filepath.Join(xdg.DataHome, "bass", "oci") } + authp := authprovider.NewDockerAuthProvider( + dockerconfig.LoadDefaultConfigFile(os.Stderr), + ) + + secrets := newSecretStore() + ociStore, err := local.NewStore(config.OCIStoreDir) if err != nil { return nil, fmt.Errorf("create oci store: %w", err) } + solveOpt := newSolveOpt(authp, secrets, ociStore) + return &Buildkit{ Config: config, Platform: ocispecs.Platform{ @@ -242,12 +253,44 @@ func NewBuildkitFrontend(gw gwclient.Client, inputs map[string]llb.State, config gateway: gw, - // NB: same as above, just assigning anyway - authp: authprovider.NewDockerAuthProvider(dockerconfig.LoadDefaultConfigFile(os.Stderr)), + authp: authp, + secrets: secrets, ociStore: ociStore, + solveOpt: solveOpt, }, nil } +func (buildkit *Buildkit) Client() (*bkclient.Client, error) { + if buildkit.client == nil { + return nil, fmt.Errorf("buildkit client unavailable") + } + + return buildkit.client, nil +} + +func (runtime *Buildkit) WithGateway(ctx context.Context, doBuild func(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error)) error { + if runtime.gateway != nil { + _, err := doBuild(ctx, runtime.gateway) + return err + } + + statusProxy := forwardStatus(progrock.RecorderFromContext(ctx)) + defer statusProxy.Wait() + + _, err := runtime.client.Build( + ctx, + runtime.solveOpt, + buildkitProduct, + doBuild, + statusProxy.Writer(), + ) + if err != nil { + return statusProxy.NiceError("resolve failed", err) + } + + return nil +} + func (runtime *Buildkit) Resolve(ctx context.Context, imageRef bass.ImageRef) (bass.Thunk, error) { // track dependent services ctx, svcs := bass.TrackRuns(ctx) @@ -352,13 +395,8 @@ func (b *buildkitBuilder) Start(ctx context.Context, thunk bass.Thunk) (StartRes Evaluate: true, Definition: def.ToPB(), }) - if err != nil { - return err - } - exited <- err - - return nil + return err }) select { @@ -577,10 +615,6 @@ func (runtime *Buildkit) build( forceExec bool, runOpts ...llb.RunOption, ) (*bkclient.SolveResponse, error) { - var secrets map[string][]byte - // var localDirs map[string]string - var allowed []entitlements.Entitlement - // build llb definition using the remote gateway for image resolution var ib IntermediateBuild err := runtime.WithGateway(ctx, func(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { @@ -592,32 +626,14 @@ func (runtime *Buildkit) build( return nil, err } - if ib.NeedsInsecure { - allowed = append(allowed, entitlements.EntitlementSecurityInsecure) - } - - // localDirs = b.localDirs - secrets = b.secrets - return &gwclient.Result{}, nil }) if err != nil { return nil, err } - solveOpt := bkclient.SolveOpt{ - // LocalDirs: localDirs, - AllowedEntitlements: allowed, - Session: []session.Attachable{ - runtime.authp, - secretsprovider.FromMap(secrets), - filesync.NewFSSyncProvider(AnyDirSource{}), - }, - Exports: exports, - OCIStores: map[string]content.Store{ - ociStoreName: runtime.ociStore, - }, - } + solveOpt := newSolveOpt(runtime.authp, runtime.secrets, runtime.ociStore) + solveOpt.Exports = exports doBuild := func(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { return cb(ctx, gw, ib) @@ -754,11 +770,9 @@ type buildkitBuilder struct { inputs map[string]llb.State certsDir string ociStore content.Store + secrets *secretStore debug bool disableCache bool - - secrets map[string][]byte - localDirs map[string]string } func (runtime *Buildkit) NewBuilder(ctx context.Context, client gwclient.Client) *buildkitBuilder { @@ -767,6 +781,7 @@ func (runtime *Buildkit) NewBuilder(ctx context.Context, client gwclient.Client) runtime.Platform, runtime.Inputs, runtime.Config.CertsDir, + runtime.secrets, runtime.ociStore, runtime.Config.Debug, runtime.Config.DisableCache, @@ -778,6 +793,7 @@ func NewBuilder( platform ocispecs.Platform, inputs map[string]llb.State, certsDir string, + secrets *secretStore, ociStore content.Store, debug, disableCache bool, ) *buildkitBuilder { @@ -786,12 +802,10 @@ func NewBuilder( platform: platform, inputs: inputs, certsDir: certsDir, + secrets: secrets, ociStore: ociStore, debug: debug, disableCache: disableCache, - - secrets: map[string][]byte{}, - localDirs: map[string]string{}, } } @@ -954,8 +968,7 @@ func (b *buildkitBuilder) Build( } for _, env := range cmd.SecretEnv { - id := env.Secret.Name - b.secrets[id] = env.Secret.Reveal() + id := b.secrets.PutSecret(env.Secret.Reveal()) runOpt = append(runOpt, llb.AddSecret(env.Name, llb.SecretID(id), llb.SecretAsEnv(true))) } @@ -1562,8 +1575,7 @@ func (b *buildkitBuilder) initializeMount(ctx context.Context, source bass.Thunk ), "", false, nil case source.Secret != nil: - id := source.Secret.Name - b.secrets[id] = source.Secret.Reveal() + id := b.secrets.PutSecret(source.Secret.Reveal()) return llb.AddSecret(targetPath, llb.SecretID(id)), "", false, nil default: @@ -1592,9 +1604,6 @@ func (b *buildkitBuilder) hostPathSt(ctx context.Context, source bass.HostPath) return input, sourcePath, nil } - fullPath := source.FromSlash() - b.localDirs[fullPath] = fullPath - var excludes []string ignorePath := filepath.Join(localName, ".bassignore") ignore, err := os.Open(ignorePath) @@ -1870,3 +1879,38 @@ func resolveIndex(ctx context.Context, store content.Store, desc ocispecs.Descri return nil, fmt.Errorf("no manifest for platform %s and tag %s", platforms.Format(platform), tag) } + +type secretStore struct { + digest2secret map[string][]byte + sync.Mutex +} + +func newSecretStore() *secretStore { + return &secretStore{ + digest2secret: make(map[string][]byte), + } +} + +func (s *secretStore) PutSecret(value []byte) string { + s.Lock() + defer s.Unlock() + // generate a digest for the secret + + digest := sha256.Sum256(value) + digestStr := hex.EncodeToString(digest[:]) + s.digest2secret[digestStr] = value + + return digestStr +} + +func (s *secretStore) GetSecret(ctx context.Context, digest string) ([]byte, error) { + s.Lock() + defer s.Unlock() + + v, ok := s.digest2secret[digest] + if !ok { + return nil, errors.WithStack(secrets.ErrNotFound) + } + + return v, nil +} From 898d73f770efc0479fd880651ff684a55f85db50 Mon Sep 17 00:00:00 2001 From: Alex Suraci Date: Thu, 13 Apr 2023 19:53:26 -0400 Subject: [PATCH 4/9] move code around --- pkg/runtimes/buildkit.go | 46 ++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/pkg/runtimes/buildkit.go b/pkg/runtimes/buildkit.go index 701aadc3..14c34ccb 100644 --- a/pkg/runtimes/buildkit.go +++ b/pkg/runtimes/buildkit.go @@ -118,8 +118,6 @@ type Buildkit struct { client *bkclient.Client gateway gwclient.Client - authp session.Attachable - solveOpt bkclient.SolveOpt secrets *secretStore @@ -198,33 +196,12 @@ func NewBuildkit(ctx context.Context, _ bass.RuntimePool, cfg *bass.Scope) (bass client: client, gateway: nil, - authp: authp, secrets: secrets, ociStore: ociStore, solveOpt: solveOpt, }, nil } -func newSolveOpt( - authp session.Attachable, - secrets *secretStore, - ociStore content.Store, -) bkclient.SolveOpt { - return bkclient.SolveOpt{ - AllowedEntitlements: []entitlements.Entitlement{ - entitlements.EntitlementSecurityInsecure, - }, - Session: []session.Attachable{ - authp, - secretsprovider.NewSecretProvider(secrets), - filesync.NewFSSyncProvider(AnyDirSource{}), - }, - OCIStores: map[string]content.Store{ - ociStoreName: ociStore, - }, - } -} - func NewBuildkitFrontend(gw gwclient.Client, inputs map[string]llb.State, config BuildkitConfig) (*Buildkit, error) { if config.OCIStoreDir == "" { config.OCIStoreDir = filepath.Join(xdg.DataHome, "bass", "oci") @@ -253,7 +230,6 @@ func NewBuildkitFrontend(gw gwclient.Client, inputs map[string]llb.State, config gateway: gw, - authp: authp, secrets: secrets, ociStore: ociStore, solveOpt: solveOpt, @@ -632,7 +608,7 @@ func (runtime *Buildkit) build( return nil, err } - solveOpt := newSolveOpt(runtime.authp, runtime.secrets, runtime.ociStore) + solveOpt := runtime.solveOpt solveOpt.Exports = exports doBuild := func(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { @@ -1914,3 +1890,23 @@ func (s *secretStore) GetSecret(ctx context.Context, digest string) ([]byte, err return v, nil } + +func newSolveOpt( + authp session.Attachable, + secrets *secretStore, + ociStore content.Store, +) bkclient.SolveOpt { + return bkclient.SolveOpt{ + AllowedEntitlements: []entitlements.Entitlement{ + entitlements.EntitlementSecurityInsecure, + }, + Session: []session.Attachable{ + authp, + secretsprovider.NewSecretProvider(secrets), + filesync.NewFSSyncProvider(AnyDirSource{}), + }, + OCIStores: map[string]content.Store{ + ociStoreName: ociStore, + }, + } +} From ef3f96b1a44a7c76fb39952f5c84611b90691c6d Mon Sep 17 00:00:00 2001 From: Alex Suraci Date: Thu, 13 Apr 2023 20:51:49 -0400 Subject: [PATCH 5/9] support loading from host OCI tarballs + use it to test exported images, since that was totally broken before --- .gitignore | 2 +- pkg/bass/encoding_test.go | 37 +++- pkg/bass/thunk_types.go | 28 ++- pkg/proto/bass.pb.go | 296 +++++++++++++++--------------- pkg/runtimes/buildkit.go | 4 +- pkg/runtimes/dagger.go | 54 +++++- pkg/runtimes/testdata/export.bass | 20 +- proto/bass.proto | 3 +- 8 files changed, 278 insertions(+), 166 deletions(-) diff --git a/.gitignore b/.gitignore index d69ad383..5ce40580 100644 --- a/.gitignore +++ b/.gitignore @@ -35,7 +35,7 @@ pkg/runtimes/bin/exe.* cmd/bass/bass # created by tests, don't want it committed -pkg/runtimes/testdata/memo/bass.lock +pkg/runtimes/testdata/*.tar hack/cni-test/plugins/ iptables.rules diff --git a/pkg/bass/encoding_test.go b/pkg/bass/encoding_test.go index f76393a6..d0806bcd 100644 --- a/pkg/bass/encoding_test.go +++ b/pkg/bass/encoding_test.go @@ -216,9 +216,11 @@ var validThunkImageRefs = []bass.ImageRef{ var validThunkImageArchives = []bass.ImageArchive{ { - File: bass.ThunkPath{ - Thunk: validBasicThunk, - Path: bass.ParseFileOrDirPath("image.tar"), + File: bass.ImageBuildInput{ + Thunk: &bass.ThunkPath{ + Thunk: validBasicThunk, + Path: bass.ParseFileOrDirPath("image.tar"), + }, }, Platform: bass.Platform{ OS: "os", @@ -227,9 +229,11 @@ var validThunkImageArchives = []bass.ImageArchive{ Tag: "tag", }, { - File: bass.ThunkPath{ - Thunk: validBasicThunk, - Path: bass.ParseFileOrDirPath("image.tar"), + File: bass.ImageBuildInput{ + Thunk: &bass.ThunkPath{ + Thunk: validBasicThunk, + Path: bass.ParseFileOrDirPath("image.tar"), + }, }, Platform: bass.Platform{ OS: "os", @@ -237,6 +241,27 @@ var validThunkImageArchives = []bass.ImageArchive{ }, // no tag }, + { + File: bass.ImageBuildInput{ + Host: &bass.HostPath{ + ContextDir: "/not/real", + Path: bass.ParseFileOrDirPath("image.tar"), + }, + }, + Platform: bass.Platform{ + OS: "os", + Architecture: "arch", + }, + }, + { + File: bass.ImageBuildInput{ + FS: bass.NewInMemoryFile("fs/mount-dir/file", "hello"), + }, + Platform: bass.Platform{ + OS: "os", + Architecture: "arch", + }, + }, } func ptr[T any](v T) *T { diff --git a/pkg/bass/thunk_types.go b/pkg/bass/thunk_types.go index 58b5ce68..8bbdc91a 100644 --- a/pkg/bass/thunk_types.go +++ b/pkg/bass/thunk_types.go @@ -722,7 +722,7 @@ func (repo *ImageRepository) FromValue(val Value) error { // ImageArchive specifies an OCI image tarball. type ImageArchive struct { // An OCI image archive tarball to load. - File ThunkPath `json:"file"` + File ImageBuildInput `json:"file"` // The platform to target; influences runtime selection. Platform Platform `json:"platform"` @@ -770,7 +770,7 @@ func (ref ImageArchive) MarshalProto() (proto.Message, error) { return nil, fmt.Errorf("file: %w", err) } - pv.File = tp.(*proto.ThunkPath) + pv.File = tp.(*proto.ImageBuildInput) return pv, nil } @@ -954,6 +954,30 @@ func (enum ImageBuildInput) ToValue() Value { } } +func (enum ImageBuildInput) ToPath() Path { + if enum.FS != nil { + return enum.FS + } else if enum.Host != nil { + return *enum.Host + } else if enum.Thunk != nil { + return *enum.Thunk + } else { + panic("empty ImageBuildInput") + } +} + +func (enum *ImageBuildInput) ToReadable() Readable { + if enum.FS != nil { + return enum.FS + } else if enum.Host != nil { + return *enum.Host + } else if enum.Thunk != nil { + return *enum.Thunk + } else { + panic("empty ImageBuildInput") + } +} + func (enum *ImageBuildInput) UnmarshalJSON(payload []byte) error { return UnmarshalJSON(payload, enum) } diff --git a/pkg/proto/bass.pb.go b/pkg/proto/bass.pb.go index e590648c..db393e85 100644 --- a/pkg/proto/bass.pb.go +++ b/pkg/proto/bass.pb.go @@ -913,9 +913,10 @@ type ImageArchive struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Platform *Platform `protobuf:"bytes,1,opt,name=platform,proto3" json:"platform,omitempty"` - File *ThunkPath `protobuf:"bytes,2,opt,name=file,proto3" json:"file,omitempty"` - Tag *string `protobuf:"bytes,3,opt,name=tag,proto3,oneof" json:"tag,omitempty"` + Platform *Platform `protobuf:"bytes,1,opt,name=platform,proto3" json:"platform,omitempty"` + // ThunkPath file = 2; + File *ImageBuildInput `protobuf:"bytes,4,opt,name=file,proto3" json:"file,omitempty"` + Tag *string `protobuf:"bytes,3,opt,name=tag,proto3,oneof" json:"tag,omitempty"` } func (x *ImageArchive) Reset() { @@ -957,7 +958,7 @@ func (x *ImageArchive) GetPlatform() *Platform { return nil } -func (x *ImageArchive) GetFile() *ThunkPath { +func (x *ImageArchive) GetFile() *ImageBuildInput { if x != nil { return x.File } @@ -2615,149 +2616,150 @@ var file_bass_proto_rawDesc = []byte{ 0x01, 0x12, 0x1b, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x88, 0x01, 0x01, 0x42, 0x08, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x74, 0x61, 0x67, - 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0x7e, 0x0a, 0x0c, 0x49, - 0x6d, 0x61, 0x67, 0x65, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x12, 0x2a, 0x0a, 0x08, 0x70, - 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, - 0x62, 0x61, 0x73, 0x73, 0x2e, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x08, 0x70, - 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x23, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x54, 0x68, 0x75, - 0x6e, 0x6b, 0x50, 0x61, 0x74, 0x68, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x15, 0x0a, 0x03, - 0x74, 0x61, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x03, 0x74, 0x61, 0x67, - 0x88, 0x01, 0x01, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x74, 0x61, 0x67, 0x22, 0xef, 0x01, 0x0a, 0x10, - 0x49, 0x6d, 0x61, 0x67, 0x65, 0x44, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x42, 0x75, 0x69, 0x6c, 0x64, - 0x12, 0x2a, 0x0a, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, - 0x72, 0x6d, 0x52, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x2f, 0x0a, 0x07, - 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, - 0x62, 0x61, 0x73, 0x73, 0x2e, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x49, - 0x6e, 0x70, 0x75, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x23, 0x0a, - 0x0a, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x48, 0x00, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x66, 0x69, 0x6c, 0x65, 0x88, - 0x01, 0x01, 0x12, 0x1b, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x48, 0x01, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x88, 0x01, 0x01, 0x12, - 0x22, 0x0a, 0x04, 0x61, 0x72, 0x67, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, - 0x62, 0x61, 0x73, 0x73, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x41, 0x72, 0x67, 0x52, 0x04, 0x61, - 0x72, 0x67, 0x73, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x66, 0x69, - 0x6c, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0x98, 0x01, - 0x0a, 0x0f, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x6e, 0x70, 0x75, - 0x74, 0x12, 0x27, 0x0a, 0x05, 0x74, 0x68, 0x75, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x0f, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x54, 0x68, 0x75, 0x6e, 0x6b, 0x50, 0x61, 0x74, - 0x68, 0x48, 0x00, 0x52, 0x05, 0x74, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x24, 0x0a, 0x04, 0x68, 0x6f, - 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, - 0x48, 0x6f, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x48, 0x00, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, - 0x12, 0x2d, 0x0a, 0x07, 0x6c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x11, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, - 0x50, 0x61, 0x74, 0x68, 0x48, 0x00, 0x52, 0x07, 0x6c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x42, - 0x07, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x22, 0x34, 0x0a, 0x08, 0x42, 0x75, 0x69, 0x6c, - 0x64, 0x41, 0x72, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x2e, - 0x0a, 0x08, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x73, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x6f, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x61, 0x72, - 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x61, 0x72, 0x63, 0x68, 0x22, 0x87, - 0x01, 0x0a, 0x08, 0x54, 0x68, 0x75, 0x6e, 0x6b, 0x44, 0x69, 0x72, 0x12, 0x25, 0x0a, 0x05, 0x6c, - 0x6f, 0x63, 0x61, 0x6c, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x62, 0x61, 0x73, - 0x73, 0x2e, 0x44, 0x69, 0x72, 0x50, 0x61, 0x74, 0x68, 0x48, 0x00, 0x52, 0x05, 0x6c, 0x6f, 0x63, - 0x61, 0x6c, 0x12, 0x27, 0x0a, 0x05, 0x74, 0x68, 0x75, 0x6e, 0x6b, 0x18, 0x0d, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x0f, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x54, 0x68, 0x75, 0x6e, 0x6b, 0x50, 0x61, - 0x74, 0x68, 0x48, 0x00, 0x52, 0x05, 0x74, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x24, 0x0a, 0x04, 0x68, - 0x6f, 0x73, 0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x61, 0x73, 0x73, - 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x48, 0x00, 0x52, 0x04, 0x68, 0x6f, 0x73, - 0x74, 0x42, 0x05, 0x0a, 0x03, 0x64, 0x69, 0x72, 0x22, 0xeb, 0x01, 0x0a, 0x10, 0x54, 0x68, 0x75, - 0x6e, 0x6b, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x27, 0x0a, - 0x05, 0x74, 0x68, 0x75, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x62, - 0x61, 0x73, 0x73, 0x2e, 0x54, 0x68, 0x75, 0x6e, 0x6b, 0x50, 0x61, 0x74, 0x68, 0x48, 0x00, 0x52, - 0x05, 0x74, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x24, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x48, 0x6f, 0x73, 0x74, - 0x50, 0x61, 0x74, 0x68, 0x48, 0x00, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x07, - 0x6c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, + 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0x84, 0x01, 0x0a, 0x0c, + 0x49, 0x6d, 0x61, 0x67, 0x65, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x12, 0x2a, 0x0a, 0x08, + 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, + 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x08, + 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x29, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x49, 0x6d, + 0x61, 0x67, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x04, 0x66, + 0x69, 0x6c, 0x65, 0x12, 0x15, 0x0a, 0x03, 0x74, 0x61, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x48, 0x00, 0x52, 0x03, 0x74, 0x61, 0x67, 0x88, 0x01, 0x01, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x74, + 0x61, 0x67, 0x22, 0xef, 0x01, 0x0a, 0x10, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x44, 0x6f, 0x63, 0x6b, + 0x65, 0x72, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x12, 0x2a, 0x0a, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, + 0x6f, 0x72, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x61, 0x73, 0x73, + 0x2e, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, + 0x6f, 0x72, 0x6d, 0x12, 0x2f, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x49, 0x6d, 0x61, 0x67, + 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x12, 0x23, 0x0a, 0x0a, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x66, 0x69, + 0x6c, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0a, 0x64, 0x6f, 0x63, 0x6b, + 0x65, 0x72, 0x66, 0x69, 0x6c, 0x65, 0x88, 0x01, 0x01, 0x12, 0x1b, 0x0a, 0x06, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x06, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x88, 0x01, 0x01, 0x12, 0x22, 0x0a, 0x04, 0x61, 0x72, 0x67, 0x73, 0x18, 0x05, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x42, 0x75, 0x69, 0x6c, + 0x64, 0x41, 0x72, 0x67, 0x52, 0x04, 0x61, 0x72, 0x67, 0x73, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x64, + 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x66, 0x69, 0x6c, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x74, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x22, 0x98, 0x01, 0x0a, 0x0f, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x42, 0x75, + 0x69, 0x6c, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x27, 0x0a, 0x05, 0x74, 0x68, 0x75, 0x6e, + 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x54, + 0x68, 0x75, 0x6e, 0x6b, 0x50, 0x61, 0x74, 0x68, 0x48, 0x00, 0x52, 0x05, 0x74, 0x68, 0x75, 0x6e, + 0x6b, 0x12, 0x24, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x0e, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x48, + 0x00, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x07, 0x6c, 0x6f, 0x67, 0x69, 0x63, + 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, + 0x4c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x48, 0x00, 0x52, 0x07, 0x6c, + 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x42, 0x07, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x22, + 0x34, 0x0a, 0x08, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x41, 0x72, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x2e, 0x0a, 0x08, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, + 0x6d, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x6f, + 0x73, 0x12, 0x12, 0x0a, 0x04, 0x61, 0x72, 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x61, 0x72, 0x63, 0x68, 0x22, 0x87, 0x01, 0x0a, 0x08, 0x54, 0x68, 0x75, 0x6e, 0x6b, 0x44, + 0x69, 0x72, 0x12, 0x25, 0x0a, 0x05, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x18, 0x0c, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x0d, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x44, 0x69, 0x72, 0x50, 0x61, 0x74, 0x68, + 0x48, 0x00, 0x52, 0x05, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x12, 0x27, 0x0a, 0x05, 0x74, 0x68, 0x75, + 0x6e, 0x6b, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, + 0x54, 0x68, 0x75, 0x6e, 0x6b, 0x50, 0x61, 0x74, 0x68, 0x48, 0x00, 0x52, 0x05, 0x74, 0x68, 0x75, + 0x6e, 0x6b, 0x12, 0x24, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x0e, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, + 0x48, 0x00, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x42, 0x05, 0x0a, 0x03, 0x64, 0x69, 0x72, 0x22, + 0xeb, 0x01, 0x0a, 0x10, 0x54, 0x68, 0x75, 0x6e, 0x6b, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x53, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x12, 0x27, 0x0a, 0x05, 0x74, 0x68, 0x75, 0x6e, 0x6b, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x54, 0x68, 0x75, 0x6e, 0x6b, + 0x50, 0x61, 0x74, 0x68, 0x48, 0x00, 0x52, 0x05, 0x74, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x24, 0x0a, + 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x61, + 0x73, 0x73, 0x2e, 0x48, 0x6f, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x48, 0x00, 0x52, 0x04, 0x68, + 0x6f, 0x73, 0x74, 0x12, 0x2d, 0x0a, 0x07, 0x6c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x4c, 0x6f, 0x67, 0x69, + 0x63, 0x61, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x48, 0x00, 0x52, 0x07, 0x6c, 0x6f, 0x67, 0x69, 0x63, + 0x61, 0x6c, 0x12, 0x27, 0x0a, 0x05, 0x63, 0x61, 0x63, 0x68, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x0f, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x50, 0x61, + 0x74, 0x68, 0x48, 0x00, 0x52, 0x05, 0x63, 0x61, 0x63, 0x68, 0x65, 0x12, 0x26, 0x0a, 0x06, 0x73, + 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x62, 0x61, + 0x73, 0x73, 0x2e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x06, 0x73, 0x65, 0x63, + 0x72, 0x65, 0x74, 0x42, 0x08, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x6a, 0x0a, + 0x0a, 0x54, 0x68, 0x75, 0x6e, 0x6b, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2e, 0x0a, 0x06, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x62, 0x61, + 0x73, 0x73, 0x2e, 0x54, 0x68, 0x75, 0x6e, 0x6b, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x53, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x2c, 0x0a, 0x06, 0x74, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x62, 0x61, + 0x73, 0x73, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x50, 0x61, 0x74, + 0x68, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0x2c, 0x0a, 0x05, 0x41, 0x72, 0x72, + 0x61, 0x79, 0x12, 0x23, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, + 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x22, 0x33, 0x0a, 0x06, 0x4f, 0x62, 0x6a, 0x65, 0x63, + 0x74, 0x12, 0x29, 0x0a, 0x08, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x42, 0x69, 0x6e, 0x64, 0x69, + 0x6e, 0x67, 0x52, 0x08, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x73, 0x22, 0x44, 0x0a, 0x07, + 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x79, 0x6d, 0x62, 0x6f, + 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x12, + 0x21, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, + 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x22, 0x06, 0x0a, 0x04, 0x4e, 0x75, 0x6c, 0x6c, 0x22, 0x1c, 0x0a, 0x04, 0x42, 0x6f, + 0x6f, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x1b, 0x0a, 0x03, 0x49, 0x6e, 0x74, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x1e, 0x0a, 0x06, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x7e, 0x0a, 0x09, 0x43, 0x61, 0x63, 0x68, 0x65, 0x50, 0x61, + 0x74, 0x68, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, + 0x69, 0x64, 0x12, 0x28, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x14, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, + 0x65, 0x6d, 0x50, 0x61, 0x74, 0x68, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x37, 0x0a, 0x0b, + 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x15, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, + 0x65, 0x6e, 0x63, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, + 0x72, 0x65, 0x6e, 0x63, 0x79, 0x22, 0x1c, 0x0a, 0x06, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, + 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x22, 0x21, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x50, 0x61, + 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x1e, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x50, 0x61, + 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x1d, 0x0a, 0x07, 0x44, 0x69, 0x72, 0x50, 0x61, 0x74, + 0x68, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x61, 0x0a, 0x0e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, + 0x74, 0x65, 0x6d, 0x50, 0x61, 0x74, 0x68, 0x12, 0x24, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x46, 0x69, 0x6c, + 0x65, 0x50, 0x61, 0x74, 0x68, 0x48, 0x00, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x21, 0x0a, + 0x03, 0x64, 0x69, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x62, 0x61, 0x73, + 0x73, 0x2e, 0x44, 0x69, 0x72, 0x50, 0x61, 0x74, 0x68, 0x48, 0x00, 0x52, 0x03, 0x64, 0x69, 0x72, + 0x42, 0x06, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x58, 0x0a, 0x09, 0x54, 0x68, 0x75, 0x6e, + 0x6b, 0x50, 0x61, 0x74, 0x68, 0x12, 0x21, 0x0a, 0x05, 0x74, 0x68, 0x75, 0x6e, 0x6b, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x54, 0x68, 0x75, 0x6e, + 0x6b, 0x52, 0x05, 0x74, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x28, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x46, 0x69, + 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x50, 0x61, 0x74, 0x68, 0x52, 0x04, 0x70, 0x61, + 0x74, 0x68, 0x22, 0x4e, 0x0a, 0x08, 0x48, 0x6f, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x12, 0x18, + 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x28, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x46, 0x69, + 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x50, 0x61, 0x74, 0x68, 0x52, 0x04, 0x70, 0x61, + 0x74, 0x68, 0x22, 0xec, 0x01, 0x0a, 0x0b, 0x4c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x50, 0x61, + 0x74, 0x68, 0x12, 0x2c, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x16, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x50, + 0x61, 0x74, 0x68, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x00, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, + 0x12, 0x29, 0x0a, 0x03, 0x64, 0x69, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, + 0x62, 0x61, 0x73, 0x73, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x50, 0x61, 0x74, 0x68, + 0x2e, 0x44, 0x69, 0x72, 0x48, 0x00, 0x52, 0x03, 0x64, 0x69, 0x72, 0x1a, 0x34, 0x0a, 0x04, 0x46, + 0x69, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x1a, 0x46, 0x0a, 0x03, 0x44, 0x69, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2b, 0x0a, 0x07, + 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x50, 0x61, 0x74, 0x68, - 0x48, 0x00, 0x52, 0x07, 0x6c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x12, 0x27, 0x0a, 0x05, 0x63, - 0x61, 0x63, 0x68, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x62, 0x61, 0x73, - 0x73, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x50, 0x61, 0x74, 0x68, 0x48, 0x00, 0x52, 0x05, 0x63, - 0x61, 0x63, 0x68, 0x65, 0x12, 0x26, 0x0a, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x53, 0x65, 0x63, 0x72, - 0x65, 0x74, 0x48, 0x00, 0x52, 0x06, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x42, 0x08, 0x0a, 0x06, - 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x6a, 0x0a, 0x0a, 0x54, 0x68, 0x75, 0x6e, 0x6b, 0x4d, - 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2e, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x54, 0x68, 0x75, 0x6e, - 0x6b, 0x4d, 0x6f, 0x75, 0x6e, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x06, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x12, 0x2c, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x46, 0x69, 0x6c, 0x65, - 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x50, 0x61, 0x74, 0x68, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x22, 0x2c, 0x0a, 0x05, 0x41, 0x72, 0x72, 0x61, 0x79, 0x12, 0x23, 0x0a, 0x06, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x62, 0x61, - 0x73, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, - 0x22, 0x33, 0x0a, 0x06, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x29, 0x0a, 0x08, 0x62, 0x69, - 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x62, - 0x61, 0x73, 0x73, 0x2e, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x08, 0x62, 0x69, 0x6e, - 0x64, 0x69, 0x6e, 0x67, 0x73, 0x22, 0x44, 0x0a, 0x07, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, - 0x12, 0x16, 0x0a, 0x06, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x12, 0x21, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x06, 0x0a, 0x04, 0x4e, - 0x75, 0x6c, 0x6c, 0x22, 0x1c, 0x0a, 0x04, 0x42, 0x6f, 0x6f, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x22, 0x1b, 0x0a, 0x03, 0x49, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x1e, - 0x0a, 0x06, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x7e, - 0x0a, 0x09, 0x43, 0x61, 0x63, 0x68, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x0e, 0x0a, 0x02, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x04, 0x70, - 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x62, 0x61, 0x73, 0x73, - 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x50, 0x61, 0x74, 0x68, 0x52, - 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x37, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, - 0x65, 0x6e, 0x63, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x62, 0x61, 0x73, - 0x73, 0x2e, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x4d, 0x6f, 0x64, - 0x65, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x22, 0x1c, - 0x0a, 0x06, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x21, 0x0a, 0x0b, - 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x50, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, - 0x1e, 0x0a, 0x08, 0x46, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x70, - 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, - 0x1d, 0x0a, 0x07, 0x44, 0x69, 0x72, 0x50, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, - 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x61, - 0x0a, 0x0e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x50, 0x61, 0x74, 0x68, - 0x12, 0x24, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, - 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x74, 0x68, 0x48, 0x00, - 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x21, 0x0a, 0x03, 0x64, 0x69, 0x72, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x44, 0x69, 0x72, 0x50, 0x61, - 0x74, 0x68, 0x48, 0x00, 0x52, 0x03, 0x64, 0x69, 0x72, 0x42, 0x06, 0x0a, 0x04, 0x70, 0x61, 0x74, - 0x68, 0x22, 0x58, 0x0a, 0x09, 0x54, 0x68, 0x75, 0x6e, 0x6b, 0x50, 0x61, 0x74, 0x68, 0x12, 0x21, - 0x0a, 0x05, 0x74, 0x68, 0x75, 0x6e, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, - 0x62, 0x61, 0x73, 0x73, 0x2e, 0x54, 0x68, 0x75, 0x6e, 0x6b, 0x52, 0x05, 0x74, 0x68, 0x75, 0x6e, - 0x6b, 0x12, 0x28, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x14, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, - 0x6d, 0x50, 0x61, 0x74, 0x68, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x4e, 0x0a, 0x08, 0x48, - 0x6f, 0x73, 0x74, 0x50, 0x61, 0x74, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, - 0x74, 0x12, 0x28, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x14, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x79, 0x73, 0x74, 0x65, - 0x6d, 0x50, 0x61, 0x74, 0x68, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0xec, 0x01, 0x0a, 0x0b, - 0x4c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x12, 0x2c, 0x0a, 0x04, 0x66, - 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x62, 0x61, 0x73, 0x73, - 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x2e, 0x46, 0x69, 0x6c, - 0x65, 0x48, 0x00, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x29, 0x0a, 0x03, 0x64, 0x69, 0x72, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x4c, 0x6f, - 0x67, 0x69, 0x63, 0x61, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x2e, 0x44, 0x69, 0x72, 0x48, 0x00, 0x52, - 0x03, 0x64, 0x69, 0x72, 0x1a, 0x34, 0x0a, 0x04, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x1a, 0x46, 0x0a, 0x03, 0x44, 0x69, - 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2b, 0x0a, 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x62, 0x61, 0x73, 0x73, 0x2e, 0x4c, 0x6f, - 0x67, 0x69, 0x63, 0x61, 0x6c, 0x50, 0x61, 0x74, 0x68, 0x52, 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, - 0x65, 0x73, 0x42, 0x06, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x2a, 0x69, 0x0a, 0x0f, 0x43, 0x6f, - 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1b, 0x0a, - 0x17, 0x43, 0x4f, 0x4e, 0x43, 0x55, 0x52, 0x52, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x4d, 0x4f, 0x44, - 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x44, 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x43, 0x4f, - 0x4e, 0x43, 0x55, 0x52, 0x52, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x50, - 0x52, 0x49, 0x56, 0x41, 0x54, 0x45, 0x10, 0x01, 0x12, 0x1b, 0x0a, 0x17, 0x43, 0x4f, 0x4e, 0x43, - 0x55, 0x52, 0x52, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x4c, 0x4f, 0x43, - 0x4b, 0x45, 0x44, 0x10, 0x02, 0x42, 0x0b, 0x5a, 0x09, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x42, 0x06, 0x0a, 0x04, 0x70, 0x61, 0x74, + 0x68, 0x2a, 0x69, 0x0a, 0x0f, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x63, 0x79, + 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x1b, 0x0a, 0x17, 0x43, 0x4f, 0x4e, 0x43, 0x55, 0x52, 0x52, 0x45, + 0x4e, 0x43, 0x59, 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x44, 0x10, + 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x43, 0x4f, 0x4e, 0x43, 0x55, 0x52, 0x52, 0x45, 0x4e, 0x43, 0x59, + 0x5f, 0x4d, 0x4f, 0x44, 0x45, 0x5f, 0x50, 0x52, 0x49, 0x56, 0x41, 0x54, 0x45, 0x10, 0x01, 0x12, + 0x1b, 0x0a, 0x17, 0x43, 0x4f, 0x4e, 0x43, 0x55, 0x52, 0x52, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x4d, + 0x4f, 0x44, 0x45, 0x5f, 0x4c, 0x4f, 0x43, 0x4b, 0x45, 0x44, 0x10, 0x02, 0x42, 0x0b, 0x5a, 0x09, + 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( @@ -2847,7 +2849,7 @@ var file_bass_proto_depIdxs = []int32{ 29, // 33: bass.ImageRef.file:type_name -> bass.ThunkPath 3, // 34: bass.ImageRef.addr:type_name -> bass.ThunkAddr 12, // 35: bass.ImageArchive.platform:type_name -> bass.Platform - 29, // 36: bass.ImageArchive.file:type_name -> bass.ThunkPath + 10, // 36: bass.ImageArchive.file:type_name -> bass.ImageBuildInput 12, // 37: bass.ImageDockerBuild.platform:type_name -> bass.Platform 10, // 38: bass.ImageDockerBuild.context:type_name -> bass.ImageBuildInput 11, // 39: bass.ImageDockerBuild.args:type_name -> bass.BuildArg diff --git a/pkg/runtimes/buildkit.go b/pkg/runtimes/buildkit.go index 14c34ccb..3d59f46c 100644 --- a/pkg/runtimes/buildkit.go +++ b/pkg/runtimes/buildkit.go @@ -1314,7 +1314,7 @@ func (b *buildkitBuilder) image(ctx context.Context, image *bass.ThunkImage) (In return b.Build(ctx, *image.Thunk, false) case image.Archive != nil: - file, err := image.Archive.File.Open(ctx) + file, err := image.Archive.File.ToReadable().Open(ctx) if err != nil { return ib, fmt.Errorf("image archive file: %w", err) } @@ -1340,7 +1340,7 @@ func (b *buildkitBuilder) image(ctx context.Context, image *bass.ThunkImage) (In // NB: the repository portion of this ref doesn't actually matter, but it's // pleasant to see something recognizable. - dummyRepo := path.Join(image.Archive.File.Thunk.Name(), image.Archive.File.Name()) + dummyRepo := path.Join("load", image.Archive.File.ToPath().Name()) st := llb.OCILayout( fmt.Sprintf("%s@%s", dummyRepo, manifestDesc.Digest), diff --git a/pkg/runtimes/dagger.go b/pkg/runtimes/dagger.go index d9105e5b..7d5400df 100644 --- a/pkg/runtimes/dagger.go +++ b/pkg/runtimes/dagger.go @@ -503,12 +503,12 @@ func (runtime *Dagger) image(ctx context.Context, client *dagger.Client, image * } if image.Archive != nil { - srcCtr, err := runtime.container(ctx, client, image.Archive.File.Thunk, true) // TODO: or false? + file, err := runtime.inputFile(ctx, client, image.Archive.File) if err != nil { return "", nil, fmt.Errorf("image thunk: %w", err) } - name := image.Archive.File.String() + name := image.Archive.File.ToValue().String() if image.Archive.Tag != "" { name += ":" + image.Archive.Tag } @@ -517,12 +517,56 @@ func (runtime *Dagger) image(ctx context.Context, client *dagger.Client, image * ctr := client.Container(dagger.ContainerOpts{ Platform: dagger.Platform(image.Archive.Platform.String()), - }).Import(srcCtr.File(image.Archive.File.Path.Slash()), dagger.ContainerImportOpts{ - Tag: image.Archive.Tag, - }) + }).Import(file) return "", ctr, nil } return "", nil, fmt.Errorf("unsupported image type: %s", image.ToValue()) } + +func (runtime *Dagger) inputFile(ctx context.Context, client *dagger.Client, input bass.ImageBuildInput) (*dagger.File, error) { + switch { + case input.Thunk != nil: + srcCtr, err := runtime.container(ctx, client, input.Thunk.Thunk, true) // TODO: or false? + if err != nil { + return nil, fmt.Errorf("image thunk: %w", err) + } + + return srcCtr.File(input.Thunk.Path.Slash()), nil + case input.Host != nil: + dir := client.Host().Directory(input.Host.ContextDir) + fsp := input.Host.Path.FilesystemPath() + return dir.File(fsp.FromSlash()), nil + case input.FS != nil: + dir := client.Directory() + + root := path.Clean(input.FS.Path.Slash()) + err := fs.WalkDir(input.FS.FS, ".", func(entry string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if d.IsDir() { + return nil + } + + content, err := fs.ReadFile(input.FS.FS, entry) + if err != nil { + return fmt.Errorf("read fs %s: %w", entry, err) + } + + dir = dir.WithNewFile(entry, string(content)) + + return nil + }) + if err != nil { + return nil, fmt.Errorf("walk %s: %w", root, err) + } + + fsp := input.FS.Path.FilesystemPath() + return dir.File(fsp.Slash()), nil + default: + return nil, fmt.Errorf("unknown input type: %T", input.ToValue()) + } +} diff --git a/pkg/runtimes/testdata/export.bass b/pkg/runtimes/testdata/export.bass index 7066fa07..342d5b23 100644 --- a/pkg/runtimes/testdata/export.bass +++ b/pkg/runtimes/testdata/export.bass @@ -7,8 +7,10 @@ fi:name)) (def thunk - (from (linux/alpine) - ($ echo "Hello, world!"))) + (-> (from (linux/alpine) + ($ sh -c "echo hello > /hello")) + (with-entrypoint ["echo" "hello from entrypoint"]) + (with-default-args []))) ; remove default ["/bin/sh"] arg (defn includes? [haystack needle] (case haystack @@ -21,3 +23,17 @@ (assert includes? res "oci-layout") (assert includes? res "blobs/") (assert includes? res "blobs/sha256/")) + +; test that we can export + re-import and everything is OK +(write (export thunk) *dir*/export.tar) + +(assert = "hello\n" + (-> (from (oci-load *dir*/export.tar {:os "linux"}) + ($ cat /hello)) + (read :raw) + next)) + +(assert = "hello from entrypoint\n" + (-> (oci-load *dir*/export.tar {:os "linux"}) + (read :raw) + next)) diff --git a/proto/bass.proto b/proto/bass.proto index dc86a675..91b83a1a 100644 --- a/proto/bass.proto +++ b/proto/bass.proto @@ -80,7 +80,8 @@ message ImageRef { message ImageArchive { Platform platform = 1; - ThunkPath file = 2; + /* ThunkPath file = 2; */ + ImageBuildInput file = 4; optional string tag = 3; }; From 2fa9ef0a36daa7cc0e2f3ba890172f7470401d6d Mon Sep 17 00:00:00 2001 From: Alex Suraci Date: Thu, 13 Apr 2023 22:03:28 -0400 Subject: [PATCH 6/9] don't assume HostPath is directly on local fs this is necessary to support loading Bass scripts from a Buildkit frontend input --- pkg/bass/host_path.go | 48 ++++++++++++++++++++++++++++++++++--------- pkg/bass/session.go | 14 ++++++------- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/pkg/bass/host_path.go b/pkg/bass/host_path.go index 561ba093..a1f3c2f8 100644 --- a/pkg/bass/host_path.go +++ b/pkg/bass/host_path.go @@ -15,15 +15,15 @@ import ( // HostPath is a Path representing an absolute path on the host machine's // filesystem. +// +// As of Bass v0.12.0, it is a bit of a misnomer; ContextDir might also be a +// logical name of a filesystem instead. It will be passed to the global FS +// instance to produce a fs.FS. type HostPath struct { ContextDir string `json:"context"` Path FileOrDirPath `json:"path"` } -func (hp HostPath) FromSlash() string { - return filepath.Join(hp.ContextDir, hp.Path.FilesystemPath().FromSlash()) -} - var _ Value = HostPath{} func NewHostDir(contextDir string) HostPath { @@ -195,17 +195,45 @@ func (path HostPath) Extend(ext Path) (Path, error) { var _ Readable = HostPath{} -func (path HostPath) CachePath(_ctx context.Context, _dest string) (string, error) { - abs, _, err := path.checkEscape() +func (path HostPath) CachePath(ctx context.Context, dest string) (string, error) { + switch FS.(type) { + // NB: this is a super leaky abstraction, but it seems wasteful to "cache" + // host directories back to the host + case HostFilesystem: + abs, _, err := path.checkEscape() + if err != nil { + return "", err + } + + return abs, nil + default: + return Cache( + ctx, + filepath.Join( + dest, + "host-paths", + path.Hash(), + path.Path.FilesystemPath().FromSlash(), + ), + path, + ) + } +} + +func (path HostPath) FSPath() (*FSPath, error) { + fs, err := FS.FS(path.ContextDir) if err != nil { - return "", err + return nil, err } - return abs, nil + return &FSPath{ + FS: fs, + Path: path.Path, + }, nil } func (path HostPath) Open(context.Context) (io.ReadCloser, error) { - // TODO: this is currently inconsistent with the Bass runtimme which allows + // TODO: this is currently inconsistent with the Bass runtime which allows // ../ to escape the context dir. // // it would be nice to ALWAYS restrict to the context dir. the runtime is @@ -224,7 +252,7 @@ func (path HostPath) Open(context.Context) (io.ReadCloser, error) { } func (path HostPath) Write(ctx context.Context, src io.Reader) error { - // TODO: this is currently inconsistent with the Bass runtimme which allows + // TODO: this is currently inconsistent with the Bass runtime which allows // ../ to escape the context dir. // // it would be nice to ALWAYS restrict to the context dir. the runtime is diff --git a/pkg/bass/session.go b/pkg/bass/session.go index c31cdca4..42fbb63b 100644 --- a/pkg/bass/session.go +++ b/pkg/bass/session.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "path/filepath" "sync" "github.com/vito/bass/std" @@ -109,17 +108,16 @@ func (session *Session) run(ctx context.Context, thunk Thunk, state RunState, ru if cmd.Decode(&hostp) == nil { ok = true - fp := filepath.Join(hostp.FromSlash()) - abs, err := filepath.Abs(filepath.Dir(fp)) + state.Dir = hostp.Dir() + + module = NewRunScope(session.Root, state) + + fsp, err := hostp.FSPath() if err != nil { return nil, err } - state.Dir = NewHostDir(abs) - - module = NewRunScope(session.Root, state) - - _, err = EvalFile(ctx, module, fp, hostp) + _, err = EvalFSFile(ctx, module, fsp) if err != nil { return nil, err } From 5c77870c5eb2f4af02fcea7830236b24fa92b3eb Mon Sep 17 00:00:00 2001 From: Alex Suraci Date: Thu, 13 Apr 2023 22:03:39 -0400 Subject: [PATCH 7/9] consolidate *context* into just *dir* --- Dockerfile | 2 +- cmd/bass/frontend.go | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4f90153a..36384bcc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ (use (*dir*/bass/bass.bass)) (def dist - (bass:dist *context* "dev" "linux" "amd64")) + (bass:dist *dir* "dev" "linux" "amd64")) (-> (from (linux/alpine) ($ cp dist/bass /usr/local/bin/bass)) diff --git a/cmd/bass/frontend.go b/cmd/bass/frontend.go index c2cfbc40..ecbf747f 100644 --- a/cmd/bass/frontend.go +++ b/cmd/bass/frontend.go @@ -174,15 +174,14 @@ func frontendBuild(ctx context.Context, gw gwclient.Client) (*gwclient.Result, e ctx = bass.WithRuntimePool(ctx, pool) runSt := bass.RunState{ - Env: bass.NewEmptyScope(), // TODO: build args - Dir: bass.NewFSPath(scriptFs, bass.ParseFileOrDirPath(path.Dir(scriptFn))), + Env: bass.NewEmptyScope(), // TODO: build args + // directly pass the context by local name, masquerading it as the host path + Dir: bass.NewHostDir(localNameContext), Stdin: bass.Stdin, Stdout: bass.Stdout, } module := bass.NewRunScope(bass.Ground, runSt) - // directly pass the context by local name, masquerading it as the host path - module.Set("*context*", bass.NewHostDir(localNameContext)) val, err := bass.EvalFSFile(ctx, module, bass.NewFSPath(scriptFs, bass.ParseFileOrDirPath(scriptFn))) if err != nil { From a19e8219c5f1d58ba6e0cf7a1431a85fab179f9f Mon Sep 17 00:00:00 2001 From: Alex Suraci Date: Thu, 13 Apr 2023 22:04:00 -0400 Subject: [PATCH 8/9] fix bass foo.bass --- pkg/cli/run.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/cli/run.go b/pkg/cli/run.go index 932c541e..3de779f9 100644 --- a/pkg/cli/run.go +++ b/pkg/cli/run.go @@ -11,6 +11,9 @@ func Run(ctx context.Context, env *bass.Scope, inputs []string, filePath string, ctx, runs := bass.TrackRuns(ctx) dir, base := filepath.Split(filePath) + if dir == "" { + dir = "." + } cmd := bass.NewHostPath( dir, @@ -32,7 +35,7 @@ func Run(ctx context.Context, env *bass.Scope, inputs []string, filePath string, } err := bass.NewBass().Run(ctx, thunk, bass.RunState{ - Dir: bass.NewHostDir(filepath.Dir(filePath)), + Dir: bass.NewHostDir(dir), Stdin: stdin, Stdout: stdout, Env: thunk.Env, From 15a94df981321e5c88272ea27071636a7cbebb15 Mon Sep 17 00:00:00 2001 From: Alex Suraci Date: Thu, 13 Apr 2023 22:12:19 -0400 Subject: [PATCH 9/9] pass build args as *env* --- cmd/bass/frontend.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/cmd/bass/frontend.go b/cmd/bass/frontend.go index ecbf747f..5bd11e25 100644 --- a/cmd/bass/frontend.go +++ b/cmd/bass/frontend.go @@ -9,6 +9,7 @@ import ( "os" "path" "runtime" + "strings" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/exporter/containerimage/exptypes" @@ -34,6 +35,7 @@ func frontend(ctx context.Context) error { // mimic dockerfile.v1 frontend const ( + buildArgPrefix = "build-arg:" localNameContext = "context" localNameDockerfile = "dockerfile" localNameBassTLS = "bass-tls" @@ -64,6 +66,7 @@ func (fs *InputsFilesystem) Write(path string, r io.Reader) error { func frontendBuild(ctx context.Context, gw gwclient.Client) (*gwclient.Result, error) { caps := gw.BuildOpts().Caps + opts := gw.BuildOpts().Opts scriptFn := gw.BuildOpts().Opts[keyFilename] if scriptFn == "" { @@ -173,8 +176,14 @@ func frontendBuild(ctx context.Context, gw gwclient.Client) (*gwclient.Result, e ctx = bass.WithRuntimePool(ctx, pool) + args := filter(opts, buildArgPrefix) + env := bass.NewEmptyScope() + for k, v := range args { + env.Set(bass.Symbol(k), bass.String(v)) + } + runSt := bass.RunState{ - Env: bass.NewEmptyScope(), // TODO: build args + Env: env, // directly pass the context by local name, masquerading it as the host path Dir: bass.NewHostDir(localNameContext), Stdin: bass.Stdin, @@ -238,3 +247,13 @@ func frontendBuild(ctx context.Context, gw gwclient.Client) (*gwclient.Result, e return outRes, nil } + +func filter(opt map[string]string, key string) map[string]string { + m := map[string]string{} + for k, v := range opt { + if strings.HasPrefix(k, key) { + m[strings.TrimPrefix(k, key)] = v + } + } + return m +}