Skip to content

Commit

Permalink
exec: use qemu emulator automatically
Browse files Browse the repository at this point in the history
Signed-off-by: Tonis Tiigi <[email protected]>
  • Loading branch information
tonistiigi committed Jun 8, 2020
1 parent 920772c commit f9e2612
Show file tree
Hide file tree
Showing 6 changed files with 132 additions and 35 deletions.
2 changes: 2 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,8 @@ RUN --mount=target=. --mount=target=/root/.cache,type=cache \

FROM scratch AS binaries-linux-helper
COPY --from=runc /usr/bin/runc /buildkit-runc
# built from https://github.com/tonistiigi/binfmt/tree/85394e2a1bf0ac9e6c291945e869322bea969445/binfmt
COPY --from=tonistiigi/binfmt:buildkit / /
FROM binaries-linux-helper AS binaries-linux
COPY --from=buildctl /usr/bin/buildctl /
COPY --from=buildkitd /usr/bin/buildkitd /
Expand Down
4 changes: 1 addition & 3 deletions solver/llbsolver/bridge.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ import (
"github.com/moby/buildkit/util/tracing"
"github.com/moby/buildkit/worker"
digest "github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
Expand All @@ -37,7 +36,6 @@ type llbBridge struct {
resolveCacheImporterFuncs map[string]remotecache.ResolveCacheImporterFunc
cms map[string]solver.CacheManager
cmsMu sync.Mutex
platforms []specs.Platform
sm *session.Manager
}

Expand Down Expand Up @@ -89,7 +87,7 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp
}
dpc := &detectPrunedCacheID{}

edge, err := Load(def, dpc.Load, ValidateEntitlements(ent), WithCacheSources(cms), RuntimePlatforms(b.platforms), WithValidateCaps())
edge, err := Load(def, dpc.Load, ValidateEntitlements(ent), WithCacheSources(cms), NormalizeRuntimePlatforms(), WithValidateCaps())
if err != nil {
return nil, errors.Wrap(err, "failed to load LLB")
}
Expand Down
14 changes: 14 additions & 0 deletions solver/llbsolver/ops/exec.go
Original file line number Diff line number Diff line change
Expand Up @@ -708,6 +708,20 @@ func (e *execOp) Exec(ctx context.Context, inputs []solver.Result) ([]solver.Res
return nil, err
}

emu, err := getEmulator(e.platform, e.cm.IdentityMapping())
if err == nil && emu != nil {
e.op.Meta.Args = append([]string{qemuMountName}, e.op.Meta.Args...)

mounts = append(mounts, executor.Mount{
Readonly: true,
Src: emu,
Dest: qemuMountName,
})
}
if err != nil {
logrus.Warn(err.Error()) // TODO: remove this with pull support
}

meta := executor.Meta{
Args: e.op.Meta.Args,
Env: e.op.Meta.Env,
Expand Down
114 changes: 114 additions & 0 deletions solver/llbsolver/ops/exec_binfmt.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
package ops

import (
"context"
"io/ioutil"
"os"
"os/exec"
"path/filepath"

"github.com/containerd/containerd/mount"
"github.com/containerd/containerd/platforms"
"github.com/docker/docker/pkg/idtools"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/binfmt_misc"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
copy "github.com/tonistiigi/fsutil/copy"
)

const qemuMountName = "/dev/.buildkit_qemu_emulator"

var qemuArchMap = map[string]string{
"arm64": "aarch64",
"amd64": "x86_64",
"riscv64": "riscv64",
"arm": "arm",
"s390x": "s390x",
"ppc64le": "ppc64le",
}

type emulator struct {
path string
idmap *idtools.IdentityMapping
}

func (e *emulator) Mount(ctx context.Context, readonly bool) (snapshot.Mountable, error) {
return &staticEmulatorMount{path: e.path, idmap: e.idmap}, nil
}

type staticEmulatorMount struct {
path string
idmap *idtools.IdentityMapping
}

func (m *staticEmulatorMount) Mount() ([]mount.Mount, func() error, error) {
tmpdir, err := ioutil.TempDir("", "buildkit-qemu-emulator")
if err != nil {
return nil, nil, err
}
var ret bool
defer func() {
if !ret {
os.RemoveAll(tmpdir)
}
}()

var uid, gid int
if m.idmap != nil {
root := m.idmap.RootPair()
uid = root.UID
gid = root.GID
}
if err := copy.Copy(context.TODO(), filepath.Dir(m.path), filepath.Base(m.path), tmpdir, qemuMountName, func(ci *copy.CopyInfo) {
m := 0555
ci.Mode = &m
}, copy.WithChown(uid, gid)); err != nil {
return nil, nil, err
}

ret = true
return []mount.Mount{{
Type: "bind",
Source: filepath.Join(tmpdir, qemuMountName),
Options: []string{"ro", "bind"},
}}, func() error {
return os.RemoveAll(tmpdir)
}, nil

}
func (m *staticEmulatorMount) IdentityMapping() *idtools.IdentityMapping {
return m.idmap
}

func getEmulator(p *pb.Platform, idmap *idtools.IdentityMapping) (*emulator, error) {
all := binfmt_misc.SupportedPlatforms(false)
m := make(map[string]struct{}, len(all))

for _, p := range all {
m[p] = struct{}{}
}

pp := platforms.Normalize(specs.Platform{
Architecture: p.Architecture,
OS: p.OS,
Variant: p.Variant,
})

if _, ok := m[platforms.Format(pp)]; ok {
return nil, nil
}

a, ok := qemuArchMap[pp.Architecture]
if !ok {
a = pp.Architecture
}

fn, err := exec.LookPath("buildkit-qemu-" + a)
if err != nil {
return nil, errors.Errorf("no emulator available for %v", pp.OS)
}

return &emulator{path: fn}, nil
}
10 changes: 0 additions & 10 deletions solver/llbsolver/solver.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/worker"
digest "github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"golang.org/x/sync/errgroup"
)
Expand All @@ -43,7 +42,6 @@ type Solver struct {
eachWorker func(func(worker.Worker) error) error
frontends map[string]frontend.Frontend
resolveCacheImporterFuncs map[string]remotecache.ResolveCacheImporterFunc
platforms []specs.Platform
gatewayForwarder *controlgateway.GatewayForwarder
sm *session.Manager
entitlements []string
Expand All @@ -61,13 +59,6 @@ func New(wc *worker.Controller, f map[string]frontend.Frontend, cache solver.Cac
entitlements: ents,
}

// executing is currently only allowed on default worker
w, err := wc.GetDefault()
if err != nil {
return nil, err
}
s.platforms = w.Platforms(false)

s.solver = solver.NewSolver(solver.SolverOpt{
ResolveOpFunc: s.resolver(),
DefaultCache: cache,
Expand All @@ -93,7 +84,6 @@ func (s *Solver) Bridge(b solver.Builder) frontend.FrontendLLBBridge {
eachWorker: s.eachWorker,
resolveCacheImporterFuncs: s.resolveCacheImporterFuncs,
cms: map[string]solver.CacheManager{},
platforms: s.platforms,
sm: s.sm,
}
}
Expand Down
23 changes: 1 addition & 22 deletions solver/llbsolver/vertex.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/source"
"github.com/moby/buildkit/util/binfmt_misc"
"github.com/moby/buildkit/util/entitlements"
digest "github.com/opencontainers/go-digest"
specs "github.com/opencontainers/image-spec/specs-go/v1"
Expand Down Expand Up @@ -69,12 +68,8 @@ func WithCacheSources(cms []solver.CacheManager) LoadOpt {
}
}

func RuntimePlatforms(p []specs.Platform) LoadOpt {
func NormalizeRuntimePlatforms() LoadOpt {
var defaultPlatform *pb.Platform
pp := make([]specs.Platform, len(p))
for i := range p {
pp[i] = platforms.Normalize(p[i])
}
return func(op *pb.Op, _ *pb.OpMetadata, opt *solver.VertexOptions) error {
if op.Platform == nil {
if defaultPlatform == nil {
Expand All @@ -96,22 +91,6 @@ func RuntimePlatforms(p []specs.Platform) LoadOpt {
Variant: normalizedPlatform.Variant,
}

if _, ok := op.Op.(*pb.Op_Exec); ok {
var found bool
for _, pp := range pp {
if pp.OS == op.Platform.OS && pp.Architecture == op.Platform.Architecture && pp.Variant == op.Platform.Variant {
found = true
break
}
}
if !found {
if !binfmt_misc.Check(normalizedPlatform) {
return errors.Errorf("runtime execution on platform %s not supported", platforms.Format(specs.Platform{OS: op.Platform.OS, Architecture: op.Platform.Architecture, Variant: op.Platform.Variant}))
} else {
pp = append(pp, normalizedPlatform)
}
}
}
return nil
}
}
Expand Down

0 comments on commit f9e2612

Please sign in to comment.