From fc186a8b89a17e5c02065212efd332e1a52ef182 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 30 Jan 2020 11:28:13 -0800 Subject: [PATCH 1/3] oci: mount whitelist of devices on insecure security mode Signed-off-by: Tonis Tiigi --- executor/oci/spec_unix.go | 4 +- solver/pb/caps.go | 8 +++ .../{ => security}/security_linux.go | 71 ++++++++++++++++++- 3 files changed, 80 insertions(+), 3 deletions(-) rename util/entitlements/{ => security}/security_linux.go (56%) diff --git a/executor/oci/spec_unix.go b/executor/oci/spec_unix.go index 5fe8d09e3734..8ab4fb47077d 100644 --- a/executor/oci/spec_unix.go +++ b/executor/oci/spec_unix.go @@ -18,7 +18,7 @@ import ( "github.com/moby/buildkit/executor" "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/solver/pb" - "github.com/moby/buildkit/util/entitlements" + "github.com/moby/buildkit/util/entitlements/security" "github.com/moby/buildkit/util/network" "github.com/moby/buildkit/util/system" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -38,7 +38,7 @@ func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mou ctx = namespaces.WithNamespace(ctx, "buildkit") } if meta.SecurityMode == pb.SecurityMode_INSECURE { - opts = append(opts, entitlements.WithInsecureSpec()) + opts = append(opts, security.WithInsecureSpec()) } else if system.SeccompSupported() && meta.SecurityMode == pb.SecurityMode_SANDBOX { opts = append(opts, seccomp.WithDefaultProfile()) } diff --git a/solver/pb/caps.go b/solver/pb/caps.go index 4a21d25cc16d..93c77b3e9add 100644 --- a/solver/pb/caps.go +++ b/solver/pb/caps.go @@ -45,6 +45,8 @@ const ( CapExecMountSSH apicaps.CapID = "exec.mount.ssh" CapExecCgroupsMounted apicaps.CapID = "exec.cgroup" + CapExecMetaSecurityDeviceWhitelistV1 apicaps.CapID = "exec.meta.security.devices.v1" + CapFileBase apicaps.CapID = "file.base" CapFileRmWildcard apicaps.CapID = "file.rm.wildcard" @@ -189,6 +191,12 @@ func init() { Status: apicaps.CapStatusExperimental, }) + Caps.Init(apicaps.Cap{ + ID: CapExecMetaSecurityDeviceWhitelistV1, + Enabled: true, + Status: apicaps.CapStatusExperimental, + }) + Caps.Init(apicaps.Cap{ ID: CapExecMountBind, Enabled: true, diff --git a/util/entitlements/security_linux.go b/util/entitlements/security/security_linux.go similarity index 56% rename from util/entitlements/security_linux.go rename to util/entitlements/security/security_linux.go index c4cfc6c6de89..23e742ef7929 100644 --- a/util/entitlements/security_linux.go +++ b/util/entitlements/security/security_linux.go @@ -1,7 +1,8 @@ -package entitlements +package security import ( "context" + "fmt" "github.com/containerd/containerd/containers" "github.com/containerd/containerd/oci" @@ -62,6 +63,74 @@ func WithInsecureSpec() oci.SpecOpts { s.Linux.MaskedPaths = []string{} s.Process.ApparmorProfile = "" + s.Linux.Resources.Devices = []specs.LinuxDeviceCgroup{ + { + Allow: true, + Type: "c", + Access: "rwm", + }, + { + Allow: false, + Type: "b", + Access: "rwm", + }, + } + + // Devices automatically mounted on insecure mode + s.Linux.Devices = append(s.Linux.Devices, []specs.LinuxDevice{ + // Writes to this come out as printk's, reads export the buffered printk records. (dmesg) + { + Path: "/dev/kmsg", + Type: "c", + Major: 1, + Minor: 11, + }, + // Cuse (character device in user-space) + { + Path: "/dev/cuse", + Type: "c", + Major: 10, + Minor: 203, + }, + // Fuse (virtual filesystem in user-space) + { + Path: "/dev/fuse", + Type: "c", + Major: 10, + Minor: 229, + }, + // Kernel-based virtual machine (hardware virtualization extensions) + { + Path: "/dev/kvm", + Type: "c", + Major: 10, + Minor: 232, + }, + // TAP/TUN network device + { + Path: "/dev/net/tun", + Type: "c", + Major: 10, + Minor: 200, + }, + // Loopback control device + { + Path: "/dev/loop-control", + Type: "c", + Major: 10, + Minor: 237, + }, + }...) + + for i := 0; i <= 7; i++ { + s.Linux.Devices = append(s.Linux.Devices, specs.LinuxDevice{ + Path: fmt.Sprintf("/dev/loop%d", i), + Type: "b", + Major: 7, + Minor: int64(i), + }) + } + return nil } } From 572a2b57189ae3ea9de56da8e6e78338ed048c0d Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 30 Jan 2020 13:47:44 -0800 Subject: [PATCH 2/3] entitlements: mount loop devices relative to next free device Signed-off-by: Tonis Tiigi --- util/entitlements/security/security_linux.go | 26 +++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/util/entitlements/security/security_linux.go b/util/entitlements/security/security_linux.go index 23e742ef7929..05b3ff31437e 100644 --- a/util/entitlements/security/security_linux.go +++ b/util/entitlements/security/security_linux.go @@ -3,10 +3,14 @@ package security import ( "context" "fmt" + "os" "github.com/containerd/containerd/containers" "github.com/containerd/containerd/oci" specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) // WithInsecureSpec sets spec with All capability. @@ -122,7 +126,12 @@ func WithInsecureSpec() oci.SpecOpts { }, }...) - for i := 0; i <= 7; i++ { + loopID, err := getFreeLoopID() + if err != nil { + logrus.Debugf("failed to get next free loop device: %v", err) + } + + for i := 0; i <= loopID+7; i++ { s.Linux.Devices = append(s.Linux.Devices, specs.LinuxDevice{ Path: fmt.Sprintf("/dev/loop%d", i), Type: "b", @@ -134,3 +143,18 @@ func WithInsecureSpec() oci.SpecOpts { return nil } } + +func getFreeLoopID() (int, error) { + fd, err := os.OpenFile("/dev/loop-control", os.O_RDWR, 0644) + if err != nil { + return 0, err + } + defer fd.Close() + + const _LOOP_CTL_GET_FREE = 0x4C82 + r1, _, uerr := unix.Syscall(unix.SYS_IOCTL, fd.Fd(), _LOOP_CTL_GET_FREE, 0) + if uerr == 0 { + return int(r1), nil + } + return 0, errors.Errorf("error getting free loop device: %v", uerr) +} From 8f52339933dd5dfd680b686882985d1339165441 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 30 Jan 2020 15:34:09 -0800 Subject: [PATCH 3/3] dockerfile: add test for whitelisted devices Signed-off-by: Tonis Tiigi --- .../dockerfile/dockerfile_runsecurity_test.go | 63 ++++++++++ frontend/dockerfile/dockerfile_test.go | 3 +- util/entitlements/security/security_linux.go | 117 +++++++++--------- 3 files changed, 125 insertions(+), 58 deletions(-) diff --git a/frontend/dockerfile/dockerfile_runsecurity_test.go b/frontend/dockerfile/dockerfile_runsecurity_test.go index 32bf3c14ba13..9804ecf1c99a 100644 --- a/frontend/dockerfile/dockerfile_runsecurity_test.go +++ b/frontend/dockerfile/dockerfile_runsecurity_test.go @@ -19,13 +19,76 @@ var runSecurityTests = []integration.Test{ testRunSecurityInsecure, testRunSecuritySandbox, testRunSecurityDefault, + testInsecureDevicesWhitelist, } func init() { + securityOpts = []integration.TestOpt{ + integration.WithMirroredImages(integration.OfficialImages("alpine:latest")), + integration.WithMirroredImages(map[string]string{ + "tonistiigi/hellofs:latest": "docker.io/tonistiigi/hellofs:latest", + }), + } + securityTests = append(securityTests, runSecurityTests...) } +func testInsecureDevicesWhitelist(t *testing.T, sb integration.Sandbox) { + if sb.Rootless() { + t.SkipNow() + } + + f := getFrontend(t, sb) + + dockerfile := []byte(` +FROM alpine +RUN apk add --no-cache fuse e2fsprogs +RUN [ ! -e /dev/fuse ] && [ ! -e /dev/loop-control ] +# https://github.com/bazil/fuse/blob/master/examples/hellofs/hello.go#L91 +COPY --from=tonistiigi/hellofs /hellofs /bin/hellofs +RUN --security=insecure [ -c /dev/fuse ] && [ -c /dev/loop-control ] +RUN --security=insecure dmesg > /dev/null +# testing fuse +RUN --security=insecure hellofs /mnt & sleep 1 && ls -l /mnt && mount && cat /mnt/hello +# testing loopbacks +RUN --security=insecure ls -l /dev && dd if=/dev/zero of=disk.img bs=20M count=1 && \ + mkfs.ext4 disk.img && \ + mount -o loop disk.img /mnt && touch /mnt/foo \ + umount /mnt && \ + rm disk.img +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + c, err := client.New(context.TODO(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + _, err = f.Solve(context.TODO(), c, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + AllowedEntitlements: []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure}, + }, nil) + + secMode := sb.Value("security.insecure") + switch secMode { + case securityInsecureGranted: + require.NoError(t, err) + case securityInsecureDenied: + require.Error(t, err) + require.Contains(t, err.Error(), "entitlement security.insecure is not allowed") + default: + require.Fail(t, "unexpected secmode") + } +} + func testRunSecurityInsecure(t *testing.T, sb integration.Sandbox) { f := getFrontend(t, sb) diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index efe7049331a2..afe1ff4f22f8 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -126,6 +126,7 @@ var securityTests = []integration.Test{} var networkTests = []integration.Test{} var opts []integration.TestOpt +var securityOpts []integration.TestOpt type frontend interface { Solve(context.Context, *client.Client, client.SolveOpt, chan *client.SolveStatus) (*client.SolveResponse, error) @@ -166,7 +167,7 @@ func TestIntegration(t *testing.T) { "true": true, "false": false, }))...) - integration.Run(t, securityTests, append(opts, + integration.Run(t, securityTests, append(append(opts, securityOpts...), integration.WithMatrix("security.insecure", map[string]interface{}{ "granted": securityInsecureGranted, "denied": securityInsecureDenied, diff --git a/util/entitlements/security/security_linux.go b/util/entitlements/security/security_linux.go index 05b3ff31437e..2b9b151247f1 100644 --- a/util/entitlements/security/security_linux.go +++ b/util/entitlements/security/security_linux.go @@ -7,6 +7,7 @@ import ( "github.com/containerd/containerd/containers" "github.com/containerd/containerd/oci" + "github.com/opencontainers/runc/libcontainer/system" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -74,70 +75,72 @@ func WithInsecureSpec() oci.SpecOpts { Access: "rwm", }, { - Allow: false, + Allow: true, Type: "b", Access: "rwm", }, } - // Devices automatically mounted on insecure mode - s.Linux.Devices = append(s.Linux.Devices, []specs.LinuxDevice{ - // Writes to this come out as printk's, reads export the buffered printk records. (dmesg) - { - Path: "/dev/kmsg", - Type: "c", - Major: 1, - Minor: 11, - }, - // Cuse (character device in user-space) - { - Path: "/dev/cuse", - Type: "c", - Major: 10, - Minor: 203, - }, - // Fuse (virtual filesystem in user-space) - { - Path: "/dev/fuse", - Type: "c", - Major: 10, - Minor: 229, - }, - // Kernel-based virtual machine (hardware virtualization extensions) - { - Path: "/dev/kvm", - Type: "c", - Major: 10, - Minor: 232, - }, - // TAP/TUN network device - { - Path: "/dev/net/tun", - Type: "c", - Major: 10, - Minor: 200, - }, - // Loopback control device - { - Path: "/dev/loop-control", - Type: "c", - Major: 10, - Minor: 237, - }, - }...) + if !system.RunningInUserNS() { + // Devices automatically mounted on insecure mode + s.Linux.Devices = append(s.Linux.Devices, []specs.LinuxDevice{ + // Writes to this come out as printk's, reads export the buffered printk records. (dmesg) + { + Path: "/dev/kmsg", + Type: "c", + Major: 1, + Minor: 11, + }, + // Cuse (character device in user-space) + { + Path: "/dev/cuse", + Type: "c", + Major: 10, + Minor: 203, + }, + // Fuse (virtual filesystem in user-space) + { + Path: "/dev/fuse", + Type: "c", + Major: 10, + Minor: 229, + }, + // Kernel-based virtual machine (hardware virtualization extensions) + { + Path: "/dev/kvm", + Type: "c", + Major: 10, + Minor: 232, + }, + // TAP/TUN network device + { + Path: "/dev/net/tun", + Type: "c", + Major: 10, + Minor: 200, + }, + // Loopback control device + { + Path: "/dev/loop-control", + Type: "c", + Major: 10, + Minor: 237, + }, + }...) - loopID, err := getFreeLoopID() - if err != nil { - logrus.Debugf("failed to get next free loop device: %v", err) - } + loopID, err := getFreeLoopID() + if err != nil { + logrus.Debugf("failed to get next free loop device: %v", err) + } - for i := 0; i <= loopID+7; i++ { - s.Linux.Devices = append(s.Linux.Devices, specs.LinuxDevice{ - Path: fmt.Sprintf("/dev/loop%d", i), - Type: "b", - Major: 7, - Minor: int64(i), - }) + for i := 0; i <= loopID+7; i++ { + s.Linux.Devices = append(s.Linux.Devices, specs.LinuxDevice{ + Path: fmt.Sprintf("/dev/loop%d", i), + Type: "b", + Major: 7, + Minor: int64(i), + }) + } } return nil