mount whitelist of devices on insecure security mode

executor/oci/spec_unix.go

@@ -18,7 +18,7 @@ import (
 	"github.com/moby/buildkit/executor"
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/solver/pb"
-	"github.com/moby/buildkit/util/entitlements"
+	"github.com/moby/buildkit/util/entitlements/security"
 	"github.com/moby/buildkit/util/network"
 	"github.com/moby/buildkit/util/system"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -38,7 +38,7 @@ func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mou
 		ctx = namespaces.WithNamespace(ctx, "buildkit")
 	}
 	if meta.SecurityMode == pb.SecurityMode_INSECURE {
-		opts = append(opts, entitlements.WithInsecureSpec())
+		opts = append(opts, security.WithInsecureSpec())
 	} else if system.SeccompSupported() && meta.SecurityMode == pb.SecurityMode_SANDBOX {
 		opts = append(opts, seccomp.WithDefaultProfile())
 	}

frontend/dockerfile/dockerfile_runsecurity_test.go

@@ -19,13 +19,76 @@ var runSecurityTests = []integration.Test{
 	testRunSecurityInsecure,
 	testRunSecuritySandbox,
 	testRunSecurityDefault,
+	testInsecureDevicesWhitelist,
 }
 
 func init() {
+	securityOpts = []integration.TestOpt{
+		integration.WithMirroredImages(integration.OfficialImages("alpine:latest")),
+		integration.WithMirroredImages(map[string]string{
+			"tonistiigi/hellofs:latest": "docker.io/tonistiigi/hellofs:latest",
+		}),
+	}
+
 	securityTests = append(securityTests, runSecurityTests...)
 
 }
 
+func testInsecureDevicesWhitelist(t *testing.T, sb integration.Sandbox) {
+	if sb.Rootless() {
+		t.SkipNow()
+	}
+
+	f := getFrontend(t, sb)
+
+	dockerfile := []byte(`
+FROM alpine
+RUN apk add --no-cache fuse e2fsprogs
+RUN [ ! -e /dev/fuse ] && [ ! -e /dev/loop-control ]
+# https://github.com/bazil/fuse/blob/master/examples/hellofs/hello.go#L91
+COPY --from=tonistiigi/hellofs /hellofs /bin/hellofs
+RUN --security=insecure [ -c /dev/fuse ] && [ -c /dev/loop-control ]
+RUN --security=insecure dmesg > /dev/null
+# testing fuse
+RUN --security=insecure hellofs /mnt & sleep 1 && ls -l /mnt && mount && cat /mnt/hello
+# testing loopbacks
+RUN --security=insecure ls -l /dev && dd if=/dev/zero of=disk.img bs=20M count=1 && \
+	mkfs.ext4 disk.img && \
+	mount -o loop disk.img /mnt && touch /mnt/foo \
+	umount /mnt && \
+	rm disk.img
+`)
+
+	dir, err := tmpdir(
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	c, err := client.New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	_, err = f.Solve(context.TODO(), c, client.SolveOpt{
+		LocalDirs: map[string]string{
+			builder.DefaultLocalNameDockerfile: dir,
+			builder.DefaultLocalNameContext:    dir,
+		},
+		AllowedEntitlements: []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure},
+	}, nil)
+
+	secMode := sb.Value("security.insecure")
+	switch secMode {
+	case securityInsecureGranted:
+		require.NoError(t, err)
+	case securityInsecureDenied:
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "entitlement security.insecure is not allowed")
+	default:
+		require.Fail(t, "unexpected secmode")
+	}
+}
+
 func testRunSecurityInsecure(t *testing.T, sb integration.Sandbox) {
 	f := getFrontend(t, sb)
 

frontend/dockerfile/dockerfile_test.go

@@ -126,6 +126,7 @@ var securityTests = []integration.Test{}
 var networkTests = []integration.Test{}
 
 var opts []integration.TestOpt
+var securityOpts []integration.TestOpt
 
 type frontend interface {
 	Solve(context.Context, *client.Client, client.SolveOpt, chan *client.SolveStatus) (*client.SolveResponse, error)
@@ -166,7 +167,7 @@ func TestIntegration(t *testing.T) {
 		"true":  true,
 		"false": false,
 	}))...)
-	integration.Run(t, securityTests, append(opts,
+	integration.Run(t, securityTests, append(append(opts, securityOpts...),
 		integration.WithMatrix("security.insecure", map[string]interface{}{
 			"granted": securityInsecureGranted,
 			"denied":  securityInsecureDenied,

solver/pb/caps.go

@@ -45,6 +45,8 @@ const (
 	CapExecMountSSH                  apicaps.CapID = "exec.mount.ssh"
 	CapExecCgroupsMounted            apicaps.CapID = "exec.cgroup"
 
+	CapExecMetaSecurityDeviceWhitelistV1 apicaps.CapID = "exec.meta.security.devices.v1"
+
 	CapFileBase       apicaps.CapID = "file.base"
 	CapFileRmWildcard apicaps.CapID = "file.rm.wildcard"
 
@@ -189,6 +191,12 @@ func init() {
 		Status:  apicaps.CapStatusExperimental,
 	})
 
+	Caps.Init(apicaps.Cap{
+		ID:      CapExecMetaSecurityDeviceWhitelistV1,
+		Enabled: true,
+		Status:  apicaps.CapStatusExperimental,
+	})
+
 	Caps.Init(apicaps.Cap{
 		ID:      CapExecMountBind,
 		Enabled: true,

util/entitlements/security/security_linux.go

@@ -0,0 +1,163 @@
+package security
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/oci"
+	"github.com/opencontainers/runc/libcontainer/system"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
+)
+
+// WithInsecureSpec sets spec with All capability.
+func WithInsecureSpec() oci.SpecOpts {
+	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
+		addCaps := []string{
+			"CAP_FSETID",
+			"CAP_KILL",
+			"CAP_FOWNER",
+			"CAP_MKNOD",
+			"CAP_CHOWN",
+			"CAP_DAC_OVERRIDE",
+			"CAP_NET_RAW",
+			"CAP_SETGID",
+			"CAP_SETUID",
+			"CAP_SETPCAP",
+			"CAP_SETFCAP",
+			"CAP_NET_BIND_SERVICE",
+			"CAP_SYS_CHROOT",
+			"CAP_AUDIT_WRITE",
+			"CAP_MAC_ADMIN",
+			"CAP_MAC_OVERRIDE",
+			"CAP_DAC_READ_SEARCH",
+			"CAP_SYS_PTRACE",
+			"CAP_SYS_MODULE",
+			"CAP_SYSLOG",
+			"CAP_SYS_RAWIO",
+			"CAP_SYS_ADMIN",
+			"CAP_LINUX_IMMUTABLE",
+			"CAP_SYS_BOOT",
+			"CAP_SYS_NICE",
+			"CAP_SYS_PACCT",
+			"CAP_SYS_TTY_CONFIG",
+			"CAP_SYS_TIME",
+			"CAP_WAKE_ALARM",
+			"CAP_AUDIT_READ",
+			"CAP_AUDIT_CONTROL",
+			"CAP_SYS_RESOURCE",
+			"CAP_BLOCK_SUSPEND",
+			"CAP_IPC_LOCK",
+			"CAP_IPC_OWNER",
+			"CAP_LEASE",
+			"CAP_NET_ADMIN",
+			"CAP_NET_BROADCAST",
+		}
+		for _, cap := range addCaps {
+			s.Process.Capabilities.Bounding = append(s.Process.Capabilities.Bounding, cap)
+			s.Process.Capabilities.Ambient = append(s.Process.Capabilities.Ambient, cap)
+			s.Process.Capabilities.Effective = append(s.Process.Capabilities.Effective, cap)
+			s.Process.Capabilities.Inheritable = append(s.Process.Capabilities.Inheritable, cap)
+			s.Process.Capabilities.Permitted = append(s.Process.Capabilities.Permitted, cap)
+		}
+		s.Linux.ReadonlyPaths = []string{}
+		s.Linux.MaskedPaths = []string{}
+		s.Process.ApparmorProfile = ""
+
+		s.Linux.Resources.Devices = []specs.LinuxDeviceCgroup{
+			{
+				Allow:  true,
+				Type:   "c",
+				Access: "rwm",
+			},
+			{
+				Allow:  true,
+				Type:   "b",
+				Access: "rwm",
+			},
+		}
+
+		if !system.RunningInUserNS() {
+			// Devices automatically mounted on insecure mode
+			s.Linux.Devices = append(s.Linux.Devices, []specs.LinuxDevice{
+				// Writes to this come out as printk's, reads export the buffered printk records. (dmesg)
+				{
+					Path:  "/dev/kmsg",
+					Type:  "c",
+					Major: 1,
+					Minor: 11,
+				},
+				// Cuse (character device in user-space)
+				{
+					Path:  "/dev/cuse",
+					Type:  "c",
+					Major: 10,
+					Minor: 203,
+				},
+				// Fuse (virtual filesystem in user-space)
+				{
+					Path:  "/dev/fuse",
+					Type:  "c",
+					Major: 10,
+					Minor: 229,
+				},
+				// Kernel-based virtual machine (hardware virtualization extensions)
+				{
+					Path:  "/dev/kvm",
+					Type:  "c",
+					Major: 10,
+					Minor: 232,
+				},
+				// TAP/TUN network device
+				{
+					Path:  "/dev/net/tun",
+					Type:  "c",
+					Major: 10,
+					Minor: 200,
+				},
+				// Loopback control device
+				{
+					Path:  "/dev/loop-control",
+					Type:  "c",
+					Major: 10,
+					Minor: 237,
+				},
+			}...)
+
+			loopID, err := getFreeLoopID()
+			if err != nil {
+				logrus.Debugf("failed to get next free loop device: %v", err)
+			}
+
+			for i := 0; i <= loopID+7; i++ {
+				s.Linux.Devices = append(s.Linux.Devices, specs.LinuxDevice{
+					Path:  fmt.Sprintf("/dev/loop%d", i),
+					Type:  "b",
+					Major: 7,
+					Minor: int64(i),
+				})
+			}
+		}
+
+		return nil
+	}
+}
+
+func getFreeLoopID() (int, error) {
+	fd, err := os.OpenFile("/dev/loop-control", os.O_RDWR, 0644)
+	if err != nil {
+		return 0, err
+	}
+	defer fd.Close()
+
+	const _LOOP_CTL_GET_FREE = 0x4C82
+	r1, _, uerr := unix.Syscall(unix.SYS_IOCTL, fd.Fd(), _LOOP_CTL_GET_FREE, 0)
+	if uerr == 0 {
+		return int(r1), nil
+	}
+	return 0, errors.Errorf("error getting free loop device: %v", uerr)
+}