diff --git a/client/build_test.go b/client/build_test.go index 10e746109432..c37d81a83429 100644 --- a/client/build_test.go +++ b/client/build_test.go @@ -1838,7 +1838,7 @@ func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox command := []string{"sh", "-c", `cat /proc/self/status | grep CapEff | cut -f 2`} mode := llb.SecurityModeSandbox - var allowedEntitlements []entitlements.Entitlement + var allowedEntitlements []string var assertCaps func(caps uint64) secMode := sb.Value("secmode") if secMode == securitySandbox { @@ -1850,7 +1850,7 @@ func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox */ require.Equal(t, uint64(0xa80425fb), caps) } - allowedEntitlements = []entitlements.Entitlement{} + allowedEntitlements = []string{} if expectFail { return } @@ -1869,9 +1869,9 @@ func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox require.Equal(t, uint64(0x3fffffffff), caps&0x3fffffffff) } mode = llb.SecurityModeInsecure - allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure} + allowedEntitlements = []string{entitlements.EntitlementSecurityInsecure.String()} if expectFail { - allowedEntitlements = []entitlements.Entitlement{} + allowedEntitlements = []string{} } } @@ -2046,13 +2046,13 @@ func testClientGatewayContainerHostNetworking(t *testing.T, sb integration.Sandb ctx := sb.Context() product := "buildkit_test" - var allowedEntitlements []entitlements.Entitlement + var allowedEntitlements []string netMode := pb.NetMode_UNSET if sb.Value("netmode") == hostNetwork { netMode = pb.NetMode_HOST - allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementNetworkHost} + allowedEntitlements = []string{entitlements.EntitlementNetworkHost.String()} if expectFail { - allowedEntitlements = []entitlements.Entitlement{} + allowedEntitlements = []string{} } } c, err := New(sb.Context(), sb.Address()) diff --git a/client/client_test.go b/client/client_test.go index a8bfa093a9ea..f266b5bac2ac 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -278,6 +278,8 @@ func testIntegration(t *testing.T, funcs ...func(t *testing.T, sb integration.Sa integration.Run(t, integration.TestFuncs( testCDI, + testCDINotAllowed, + testCDIEntitlement, testCDIFirst, testCDIWildcard, testCDIClass, @@ -400,9 +402,9 @@ func testHostNetworking(t *testing.T, sb integration.Sandbox) { t.SkipNow() } netMode := sb.Value("netmode") - var allowedEntitlements []entitlements.Entitlement + var allowedEntitlements []string if netMode == hostNetwork { - allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementNetworkHost} + allowedEntitlements = []string{entitlements.EntitlementNetworkHost.String()} } c, err := New(sb.Context(), sb.Address()) require.NoError(t, err) @@ -1063,7 +1065,7 @@ func testSecurityMode(t *testing.T, sb integration.Sandbox) { workers.CheckFeatureCompat(t, sb, workers.FeatureSecurityMode) command := `sh -c 'cat /proc/self/status | grep CapEff | cut -f 2 > /out'` mode := llb.SecurityModeSandbox - var allowedEntitlements []entitlements.Entitlement + var allowedEntitlements []string var assertCaps func(caps uint64) secMode := sb.Value("secmode") if secMode == securitySandbox { @@ -1075,7 +1077,7 @@ func testSecurityMode(t *testing.T, sb integration.Sandbox) { */ require.Equal(t, uint64(0xa80425fb), caps) } - allowedEntitlements = []entitlements.Entitlement{} + allowedEntitlements = []string{} } else { assertCaps = func(caps uint64) { /* @@ -1091,7 +1093,7 @@ func testSecurityMode(t *testing.T, sb integration.Sandbox) { require.Equal(t, uint64(0x3fffffffff), caps&0x3fffffffff) } mode = llb.SecurityModeInsecure - allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure} + allowedEntitlements = []string{entitlements.EntitlementSecurityInsecure.String()} } c, err := New(sb.Context(), sb.Address()) @@ -1138,13 +1140,13 @@ func testSecurityModeSysfs(t *testing.T, sb integration.Sandbox) { } mode := llb.SecurityModeSandbox - var allowedEntitlements []entitlements.Entitlement + var allowedEntitlements []string secMode := sb.Value("secmode") if secMode == securitySandbox { - allowedEntitlements = []entitlements.Entitlement{} + allowedEntitlements = []string{} } else { mode = llb.SecurityModeInsecure - allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure} + allowedEntitlements = []string{entitlements.EntitlementSecurityInsecure.String()} } c, err := New(sb.Context(), sb.Address()) @@ -1191,7 +1193,7 @@ func testSecurityModeErrors(t *testing.T, sb integration.Sandbox) { require.NoError(t, err) _, err = c.Solve(sb.Context(), def, SolveOpt{ - AllowedEntitlements: []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure}, + AllowedEntitlements: []string{entitlements.EntitlementSecurityInsecure.String()}, }, nil) require.Error(t, err) require.Contains(t, err.Error(), "security.insecure is not allowed") @@ -11054,22 +11056,26 @@ func testCDI(t *testing.T, sb integration.Sandbox) { defer c.Close() require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor1-device.yaml"), []byte(` -cdiVersion: "0.3.0" +cdiVersion: "0.6.0" kind: "vendor1.com/device" devices: - name: foo containerEdits: env: - FOO=injected +annotations: + org.mobyproject.buildkit.device.autoallow: true `), 0600)) require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor2-device.yaml"), []byte(` -cdiVersion: "0.3.0" +cdiVersion: "0.6.0" kind: "vendor2.com/device" devices: - name: bar containerEdits: env: - BAR=injected +annotations: + org.mobyproject.buildkit.device.autoallow: true `), 0600)) busybox := llb.Image("busybox:latest") @@ -11107,6 +11113,104 @@ devices: require.Contains(t, strings.TrimSpace(string(dt2)), `BAR=injected`) } +func testCDINotAllowed(t *testing.T, sb integration.Sandbox) { + if sb.Rootless() { + t.SkipNow() + } + + integration.SkipOnPlatform(t, "windows") + workers.CheckFeatureCompat(t, sb, workers.FeatureCDI) + c, err := New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor1-device.yaml"), []byte(` +cdiVersion: "0.6.0" +kind: "vendor1.com/device" +devices: +- name: foo + containerEdits: + env: + - FOO=injected +`), 0600)) + + busybox := llb.Image("busybox:latest") + st := llb.Scratch() + + run := func(cmd string, ro ...llb.RunOption) { + st = busybox.Run(append(ro, llb.Shlex(cmd), llb.Dir("/wd"))...).AddMount("/wd", st) + } + + run(`sh -c 'env|sort | tee foo.env'`, llb.AddCDIDevice(llb.CDIDeviceName("vendor1.com/device=foo"))) + + def, err := st.Marshal(sb.Context()) + require.NoError(t, err) + + destDir := t.TempDir() + + _, err = c.Solve(sb.Context(), def, SolveOpt{ + Exports: []ExportEntry{ + { + Type: ExporterLocal, + OutputDir: destDir, + }, + }, + }, nil) + require.Error(t, err) + require.ErrorContains(t, err, "requested by the build but not allowed") +} + +func testCDIEntitlement(t *testing.T, sb integration.Sandbox) { + if sb.Rootless() { + t.SkipNow() + } + + integration.SkipOnPlatform(t, "windows") + workers.CheckFeatureCompat(t, sb, workers.FeatureCDI) + c, err := New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor1-device.yaml"), []byte(` +cdiVersion: "0.6.0" +kind: "vendor1.com/device" +devices: +- name: foo + containerEdits: + env: + - FOO=injected +`), 0600)) + + busybox := llb.Image("busybox:latest") + st := llb.Scratch() + + run := func(cmd string, ro ...llb.RunOption) { + st = busybox.Run(append(ro, llb.Shlex(cmd), llb.Dir("/wd"))...).AddMount("/wd", st) + } + + run(`sh -c 'env|sort | tee foo.env'`, llb.AddCDIDevice(llb.CDIDeviceName("vendor1.com/device=foo"))) + + def, err := st.Marshal(sb.Context()) + require.NoError(t, err) + + destDir := t.TempDir() + + _, err = c.Solve(sb.Context(), def, SolveOpt{ + AllowedEntitlements: []string{"device=vendor1.com/device"}, + Exports: []ExportEntry{ + { + Type: ExporterLocal, + OutputDir: destDir, + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := os.ReadFile(filepath.Join(destDir, "foo.env")) + require.NoError(t, err) + require.Contains(t, strings.TrimSpace(string(dt)), `FOO=injected`) +} + func testCDIFirst(t *testing.T, sb integration.Sandbox) { if sb.Rootless() { t.SkipNow() @@ -11119,7 +11223,7 @@ func testCDIFirst(t *testing.T, sb integration.Sandbox) { defer c.Close() require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor1-device.yaml"), []byte(` -cdiVersion: "0.3.0" +cdiVersion: "0.6.0" kind: "vendor1.com/device" devices: - name: foo @@ -11138,6 +11242,8 @@ devices: containerEdits: env: - QUX=injected +annotations: + org.mobyproject.buildkit.device.autoallow: true `), 0600)) busybox := llb.Image("busybox:latest") @@ -11184,7 +11290,7 @@ func testCDIWildcard(t *testing.T, sb integration.Sandbox) { defer c.Close() require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor1-device.yaml"), []byte(` -cdiVersion: "0.3.0" +cdiVersion: "0.6.0" kind: "vendor1.com/device" devices: - name: foo @@ -11195,6 +11301,8 @@ devices: containerEdits: env: - BAR=injected +annotations: + org.mobyproject.buildkit.device.autoallow: true `), 0600)) busybox := llb.Image("busybox:latest") @@ -11243,6 +11351,7 @@ cdiVersion: "0.6.0" kind: "vendor1.com/device" annotations: foo.bar.baz: FOO + org.mobyproject.buildkit.device.autoallow: true devices: - name: foo annotations: diff --git a/client/solve.go b/client/solve.go index efdf9fa9f105..57ee82d05669 100644 --- a/client/solve.go +++ b/client/solve.go @@ -7,6 +7,7 @@ import ( "io" "maps" "os" + "slices" "strings" "time" @@ -24,7 +25,6 @@ import ( "github.com/moby/buildkit/solver/pb" spb "github.com/moby/buildkit/sourcepolicy/pb" "github.com/moby/buildkit/util/bklog" - "github.com/moby/buildkit/util/entitlements" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/tonistiigi/fsutil" @@ -45,7 +45,7 @@ type SolveOpt struct { CacheExports []CacheOptionsEntry CacheImports []CacheOptionsEntry Session []session.Attachable - AllowedEntitlements []entitlements.Entitlement + AllowedEntitlements []string SharedSession *session.Session // TODO: refactor to better session syncing SessionPreInitialized bool // TODO: refactor to better session syncing Internal bool @@ -277,7 +277,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG FrontendAttrs: frontendAttrs, FrontendInputs: frontendInputs, Cache: &cacheOpt.options, - Entitlements: entitlementsToPB(opt.AllowedEntitlements), + Entitlements: slices.Clone(opt.AllowedEntitlements), Internal: opt.Internal, SourcePolicy: opt.SourcePolicy, }) @@ -553,11 +553,3 @@ func prepareMounts(opt *SolveOpt) (map[string]fsutil.FS, error) { } return mounts, nil } - -func entitlementsToPB(entitlements []entitlements.Entitlement) []string { - clone := make([]string, len(entitlements)) - for i, e := range entitlements { - clone[i] = string(e) - } - return clone -} diff --git a/cmd/buildctl/build.go b/cmd/buildctl/build.go index fd6a4bf0d8b7..bc95f01cf380 100644 --- a/cmd/buildctl/build.go +++ b/cmd/buildctl/build.go @@ -213,8 +213,7 @@ func buildAction(clicontext *cli.Context) error { attachable = append(attachable, secretProvider) } - allowed, err := build.ParseAllow(clicontext.StringSlice("allow")) - if err != nil { + if err := build.ValidateAllow(clicontext.StringSlice("allow")); err != nil { return err } @@ -258,7 +257,7 @@ func buildAction(clicontext *cli.Context) error { CacheExports: cacheExports, CacheImports: cacheImports, Session: attachable, - AllowedEntitlements: allowed, + AllowedEntitlements: clicontext.StringSlice("allow"), SourcePolicy: srcPol, Ref: ref, } diff --git a/cmd/buildctl/build/allow.go b/cmd/buildctl/build/allow.go index fe43e5676dad..70c66b0efb9a 100644 --- a/cmd/buildctl/build/allow.go +++ b/cmd/buildctl/build/allow.go @@ -4,15 +4,13 @@ import ( "github.com/moby/buildkit/util/entitlements" ) -// ParseAllow parses --allow -func ParseAllow(inp []string) ([]entitlements.Entitlement, error) { - ent := make([]entitlements.Entitlement, 0, len(inp)) +// ValidateAllow parses --allow +func ValidateAllow(inp []string) error { for _, v := range inp { - e, err := entitlements.Parse(v) + _, _, err := entitlements.Parse(v) if err != nil { - return nil, err + return err } - ent = append(ent, e) } - return ent, nil + return nil } diff --git a/cmd/buildctl/debug/workers.go b/cmd/buildctl/debug/workers.go index 58fc6ad10d1a..6045d873ebca 100644 --- a/cmd/buildctl/debug/workers.go +++ b/cmd/buildctl/debug/workers.go @@ -94,7 +94,7 @@ func printWorkersVerbose(tw *tabwriter.Writer, winfo []*client.WorkerInfo) { for _, k := range sortedKeys(d.Annotations) { v := d.Annotations[k] - fmt.Fprintf(tw, "\t\t%s:\t%s\n", k, v) + fmt.Fprintf(tw, "\tAnnotations:\t%s:\t%s\n", k, v) } } fmt.Fprint(tw, "\n") diff --git a/cmd/buildkitd/config/config.go b/cmd/buildkitd/config/config.go index 381effcdc9b7..3222406a42da 100644 --- a/cmd/buildkitd/config/config.go +++ b/cmd/buildkitd/config/config.go @@ -77,8 +77,9 @@ type OTELConfig struct { } type CDIConfig struct { - Disabled *bool `toml:"disabled"` - SpecDirs []string `toml:"specDirs"` + Disabled *bool `toml:"disabled"` + SpecDirs []string `toml:"specDirs"` + AutoAllowed []string `toml:"autoAllowed"` } type GCConfig struct { diff --git a/cmd/buildkitd/main.go b/cmd/buildkitd/main.go index 77d48620f7a8..364252a59b34 100644 --- a/cmd/buildkitd/main.go +++ b/cmd/buildkitd/main.go @@ -39,6 +39,7 @@ import ( "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/bboltcachestorage" + "github.com/moby/buildkit/solver/llbsolver/cdidevices" "github.com/moby/buildkit/util/apicaps" "github.com/moby/buildkit/util/appcontext" "github.com/moby/buildkit/util/appdefaults" @@ -846,6 +847,11 @@ func newController(ctx context.Context, c *cli.Context, cfg *config.Config) (*co "s3": s3remotecache.ResolveCacheImporterFunc(), "azblob": azblob.ResolveCacheImporterFunc(), } + + if cfg.CDI.Disabled == nil || !*cfg.CDI.Disabled { + cfg.Entitlements = append(cfg.Entitlements, "device") + } + return control.NewController(control.Opt{ SessionManager: sessionManager, WorkerController: wc, @@ -1046,19 +1052,15 @@ func newMeterProvider(ctx context.Context) (*sdkmetric.MeterProvider, error) { return sdkmetric.NewMeterProvider(opts...), nil } -// getCDIManager returns a new CDI registry with disabled auto-refresh. -func getCDIManager(disabled *bool, specDirs []string) (*cdi.Cache, error) { - if disabled != nil && *disabled { +func getCDIManager(cfg config.CDIConfig) (*cdidevices.Manager, error) { + if cfg.Disabled != nil && *cfg.Disabled { return nil, nil } - if len(specDirs) == 0 { - return nil, errors.New("No CDI specification directories specified") + if len(cfg.SpecDirs) == 0 { + return nil, errors.New("no CDI specification directories specified") } cdiCache, err := func() (*cdi.Cache, error) { - cdiCache, err := cdi.NewCache( - cdi.WithSpecDirs(specDirs...), - cdi.WithAutoRefresh(false), - ) + cdiCache, err := cdi.NewCache(cdi.WithSpecDirs(cfg.SpecDirs...)) if err != nil { return nil, err } @@ -1070,5 +1072,5 @@ func getCDIManager(disabled *bool, specDirs []string) (*cdi.Cache, error) { if err != nil { return nil, errors.Wrapf(err, "CDI registry initialization failure") } - return cdiCache, nil + return cdidevices.NewManager(cdiCache, cfg.AutoAllowed), nil } diff --git a/cmd/buildkitd/main_containerd_worker.go b/cmd/buildkitd/main_containerd_worker.go index 13694a48d5bc..38f0844bb505 100644 --- a/cmd/buildkitd/main_containerd_worker.go +++ b/cmd/buildkitd/main_containerd_worker.go @@ -12,7 +12,6 @@ import ( ctd "github.com/containerd/containerd/v2/client" "github.com/containerd/containerd/v2/defaults" "github.com/moby/buildkit/cmd/buildkitd/config" - "github.com/moby/buildkit/solver/llbsolver/cdidevices" "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/disk" "github.com/moby/buildkit/util/network/cniprovider" @@ -283,7 +282,7 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([ dns := getDNSConfig(common.config.DNS) - cdiManager, err := getCDIManager(common.config.CDI.Disabled, common.config.CDI.SpecDirs) + cdiManager, err := getCDIManager(common.config.CDI) if err != nil { return nil, err } @@ -345,7 +344,7 @@ func containerdWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([ ParallelismSem: parallelismSem, TraceSocket: common.traceSocket, Runtime: runtime, - CDIManager: cdidevices.NewManager(cdiManager), + CDIManager: cdiManager, } opt, err := containerd.NewWorkerOpt(workerOpts, ctd.WithTimeout(60*time.Second)) diff --git a/cmd/buildkitd/main_oci_worker.go b/cmd/buildkitd/main_oci_worker.go index ee1ed9eb962f..ec2f36e47916 100644 --- a/cmd/buildkitd/main_oci_worker.go +++ b/cmd/buildkitd/main_oci_worker.go @@ -298,7 +298,7 @@ func ociWorkerInitializer(c *cli.Context, common workerInitializerOpt) ([]worker dns := getDNSConfig(common.config.DNS) - cdiManager, err := getCDIManager(common.config.CDI.Disabled, common.config.CDI.SpecDirs) + cdiManager, err := getCDIManager(common.config.CDI) if err != nil { return nil, err } diff --git a/control/control.go b/control/control.go index ca50913190b9..a54425168d6e 100644 --- a/control/control.go +++ b/control/control.go @@ -695,7 +695,7 @@ func toPBCDIDevices(manager *cdidevices.Manager) []*apitypes.CDIDevice { for _, dev := range devs { out = append(out, &apitypes.CDIDevice{ Name: dev.Name, - AutoAllow: true, // TODO + AutoAllow: dev.AutoAllow, Annotations: dev.Annotations, OnDemand: dev.OnDemand, }) diff --git a/frontend/dockerfile/dockerfile_rundevice_test.go b/frontend/dockerfile/dockerfile_rundevice_test.go index 4975fb08fbdc..dfc4cdf0035c 100644 --- a/frontend/dockerfile/dockerfile_rundevice_test.go +++ b/frontend/dockerfile/dockerfile_rundevice_test.go @@ -30,13 +30,15 @@ func testDeviceRunEnv(t *testing.T, sb integration.Sandbox) { f := getFrontend(t, sb) require.NoError(t, os.WriteFile(filepath.Join(sb.CDISpecDir(), "vendor1-device.yaml"), []byte(` -cdiVersion: "0.3.0" +cdiVersion: "0.6.0" kind: "vendor1.com/device" devices: - name: foo containerEdits: env: - FOO=injected + annotations: + org.mobyproject.buildkit.device.autoallow: true `), 0600)) dockerfile := []byte(` diff --git a/frontend/dockerfile/dockerfile_runnetwork_test.go b/frontend/dockerfile/dockerfile_runnetwork_test.go index a2f360cd3f04..e0ba322d0b0a 100644 --- a/frontend/dockerfile/dockerfile_runnetwork_test.go +++ b/frontend/dockerfile/dockerfile_runnetwork_test.go @@ -132,7 +132,7 @@ RUN --network=host nc 127.0.0.1 %s | grep foo dockerui.DefaultLocalNameDockerfile: dir, dockerui.DefaultLocalNameContext: dir, }, - AllowedEntitlements: []entitlements.Entitlement{entitlements.EntitlementNetworkHost}, + AllowedEntitlements: []string{entitlements.EntitlementNetworkHost.String()}, }, nil) hostAllowed := sb.Value("network.host") @@ -180,7 +180,7 @@ RUN --network=none ! nc -z 127.0.0.1 %s dockerui.DefaultLocalNameDockerfile: dir, dockerui.DefaultLocalNameContext: dir, }, - AllowedEntitlements: []entitlements.Entitlement{entitlements.EntitlementNetworkHost}, + AllowedEntitlements: []string{entitlements.EntitlementNetworkHost.String()}, FrontendAttrs: map[string]string{ "force-network-mode": "host", }, diff --git a/frontend/dockerfile/dockerfile_runsecurity_test.go b/frontend/dockerfile/dockerfile_runsecurity_test.go index 2b1cdbe03130..73203910c96a 100644 --- a/frontend/dockerfile/dockerfile_runsecurity_test.go +++ b/frontend/dockerfile/dockerfile_runsecurity_test.go @@ -71,7 +71,7 @@ RUN --security=insecure ls -l /dev && dd if=/dev/zero of=disk.img bs=20M count=1 dockerui.DefaultLocalNameDockerfile: dir, dockerui.DefaultLocalNameContext: dir, }, - AllowedEntitlements: []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure}, + AllowedEntitlements: []string{entitlements.EntitlementSecurityInsecure.String()}, }, nil) secMode := sb.Value("security.insecure") @@ -109,7 +109,7 @@ RUN [ "$(cat /proc/self/status | grep CapBnd)" == "CapBnd: 00000000a80425fb" ] dockerui.DefaultLocalNameDockerfile: dir, dockerui.DefaultLocalNameContext: dir, }, - AllowedEntitlements: []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure}, + AllowedEntitlements: []string{entitlements.EntitlementSecurityInsecure.String()}, }, nil) secMode := sb.Value("security.insecure") @@ -173,7 +173,7 @@ RUN [ "$(cat /proc/self/status | grep CapBnd)" == "CapBnd: 00000000a80425fb" ] dockerui.DefaultLocalNameDockerfile: dir, dockerui.DefaultLocalNameContext: dir, }, - AllowedEntitlements: []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure}, + AllowedEntitlements: []string{entitlements.EntitlementSecurityInsecure.String()}, }, nil) secMode := sb.Value("security.insecure") diff --git a/solver/llbsolver/bridge.go b/solver/llbsolver/bridge.go index 18b20ba0c785..b3466c65066f 100644 --- a/solver/llbsolver/bridge.go +++ b/solver/llbsolver/bridge.go @@ -138,7 +138,7 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp } dpc := &detectPrunedCacheID{} - edge, err := Load(ctx, def, polEngine, dpc.Load, ValidateEntitlements(ent), WithCacheSources(cms), NormalizeRuntimePlatforms(), WithValidateCaps()) + edge, err := Load(ctx, def, polEngine, dpc.Load, ValidateEntitlements(ent, w.CDIManager()), WithCacheSources(cms), NormalizeRuntimePlatforms(), WithValidateCaps()) if err != nil { return nil, errors.Wrap(err, "failed to load LLB") } diff --git a/solver/llbsolver/cdidevices/fixtures/vendor1-device.yaml b/solver/llbsolver/cdidevices/fixtures/vendor1-device.yaml new file mode 100644 index 000000000000..bf76973204c8 --- /dev/null +++ b/solver/llbsolver/cdidevices/fixtures/vendor1-device.yaml @@ -0,0 +1,9 @@ +cdiVersion: "0.6.0" +kind: "vendor1.com/device" +devices: + - name: foo + containerEdits: + env: + - FOO=injected +annotations: + org.mobyproject.buildkit.device.autoallow: true diff --git a/solver/llbsolver/cdidevices/fixtures/vendor1-deviceclass.yaml b/solver/llbsolver/cdidevices/fixtures/vendor1-deviceclass.yaml new file mode 100644 index 000000000000..2b2ee4b8cf45 --- /dev/null +++ b/solver/llbsolver/cdidevices/fixtures/vendor1-deviceclass.yaml @@ -0,0 +1,28 @@ +cdiVersion: "0.6.0" +kind: "vendor1.com/deviceclass" +annotations: + foo.bar.baz: FOO + org.mobyproject.buildkit.device.autoallow: true +devices: + - name: foo + annotations: + org.mobyproject.buildkit.device.class: class1 + containerEdits: + env: + - FOO=injected + - name: bar + annotations: + org.mobyproject.buildkit.device.class: class1 + containerEdits: + env: + - BAR=injected + - name: baz + annotations: + org.mobyproject.buildkit.device.class: class2 + containerEdits: + env: + - BAZ=injected + - name: qux + containerEdits: + env: + - QUX=injected diff --git a/solver/llbsolver/cdidevices/fixtures/vendor1-devicemulti.yaml b/solver/llbsolver/cdidevices/fixtures/vendor1-devicemulti.yaml new file mode 100644 index 000000000000..f6940569ddcb --- /dev/null +++ b/solver/llbsolver/cdidevices/fixtures/vendor1-devicemulti.yaml @@ -0,0 +1,21 @@ +cdiVersion: "0.6.0" +kind: "vendor1.com/devicemulti" +devices: + - name: foo + containerEdits: + env: + - FOO=injected + - name: bar + containerEdits: + env: + - BAR=injected + - name: baz + containerEdits: + env: + - BAZ=injected + - name: qux + containerEdits: + env: + - QUX=injected +annotations: + org.mobyproject.buildkit.device.autoallow: true diff --git a/solver/llbsolver/cdidevices/manager.go b/solver/llbsolver/cdidevices/manager.go index 1996a41c8f7f..673c823f25c8 100644 --- a/solver/llbsolver/cdidevices/manager.go +++ b/solver/llbsolver/cdidevices/manager.go @@ -2,6 +2,7 @@ package cdidevices import ( "context" + "strconv" "strings" "github.com/moby/buildkit/solver/pb" @@ -13,7 +14,10 @@ import ( "tags.cncf.io/container-device-interface/pkg/parser" ) -const deviceAnnotationClass = "org.mobyproject.buildkit.device.class" +const ( + deviceAnnotationClass = "org.mobyproject.buildkit.device.class" + deviceAnnotationAutoAllow = "org.mobyproject.buildkit.device.autoallow" +) var installers = map[string]Setup{} @@ -35,17 +39,38 @@ type Device struct { } type Manager struct { - cache *cdi.Cache - locker *locker.Locker + cache *cdi.Cache + locker *locker.Locker + autoAllowed map[string]struct{} } -func NewManager(cache *cdi.Cache) *Manager { +func NewManager(cache *cdi.Cache, autoAllowed []string) *Manager { + m := make(map[string]struct{}) + for _, d := range autoAllowed { + m[d] = struct{}{} + } return &Manager{ - cache: cache, - locker: locker.New(), + cache: cache, + locker: locker.New(), + autoAllowed: m, } } +func (m *Manager) isAutoAllowed(kind, name string, annotations map[string]string) bool { + if _, ok := m.autoAllowed[name]; ok { + return true + } + if _, ok := m.autoAllowed[kind]; ok { + return true + } + if v, ok := annotations[deviceAnnotationAutoAllow]; ok { + if b, err := strconv.ParseBool(v); err == nil && b { + return true + } + } + return false +} + func (m *Manager) ListDevices() []Device { devs := m.cache.ListDevices() out := make([]Device, 0, len(devs)) @@ -53,10 +78,11 @@ func (m *Manager) ListDevices() []Device { for _, dev := range devs { kind, _, _ := strings.Cut(dev, "=") dd := m.cache.GetDevice(dev) + annotations := deviceAnnotations(dd) out = append(out, Device{ Name: dev, - AutoAllow: true, // TODO - Annotations: deviceAnnotations(dd), + AutoAllow: m.isAutoAllowed(kind, dev, annotations), + Annotations: annotations, }) kinds[kind] = struct{}{} } @@ -69,20 +95,31 @@ func (m *Manager) ListDevices() []Device { continue } out = append(out, Device{ - Name: k, - OnDemand: true, + Name: k, + OnDemand: true, + AutoAllow: true, }) } return out } +func (m *Manager) GetDevice(name string) Device { + kind, _, _ := strings.Cut(name, "=") + annotations := deviceAnnotations(m.cache.GetDevice(name)) + return Device{ + Name: name, + AutoAllow: m.isAutoAllowed(kind, name, annotations), + Annotations: annotations, + } +} + func (m *Manager) Refresh() error { return m.cache.Refresh() } func (m *Manager) InjectDevices(spec *specs.Spec, devs ...*pb.CDIDevice) error { - pdevs, err := m.parseDevices(devs...) + pdevs, err := m.FindDevices(devs...) if err != nil { return err } else if len(pdevs) == 0 { @@ -93,13 +130,17 @@ func (m *Manager) InjectDevices(spec *specs.Spec, devs ...*pb.CDIDevice) error { return err } -func (m *Manager) parseDevices(devs ...*pb.CDIDevice) ([]string, error) { +func (m *Manager) FindDevices(devs ...*pb.CDIDevice) ([]string, error) { var out []string + if len(devs) == 0 { + return out, nil + } + list := m.cache.ListDevices() for _, dev := range devs { if dev == nil { continue } - pdev, err := m.parseDevice(dev) + pdev, err := m.parseDevice(dev, list) if err != nil { return nil, err } else if len(pdev) == 0 { @@ -110,7 +151,7 @@ func (m *Manager) parseDevices(devs ...*pb.CDIDevice) ([]string, error) { return dedupSlice(out), nil } -func (m *Manager) parseDevice(dev *pb.CDIDevice) ([]string, error) { +func (m *Manager) parseDevice(dev *pb.CDIDevice, all []string) ([]string, error) { var out []string kind, name, _ := strings.Cut(dev.Name, "=") @@ -127,7 +168,7 @@ func (m *Manager) parseDevice(dev *pb.CDIDevice) ([]string, error) { switch name { case "": // first device of kind if no name is specified - for _, d := range m.cache.ListDevices() { + for _, d := range all { if strings.HasPrefix(d, kind+"=") { out = append(out, d) break @@ -135,14 +176,14 @@ func (m *Manager) parseDevice(dev *pb.CDIDevice) ([]string, error) { } case "*": // all devices of kind if the name is a wildcard - for _, d := range m.cache.ListDevices() { + for _, d := range all { if strings.HasPrefix(d, kind+"=") { out = append(out, d) } } default: // the specified device - for _, d := range m.cache.ListDevices() { + for _, d := range all { if d == dev.Name { out = append(out, d) break @@ -150,7 +191,7 @@ func (m *Manager) parseDevice(dev *pb.CDIDevice) ([]string, error) { } if len(out) == 0 { // check class annotation if name unknown - for _, d := range m.cache.ListDevices() { + for _, d := range all { if !strings.HasPrefix(d, kind+"=") { continue } @@ -214,6 +255,9 @@ func (m *Manager) OnDemandInstaller(name string) (func(context.Context) error, b return errors.Wrapf(err, "failed to refresh CDI cache") } + // TODO: this needs to be set as annotation to survive reboot + m.autoAllowed[name] = struct{}{} + return nil }, true } diff --git a/solver/llbsolver/cdidevices/manager_test.go b/solver/llbsolver/cdidevices/manager_test.go new file mode 100644 index 000000000000..057a17811bcb --- /dev/null +++ b/solver/llbsolver/cdidevices/manager_test.go @@ -0,0 +1,70 @@ +package cdidevices + +import ( + "testing" + + "github.com/moby/buildkit/solver/pb" + "github.com/stretchr/testify/require" + "tags.cncf.io/container-device-interface/pkg/cdi" +) + +func TestFindDevices(t *testing.T) { + tests := []struct { + name string + devices []*pb.CDIDevice + expected []string + err bool + }{ + { + name: "Find specific device", + devices: []*pb.CDIDevice{ + {Name: "vendor1.com/device=foo"}, + }, + expected: []string{"vendor1.com/device=foo"}, + }, + { + name: "Find first devices", + devices: []*pb.CDIDevice{ + {Name: "vendor1.com/devicemulti"}, + }, + expected: []string{"vendor1.com/devicemulti=bar"}, + }, + { + name: "Find all devices of a kind", + devices: []*pb.CDIDevice{ + {Name: "vendor1.com/devicemulti=*"}, + }, + expected: []string{"vendor1.com/devicemulti=foo", "vendor1.com/devicemulti=bar", "vendor1.com/devicemulti=baz", "vendor1.com/devicemulti=qux"}, + }, + { + name: "Find devices by class", + devices: []*pb.CDIDevice{ + {Name: "vendor1.com/deviceclass=class1"}, + }, + expected: []string{"vendor1.com/deviceclass=foo", "vendor1.com/deviceclass=bar"}, + }, + { + name: "Device not found", + devices: []*pb.CDIDevice{ + {Name: "vendor1.com/device=unknown"}, + }, + expected: nil, + err: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cache, err := cdi.NewCache(cdi.WithSpecDirs("./fixtures")) + require.NoError(t, err) + manager := NewManager(cache, nil) + result, err := manager.FindDevices(tt.devices...) + if tt.err { + require.Error(t, err) + } else { + require.NoError(t, err) + require.ElementsMatch(t, tt.expected, result) + } + }) + } +} diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 15a1f0911372..2ac9070a1fcf 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -1110,19 +1110,26 @@ func supportedEntitlements(ents []string) []entitlements.Entitlement { if e == string(entitlements.EntitlementSecurityInsecure) { out = append(out, entitlements.EntitlementSecurityInsecure) } + if e == string(entitlements.EntitlementDevice) { + out = append(out, entitlements.EntitlementDevice) + } } return out } func loadEntitlements(b solver.Builder) (entitlements.Set, error) { - var ent entitlements.Set = map[entitlements.Entitlement]struct{}{} + var ent entitlements.Set = map[entitlements.Entitlement]entitlements.EntitlementsConfig{} err := b.EachValue(context.TODO(), keyEntitlements, func(v interface{}) error { set, ok := v.(entitlements.Set) if !ok { return errors.Errorf("invalid entitlements %T", v) } - for k := range set { - ent[k] = struct{}{} + for k, v := range set { + if prev, ok := ent[k]; ok && prev != nil { + prev.Merge(v) + } else { + ent[k] = v + } } return nil }) diff --git a/solver/llbsolver/vertex.go b/solver/llbsolver/vertex.go index 21ae0f9f22b8..c61f3b9ea719 100644 --- a/solver/llbsolver/vertex.go +++ b/solver/llbsolver/vertex.go @@ -7,6 +7,7 @@ import ( "github.com/containerd/platforms" "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/solver/llbsolver/cdidevices" "github.com/moby/buildkit/solver/llbsolver/ops/opsutils" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/apicaps" @@ -109,7 +110,7 @@ func NormalizeRuntimePlatforms() LoadOpt { } } -func ValidateEntitlements(ent entitlements.Set) LoadOpt { +func ValidateEntitlements(ent entitlements.Set, cdiManager *cdidevices.Manager) LoadOpt { return func(op *pb.Op, _ *pb.OpMetadata, opt *solver.VertexOptions) error { switch op := op.Op.(type) { case *pb.Op_Exec: @@ -120,6 +121,75 @@ func ValidateEntitlements(ent entitlements.Set) LoadOpt { if err := ent.Check(v); err != nil { return err } + if device := op.Exec.CdiDevices; len(device) > 0 { + var cfg *entitlements.DevicesConfig + if ent, ok := ent[entitlements.EntitlementDevice]; ok { + cfg, ok = ent.(*entitlements.DevicesConfig) + if !ok { + return errors.Errorf("invalid device entitlement config %T", ent) + } + } + if cfg != nil && cfg.All { + return nil + } + + var allowedDevices []*pb.CDIDevice + var nonAliasedDevices []*pb.CDIDevice + if cfg != nil { + for _, d := range op.Exec.CdiDevices { + if newName, ok := cfg.Devices[d.Name]; ok && newName != "" { + d.Name = newName + allowedDevices = append(allowedDevices, d) + } else { + nonAliasedDevices = append(nonAliasedDevices, d) + } + } + } else { + nonAliasedDevices = op.Exec.CdiDevices + } + + mountedDevices, err := cdiManager.FindDevices(nonAliasedDevices...) + if err != nil { + return err + } + if len(mountedDevices) == 0 { + op.Exec.CdiDevices = allowedDevices + return nil + } + + grantedDevices := make(map[string]struct{}) + if cfg != nil { + for d := range cfg.Devices { + resolved, err := cdiManager.FindDevices(&pb.CDIDevice{Name: d}) + if err != nil { + return err + } + for _, r := range resolved { + grantedDevices[r] = struct{}{} + } + } + } + + var forbidden []string + for _, d := range mountedDevices { + if _, ok := grantedDevices[d]; !ok { + if dev := cdiManager.GetDevice(d); !dev.AutoAllow { + forbidden = append(forbidden, d) + continue + } + } + allowedDevices = append(allowedDevices, &pb.CDIDevice{Name: d}) + } + + if len(forbidden) > 0 { + if len(forbidden) == 1 { + return errors.Errorf("device %s is requested by the build but not allowed", forbidden[0]) + } + return errors.Errorf("devices %s are requested by the build but not allowed", strings.Join(forbidden, ", ")) + } + + op.Exec.CdiDevices = allowedDevices + } } return nil } diff --git a/util/entitlements/entitlements.go b/util/entitlements/entitlements.go index 328580c326df..106f492ceee2 100644 --- a/util/entitlements/entitlements.go +++ b/util/entitlements/entitlements.go @@ -1,31 +1,119 @@ package entitlements import ( + "strings" + "github.com/pkg/errors" + "github.com/tonistiigi/go-csvvalue" ) type Entitlement string +func (e Entitlement) String() string { + return string(e) +} + const ( EntitlementSecurityInsecure Entitlement = "security.insecure" EntitlementNetworkHost Entitlement = "network.host" + EntitlementDevice Entitlement = "device" ) var all = map[Entitlement]struct{}{ EntitlementSecurityInsecure: {}, EntitlementNetworkHost: {}, + EntitlementDevice: {}, +} + +type EntitlementsConfig interface { + Merge(EntitlementsConfig) error } -func Parse(s string) (Entitlement, error) { +type DevicesConfig struct { + Devices map[string]string + All bool +} + +var _ EntitlementsConfig = &DevicesConfig{} + +func ParseDevicesConfig(s string) (*DevicesConfig, error) { + if s == "" { + return &DevicesConfig{All: true}, nil + } + + fields, err := csvvalue.Fields(s, nil) + if err != nil { + return nil, err + } + deviceName := fields[0] + var deviceAlias string + + for _, field := range fields[1:] { + k, v, ok := strings.Cut(field, "=") + if !ok { + return nil, errors.Errorf("invalid device config %q", field) + } + switch k { + case "alias": + deviceAlias = v + default: + return nil, errors.Errorf("unknown device config key %q", k) + } + } + + cfg := &DevicesConfig{Devices: map[string]string{}} + + if deviceAlias != "" { + cfg.Devices[deviceAlias] = deviceName + } else { + cfg.Devices[deviceName] = "" + } + return cfg, nil +} + +func (c *DevicesConfig) Merge(in EntitlementsConfig) error { + c2, ok := in.(*DevicesConfig) + if !ok { + return errors.Errorf("cannot merge %T into %T", in, c) + } + + if c2.All { + c.All = true + return nil + } + + for k, v := range c2.Devices { + if c.Devices == nil { + c.Devices = map[string]string{} + } + c.Devices[k] = v + } + return nil +} + +func Parse(s string) (Entitlement, EntitlementsConfig, error) { + var cfg EntitlementsConfig + key, rest, _ := strings.Cut(s, "=") + switch Entitlement(key) { + case EntitlementDevice: + s = key + var err error + cfg, err = ParseDevicesConfig(rest) + if err != nil { + return "", nil, err + } + default: + } + _, ok := all[Entitlement(s)] if !ok { - return "", errors.Errorf("unknown entitlement %s", s) + return "", nil, errors.Errorf("unknown entitlement %s", s) } - return Entitlement(s), nil + return Entitlement(s), cfg, nil } func WhiteList(allowed, supported []Entitlement) (Set, error) { - m := map[Entitlement]struct{}{} + m := map[Entitlement]EntitlementsConfig{} var supm Set if supported != nil { @@ -37,7 +125,7 @@ func WhiteList(allowed, supported []Entitlement) (Set, error) { } for _, e := range allowed { - e, err := Parse(string(e)) + e, cfg, err := Parse(string(e)) if err != nil { return nil, err } @@ -46,13 +134,19 @@ func WhiteList(allowed, supported []Entitlement) (Set, error) { return nil, errors.Errorf("granting entitlement %s is not allowed by build daemon configuration", e) } } - m[e] = struct{}{} + if prev, ok := m[e]; ok && prev != nil { + if err := prev.Merge(cfg); err != nil { + return nil, err + } + } else { + m[e] = cfg + } } return Set(m), nil } -type Set map[Entitlement]struct{} +type Set map[Entitlement]EntitlementsConfig func (s Set) Allowed(e Entitlement) bool { _, ok := s[e] @@ -77,4 +171,5 @@ func (s Set) Check(v Values) error { type Values struct { NetworkHost bool SecurityInsecure bool + Devices map[string]struct{} } diff --git a/worker/runc/runc.go b/worker/runc/runc.go index 0ea36169bdd2..1626530d5e7f 100644 --- a/worker/runc/runc.go +++ b/worker/runc/runc.go @@ -30,7 +30,6 @@ import ( ocispecs "github.com/opencontainers/image-spec/specs-go/v1" bolt "go.etcd.io/bbolt" "golang.org/x/sync/semaphore" - "tags.cncf.io/container-device-interface/pkg/cdi" ) // SnapshotterFactory instantiates a snapshotter @@ -40,7 +39,7 @@ type SnapshotterFactory struct { } // NewWorkerOpt creates a WorkerOpt. -func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, processMode oci.ProcessMode, labels map[string]string, idmap *idtools.IdentityMapping, nopt netproviders.Opt, dns *oci.DNSConfig, binary, apparmorProfile string, selinux bool, parallelismSem *semaphore.Weighted, traceSocket, defaultCgroupParent string, cdiManager *cdi.Cache) (base.WorkerOpt, error) { +func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, processMode oci.ProcessMode, labels map[string]string, idmap *idtools.IdentityMapping, nopt netproviders.Opt, dns *oci.DNSConfig, binary, apparmorProfile string, selinux bool, parallelismSem *semaphore.Weighted, traceSocket, defaultCgroupParent string, cdiManager *cdidevices.Manager) (base.WorkerOpt, error) { var opt base.WorkerOpt name := "runc-" + snFactory.Name root = filepath.Join(root, name) @@ -80,7 +79,7 @@ func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, proc TracingSocket: traceSocket, DefaultCgroupParent: defaultCgroupParent, ResourceMonitor: rm, - CDIManager: cdidevices.NewManager(cdiManager), + CDIManager: cdiManager, }, np) if err != nil { return opt, err @@ -169,7 +168,7 @@ func NewWorkerOpt(root string, snFactory SnapshotterFactory, rootless bool, proc ParallelismSem: parallelismSem, MountPoolRoot: filepath.Join(root, "cachemounts"), ResourceMonitor: rm, - CDIManager: cdidevices.NewManager(cdiManager), + CDIManager: cdiManager, } return opt, nil }