keep local dns in resolv.conf when host network enabled

executor/containerdexecutor/executor.go

@@ -118,7 +118,16 @@ func (w *containerdExecutor) Run(ctx context.Context, id string, root executor.M
 	}()
 
 	meta := process.Meta
-	resolvConf, hostsFile, releasers, err := w.prepareExecutionEnv(ctx, root, mounts, meta, details)
+	if meta.NetMode == pb.NetMode_HOST {
+		bklog.G(ctx).Info("enabling HostNetworking")
+	}
+
+	provider, ok := w.networkProviders[meta.NetMode]
+	if !ok {
+		return nil, errors.Errorf("unknown network mode %s", meta.NetMode)
+	}
+
+	resolvConf, hostsFile, releasers, err := w.prepareExecutionEnv(ctx, root, mounts, meta, details, meta.NetMode)
 	if err != nil {
 		return nil, err
 	}
@@ -131,20 +140,12 @@ func (w *containerdExecutor) Run(ctx context.Context, id string, root executor.M
 		return nil, err
 	}
 
-	provider, ok := w.networkProviders[meta.NetMode]
-	if !ok {
-		return nil, errors.Errorf("unknown network mode %s", meta.NetMode)
-	}
 	namespace, err := provider.New(ctx, meta.Hostname)
 	if err != nil {
 		return nil, err
 	}
 	defer namespace.Close()
 
-	if meta.NetMode == pb.NetMode_HOST {
-		bklog.G(ctx).Info("enabling HostNetworking")
-	}
-
 	spec, releaseSpec, err := w.createOCISpec(ctx, id, resolvConf, hostsFile, namespace, mounts, meta, details)
 	if err != nil {
 		return nil, err

executor/containerdexecutor/executor_unix.go

@@ -16,6 +16,7 @@ import (
 	"github.com/moby/buildkit/executor"
 	"github.com/moby/buildkit/executor/oci"
 	"github.com/moby/buildkit/snapshot"
+	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/bklog"
 	"github.com/moby/buildkit/util/network"
 	rootlessspecconv "github.com/moby/buildkit/util/rootless/specconv"
@@ -42,15 +43,15 @@ func getUserSpec(user, rootfsPath string) (specs.User, error) {
 	}, nil
 }
 
-func (w *containerdExecutor) prepareExecutionEnv(ctx context.Context, rootMount executor.Mount, mounts []executor.Mount, meta executor.Meta, details *containerState) (string, string, func(), error) {
+func (w *containerdExecutor) prepareExecutionEnv(ctx context.Context, rootMount executor.Mount, mounts []executor.Mount, meta executor.Meta, details *containerState, netMode pb.NetMode) (string, string, func(), error) {
 	var releasers []func()
 	releaseAll := func() {
 		for i := len(releasers) - 1; i >= 0; i-- {
 			releasers[i]()
 		}
 	}
 
-	resolvConf, err := oci.GetResolvConf(ctx, w.root, nil, w.dnsConfig)
+	resolvConf, err := oci.GetResolvConf(ctx, w.root, nil, w.dnsConfig, netMode)
 	if err != nil {
 		releaseAll()
 		return "", "", nil, err

executor/containerdexecutor/executor_windows.go

@@ -12,6 +12,7 @@ import (
 	"github.com/moby/buildkit/executor"
 	"github.com/moby/buildkit/executor/oci"
 	"github.com/moby/buildkit/snapshot"
+	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/network"
 	"github.com/moby/buildkit/util/windows"
 	"github.com/opencontainers/runtime-spec/specs-go"
@@ -24,7 +25,7 @@ func getUserSpec(user, rootfsPath string) (specs.User, error) {
 	}, nil
 }
 
-func (w *containerdExecutor) prepareExecutionEnv(ctx context.Context, rootMount executor.Mount, mounts []executor.Mount, meta executor.Meta, details *containerState) (string, string, func(), error) {
+func (w *containerdExecutor) prepareExecutionEnv(ctx context.Context, rootMount executor.Mount, mounts []executor.Mount, meta executor.Meta, details *containerState, netMode pb.NetMode) (string, string, func(), error) {
 	var releasers []func() error
 	releaseAll := func() {
 		for _, release := range releasers {

executor/oci/resolvconf.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/docker/docker/libnetwork/resolvconf"
 	"github.com/docker/docker/pkg/idtools"
+	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/flightcontrol"
 	"github.com/pkg/errors"
 )
@@ -24,9 +25,13 @@ type DNSConfig struct {
 	SearchDomains []string
 }
 
-func GetResolvConf(ctx context.Context, stateDir string, idmap *idtools.IdentityMapping, dns *DNSConfig) (string, error) {
+func GetResolvConf(ctx context.Context, stateDir string, idmap *idtools.IdentityMapping, dns *DNSConfig, netMode pb.NetMode) (string, error) {
 	p := filepath.Join(stateDir, "resolv.conf")
-	_, err := g.Do(ctx, stateDir, func(ctx context.Context) (struct{}, error) {
+	if netMode == pb.NetMode_HOST {
+		p = filepath.Join(stateDir, "resolv-host.conf")
+	}
+
+	_, err := g.Do(ctx, p, func(ctx context.Context) (struct{}, error) {
 		generate := !notFirstRun
 		notFirstRun = true
 
@@ -65,7 +70,6 @@ func GetResolvConf(ctx context.Context, stateDir string, idmap *idtools.Identity
 			return struct{}{}, err
 		}
 
-		var f *resolvconf.File
 		tmpPath := p + ".tmp"
 		if dns != nil {
 			var (
@@ -83,19 +87,22 @@ func GetResolvConf(ctx context.Context, stateDir string, idmap *idtools.Identity
 				dnsOptions = resolvconf.GetOptions(dt)
 			}
 
-			f, err = resolvconf.Build(tmpPath, dnsNameservers, dnsSearchDomains, dnsOptions)
+			f, err := resolvconf.Build(tmpPath, dnsNameservers, dnsSearchDomains, dnsOptions)
 			if err != nil {
 				return struct{}{}, err
 			}
 			dt = f.Content
 		}
 
-		f, err = resolvconf.FilterResolvDNS(dt, true)
-		if err != nil {
-			return struct{}{}, err
+		if netMode != pb.NetMode_HOST || len(resolvconf.GetNameservers(dt, resolvconf.IP)) == 0 {
+			f, err := resolvconf.FilterResolvDNS(dt, true)
+			if err != nil {
+				return struct{}{}, err
+			}
+			dt = f.Content
 		}
 
-		if err := os.WriteFile(tmpPath, f.Content, 0644); err != nil {
+		if err := os.WriteFile(tmpPath, dt, 0644); err != nil {
 			return struct{}{}, err
 		}
 

executor/oci/resolvconf_test.go

@@ -2,33 +2,134 @@ package oci
 
 import (
 	"context"
+	"fmt"
 	"os"
+	"path"
 	"testing"
+	"time"
 
+	"github.com/moby/buildkit/solver/pb"
 	"github.com/stretchr/testify/require"
 )
 
-// TestResolvConfNotExist modifies a global variable
-// It must not run in parallel.
-func TestResolvConfNotExist(t *testing.T) {
-	oldResolvconfPath := resolvconfPath
-	t.Cleanup(func() {
-		resolvconfPath = oldResolvconfPath
-	})
-	resolvconfPath = func() string {
-		return "no-such-file"
-	}
-
-	defaultResolvConf := `
+const defaultResolvConf = `
 nameserver 8.8.8.8
 nameserver 8.8.4.4
 nameserver 2001:4860:4860::8888
 nameserver 2001:4860:4860::8844`
 
-	ctx := context.Background()
-	p, err := GetResolvConf(ctx, t.TempDir(), nil, nil)
-	require.NoError(t, err)
-	b, err := os.ReadFile(p)
-	require.NoError(t, err)
-	require.Equal(t, string(b), defaultResolvConf)
+const dnsOption = `
+options ndots:0`
+
+const localDNSResolvConf = `
+nameserver 127.0.0.11
+options ndots:0`
+
+const regularResolvConf = `
+# DNS requests are forwarded to the host. DHCP DNS options are ignored.
+nameserver 192.168.65.5`
+
+// TestResolvConf modifies a global variable
+// It must not run in parallel.
+func TestResolvConf(t *testing.T) {
+	cases := []struct {
+		name        string
+		dt          []byte
+		execution   int
+		networkMode []pb.NetMode
+		expected    []string
+	}{
+		{
+			name:        "TestResolvConfNotExist",
+			dt:          nil,
+			execution:   1,
+			networkMode: []pb.NetMode{pb.NetMode_UNSET},
+			expected:    []string{defaultResolvConf},
+		},
+		{
+			name:        "TestNetModeIsHostResolvConfNotExist",
+			dt:          nil,
+			execution:   1,
+			networkMode: []pb.NetMode{pb.NetMode_HOST},
+			expected:    []string{defaultResolvConf},
+		},
+		{
+			name:        "TestNetModeIsHostWithoutLocalDNS",
+			dt:          []byte(regularResolvConf),
+			execution:   1,
+			networkMode: []pb.NetMode{pb.NetMode_HOST},
+			expected:    []string{regularResolvConf},
+		},
+		{
+			name:        "TestNetModeIsHostWithLocalDNS",
+			dt:          []byte(localDNSResolvConf),
+			execution:   1,
+			networkMode: []pb.NetMode{pb.NetMode_HOST},
+			expected:    []string{localDNSResolvConf},
+		},
+		{
+			name:        "TestNetModeNotHostWithoutLocalDNS",
+			dt:          []byte(regularResolvConf),
+			execution:   1,
+			networkMode: []pb.NetMode{pb.NetMode_UNSET},
+			expected:    []string{regularResolvConf},
+		},
+		{
+			name:        "TestNetModeNotHostWithLocalDNS",
+			dt:          []byte(localDNSResolvConf),
+			execution:   1,
+			networkMode: []pb.NetMode{pb.NetMode_UNSET},
+			expected:    []string{fmt.Sprintf("%s%s", dnsOption, defaultResolvConf)},
+		},
+		{
+			name:        "TestRegenerateResolvconfToRemoveLocalDNS",
+			dt:          []byte(localDNSResolvConf),
+			execution:   2,
+			networkMode: []pb.NetMode{pb.NetMode_HOST, pb.NetMode_UNSET},
+			expected: []string{
+				localDNSResolvConf,
+				fmt.Sprintf("%s%s", dnsOption, defaultResolvConf),
+			},
+		},
+		{
+			name:        "TestRegenerateResolvconfToAddLocalDNS",
+			dt:          []byte(localDNSResolvConf),
+			execution:   2,
+			networkMode: []pb.NetMode{pb.NetMode_UNSET, pb.NetMode_HOST},
+			expected: []string{
+				fmt.Sprintf("%s%s", dnsOption, defaultResolvConf),
+				localDNSResolvConf,
+			},
+		},
+	}
+
+	for _, tt := range cases {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := context.Background()
+			tempDir := t.TempDir()
+			oldResolvconfPath := resolvconfPath
+			t.Cleanup(func() {
+				resolvconfPath = oldResolvconfPath
+			})
+			resolvconfPath = func() string {
+				if tt.dt == nil {
+					return "no-such-file"
+				}
+				rpath := path.Join(t.TempDir(), "resolv.conf")
+				require.NoError(t, os.WriteFile(rpath, tt.dt, 0600))
+				return rpath
+			}
+			for i := 0; i < tt.execution; i++ {
+				if i > 0 {
+					time.Sleep(100 * time.Millisecond)
+				}
+				p, err := GetResolvConf(ctx, tempDir, nil, nil, tt.networkMode[i])
+				require.NoError(t, err)
+				b, err := os.ReadFile(p)
+				require.NoError(t, err)
+				require.Equal(t, tt.expected[i], string(b))
+			}
+		})
+	}
 }

executor/runcexecutor/executor.go

@@ -146,8 +146,6 @@ func New(opt Opt, networkProviders map[pb.NetMode]network.Provider) (executor.Ex
 }
 
 func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount, mounts []executor.Mount, process executor.ProcessInfo, started chan<- struct{}) (rec resourcestypes.Recorder, err error) {
-	meta := process.Meta
-
 	startedOnce := sync.Once{}
 	done := make(chan error, 1)
 	w.mu.Lock()
@@ -166,6 +164,11 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount,
 		}
 	}()
 
+	meta := process.Meta
+	if meta.NetMode == pb.NetMode_HOST {
+		bklog.G(ctx).Info("enabling HostNetworking")
+	}
+
 	provider, ok := w.networkProviders[meta.NetMode]
 	if !ok {
 		return nil, errors.Errorf("unknown network mode %s", meta.NetMode)
@@ -181,11 +184,7 @@ func (w *runcExecutor) Run(ctx context.Context, id string, root executor.Mount,
 		}
 	}()
 
-	if meta.NetMode == pb.NetMode_HOST {
-		bklog.G(ctx).Info("enabling HostNetworking")
-	}
-
-	resolvConf, err := oci.GetResolvConf(ctx, w.root, w.idmap, w.dns)
+	resolvConf, err := oci.GetResolvConf(ctx, w.root, w.idmap, w.dns, meta.NetMode)
 	if err != nil {
 		return nil, err
 	}