windows integration tests: plumbing work to be able to run on windows

Co-authored-by: Gabriel Samfira <[email protected]>

This PR does most of the plumbing work requred for us to start
running integration tests on Windows. We will need this to land
before we can do the follow-up PRs to handle the specific tests.

Signed-off-by: Anthony Nandaa <[email protected]>

cmd/buildkitd/main_containerd_worker.go

@@ -355,6 +355,7 @@ func validContainerdSocket(cfg config.ContainerdConfig) bool {
 		return true
 	}
 	socketPath := strings.TrimPrefix(socket, "unix://")
+	socketPath = strings.TrimPrefix(socketPath, "npipe://")
 	if _, err := os.Stat(socketPath); errors.Is(err, os.ErrNotExist) {
 		// FIXME(AkihiroSuda): add more conditions
 		bklog.L.Warnf("skipping containerd worker, as %q does not exist", socketPath)

util/testutil/integration/sandbox.go

@@ -99,7 +99,7 @@ func newSandbox(ctx context.Context, w Worker, mirror string, mv matrixValue) (s
 
 	b, closer, err := w.New(ctx, cfg)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errors.Wrap(err, "creating worker")
 	}
 	deferF.Append(closer)
 

util/testutil/integration/util.go

@@ -5,10 +5,8 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"net"
 	"os"
 	"os/exec"
-	"strings"
 	"sync"
 	"syscall"
 	"testing"
@@ -100,31 +98,8 @@ func StartCmd(cmd *exec.Cmd, logs map[string]*bytes.Buffer) (func() error, error
 	}, nil
 }
 
-func WaitUnix(address string, d time.Duration, cmd *exec.Cmd) error {
-	address = strings.TrimPrefix(address, "unix://")
-	addr, err := net.ResolveUnixAddr("unix", address)
-	if err != nil {
-		return errors.Wrapf(err, "failed resolving unix addr: %s", address)
-	}
-
-	step := 50 * time.Millisecond
-	i := 0
-	for {
-		if cmd != nil && cmd.ProcessState != nil {
-			return errors.Errorf("process exited: %s", cmd.String())
-		}
-
-		if conn, err := net.DialUnix("unix", nil, addr); err == nil {
-			conn.Close()
-			break
-		}
-		i++
-		if time.Duration(i)*step > d {
-			return errors.Errorf("failed dialing: %s", address)
-		}
-		time.Sleep(step)
-	}
-	return nil
+func WaitSocket(address string, d time.Duration, cmd *exec.Cmd) error {
+	return waitSocket(address, d, cmd)
 }
 
 func LookupBinary(name string) error {

util/testutil/integration/util_unix.go

@@ -0,0 +1,40 @@
+//go:build !windows
+// +build !windows
+
+package integration
+
+import (
+	"net"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+func waitSocket(address string, d time.Duration, cmd *exec.Cmd) error {
+	address = strings.TrimPrefix(address, "unix://")
+	addr, err := net.ResolveUnixAddr("unix", address)
+	if err != nil {
+		return errors.Wrapf(err, "failed resolving unix addr: %s", address)
+	}
+
+	step := 50 * time.Millisecond
+	i := 0
+	for {
+		if cmd != nil && cmd.ProcessState != nil {
+			return errors.Errorf("process exited: %s", cmd.String())
+		}
+
+		if conn, err := net.DialUnix("unix", nil, addr); err == nil {
+			conn.Close()
+			break
+		}
+		i++
+		if time.Duration(i)*step > d {
+			return errors.Errorf("failed dialing: %s", address)
+		}
+		time.Sleep(step)
+	}
+	return nil
+}

util/testutil/integration/util_windows.go

@@ -0,0 +1,33 @@
+package integration
+
+import (
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/Microsoft/go-winio"
+	"github.com/pkg/errors"
+)
+
+func waitSocket(address string, d time.Duration, cmd *exec.Cmd) error {
+	address = strings.TrimPrefix(address, "npipe://")
+	step := 50 * time.Millisecond
+	i := 0
+
+	for {
+		if cmd != nil && cmd.ProcessState != nil {
+			return errors.Errorf("process exited: %s", cmd.String())
+		}
+
+		if conn, err := winio.DialPipe(address, nil); err == nil {
+			conn.Close()
+			break
+		}
+		i++
+		if time.Duration(i)*step > d {
+			return errors.Errorf("failed dialing: %s", address)
+		}
+		time.Sleep(step)
+	}
+	return nil
+}

util/testutil/workers/containerd.go

@@ -88,9 +88,11 @@ func (c *Containerd) New(ctx context.Context, cfg *integration.BackendConfig) (b
 	if err := integration.LookupBinary(c.Containerd); err != nil {
 		return nil, nil, err
 	}
+
 	if err := integration.LookupBinary("buildkitd"); err != nil {
 		return nil, nil, err
 	}
+
 	if err := requireRoot(); err != nil {
 		return nil, nil, err
 	}
@@ -106,7 +108,7 @@ func (c *Containerd) New(ctx context.Context, cfg *integration.BackendConfig) (b
 	}()
 
 	rootless := false
-	if c.UID != 0 {
+	if runtime.GOOS != "windows" && c.UID != 0 {
 		if c.GID == 0 {
 			return nil, nil, errors.Errorf("unsupported id pair: uid=%d, gid=%d", c.UID, c.GID)
 		}
@@ -117,6 +119,7 @@ func (c *Containerd) New(ctx context.Context, cfg *integration.BackendConfig) (b
 	if err != nil {
 		return nil, nil, err
 	}
+
 	if rootless {
 		if err := os.Chown(tmpdir, c.UID, c.GID); err != nil {
 			return nil, nil, err
@@ -125,7 +128,7 @@ func (c *Containerd) New(ctx context.Context, cfg *integration.BackendConfig) (b
 
 	deferF.Append(func() error { return os.RemoveAll(tmpdir) })
 
-	address := filepath.Join(tmpdir, "containerd.sock")
+	address := getContainerdSock(tmpdir)
 	config := fmt.Sprintf(`root = %q
 state = %q
 # CRI plugins listens on 10010/tcp for stream server.
@@ -137,8 +140,11 @@ disabled_plugins = ["cri"]
 
 [debug]
   level = "debug"
-  address = %q
-`, filepath.Join(tmpdir, "root"), filepath.Join(tmpdir, "state"), address, filepath.Join(tmpdir, "debug.sock"))
+  address = %q`,
+		filepath.Join(tmpdir, "root"),
+		filepath.Join(tmpdir, "state"),
+		address, getContainerdDebugSock(tmpdir),
+	)
 
 	var snBuildkitdArgs []string
 	if c.Snapshotter != "" {
@@ -185,19 +191,13 @@ disabled_plugins = ["cri"]
 	if err != nil {
 		return nil, nil, err
 	}
-	if err := integration.WaitUnix(address, 10*time.Second, cmd); err != nil {
+	if err := integration.WaitSocket(address, 10*time.Second, cmd); err != nil {
 		ctdStop()
 		return nil, nil, errors.Wrapf(err, "containerd did not start up: %s", integration.FormatLogs(cfg.Logs))
 	}
 	deferF.Append(ctdStop)
 
-	buildkitdArgs := append([]string{"buildkitd",
-		"--oci-worker=false",
-		"--containerd-worker-gc=false",
-		"--containerd-worker=true",
-		"--containerd-worker-addr", address,
-		"--containerd-worker-labels=org.mobyproject.buildkit.worker.sandbox=true", // Include use of --containerd-worker-labels to trigger https://github.com/moby/buildkit/pull/603
-	}, snBuildkitdArgs...)
+	buildkitdArgs := append(getBuildkitdArgs(address), snBuildkitdArgs...)
 
 	if runtime.GOOS != "windows" && c.Snapshotter != "native" {
 		c.ExtraEnv = append(c.ExtraEnv, "BUILDKIT_DEBUG_FORCE_OVERLAY_DIFF=true")
@@ -266,9 +266,13 @@ func runStargzSnapshotter(cfg *integration.BackendConfig) (address string, cl fu
 	if err != nil {
 		return "", nil, err
 	}
-	if err = integration.WaitUnix(address, 10*time.Second, cmd); err != nil {
+	if err = integration.WaitSocket(address, 10*time.Second, cmd); err != nil {
 		snStop()
-		return "", nil, errors.Wrapf(err, "containerd-stargz-grpc did not start up: %s", integration.FormatLogs(cfg.Logs))
+		errMsg := fmt.Sprintf(
+			"containerd-stargz-grpc did not start up: %s",
+			integration.FormatLogs(cfg.Logs),
+		)
+		return "", nil, errors.Wrapf(err, errMsg)
 	}
 	deferF.Append(snStop)
 

util/testutil/workers/dockerd.go

@@ -159,7 +159,7 @@ func (c Moby) New(ctx context.Context, cfg *integration.BackendConfig) (b integr
 	}
 	deferF.Append(d.StopWithError)
 
-	if err := integration.WaitUnix(d.Sock(), 5*time.Second, nil); err != nil {
+	if err := integration.WaitSocket(d.Sock(), 5*time.Second, nil); err != nil {
 		return nil, nil, errors.Errorf("dockerd did not start up: %q, %s", err, integration.FormatLogs(cfg.Logs))
 	}
 

util/testutil/workers/sysprocattr_unix.go

@@ -21,3 +21,21 @@ func getBuildkitdAddr(tmpdir string) string {
 func getTraceSocketPath(tmpdir string) string {
 	return filepath.Join(tmpdir, "otel-grpc.sock")
 }
+
+func getContainerdSock(tmpdir string) string {
+	return filepath.Join(tmpdir, "containerd.sock")
+}
+
+func getContainerdDebugSock(tmpdir string) string {
+	return filepath.Join(tmpdir, "debug.sock")
+}
+
+func getBuildkitdArgs(address string) []string {
+	return []string{"buildkitd",
+		"--oci-worker=false",
+		"--containerd-worker-gc=false",
+		"--containerd-worker=true",
+		"--containerd-worker-addr", address,
+		"--containerd-worker-labels=org.mobyproject.buildkit.worker.sandbox=true", // Include use of --containerd-worker-labels to trigger https://github.com/moby/buildkit/pull/603
+	}
+}

util/testutil/workers/sysprocattr_windows.go

@@ -5,6 +5,7 @@ package workers
 
 import (
 	"path/filepath"
+	"strings"
 	"syscall"
 )
 
@@ -13,9 +14,30 @@ func getSysProcAttr() *syscall.SysProcAttr {
 }
 
 func getBuildkitdAddr(tmpdir string) string {
-	return "//./pipe/buildkitd-" + filepath.Base(tmpdir)
+	return "npipe:////./pipe/buildkitd-" + filepath.Base(tmpdir)
 }
 
 func getTraceSocketPath(tmpdir string) string {
 	return `\\.\pipe\buildkit-otel-grpc-` + filepath.Base(tmpdir)
 }
+
+func getContainerdSock(tmpdir string) string {
+	return `\\.\pipe\containerd-` + filepath.Base(tmpdir)
+}
+
+func getContainerdDebugSock(tmpdir string) string {
+	return `\\.\pipe\containerd-` + filepath.Base(tmpdir) + `debug`
+}
+
+func getBuildkitdArgs(address string) []string {
+	address = filepath.ToSlash(address)
+	if !strings.HasPrefix(address, "npipe://") {
+		address = "npipe://" + address
+	}
+	return []string{"buildkitd",
+		"--containerd-worker-gc=false",
+		"--containerd-worker=true",
+		"--containerd-worker-addr", address,
+		"--containerd-worker-labels=org.mobyproject.buildkit.worker.sandbox=true", // Include use of --containerd-worker-labels to trigger https://github.com/moby/buildkit/pull/603
+	}
+}

util/testutil/workers/util.go

@@ -1,98 +1,11 @@
 package workers
 
 import (
-	"bufio"
-	"bytes"
-	"context"
 	"fmt"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"time"
 
 	"github.com/moby/buildkit/util/testutil/integration"
-	"github.com/pkg/errors"
 )
 
-func requireRoot() error {
-	if os.Getuid() != 0 {
-		return errors.Wrap(integration.ErrRequirements, "requires root")
-	}
-	return nil
-}
-
-func runBuildkitd(ctx context.Context, conf *integration.BackendConfig, args []string, logs map[string]*bytes.Buffer, uid, gid int, extraEnv []string) (address string, cl func() error, err error) {
-	deferF := &integration.MultiCloser{}
-	cl = deferF.F()
-
-	defer func() {
-		if err != nil {
-			deferF.F()()
-			cl = nil
-		}
-	}()
-
-	tmpdir, err := os.MkdirTemp("", "bktest_buildkitd")
-	if err != nil {
-		return "", nil, err
-	}
-	if err := os.Chown(tmpdir, uid, gid); err != nil {
-		return "", nil, err
-	}
-	if err := os.MkdirAll(filepath.Join(tmpdir, "tmp"), 0711); err != nil {
-		return "", nil, err
-	}
-	if err := os.Chown(filepath.Join(tmpdir, "tmp"), uid, gid); err != nil {
-		return "", nil, err
-	}
-	deferF.Append(func() error { return os.RemoveAll(tmpdir) })
-
-	cfgfile, err := integration.WriteConfig(append(conf.DaemonConfig, withOTELSocketPath(getTraceSocketPath(tmpdir))))
-	if err != nil {
-		return "", nil, err
-	}
-	deferF.Append(func() error {
-		return os.RemoveAll(filepath.Dir(cfgfile))
-	})
-	args = append(args, "--config="+cfgfile)
-
-	address = getBuildkitdAddr(tmpdir)
-
-	args = append(args, "--root", tmpdir, "--addr", address, "--debug")
-	cmd := exec.Command(args[0], args[1:]...) //nolint:gosec // test utility
-	cmd.Env = append(os.Environ(), "BUILDKIT_DEBUG_EXEC_OUTPUT=1", "BUILDKIT_DEBUG_PANIC_ON_ERROR=1", "TMPDIR="+filepath.Join(tmpdir, "tmp"))
-	cmd.Env = append(cmd.Env, extraEnv...)
-	cmd.SysProcAttr = getSysProcAttr()
-
-	stop, err := integration.StartCmd(cmd, logs)
-	if err != nil {
-		return "", nil, err
-	}
-	deferF.Append(stop)
-
-	if err := integration.WaitUnix(address, 15*time.Second, cmd); err != nil {
-		return "", nil, err
-	}
-
-	deferF.Append(func() error {
-		f, err := os.Open("/proc/self/mountinfo")
-		if err != nil {
-			return errors.Wrap(err, "failed to open mountinfo")
-		}
-		defer f.Close()
-		s := bufio.NewScanner(f)
-		for s.Scan() {
-			if strings.Contains(s.Text(), tmpdir) {
-				return errors.Errorf("leaked mountpoint for %s", tmpdir)
-			}
-		}
-		return s.Err()
-	})
-
-	return address, cl, err
-}
-
 func withOTELSocketPath(socketPath string) integration.ConfigUpdater {
 	return otelSocketPath(socketPath)
 }