Merge pull request #571 from buildpacks/feature/469-windows-create-bu…

…ilder

Support windows images for create-builder

.github/workflows/build.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [macos, linux, windows]
+        os: [macos, linux, windows-lcow, windows-wcow]
         include:
           - os: macos
             runner: macos-latest
@@ -25,10 +25,14 @@ jobs:
             runner: ubuntu-latest
             no_docker: "false"
             pack_bin: pack
-          - os: windows
+          - os: windows-lcow
             runner: [self-hosted, windows]
             no_docker: "false"
             pack_bin: pack.exe
+          - os: windows-wcow
+            runner: windows-latest
+            no_docker: "false"
+            pack_bin: pack.exe
     runs-on: ${{ matrix.runner }}
     env:
       PACK_BIN: ${{ matrix.pack_bin }}
@@ -50,7 +54,7 @@ jobs:
           echo "::add-path::$(go env GOPATH)/bin"
       - name: Verify
         # disabled for windows due to format verification failing
-        if: matrix.os != 'windows'
+        if: ${{ !contains(matrix.os, 'windows') }}
         run: make verify
       - name: Test
         env:
@@ -91,7 +95,8 @@ jobs:
       - name: Download artifacts - windows
         uses: actions/download-artifact@v1
         with:
-          name: pack-windows
+          name: pack-windows-lcow
+          path: pack-windows
       - name: Package artifacts - macos
         run: |
           chmod +x pack-macos/pack

acceptance/acceptance_test.go

@@ -300,6 +300,8 @@ func testWithoutSpecificBuilderRequirement(
 				"pack does not support 'package-buildpack'",
 			)
 
+			h.SkipIf(t, dockerHostOS() == "windows", "These tests are not yet compatible with Windows-based containers")
+
 			var err error
 			tmpDir, err = ioutil.TempDir("", "package-buildpack-tests")
 			h.AssertNil(t, err)
@@ -553,13 +555,31 @@ func testAcceptance(
 			runImageMirror = value
 		})
 
+		when("creating a windows builder", func() {
+			it.Before(func() {
+				h.SkipIf(t, dockerHostOS() != "windows", "The current Docker daemon does not support Windows-based containers")
+			})
+
+			it("succeeds", func() {
+				builderName := createBuilder(t, runImageMirror, configDir, packCreateBuilderPath, lifecyclePath, lifecycleDescriptor)
+				defer h.DockerRmi(dockerCli, builderName)
+
+				inspect, _, err := dockerCli.ImageInspectWithRaw(context.TODO(), builderName)
+				h.AssertNil(t, err)
+
+				h.AssertEq(t, inspect.Os, "windows")
+			})
+		})
+
 		when("builder is created", func() {
 			var (
 				builderName string
 				tmpDir      string
 			)
 
 			it.Before(func() {
+				h.SkipIf(t, dockerHostOS() == "windows", "These tests are not yet compatible with Windows-based containers")
+
 				var err error
 				tmpDir, err = ioutil.TempDir("", "package-buildpack-tests")
 				h.AssertNil(t, err)
@@ -1672,43 +1692,42 @@ func createBuilder(t *testing.T, runImageMirror, configDir, packPath, lifecycleP
 		h.AssertNil(t, err)
 	}
 
-	// CREATE PACKAGE
-	packageImageName := packageBuildpackAsImage(t,
-		packPath,
-		filepath.Join(configDir, "package.toml"),
-		lifecycleDescriptor,
-		[]string{"simple-layers-buildpack"},
-	)
-
-	// RENDER builder.toml
-	cfgData := fillTemplate(t, filepath.Join(configDir, "builder.toml"), map[string]interface{}{
-		"package_name": packageImageName,
-	})
-	err = ioutil.WriteFile(filepath.Join(tmpDir, "builder.toml"), []byte(cfgData), os.ModePerm)
-	h.AssertNil(t, err)
-
-	builderConfigFile, err := os.OpenFile(filepath.Join(tmpDir, "builder.toml"), os.O_RDWR|os.O_APPEND, os.ModePerm)
-	h.AssertNil(t, err)
+	var packageImageName string
+	var packageId string
+	if dockerHostOS() != "windows" {
+		// CREATE PACKAGE
+		packageImageName = packageBuildpackAsImage(t,
+			packPath,
+			filepath.Join(configDir, "package.toml"),
+			lifecycleDescriptor,
+			[]string{"simple-layers-buildpack"},
+		)
 
-	// ADD run-image-mirrors
-	_, err = builderConfigFile.Write([]byte(fmt.Sprintf("run-image-mirrors = [\"%s\"]\n", runImageMirror)))
-	h.AssertNil(t, err)
+		packageId = "simple/layers"
+	}
 
 	// ADD lifecycle
-	_, err = builderConfigFile.Write([]byte("[lifecycle]\n"))
-	h.AssertNil(t, err)
-
+	var lifecycleURI string
+	var lifecycleVersion string
 	if lifecyclePath != "" {
 		t.Logf("adding lifecycle path '%s' to builder config", lifecyclePath)
-		_, err = builderConfigFile.Write([]byte(fmt.Sprintf("uri = \"%s\"\n", strings.ReplaceAll(lifecyclePath, `\`, `\\`))))
-		h.AssertNil(t, err)
+		lifecycleURI = strings.ReplaceAll(lifecyclePath, `\`, `\\`)
 	} else {
 		t.Logf("adding lifecycle version '%s' to builder config", lifecycleDescriptor.Info.Version.String())
-		_, err = builderConfigFile.Write([]byte(fmt.Sprintf("version = \"%s\"\n", lifecycleDescriptor.Info.Version.String())))
-		h.AssertNil(t, err)
+		lifecycleVersion = lifecycleDescriptor.Info.Version.String()
 	}
 
-	builderConfigFile.Close()
+	// RENDER builder.toml
+	cfgData := fillTemplate(t, filepath.Join(configDir, "builder.toml"), map[string]interface{}{
+		"package_image_name": packageImageName,
+		"package_id":         packageId,
+		"run_image_mirror":   runImageMirror,
+		"lifecycle_uri":      lifecycleURI,
+		"lifecycle_version":  lifecycleVersion,
+	})
+
+	err = ioutil.WriteFile(filepath.Join(tmpDir, "builder.toml"), []byte(cfgData), os.ModePerm)
+	h.AssertNil(t, err)
 
 	// NAME BUILDER
 	bldr := registryConfig.RepoName("test/builder-" + h.RandString(10))
@@ -1793,10 +1812,12 @@ func createStack(t *testing.T, dockerCli client.CommonAPIClient, runImageMirror
 	t.Helper()
 	t.Log("creating stack images...")
 
-	if err := createStackImage(dockerCli, runImage, filepath.Join("testdata", "mock_stack", "run")); err != nil {
+	stackBaseDir := filepath.Join("testdata", "mock_stack", dockerHostOS())
+
+	if err := createStackImage(dockerCli, runImage, filepath.Join(stackBaseDir, "run")); err != nil {
 		return err
 	}
-	if err := createStackImage(dockerCli, buildImage, filepath.Join("testdata", "mock_stack", "build")); err != nil {
+	if err := createStackImage(dockerCli, buildImage, filepath.Join(stackBaseDir, "build")); err != nil {
 		return err
 	}
 
@@ -1991,6 +2012,14 @@ func fillTemplate(t *testing.T, templatePath string, data map[string]interface{}
 	return expectedOutput.String()
 }
 
+func dockerHostOS() string {
+	daemonInfo, err := dockerCli.Info(context.TODO())
+	if err != nil {
+		panic(err.Error())
+	}
+	return daemonInfo.OSType
+}
+
 // taskKey creates a key from the prefix and all arguments to be unique
 func taskKey(prefix string, args ...string) string {
 	hash := sha256.New()

acceptance/testdata/mock_stack/create-stack.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 dir="$(cd $(dirname $0) && pwd)"
+os=$(docker info --format '{{json .}}' | jq -r .OSType)
 
-docker build --tag pack-test/build "$dir"/build
-docker build --tag pack-test/run "$dir"/run
+docker build --tag pack-test/build "$dir"/$os/build
+docker build --tag pack-test/run "$dir"/$os/run

acceptance/testdata/mock_stack/windows/build/Dockerfile

@@ -0,0 +1,14 @@
+FROM mcr.microsoft.com/windows/nanoserver:1809
+
+# placeholder values until correct values are deteremined
+ENV CNB_USER_ID=0
+ENV CNB_GROUP_ID=0
+
+USER ContainerAdministrator
+
+RUN net users /ADD pack /passwordreq:no /expires:never
+
+LABEL io.buildpacks.stack.id=pack.test.stack
+LABEL io.buildpacks.stack.mixins="[\"mixinA\", \"build:mixinTwo\", \"netcat\", \"mixin3\"]"
+
+USER pack

acceptance/testdata/mock_stack/windows/run/Dockerfile

@@ -0,0 +1,14 @@
+FROM mcr.microsoft.com/windows/nanoserver:1809
+
+# placeholder values until correct values are deteremined
+ENV CNB_USER_ID=0
+ENV CNB_GROUP_ID=0
+
+USER ContainerAdministrator
+
+RUN net users /ADD pack /passwordreq:no /expires:never
+
+LABEL io.buildpacks.stack.id=pack.test.stack
+LABEL io.buildpacks.stack.mixins="[\"mixinA\", \"netcat\", \"mixin3\"]"
+
+USER pack

acceptance/testdata/pack_fixtures/.gitattributes

@@ -0,0 +1 @@
+*.txt text eol=lf

acceptance/testdata/pack_fixtures/builder.toml

@@ -11,13 +11,17 @@
   # noop-buildpack-2 has the same id but a different version compared to noop-buildpack
   uri = "noop-buildpack-2.tgz"
 
+{{- if .package_image_name}}
 [[buildpacks]]
-  image = "{{ .package_name }}"
+  image = "{{.package_image_name}}"
+{{- end}}
 
 [[order]]
+{{- if .package_id}}
 [[order.group]]
-  id = "simple/layers"
+  id = "{{.package_id}}"
   # intentionlly missing version to test support
+{{- end}}
 
 [[order.group]]
   id = "read/env"
@@ -28,5 +32,12 @@
   id = "pack.test.stack"
   build-image = "pack-test/build"
   run-image = "pack-test/run"
+  run-image-mirrors = ["{{.run_image_mirror}}"]
 
-# run-image-mirror and lifecycle are appended by acceptance tests
+[lifecycle]
+{{- if .lifecycle_uri}}
+  uri = "{{.lifecycle_uri}}"
+{{- end}}
+{{- if .lifecycle_version}}
+  version = "{{.lifecycle_version}}"
+{{- end}}

acceptance/testdata/pack_previous_fixtures_overrides/builder.toml

@@ -0,0 +1,43 @@
+[[buildpacks]]
+  id = "read/env"
+  version = "read-env-version"
+  uri = "read-env-buildpack.tgz"
+
+[[buildpacks]]
+  # intentionally missing id/version as they are optional
+  uri = "noop-buildpack.tgz"
+
+[[buildpacks]]
+  # noop-buildpack-2 has the same id but a different version compared to noop-buildpack
+  uri = "noop-buildpack-2.tgz"
+
+{{- if .package_image_name}}
+[[buildpacks]]
+  image = "{{.package_image_name}}"
+{{- end}}
+
+[[order]]
+{{- if .package_id}}
+[[order.group]]
+  id = "{{.package_id}}"
+  # intentionlly missing version to test support
+{{- end}}
+
+[[order.group]]
+  id = "read/env"
+  version = "read-env-version"
+  optional = true
+
+[stack]
+  id = "pack.test.stack"
+  build-image = "pack-test/build"
+  run-image = "pack-test/run"
+  run-image-mirrors = ["{{.run_image_mirror}}"]
+
+[lifecycle]
+{{- if .lifecycle_uri}}
+  uri = "{{.lifecycle_uri}}"
+{{- end}}
+{{- if .lifecycle_version}}
+  version = "{{.lifecycle_version}}"
+{{- end}}

build.go

@@ -26,6 +26,7 @@ import (
 	"github.com/buildpacks/pack/internal/buildpack"
 	"github.com/buildpacks/pack/internal/buildpackage"
 	"github.com/buildpacks/pack/internal/dist"
+	"github.com/buildpacks/pack/internal/layer"
 	"github.com/buildpacks/pack/internal/paths"
 	"github.com/buildpacks/pack/internal/stack"
 	"github.com/buildpacks/pack/internal/stringset"
@@ -104,7 +105,7 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error {
 		return err
 	}
 
-	fetchedBPs, order, err := c.processBuildpacks(ctx, bldr.Buildpacks(), bldr.Order(), opts.Buildpacks, opts.NoPull, opts.Publish, opts.Registry)
+	fetchedBPs, order, err := c.processBuildpacks(ctx, bldr.Image(), bldr.Buildpacks(), bldr.Order(), opts.Buildpacks, opts.NoPull, opts.Publish, opts.Registry)
 	if err != nil {
 		return err
 	}
@@ -396,7 +397,7 @@ func (c *Client) processProxyConfig(config *ProxyConfig) ProxyConfig {
 // 	----------
 // 	- group:
 //		- A
-func (c *Client) processBuildpacks(ctx context.Context, builderBPs []dist.BuildpackInfo, builderOrder dist.Order, declaredBPs []string, noPull bool, publish bool, registry string) (fetchedBPs []dist.Buildpack, order dist.Order, err error) {
+func (c *Client) processBuildpacks(ctx context.Context, builderImage imgutil.Image, builderBPs []dist.BuildpackInfo, builderOrder dist.Order, declaredBPs []string, noPull bool, publish bool, registry string) (fetchedBPs []dist.Buildpack, order dist.Order, err error) {
 	order = dist.Order{{Group: []dist.BuildpackRef{}}}
 	for _, bp := range declaredBPs {
 		locatorType, err := buildpack.GetLocatorType(bp, builderBPs)
@@ -453,7 +454,11 @@ func (c *Client) processBuildpacks(ctx context.Context, builderBPs []dist.Buildp
 					return fetchedBPs, order, errors.Wrapf(err, "extracting buildpacks from %s", style.Symbol(bp))
 				}
 			} else {
-				mainBP, err = dist.BuildpackFromRootBlob(blob)
+				layerWriterFactory, err := layer.NewWriterFactory(builderImage)
+				if err != nil {
+					return fetchedBPs, order, errors.Wrapf(err, "get tar writer factory for image %s", style.Symbol(builderImage.Name()))
+				}
+				mainBP, err = dist.BuildpackFromRootBlob(blob, layerWriterFactory)
 				if err != nil {
 					return fetchedBPs, order, errors.Wrapf(err, "creating buildpack from %s", style.Symbol(bp))
 				}

create_builder.go

@@ -14,6 +14,7 @@ import (
 	"github.com/buildpacks/pack/internal/builder"
 	"github.com/buildpacks/pack/internal/dist"
 	"github.com/buildpacks/pack/internal/image"
+	"github.com/buildpacks/pack/internal/layer"
 	"github.com/buildpacks/pack/internal/style"
 )
 
@@ -101,7 +102,11 @@ func (c *Client) CreateBuilder(ctx context.Context, opts CreateBuilderOptions) e
 				return errors.Wrapf(err, "downloading buildpack from %s", style.Symbol(b.URI))
 			}
 
-			fetchedBp, err := dist.BuildpackFromRootBlob(blob)
+			layerWriterFactory, err := layer.NewWriterFactory(bldr.Image())
+			if err != nil {
+				return errors.Wrapf(err, "get tar writer factory for image %s", style.Symbol(bldr.Name()))
+			}
+			fetchedBp, err := dist.BuildpackFromRootBlob(blob, layerWriterFactory)
 			if err != nil {
 				return errors.Wrapf(err, "creating buildpack from %s", style.Symbol(b.URI))
 			}

go.mod

@@ -5,8 +5,8 @@ require (
 	github.com/Masterminds/semver v1.5.0
 	github.com/Microsoft/hcsshim v0.8.7 // indirect
 	github.com/apex/log v1.1.2
-	github.com/buildpacks/imgutil v0.0.0-20200313170640-a02052f47d62
-	github.com/buildpacks/lifecycle v0.7.1
+	github.com/buildpacks/imgutil v0.0.0-20200424215026-dfdc82949704
+	github.com/buildpacks/lifecycle v0.7.2
 	github.com/containerd/containerd v1.3.3 // indirect
 	github.com/containerd/continuity v0.0.0-20200107194136-26c1120b8d41 // indirect
 	github.com/dgodd/dockerdial v1.0.1