diff --git a/build.go b/build.go index e59a5646a..33a2bb078 100644 --- a/build.go +++ b/build.go @@ -8,7 +8,6 @@ import ( "io/ioutil" "os" "path/filepath" - "strconv" "strings" "github.com/buildpack/pack/build" @@ -20,8 +19,6 @@ import ( "github.com/buildpack/pack/style" lcimg "github.com/buildpack/lifecycle/image" - dockertypes "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" "github.com/pkg/errors" ) @@ -67,16 +64,6 @@ type BuildConfig struct { LifecycleConfig build.LifecycleConfig } -const ( - layersDir = "/workspace" - buildpacksDir = "/buildpacks" - platformDir = "/platform" - orderPath = "/buildpacks/order.toml" - groupPath = `/workspace/group.toml` - planPath = "/workspace/plan.toml" - appDir = "/workspace/app" -) - func DefaultBuildFactory(logger *logging.Logger, cache Cache, dockerClient Docker, fetcher Fetcher) (*BuildFactory, error) { f := &BuildFactory{ Logger: logger, @@ -310,7 +297,6 @@ func (b *BuildConfig) Run(ctx context.Context) error { if b.ClearCache { b.Logger.Verbose("Skipping 'analyze' due to clearing cache") } else { - b.Logger.Verbose("Reading information from previous image for possible re-use") if err := b.analyze(ctx, lifecycle); err != nil { return err } @@ -335,244 +321,57 @@ func (b *BuildConfig) Run(ctx context.Context) error { } func (b *BuildConfig) detect(ctx context.Context, lifecycle *build.Lifecycle) error { - phase, err := lifecycle.NewPhase( - "detector", - build.WithArgs( - "-buildpacks", buildpacksDir, - "-order", orderPath, - "-group", groupPath, - "-plan", planPath, - "-app", appDir, - ), - ) + detect, err := lifecycle.NewDetect() if err != nil { return err } - defer phase.Cleanup() - - if err := phase.Run(ctx); err != nil { - return errors.Wrap(err, "run detect container") - } - return nil + defer detect.Cleanup() + return detect.Run(ctx) } func (b *BuildConfig) restore(ctx context.Context, lifecycle *build.Lifecycle) error { - phase, err := lifecycle.NewPhase( - "restorer", - build.WithArgs( - "-image", b.Cache.Image(), - "-group", groupPath, - "-layers", layersDir, - ), - build.WithDaemonAccess(), - ) - + restore, err := lifecycle.NewRestore(b.Cache.Image()) if err != nil { return err } - defer phase.Cleanup() - - if err := phase.Run(ctx); err != nil { - return errors.Wrap(err, "run restorer container") - } - - return nil + defer restore.Cleanup() + return restore.Run(ctx) } func (b *BuildConfig) analyze(ctx context.Context, lifecycle *build.Lifecycle) error { - var analyze *build.Phase - var err error - if b.Publish { - analyze, err = lifecycle.NewPhase( - "analyzer", - build.WithRegistryAccess(b.RepoName, b.RunImage), - build.WithArgs( - "-layers", layersDir, - "-group", groupPath, - b.RepoName, - ), - ) - } else { - analyze, err = lifecycle.NewPhase( - "analyzer", - build.WithDaemonAccess(), - build.WithArgs( - "-layers", layersDir, - "-group", groupPath, - "-daemon", - b.RepoName, - ), - ) - } - defer analyze.Cleanup() - if err = analyze.Run(ctx); err != nil { - return err - } - - uid, gid, err := b.packUidGid(ctx, b.Builder) + analyze, err := lifecycle.NewAnalyze(b.RepoName, b.Publish) if err != nil { - return errors.Wrap(err, "get pack uid and gid") - } - if err := b.chownDir(ctx, lifecycle, layersDir, uid, gid); err != nil { - return errors.Wrap(err, "chown launch dir") + return err } - - return nil + defer analyze.Cleanup() + return analyze.Run(ctx) } func (b *BuildConfig) build(ctx context.Context, lifecycle *build.Lifecycle) error { - build, err := lifecycle.NewPhase( - "builder", - build.WithArgs( - "-buildpacks", buildpacksDir, - "-layers", layersDir, - "-app", appDir, - "-group", groupPath, - "-plan", planPath, - "-platform", platformDir, - ), - ) + build, err := lifecycle.NewBuild() if err != nil { return err } defer build.Cleanup() - if err := build.Run(ctx); err != nil { - return errors.Wrap(err, "run build container") - } - return nil -} - -type exporterArgs struct { - args []string - repoName string -} - -func (e *exporterArgs) add(args ...string) { - e.args = append(e.args, args...) -} - -func (e *exporterArgs) daemon() { - e.args = append(e.args, "-daemon") -} - -func (e *exporterArgs) list() []string { - e.args = append(e.args, e.repoName) - return e.args + return build.Run(ctx) } func (b *BuildConfig) export(ctx context.Context, lifecycle *build.Lifecycle) error { - var ( - export *build.Phase - err error - ) - - args := &exporterArgs{repoName: b.RepoName} - args.add( - "-image", b.RunImage, - "-layers", layersDir, - "-app", appDir, - "-group", groupPath, - ) - - if b.Publish { - export, err = lifecycle.NewPhase( - "exporter", - build.WithRegistryAccess(b.RepoName, b.RunImage), - build.WithArgs(args.list()...), - ) - } else { - args.daemon() - export, err = lifecycle.NewPhase( - "exporter", - build.WithDaemonAccess(), - build.WithArgs(args.list()...), - ) - } - defer export.Cleanup() - - uid, gid, err := b.packUidGid(ctx, b.Builder) + export, err := lifecycle.NewExport(b.RepoName, b.RunImage, b.Publish) if err != nil { - return errors.Wrap(err, "get pack uid and gid") - } - - if err := b.chownDir(ctx, lifecycle, layersDir, uid, gid); err != nil { - return errors.Wrap(err, "chown launch dir") + return err } - + defer export.Cleanup() return export.Run(ctx) } func (b *BuildConfig) cache(ctx context.Context, lifecycle *build.Lifecycle) error { - phase, err := lifecycle.NewPhase( - "cacher", - build.WithArgs( - "-image", b.Cache.Image(), - "-group", groupPath, - "-layers", layersDir, - ), - build.WithDaemonAccess(), - ) - - if err != nil { - return err - } - defer phase.Cleanup() - - if err := phase.Run(ctx); err != nil { - return errors.Wrap(err, "run cacher container") - } - - return nil -} - -func (b *BuildConfig) packUidGid(ctx context.Context, builder string) (int, int, error) { - i, _, err := b.Cli.ImageInspectWithRaw(ctx, builder) - if err != nil { - return 0, 0, errors.Wrap(err, "reading builder env variables") - } - var sUID, sGID string - for _, kv := range i.Config.Env { - kv2 := strings.SplitN(kv, "=", 2) - if len(kv2) == 2 && kv2[0] == "CNB_USER_ID" { - sUID = kv2[1] - } else if len(kv2) == 2 && kv2[0] == "CNB_GROUP_ID" { - sGID = kv2[1] - } - } - if sUID == "" || sGID == "" { - return 0, 0, errors.New("not found pack uid & gid") - } - var uid, gid int - uid, err = strconv.Atoi(sUID) + cache, err := lifecycle.NewCache(b.Cache.Image()) if err != nil { - return 0, 0, errors.Wrapf(err, "parsing pack uid: %s", sUID) - } - gid, err = strconv.Atoi(sGID) - if err != nil { - return 0, 0, errors.Wrapf(err, "parsing pack gid: %s", sGID) - } - return uid, gid, nil -} - -func (b *BuildConfig) chownDir(ctx context.Context, lifecycle *build.Lifecycle, path string, uid, gid int) error { - ctr, err := b.Cli.ContainerCreate(ctx, &container.Config{ - Image: b.Builder, - Cmd: []string{"chown", "-R", fmt.Sprintf("%d:%d", uid, gid), path}, - User: "root", - Labels: map[string]string{"author": "pack"}, - }, &container.HostConfig{ - Binds: []string{ - fmt.Sprintf("%s:%s:", lifecycle.WorkspaceVolume, layersDir), - }, - }, nil, "") - if err != nil { - return err - } - defer b.Cli.ContainerRemove(context.Background(), ctr.ID, dockertypes.ContainerRemoveOptions{Force: true}) - if err := b.Cli.RunContainer(ctx, ctr.ID, b.Logger.VerboseWriter(), b.Logger.VerboseErrorWriter()); err != nil { return err } - return nil + defer cache.Cleanup() + return cache.Run(ctx) } func parseEnvFile(filename string) (map[string]string, error) { diff --git a/build/lifecycle.go b/build/lifecycle.go index b11df4f13..5f539dc8d 100644 --- a/build/lifecycle.go +++ b/build/lifecycle.go @@ -4,8 +4,6 @@ import ( "archive/tar" "context" "fmt" - "github.com/docker/docker/api/types/filters" - "github.com/docker/docker/api/types/volume" "io" "io/ioutil" "math/rand" @@ -20,27 +18,28 @@ import ( "github.com/BurntSushi/toml" "github.com/buildpack/lifecycle" "github.com/buildpack/lifecycle/image" - - "github.com/buildpack/pack/archive" - "github.com/buildpack/pack/docker" - "github.com/buildpack/pack/style" - "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/network" + "github.com/docker/docker/api/types/volume" "github.com/pkg/errors" + "github.com/buildpack/pack/archive" + "github.com/buildpack/pack/docker" "github.com/buildpack/pack/logging" + "github.com/buildpack/pack/style" ) type Lifecycle struct { - BuilderImage string - Logger *logging.Logger - Docker Docker - WorkspaceVolume string - uid, gid int - appDir string - appOnce *sync.Once + BuilderImage string + Logger *logging.Logger + Docker Docker + LayersVolume string + AppVolume string + uid, gid int + appDir string + appOnce *sync.Once } type Docker interface { @@ -54,10 +53,6 @@ type Docker interface { VolumeList(ctx context.Context, filter filters.Args) (volume.VolumeListOKBody, error) } -const ( - launchDir = "/workspace" -) - type LifecycleConfig struct { BuilderImage string Logger *logging.Logger @@ -123,23 +118,30 @@ func NewLifecycle(c LifecycleConfig) (*Lifecycle, error) { } return &Lifecycle{ - BuilderImage: builder.Name(), - Logger: c.Logger, - Docker: client, - WorkspaceVolume: "pack-workspace-" + randString(10), - appDir: c.AppDir, - uid: uid, - gid: gid, - appOnce: &sync.Once{}, + BuilderImage: builder.Name(), + Logger: c.Logger, + Docker: client, + LayersVolume: "pack-layers-" + randString(10), + AppVolume: "pack-app-" + randString(10), + appDir: c.AppDir, + uid: uid, + gid: gid, + appOnce: &sync.Once{}, }, nil } func (l *Lifecycle) Cleanup() error { - _, err := l.Docker.ImageRemove(context.Background(), l.BuilderImage, types.ImageRemoveOptions{}) - if err != nil { - return err + var reterr error + if _, err := l.Docker.ImageRemove(context.Background(), l.BuilderImage, types.ImageRemoveOptions{}); err != nil { + reterr = errors.Wrapf(err, "failed to clean up builder image %s", l.BuilderImage) + } + if err := l.Docker.VolumeRemove(context.Background(), l.LayersVolume, true); err != nil { + reterr = errors.Wrapf(err, "failed to clean up layers volume %s", l.LayersVolume) + } + if err := l.Docker.VolumeRemove(context.Background(), l.AppVolume, true); err != nil { + reterr = errors.Wrapf(err, "failed to clean up app volume %s", l.AppVolume) } - return l.Docker.VolumeRemove(context.Background(), l.WorkspaceVolume, true) + return reterr } func randString(n int) string { @@ -181,14 +183,17 @@ func tarEnvFile(tmpDir string, env map[string]string) (string, error) { tw := tar.NewWriter(fh) defer tw.Close() for k, v := range env { - if err := tw.WriteHeader(&tar.Header{Name: "/platform/env/" + k, Size: int64(len(v)), Mode: 0444, ModTime: now}); err != nil { + if err := tw.WriteHeader(&tar.Header{Name: filepath.Join(platformDir, "env", k), Size: int64(len(v)), Mode: 0444, ModTime: now}); err != nil { return "", err } if _, err := tw.Write([]byte(v)); err != nil { return "", err } } - if err := tw.WriteHeader(&tar.Header{Typeflag: tar.TypeDir, Name: "/platform/env/", Mode: 0555, ModTime: now}); err != nil { + if err := tw.WriteHeader(&tar.Header{Typeflag: tar.TypeDir, Name: filepath.Join(platformDir, "env"), Mode: 0555, ModTime: now}); err != nil { + return "", err + } + if err := tw.WriteHeader(&tar.Header{Typeflag: tar.TypeDir, Name: platformDir, Mode: 0555, ModTime: now}); err != nil { return "", err } return fh.Name(), nil @@ -217,7 +222,7 @@ func createBuildpacksTars(tmpDir string, buildpacks []string, logger *logging.Lo tarFile := filepath.Join(tmpDir, fmt.Sprintf("%s.%s.tar", buildpackTOML.Buildpack.EscapedID(), version)) - if err := archive.CreateTar(tarFile, bp, filepath.Join("/buildpacks", buildpackTOML.Buildpack.EscapedID(), version), uid, gid); err != nil { + if err := archive.CreateTar(tarFile, bp, filepath.Join(buildpacksDir, buildpackTOML.Buildpack.EscapedID(), version), uid, gid); err != nil { return nil, err } @@ -254,7 +259,7 @@ func orderTar(tmpDir string, buildpacks []*lifecycle.Buildpack) (string, error) orderToml := tomlBuilder.String() err := archive.CreateSingleFileTar( filepath.Join(tmpDir, "order.tar"), - "/buildpacks/order.toml", + orderPath, orderToml, ) if err != nil { diff --git a/build/lifecycle_test.go b/build/lifecycle_test.go index 371b01e69..dd34ed6b9 100644 --- a/build/lifecycle_test.go +++ b/build/lifecycle_test.go @@ -102,28 +102,39 @@ func testLifecycle(t *testing.T, when spec.G, it spec.S) { h.AssertContains(t, outBuf.String(), "[phase] other-key=other-val") }) - it("attaches the same workspace volume to each phase", func() { - writePhase, err := lifecycle.NewPhase("phase", build.WithArgs("write", "/workspace/test.txt", "test-workspace")) + it("attaches the same layers volume to each phase", func() { + writePhase, err := lifecycle.NewPhase("phase", build.WithArgs("write", "/layers/test.txt", "test-layers")) + h.AssertNil(t, err) + assertRunSucceeds(t, writePhase, &outBuf, &errBuf) + h.AssertContains(t, outBuf.String(), "[phase] write test") + readPhase, err := lifecycle.NewPhase("phase", build.WithArgs("read", "/layers/test.txt")) + h.AssertNil(t, err) + assertRunSucceeds(t, readPhase, &outBuf, &errBuf) + h.AssertContains(t, outBuf.String(), "[phase] file contents: test-layers") + }) + + it("attaches the same app volume to each phase", func() { + writePhase, err := lifecycle.NewPhase("phase", build.WithArgs("write", "/workspace/test.txt", "test-app")) h.AssertNil(t, err) assertRunSucceeds(t, writePhase, &outBuf, &errBuf) h.AssertContains(t, outBuf.String(), "[phase] write test") readPhase, err := lifecycle.NewPhase("phase", build.WithArgs("read", "/workspace/test.txt")) h.AssertNil(t, err) assertRunSucceeds(t, readPhase, &outBuf, &errBuf) - h.AssertContains(t, outBuf.String(), "[phase] file contents: test-workspace") + h.AssertContains(t, outBuf.String(), "[phase] file contents: test-app") }) - it("copies the app into the workspace volume before the first phase", func() { - readPhase, err := lifecycle.NewPhase("phase", build.WithArgs("read", "/workspace/app/fake-app-file")) + it("copies the app into the app volume before the first phase", func() { + readPhase, err := lifecycle.NewPhase("phase", build.WithArgs("read", "/workspace/fake-app-file")) h.AssertNil(t, err) assertRunSucceeds(t, readPhase, &outBuf, &errBuf) h.AssertContains(t, outBuf.String(), "[phase] file contents: fake-app-contents") h.AssertContains(t, outBuf.String(), "[phase] file uid/gid 111/222") - deletePhase, err := lifecycle.NewPhase("phase", build.WithArgs("delete", "/workspace/app/fake-app-file")) + deletePhase, err := lifecycle.NewPhase("phase", build.WithArgs("delete", "/workspace/fake-app-file")) h.AssertNil(t, err) assertRunSucceeds(t, deletePhase, &outBuf, &errBuf) h.AssertContains(t, outBuf.String(), "[phase] delete test") - readPhase2, err := lifecycle.NewPhase("phase", build.WithArgs("read", "/workspace/app/fake-app-file")) + readPhase2, err := lifecycle.NewPhase("phase", build.WithArgs("read", "/workspace/fake-app-file")) h.AssertNil(t, err) err = readPhase2.Run(context.TODO()) readPhase2.Cleanup() @@ -309,11 +320,21 @@ func testLifecycle(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, subject.Cleanup()) }) - it("should delete the workspace volume", func() { + it("should delete the layers volume", func() { body, err := subject.Docker.VolumeList(context.TODO(), filters.NewArgs(filters.KeyValuePair{ Key: "name", - Value: subject.WorkspaceVolume, + Value: subject.LayersVolume, + })) + h.AssertNil(t, err) + h.AssertEq(t, len(body.Volumes), 0) + }) + + it("should delete the app volume", func() { + body, err := subject.Docker.VolumeList(context.TODO(), + filters.NewArgs(filters.KeyValuePair{ + Key: "name", + Value: subject.AppVolume, })) h.AssertNil(t, err) h.AssertEq(t, len(body.Volumes), 0) @@ -335,14 +356,13 @@ func testLifecycle(t *testing.T, when spec.G, it spec.S) { break } } - h.AssertEq(t, found, false) }) }) - } func assertRunSucceeds(t *testing.T, phase *build.Phase, outBuf *bytes.Buffer, errBuf *bytes.Buffer) { + t.Helper() if err := phase.Run(context.TODO()); err != nil { phase.Cleanup() t.Fatalf("Failed to run phase '%s' \n stdout: '%s' \n stderr '%s'", err, outBuf.String(), errBuf.String()) diff --git a/build/phase.go b/build/phase.go index a4e0f06d9..870bc34f3 100644 --- a/build/phase.go +++ b/build/phase.go @@ -35,7 +35,8 @@ func (l *Lifecycle) NewPhase(name string, ops ...func(*Phase) (*Phase, error)) ( } hostConf := &container.HostConfig{ Binds: []string{ - fmt.Sprintf("%s:%s:", l.WorkspaceVolume, launchDir), + fmt.Sprintf("%s:%s:", l.LayersVolume, layersDir), + fmt.Sprintf("%s:%s:", l.AppVolume, appDir), }, } ctrConf.Cmd = []string{"/lifecycle/" + name} @@ -54,7 +55,7 @@ func (l *Lifecycle) NewPhase(name string, ops ...func(*Phase) (*Phase, error)) ( for _, op := range ops { phase, err = op(phase) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "create %s phase", name) } } return phase, nil @@ -94,13 +95,13 @@ func (p *Phase) Run(context context.Context) error { return errors.Wrapf(err, "failed to create '%s' container", p.name) } p.appOnce.Do(func() { - appReader, _ := archive.CreateTarReader(p.appDir, launchDir+"/app", p.uid, p.gid) + appReader, _ := archive.CreateTarReader(p.appDir, appDir, p.uid, p.gid) if err := p.docker.CopyToContainer(context, p.ctr.ID, "/", appReader, types.CopyToContainerOptions{}); err != nil { err = errors.Wrapf(err, "failed to copy files to '%s' container", p.name) } }) if err != nil { - return err + return errors.Wrapf(err, "run %s container", p.name) } return p.docker.RunContainer( context, diff --git a/build/phases.go b/build/phases.go new file mode 100644 index 000000000..0be12c99a --- /dev/null +++ b/build/phases.go @@ -0,0 +1,117 @@ +package build + +const ( + layersDir = "/layers" + buildpacksDir = "/buildpacks" + platformDir = "/platform" + orderPath = "/buildpacks/order.toml" + groupPath = `/layers/group.toml` + planPath = "/layers/plan.toml" + appDir = "/workspace" +) + +func (l *Lifecycle) NewDetect() (*Phase, error) { + return l.NewPhase( + "detector", + WithArgs( + "-buildpacks", buildpacksDir, + "-order", orderPath, + "-group", groupPath, + "-plan", planPath, + "-app", appDir, + ), + ) +} + +func (l *Lifecycle) NewRestore(cacheImage string) (*Phase, error) { + return l.NewPhase( + "restorer", + WithDaemonAccess(), + WithArgs( + "-image", cacheImage, + "-group", groupPath, + "-layers", layersDir, + ), + ) +} + +func (l *Lifecycle) NewAnalyze(repoName string, publish bool) (*Phase, error) { + if publish { + return l.NewPhase( + "analyzer", + WithRegistryAccess(repoName), + WithArgs( + "-layers", layersDir, + "-group", groupPath, + repoName, + ), + ) + } else { + return l.NewPhase( + "analyzer", + WithDaemonAccess(), + WithArgs( + "-layers", layersDir, + "-group", groupPath, + "-daemon", + repoName, + ), + ) + } +} + +func (l *Lifecycle) NewBuild() (*Phase, error) { + return l.NewPhase( + "builder", + WithArgs( + "-buildpacks", buildpacksDir, + "-layers", layersDir, + "-app", appDir, + "-group", groupPath, + "-plan", planPath, + "-platform", platformDir, + ), + ) +} + +func (l *Lifecycle) NewExport(repoName, runImage string, publish bool) (*Phase, error) { + if publish { + return l.NewPhase( + "exporter", + WithRegistryAccess(repoName, runImage), + WithArgs( + "-image", runImage, + "-layers", layersDir, + "-app", appDir, + "-group", groupPath, + repoName, + ), + ) + } else { + return l.NewPhase( + "exporter", + WithDaemonAccess(), + WithArgs( + "-image", runImage, + "-layers", layersDir, + "-app", appDir, + "-group", groupPath, + "-daemon", + repoName, + ), + ) + } +} + + +func (l *Lifecycle) NewCache(cacheImage string) (*Phase, error) { + return l.NewPhase( + "cacher", + WithDaemonAccess(), + WithArgs( + "-image", cacheImage, + "-group", groupPath, + "-layers", layersDir, + ), + ) +}