Support for building overlaybd images

Signed-off-by: Haoqi Miao <[email protected]>

cache/blobs.go

@@ -1,11 +1,17 @@
 package cache
 
 import (
+	"bufio"
 	"context"
 	"fmt"
+	"io"
 	"os"
+	"path"
 	"strconv"
 
+	obdlabel "github.com/containerd/accelerated-container-image/pkg/label"
+	obdcmd "github.com/containerd/accelerated-container-image/pkg/utils"
+	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/diff"
 	"github.com/containerd/containerd/diff/walking"
 	"github.com/containerd/containerd/leases"
@@ -96,7 +102,17 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool
 
 				compressorFunc, finalize := comp.Type.Compress(ctx, comp)
 				mediaType := comp.Type.MediaType()
-
+				var isOverlaybd = false
+				if sr.cm.Snapshotter.Name() == "overlaybd" {
+					snStat, err := sr.cm.Snapshotter.Stat(ctx, sr.getSnapshotID())
+					if err != nil {
+						return nil, err
+					}
+					if snStat.Labels[obdlabel.LocalOverlayBDPath] != "" {
+						isOverlaybd = true
+						mediaType = ocispecs.MediaTypeImageLayer
+					}
+				}
 				var lowerRef *immutableRef
 				switch sr.kind() {
 				case Diff:
@@ -162,7 +178,7 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool
 						// (in which case lower and upper may differ by more than one layer), so print warn log on unexpected
 						// failure.
 						logWarnOnErr = sr.kind() != Diff
-					case "fuse-overlayfs", "native":
+					case "fuse-overlayfs", "native", "overlaybd":
 						// not supported with fuse-overlayfs snapshotter which doesn't provide overlayfs mounts.
 						// TODO: add support for fuse-overlayfs
 						enableOverlay = false
@@ -188,6 +204,13 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool
 					}
 				}
 
+				if isOverlaybd {
+					if err := commitOverlaybd(ctx, sr, &desc); err != nil {
+						return nil, err
+					}
+					desc.MediaType = mediaType
+				}
+
 				if desc.Digest == "" && !isTypeWindows(sr) && comp.Type.NeedsComputeDiffBySelf() {
 					// These compression types aren't supported by containerd differ. So try to compute diff on buildkit side.
 					// This case can be happen on containerd worker + non-overlayfs snapshotter (e.g. native).
@@ -462,3 +485,45 @@ func ensureCompression(ctx context.Context, ref *immutableRef, comp compression.
 	})
 	return err
 }
+
+func commitOverlaybd(ctx context.Context, sr *immutableRef, desc *ocispecs.Descriptor) error {
+	snStat, err := sr.cm.Snapshotter.Stat(ctx, sr.getSnapshotID())
+	if err != nil {
+		return errors.Wrapf(err, "failed to Stat overlaybd")
+	}
+	dir := path.Dir(snStat.Labels[obdlabel.LocalOverlayBDPath])
+	commitPath := path.Join(dir, "overlaybd.commit")
+	err = obdcmd.Commit(ctx, dir, dir, true, "-t", "-z")
+	if err != nil {
+		return errors.Wrapf(err, "failed to overlaybd-commit")
+	}
+	cw, err := sr.cm.ContentStore.Writer(ctx, content.WithRef(sr.ID()))
+	if err != nil {
+		return errors.Wrapf(err, "failed to open writer")
+	}
+	fi, err := os.Open(commitPath)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open overlaybd commit file")
+	}
+	sz, err := io.Copy(cw, bufio.NewReader(fi))
+	if err != nil {
+		return errors.Wrapf(err, "failed to do io.Copy()")
+	}
+	dgst := cw.Digest()
+	labels := map[string]string{
+		containerdUncompressed:       dgst.String(),
+		obdlabel.OverlayBDBlobDigest: dgst.String(),
+		obdlabel.OverlayBDBlobSize:   fmt.Sprintf("%d", sz),
+	}
+	err = cw.Commit(ctx, sz, dgst, content.WithLabels(labels))
+	if err != nil {
+		return errors.Wrapf(err, "failed to do cw.Commit")
+	}
+	desc.Digest = dgst
+	desc.Size = sz
+	desc.Annotations = map[string]string{
+		obdlabel.OverlayBDBlobDigest: string(desc.Digest),
+		obdlabel.OverlayBDBlobSize:   fmt.Sprintf("%d", desc.Size),
+	}
+	return nil
+}

cache/manager.go

@@ -8,12 +8,14 @@ import (
 	"sync"
 	"time"
 
+	obdlabel "github.com/containerd/accelerated-container-image/pkg/label"
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/diff"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/filters"
 	"github.com/containerd/containerd/gc"
 	"github.com/containerd/containerd/leases"
+	"github.com/containerd/containerd/snapshots"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/moby/buildkit/cache/metadata"
 	"github.com/moby/buildkit/client"
@@ -615,6 +617,10 @@ func (cm *cacheManager) New(ctx context.Context, s ImmutableRef, sess session.Gr
 		}); rerr != nil {
 			return nil, rerr
 		}
+	} else if cm.Snapshotter.Name() == "overlaybd" && parent != nil {
+		// Snapshotter will create a R/W block device directly as rootfs with this label
+		rwLabels := map[string]string{obdlabel.SupportReadWriteMode: "dev"}
+		err = cm.Snapshotter.Prepare(ctx, snapshotID, parentSnapshotID, snapshots.WithLabels(rwLabels))
 	} else {
 		err = cm.Snapshotter.Prepare(ctx, snapshotID, parentSnapshotID)
 	}

cache/refs.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 	"time"
 
+	obdlabel "github.com/containerd/accelerated-container-image/pkg/label"
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
@@ -39,7 +40,7 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
-var additionalAnnotations = append(compression.EStargzAnnotations, containerdUncompressed)
+var additionalAnnotations = append(append(compression.EStargzAnnotations, obdlabel.OverlayBDAnnotations...), containerdUncompressed)
 
 // Ref is a reference to cacheable objects.
 type Ref interface {
@@ -1010,6 +1011,10 @@ func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr erro
 			return err
 		}
 		return rerr
+	} else if sr.cm.Snapshotter.Name() == "overlaybd" {
+		if rerr = sr.prepareRemoteSnapshotsOverlaybdMode(ctx, s); rerr == nil {
+			return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true)
+		}
 	}
 
 	return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true)
@@ -1126,6 +1131,60 @@ func (sr *immutableRef) prepareRemoteSnapshotsStargzMode(ctx context.Context, s
 	return err
 }
 
+func (sr *immutableRef) prepareRemoteSnapshotsOverlaybdMode(ctx context.Context, s session.Group) error {
+	_, err := sr.sizeG.Do(ctx, sr.ID()+"-prepare-remote-snapshot", func(ctx context.Context) (_ interface{}, rerr error) {
+		dhs := sr.descHandlers
+		for _, r := range sr.layerChain() {
+			r := r
+			snapshotID := r.getSnapshotID()
+			if _, err := r.cm.Snapshotter.Stat(ctx, snapshotID); err == nil {
+				continue
+			}
+
+			dh := dhs[digest.Digest(r.getBlob())]
+			if dh == nil {
+				// We cannot prepare remote snapshots without descHandler.
+				return nil, nil
+			}
+
+			defaultLabels := snapshots.FilterInheritedLabels(dh.SnapshotLabels)
+			if defaultLabels == nil {
+				defaultLabels = make(map[string]string)
+			}
+			defaultLabels["containerd.io/snapshot.ref"] = snapshotID
+
+			// Prepare remote snapshots
+			var (
+				key  = fmt.Sprintf("tmp-%s %s", identity.NewID(), r.getChainID())
+				opts = []snapshots.Opt{
+					snapshots.WithLabels(defaultLabels),
+				}
+			)
+			parentID := ""
+			if r.layerParent != nil {
+				parentID = r.layerParent.getSnapshotID()
+			}
+			if err := r.cm.Snapshotter.Prepare(ctx, key, parentID, opts...); err != nil {
+				if errdefs.IsAlreadyExists(err) {
+					// Check if the targeting snapshot ID has been prepared as
+					// a remote snapshot in the snapshotter.
+					_, err := r.cm.Snapshotter.Stat(ctx, snapshotID)
+					if err == nil { // usable as remote snapshot without unlazying.
+						// Try the next layer as well.
+						continue
+					}
+				}
+			}
+
+			// This layer and all upper layers cannot be prepared without unlazying.
+			break
+		}
+
+		return nil, nil
+	})
+	return err
+}
+
 func makeTmpLabelsStargzMode(labels map[string]string, s session.Group) (fields []string, res map[string]string) {
 	res = make(map[string]string)
 	// Append unique ID to labels for avoiding collision of labels among calls
@@ -1284,7 +1343,12 @@ func (sr *immutableRef) unlazyLayer(ctx context.Context, dhs DescHandlers, pg pr
 
 	key := fmt.Sprintf("extract-%s %s", identity.NewID(), sr.getChainID())
 
-	err = sr.cm.Snapshotter.Prepare(ctx, key, parentID)
+	if sr.cm.Snapshotter.Name() == "overlaybd" {
+		err = sr.cm.Snapshotter.Prepare(ctx, key, parentID,
+			snapshots.WithLabels(map[string]string{"containerd.io/snapshot.ref": string(desc.Digest)}))
+	} else {
+		err = sr.cm.Snapshotter.Prepare(ctx, key, parentID)
+	}
 	if err != nil {
 		return err
 	}

docs/overlaybd.md

@@ -0,0 +1,40 @@
+# Overlaybd
+
+[Overlaybd](https://github.com/containerd/overlaybd) is a novel layering block-level image format, which is design for container, secure container and applicable to virtual machine. And it is an open-source implementation of paper [DADI: Block-Level Image Service for Agile and Elastic Application Deployment. USENIX ATC'20"](https://www.usenix.org/conference/atc20/presentation/li-huiba).
+
+## Build Overlaybd Images
+
+Before building overlaybd images, ensure that `overlaybd-tcmu` and `overlaybd-snapshotter` are active by referring to the [QUICKSTART](https://github.com/containerd/accelerated-container-image/blob/main/docs/QUICKSTART.md#install) guide.
+
+To use buildkit to build overlaybd images, you should specify `--oci-worker-snapshotter=overlaybd` and `--oci-worker-proxy-snapshotter-path=/run/overlaybd-snapshotter/overlaybd.sock` when start buildkitd:
+
+```bash
+buildkitd  --oci-worker-snapshotter=overlaybd  --oci-worker-proxy-snapshotter-path=/run/overlaybd-snapshotter/overlaybd.sock
+```
+
+After that, you can build a new overlaybd image from an existing overlaybd image. It is essential to include `--oci-mediatypes=true` and `--compression=uncompressed` while running buildctl:
+
+```bash
+buildctl build ... \
+--output type=image,name=<new image name>,push=true,oci-mediatypes=true,compression=uncompressed
+```
+
+## Performance
+
+In our test case Dockerfile, we used a 5GB OCI image (and corresponding overlaybd format), wrote some new layers of identical size, and recorded the time cost of image pull (as **pull** in the table below), building all lines in Dockerfile (as **build**), and exporting to image and pushing (as **push**).
+
+OCI:
+
+| **size per layer** | **layers** | **pull** | **build** | **push** | **total**  |
+| -------- | ---- | ---- | ----- | ---- | ---- |
+| 4GB      | 1    | 105.7| 23.5  | 219.4| 348.6|
+| 1GB      | 4    | 88.5 | 34.0  | 123.8| 246.3|
+| 256MB    | 10   | 92.1 | 20.7  | 63.6 | 176.4|
+
+Overlaybd:
+
+| **size per layer** | **layers** | **pull** | **build** | **push** | **total**  |
+| -------- | ---- | ---- | ----- | ---- | ---- |
+| 4GB      | 1    | 0.9  | 21.5  | 166.2| 188.6|
+| 1GB      | 4    | 0.9	 | 24.9	 | 72.9 | 98.7 |
+| 256MB    | 10   | 0.7  | 18.4  | 48.9 | 68.0 |

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.56
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.30.6
 	github.com/aws/smithy-go v1.13.5
+	github.com/containerd/accelerated-container-image v0.6.5
 	github.com/containerd/console v1.0.3
 	github.com/containerd/containerd v1.7.1
 	github.com/containerd/continuity v0.4.1

go.sum

@@ -285,6 +285,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE=
+github.com/containerd/accelerated-container-image v0.6.5 h1:XArcWyUbQ4pD7f9rh3qDQCdyu85ezdSvTZHvNb74VsQ=
+github.com/containerd/accelerated-container-image v0.6.5/go.mod h1:RZB4r386ZnUEuYdwJplVoI4avEE+oxWB75uOB3quctA=
 github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
 github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
 github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=

source/containerimage/pull.go

@@ -307,6 +307,7 @@ func (p *puller) CacheKey(ctx context.Context, g session.Group, index int) (cach
 				for k, v := range estargz.SnapshotLabels(p.manifest.Ref, p.manifest.Descriptors, i) {
 					labels[k] = v
 				}
+				labels["containerd.io/snapshot/image-ref"] = p.manifest.Ref
 				p.descHandlers[desc.Digest] = &cache.DescHandler{
 					Provider:       p.manifest.Provider,
 					Progress:       progressController,