solver: add sqlcachestorage as an alternative to the bolt db

This creates an sql cache storage that stores the cache key index in an
sqlite3 database instead of in bolt. At the moment, this is just
functional in the bare minimum as a way to test the feasibility of this
method. There are ways to configure the sqlite database for more
efficient transactions.

This can be used as an alternative storage for bolt db and is
potentially usable as a distributed cache key storage for another
implementation in the future.

Signed-off-by: Jonathan A. Sternberg <[email protected]>

cmd/buildkitd/config/config.go

@@ -40,6 +40,8 @@ type Config struct {
 	} `toml:"frontend"`
 
 	System *SystemConfig `toml:"system"`
+
+	Cache CacheConfig `toml:"cache"`
 }
 
 type SystemConfig struct {
@@ -179,3 +181,7 @@ type GatewayFrontendConfig struct {
 	Enabled             *bool    `toml:"enabled"`
 	AllowedRepositories []string `toml:"allowedRepositories"`
 }
+
+type CacheConfig struct {
+	IndexFormat string `toml:"index-format"`
+}

cmd/buildkitd/main.go

@@ -41,6 +41,7 @@ import (
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/solver/bboltcachestorage"
+	"github.com/moby/buildkit/solver/sqlcachestorage"
 	"github.com/moby/buildkit/util/apicaps"
 	"github.com/moby/buildkit/util/appcontext"
 	"github.com/moby/buildkit/util/appdefaults"
@@ -540,10 +541,16 @@ func setDefaultConfig(cfg *config.Config) {
 	if cfg.OTEL.SocketPath == "" {
 		cfg.OTEL.SocketPath = appdefaults.TraceSocketPath(isRootlessConfig())
 	}
+
+	if cfg.Cache.IndexFormat == "" {
+		cfg.Cache.IndexFormat = appdefaults.CacheIndexFormat
+	}
 }
 
-var isRootlessConfigOnce sync.Once
-var isRootlessConfigValue bool
+var (
+	isRootlessConfigOnce  sync.Once
+	isRootlessConfigValue bool
+)
 
 // isRootlessConfig is true if we should be using the rootless config
 // defaults instead of the normal defaults.
@@ -799,7 +806,7 @@ func newController(c *cli.Context, cfg *config.Config) (*control.Controller, err
 		frontends["gateway.v0"] = gwfe
 	}
 
-	cacheStorage, err := bboltcachestorage.NewStore(filepath.Join(cfg.Root, "cache.db"))
+	cacheStorage, err := newCacheKeyStorage(cfg.Root, cfg.Cache)
 	if err != nil {
 		return nil, err
 	}
@@ -962,6 +969,17 @@ func parseBoolOrAuto(s string) (*bool, error) {
 	return &b, err
 }
 
+func newCacheKeyStorage(root string, cfg config.CacheConfig) (solver.PersistentCacheKeyStorage, error) {
+	switch cfg.IndexFormat {
+	case "bolt":
+		return bboltcachestorage.NewStore(filepath.Join(root, "cache.db"))
+	case "sqlite":
+		return sqlcachestorage.NewStore(filepath.Join(root, "cache.db"))
+	default:
+		return nil, errors.Errorf("invalid cache index format: %s", cfg.IndexFormat)
+	}
+}
+
 func runTraceController(p string, exp sdktrace.SpanExporter) error {
 	server := grpc.NewServer()
 	tracev1.RegisterTraceServiceServer(server, &traceCollector{exporter: exp})

control/control.go

@@ -30,7 +30,6 @@ import (
 	"github.com/moby/buildkit/session/grpchijack"
 	containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
 	"github.com/moby/buildkit/solver"
-	"github.com/moby/buildkit/solver/bboltcachestorage"
 	"github.com/moby/buildkit/solver/llbsolver"
 	"github.com/moby/buildkit/solver/llbsolver/proc"
 	"github.com/moby/buildkit/solver/pb"
@@ -63,7 +62,7 @@ type Opt struct {
 	Entitlements              []string
 	TraceCollector            sdktrace.SpanExporter
 	HistoryDB                 db.DB
-	CacheStore                *bboltcachestorage.Store
+	CacheStore                solver.PersistentCacheKeyStorage
 	LeaseManager              *leaseutil.Manager
 	ContentStore              *containerdsnapshot.Store
 	HistoryConfig             *config.HistoryConfig
@@ -673,7 +672,7 @@ func cacheOptKey(opt controlapi.CacheOptionsEntry) (string, error) {
 	if opt.Type == "registry" && opt.Attrs["ref"] != "" {
 		return opt.Attrs["ref"], nil
 	}
-	var rawOpt = struct {
+	rawOpt := struct {
 		Type  string
 		Attrs map[string]string
 	}{

go.mod

@@ -47,6 +47,7 @@ require (
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/in-toto/in-toto-golang v0.5.0
 	github.com/klauspost/compress v1.17.9
+	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/moby/docker-image-spec v1.3.1
 	github.com/moby/locker v1.0.1

go.sum

@@ -276,6 +276,8 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=

hack/composefiles/buildkitd.toml

@@ -1,4 +1,7 @@
 debug = true
 
 [grpc]
-  debugAddress = "0.0.0.0:6060"
+debugAddress = "0.0.0.0:6060"
+
+[cache]
+index-format = "sqlite"

hack/composefiles/compose.yaml

@@ -5,7 +5,8 @@ services:
     build:
       context: ../..
       args:
-        BUILDKIT_DEBUG: 1
+        #BUILDKIT_DEBUG: 1
+        CGO_ENABLED: 1
     image: moby/buildkit:local
     ports:
       - 127.0.0.1:5000:5000

solver/bboltcachestorage/storage.go

@@ -46,6 +46,7 @@ func NewStore(dbPath string) (*Store, error) {
 	}); err != nil {
 		return nil, err
 	}
+
 	return &Store{db: db}, nil
 }
 

solver/bboltcachestorage/storage_test.go

@@ -3,7 +3,9 @@ package bboltcachestorage
 import (
 	"path/filepath"
 	"testing"
+	"time"
 
+	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/solver/testutil"
 	"github.com/stretchr/testify/require"
@@ -22,3 +24,24 @@ func TestBoltCacheStorage(t *testing.T) {
 		return st
 	})
 }
+
+func BenchmarkAddResult(b *testing.B) {
+	tmpdir := b.TempDir()
+	store, err := NewStore(filepath.Join(tmpdir, "cache.db"))
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer store.Close()
+
+	for i := 0; i < b.N; i++ {
+		id, res := newResult()
+		store.AddResult(id, res)
+	}
+}
+
+func newResult() (string, solver.CacheResult) {
+	return identity.NewID(), solver.CacheResult{
+		CreatedAt: time.Now(),
+		ID:        identity.NewID(),
+	}
+}

solver/cachemanager.go

@@ -306,7 +306,11 @@ func (c *cacheManager) Save(k *CacheKey, r Result, createdAt time.Time) (rck *Ex
 		"stack":         bklog.TraceLevelOnlyStack(),
 	})
 	defer func() {
-		lg.WithError(rerr).WithField("return_cachekey", rck.TraceFields()).Trace("cache manager")
+		l := lg.WithError(rerr)
+		if rck != nil {
+			l = l.WithField("return_cachekey", rck.TraceFields())
+		}
+		l.Trace("cache manager")
 	}()
 
 	c.mu.Lock()

solver/cachestorage.go

@@ -2,6 +2,7 @@ package solver
 
 import (
 	"context"
+	"io"
 	"time"
 
 	"github.com/moby/buildkit/session"
@@ -29,6 +30,11 @@ type CacheKeyStorage interface {
 	WalkBacklinks(id string, fn func(id string, link CacheInfoLink) error) error
 }
 
+type PersistentCacheKeyStorage interface {
+	CacheKeyStorage
+	io.Closer
+}
+
 // CacheResult is a record for a single solve result
 type CacheResult struct {
 	CreatedAt time.Time

solver/sqlcachestorage/sql.go

@@ -0,0 +1,116 @@
+package sqlcachestorage
+
+const (
+	createResultsTableSQL = `
+CREATE TABLE IF NOT EXISTS results (
+	id text NOT NULL,
+	created_at timestamp,
+	worker_ref_id text NOT NULL
+);
+	`
+
+	createResultsIndexSQL = `
+CREATE INDEX IF NOT EXISTS results_index
+ON results (id);
+	`
+
+	createResultsByRefIndexSQL = `
+CREATE INDEX IF NOT EXISTS results_by_ref_index
+ON results (worker_ref_id);
+	`
+
+	createLinksTableSQL = `
+CREATE TABLE IF NOT EXISTS links (
+	source_result_id text NOT NULL,
+	vertex_input integer DEFAULT 0,
+	vertex_output integer DEFAULT 0,
+	vertex_digest text NOT NULL,
+	vertex_selector text DEFAULT '',
+	target_result_id text NOT NULL,
+	UNIQUE (source_result_id, vertex_input, vertex_output, vertex_digest, vertex_selector, target_result_id)
+);
+	`
+
+	createUnreferencedLinksTriggerSQL = `
+CREATE TRIGGER IF NOT EXISTS unreferenced_links_sql AFTER DELETE ON results
+BEGIN
+	DELETE FROM links WHERE source_result_id = old.id OR target_result_id = old.id;
+END
+	`
+
+	createLinksIndexSQL = `
+CREATE INDEX IF NOT EXISTS links_index
+ON links (source_result_id, vertex_input, vertex_output, vertex_digest, vertex_selector, target_result_id);
+	`
+
+	createBacklinksIndexSQL = `
+CREATE INDEX IF NOT EXISTS backlinks_index
+ON links (target_result_id);
+	`
+
+	existsSQL = `
+SELECT 1 FROM results WHERE id = ? LIMIT 1;
+	`
+
+	walkSQL = `
+SELECT id FROM results;
+	`
+
+	walkResultsSQL = `
+SELECT worker_ref_id, created_at FROM results WHERE id = ?;
+	`
+
+	loadSQL = `
+SELECT worker_ref_id, created_at FROM results
+WHERE id = ? AND worker_ref_id = ?
+LIMIT 1;
+	`
+
+	addResultSQL = `
+INSERT INTO results (id, created_at, worker_ref_id)
+VALUES(?, ?, ?);
+	`
+
+	deleteResultByRefIDSQL = `
+DELETE FROM results WHERE worker_ref_id = ?;
+	`
+
+	walkIDsByResultSQL = `
+SELECT DISTINCT id FROM results WHERE worker_ref_id = ?;
+	`
+
+	addLinkSQL = `
+INSERT INTO links (source_result_id, vertex_input, vertex_output, vertex_digest, vertex_selector, target_result_id)
+VALUES (?, ?, ?, ?, ?, ?);
+	`
+
+	walkLinksSQL = `
+SELECT target_result_id FROM links
+WHERE source_result_id = ? AND vertex_input = ? AND vertex_output = ? AND vertex_digest = ? AND vertex_selector = ?;
+	`
+
+	hasLinkSQL = `
+SELECT 1 FROM links
+WHERE source_result_id = ?
+AND vertex_input = ?
+AND vertex_output = ?
+AND vertex_digest = ?
+AND vertex_selector = ?
+AND target_result_id = ?
+LIMIT 1;
+	`
+
+	walkBacklinksSQL = `
+SELECT source_result_id, vertex_input, vertex_output, vertex_digest, vertex_selector FROM links WHERE target_result_id = ?;
+	`
+)
+
+var createSQL = []string{
+	createResultsTableSQL,
+	createResultsIndexSQL,
+	createResultsByRefIndexSQL,
+	createLinksTableSQL,
+	createUnreferencedLinksTriggerSQL,
+	createLinksIndexSQL,
+	createBacklinksIndexSQL,
+}