Add initial schema, storage provider and quota manager for CockroachDB (

#2834) This creates an initial schema for CRDB by mirroring a lot of the existing constructs from the MySQL storage provider. It also adds scripts and test code to run unit and integration tests with CockRoachDB While this lead to a lot of code duplication, the intention is to address that in a separate PR in order to keep the work separated into logical batches. Doing all the refactoring to create common constructs for SQL backends would lead to a patch too big for folks to review in a reasonable time and would create more risk. Signed-off-by: Juan Antonio Osorio <[email protected]>
google · Nov 21, 2022 · ca6e62d · ca6e62d
1 parent 03dad73
commit ca6e62d
Show file tree

Hide file tree

Showing 35 changed files with 4,595 additions and 28 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -0,0 +1,71 @@
+---
+name: Test CRDB
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/setup-go@v3
+      with:
+        go-version: 1.19
+
+    - uses: golangci/golangci-lint-action@v3
+      with:
+        args: ./storage/crdb
+
+  unit-test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/setup-go@v3
+      with:
+        go-version: '1.19'
+        check-latest: true
+        cache: true
+
+    - name: Run tests
+      run: go test -v ./storage/crdb/... ./quota/crdbqm/...
+
+  integration:
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/setup-go@v3
+      with:
+        go-version: '1.19'
+        check-latest: true
+        cache: true
+
+    - name: Build before tests
+      run: go mod download && go build ./...
+
+    - name: Run CockroachDB
+      run: docker run --rm -d --name=roach -p 8080:8080 -p 26257:26257 -v "${PWD}/cockroach-data:/cockroach/cockroach-data"  cockroachdb/cockroach:latest start-single-node --insecure
+
+    - name: Wait for CockroachDB
+      uses: nick-fields/retry@v2
+      with:
+        timeout_seconds: 15
+        max_attempts: 3
+        retry_on: error
+        command: docker exec roach ./cockroach sql --insecure -e "SELECT 1"
+
+    - name: Get crdb logs
+      run: docker logs roach
+
+    - name: Run tests
+      run: ./integration/integration_test.sh
+      env:
+        TEST_COCKROACHDB_URI: postgresql://root@localhost:26257/defaultdb?sslmode=disable
+        CRDB_IN_CONTAINER: true
+        CRDB_CONTAINER_NAME: roach
diff --git a/.gitignore b/.gitignore
@@ -22,3 +22,4 @@
 /trillian_log_signer
 /trillian_map_server
 default.etcd
+cockroach-data/
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,11 @@
 
 ## HEAD
 
+### Storage
+
+A new storage driver for CockroachDB has been added. It's currently in alpha stage
+with support provided by Equinix Metal.
+
 ### Misc
 * Fix log server not exiting properly on SIGINT
 

diff --git a/cmd/trillian_log_server/main.go b/cmd/trillian_log_server/main.go
@@ -47,9 +47,11 @@ import (
 
 	// Register supported storage providers.
 	_ "github.com/google/trillian/storage/cloudspanner"
+	_ "github.com/google/trillian/storage/crdb"
 	_ "github.com/google/trillian/storage/mysql"
 
-	// Load MySQL quota provider
+	// Load quota providers
+	_ "github.com/google/trillian/quota/crdbqm"
 	_ "github.com/google/trillian/quota/mysqlqm"
 )
 
@@ -92,6 +94,7 @@ func main() {
 			klog.Exitf("Failed to load flags from config file %q: %s", *configFile, err)
 		}
 	}
+	klog.Info("**** Log Server Starting ****")
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()

diff --git a/cmd/trillian_log_signer/main.go b/cmd/trillian_log_signer/main.go
@@ -53,9 +53,11 @@ import (
 
 	// Register supported storage providers.
 	_ "github.com/google/trillian/storage/cloudspanner"
+	_ "github.com/google/trillian/storage/crdb"
 	_ "github.com/google/trillian/storage/mysql"
 
-	// Load MySQL quota provider
+	// Load quota providers
+	_ "github.com/google/trillian/quota/crdbqm"
 	_ "github.com/google/trillian/quota/mysqlqm"
 )
 

diff --git a/docs/Feature_Implementation_Matrix.md b/docs/Feature_Implementation_Matrix.md
@@ -60,6 +60,7 @@ The Log storage implementations supporting the original Trillian log.
 | Spanner          | GA      | ✓                   | Google internal-only, see CloudSpanner for external use.                    |
 | CloudSpanner    | Beta     |                     | Google maintains continuous-integration environment based on CloudSpanner.  |
 | MySQL            | GA      | ✓                   |                                                                             |
+| CockroachDB      | Alpha   |                     | Supported by [Equinix Metal](https://deploy.equinix.com/).                  |
 
 ##### Spanner
 This is a Google-internal implementation, and is used by all of Google's current Trillian deployments.
@@ -79,6 +80,12 @@ It's currently in production use by at least one CT log operator.
 
 Write throughput of 4-500 entries/s has been observed.
 
+##### CockroachDB
+
+This implementation has been tested with CockroachDB 22.1.10.
+
+It's currently in alpha mode and is not yet in production use.
+
 ### Monitoring
 
 Supported monitoring frameworks, allowing for production monitoring and alerting.
@@ -107,6 +114,7 @@ Supported frameworks for quota management.
 | etcd            | GA      | ✓                   |                                                                             |
 | MySQL           | Beta    | ?                   |                                                                             |
 | Redis           | Alpha   | ✓                   |                                                                             |
+| CockroachDB     | Alpha   |                     | Supported by [Equinix Metal](https://deploy.equinix.com/).                  |
 
 ### Key management
 

diff --git a/go.mod b/go.mod
@@ -7,6 +7,7 @@ require (
 	cloud.google.com/go/spanner v1.40.0
 	contrib.go.opencensus.io/exporter/stackdriver v0.13.12
 	github.com/apache/beam/sdks/v2 v2.0.0-20211012030016-ef4364519c94
+	github.com/cockroachdb/cockroach-go/v2 v2.2.16
 	github.com/fullstorydev/grpcurl v1.8.7
 	github.com/go-redis/redis v6.15.9+incompatible
 	github.com/go-sql-driver/mysql v1.6.0
@@ -16,6 +17,7 @@ require (
 	github.com/google/go-licenses v0.0.0-20210329231322-ce1d9163b77d
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/letsencrypt/pkcs11key/v4 v4.0.0
+	github.com/lib/pq v1.10.7
 	github.com/prometheus/client_golang v1.14.0
 	github.com/prometheus/client_model v0.3.0
 	github.com/pseudomuto/protoc-gen-doc v1.5.1
@@ -68,6 +70,7 @@ require (
 	github.com/fatih/color v1.9.0 // indirect
 	github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect
 	github.com/go-logr/logr v1.2.0 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
@@ -83,13 +86,20 @@ require (
 	github.com/huandu/xstrings v1.2.0 // indirect
 	github.com/imdario/mergo v0.3.9 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
+	github.com/jackc/pgconn v1.12.1 // indirect
+	github.com/jackc/pgio v1.0.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.3.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
+	github.com/jackc/pgtype v1.11.0 // indirect
+	github.com/jackc/pgx/v4 v4.16.1 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jhump/protoreflect v1.12.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jonboulle/clockwork v0.3.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd // indirect
-	github.com/mattn/go-colorable v0.1.4 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/miekg/pkcs11 v1.0.3 // indirect
@@ -103,6 +113,7 @@ require (
 	github.com/onsi/ginkgo v1.10.3 // indirect
 	github.com/onsi/gomega v1.7.1 // indirect
 	github.com/otiai10/copy v1.2.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/prometheus/prometheus v2.5.0+incompatible // indirect