PostgreSQL quota manager and storage backend

.github/workflows/test_pgdb.yaml

@@ -0,0 +1,70 @@
+---
+name: Test PostgreSQL
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    permissions:
+      contents: read  # for actions/checkout to fetch code
+      pull-requests: read  # for golangci/golangci-lint-action to fetch pull requests
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+
+    - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      with:
+        go-version-file: go.mod
+        check-latest: true
+        cache: true
+
+    - uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1
+      with:
+        version: 'v1.55.1'
+        args: ./storage/postgresql
+
+  integration-and-unit-tests:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
+
+    - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      with:
+        go-version-file: go.mod
+        check-latest: true
+        cache: true
+
+    - name: Build before tests
+      run: go mod download && go build ./...
+
+    - name: Run PostgreSQL
+      run: docker run --rm -d --name=pgsql -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust postgres:latest
+
+    - name: Wait for PostgreSQL
+      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
+      with:
+        timeout_seconds: 15
+        max_attempts: 3
+        retry_on: error
+        command: docker exec pgsql psql -U postgres -c "SELECT 1"
+
+    - name: Get PostgreSQL logs
+      run: docker logs pgsql 2>&1
+
+    - name: Run integration tests
+      run: ./integration/integration_test.sh
+      env:
+        TEST_POSTGRESQL_URI: postgresql:///defaultdb?host=localhost&user=postgres&password=postgres
+        POSTGRESQL_IN_CONTAINER: true
+        POSTGRESQL_CONTAINER_NAME: pgsql
+
+    - name: Run unit tests
+      run: go test -v ./storage/postgresql/... ./quota/postgresqlqm/...
+

AUTHORS

@@ -11,4 +11,5 @@
 Antonio Marcedone <[email protected]>
 Google LLC
 Internet Security Research Group
+Sectigo Limited
 Vishal Kuo <[email protected]>

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## HEAD
 
+* Add PostgreSQL quota manager and storage backend by @robstradling in https://github.com/google/trillian/pull/3644
+
 ## v1.6.1
 
 * Recommended go version for development: 1.22

CODEOWNERS

@@ -19,3 +19,4 @@
 # upgrade schema instances.
 /storage/mysql/schema/*          @mhutchinson @AlCutter
 /storage/cloudspanner/spanner.sdl @mhutchinson @AlCutter
+/storage/postgresql/schema/*     @mhutchinson @AlCutter

CONTRIBUTORS

@@ -35,5 +35,6 @@ Paul Hadfield <[email protected]> <[email protected]>
 Pavel Kalinnikov <[email protected]> <[email protected]>
 Pierre Phaneuf <[email protected]> <[email protected]>
 Rob Percival <[email protected]>
+Rob Stradling <[email protected]>
 Roger Ng <[email protected]> <[email protected]>
 Vishal Kuo <[email protected]>

cmd/trillian_log_server/main.go

@@ -49,10 +49,12 @@ import (
 	_ "github.com/google/trillian/storage/cloudspanner"
 	_ "github.com/google/trillian/storage/crdb"
 	_ "github.com/google/trillian/storage/mysql"
+	_ "github.com/google/trillian/storage/postgresql"
 
 	// Load quota providers
 	_ "github.com/google/trillian/quota/crdbqm"
 	_ "github.com/google/trillian/quota/mysqlqm"
+	_ "github.com/google/trillian/quota/postgresqlqm"
 )
 
 var (

cmd/trillian_log_signer/main.go

@@ -55,10 +55,12 @@ import (
 	_ "github.com/google/trillian/storage/cloudspanner"
 	_ "github.com/google/trillian/storage/crdb"
 	_ "github.com/google/trillian/storage/mysql"
+	_ "github.com/google/trillian/storage/postgresql"
 
 	// Load quota providers
 	_ "github.com/google/trillian/quota/crdbqm"
 	_ "github.com/google/trillian/quota/mysqlqm"
+	_ "github.com/google/trillian/quota/postgresqlqm"
 )
 
 var (

docs/Feature_Implementation_Matrix.md

@@ -61,6 +61,7 @@ The Log storage implementations supporting the original Trillian log.
 | CloudSpanner    | Beta     |                     | Google maintains continuous-integration environment based on CloudSpanner.  |
 | MySQL            | GA      | ✓                   |                                                                             |
 | CockroachDB      | Alpha   |                     | Supported by [Equinix Metal](https://deploy.equinix.com/).                  |
+| PostgreSQL       | Alpha   |                     | Supported by [Rob Stradling](https://github.com/robstradling) at [Sectigo](https://github.com/sectigo). |
 
 ##### Spanner
 This is a Google-internal implementation, and is used by all of Google's current Trillian deployments.
@@ -86,6 +87,12 @@ This implementation has been tested with CockroachDB 22.1.10.
 
 It's currently in alpha mode and is not yet in production use.
 
+##### PostgreSQL
+
+This implementation has been tested with PostgreSQL 17.0.
+
+It's currently in alpha mode and is not yet in production use.
+
 ### Monitoring
 
 Supported monitoring frameworks, allowing for production monitoring and alerting.
@@ -115,6 +122,7 @@ Supported frameworks for quota management.
 | MySQL           | Beta    | ?                   |                                                                             |
 | Redis           | Alpha   | ✓                   |                                                                             |
 | CockroachDB     | Alpha   |                     | Supported by [Equinix Metal](https://deploy.equinix.com/).                  |
+| PostgreSQL      | Alpha   |                     | Supported by [Rob Stradling](https://github.com/robstradling) at [Sectigo](https://github.com/sectigo). |
 
 ### Key management
 

go.mod

@@ -16,6 +16,8 @@ require (
 	github.com/google/go-cmp v0.6.0
 	github.com/google/go-licenses/v2 v2.0.0-alpha.1
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
+	github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438
+	github.com/jackc/pgx/v5 v5.7.1
 	github.com/letsencrypt/pkcs11key/v4 v4.0.0
 	github.com/lib/pq v1.10.9
 	github.com/prometheus/client_golang v1.20.5
@@ -107,9 +109,10 @@ require (
 	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgproto3/v2 v2.3.3 // indirect
-	github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/pgtype v1.14.3 // indirect
 	github.com/jackc/pgx/v4 v4.18.3 // indirect
+	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/jhump/protoreflect v1.16.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect
 	github.com/jonboulle/clockwork v0.4.0 // indirect

go.sum

@@ -974,6 +974,8 @@ github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8
 github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
 github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w=
 github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM=
+github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 h1:Dj0L5fhJ9F82ZJyVOmBx6msDp/kfd1t9GRfny/mfJA0=
+github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds=
 github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
 github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
 github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
@@ -995,6 +997,8 @@ github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4
 github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
 github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA=
 github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
 github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
 github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
 github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
@@ -1009,11 +1013,19 @@ github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgS
 github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw=
 github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA=
 github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw=
+github.com/jackc/pgx/v5 v5.5.2 h1:iLlpgp4Cp/gC9Xuscl7lFL1PhhW+ZLtXZcrfCt4C3tA=
+github.com/jackc/pgx/v5 v5.5.2/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
+github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs=
+github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA=
 github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0=
 github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
+github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
+github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jhump/protoreflect v1.16.0 h1:54fZg+49widqXYQ0b+usAFHbMkBGR4PpXrsHc8+TBDg=
 github.com/jhump/protoreflect v1.16.0/go.mod h1:oYPd7nPvcBw/5wlDfm/AVmU9zH9BgqGCI469pGxfj/8=
 github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 h1:liMMTbpW34dhU4az1GN0pTPADwNmvoRSeoZ6PItiqnY=

integration/functions.sh

@@ -127,6 +127,8 @@ log_prep_test() {
     yes | bash "${TRILLIAN_PATH}/scripts/resetdb.sh"
   elif [[ "${TEST_COCKROACHDB_URI}" != "" ]]; then
     yes | bash "${TRILLIAN_PATH}/scripts/resetcrdb.sh"
+  elif [[ "${TEST_POSTGRESQL_URI}" != "" ]]; then
+    yes | bash "${TRILLIAN_PATH}/scripts/resetpgdb.sh"
   fi
 
   local logserver_opts=''
@@ -139,6 +141,9 @@ log_prep_test() {
   elif [[ "${TEST_COCKROACHDB_URI}" != "" ]]; then
     logserver_opts+="--quota_system=crdb --storage_system=crdb --crdb_uri=${TEST_COCKROACHDB_URI}"
     logsigner_opts+="--quota_system=crdb --storage_system=crdb --crdb_uri=${TEST_COCKROACHDB_URI}"
+  elif [[ "${TEST_POSTGRESQL_URI}" != "" ]]; then
+    logserver_opts+="--quota_system=postgresql --storage_system=postgresql --postgresql_uri=${TEST_POSTGRESQL_URI}"
+    logsigner_opts+="--quota_system=postgresql --storage_system=postgresql --postgresql_uri=${TEST_POSTGRESQL_URI}"
   fi
 
   # Start a local etcd instance (if configured).

quota/postgresqlqm/postgresql_quota.go

@@ -0,0 +1,115 @@
+// Copyright 2024 Trillian Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package postgresqlqm defines a PostgreSQL-based quota.Manager implementation.
+package postgresqlqm
+
+import (
+	"context"
+	"errors"
+
+	"github.com/google/trillian/quota"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+const (
+	// DefaultMaxUnsequenced is a suggested value for MaxUnsequencedRows.
+	// Note that this is a Global/Write quota suggestion, so it applies across trees.
+	DefaultMaxUnsequenced = 500000 // About 2h of non-stop signing at 70QPS.
+
+	countFromExplainOutputQuery = "SELECT count_estimate($1)"
+	countFromUnsequencedQuery   = "SELECT COUNT(*) FROM Unsequenced"
+)
+
+// ErrTooManyUnsequencedRows is returned when tokens are requested but Unsequenced has grown
+// beyond the configured limit.
+var ErrTooManyUnsequencedRows = errors.New("too many unsequenced rows")
+
+// QuotaManager is a PostgreSQL-based quota.Manager implementation.
+//
+// QuotaManager only implements Global/Write quotas, which is based on the number of Unsequenced
+// rows (to be exact, tokens = MaxUnsequencedRows - actualUnsequencedRows).
+// Other quotas are considered infinite. In other words, it attempts to protect the MMD SLO of all
+// logs in the instance, but it does not make any attempt to ensure fairness, whether per-tree,
+// per-intermediate-CA (in the case of Certificate Transparency), or any other dimension.
+//
+// It has two working modes: one estimates the number of Unsequenced rows by collecting information
+// from EXPLAIN output; the other does a select count(*) on the Unsequenced table. Estimates are
+// default, even though they are approximate, as they're constant time (select count(*) on
+// PostgreSQL needs to traverse the index and may take quite a while to complete).
+// Other estimation methods exist (see https://wiki.postgresql.org/wiki/Count_estimate), but using
+// EXPLAIN output is the most accurate because it "fetches the actual current number of pages in
+// the table (this is a cheap operation, not requiring a table scan). If that is different from
+// relpages then reltuples is scaled accordingly to arrive at a current number-of-rows estimate."
+// (quoting https://www.postgresql.org/docs/current/row-estimation-examples.html)
+type QuotaManager struct {
+	DB                 *pgxpool.Pool
+	MaxUnsequencedRows int
+	UseSelectCount     bool
+}
+
+// GetTokens implements quota.Manager.GetTokens.
+// It doesn't actually reserve or retrieve tokens, instead it allows access based on the number of
+// rows in the Unsequenced table.
+func (m *QuotaManager) GetTokens(ctx context.Context, numTokens int, specs []quota.Spec) error {
+	for _, spec := range specs {
+		if spec.Group != quota.Global || spec.Kind != quota.Write {
+			continue
+		}
+		// Only allow global writes if Unsequenced is under the expected limit
+		count, err := m.countUnsequenced(ctx)
+		if err != nil {
+			return err
+		}
+		if count+numTokens > m.MaxUnsequencedRows {
+			return ErrTooManyUnsequencedRows
+		}
+	}
+	return nil
+}
+
+// PutTokens implements quota.Manager.PutTokens.
+// It's a noop for QuotaManager.
+func (m *QuotaManager) PutTokens(ctx context.Context, numTokens int, specs []quota.Spec) error {
+	return nil
+}
+
+// ResetQuota implements quota.Manager.ResetQuota.
+// It's a noop for QuotaManager.
+func (m *QuotaManager) ResetQuota(ctx context.Context, specs []quota.Spec) error {
+	return nil
+}
+
+func (m *QuotaManager) countUnsequenced(ctx context.Context) (int, error) {
+	if m.UseSelectCount {
+		return countFromTable(ctx, m.DB)
+	}
+	return countFromExplainOutput(ctx, m.DB)
+}
+
+func countFromExplainOutput(ctx context.Context, db *pgxpool.Pool) (int, error) {
+	var count int
+	if err := db.QueryRow(ctx, countFromExplainOutputQuery, "Unsequenced").Scan(&count); err != nil {
+		return 0, err
+	}
+	return count, nil
+}
+
+func countFromTable(ctx context.Context, db *pgxpool.Pool) (int, error) {
+	var count int
+	if err := db.QueryRow(ctx, countFromUnsequencedQuery).Scan(&count); err != nil {
+		return 0, err
+	}
+	return count, nil
+}