SA: Add and use revokedCertificates table (#7095)

Add a new "revokedCertificates" table to the database schema. This table
is similar to the existing "certificateStatus" table in many ways, but
the idea is that it will only have rows added to it when certificates
are revoked, not when they're issued. Thus, it will grow many orders of
magnitude slower than the certificateStatus table does. Eventually, it
will replace that table entirely.

The one column that revokedCertificates adds is the new "ShardIdx"
column, which is the CRL shard in which the revoked certificate will
appear. This way we can assign certificates to CRL shards at the time
they are revoked, and guarantee that they will never move to a different
shard even if we change the number of shards we produce. This will
eventually allow us to put CRL URLs directly into our certificates,
replacing OCSP URLs.

Add new logic to the SA's RevokeCertificate and UpdateRevokedCertificate
methods to handle this new table. If these methods receive a request
which specifies a CRL shard (our CRL shards are 1-indexed, so shard 0
does not exist), then they will ensure that the new revocation status is
written into both the certificateStatus and revokedCertificates tables.
This logic will not function until the RA is updated to take advantage
of it, so it is not a risk for it to appear in Boulder before the new
table has been created.

Also add new logic to the SA's GetRevokedCertificates method. Similar to
the above, this reads from the new table if the ShardIdx field is
supplied in the request message. This code will not operate until the
crl-updater is updated to include this field. We will not perform this
update for a minimum of 100 days after this code is deployed, to ensure
that all unexpired revoked certificates are present in the
revokedCertificates table.

Part of #7094

sa/database.go

@@ -282,6 +282,7 @@ func initTables(dbMap *borp.DbMap) {
 	dbMap.AddTableWithName(incidentModel{}, "incidents").SetKeys(true, "ID")
 	dbMap.AddTable(incidentSerialModel{})
 	dbMap.AddTableWithName(crlShardModel{}, "crlShards").SetKeys(true, "ID")
+	dbMap.AddTableWithName(revokedCertModel{}, "revokedCertificates").SetKeys(true, "ID")
 
 	// Read-only maps used for selecting subsets of columns.
 	dbMap.AddTableWithName(CertStatusMetadata{}, "certificateStatus")

sa/db-next/boulder_sa/20230919000000_RevokedCertificates.sql

@@ -0,0 +1,21 @@
+-- +migrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+
+CREATE TABLE `revokedCertificates` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `issuerID` bigint(20) NOT NULL,
+  `serial` varchar(255) NOT NULL,
+  `notAfterHour` datetime NOT NULL,
+  `shardIdx` bigint(20) NOT NULL,
+  `revokedDate` datetime NOT NULL,
+  `revokedReason` int(11) NOT NULL,
+  PRIMARY KEY (`id`),
+  KEY `issuerID_shardIdx_notAfterHour_idx` (`issuerID`, `shardIdx`, `notAfterHour`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+ PARTITION BY RANGE(id)
+(PARTITION p_start VALUES LESS THAN (MAXVALUE));
+
+-- +migrate Down
+-- SQL section 'Down' is executed when this migration is rolled back
+
+DROP TABLE `revokedCertificates`;

sa/db-users/boulder_sa.sql

@@ -33,6 +33,7 @@ GRANT SELECT,INSERT ON blockedKeys TO 'sa'@'localhost';
 GRANT SELECT,INSERT,UPDATE ON newOrdersRL TO 'sa'@'localhost';
 GRANT SELECT ON incidents TO 'sa'@'localhost';
 GRANT SELECT,INSERT,UPDATE ON crlShards TO 'sa'@'localhost';
+GRANT SELECT,INSERT,UPDATE ON revokedCertificates TO 'sa'@'localhost';
 
 GRANT SELECT ON certificates TO 'sa_ro'@'localhost';
 GRANT SELECT ON certificateStatus TO 'sa_ro'@'localhost';
@@ -52,6 +53,7 @@ GRANT SELECT ON blockedKeys TO 'sa_ro'@'localhost';
 GRANT SELECT ON newOrdersRL TO 'sa_ro'@'localhost';
 GRANT SELECT ON incidents TO 'sa_ro'@'localhost';
 GRANT SELECT ON crlShards TO 'sa_ro'@'localhost';
+GRANT SELECT ON revokedCertificates TO 'sa_ro'@'localhost';
 
 -- OCSP Responder
 GRANT SELECT ON certificateStatus TO 'ocsp_resp'@'localhost';

sa/model.go

@@ -1179,3 +1179,16 @@ type crlShardModel struct {
 	NextUpdate  *time.Time `db:"nextUpdate"`
 	LeasedUntil time.Time  `db:"leasedUntil"`
 }
+
+// revokedCertModel represents one row in the revokedCertificates table. It
+// contains all of the information necessary to populate a CRL entry or OCSP
+// response for the indicated certificate.
+type revokedCertModel struct {
+	ID            int64             `db:"id"`
+	IssuerID      int64             `db:"issuerID"`
+	Serial        string            `db:"serial"`
+	NotAfterHour  time.Time         `db:"notAfterHour"`
+	ShardIdx      int64             `db:"shardIdx"`
+	RevokedDate   time.Time         `db:"revokedDate"`
+	RevokedReason revocation.Reason `db:"revokedReason"`
+}

sa/proto/sa.proto

@@ -291,6 +291,7 @@ message RevokeCertificateRequest {
   int64 backdateNS = 5; // Unix timestamp (nanoseconds)
   bytes response = 4;
   int64 issuerID = 6;
+  int64 shardIdx = 7;
 }
 
 message FinalizeAuthorizationRequest {
@@ -343,6 +344,7 @@ message GetRevokedCertsRequest {
   int64 expiresAfterNS = 2; // Unix timestamp (nanoseconds), inclusive
   int64 expiresBeforeNS = 3; // Unix timestamp (nanoseconds), exclusive
   int64 revokedBeforeNS = 4; // Unix timestamp (nanoseconds)
+  int64 shardIdx = 5; // Must not be set until the revokedCertificates table has 90+ days of entries.
 }
 
 message RevocationStatus {

sa/sa.go

@@ -787,38 +787,89 @@ func (ssa *SQLStorageAuthority) FinalizeAuthorization2(ctx context.Context, req
 	return &emptypb.Empty{}, nil
 }
 
+// addRevokedCertificate is a helper used by both RevokeCertificate and
+// UpdateRevokedCertificate. It inserts a new row into the revokedCertificates
+// table based on the contents of the input request. The second argument must be
+// a transaction object so that it is safe to conduct multiple queries with a
+// consistent view of the database. It must only be called when the request
+// specifies a non-zero ShardIdx.
+func addRevokedCertificate(ctx context.Context, tx db.Executor, req *sapb.RevokeCertificateRequest, revokedDate time.Time) error {
+	if req.ShardIdx == 0 {
+		return errors.New("cannot add revoked certificate with shard index 0")
+	}
+
+	var serial struct {
+		Expires time.Time
+	}
+	err := tx.SelectOne(
+		ctx, &serial, `SELECT expires FROM serials WHERE serial = ?`, req.Serial)
+	if err != nil {
+		return fmt.Errorf("retrieving revoked certificate expiration: %w", err)
+	}
+
+	err = tx.Insert(ctx, &revokedCertModel{
+		IssuerID:      req.IssuerID,
+		Serial:        req.Serial,
+		ShardIdx:      req.ShardIdx,
+		RevokedDate:   revokedDate,
+		RevokedReason: revocation.Reason(req.Reason),
+		// Round the notAfter up to the next hour, to reduce index size while still
+		// ensuring we correctly serve revocation info past the actual expiration.
+		NotAfterHour: serial.Expires.Add(time.Hour).Truncate(time.Hour),
+	})
+	if err != nil {
+		return fmt.Errorf("inserting revoked certificate row: %w", err)
+	}
+
+	return nil
+}
+
 // RevokeCertificate stores revocation information about a certificate. It will only store this
 // information if the certificate is not already marked as revoked.
 func (ssa *SQLStorageAuthority) RevokeCertificate(ctx context.Context, req *sapb.RevokeCertificateRequest) (*emptypb.Empty, error) {
-	if req.Serial == "" || req.DateNS == 0 {
+	if req.Serial == "" || req.DateNS == 0 || req.IssuerID == 0 {
 		return nil, errIncompleteRequest
 	}
 
-	revokedDate := time.Unix(0, req.DateNS)
+	_, overallError := db.WithTransaction(ctx, ssa.dbMap, func(tx db.Executor) (interface{}, error) {
+		revokedDate := time.Unix(0, req.DateNS)
 
-	res, err := ssa.dbMap.ExecContext(ctx,
-		`UPDATE certificateStatus SET
+		res, err := tx.ExecContext(ctx,
+			`UPDATE certificateStatus SET
 				status = ?,
 				revokedReason = ?,
 				revokedDate = ?,
 				ocspLastUpdated = ?
 			WHERE serial = ? AND status != ?`,
-		string(core.OCSPStatusRevoked),
-		revocation.Reason(req.Reason),
-		revokedDate,
-		revokedDate,
-		req.Serial,
-		string(core.OCSPStatusRevoked),
-	)
-	if err != nil {
-		return nil, err
-	}
-	rows, err := res.RowsAffected()
-	if err != nil {
-		return nil, err
-	}
-	if rows == 0 {
-		return nil, berrors.AlreadyRevokedError("no certificate with serial %s and status other than %s", req.Serial, string(core.OCSPStatusRevoked))
+			string(core.OCSPStatusRevoked),
+			revocation.Reason(req.Reason),
+			revokedDate,
+			revokedDate,
+			req.Serial,
+			string(core.OCSPStatusRevoked),
+		)
+		if err != nil {
+			return nil, err
+		}
+		rows, err := res.RowsAffected()
+		if err != nil {
+			return nil, err
+		}
+		if rows == 0 {
+			return nil, berrors.AlreadyRevokedError("no certificate with serial %s and status other than %s", req.Serial, string(core.OCSPStatusRevoked))
+		}
+
+		if req.ShardIdx != 0 {
+			err = addRevokedCertificate(ctx, tx, req, revokedDate)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		return nil, nil
+	})
+	if overallError != nil {
+		return nil, overallError
 	}
 
 	return &emptypb.Empty{}, nil
@@ -829,39 +880,80 @@ func (ssa *SQLStorageAuthority) RevokeCertificate(ctx context.Context, req *sapb
 // cert is already revoked, if the new revocation reason is `KeyCompromise`,
 // and if the revokedDate is identical to the current revokedDate.
 func (ssa *SQLStorageAuthority) UpdateRevokedCertificate(ctx context.Context, req *sapb.RevokeCertificateRequest) (*emptypb.Empty, error) {
-	if req.Serial == "" || req.DateNS == 0 || req.BackdateNS == 0 {
+	if req.Serial == "" || req.DateNS == 0 || req.BackdateNS == 0 || req.IssuerID == 0 {
 		return nil, errIncompleteRequest
 	}
 	if req.Reason != ocsp.KeyCompromise {
 		return nil, fmt.Errorf("cannot update revocation for any reason other than keyCompromise (1); got: %d", req.Reason)
 	}
 
-	thisUpdate := time.Unix(0, req.DateNS)
-	revokedDate := time.Unix(0, req.BackdateNS)
+	_, overallError := db.WithTransaction(ctx, ssa.dbMap, func(tx db.Executor) (interface{}, error) {
+		thisUpdate := time.Unix(0, req.DateNS)
+		revokedDate := time.Unix(0, req.BackdateNS)
 
-	res, err := ssa.dbMap.ExecContext(ctx,
-		`UPDATE certificateStatus SET
-				revokedReason = ?,
-				ocspLastUpdated = ?
-			WHERE serial = ? AND status = ? AND revokedReason != ? AND revokedDate = ?`,
-		revocation.Reason(ocsp.KeyCompromise),
-		thisUpdate,
-		req.Serial,
-		string(core.OCSPStatusRevoked),
-		revocation.Reason(ocsp.KeyCompromise),
-		revokedDate,
-	)
-	if err != nil {
-		return nil, err
-	}
-	rows, err := res.RowsAffected()
-	if err != nil {
-		return nil, err
-	}
-	if rows == 0 {
-		// InternalServerError because we expected this certificate status to exist,
-		// to already be revoked for a different reason, and to have a matching date.
-		return nil, berrors.InternalServerError("no certificate with serial %s and revoked reason other than keyCompromise", req.Serial)
+		res, err := tx.ExecContext(ctx,
+			`UPDATE certificateStatus SET
+					revokedReason = ?,
+					ocspLastUpdated = ?
+				WHERE serial = ? AND status = ? AND revokedReason != ? AND revokedDate = ?`,
+			revocation.Reason(ocsp.KeyCompromise),
+			thisUpdate,
+			req.Serial,
+			string(core.OCSPStatusRevoked),
+			revocation.Reason(ocsp.KeyCompromise),
+			revokedDate,
+		)
+		if err != nil {
+			return nil, err
+		}
+		rows, err := res.RowsAffected()
+		if err != nil {
+			return nil, err
+		}
+		if rows == 0 {
+			// InternalServerError because we expected this certificate status to exist,
+			// to already be revoked for a different reason, and to have a matching date.
+			return nil, berrors.InternalServerError("no certificate with serial %s and revoked reason other than keyCompromise", req.Serial)
+		}
+
+		// Only update the revokedCertificates table if the revocation request
+		// specifies the CRL shard that this certificate belongs in. Our shards are
+		// one-indexed, so a ShardIdx of zero means no value was set.
+		if req.ShardIdx != 0 {
+			var rcm revokedCertModel
+			// Note: this query MUST be updated to enforce the same preconditions as
+			// the "UPDATE certificateStatus SET revokedReason..." above if this
+			// query ever becomes the first or only query in this transaction. We are
+			// currently relying on the query above to exit early if the certificate
+			// does not have an appropriate status.
+			err = tx.SelectOne(
+				ctx, &rcm, `SELECT * FROM revokedCertificates WHERE serial = ?`, req.Serial)
+			if db.IsNoRows(err) {
+				// TODO: Remove this fallback codepath once we know that all unexpired
+				// certs marked as revoked in the certificateStatus table have
+				// corresponding rows in the revokedCertificates table. That should be
+				// 90+ days after the RA starts sending ShardIdx in its
+				// RevokeCertificateRequest messages.
+				err = addRevokedCertificate(ctx, tx, req, revokedDate)
+				if err != nil {
+					return nil, err
+				}
+				return nil, nil
+			} else if err != nil {
+				return nil, fmt.Errorf("retrieving revoked certificate row: %w", err)
+			}
+
+			rcm.RevokedReason = revocation.Reason(ocsp.KeyCompromise)
+			_, err = tx.Update(ctx, &rcm)
+			if err != nil {
+				return nil, fmt.Errorf("updating revoked certificate row: %w", err)
+			}
+		}
+
+		return nil, nil
+	})
+	if overallError != nil {
+		return nil, overallError
 	}
 
 	return &emptypb.Empty{}, nil