Structure of ACME Tidy (hashicorp#20494)

* Structure of ACME Tidy.

* The tidy endpoints/call information.

* Counts for status plumbing.

* Update typo calls, add information saving date of account creation.

* Missed some field locations.

* Set-up of Tidy command.

* Proper tidy function; lock to work with

* Remove order safety buffer.

* Missed a field.

* Read lock for account creation; Write lock for tidy (account deletion)

* Type issues fixed.

* fix range operator.

* Fix path_tidy read.

* Add fields to auto-tidy config.

* Add (and standardize) Tidy Config Response

* Test pass, consistent fields

* Changes from PR-Reviews.

* Update test to updated default due to PR-Review.

builtin/logical/pki/acme_state.go

@@ -217,6 +217,8 @@ type acmeAccount struct {
 	TermsOfServiceAgreed bool              `json:"termsOfServiceAgreed"`
 	Jwk                  []byte            `json:"jwk"`
 	AcmeDirectory        string            `json:"acme-directory"`
+	AccountCreatedDate   time.Time         `json:"account_created_date"`
+	AccountRevokedDate   time.Time         `json:"account_revoked_date"`
 }
 
 type acmeOrder struct {
@@ -288,6 +290,7 @@ func (a *acmeState) CreateAccount(ac *acmeContext, c *jwsCtx, contact []string,
 		Jwk:                  c.Jwk,
 		Status:               StatusValid,
 		AcmeDirectory:        ac.acmeDirectory,
+		AccountCreatedDate:   time.Now(),
 	}
 	json, err := logical.StorageEntryJSON(acmeAccountPrefix+c.Kid, acct)
 	if err != nil {

builtin/logical/pki/backend.go

@@ -345,7 +345,9 @@ type backend struct {
 	issuersLock sync.RWMutex
 
 	// Context around ACME operations
-	acmeState *acmeState
+	acmeState       *acmeState
+	acmeAccountLock sync.RWMutex // (Write) Locked on Tidy, (Read) Locked on Account Creation
+	// TODO: Stress test this - eg. creating an order while an account is being revoked
 }
 
 type roleOperation func(ctx context.Context, req *logical.Request, data *framework.FieldData, role *roleEntry) (*logical.Response, error)

builtin/logical/pki/backend_test.go

@@ -4030,6 +4030,12 @@ func TestBackend_RevokePlusTidy_Intermediate(t *testing.T) {
 			"revocation_queue_deleted_count":        json.Number("0"),
 			"cross_revoked_cert_deleted_count":      json.Number("0"),
 			"internal_backend_uuid":                 backendUUID,
+			"tidy_acme":                             false,
+			"acme_account_safety_buffer":            json.Number("2592000"),
+			"acme_orders_deleted_count":             json.Number("0"),
+			"acme_account_revoked_count":            json.Number("0"),
+			"acme_account_deleted_count":            json.Number("0"),
+			"total_acme_account_count":              json.Number("0"),
 		}
 		// Let's copy the times from the response so that we can use deep.Equal()
 		timeStarted, ok := tidyStatus.Data["time_started"]

builtin/logical/pki/fields.go

@@ -491,6 +491,23 @@ this removes ALL issuers within the mount (and is thus not desirable
 in most operational scenarios).`,
 	}
 
+	fields["tidy_acme"] = &framework.FieldSchema{
+		Type: framework.TypeBool,
+		Description: `Set to true to enable tidying ACME accounts,
+orders and authorizations.  ACME orders are tidied (deleted) 
+safety_buffer after the certificate associated with them expires,
+or after the order and relevant authorizations have expired if no 
+certificate was produced.  Authorizations are tidied with the 
+corresponding order.
+
+When a valid ACME Account is at least acme_account_safety_buffer
+old, and has no remaining orders associated with it, the account is
+marked as revoked.  After another acme_account_safety_buffer has 
+passed from the revocation or deactivation date, a revoked or 
+deactivated ACME account is deleted.`,
+		Default: false,
+	}
+
 	fields["safety_buffer"] = &framework.FieldSchema{
 		Type: framework.TypeDurationSecond,
 		Description: `The amount of extra time that must have passed
@@ -509,6 +526,14 @@ Defaults to 8760 hours (1 year).`,
 		Default: int(defaultTidyConfig.IssuerSafetyBuffer / time.Second), // TypeDurationSecond currently requires defaults to be int
 	}
 
+	fields["acme_account_safety_buffer"] = &framework.FieldSchema{
+		Type: framework.TypeDurationSecond,
+		Description: `The amount of time that must pass after creation
+that an account with no orders is marked revoked, and the amount of time
+after being marked revoked or dea`,
+		Default: int(defaultTidyConfig.AcmeAccountSafetyBuffer / time.Second), // TypeDurationSecond currently requires defaults to be int
+	}
+
 	fields["pause_duration"] = &framework.FieldSchema{
 		Type: framework.TypeString,
 		Description: `The amount of time to wait between processing

builtin/logical/pki/path_acme_account.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
+	"time"
 
 	"github.com/hashicorp/go-secure-stdlib/strutil"
 
@@ -236,6 +237,8 @@ func (b *backend) acmeNewAccountCreateHandler(acmeCtx *acmeContext, userCtx *jws
 	//	return nil, fmt.Errorf("terms of service not agreed to: %w", ErrUserActionRequired)
 	//}
 
+	b.acmeAccountLock.RLock() // Prevents Account Creation and Tidy Interfering
+	defer b.acmeAccountLock.RUnlock()
 	accountByKid, err := b.acmeState.CreateAccount(acmeCtx, userCtx, contact, termsOfServiceAgreed)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create account: %w", err)
@@ -282,6 +285,7 @@ func (b *backend) acmeNewAccountUpdateHandler(acmeCtx *acmeContext, userCtx *jws
 		// TODO: This should cancel any ongoing operations (do not revoke certs),
 		//       perhaps we should delete this account here?
 		account.Status = StatusDeactivated
+		account.AccountRevokedDate = time.Now()
 	}
 
 	if shouldUpdate {
@@ -294,3 +298,90 @@ func (b *backend) acmeNewAccountUpdateHandler(acmeCtx *acmeContext, userCtx *jws
 	resp := formatAccountResponse(acmeCtx, account)
 	return resp, nil
 }
+
+func (b *backend) tidyAcmeAccountByThumbprint(as *acmeState, ac *acmeContext, keyThumbprint string, certTidyBuffer, accountTidyBuffer time.Duration) error {
+	thumbprintEntry, err := ac.sc.Storage.Get(ac.sc.Context, acmeThumbprintPrefix+keyThumbprint)
+	if err != nil {
+		return fmt.Errorf("error retrieving thumbprint entry %v, unable to find corresponding account entry: %w", keyThumbprint, err)
+	}
+	if thumbprintEntry == nil {
+		return fmt.Errorf("empty thumbprint entry %v, unable to find corresponding account entry", keyThumbprint)
+	}
+
+	var thumbprint acmeThumbprint
+	err = thumbprintEntry.DecodeJSON(&thumbprint)
+	if err != nil {
+		return fmt.Errorf("unable to decode thumbprint entry %v to find account entry: %w", keyThumbprint, err)
+	}
+
+	if len(thumbprint.Kid) == 0 {
+		return fmt.Errorf("unable to find account entry: empty kid within thumbprint entry: %s", keyThumbprint)
+	}
+
+	// Now Get the Account:
+	accountEntry, err := ac.sc.Storage.Get(ac.sc.Context, acmeAccountPrefix+thumbprint.Kid)
+	if err != nil {
+		return err
+	}
+	if accountEntry == nil {
+		// We delete the Thumbprint Associated with the Account, and we are done
+		err = ac.sc.Storage.Delete(ac.sc.Context, acmeThumbprintPrefix+keyThumbprint)
+		if err != nil {
+			return err
+		}
+		b.tidyStatusIncDeletedAcmeAccountCount()
+		return nil
+	}
+
+	var account acmeAccount
+	err = accountEntry.DecodeJSON(&account)
+	if err != nil {
+		return err
+	}
+
+	// Tidy Orders On the Account
+	orderIds, err := as.ListOrderIds(ac, thumbprint.Kid)
+	if err != nil {
+		return err
+	}
+	allOrdersTidied := true
+	for _, orderId := range orderIds {
+		wasTidied, err := b.acmeTidyOrder(ac, thumbprint.Kid, acmeAccountPrefix+thumbprint.Kid+"/orders/"+orderId, ac.sc, certTidyBuffer)
+		if err != nil {
+			return err
+		}
+		if !wasTidied {
+			allOrdersTidied = false
+		}
+	}
+
+	if allOrdersTidied && time.Now().After(account.AccountCreatedDate.Add(accountTidyBuffer)) {
+		// Tidy this account
+		// If it is Revoked or Deactivated:
+		if (account.Status == StatusRevoked || account.Status == StatusDeactivated) && time.Now().After(account.AccountRevokedDate.Add(accountTidyBuffer)) {
+			// We Delete the Account Associated with this Thumbprint:
+			err = ac.sc.Storage.Delete(ac.sc.Context, acmeAccountPrefix+thumbprint.Kid)
+			if err != nil {
+				return err
+			}
+
+			// Now we delete the Thumbprint Associated with the Account:
+			err = ac.sc.Storage.Delete(ac.sc.Context, acmeThumbprintPrefix+keyThumbprint)
+			if err != nil {
+				return err
+			}
+			b.tidyStatusIncDeletedAcmeAccountCount()
+		} else if account.Status == StatusValid {
+			// Revoke This Account
+			account.AccountRevokedDate = time.Now()
+			account.Status = StatusRevoked
+			err := as.UpdateAccount(ac, &account)
+			if err != nil {
+				return err
+			}
+			b.tidyStatusIncRevAcmeAccountCount()
+		}
+	}
+
+	return nil
+}

builtin/logical/pki/path_acme_order.go

@@ -884,3 +884,64 @@ func parseOrderIdentifiers(data map[string]interface{}) ([]*ACMEIdentifier, erro
 
 	return identifiers, nil
 }
+
+func (b *backend) acmeTidyOrder(ac *acmeContext, accountId string, orderPath string, sc *storageContext, certTidyBuffer time.Duration) (wasTidied bool, err error) {
+	// First we get the order; note that the orderPath includes the account
+	// It's only accessed at acme/orders/<order_id> with the account context
+	// It's saved at acme/<account_id>/orders/<orderId>
+	entry, err := ac.sc.Storage.Get(ac.sc.Context, orderPath)
+	if err != nil {
+		return false, fmt.Errorf("error loading order: %w", err)
+	}
+	if entry == nil {
+		return false, fmt.Errorf("order does not exist: %w", ErrMalformed)
+	}
+	var order acmeOrder
+	err = entry.DecodeJSON(&order)
+	if err != nil {
+		return false, fmt.Errorf("error decoding order: %w", err)
+	}
+
+	// Determine whether we should tidy this order
+	shouldTidy := false
+	// It is faster to check certificate information on the order entry rather than fetch the cert entry to parse:
+	if !order.CertificateExpiry.IsZero() {
+		// This implies that a certificate exists
+		// When a certificate exists, we want to expire and tidy the order when we tidy the certificate:
+		if time.Now().After(order.CertificateExpiry.Add(certTidyBuffer)) { // It's time to clean
+			shouldTidy = true
+		}
+	} else {
+		// This implies that no certificate exists
+		// In this case, we want to expire the order after it has expired (+ some safety buffer)
+		if time.Now().After(order.Expires) {
+			shouldTidy = true
+		}
+	}
+	if shouldTidy == false {
+		return shouldTidy, nil
+	}
+
+	// Tidy this Order
+	// That includes any certificate acme/<account_id>/orders/orderPath/cert
+	// That also includes any related authorizations: acme/<account_id>/authorizations/<auth_id>
+
+	// First Authorizations
+	for _, authorizationId := range order.AuthorizationIds {
+		err = ac.sc.Storage.Delete(ac.sc.Context, getAuthorizationPath(accountId, authorizationId))
+		if err != nil {
+			return false, err
+		}
+	}
+
+	// Normal Tidy will Take Care of the Certificate
+
+	// And Finally, the order:
+	err = ac.sc.Storage.Delete(ac.sc.Context, orderPath)
+	if err != nil {
+		return false, err
+	}
+	b.tidyStatusIncDelAcmeOrderCount()
+
+	return true, nil
+}