Implement Auto-Config certificate generation

Most of the groundwork was layed in previous PRs between adding the cert-monitor package to extracting the logic of signing certificates out of the connect_ca_endpoint.go code and into a method on the server.

This also refactors the auto-config package a bit to split things out into multiple files.

# Conflicts:
#	agent/agent.go

agent/agent.go

@@ -460,6 +460,10 @@ func New(options ...AgentOption) (*Agent, error) {
 	// loaded any auto-config sources yet.
 	a.config = config
 
+	// create the cache using the rate limiting settings from the config. Note that this means
+	// that these limits are not reloadable.
+	a.cache = cache.New(a.config.Cache)
+
 	if flat.logger == nil {
 		logConf := &logging.Config{
 			LogLevel:          config.LogLevel,
@@ -525,14 +529,34 @@ func New(options ...AgentOption) (*Agent, error) {
 		return nil, fmt.Errorf("Failed to setup node ID: %v", err)
 	}
 
-	acOpts := []autoconf.Option{
-		autoconf.WithDirectRPC(a.connPool),
-		autoconf.WithTLSConfigurator(a.tlsConfigurator),
-		autoconf.WithBuilderOpts(flat.builderOpts),
-		autoconf.WithLogger(a.logger),
-		autoconf.WithOverrides(flat.overrides...),
+	// We used to do this in the Start method. However it doesn't need to go
+	// there any longer. Originally it did because we passed the agent
+	// delegate to some of the cache registrations. Now we just
+	// pass the agent itself so its safe to move here.
+	a.registerCache()
+
+	cmConf := new(certmon.Config).
+		WithCache(a.cache).
+		WithTLSConfigurator(a.tlsConfigurator).
+		WithDNSSANs(a.config.AutoConfig.DNSSANs).
+		WithIPSANs(a.config.AutoConfig.IPSANs).
+		WithDatacenter(a.config.Datacenter).
+		WithNodeName(a.config.NodeName).
+		WithFallback(a.autoConfigFallbackTLS).
+		WithLogger(a.logger.Named(logging.AutoConfig)).
+		WithTokens(a.tokens)
+	acCertMon, err := certmon.New(cmConf)
+	if err != nil {
+		return nil, err
 	}
-	ac, err := autoconf.New(acOpts...)
+
+	acConf := new(autoconf.Config).
+		WithDirectRPC(a.connPool).
+		WithBuilderOpts(flat.builderOpts).
+		WithLogger(a.logger).
+		WithOverrides(flat.overrides...).
+		WithCertMonitor(acCertMon)
+	ac, err := autoconf.New(acConf)
 	if err != nil {
 		return nil, err
 	}
@@ -663,9 +687,6 @@ func (a *Agent) Start(ctx context.Context) error {
 	// regular and on-demand state synchronizations (anti-entropy).
 	a.sync = ae.NewStateSyncer(a.State, c.AEInterval, a.shutdownCh, a.logger)
 
-	// create the cache
-	a.cache = cache.New(c.Cache)
-
 	// create the config for the rpc server/client
 	consulCfg, err := a.consulConfig()
 	if err != nil {
@@ -714,10 +735,6 @@ func (a *Agent) Start(ctx context.Context) error {
 	a.State.Delegate = a.delegate
 	a.State.TriggerSyncChanges = a.sync.SyncChanges.Trigger
 
-	// Register the cache. We do this much later so the delegate is
-	// populated from above.
-	a.registerCache()
-
 	if a.config.AutoEncryptTLS && !a.config.ServerMode {
 		reply, err := a.autoEncryptInitialCertificate(ctx)
 		if err != nil {
@@ -755,6 +772,9 @@ func (a *Agent) Start(ctx context.Context) error {
 		a.logger.Info("automatically upgraded to TLS")
 	}
 
+	if err := a.autoConf.Start(&lib.StopChannelContext{StopCh: a.shutdownCh}); err != nil {
+		return fmt.Errorf("AutoConf failed to start certificate monitor: %w", err)
+	}
 	a.serviceManager.Start()
 
 	// Load checks/services/metadata.
@@ -867,6 +887,10 @@ func (a *Agent) autoEncryptInitialCertificate(ctx context.Context) (*structs.Sig
 	return client.RequestAutoEncryptCerts(ctx, addrs, a.config.ServerPort, a.tokens.AgentToken(), a.config.AutoEncryptDNSSAN, a.config.AutoEncryptIPSAN)
 }
 
+func (a *Agent) autoConfigFallbackTLS(ctx context.Context) (*structs.SignedResponse, error) {
+	return a.autoConf.FallbackTLS(ctx)
+}
+
 func (a *Agent) listenAndServeGRPC() error {
 	if len(a.config.GRPCAddrs) < 1 {
 		return nil
@@ -1827,6 +1851,16 @@ func (a *Agent) ShutdownAgent() error {
 	// Stop the watches to avoid any notification/state change during shutdown
 	a.stopAllWatches()
 
+	// this would be cancelled anyways (by the closing of the shutdown ch) but
+	// this should help them to be stopped more quickly
+	a.autoConf.Stop()
+
+	if a.certMonitor != nil {
+		// this would be cancelled anyways  (by the closing of the shutdown ch)
+		// but this should help them to be stopped more quickly
+		a.certMonitor.Stop()
+	}
+
 	// Stop the service manager (must happen before we take the stateLock to avoid deadlock)
 	if a.serviceManager != nil {
 		a.serviceManager.Stop()

agent/agent_test.go

@@ -4645,7 +4645,8 @@ func TestAutoConfig_Integration(t *testing.T) {
 	// eventually this test should really live with integration tests
 	// the goal here is to have one test server and another test client
 	// spin up both agents and allow the server to authorize the auto config
-	// request and then see the client joined
+	// request and then see the client joined. Finally we force a CA roots
+	// update and wait to see that the agents TLS certificate gets updated.
 
 	cfgDir := testutil.TempDir(t, "auto-config")
 
@@ -4683,7 +4684,6 @@ func TestAutoConfig_Integration(t *testing.T) {
 		cert_file = "` + certFile + `"
 		key_file = "` + keyFile + `"
 		connect { enabled = true }
-		auto_encrypt { allow_tls = true }
 		auto_config {
 			authorization {
 				enabled = true
@@ -4740,11 +4740,44 @@ func TestAutoConfig_Integration(t *testing.T) {
 
 	defer client.Shutdown()
 
+	retry.Run(t, func(r *retry.R) {
+		require.NotNil(r, client.Agent.tlsConfigurator.Cert())
+	})
+
 	// when this is successful we managed to get the gossip key and serf addresses to bind to
 	// and then connect. Additionally we would have to have certificates or else the
 	// verify_incoming config on the server would not let it work.
 	testrpc.WaitForTestAgent(t, client.RPC, "dc1", testrpc.WithToken(TestDefaultMasterToken))
 
+	// grab the existing cert
+	cert1 := client.Agent.tlsConfigurator.Cert()
+	require.NotNil(t, cert1)
+
+	// force a roots rotation by updating the CA config
+	t.Logf("Forcing roots rotation on the server")
+	ca := connect.TestCA(t, nil)
+	req := &structs.CARequest{
+		Datacenter:   "dc1",
+		WriteRequest: structs.WriteRequest{Token: TestDefaultMasterToken},
+		Config: &structs.CAConfiguration{
+			Provider: "consul",
+			Config: map[string]interface{}{
+				"LeafCertTTL":         "1h",
+				"PrivateKey":          ca.SigningKey,
+				"RootCert":            ca.RootCert,
+				"RotationPeriod":      "6h",
+				"IntermediateCertTTL": "3h",
+			},
+		},
+	}
+	var reply interface{}
+	require.NoError(t, srv.RPC("ConnectCA.ConfigurationSet", &req, &reply))
+
+	// ensure that a new cert gets generated and pushed into the TLS configurator
+	retry.Run(t, func(r *retry.R) {
+		require.NotEqual(r, cert1, client.Agent.tlsConfigurator.Cert())
+	})
+
 	// spot check that we now have an ACL token
 	require.NotEmpty(t, client.tokens.AgentToken())
 }