Wait for handler startup until TLS has been established

agent-inject/agent/agent.go

@@ -32,6 +32,7 @@ const (
 	DefaultAgentUseLeaderElector            = false
 	DefaultAgentInjectToken                 = false
 	DefaultTemplateConfigExitOnRetryFailure = true
+	DefaultServerWaitForTLSCert             = true
 	DefaultServiceAccountMount              = "/var/run/secrets/vault.hashicorp.com/serviceaccount"
 )
 

deploy/injector-deployment.yaml

@@ -42,6 +42,8 @@ spec:
               value: "https://vault.$(NAMESPACE).svc:8200"
             - name: AGENT_INJECT_VAULT_IMAGE
               value: "hashicorp/vault:1.9.2"
+            - name: AGENT_INJECT_SERVER_WAIT_FOR_TLS_CERT
+              value: "true"
             - name: AGENT_INJECT_TLS_AUTO
               value: vault-agent-injector-cfg
             - name: AGENT_INJECT_TLS_AUTO_HOSTS

go.mod

@@ -65,6 +65,7 @@ require (
 	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 // indirect
 	golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 // indirect
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect
 	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
 	golang.org/x/text v0.3.6 // indirect

go.sum

@@ -682,6 +682,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

subcommand/injector/command.go

@@ -22,6 +22,7 @@ import (
 	"github.com/hashicorp/vault-k8s/leader"
 	"github.com/mitchellh/cli"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"golang.org/x/sync/semaphore"
 	adminv1 "k8s.io/api/admissionregistration/v1"
 	adminv1beta "k8s.io/api/admissionregistration/v1beta1"
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
@@ -42,6 +43,7 @@ type Command struct {
 	flagLogFormat                  string // Log format
 	flagCertFile                   string // TLS Certificate to serve
 	flagKeyFile                    string // TLS private key to serve
+	flagServerWaitForTLSCert       bool   // HTTP server waits for TLS cert to be updated before starting up
 	flagExitOnRetryFailure         bool   // Set template_config.exit_on_retry_failure on agent
 	flagStaticSecretRenderInterval string // Set template_config.static_secret_render_interval on agent
 	flagAutoName                   string // MutatingWebhookConfiguration for updating
@@ -149,6 +151,12 @@ func (c *Command) Run(args []string) int {
 		logger.Warn(fmt.Sprintf("failed to determine Admissionregistration API version, defaulting to %s", adminAPIVersion), "error", err)
 	}
 
+	tlsConfig, err := c.makeTLSConfig()
+	if err != nil {
+		c.UI.Error(fmt.Sprintf("Failed to configure TLS: %s", err))
+		return 1
+	}
+
 	// Determine where to source the certificates from
 	var certSource cert.Source = &cert.GenSource{
 		Name:            "Agent Inject",
@@ -172,8 +180,15 @@ func (c *Command) Run(args []string) int {
 	// then start all the background routines for updating certificates.
 	certCh := make(chan cert.Bundle)
 	certNotify := cert.NewNotify(ctx, certCh, certSource, logger.Named("notify"))
+	// Create a mutex to wait for certificate to be updated.
+	certWaitSemaphore := semaphore.NewWeighted(1)
+
 	go certNotify.Run()
-	go c.certWatcher(ctx, certCh, clientset, leaderElector, adminAPIVersion, logger.Named("certwatcher"))
+	go c.certWatcher(ctx, certCh, certWaitSemaphore, clientset, leaderElector, adminAPIVersion, logger.Named("certwatcher"))
+
+	if c.flagServerWaitForTLSCert {
+		c.waitForTLSCert(ctx, certWaitSemaphore)
+	}
 
 	// Build the HTTP handler and server
 	injector := agentInject.Handler{
@@ -210,11 +225,6 @@ func (c *Command) Run(args []string) int {
 	}
 
 	var handler http.Handler = mux
-	tlsConfig, err := c.makeTLSConfig()
-	if err != nil {
-		c.UI.Error(fmt.Sprintf("Failed to configure TLS: %s", err))
-		return 1
-	}
 	server := &http.Server{
 		Addr:      c.flagListen,
 		Handler:   handler,
@@ -316,7 +326,19 @@ func getAdminAPIVersion(ctx context.Context, clientset *kubernetes.Clientset) (s
 	return adminAPIVersion, err
 }
 
-func (c *Command) certWatcher(ctx context.Context, ch <-chan cert.Bundle, clientset *kubernetes.Clientset, leaderElector leader.Elector, adminAPIVersion string, log hclog.Logger) {
+func (c *Command) waitForTLSCert(ctx context.Context, certWaitSemaphore *semaphore.Weighted) error {
+	c.UI.Info("Waiting for TLS certificate before continuing with starting handler")
+
+	err := certWaitSemaphore.Acquire(ctx, 1)
+	if err != nil {
+		return fmt.Errorf("something went wrong while waiting for the TLS certificate: %s", err)
+	}
+
+	c.UI.Info("Updated TLS certificate.. continuing with starting handler")
+	return nil
+}
+
+func (c *Command) certWatcher(ctx context.Context, ch <-chan cert.Bundle, certWaitSemaphore *semaphore.Weighted, clientset *kubernetes.Clientset, leaderElector leader.Elector, adminAPIVersion string, log hclog.Logger) {
 	var bundle cert.Bundle
 
 	for {
@@ -390,7 +412,10 @@ func (c *Command) certWatcher(ctx context.Context, ch <-chan cert.Bundle, client
 		}
 
 		// Update the certificate
-		c.cert.Store(&crt)
+		// Check if previously no certificate was loaded in to prevent panic
+		if c.cert.Swap(&crt) == nil && c.flagServerWaitForTLSCert {
+			certWaitSemaphore.Release(1)
+		}
 	}
 }
 

subcommand/injector/flags.go

@@ -53,6 +53,9 @@ type Specification struct {
 	// TLSKeyFile is the AGENT_INJECT_TLS_KEY_FILE environment variable.
 	TLSKeyFile string `envconfig:"tls_key_file"`
 
+	// ServerWaitForTLSCert is the AGENT_INJECT_SERVER_WAIT_FOR_TLS_CERT
+	ServerWaitForTLSCert string `envconfig:"server_wait_for_tls_cert"`
+
 	// VaultAddr is the AGENT_INJECT_VAULT_ADDR environment variable.
 	VaultAddr string `split_words:"true"`
 
@@ -130,6 +133,8 @@ func (c *Command) init() {
 		"PEM-encoded TLS certificate to serve. If blank, will generate random cert.")
 	c.flagSet.StringVar(&c.flagKeyFile, "tls-key-file", "",
 		"PEM-encoded TLS private key to serve. If blank, will generate random cert.")
+	c.flagSet.BoolVar(&c.flagServerWaitForTLSCert, "server-wait-for-tls-cert", agent.DefaultServerWaitForTLSCert,
+		fmt.Sprintf("Whether or not injector's HTTP server should wait for TLS certificate to be updated. Defaults to %t.", agent.DefaultServerWaitForTLSCert))
 	c.flagSet.StringVar(&c.flagVaultImage, "vault-image", agent.DefaultVaultImage,
 		fmt.Sprintf("Docker image for Vault. Defaults to %q.", agent.DefaultVaultImage))
 	c.flagSet.StringVar(&c.flagVaultService, "vault-address", "",
@@ -251,6 +256,13 @@ func (c *Command) parseEnvs() error {
 		c.flagKeyFile = envs.TLSKeyFile
 	}
 
+	if envs.ServerWaitForTLSCert != "" {
+		c.flagServerWaitForTLSCert, err = strconv.ParseBool(envs.ServerWaitForTLSCert)
+		if err != nil {
+			return err
+		}
+	}
+
 	if envs.VaultImage != "" {
 		c.flagVaultImage = envs.VaultImage
 	}

subcommand/injector/flags_test.go

@@ -170,6 +170,7 @@ func TestCommandEnvBools(t *testing.T) {
 		{env: "AGENT_INJECT_USE_LEADER_ELECTOR", value: false, cmdPtr: &cmd.flagUseLeaderElector},
 		{env: "AGENT_INJECT_TEMPLATE_CONFIG_EXIT_ON_RETRY_FAILURE", value: true, cmdPtr: &cmd.flagExitOnRetryFailure},
 		{env: "AGENT_INJECT_TEMPLATE_CONFIG_EXIT_ON_RETRY_FAILURE", value: false, cmdPtr: &cmd.flagExitOnRetryFailure},
+		{env: "AGENT_INJECT_SERVER_WAIT_FOR_TLS_CERT", value: true, cmdPtr: &cmd.flagServerWaitForTLSCert},
 	}
 
 	for _, tt := range tests {