From 56874dc99cbd88fa96625ddacb7c6bbc993d4058 Mon Sep 17 00:00:00 2001 From: Brian Kassouf Date: Fri, 5 Jul 2019 09:53:16 -0700 Subject: [PATCH 1/2] core: Don't shutdown if key upgrades fail due to canceled context --- vault/ha.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/vault/ha.go b/vault/ha.go index 020a1543f3eb..d615ef39287c 100644 --- a/vault/ha.go +++ b/vault/ha.go @@ -7,6 +7,7 @@ import ( "encoding/base64" "errors" "fmt" + "strings" "sync/atomic" "time" @@ -472,12 +473,18 @@ func (c *Core) waitForLeadership(newLeaderCh chan func(), manualStepDownCh, stop } if err := c.performKeyUpgrades(activeCtx); err != nil { - // We call this in a goroutine so that we can give up the - // statelock and have this shut us down; sealInternal has a - // workflow where it watches for the stopCh to close so we want - // to return from here c.logger.Error("error performing key upgrades", "error", err) - go c.Shutdown() + + // If we fail due to anything other than a context canceled + // error we should shutdown as we may have the incorrect Keys. + if !strings.Contains(err.Error(), context.Canceled.Error()) { + // We call this in a goroutine so that we can give up the + // statelock and have this shut us down; sealInternal has a + // workflow where it watches for the stopCh to close so we want + // to return from here + go c.Shutdown() + } + c.heldHALock = nil lock.Unlock() close(continueCh) From c4d031f75a816678eeabb70a4646f9ffddee5c01 Mon Sep 17 00:00:00 2001 From: Brian Kassouf Date: Fri, 5 Jul 2019 10:18:58 -0700 Subject: [PATCH 2/2] Continue if we are not shutting down --- vault/ha.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/vault/ha.go b/vault/ha.go index d615ef39287c..6e31b78a52f8 100644 --- a/vault/ha.go +++ b/vault/ha.go @@ -490,7 +490,14 @@ func (c *Core) waitForLeadership(newLeaderCh chan func(), manualStepDownCh, stop close(continueCh) c.stateLock.Unlock() metrics.MeasureSince([]string{"core", "leadership_setup_failed"}, activeTime) - return + + // If we are shutting down we should return from this function, + // otherwise continue + if !strings.Contains(err.Error(), context.Canceled.Error()) { + continue + } else { + return + } } }