From 09f272510f9158df9737f3ed767cc3340f983a60 Mon Sep 17 00:00:00 2001 From: Chris Hoffman Date: Mon, 11 Sep 2017 14:49:08 -0400 Subject: [PATCH 1/3] Adding latency injector option to -dev mode for storage operations (#3289) --- command/server.go | 12 +++++- physical/latency.go | 90 ++++++++++++++++++++++++++++++++++++++++ vault/expiration.go | 13 ++---- vault/expiration_test.go | 6 +-- 4 files changed, 108 insertions(+), 13 deletions(-) create mode 100644 physical/latency.go diff --git a/command/server.go b/command/server.go index 0b5f8081009c..e5101fd124da 100644 --- a/command/server.go +++ b/command/server.go @@ -75,12 +75,15 @@ func (c *ServerCommand) Run(args []string) int { var dev, verifyOnly, devHA, devTransactional, devLeasedGeneric, devThreeNode bool var configPath []string var logLevel, devRootTokenID, devListenAddress, devPluginDir string + var devLatency, devLatencyJitter int flags := c.Meta.FlagSet("server", meta.FlagSetDefault) flags.BoolVar(&dev, "dev", false, "") flags.StringVar(&devRootTokenID, "dev-root-token-id", "", "") flags.StringVar(&devListenAddress, "dev-listen-address", "", "") flags.StringVar(&devPluginDir, "dev-plugin-dir", "", "") flags.StringVar(&logLevel, "log-level", "info", "") + flags.IntVar(&devLatency, "dev-latency", 0, "") + flags.IntVar(&devLatencyJitter, "dev-latency-jitter", 20, "") flags.BoolVar(&verifyOnly, "verify-only", false, "") flags.BoolVar(&devHA, "dev-ha", false, "") flags.BoolVar(&devTransactional, "dev-transactional", false, "") @@ -266,7 +269,14 @@ func (c *ServerCommand) Run(args []string) int { if devPluginDir != "" { coreConfig.PluginDirectory = devPluginDir } - + if devLatency > 0 { + injectLatency := time.Duration(devLatency) * time.Millisecond + if _, txnOK := backend.(physical.Transactional); txnOK { + coreConfig.Physical = physical.NewTransactionalLatencyInjector(backend, injectLatency, devLatencyJitter, c.logger) + } else { + coreConfig.Physical = physical.NewLatencyInjector(backend, injectLatency, devLatencyJitter, c.logger) + } + } } if devThreeNode { diff --git a/physical/latency.go b/physical/latency.go new file mode 100644 index 000000000000..3253036da05e --- /dev/null +++ b/physical/latency.go @@ -0,0 +1,90 @@ +package physical + +import ( + "math/rand" + "time" + + log "github.com/mgutz/logxi/v1" +) + +const ( + // DefaultJitterPercent is used if no cache size is specified for NewCache + DefaultJitterPercent = 20 +) + +// LatencyInjector is used to add latency into underlying physical requests +type LatencyInjector struct { + backend Backend + latency time.Duration + jitterPercent int + random *rand.Rand +} + +// TransactionalLatencyInjector is the transactional version of the latency +// injector +type TransactionalLatencyInjector struct { + *LatencyInjector + Transactional +} + +// NewLatencyInjector returns a wrapped physical backend to simulate latency +func NewLatencyInjector(b Backend, latency time.Duration, jitter int, logger log.Logger) *LatencyInjector { + if jitter < 0 || jitter > 100 { + jitter = DefaultJitterPercent + } + logger.Info("physical/latency: creating latency injector") + + return &LatencyInjector{ + backend: b, + latency: latency, + jitterPercent: jitter, + random: rand.New(rand.NewSource(int64(time.Now().Nanosecond()))), + } +} + +// NewTransactionalLatencyInjector creates a new transactional LatencyInjector +func NewTransactionalLatencyInjector(b Backend, latency time.Duration, jitter int, logger log.Logger) *TransactionalLatencyInjector { + return &TransactionalLatencyInjector{ + LatencyInjector: NewLatencyInjector(b, latency, jitter, logger), + Transactional: b.(Transactional), + } +} + +func (l *LatencyInjector) addLatency() { + // Calculate a value between 1 +- jitter% + min := 100 - l.jitterPercent + max := 100 + l.jitterPercent + percent := l.random.Intn(max-min) + min + latencyDuration := time.Duration(int(l.latency) * percent / 100) + time.Sleep(latencyDuration) +} + +// Put is a latent put request +func (l *LatencyInjector) Put(entry *Entry) error { + l.addLatency() + return l.backend.Put(entry) +} + +// Get is a latent get request +func (l *LatencyInjector) Get(key string) (*Entry, error) { + l.addLatency() + return l.backend.Get(key) +} + +// Delete is a latent delete request +func (l *LatencyInjector) Delete(key string) error { + l.addLatency() + return l.backend.Delete(key) +} + +// List is a latent list request +func (l *LatencyInjector) List(prefix string) ([]string, error) { + l.addLatency() + return l.backend.List(prefix) +} + +// Transaction is a latent transaction request +func (l *TransactionalLatencyInjector) Transaction(txns []TxnEntry) error { + l.addLatency() + return l.Transactional.Transaction(txns) +} diff --git a/vault/expiration.go b/vault/expiration.go index 41a697291749..628df8e973b0 100644 --- a/vault/expiration.go +++ b/vault/expiration.go @@ -117,7 +117,7 @@ func (c *Core) setupExpiration() error { c.logger.Error("expiration: error shutting down core: %v", err) } } - go c.expiration.Restore(errorFunc, 0) + go c.expiration.Restore(errorFunc) return nil } @@ -268,7 +268,7 @@ func (m *ExpirationManager) Tidy() error { // Restore is used to recover the lease states when starting. // This is used after starting the vault. -func (m *ExpirationManager) Restore(errorFunc func(), loadDelay time.Duration) (retErr error) { +func (m *ExpirationManager) Restore(errorFunc func()) (retErr error) { defer func() { // Turn off restore mode. We can do this safely without the lock because // if restore mode finished successfully, restore mode was already @@ -322,7 +322,7 @@ func (m *ExpirationManager) Restore(errorFunc func(), loadDelay time.Duration) ( return } - err := m.processRestore(leaseID, loadDelay) + err := m.processRestore(leaseID) if err != nil { errs <- err continue @@ -398,7 +398,7 @@ func (m *ExpirationManager) Restore(errorFunc func(), loadDelay time.Duration) ( // processRestore takes a lease and restores it in the expiration manager if it has // not already been seen -func (m *ExpirationManager) processRestore(leaseID string, loadDelay time.Duration) error { +func (m *ExpirationManager) processRestore(leaseID string) error { m.restoreRequestLock.RLock() defer m.restoreRequestLock.RUnlock() @@ -415,11 +415,6 @@ func (m *ExpirationManager) processRestore(leaseID string, loadDelay time.Durati return nil } - // Useful for testing to add latency to all load requests - if loadDelay > 0 { - time.Sleep(loadDelay) - } - // Load lease and restore expiration timer _, err := m.loadEntryInternal(leaseID, true, false) if err != nil { diff --git a/vault/expiration_test.go b/vault/expiration_test.go index 4df5fb42699f..144bd16b045f 100644 --- a/vault/expiration_test.go +++ b/vault/expiration_test.go @@ -37,7 +37,7 @@ func TestExpiration_Tidy(t *testing.T) { var err error exp := mockExpiration(t) - if err := exp.Restore(nil, 0); err != nil { + if err := exp.Restore(nil); err != nil { t.Fatal(err) } @@ -341,7 +341,7 @@ func benchmarkExpirationBackend(b *testing.B, physicalBackend physical.Backend, b.ResetTimer() for i := 0; i < b.N; i++ { - err = exp.Restore(nil, 0) + err = exp.Restore(nil) // Restore if err != nil { b.Fatalf("err: %v", err) @@ -399,7 +399,7 @@ func TestExpiration_Restore(t *testing.T) { } // Restore - err = exp.Restore(nil, 0) + err = exp.Restore(nil) if err != nil { t.Fatalf("err: %v", err) } From 6553f31f17c6bd7441979e56a574d1eeebc6c88d Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Mon, 11 Sep 2017 14:56:25 -0400 Subject: [PATCH 2/3] changelog++ --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f36b0e39805..4473af6a8803 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.8.2.1 (September 11th, 2017) (Enterprise Only) + +BUG FIXES: + + * Fix an issue upgrading to 0.8.2 for Enterprise customers. + ## 0.8.2 (September 5th, 2017) SECURITY: From bfff8b42441f57ce6a8f4507f93ab9ac4ab75d2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Miguel=20Cust=C3=B3dio?= Date: Mon, 11 Sep 2017 20:26:24 +0100 Subject: [PATCH 3/3] Fix navigation and prameters in the 'gcp' auth backend docs. (#3317) --- website/source/docs/auth/gcp.html.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/website/source/docs/auth/gcp.html.md b/website/source/docs/auth/gcp.html.md index 144846d3d561..31cab0852593 100644 --- a/website/source/docs/auth/gcp.html.md +++ b/website/source/docs/auth/gcp.html.md @@ -33,10 +33,10 @@ v0.8.0+ to use plugins. The Vault authentication workflow for IAM service accounts is as follows: - 1. A client with IAM service account credentials generates a signed JWT using the IAM [projects.serviceAccounts.signJwt](https://cloud.google.com/iam/reference/rest/v1/projects.serviceAccounts/signJwt) method. See [usage](#iam-authentication-token) for the expected format and example code. + 1. A client with IAM service account credentials generates a signed JWT using the IAM [projects.serviceAccounts.signJwt](https://cloud.google.com/iam/reference/rest/v1/projects.serviceAccounts/signJwt) method. See [usage](#the-iam-authentication-token) for the expected format and example code. 2. The client sends this JWT to Vault in a login request with a role name. This role should have type `iam` 3. Vault grabs the `kid` header value, which contains the ID of the key-pair used to generate the JWT, and the `sub` ID/email to find the service account key. If the service account does not exist or the key is not linked to the service account, Vault will deny authentication. - 4. Vault authorizes the confirmed service account against the given role. See [authorization section](#authorization) to see how each type of role handles authorization. + 4. Vault authorizes the confirmed service account against the given role. See [authorization section](#authorization-workflow) to see how each type of role handles authorization. [![IAM Login Workflow](/assets/images/vault-gcp-iam-auth-workflow.svg)](/assets/images/vault-gcp-iam-auth-workflow.svg) @@ -241,7 +241,7 @@ to learn more about parameters. ``` $ vault write auth/gcp/role/dev-role \ type="iam" \ - project="project-123456" \ + project_id="project-123456" \ policies="prod,dev" \ service_accounts="serviceaccount1@project1234.iam.gserviceaccount.com,uuid123,..." ... @@ -300,12 +300,12 @@ $ curl $VAULT_ADDR/v1/auth/gcp/config \ ``` $ curl $VAULT_ADDR/v1/auth/gcp/role/dev-role \ --d '{ "type": "iam", "project": "project-123456", ...}' +-d '{ "type": "iam", "project_id": "project-123456", ...}' ``` #### Login to get a Vault Token -The endpoint for the GitHub login is `auth/gcp/login`. +The endpoint for the GCP login is `auth/gcp/login`. The `gcp` mountpoint value in the url is the default mountpoint value. If you have mounted the `gcp` backend with a different mountpoint, use that value.