From d86b3352283a0da26c33eae585e3a1367e916fcb Mon Sep 17 00:00:00 2001 From: Pavlos Ratis Date: Tue, 17 Sep 2019 22:39:00 +0200 Subject: [PATCH 01/46] add more gcp examples (#6358) --- website/source/docs/secrets/gcp/index.html.md | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/website/source/docs/secrets/gcp/index.html.md b/website/source/docs/secrets/gcp/index.html.md index 54a0e92c1ec7..5b8d33c0762d 100644 --- a/website/source/docs/secrets/gcp/index.html.md +++ b/website/source/docs/secrets/gcp/index.html.md @@ -188,6 +188,30 @@ resource "buckets/my-bucket" { "roles/storage.legacyBucketReader", ] } + +# At instance level, using self-link +resource "https://www.googleapis.com/compute/v1/projects/my-project/zone/my-zone/instances/my-instance" { + roles = [ + "roles/compute.instanceAdmin.v1" + ] +} + +# At project level +resource "//cloudresourcemanager.googleapis.com/projects/my-project" { + roles = [ + "roles/compute.instanceAdmin.v1", + "roles/iam.serviceAccountUser", # required if managing instances that run as service accounts + ] +} + +# At folder level +resource "//cloudresourcemanager.googleapis.com/folders/123456" { + roles = [ + "roles/compute.viewer", + "roles/deploymentmanager.viewer", + ] +} + ``` The top-level `resource` block defines the resource or resource path for which @@ -213,6 +237,9 @@ few different formats: # Pubsub snapshot //pubsub.googleapis.com/project/my-project/snapshots/my-pubsub-snapshot + + # Resource manager + //cloudresourcemanager.googleapis.com/projects/my-project" ``` - **Relative resource name** - A path-noscheme URI path, usually as accepted by From 9de946780beebc9cdc1a72fc6bb06177b9799840 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Wed, 18 Sep 2019 09:09:44 -0400 Subject: [PATCH 02/46] Tidy sdk --- sdk/go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/go.mod b/sdk/go.mod index cd382d9e9b8a..69d135bf8f53 100644 --- a/sdk/go.mod +++ b/sdk/go.mod @@ -31,7 +31,6 @@ require ( golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 // indirect golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db // indirect - google.golang.org/appengine v1.4.0 // indirect google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107 // indirect google.golang.org/grpc v1.22.0 gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect From e430c836412e33195982dc386634595381f481da Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Wed, 18 Sep 2019 09:10:23 -0400 Subject: [PATCH 03/46] Bump API's sdk --- api/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/go.mod b/api/go.mod index 301a42335c2b..9125a9757c22 100644 --- a/api/go.mod +++ b/api/go.mod @@ -11,7 +11,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.6.2 github.com/hashicorp/go-rootcerts v1.0.1 github.com/hashicorp/hcl v1.0.0 - github.com/hashicorp/vault/sdk v0.1.14-0.20190904164450-29f2490f986b + github.com/hashicorp/vault/sdk v0.1.14-0.20190918130944-9de946780bee github.com/mitchellh/mapstructure v1.1.2 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 From c9613203f58165c8a8b68f691f2a6c44a5ff6be1 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Wed, 18 Sep 2019 09:12:57 -0400 Subject: [PATCH 04/46] Update api/sdk. Let kr/pty stay for now so it stops going in on every build --- go.mod | 5 +++-- vendor/github.com/hashicorp/vault/api/go.mod | 2 +- vendor/modules.txt | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 5a6dad0d75d9..fe7b7ab86cc3 100644 --- a/go.mod +++ b/go.mod @@ -81,8 +81,8 @@ require ( github.com/hashicorp/vault-plugin-secrets-gcp v0.5.3-0.20190814210141-d2086ff79b04 github.com/hashicorp/vault-plugin-secrets-gcpkms v0.5.2-0.20190814210149-315cdbf5de6e github.com/hashicorp/vault-plugin-secrets-kv v0.5.2-0.20190814210155-e060c2a001a8 - github.com/hashicorp/vault/api v1.0.5-0.20190904164530-82f8309ab640 - github.com/hashicorp/vault/sdk v0.1.14-0.20190917153655-ea21a2a1bf3f + github.com/hashicorp/vault/api v1.0.5-0.20190918131023-e430c836412e + github.com/hashicorp/vault/sdk v0.1.14-0.20190918130944-9de946780bee github.com/influxdata/influxdb v0.0.0-20190411212539-d24b7ba8c4c4 github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 // indirect github.com/jackc/pgx v3.3.0+incompatible // indirect @@ -91,6 +91,7 @@ require ( github.com/joyent/triton-go v0.0.0-20190112182421-51ffac552869 github.com/keybase/go-crypto v0.0.0-20190403132359-d65b6b94177f github.com/kr/pretty v0.1.0 + github.com/kr/pty v1.1.3 // indirect github.com/kr/text v0.1.0 github.com/lib/pq v1.2.0 github.com/mattn/go-colorable v0.1.2 diff --git a/vendor/github.com/hashicorp/vault/api/go.mod b/vendor/github.com/hashicorp/vault/api/go.mod index 301a42335c2b..9125a9757c22 100644 --- a/vendor/github.com/hashicorp/vault/api/go.mod +++ b/vendor/github.com/hashicorp/vault/api/go.mod @@ -11,7 +11,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.6.2 github.com/hashicorp/go-rootcerts v1.0.1 github.com/hashicorp/hcl v1.0.0 - github.com/hashicorp/vault/sdk v0.1.14-0.20190904164450-29f2490f986b + github.com/hashicorp/vault/sdk v0.1.14-0.20190918130944-9de946780bee github.com/mitchellh/mapstructure v1.1.2 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 diff --git a/vendor/modules.txt b/vendor/modules.txt index 91cfc6ef3efa..b5f41a6749b4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -373,9 +373,9 @@ github.com/hashicorp/vault-plugin-secrets-gcp/plugin/util github.com/hashicorp/vault-plugin-secrets-gcpkms # github.com/hashicorp/vault-plugin-secrets-kv v0.5.2-0.20190814210155-e060c2a001a8 github.com/hashicorp/vault-plugin-secrets-kv -# github.com/hashicorp/vault/api v1.0.5-0.20190904164530-82f8309ab640 => ./api +# github.com/hashicorp/vault/api v1.0.5-0.20190918131023-e430c836412e => ./api github.com/hashicorp/vault/api -# github.com/hashicorp/vault/sdk v0.1.14-0.20190917153655-ea21a2a1bf3f => ./sdk +# github.com/hashicorp/vault/sdk v0.1.14-0.20190918130944-9de946780bee => ./sdk github.com/hashicorp/vault/sdk/helper/salt github.com/hashicorp/vault/sdk/helper/strutil github.com/hashicorp/vault/sdk/helper/wrapping From c69481ebe289356769a14c8e50cf7206b7ffe517 Mon Sep 17 00:00:00 2001 From: Michael Gaffney Date: Wed, 18 Sep 2019 09:29:58 -0400 Subject: [PATCH 05/46] Fix the transit trim key api doc (#7453) --- website/source/api/secret/transit/index.html.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/website/source/api/secret/transit/index.html.md b/website/source/api/secret/transit/index.html.md index 4a2404796358..70bfa71f10ec 100644 --- a/website/source/api/secret/transit/index.html.md +++ b/website/source/api/secret/transit/index.html.md @@ -1229,11 +1229,12 @@ keyring. Once trimmed, previous versions of the key cannot be recovered. ### Parameters -- `min_version` `(int: )` - The minimum version for the key ring. All - versions before this version will be permanently deleted. This value can at - most be equal to the lesser of `min_decryption_version` and - `min_encryption_version`. This is not allowed to be set when either - `min_encryption_version` or `min_decryption_version` is set to zero. +- `min_available_version` `(int: )` - The minimum available version + for the key ring. All versions before this version will be permanently + deleted. This value can at most be equal to the lesser of + `min_decryption_version` and `min_encryption_version`. This is not allowed to + be set when either `min_encryption_version` or `min_decryption_version` is set + to zero. ### Sample Payload @@ -1312,4 +1313,4 @@ $ curl \ }, ``` -[sys-plugin-reload-backend]: /api/system/plugins-reload-backend.html#reload-plugins \ No newline at end of file +[sys-plugin-reload-backend]: /api/system/plugins-reload-backend.html#reload-plugins From 545a4b1242bf69e7fee8c068da70eba37bf43e45 Mon Sep 17 00:00:00 2001 From: Matthew Irish Date: Wed, 18 Sep 2019 14:07:18 -0500 Subject: [PATCH 06/46] expose 'storage_type' on the sys/seal-status endpoint (#7486) * expose 'storage_type' on the sys/seal-status endpoint * add comments * Update vault/core.go Co-Authored-By: Brian Kassouf --- api/sys_seal.go | 1 + command/server.go | 1 + http/sys_seal.go | 3 +++ vault/core.go | 12 ++++++++++++ 4 files changed, 17 insertions(+) diff --git a/api/sys_seal.go b/api/sys_seal.go index 301d3f26a102..20d41a28f343 100644 --- a/api/sys_seal.go +++ b/api/sys_seal.go @@ -77,6 +77,7 @@ type SealStatusResponse struct { ClusterName string `json:"cluster_name,omitempty"` ClusterID string `json:"cluster_id,omitempty"` RecoverySeal bool `json:"recovery_seal"` + StorageType string `json:"storage_type,omitempty"` } type UnsealOpts struct { diff --git a/command/server.go b/command/server.go index 32e43fd4a0ad..ad0a40e5a8d5 100644 --- a/command/server.go +++ b/command/server.go @@ -664,6 +664,7 @@ func (c *ServerCommand) Run(args []string) int { coreConfig := &vault.CoreConfig{ Physical: backend, RedirectAddr: config.Storage.RedirectAddr, + StorageType: config.Storage.Type, HAPhysical: nil, Seal: barrierSeal, AuditBackends: c.AuditBackends, diff --git a/http/sys_seal.go b/http/sys_seal.go index 7b657f0487cf..384d6c5ab770 100644 --- a/http/sys_seal.go +++ b/http/sys_seal.go @@ -198,6 +198,7 @@ func handleSysSealStatusRaw(core *vault.Core, w http.ResponseWriter, r *http.Req Initialized: false, Sealed: true, RecoverySeal: core.SealAccess().RecoveryKeySupported(), + StorageType: core.StorageType(), }) return } @@ -233,6 +234,7 @@ func handleSysSealStatusRaw(core *vault.Core, w http.ResponseWriter, r *http.Req ClusterName: clusterName, ClusterID: clusterID, RecoverySeal: core.SealAccess().RecoveryKeySupported(), + StorageType: core.StorageType(), }) } @@ -249,6 +251,7 @@ type SealStatusResponse struct { ClusterName string `json:"cluster_name,omitempty"` ClusterID string `json:"cluster_id,omitempty"` RecoverySeal bool `json:"recovery_seal"` + StorageType string `json:"storage_type,omitempty"` } // Note: because we didn't provide explicit tagging in the past we can't do it diff --git a/vault/core.go b/vault/core.go index 8da65417cb61..7979361585e4 100644 --- a/vault/core.go +++ b/vault/core.go @@ -172,6 +172,9 @@ type Core struct { // HABackend may be available depending on the physical backend ha physical.HABackend + // storageType is the the storage type set in the storage configuration + storageType string + // redirectAddr is the address we advertise as leader if held redirectAddr string @@ -474,6 +477,8 @@ type CoreConfig struct { Physical physical.Backend `json:"physical" structs:"physical" mapstructure:"physical"` + StorageType string `json:"storage_type" structs:"storage_type" mapstructure:"storage_type"` + // May be nil, which disables HA operations HAPhysical physical.HABackend `json:"ha_physical" structs:"ha_physical" mapstructure:"ha_physical"` @@ -546,6 +551,7 @@ func (c *CoreConfig) Clone() *CoreConfig { DisableCache: c.DisableCache, DisableMlock: c.DisableMlock, CacheSize: c.CacheSize, + StorageType: c.StorageType, RedirectAddr: c.RedirectAddr, ClusterAddr: c.ClusterAddr, DefaultLeaseTTL: c.DefaultLeaseTTL, @@ -613,6 +619,7 @@ func NewCore(conf *CoreConfig) (*Core, error) { devToken: conf.DevToken, physical: conf.Physical, underlyingPhysical: conf.Physical, + storageType: conf.StorageType, redirectAddr: conf.RedirectAddr, clusterAddr: new(atomic.Value), clusterListener: new(atomic.Value), @@ -1820,6 +1827,11 @@ func (c *Core) SealAccess() *SealAccess { return NewSealAccess(c.seal) } +// StorageType returns a string equal to the storage configuration's type. +func (c *Core) StorageType() string { + return c.storageType +} + func (c *Core) Logger() log.Logger { return c.logger } From acbff2d5ee070a18a0809aac00c3f06ea3699a27 Mon Sep 17 00:00:00 2001 From: Matthew Irish Date: Wed, 18 Sep 2019 14:09:10 -0500 Subject: [PATCH 07/46] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2bb6bb90aa1..25fb04149fa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ FEATURES: [Stackdriver](https://cloud.google.com/stackdriver/). See the [configuration documentation](https://www.vaultproject.io/docs/config/index.html) for details. [GH-6957] + +CHANGES: + * sys/seal-status now has a `storage_type` field denoting what type of storage + the cluster is configured to use IMPROVEMENTS: From c4f53775afdf8cccb67d845901f1976caf3d476e Mon Sep 17 00:00:00 2001 From: Lars Lehtonen Date: Wed, 18 Sep 2019 14:18:08 -0700 Subject: [PATCH 08/46] Fix token_store_test.go (#7490) * vault: fix dropped error in test goroutine * vault: fix dropped test errors --- vault/token_store_test.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/vault/token_store_test.go b/vault/token_store_test.go index d03975d30b5d..4b16409eb296 100644 --- a/vault/token_store_test.go +++ b/vault/token_store_test.go @@ -849,6 +849,9 @@ func TestTokenStore_HandleRequest_RevokeAccessor(t *testing.T) { ts := exp.tokenStore rootToken, err := ts.rootToken(namespace.RootContext(nil)) + if err != nil { + t.Fatal(err) + } root := rootToken.ID testMakeServiceTokenViaBackend(t, ts, root, "tokenid", "", []string{"foo"}) @@ -2117,6 +2120,9 @@ func TestTokenStore_HandleRequest_Revoke(t *testing.T) { ts := exp.tokenStore rootToken, err := ts.rootToken(namespace.RootContext(nil)) + if err != nil { + t.Fatal(err) + } root := rootToken.ID testMakeServiceTokenViaBackend(t, ts, root, "child", "", []string{"root", "foo"}) @@ -2657,6 +2663,9 @@ func TestTokenStore_HandleRequest_CreateToken_ExistingEntityAlias(t *testing.T) "mount_accessor": tokenMountAccessor, }, }) + if err != nil { + t.Fatalf("error handling request: %v", err) + } // Create token role resp, err = core.HandleRequest(ctx, &logical.Request{ @@ -4992,17 +5001,18 @@ func TestTokenStore_RevokeUseCountToken(t *testing.T) { } cubbyFuncLock.Unlock() + errCh := make(chan error) + defer close(errCh) go func() { cubbyFuncLock.RLock() err := ts.revokeInternal(namespace.RootContext(nil), saltTut, false) cubbyFuncLock.RUnlock() - if err == nil { - t.Fatalf("expected error") - } + errCh <- err }() // Give time for the function to start and grab locks time.Sleep(200 * time.Millisecond) + te, err = ts.lookupInternal(namespace.RootContext(nil), saltTut, true, true) if err != nil { t.Fatal(err) @@ -5011,8 +5021,10 @@ func TestTokenStore_RevokeUseCountToken(t *testing.T) { t.Fatal("nil token entry") } - // Let things catch up - time.Sleep(2 * time.Second) + err = <-errCh + if err == nil { + t.Fatal("expected error on ts.revokeInternal() in anonymous goroutine") + } // Put back to normal cubbyFuncLock.Lock() From a65f4fed92dc58c21c69a9f07448693ee93128ac Mon Sep 17 00:00:00 2001 From: Graham Land Date: Wed, 18 Sep 2019 22:19:32 +0100 Subject: [PATCH 09/46] Early indication of storage backend requirements (#7472) A Vault Enterprise Pro customer in Japan has tried to get Vault DR replication working using Google Cloud Storage. They were frustrated to learn that GCS may not have support for transactional updates which has resulted in a lot of wasted time. The complaint was that this was not clear from our documentation. This note may help customers to understand sooner that not all highly available backends support transactional updates. --- website/source/docs/enterprise/replication/index.html.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/source/docs/enterprise/replication/index.html.md b/website/source/docs/enterprise/replication/index.html.md index c35dd0cf9ade..0549324e45da 100644 --- a/website/source/docs/enterprise/replication/index.html.md +++ b/website/source/docs/enterprise/replication/index.html.md @@ -23,6 +23,8 @@ applications that need to interoperate. Vault replication addresses both of these needs in providing consistency, scalability, and highly-available disaster recovery. +Note: Using replication requires a storage backend that supports transactional updates, such as Consul. + ## Architecture The core unit of Vault replication is a **cluster**, which is comprised of a From d6135f743fd604066c298c43c1631da0fd2a3b38 Mon Sep 17 00:00:00 2001 From: Jim Kalafut Date: Wed, 18 Sep 2019 14:24:41 -0700 Subject: [PATCH 10/46] Fix Agent handling of gzipped responses (#7470) * Fix Agent handling of gzipped responses Fixes #6606 * Only remove "gzip" member, if present * Simplify to just removing Accept-Encoding altogether --- command/agent/cache/api_proxy.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/command/agent/cache/api_proxy.go b/command/agent/cache/api_proxy.go index f0a919121a0a..580c14f75f2c 100644 --- a/command/agent/cache/api_proxy.go +++ b/command/agent/cache/api_proxy.go @@ -36,6 +36,11 @@ func (ap *APIProxy) Send(ctx context.Context, req *SendRequest) (*SendResponse, return nil, err } client.SetToken(req.Token) + + // http.Transport will transparently request gzip and decompress the response, but only if + // the client doesn't manually set the header. Removing any Accept-Encoding header allows the + // transparent compression to occur. + req.Request.Header.Del("Accept-Encoding") client.SetHeaders(req.Request.Header) fwReq := client.NewRequest(req.Request.Method, req.Request.URL.Path) From 645ac174deeb68545d65c4414ba89e02f5fe69d2 Mon Sep 17 00:00:00 2001 From: Yahya <5457202+anakaiti@users.noreply.github.com> Date: Thu, 19 Sep 2019 15:14:34 +0700 Subject: [PATCH 11/46] [Docs] Fix typo in database sample request (#7492) --- website/source/api/secret/databases/index.html.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/api/secret/databases/index.html.md b/website/source/api/secret/databases/index.html.md index e40e9aa0b53e..9a19f4e154d4 100644 --- a/website/source/api/secret/databases/index.html.md +++ b/website/source/api/secret/databases/index.html.md @@ -300,7 +300,7 @@ $ curl \ ```json { "data": { - "creation_statements": ["CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}';"], "GRANT SELECT ON ALL TABLES IN SCHEMA public TO \"{{name}}\";"], + "creation_statements": ["CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}';", "GRANT SELECT ON ALL TABLES IN SCHEMA public TO \"{{name}}\";"], "db_name": "mysql", "default_ttl": 3600, "max_ttl": 86400, From 4eefe0ebe1a1bf291c1d43927b63ad98c656b5f1 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Thu, 19 Sep 2019 09:42:45 -0400 Subject: [PATCH 12/46] Bump API's SDK --- api/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/go.mod b/api/go.mod index 9125a9757c22..8f92a43bc295 100644 --- a/api/go.mod +++ b/api/go.mod @@ -11,7 +11,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.6.2 github.com/hashicorp/go-rootcerts v1.0.1 github.com/hashicorp/hcl v1.0.0 - github.com/hashicorp/vault/sdk v0.1.14-0.20190918130944-9de946780bee + github.com/hashicorp/vault/sdk v0.1.14-0.20190919081434-645ac174deeb github.com/mitchellh/mapstructure v1.1.2 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 From 1aa86aa3fbc70cb242e10d4ebc62432844624c68 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Thu, 19 Sep 2019 09:43:23 -0400 Subject: [PATCH 13/46] Bump api/sdk and vendoring --- go.mod | 5 ++--- vendor/github.com/hashicorp/vault/api/go.mod | 2 +- vendor/github.com/hashicorp/vault/api/sys_seal.go | 1 + vendor/modules.txt | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index fe7b7ab86cc3..befb49285a50 100644 --- a/go.mod +++ b/go.mod @@ -81,8 +81,8 @@ require ( github.com/hashicorp/vault-plugin-secrets-gcp v0.5.3-0.20190814210141-d2086ff79b04 github.com/hashicorp/vault-plugin-secrets-gcpkms v0.5.2-0.20190814210149-315cdbf5de6e github.com/hashicorp/vault-plugin-secrets-kv v0.5.2-0.20190814210155-e060c2a001a8 - github.com/hashicorp/vault/api v1.0.5-0.20190918131023-e430c836412e - github.com/hashicorp/vault/sdk v0.1.14-0.20190918130944-9de946780bee + github.com/hashicorp/vault/api v1.0.5-0.20190919134245-4eefe0ebe1a1 + github.com/hashicorp/vault/sdk v0.1.14-0.20190919081434-645ac174deeb github.com/influxdata/influxdb v0.0.0-20190411212539-d24b7ba8c4c4 github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 // indirect github.com/jackc/pgx v3.3.0+incompatible // indirect @@ -91,7 +91,6 @@ require ( github.com/joyent/triton-go v0.0.0-20190112182421-51ffac552869 github.com/keybase/go-crypto v0.0.0-20190403132359-d65b6b94177f github.com/kr/pretty v0.1.0 - github.com/kr/pty v1.1.3 // indirect github.com/kr/text v0.1.0 github.com/lib/pq v1.2.0 github.com/mattn/go-colorable v0.1.2 diff --git a/vendor/github.com/hashicorp/vault/api/go.mod b/vendor/github.com/hashicorp/vault/api/go.mod index 9125a9757c22..8f92a43bc295 100644 --- a/vendor/github.com/hashicorp/vault/api/go.mod +++ b/vendor/github.com/hashicorp/vault/api/go.mod @@ -11,7 +11,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.6.2 github.com/hashicorp/go-rootcerts v1.0.1 github.com/hashicorp/hcl v1.0.0 - github.com/hashicorp/vault/sdk v0.1.14-0.20190918130944-9de946780bee + github.com/hashicorp/vault/sdk v0.1.14-0.20190919081434-645ac174deeb github.com/mitchellh/mapstructure v1.1.2 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 diff --git a/vendor/github.com/hashicorp/vault/api/sys_seal.go b/vendor/github.com/hashicorp/vault/api/sys_seal.go index 301d3f26a102..20d41a28f343 100644 --- a/vendor/github.com/hashicorp/vault/api/sys_seal.go +++ b/vendor/github.com/hashicorp/vault/api/sys_seal.go @@ -77,6 +77,7 @@ type SealStatusResponse struct { ClusterName string `json:"cluster_name,omitempty"` ClusterID string `json:"cluster_id,omitempty"` RecoverySeal bool `json:"recovery_seal"` + StorageType string `json:"storage_type,omitempty"` } type UnsealOpts struct { diff --git a/vendor/modules.txt b/vendor/modules.txt index b5f41a6749b4..e356d41ad552 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -373,9 +373,9 @@ github.com/hashicorp/vault-plugin-secrets-gcp/plugin/util github.com/hashicorp/vault-plugin-secrets-gcpkms # github.com/hashicorp/vault-plugin-secrets-kv v0.5.2-0.20190814210155-e060c2a001a8 github.com/hashicorp/vault-plugin-secrets-kv -# github.com/hashicorp/vault/api v1.0.5-0.20190918131023-e430c836412e => ./api +# github.com/hashicorp/vault/api v1.0.5-0.20190919134245-4eefe0ebe1a1 => ./api github.com/hashicorp/vault/api -# github.com/hashicorp/vault/sdk v0.1.14-0.20190918130944-9de946780bee => ./sdk +# github.com/hashicorp/vault/sdk v0.1.14-0.20190919081434-645ac174deeb => ./sdk github.com/hashicorp/vault/sdk/helper/salt github.com/hashicorp/vault/sdk/helper/strutil github.com/hashicorp/vault/sdk/helper/wrapping From 13c56f5f92a91bcc9a2ab02daed0910b1828c94f Mon Sep 17 00:00:00 2001 From: Dilan Bellinghoven Date: Thu, 19 Sep 2019 16:03:30 -0400 Subject: [PATCH 14/46] chore: Do not need logger for command/agent/config.LoadConfig (#7496) --- command/agent.go | 2 +- command/agent/config/config.go | 3 +- command/agent/config/config_test.go | 46 ++++++++--------------------- 3 files changed, 14 insertions(+), 37 deletions(-) diff --git a/command/agent.go b/command/agent.go index aba2319cba3a..e338ad86ed1a 100644 --- a/command/agent.go +++ b/command/agent.go @@ -192,7 +192,7 @@ func (c *AgentCommand) Run(args []string) int { } // Load the configuration - config, err := config.LoadConfig(c.flagConfigs[0], c.logger) + config, err := config.LoadConfig(c.flagConfigs[0]) if err != nil { c.UI.Error(fmt.Sprintf("Error loading configuration from %s: %s", c.flagConfigs[0], err)) return 1 diff --git a/command/agent/config/config.go b/command/agent/config/config.go index b8450b51e185..9f9fafa1a83a 100644 --- a/command/agent/config/config.go +++ b/command/agent/config/config.go @@ -9,7 +9,6 @@ import ( "time" "github.com/hashicorp/errwrap" - log "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-multierror" "github.com/hashicorp/hcl" "github.com/hashicorp/hcl/hcl/ast" @@ -77,7 +76,7 @@ type Sink struct { // LoadConfig loads the configuration at the given path, regardless if // its a file or directory. -func LoadConfig(path string, logger log.Logger) (*Config, error) { +func LoadConfig(path string) (*Config, error) { fi, err := os.Stat(path) if err != nil { return nil, err diff --git a/command/agent/config/config_test.go b/command/agent/config/config_test.go index aee2a0108ddf..ff133ae576c8 100644 --- a/command/agent/config/config_test.go +++ b/command/agent/config/config_test.go @@ -6,14 +6,10 @@ import ( "time" "github.com/go-test/deep" - log "github.com/hashicorp/go-hclog" - "github.com/hashicorp/vault/sdk/helper/logging" ) func TestLoadConfigFile_AgentCache(t *testing.T) { - logger := logging.NewVaultLogger(log.Debug) - - config, err := LoadConfig("./test-fixtures/config-cache.hcl", logger) + config, err := LoadConfig("./test-fixtures/config-cache.hcl") if err != nil { t.Fatal(err) } @@ -85,7 +81,7 @@ func TestLoadConfigFile_AgentCache(t *testing.T) { t.Fatal(diff) } - config, err = LoadConfig("./test-fixtures/config-cache-embedded-type.hcl", logger) + config, err = LoadConfig("./test-fixtures/config-cache-embedded-type.hcl") if err != nil { t.Fatal(err) } @@ -97,12 +93,10 @@ func TestLoadConfigFile_AgentCache(t *testing.T) { } func TestLoadConfigFile(t *testing.T) { - logger := logging.NewVaultLogger(log.Debug) - os.Setenv("TEST_AAD_ENV", "aad") defer os.Unsetenv("TEST_AAD_ENV") - config, err := LoadConfig("./test-fixtures/config.hcl", logger) + config, err := LoadConfig("./test-fixtures/config.hcl") if err != nil { t.Fatalf("err: %s", err) } @@ -146,7 +140,7 @@ func TestLoadConfigFile(t *testing.T) { t.Fatal(diff) } - config, err = LoadConfig("./test-fixtures/config-embedded-type.hcl", logger) + config, err = LoadConfig("./test-fixtures/config-embedded-type.hcl") if err != nil { t.Fatalf("err: %s", err) } @@ -157,9 +151,7 @@ func TestLoadConfigFile(t *testing.T) { } func TestLoadConfigFile_Method_Wrapping(t *testing.T) { - logger := logging.NewVaultLogger(log.Debug) - - config, err := LoadConfig("./test-fixtures/config-method-wrapping.hcl", logger) + config, err := LoadConfig("./test-fixtures/config-method-wrapping.hcl") if err != nil { t.Fatalf("err: %s", err) } @@ -192,9 +184,7 @@ func TestLoadConfigFile_Method_Wrapping(t *testing.T) { } func TestLoadConfigFile_AgentCache_NoAutoAuth(t *testing.T) { - logger := logging.NewVaultLogger(log.Debug) - - config, err := LoadConfig("./test-fixtures/config-cache-no-auto_auth.hcl", logger) + config, err := LoadConfig("./test-fixtures/config-cache-no-auto_auth.hcl") if err != nil { t.Fatalf("err: %s", err) } @@ -219,54 +209,42 @@ func TestLoadConfigFile_AgentCache_NoAutoAuth(t *testing.T) { } func TestLoadConfigFile_Bad_AgentCache_InconsisentAutoAuth(t *testing.T) { - logger := logging.NewVaultLogger(log.Debug) - - _, err := LoadConfig("./test-fixtures/bad-config-cache-inconsistent-auto_auth.hcl", logger) + _, err := LoadConfig("./test-fixtures/bad-config-cache-inconsistent-auto_auth.hcl") if err == nil { t.Fatal("LoadConfig should return an error when use_auto_auth_token=true and no auto_auth section present") } } func TestLoadConfigFile_Bad_AgentCache_NoListeners(t *testing.T) { - logger := logging.NewVaultLogger(log.Debug) - - _, err := LoadConfig("./test-fixtures/bad-config-cache-no-listeners.hcl", logger) + _, err := LoadConfig("./test-fixtures/bad-config-cache-no-listeners.hcl") if err == nil { t.Fatal("LoadConfig should return an error when cache section present and no listeners present") } } func TestLoadConfigFile_Bad_AutoAuth_Wrapped_Multiple_Sinks(t *testing.T) { - logger := logging.NewVaultLogger(log.Debug) - - _, err := LoadConfig("./test-fixtures/bad-config-auto_auth-wrapped-multiple-sinks", logger) + _, err := LoadConfig("./test-fixtures/bad-config-auto_auth-wrapped-multiple-sinks") if err == nil { t.Fatal("LoadConfig should return an error when auth_auth.method.wrap_ttl nonzero and multiple sinks defined") } } func TestLoadConfigFile_Bad_AutoAuth_Both_Wrapping_Types(t *testing.T) { - logger := logging.NewVaultLogger(log.Debug) - - _, err := LoadConfig("./test-fixtures/bad-config-method-wrapping-and-sink-wrapping.hcl", logger) + _, err := LoadConfig("./test-fixtures/bad-config-method-wrapping-and-sink-wrapping.hcl") if err == nil { t.Fatal("LoadConfig should return an error when auth_auth.method.wrap_ttl nonzero and sinks.wrap_ttl nonzero") } } func TestLoadConfigFile_Bad_AgentCache_AutoAuth_Method_wrapping(t *testing.T) { - logger := logging.NewVaultLogger(log.Debug) - - _, err := LoadConfig("./test-fixtures/bad-config-cache-auto_auth-method-wrapping.hcl", logger) + _, err := LoadConfig("./test-fixtures/bad-config-cache-auto_auth-method-wrapping.hcl") if err == nil { t.Fatal("LoadConfig should return an error when auth_auth.method.wrap_ttl nonzero and cache.use_auto_auth_token=true") } } func TestLoadConfigFile_AgentCache_AutoAuth_NoSink(t *testing.T) { - logger := logging.NewVaultLogger(log.Debug) - - config, err := LoadConfig("./test-fixtures/config-cache-auto_auth-no-sink.hcl", logger) + config, err := LoadConfig("./test-fixtures/config-cache-auto_auth-no-sink.hcl") if err != nil { t.Fatalf("err: %s", err) } From ad9e5ebee2acd5bca3c5c43e172037c7eb6d6867 Mon Sep 17 00:00:00 2001 From: Calvin Leung Huang Date: Thu, 19 Sep 2019 13:44:37 -0700 Subject: [PATCH 15/46] sys: add pprof endpoint (#7473) * sys/pprof: add pprof routes to the system backend * sys/pprof: add pprof paths to handler with local-only check * fix trailing slash on pprof index endpoint * use new no-forward handler on pprof * go mod tidy * add pprof external tests * disallow streaming requests to exceed DefaultMaxRequestDuration * add max request duration test --- http/handler.go | 4 + http/logical.go | 6 +- vault/external_tests/pprof/pprof_test.go | 173 +++++++++++++++++++ vault/logical_system.go | 1 + vault/logical_system_pprof.go | 210 +++++++++++++++++++++++ 5 files changed, 393 insertions(+), 1 deletion(-) create mode 100644 vault/external_tests/pprof/pprof_test.go create mode 100644 vault/logical_system_pprof.go diff --git a/http/handler.go b/http/handler.go index 2562c6a052c2..ef568065bb44 100644 --- a/http/handler.go +++ b/http/handler.go @@ -110,6 +110,10 @@ func Handler(props *vault.HandlerProperties) http.Handler { // Create the muxer to handle the actual endpoints mux := http.NewServeMux() + + // Handle pprof paths + mux.Handle("/v1/sys/pprof/", handleLogicalNoForward(core)) + mux.Handle("/v1/sys/init", handleSysInit(core)) mux.Handle("/v1/sys/seal-status", handleSysSealStatus(core)) mux.Handle("/v1/sys/seal", handleSysSeal(core)) diff --git a/http/logical.go b/http/logical.go index 156b8c4388a8..8b04dbb59ff4 100644 --- a/http/logical.go +++ b/http/logical.go @@ -59,7 +59,11 @@ func buildLogicalRequest(core *vault.Core, w http.ResponseWriter, r *http.Reques data = parseQuery(queryVals) } - if path == "sys/storage/raft/snapshot" { + switch { + case strings.HasPrefix(path, "sys/pprof/"): + passHTTPReq = true + responseWriter = w + case path == "sys/storage/raft/snapshot": responseWriter = w } diff --git a/vault/external_tests/pprof/pprof_test.go b/vault/external_tests/pprof/pprof_test.go new file mode 100644 index 000000000000..5da8c2bb3aca --- /dev/null +++ b/vault/external_tests/pprof/pprof_test.go @@ -0,0 +1,173 @@ +package pprof + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "strconv" + "strings" + "testing" + + "github.com/hashicorp/go-cleanhttp" + vaulthttp "github.com/hashicorp/vault/http" + "github.com/hashicorp/vault/vault" + "golang.org/x/net/http2" +) + +func TestSysPprof(t *testing.T) { + cluster := vault.NewTestCluster(t, nil, &vault.TestClusterOptions{ + HandlerFunc: vaulthttp.Handler, + }) + cluster.Start() + defer cluster.Cleanup() + + core := cluster.Cores[0].Core + vault.TestWaitActive(t, core) + client := cluster.Cores[0].Client + + transport := cleanhttp.DefaultPooledTransport() + transport.TLSClientConfig = cluster.Cores[0].TLSConfig.Clone() + if err := http2.ConfigureTransport(transport); err != nil { + t.Fatal(err) + } + httpClient := &http.Client{ + Transport: transport, + } + + cases := []struct { + name string + path string + seconds string + }{ + { + "index", + "/v1/sys/pprof/", + "", + }, + { + "cmdline", + "/v1/sys/pprof/cmdline", + "", + }, + { + "goroutine", + "/v1/sys/pprof/goroutine", + "", + }, + { + "heap", + "/v1/sys/pprof/heap", + "", + }, + { + "profile", + "/v1/sys/pprof/profile", + "1", + }, + { + "symbol", + "/v1/sys/pprof/symbol", + "", + }, + { + "trace", + "/v1/sys/pprof/trace", + "1", + }, + } + + pprofRequest := func(path string, seconds string) { + req := client.NewRequest("GET", path) + if seconds != "" { + req.Params.Set("seconds", seconds) + } + httpReq, err := req.ToHTTP() + if err != nil { + t.Fatal(err) + } + resp, err := httpClient.Do(httpReq) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + httpRespBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + + httpResp := make(map[string]interface{}) + + // Skip this error check since some endpoints return binary blobs, we + // only care about the ok check right after as an existence check. + _ = json.Unmarshal(httpRespBody, &httpResp) + + // Make sure that we don't get a error response + if _, ok := httpResp["errors"]; ok { + t.Fatalf("unexpected error response: %v", httpResp["errors"]) + } + + if len(httpRespBody) == 0 { + t.Fatal("no pprof index returned") + } + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + pprofRequest(tc.path, tc.seconds) + }) + } +} + +func TestSysPprof_MaxRequestDuration(t *testing.T) { + cluster := vault.NewTestCluster(t, nil, &vault.TestClusterOptions{ + HandlerFunc: vaulthttp.Handler, + }) + cluster.Start() + defer cluster.Cleanup() + client := cluster.Cores[0].Client + + transport := cleanhttp.DefaultPooledTransport() + transport.TLSClientConfig = cluster.Cores[0].TLSConfig.Clone() + if err := http2.ConfigureTransport(transport); err != nil { + t.Fatal(err) + } + httpClient := &http.Client{ + Transport: transport, + } + + sec := strconv.Itoa(int(vault.DefaultMaxRequestDuration.Seconds()) + 1) + + req := client.NewRequest("GET", "/v1/sys/pprof/profile") + req.Params.Set("seconds", sec) + httpReq, err := req.ToHTTP() + if err != nil { + t.Fatal(err) + } + resp, err := httpClient.Do(httpReq) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + httpRespBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + + httpResp := make(map[string]interface{}) + + // If we error here, it means that profiling likely happened, which is not + // what we're checking for in this case. + if err := json.Unmarshal(httpRespBody, &httpResp); err != nil { + t.Fatalf("expected valid error response, got: %v", err) + } + + errs, ok := httpResp["errors"].([]interface{}) + if !ok { + t.Fatalf("expected error response, got: %v", httpResp) + } + if len(errs) == 0 || !strings.Contains(errs[0].(string), "exceeds max request duration") { + t.Fatalf("unexptected error returned: %v", errs) + } +} diff --git a/vault/logical_system.go b/vault/logical_system.go index d331c253c4ae..77a47fbced4e 100644 --- a/vault/logical_system.go +++ b/vault/logical_system.go @@ -164,6 +164,7 @@ func NewSystemBackend(core *Core, logger log.Logger) *SystemBackend { b.Backend.Paths = append(b.Backend.Paths, b.toolsPaths()...) b.Backend.Paths = append(b.Backend.Paths, b.capabilitiesPaths()...) b.Backend.Paths = append(b.Backend.Paths, b.internalPaths()...) + b.Backend.Paths = append(b.Backend.Paths, b.pprofPaths()...) b.Backend.Paths = append(b.Backend.Paths, b.remountPath()) b.Backend.Paths = append(b.Backend.Paths, b.metricsPath()) diff --git a/vault/logical_system_pprof.go b/vault/logical_system_pprof.go new file mode 100644 index 000000000000..87c892dc2f41 --- /dev/null +++ b/vault/logical_system_pprof.go @@ -0,0 +1,210 @@ +package vault + +import ( + "context" + "errors" + "fmt" + "net/http/pprof" + "strconv" + + "github.com/hashicorp/vault/sdk/framework" + "github.com/hashicorp/vault/sdk/logical" +) + +func (b *SystemBackend) pprofPaths() []*framework.Path { + return []*framework.Path{ + { + Pattern: "pprof/$", + + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.handlePprofIndex, + Summary: "Returns the running program's command line.", + Description: "Returns the running program's command line, with arguments separated by NUL bytes.", + }, + }, + }, + { + Pattern: "pprof/cmdline", + + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.handlePprofCmdline, + Summary: "Returns the running program's command line.", + Description: "Returns the running program's command line, with arguments separated by NUL bytes.", + }, + }, + }, + { + Pattern: "pprof/goroutine", + + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.handlePprofGoroutine, + Summary: "Returns stack traces of all current goroutines.", + Description: "Returns stack traces of all current goroutines.", + }, + }, + }, + { + Pattern: "pprof/heap", + + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.handlePprofHeap, + Summary: "Returns a sampling of memory allocations of live object.", + Description: "Returns a sampling of memory allocations of live object.", + }, + }, + }, + { + Pattern: "pprof/profile", + + Fields: map[string]*framework.FieldSchema{ + "seconds": { + Type: framework.TypeInt, + Description: "If provided, specifies the duration to run the profiling command.", + }, + }, + + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.handlePprofProfile, + Summary: "Returns a pprof-formatted cpu profile payload.", + Description: "Returns a pprof-formatted cpu profile payload. Profiling lasts for duration specified in seconds GET parameter, or for 30 seconds if not specified.", + }, + }, + }, + { + Pattern: "pprof/symbol", + + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.handlePprofSymbol, + Summary: "Returns the program counters listed in the request.", + Description: "Returns the program counters listed in the request.", + }, + }, + }, + + { + Pattern: "pprof/trace", + + Fields: map[string]*framework.FieldSchema{ + "seconds": { + Type: framework.TypeInt, + Description: "If provided, specifies the duration to run the tracing command.", + }, + }, + + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.handlePprofTrace, + Summary: "Returns the execution trace in binary form.", + Description: "Returns the execution trace in binary form. Tracing lasts for duration specified in seconds GET parameter, or for 1 second if not specified.", + }, + }, + }, + } +} + +func (b *SystemBackend) handlePprofIndex(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { + if err := checkRequestHandlerParams(req); err != nil { + return nil, err + } + + pprof.Index(req.ResponseWriter, req.HTTPRequest) + return nil, nil +} + +func (b *SystemBackend) handlePprofCmdline(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { + if err := checkRequestHandlerParams(req); err != nil { + return nil, err + } + + pprof.Cmdline(req.ResponseWriter, req.HTTPRequest) + return nil, nil +} + +func (b *SystemBackend) handlePprofGoroutine(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { + if err := checkRequestHandlerParams(req); err != nil { + return nil, err + } + + pprof.Handler("goroutine").ServeHTTP(req.ResponseWriter, req.HTTPRequest) + return nil, nil +} + +func (b *SystemBackend) handlePprofHeap(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { + if err := checkRequestHandlerParams(req); err != nil { + return nil, err + } + + pprof.Handler("heap").ServeHTTP(req.ResponseWriter, req.HTTPRequest) + return nil, nil +} + +func (b *SystemBackend) handlePprofProfile(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { + if err := checkRequestHandlerParams(req); err != nil { + return nil, err + } + + // Return an error if seconds exceeds max request duration. This follows a + // similar behavior to how pprof treats seconds > WriteTimeout (i.e. it + // error with a 400), and avoids drift between what gets audited vs what + // ends up happening. + if secQueryVal := req.HTTPRequest.FormValue("seconds"); secQueryVal != "" { + maxDur := int64(DefaultMaxRequestDuration.Seconds()) + sec, _ := strconv.ParseInt(secQueryVal, 10, 64) + if sec > maxDur { + return logical.ErrorResponse(fmt.Sprintf("seconds %d exceeds max request duration of %d", sec, maxDur)), nil + } + } + + pprof.Profile(req.ResponseWriter, req.HTTPRequest) + return nil, nil +} + +func (b *SystemBackend) handlePprofSymbol(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { + if err := checkRequestHandlerParams(req); err != nil { + return nil, err + } + + pprof.Symbol(req.ResponseWriter, req.HTTPRequest) + return nil, nil +} + +func (b *SystemBackend) handlePprofTrace(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { + if err := checkRequestHandlerParams(req); err != nil { + return nil, err + } + + // Return an error if seconds exceeds max request duration. This follows a + // similar behavior to how pprof treats seconds > WriteTimeout (i.e. it + // error with a 400), and avoids drift between what gets audited vs what + // ends up happening. + if secQueryVal := req.HTTPRequest.FormValue("seconds"); secQueryVal != "" { + maxDur := int64(DefaultMaxRequestDuration.Seconds()) + sec, _ := strconv.ParseInt(secQueryVal, 10, 64) + if sec > maxDur { + return logical.ErrorResponse(fmt.Sprintf("seconds %d exceeds max request duration of %d", sec, maxDur)), nil + } + } + + pprof.Trace(req.ResponseWriter, req.HTTPRequest) + return nil, nil +} + +// checkRequestHandlerParams is a helper that checks for the existence of the +// HTTP request and response writer in a logical.Request. +func checkRequestHandlerParams(req *logical.Request) error { + if req.ResponseWriter == nil { + return errors.New("no writer for request") + } + + if req.HTTPRequest == nil || req.HTTPRequest.Body == nil { + return errors.New("no reader for request") + } + + return nil +} From 961f4468385aa08848baee1b27e64b93904c8bd7 Mon Sep 17 00:00:00 2001 From: Joel Thompson Date: Thu, 19 Sep 2019 16:35:12 -0700 Subject: [PATCH 16/46] secret/aws: Support permissions boundaries on iam_user creds (#6786) * secrets/aws: Support permissions boundaries on iam_user creds This allows configuring Vault to attach a permissions boundary policy to IAM users that it creates, configured on a per-Vault-role basis. * Fix indentation of policy in docs Use spaces instead of tabs --- builtin/logical/aws/backend_test.go | 65 ++++++--- builtin/logical/aws/path_roles.go | 54 +++++++- builtin/logical/aws/path_roles_test.go | 131 +++++++++++++++++- builtin/logical/aws/secret_access_keys.go | 11 +- website/source/api/secret/aws/index.html.md | 6 + website/source/docs/secrets/aws/index.html.md | 63 ++++++++- 6 files changed, 289 insertions(+), 41 deletions(-) diff --git a/builtin/logical/aws/backend_test.go b/builtin/logical/aws/backend_test.go index ec14fc198e1e..5c372e513f23 100644 --- a/builtin/logical/aws/backend_test.go +++ b/builtin/logical/aws/backend_test.go @@ -57,6 +57,25 @@ func TestBackend_basic(t *testing.T) { }) } +func TestBackend_IamUserWithPermissionsBoundary(t *testing.T) { + t.Parallel() + roleData := map[string]interface{}{ + "credential_type": iamUserCred, + "policy_arns": adminAccessPolicyArn, + "permissions_boundary_arn": iamPolicyArn, + } + logicaltest.Test(t, logicaltest.TestCase{ + AcceptanceTest: true, + PreCheck: func() { testAccPreCheck(t) }, + LogicalBackend: getBackend(t), + Steps: []logicaltest.TestStep{ + testAccStepConfig(t), + testAccStepWriteRole(t, "test", roleData), + testAccStepRead(t, "creds", "test", []credentialTestFunc{listIamUsersTest, describeAzsTestUnauthorized}), + }, + }) +} + func TestBackend_basicSTS(t *testing.T) { t.Parallel() awsAccountID, err := getAccountID() @@ -681,13 +700,14 @@ func testAccStepReadPolicy(t *testing.T, name string, value string) logicaltest. } expected := map[string]interface{}{ - "policy_arns": []string(nil), - "role_arns": []string(nil), - "policy_document": value, - "credential_type": strings.Join([]string{iamUserCred, federationTokenCred}, ","), - "default_sts_ttl": int64(0), - "max_sts_ttl": int64(0), - "user_path": "", + "policy_arns": []string(nil), + "role_arns": []string(nil), + "policy_document": value, + "credential_type": strings.Join([]string{iamUserCred, federationTokenCred}, ","), + "default_sts_ttl": int64(0), + "max_sts_ttl": int64(0), + "user_path": "", + "permissions_boundary_arn": "", } if !reflect.DeepEqual(resp.Data, expected) { return fmt.Errorf("bad: got: %#v\nexpected: %#v", resp.Data, expected) @@ -714,6 +734,7 @@ const testDynamoPolicy = `{ } ` +const adminAccessPolicyArn = "arn:aws:iam::aws:policy/AdministratorAccess" const ec2PolicyArn = "arn:aws:iam::aws:policy/AmazonEC2ReadOnlyAccess" const iamPolicyArn = "arn:aws:iam::aws:policy/IAMReadOnlyAccess" const dynamoPolicyArn = "arn:aws:iam::aws:policy/AmazonDynamoDBReadOnlyAccess" @@ -782,13 +803,14 @@ func TestBackend_iamUserManagedInlinePolicies(t *testing.T) { "user_path": "/path/", } expectedRoleData := map[string]interface{}{ - "policy_document": compacted, - "policy_arns": []string{ec2PolicyArn, iamPolicyArn}, - "credential_type": iamUserCred, - "role_arns": []string(nil), - "default_sts_ttl": int64(0), - "max_sts_ttl": int64(0), - "user_path": "/path/", + "policy_document": compacted, + "policy_arns": []string{ec2PolicyArn, iamPolicyArn}, + "credential_type": iamUserCred, + "role_arns": []string(nil), + "default_sts_ttl": int64(0), + "max_sts_ttl": int64(0), + "user_path": "/path/", + "permissions_boundary_arn": "", } logicaltest.Test(t, logicaltest.TestCase{ @@ -986,13 +1008,14 @@ func testAccStepReadArnPolicy(t *testing.T, name string, value string) logicalte } expected := map[string]interface{}{ - "policy_arns": []string{value}, - "role_arns": []string(nil), - "policy_document": "", - "credential_type": iamUserCred, - "default_sts_ttl": int64(0), - "max_sts_ttl": int64(0), - "user_path": "", + "policy_arns": []string{value}, + "role_arns": []string(nil), + "policy_document": "", + "credential_type": iamUserCred, + "default_sts_ttl": int64(0), + "max_sts_ttl": int64(0), + "user_path": "", + "permissions_boundary_arn": "", } if !reflect.DeepEqual(resp.Data, expected) { return fmt.Errorf("bad: got: %#v\nexpected: %#v", resp.Data, expected) diff --git a/builtin/logical/aws/path_roles.go b/builtin/logical/aws/path_roles.go index 915914222403..6df1459cc1c9 100644 --- a/builtin/logical/aws/path_roles.go +++ b/builtin/logical/aws/path_roles.go @@ -96,6 +96,14 @@ GetFederationToken API call, acting as a filter on permissions available.`, }, }, + "permissions_boundary_arn": &framework.FieldSchema{ + Type: framework.TypeString, + Description: "ARN of an IAM policy to attach as a permissions boundary on IAM user credentials; only valid when credential_type is" + iamUserCred, + DisplayAttrs: &framework.DisplayAttributes{ + Name: "Permissions Boundary ARN", + }, + }, + "arn": &framework.FieldSchema{ Type: framework.TypeString, Description: `Use role_arns or policy_arns instead.`, @@ -269,6 +277,13 @@ func (b *backend) pathRolesWrite(ctx context.Context, req *logical.Request, d *f roleEntry.UserPath = userPathRaw.(string) } + if permissionsBoundaryARNRaw, ok := d.GetOk("permissions_boundary_arn"); ok { + if legacyRole != "" { + return logical.ErrorResponse("cannot supply deprecated role or policy parameters with permissions_boundary_arn"), nil + } + roleEntry.PermissionsBoundaryARN = permissionsBoundaryARNRaw.(string) + } + if legacyRole != "" { roleEntry = upgradeLegacyPolicyEntry(legacyRole) if roleEntry.InvalidData != "" { @@ -414,6 +429,20 @@ func upgradeLegacyPolicyEntry(entry string) *awsRoleEntry { return newRoleEntry } +func validateAWSManagedPolicy(policyARN string) error { + parsedARN, err := arn.Parse(policyARN) + if err != nil { + return err + } + if parsedARN.Service != "iam" { + return fmt.Errorf("expected a service of iam but got %s", parsedARN.Service) + } + if !strings.HasPrefix(parsedARN.Resource, "policy/") { + return fmt.Errorf("expected a resource type of policy but got %s", parsedARN.Resource) + } + return nil +} + func setAwsRole(ctx context.Context, s logical.Storage, roleName string, roleEntry *awsRoleEntry) error { if roleName == "" { return fmt.Errorf("empty role name") @@ -445,17 +474,19 @@ type awsRoleEntry struct { DefaultSTSTTL time.Duration `json:"default_sts_ttl"` // Default TTL for STS credentials MaxSTSTTL time.Duration `json:"max_sts_ttl"` // Max allowed TTL for STS credentials UserPath string `json:"user_path"` // The path for the IAM user when using "iam_user" credential type + PermissionsBoundaryARN string `json:"permissions_boundary_arn"` // ARN of an IAM policy to attach as a permissions boundary } func (r *awsRoleEntry) toResponseData() map[string]interface{} { respData := map[string]interface{}{ - "credential_type": strings.Join(r.CredentialTypes, ","), - "policy_arns": r.PolicyArns, - "role_arns": r.RoleArns, - "policy_document": r.PolicyDocument, - "default_sts_ttl": int64(r.DefaultSTSTTL.Seconds()), - "max_sts_ttl": int64(r.MaxSTSTTL.Seconds()), - "user_path": r.UserPath, + "credential_type": strings.Join(r.CredentialTypes, ","), + "policy_arns": r.PolicyArns, + "role_arns": r.RoleArns, + "policy_document": r.PolicyDocument, + "default_sts_ttl": int64(r.DefaultSTSTTL.Seconds()), + "max_sts_ttl": int64(r.MaxSTSTTL.Seconds()), + "user_path": r.UserPath, + "permissions_boundary_arn": r.PermissionsBoundaryARN, } if r.InvalidData != "" { @@ -501,6 +532,15 @@ func (r *awsRoleEntry) validate() error { } } + if r.PermissionsBoundaryARN != "" { + if !strutil.StrListContains(r.CredentialTypes, iamUserCred) { + errors = multierror.Append(errors, fmt.Errorf("cannot supply permissions_boundary_arn when credential_type isn't %s", iamUserCred)) + } + if err := validateAWSManagedPolicy(r.PermissionsBoundaryARN); err != nil { + errors = multierror.Append(fmt.Errorf("invalid permissions_boundary_arn parameter: %v", err)) + } + } + if len(r.RoleArns) > 0 && !strutil.StrListContains(r.CredentialTypes, assumedRoleCred) { errors = multierror.Append(errors, fmt.Errorf("cannot supply role_arns when credential_type isn't %s", assumedRoleCred)) } diff --git a/builtin/logical/aws/path_roles_test.go b/builtin/logical/aws/path_roles_test.go index 246dfa2d3698..d280dfcd0d1e 100644 --- a/builtin/logical/aws/path_roles_test.go +++ b/builtin/logical/aws/path_roles_test.go @@ -10,6 +10,8 @@ import ( "github.com/hashicorp/vault/sdk/logical" ) +const adminAccessPolicyARN = "arn:aws:iam::aws:policy/AdministratorAccess" + func TestBackend_PathListRoles(t *testing.T) { var resp *logical.Response var err error @@ -214,10 +216,114 @@ func TestUserPathValidity(t *testing.T) { } } +func TestRoleCRUDWithPermissionsBoundary(t *testing.T) { + roleName := "test_perm_boundary" + + config := logical.TestBackendConfig() + config.StorageView = &logical.InmemStorage{} + + b := Backend() + if err := b.Setup(context.Background(), config); err != nil { + t.Fatal(err) + } + + permissionsBoundaryARN := "arn:aws:iam::aws:policy/EC2FullAccess" + + roleData := map[string]interface{}{ + "credential_type": iamUserCred, + "policy_arns": []string{adminAccessPolicyARN}, + "permissions_boundary_arn": permissionsBoundaryARN, + } + request := &logical.Request{ + Operation: logical.UpdateOperation, + Path: "roles/" + roleName, + Storage: config.StorageView, + Data: roleData, + } + resp, err := b.HandleRequest(context.Background(), request) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatalf("bad: role creation failed. resp:%#v\nerr:%v", resp, err) + } + + request = &logical.Request{ + Operation: logical.ReadOperation, + Path: "roles/" + roleName, + Storage: config.StorageView, + } + resp, err = b.HandleRequest(context.Background(), request) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatalf("bad: reading role failed. resp:%#v\nerr:%v", resp, err) + } + if resp.Data["credential_type"] != iamUserCred { + t.Errorf("bad: expected credential_type of %s, got %s instead", iamUserCred, resp.Data["credential_type"]) + } + if resp.Data["permissions_boundary_arn"] != permissionsBoundaryARN { + t.Errorf("bad: expected permissions_boundary_arn of %s, got %s instead", permissionsBoundaryARN, resp.Data["permissions_boundary_arn"]) + } +} + +func TestRoleWithPermissionsBoundaryValidation(t *testing.T) { + config := logical.TestBackendConfig() + config.StorageView = &logical.InmemStorage{} + + b := Backend() + if err := b.Setup(context.Background(), config); err != nil { + t.Fatal(err) + } + + roleData := map[string]interface{}{ + "credential_type": assumedRoleCred, // only iamUserCred supported with permissions_boundary_arn + "role_arns": []string{"arn:aws:iam::123456789012:role/VaultRole"}, + "permissions_boundary_arn": "arn:aws:iam::aws:policy/FooBar", + } + request := &logical.Request{ + Operation: logical.UpdateOperation, + Path: "roles/test_perm_boundary", + Storage: config.StorageView, + Data: roleData, + } + resp, err := b.HandleRequest(context.Background(), request) + if err == nil && (resp == nil || !resp.IsError()) { + t.Fatalf("bad: expected role creation to fail due to bad credential_type, but it didn't. resp:%#v\nerr:%v", resp, err) + } + + roleData = map[string]interface{}{ + "credential_type": iamUserCred, + "policy_arns": []string{adminAccessPolicyARN}, + "permissions_boundary_arn": "arn:aws:notiam::aws:policy/FooBar", + } + request.Data = roleData + resp, err = b.HandleRequest(context.Background(), request) + if err == nil && (resp == nil || !resp.IsError()) { + t.Fatalf("bad: expected role creation to fail due to malformed permissions_boundary_arn, but it didn't. resp:%#v\nerr:%v", resp, err) + } +} + +func TestValidateAWSManagedPolicy(t *testing.T) { + expectErr := func(arn string) { + err := validateAWSManagedPolicy(arn) + if err == nil { + t.Errorf("bad: expected arn of %s to return an error but it didn't", arn) + } + } + + expectErr("not_an_arn") + expectErr("notarn:aws:iam::aws:policy/FooBar") + expectErr("arn:aws:notiam::aws:policy/FooBar") + expectErr("arn:aws:iam::aws:notpolicy/FooBar") + expectErr("arn:aws:iam::aws:policynot/FooBar") + + arn := "arn:aws:iam::aws:policy/FooBar" + err := validateAWSManagedPolicy(arn) + if err != nil { + t.Errorf("bad: expected arn of %s to not return an error but it did: %#v", arn, err) + } +} + func TestRoleEntryValidationCredTypes(t *testing.T) { roleEntry := awsRoleEntry{ CredentialTypes: []string{}, - PolicyArns: []string{"arn:aws:iam::aws:policy/AdministratorAccess"}, + PolicyArns: []string{adminAccessPolicyARN}, } if roleEntry.validate() == nil { t.Errorf("bad: invalid roleEntry with no CredentialTypes %#v passed validation", roleEntry) @@ -234,10 +340,10 @@ func TestRoleEntryValidationCredTypes(t *testing.T) { func TestRoleEntryValidationIamUserCred(t *testing.T) { var allowAllPolicyDocument = `{"Version": "2012-10-17", "Statement": [{"Sid": "AllowAll", "Effect": "Allow", "Action": "*", "Resource": "*"}]}` - roleEntry := awsRoleEntry{ - CredentialTypes: []string{iamUserCred}, - PolicyArns: []string{"arn:aws:iam::aws:policy/AdministratorAccess"}, + CredentialTypes: []string{iamUserCred}, + PolicyArns: []string{adminAccessPolicyARN}, + PermissionsBoundaryARN: adminAccessPolicyARN, } err := roleEntry.validate() if err != nil { @@ -264,7 +370,7 @@ func TestRoleEntryValidationIamUserCred(t *testing.T) { roleEntry = awsRoleEntry{ CredentialTypes: []string{iamUserCred}, - PolicyArns: []string{"arn:aws:iam::aws:policy/AdministratorAccess"}, + PolicyArns: []string{adminAccessPolicyARN}, DefaultSTSTTL: 1, } if roleEntry.validate() == nil { @@ -282,7 +388,7 @@ func TestRoleEntryValidationAssumedRoleCred(t *testing.T) { roleEntry := awsRoleEntry{ CredentialTypes: []string{assumedRoleCred}, RoleArns: []string{"arn:aws:iam::123456789012:role/SomeRole"}, - PolicyArns: []string{"arn:aws:iam::aws:policy/AdministratorAccess"}, + PolicyArns: []string{adminAccessPolicyARN}, PolicyDocument: allowAllPolicyDocument, DefaultSTSTTL: 2, MaxSTSTTL: 3, @@ -300,6 +406,11 @@ func TestRoleEntryValidationAssumedRoleCred(t *testing.T) { if roleEntry.validate() == nil { t.Errorf("bad: invalid roleEntry with unrecognized UserPath %#v passed validation", roleEntry) } + roleEntry.UserPath = "" + roleEntry.PermissionsBoundaryARN = adminAccessPolicyARN + if roleEntry.validate() == nil { + t.Errorf("bad: invalid roleEntry with unrecognized PermissionsBoundary %#v passed validation", roleEntry) + } } func TestRoleEntryValidationFederationTokenCred(t *testing.T) { @@ -307,7 +418,7 @@ func TestRoleEntryValidationFederationTokenCred(t *testing.T) { roleEntry := awsRoleEntry{ CredentialTypes: []string{federationTokenCred}, PolicyDocument: allowAllPolicyDocument, - PolicyArns: []string{"arn:aws:iam::aws:policy/AdministratorAccess"}, + PolicyArns: []string{adminAccessPolicyARN}, DefaultSTSTTL: 2, MaxSTSTTL: 3, } @@ -330,4 +441,10 @@ func TestRoleEntryValidationFederationTokenCred(t *testing.T) { if roleEntry.validate() == nil { t.Errorf("bad: invalid roleEntry with MaxSTSTTL < DefaultSTSTTL %#v passed validation", roleEntry) } + roleEntry.MaxSTSTTL = 0 + roleEntry.PermissionsBoundaryARN = adminAccessPolicyARN + if roleEntry.validate() == nil { + t.Errorf("bad: invalid roleEntry with unrecognized PermissionsBoundary %#v passed validation", roleEntry) + } + } diff --git a/builtin/logical/aws/secret_access_keys.go b/builtin/logical/aws/secret_access_keys.go index 3d8f1261781b..c512aaa4f723 100644 --- a/builtin/logical/aws/secret_access_keys.go +++ b/builtin/logical/aws/secret_access_keys.go @@ -202,11 +202,16 @@ func (b *backend) secretAccessKeysCreate( userPath = "/" } - // Create the user - _, err = iamClient.CreateUser(&iam.CreateUserInput{ + createUserRequest := &iam.CreateUserInput{ UserName: aws.String(username), Path: aws.String(userPath), - }) + } + if role.PermissionsBoundaryARN != "" { + createUserRequest.PermissionsBoundary = aws.String(role.PermissionsBoundaryARN) + } + + // Create the user + _, err = iamClient.CreateUser(createUserRequest) if err != nil { if walErr := framework.DeleteWAL(ctx, s, walID); walErr != nil { iamErr := errwrap.Wrapf("error creating IAM user: {{err}}", err) diff --git a/website/source/api/secret/aws/index.html.md b/website/source/api/secret/aws/index.html.md index e2bbbf30b4f1..dc7f731a8e9a 100644 --- a/website/source/api/secret/aws/index.html.md +++ b/website/source/api/secret/aws/index.html.md @@ -266,6 +266,12 @@ updated with the new attributes. - `user_path` `(string)` - The path for the user name. Valid only when `credential_type` is `iam_user`. Default is `/` +- `permissions_boundary_arn` `(string)` - The ARN of the [AWS Permissions + Boundary](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_boundaries.html) + to attach to IAM users created in the role. Valid only when `credential_type` + is `iam_user`. If not specified, then no permissions boundary policy will be + attached. + Legacy parameters: These parameters are supported for backwards compatibility only. They cannot be diff --git a/website/source/docs/secrets/aws/index.html.md b/website/source/docs/secrets/aws/index.html.md index 68cfcc8cf9da..389e5c6e85de 100644 --- a/website/source/docs/secrets/aws/index.html.md +++ b/website/source/docs/secrets/aws/index.html.md @@ -19,9 +19,13 @@ and are automatically revoked when the Vault lease expires. Vault supports three different types of credentials to retrieve from AWS: 1. `iam_user`: Vault will create an IAM user for each lease, attach the managed - and inline IAM policies as specified in the role to the user, and then return - the access key and secret key to the caller. IAM users have no session tokens - and so no session token will be returned. + and inline IAM policies as specified in the role to the user, and if a + [permissions + boundary](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_boundaries.html) + is specified on the role, the permissions boundary will also be attached. + Vault will then generate an access key and secret key for the IAM user and + return them to the caller. IAM users have no session tokens and so no + session token will be returned. 2. `assumed_role`: Vault will call [sts:AssumeRole](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html) and return the access key, secret key, and session token to the caller. @@ -185,6 +189,59 @@ permissions Vault needs: } ``` +Vault also supports AWS Permissions Boundaries when creating IAM users. If you +wish to enforce that Vault always attaches a permissions boundary to an IAM +user, you can use a policy like: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "iam:CreateAccessKey", + "iam:DeleteAccessKey", + "iam:DeleteUser", + "iam:ListAccessKeys", + "iam:ListAttachedUserPolicies", + "iam:ListGroupsForUser", + "iam:ListUserPolicies", + "iam:RemoveUserFromGroup" + ], + "Resource": [ + "arn:aws:iam::ACCOUNT-ID-WITHOUT-HYPHENS:user/vault-*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "iam:AttachUserPolicy", + "iam:CreateUser", + "iam:DeleteUserPolicy", + "iam:DetachUserPolicy", + "iam:PutUserPolicy" + ], + "Resource": [ + "arn:aws:iam::ACCOUNT-ID-WITHOUT-HYPHENS:user/vault-*" + ], + "Condition": { + "StringEquals": { + "iam:PermissionsBoundary": [ + "arn:aws:iam::ACCOUNT-ID-WITHOUT-HYPHENS:policy/PolicyName" + ] + } + } + } + ] +} +``` + +where the "iam:PermissionsBoundary" condition contains the list of permissions +boundary policies that you wish to ensure that Vault uses. This policy will +ensure that Vault uses one of the permissions boundaries specified (not all of +them). + ## STS credentials The above demonstrated usage with `iam_user` credential types. As mentioned, From ba834f8a0fc4bc6e05e12526e7d466d7f773708f Mon Sep 17 00:00:00 2001 From: Vu Pham <51802661+vuhphamw@users.noreply.github.com> Date: Mon, 23 Sep 2019 16:02:08 -0700 Subject: [PATCH 17/46] Use snake case for HA example (#7505) --- .../docs/configuration/storage/oci-object-storage.html.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/source/docs/configuration/storage/oci-object-storage.html.md b/website/source/docs/configuration/storage/oci-object-storage.html.md index f2c3986f5842..9f73a3f5e0c4 100644 --- a/website/source/docs/configuration/storage/oci-object-storage.html.md +++ b/website/source/docs/configuration/storage/oci-object-storage.html.md @@ -75,10 +75,10 @@ This example shows configuring OCI Object Storage with high availability enabled ```hcl storage "oci_objectstorage" { - namespaceName = "MyNamespace - bucketName = "DataBucket" + namespace_name = "MyNamespace + bucket_name = "DataBucket" ha_enabled = "true" - lockBucketName = "LockBucket" + lock_bucket_name = "LockBucket" } ``` From 6a14065f739708624f7add8ac4ff4c4361e4e4c7 Mon Sep 17 00:00:00 2001 From: Jim Kalafut Date: Wed, 25 Sep 2019 10:59:42 -0700 Subject: [PATCH 18/46] Ignore any existing token during CLI login (#7508) Fixes #6694 --- command/login.go | 6 ++++++ command/login_test.go | 3 +++ 2 files changed, 9 insertions(+) diff --git a/command/login.go b/command/login.go index bafc670c6d29..dc9a2daafb9b 100644 --- a/command/login.go +++ b/command/login.go @@ -215,6 +215,12 @@ func (c *LoginCommand) Run(args []string) int { return 2 } + // Evolving token formats across Vault versions have caused issues during CLI logins. Unless + // token auth is being used, omit any token picked up from TokenHelper. + if authMethod != "token" { + client.SetToken("") + } + // Authenticate delegation to the auth handler secret, err := authHandler.Auth(client, config) if err != nil { diff --git a/command/login_test.go b/command/login_test.go index c87add5655f1..cf8325dca912 100644 --- a/command/login_test.go +++ b/command/login_test.go @@ -58,6 +58,9 @@ func TestLoginCommand_Run(t *testing.T) { t.Fatal(err) } + // Emulate an unknown token format present in ~/.vault-token, for example + client.SetToken("a.a") + code := cmd.Run([]string{ "-method", "userpass", "-path", "my-auth", From 5a3aa0ebcd40d76aeb1b809c29d2ebdd77fd23cc Mon Sep 17 00:00:00 2001 From: Jim Kalafut Date: Wed, 25 Sep 2019 11:10:37 -0700 Subject: [PATCH 19/46] changelog++ --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25fb04149fa1..3a2582e813d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ IMPROVEMENTS: * secrets/aws: The root config can now be read [GH-7245] * storage/cassandra: Improve storage efficiency by eliminating unnecessary copies of value data [GH-7199] + +BUG FIXES: + * cli: Fix a bug where a token of an unknown format (e.g. in ~/.vault-token) + could cause confusing error messages during `vault login` [GH-7508] ## 1.2.3 (September 12, 2019) From 8c2a123d183b3d09b931430e67946ee8e5e859c5 Mon Sep 17 00:00:00 2001 From: minitux Date: Wed, 25 Sep 2019 22:27:27 +0200 Subject: [PATCH 20/46] Fix api auth approle documentation (#7382) Change policies to token_policies --- website/source/api/auth/approle/index.html.md.erb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/source/api/auth/approle/index.html.md.erb b/website/source/api/auth/approle/index.html.md.erb index efd6440ca95a..c3bbc026b18c 100644 --- a/website/source/api/auth/approle/index.html.md.erb +++ b/website/source/api/auth/approle/index.html.md.erb @@ -91,7 +91,7 @@ enabled while creating or updating a role. { "token_ttl": "10m", "token_max_ttl": "15m", - "policies": [ + "token_policies": [ "default" ], "period": 0, @@ -141,7 +141,7 @@ $ curl \ "token_max_ttl": 1800, "secret_id_ttl": 600, "secret_id_num_uses": 40, - "policies": [ + "token_policies": [ "default" ], "period": 0, @@ -589,7 +589,7 @@ $ curl \ "renewable": true, "lease_duration": 1200, "metadata": null, - "policies": [ + "token_policies": [ "default" ], "accessor": "fd6c9a00-d2dc-3b11-0be5-af7ae0e1d374", From 22e8af868240e027f1642b7b797938f229c2bb61 Mon Sep 17 00:00:00 2001 From: Brian Shumate Date: Wed, 25 Sep 2019 16:32:42 -0400 Subject: [PATCH 21/46] Update sample request (#7431) - Format curl command to be similar to other sample requests - Add single quotes to URL for '?' so that example is functional - Delete trailing space --- website/source/api/system/metrics.html.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/website/source/api/system/metrics.html.md b/website/source/api/system/metrics.html.md index 241aedd6bd6a..9e8bf0ebc523 100644 --- a/website/source/api/system/metrics.html.md +++ b/website/source/api/system/metrics.html.md @@ -26,12 +26,13 @@ model for metrics collection. - `format` `(string: "")` – Specifies the format used for the returned metrics. The default metrics format is JSON. Setting `format` to `prometheus` will return the metrics in [Prometheus format](https://prometheus.io/docs/instrumenting/exposition_formats/#text-based-format). - + ### Sample Request ``` -$ curl -H "X-Vault-Token: f3b09679-3001-009d-2b80-9c306ab81aa6" \ - http://127.0.0.1:8200/v1/sys/metrics?format=prometheus +$ curl \ + --header "X-Vault-Token: ..." \ + 'http://127.0.0.1:8200/v1/sys/metrics?format=prometheus' ``` ### Sample Response @@ -66,4 +67,5 @@ vault_barrier_get{quantile="0.9"} 0.011938000097870827 vault_barrier_get{quantile="0.99"} 0.011938000097870827 vault_barrier_get_sum 0.1814980012131855 vault_barrier_get_count 36 +... ``` From 147605044229c4a8402882624746443d29b3f6fe Mon Sep 17 00:00:00 2001 From: Yoko Date: Wed, 25 Sep 2019 13:34:58 -0700 Subject: [PATCH 22/46] Fixed the hyperlink typo to blog (#7354) --- website/source/docs/vs/chef-puppet-etc.html.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/docs/vs/chef-puppet-etc.html.md b/website/source/docs/vs/chef-puppet-etc.html.md index 28e009f5c86f..a0f6b0611d8c 100644 --- a/website/source/docs/vs/chef-puppet-etc.html.md +++ b/website/source/docs/vs/chef-puppet-etc.html.md @@ -46,6 +46,6 @@ accessed. It is rare to have a single key that can access all secrets. This makes it easier to have fine-grained access for consumers of Vault. For tips on how to integrate Vault using configuration management, please see -[Using HashiCorp's Vault with Chef](https://www.hashicorp.com/blog/using-hashicorp-vault-with-chef.html). +[Using HashiCorp's Vault with Chef](https://www.hashicorp.com/blog/using-hashicorps-vault-with-chef.html). Although this post is about Chef, the principles can be broadly applied to many of the tools listed here. From 6d5fbab775a134189d05123df335b1232672fa47 Mon Sep 17 00:00:00 2001 From: Noel Quiles <3746694+EnMod@users.noreply.github.com> Date: Wed, 25 Sep 2019 16:38:19 -0400 Subject: [PATCH 23/46] Update hashi-docs-sitemap to v0.1.6 (#7413) --- website/assets/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/assets/package.json b/website/assets/package.json index 00355f34322e..b27795e3e426 100644 --- a/website/assets/package.json +++ b/website/assets/package.json @@ -13,7 +13,7 @@ "@hashicorp/hashi-consent-manager": "^1.0.8", "@hashicorp/hashi-content": "^0.1.0", "@hashicorp/hashi-docs-sidenav": "^1.0.2", - "@hashicorp/hashi-docs-sitemap": "^0.1.4", + "@hashicorp/hashi-docs-sitemap": "^0.1.6", "@hashicorp/hashi-footer": "^1.1.3", "@hashicorp/hashi-ga-form-fields": "1.0.2", "@hashicorp/hashi-global-styles": "^1.0.9", From 98fc6a501a29fa008704fc12701ebe995e5848b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Aur=C3=A8le=20Brothier?= Date: Wed, 25 Sep 2019 22:57:57 +0200 Subject: [PATCH 24/46] docs: add -verify documentation on operator rekey command (#7190) --- .../docs/commands/operator/rekey.html.md | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/website/source/docs/commands/operator/rekey.html.md b/website/source/docs/commands/operator/rekey.html.md index 41af3f1e13ff..3e7800f4475a 100644 --- a/website/source/docs/commands/operator/rekey.html.md +++ b/website/source/docs/commands/operator/rekey.html.md @@ -37,6 +37,16 @@ $ vault operator rekey \ -key-threshold=9 ``` +Initialize a rekey and activate the verification process: + +```text +$ vault operator rekey \ + -init \ + -key-shares=15 \ + -key-threshold=9 \ + -verify +``` + Rekey and encrypt the resulting unseal keys with PGP: ```text @@ -79,6 +89,12 @@ Delete backed-up unseal keys: $ vault operator rekey -backup-delete ``` +Perform the verification of the rekey using the verification nonce: + +```text +$ vault operator rekey -verify -nonce="..." +``` + ## Usage The following flags are available in addition to the [standard set of @@ -121,6 +137,10 @@ flags](/docs/commands/index.html) included on all commands. - `-target` `(string: "barrier")` - Target for rekeying. "recovery" only applies when HSM support is enabled. +- `-verify` `(bool: false)` - Indicate during the phase `-init` that the + verification process is activated for the rekey. Along with `-nonce` option + it indicates that the nonce given is for the verification process. + ### Backup Options - `-backup` `(bool: false)` - Store a backup of the current PGP encrypted unseal From 0c86d909b47493bc9d497cc5e0bc89ce01f5f718 Mon Sep 17 00:00:00 2001 From: Ivan Kurnosov Date: Fri, 27 Sep 2019 00:46:41 +1200 Subject: [PATCH 25/46] Fixed github-prod path (#7516) --- command/login.go | 2 +- website/source/docs/commands/login.html.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/command/login.go b/command/login.go index dc9a2daafb9b..1128bf962ad7 100644 --- a/command/login.go +++ b/command/login.go @@ -60,7 +60,7 @@ Usage: vault login [options] [AUTH K=V...] If an auth method is enabled at a non-standard path, the -method flag still refers to the canonical type, but the -path flag refers to the enabled path. - If a github auth method was enabled at "github-ent", authenticate like this: + If a github auth method was enabled at "github-prod", authenticate like this: $ vault login -method=github -path=github-prod diff --git a/website/source/docs/commands/login.html.md b/website/source/docs/commands/login.html.md index 1111b6779d86..e98e31b29b21 100644 --- a/website/source/docs/commands/login.html.md +++ b/website/source/docs/commands/login.html.md @@ -80,7 +80,7 @@ token_meta_username my-username ~> Notice that the command option (`-method=userpass`) precedes the command argument (`username=my-username`). -If a github auth method was enabled at the path "github-ent": +If a github auth method was enabled at the path "github-prod": ```text $ vault login -method=github -path=github-prod From d14fa56c220028ea50a255583d83715743cad40f Mon Sep 17 00:00:00 2001 From: Mike Jarmy Date: Thu, 26 Sep 2019 10:01:45 -0400 Subject: [PATCH 26/46] Add a unit test for plugin initialization (#7158) * stub out backend lazy load test * stub out backend lazy-load test * test startBackend * test lazyLoadBackend * clean up comments in test suite --- builtin/credential/aws/backend_e2e_test.go | 1 - builtin/plugin/backend_lazyLoad_test.go | 185 +++++++++++++++++++++ 2 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 builtin/plugin/backend_lazyLoad_test.go diff --git a/builtin/credential/aws/backend_e2e_test.go b/builtin/credential/aws/backend_e2e_test.go index 41b37022653c..a252d27dc6a1 100644 --- a/builtin/credential/aws/backend_e2e_test.go +++ b/builtin/credential/aws/backend_e2e_test.go @@ -38,7 +38,6 @@ func TestBackend_E2E_Initialize(t *testing.T) { // Make sure that the upgrade happened, by fishing the 'config/version' // entry out of storage. We can't use core.Client.Logical().Read() to do // this, because 'config/version' hasn't been exposed as a path. - // TODO: should we expose 'config/version' as a path? version, err := core.UnderlyingStorage.Get(ctx, awsPath+"config/version") if err != nil { t.Fatal(err) diff --git a/builtin/plugin/backend_lazyLoad_test.go b/builtin/plugin/backend_lazyLoad_test.go new file mode 100644 index 000000000000..b9a97dd994be --- /dev/null +++ b/builtin/plugin/backend_lazyLoad_test.go @@ -0,0 +1,185 @@ +package plugin + +import ( + "context" + "testing" + + "github.com/hashicorp/vault/sdk/helper/logging" + + "github.com/hashicorp/vault/sdk/helper/pluginutil" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/vault/sdk/helper/consts" + "github.com/hashicorp/vault/sdk/logical" + "github.com/hashicorp/vault/sdk/plugin" +) + +func TestBackend_lazyLoad(t *testing.T) { + + // normal load + var invocations int + b := testLazyLoad(t, func() error { + invocations++ + return nil + }) + if invocations != 1 { + t.Fatalf("expected 1 invocation") + } + if b.canary != "" { + t.Fatalf("expected empty canary") + } + + // load with plugin shutdown + invocations = 0 + b = testLazyLoad(t, func() error { + invocations++ + if invocations == 1 { + return plugin.ErrPluginShutdown + } + return nil + }) + if invocations != 2 { + t.Fatalf("expected 2 invocations") + } + if b.canary == "" { + t.Fatalf("expected canary") + } +} + +func testLazyLoad(t *testing.T, methodWrapper func() error) *PluginBackend { + + sysView := newTestSystemView() + + ctx := context.Background() + config := &logical.BackendConfig{ + Logger: logging.NewVaultLogger(hclog.Trace), + System: sysView, + Config: map[string]string{ + "plugin_name": "test-plugin", + "plugin_type": "secret", + }, + } + + // this is a dummy plugin that hasn't really been loaded yet + orig, err := plugin.NewBackend(ctx, "test-plugin", consts.PluginTypeSecrets, sysView, config, true) + if err != nil { + t.Fatal(err) + } + + b := &PluginBackend{ + Backend: orig, + config: config, + } + + // lazy load + err = b.lazyLoadBackend(ctx, &logical.InmemStorage{}, methodWrapper) + if err != nil { + t.Fatal(err) + } + if !b.loaded { + t.Fatalf("not loaded") + } + + // make sure dummy plugin was handled properly + ob := orig.(*testBackend) + if !ob.cleaned { + t.Fatalf("not cleaned") + } + if ob.setup { + t.Fatalf("setup") + } + if ob.initialized { + t.Fatalf("initialized") + } + + // make sure our newly initialized plugin was handled properly + nb := b.Backend.(*testBackend) + if nb.cleaned { + t.Fatalf("cleaned") + } + if !nb.setup { + t.Fatalf("not setup") + } + if !nb.initialized { + t.Fatalf("not initialized") + } + + return b +} + +//------------------------------------------------------------------ + +type testBackend struct { + cleaned bool + setup bool + initialized bool +} + +var _ logical.Backend = (*testBackend)(nil) + +func (b *testBackend) Cleanup(context.Context) { + b.cleaned = true +} + +func (b *testBackend) Setup(context.Context, *logical.BackendConfig) error { + b.setup = true + return nil +} + +func (b *testBackend) Initialize(context.Context, *logical.InitializationRequest) error { + b.initialized = true + return nil +} + +func (b *testBackend) Type() logical.BackendType { + return logical.TypeLogical +} + +func (b *testBackend) SpecialPaths() *logical.Paths { + return &logical.Paths{ + Root: []string{"test-root"}, + } +} + +func (b *testBackend) Logger() hclog.Logger { + return logging.NewVaultLogger(hclog.Trace) +} + +func (b *testBackend) HandleRequest(context.Context, *logical.Request) (*logical.Response, error) { + panic("not needed") +} +func (b *testBackend) System() logical.SystemView { + panic("not needed") +} +func (b *testBackend) HandleExistenceCheck(context.Context, *logical.Request) (bool, bool, error) { + panic("not needed") +} +func (b *testBackend) InvalidateKey(context.Context, string) { + panic("not needed") +} + +//------------------------------------------------------------------ + +type testSystemView struct { + logical.StaticSystemView + factory logical.Factory +} + +func newTestSystemView() testSystemView { + return testSystemView{ + factory: func(_ context.Context, _ *logical.BackendConfig) (logical.Backend, error) { + return &testBackend{}, nil + }, + } +} + +func (v testSystemView) LookupPlugin(context.Context, string, consts.PluginType) (*pluginutil.PluginRunner, error) { + + return &pluginutil.PluginRunner{ + Name: "test-plugin-runner", + Builtin: true, + BuiltinFactory: func() (interface{}, error) { + return v.factory, nil + }, + }, nil +} From 44e20fe0ab8b8be61e33aa58c6ef1d73050c9d15 Mon Sep 17 00:00:00 2001 From: Jim Kalafut Date: Thu, 26 Sep 2019 08:24:10 -0700 Subject: [PATCH 27/46] changelog++ --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a2582e813d8..5018f3d77458 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ IMPROVEMENTS: copies of value data [GH-7199] BUG FIXES: + * agent: Fix handling of gzipped responses [GH-7470] * cli: Fix a bug where a token of an unknown format (e.g. in ~/.vault-token) could cause confusing error messages during `vault login` [GH-7508] From 9d41daf4d9127e93688edb75bbc756cefbc27afb Mon Sep 17 00:00:00 2001 From: Andy Manoske Date: Thu, 26 Sep 2019 09:53:07 -0700 Subject: [PATCH 28/46] Update index.html.md (#7506) Feedback from customers re: audit information to explicitly expose where credential password creation takes place in the source code. --- website/source/docs/secrets/databases/index.html.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/website/source/docs/secrets/databases/index.html.md b/website/source/docs/secrets/databases/index.html.md index 8691f2f7cc8b..6e2bf21eb874 100644 --- a/website/source/docs/secrets/databases/index.html.md +++ b/website/source/docs/secrets/databases/index.html.md @@ -107,13 +107,18 @@ of the role: password 8cab931c-d62e-a73d-60d3-5ee85139cd66 username v-root-e2978cd0- ``` - ## Custom Plugins This secrets engine allows custom database types to be run through the exposed plugin interface. Please see the [custom database plugin](/docs/secrets/databases/custom.html) for more information. +## Password Generation + +Password generation for both static and dynamic database credentials occurs via the ''GetPassword()'' function. For static credentials, the ''SetCredentials()''function is also used on the output of ''GetPassword().'' + +Please see the [DB plugin credentials source code](https://github.com/hashicorp/vault/blob/master/sdk/database/dbplugin/database.pb.go) for more information. + ## API The database secrets engine has a full HTTP API. Please see the [Database secret From 4c20292fd4ea3feab37d7a4310062bd8dbdd799a Mon Sep 17 00:00:00 2001 From: Connor Zapfel Date: Thu, 26 Sep 2019 16:16:21 -0400 Subject: [PATCH 29/46] Added sys/health path-help content (#7360) --- vault/logical_system.go | 9 +++++++++ vault/logical_system_paths.go | 38 ++++++++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/vault/logical_system.go b/vault/logical_system.go index 77a47fbced4e..d92a96009f2e 100644 --- a/vault/logical_system.go +++ b/vault/logical_system.go @@ -3428,6 +3428,15 @@ This path responds to the following HTTP methods. Initializes a new vault. `, }, + "health": { + "Checks the health status of the Vault.", + ` +This path responds to the following HTTP methods. + + GET / + Returns health information about the Vault. + `, + }, "generate-root": { "Reads, generates, or deletes a root token regeneration process.", ` diff --git a/vault/logical_system_paths.go b/vault/logical_system_paths.go index c52e3a3bf87d..4a7e56f4c171 100644 --- a/vault/logical_system_paths.go +++ b/vault/logical_system_paths.go @@ -143,7 +143,40 @@ func (b *SystemBackend) configPaths() []*framework.Path { }, { Pattern: "health$", - + Fields: map[string]*framework.FieldSchema{ + "standbyok": &framework.FieldSchema{ + Type: framework.TypeBool, + Description: "Specifies if being a standby should still return the active status code.", + }, + "perfstandbyok": &framework.FieldSchema{ + Type: framework.TypeBool, + Description: "Specifies if being a performance standby should still return the active status code.", + }, + "activecode": &framework.FieldSchema{ + Type: framework.TypeInt, + Description: "Specifies the status code for an active node.", + }, + "standbycode": &framework.FieldSchema{ + Type: framework.TypeInt, + Description: "Specifies the status code for a standby node.", + }, + "drsecondarycode": &framework.FieldSchema{ + Type: framework.TypeInt, + Description: "Specifies the status code for a DR secondary node.", + }, + "performancestandbycode": &framework.FieldSchema{ + Type: framework.TypeInt, + Description: "Specifies the status code for a performance standby node.", + }, + "sealedcode": &framework.FieldSchema{ + Type: framework.TypeInt, + Description: "Specifies the status code for a sealed node.", + }, + "uninitcode": &framework.FieldSchema{ + Type: framework.TypeInt, + Description: "Specifies the status code for an uninitialized node.", + }, + }, Operations: map[logical.Operation]framework.OperationHandler{ logical.ReadOperation: &framework.PathOperation{ Summary: "Returns the health status of Vault.", @@ -156,6 +189,9 @@ func (b *SystemBackend) configPaths() []*framework.Path { }, }, }, + + HelpSynopsis: strings.TrimSpace(sysHelp["health"][0]), + HelpDescription: strings.TrimSpace(sysHelp["health"][1]), }, { From 9b05bfd882134f16890f55ac2f50c0c03e679c58 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Thu, 26 Sep 2019 17:18:03 -0400 Subject: [PATCH 30/46] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5018f3d77458..a6a64864f86e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ FEATURES: - * **Stackdriver Metrics Sync**: Vault can now send metrics to + * **Stackdriver Metrics Sink**: Vault can now send metrics to [Stackdriver](https://cloud.google.com/stackdriver/). See the [configuration documentation](https://www.vaultproject.io/docs/config/index.html) for details. [GH-6957] From 2b8aeb10cc859b5369781b7ad4a45fc774817a09 Mon Sep 17 00:00:00 2001 From: Jim Kalafut Date: Fri, 27 Sep 2019 08:30:02 -0700 Subject: [PATCH 31/46] Update Go version in readme Fixes #7525 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7169d05e12fa..bfce4ee17fce 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,8 @@ Developing Vault -------------------- If you wish to work on Vault itself or any of its built-in systems, you'll -first need [Go](https://www.golang.org) installed on your machine (version -1.12.1+ is *required*). +first need [Go](https://www.golang.org) installed on your machine. Go version +1.12.7+ is *required*. Note: version 1.13.x is not yet supported. For local dev first make sure Go is properly installed, including setting up a [GOPATH](https://golang.org/doc/code.html#GOPATH). Ensure that `$GOPATH/bin` is in From 58b1b237fd328749e014c6fdfc9a6b43ef396f85 Mon Sep 17 00:00:00 2001 From: Vishal Nayak Date: Mon, 30 Sep 2019 10:27:25 -0400 Subject: [PATCH 32/46] Fix identity case sensitivity loading in secondary cluster (#7327) * Fix identity case sensitivity loading in secondary cluster * Add nil check --- vault/identity_store.go | 54 +++++++++++++++++++++++++++++++-- vault/identity_store_structs.go | 4 +++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/vault/identity_store.go b/vault/identity_store.go index 34e4fcc54012..d04e9f187a11 100644 --- a/vault/identity_store.go +++ b/vault/identity_store.go @@ -23,6 +23,7 @@ const ( ) var ( + caseSensitivityKey = "casesensitivity" sendGroupUpgrade = func(*IdentityStore, *identity.Group) (bool, error) { return false, nil } parseExtraEntityFromBucket = func(context.Context, *IdentityStore, *identity.Entity) (bool, error) { return false, nil } addExtraEntityDataToResponse = func(*identity.Entity, map[string]interface{}) {} @@ -72,9 +73,10 @@ func NewIdentityStore(ctx context.Context, core *Core, config *logical.BackendCo } iStore.Backend = &framework.Backend{ - BackendType: logical.TypeLogical, - Paths: iStore.paths(), - Invalidate: iStore.Invalidate, + BackendType: logical.TypeLogical, + Paths: iStore.paths(), + Invalidate: iStore.Invalidate, + InitializeFunc: iStore.initialize, PathsSpecial: &logical.Paths{ Unauthenticated: []string{ "oidc/.well-known/*", @@ -109,6 +111,17 @@ func (i *IdentityStore) paths() []*framework.Path { ) } +func (i *IdentityStore) initialize(ctx context.Context, req *logical.InitializationRequest) error { + entry, err := logical.StorageEntryJSON(caseSensitivityKey, &casesensitivity{ + DisableLowerCasedNames: i.disableLowerCasedNames, + }) + if err != nil { + return err + } + + return i.view.Put(ctx, entry) +} + // Invalidate is a callback wherein the backend is informed that the value at // the given key is updated. In identity store's case, it would be the entity // storage entries that get updated. The value needs to be read and MemDB needs @@ -120,6 +133,41 @@ func (i *IdentityStore) Invalidate(ctx context.Context, key string) { defer i.lock.Unlock() switch { + case key == caseSensitivityKey: + entry, err := i.view.Get(ctx, caseSensitivityKey) + if err != nil { + i.logger.Error("failed to read case sensitivity setting during invalidation", "error", err) + return + } + if entry == nil { + return + } + + var setting casesensitivity + if err := entry.DecodeJSON(&setting); err != nil { + i.logger.Error("failed to decode case sensitivity setting during invalidation", "error", err) + return + } + + // Fast return if the setting is the same + if i.disableLowerCasedNames == setting.DisableLowerCasedNames { + return + } + + // If the setting is different, reset memdb and reload all the artifacts + i.disableLowerCasedNames = setting.DisableLowerCasedNames + if err := i.resetDB(ctx); err != nil { + i.logger.Error("failed to reset memdb during invalidation", "error", err) + return + } + if err := i.loadEntities(ctx); err != nil { + i.logger.Error("failed to load entities during invalidation", "error", err) + return + } + if err := i.loadGroups(ctx); err != nil { + i.logger.Error("failed to load groups during invalidation", "error", err) + return + } // Check if the key is a storage entry key for an entity bucket case strings.HasPrefix(key, storagepacker.StoragePackerBucketsPrefix): // Create a MemDB transaction diff --git a/vault/identity_store_structs.go b/vault/identity_store_structs.go index f6a32b51788c..c0f1f97abaf9 100644 --- a/vault/identity_store_structs.go +++ b/vault/identity_store_structs.go @@ -87,3 +87,7 @@ type groupDiff struct { Deleted []*identity.Group Unmodified []*identity.Group } + +type casesensitivity struct { + DisableLowerCasedNames bool `json:"disable_lower_cased_names"` +} From cdeb3f0eb9055c7bfdf4c7396430b6f42c0f6ee2 Mon Sep 17 00:00:00 2001 From: Vishal Nayak Date: Mon, 30 Sep 2019 10:51:07 -0400 Subject: [PATCH 33/46] changelog++ --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6a64864f86e..0dbe32429c71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ BUG FIXES: * agent: Fix handling of gzipped responses [GH-7470] * cli: Fix a bug where a token of an unknown format (e.g. in ~/.vault-token) could cause confusing error messages during `vault login` [GH-7508] + * identity (enterprise): Fixed identity case sensitive loading in secondary + cluster [GH-7327] ## 1.2.3 (September 12, 2019) From b68573fa746f43ca7e7592a8049973d002d8571d Mon Sep 17 00:00:00 2001 From: Jim Kalafut Date: Mon, 30 Sep 2019 08:46:42 -0700 Subject: [PATCH 34/46] Log proxy settings from environment on startup (#7528) --- command/server.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/command/server.go b/command/server.go index ad0a40e5a8d5..279a643ce62b 100644 --- a/command/server.go +++ b/command/server.go @@ -20,6 +20,8 @@ import ( "sync" "time" + "golang.org/x/net/http/httpproxy" + "github.com/hashicorp/vault/helper/metricsutil" monitoring "cloud.google.com/go/monitoring/apiv3" @@ -549,6 +551,11 @@ func (c *ServerCommand) Run(args []string) int { vault.DefaultMaxRequestDuration = config.DefaultMaxRequestDuration } + // log proxy settings + proxyCfg := httpproxy.FromEnvironment() + c.logger.Info("proxy environment", "http_proxy", proxyCfg.HTTPProxy, + "https_proxy", proxyCfg.HTTPSProxy, "no_proxy", proxyCfg.NoProxy) + // If mlockall(2) isn't supported, show a warning. We disable this in dev // because it is quite scary to see when first using Vault. We also disable // this if the user has explicitly disabled mlock in configuration. From 6975e5d888f318a90b42a7e6099a5c0eba0701c8 Mon Sep 17 00:00:00 2001 From: Jim Kalafut Date: Mon, 30 Sep 2019 13:55:05 -0700 Subject: [PATCH 35/46] changelog++ --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dbe32429c71..3146686c57bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ IMPROVEMENTS: BUG FIXES: * agent: Fix handling of gzipped responses [GH-7470] + * auth/gcp: Fix a bug where region information in instance groups names could + cause an authorization attempt to fail [GCP-74] * cli: Fix a bug where a token of an unknown format (e.g. in ~/.vault-token) could cause confusing error messages during `vault login` [GH-7508] * identity (enterprise): Fixed identity case sensitive loading in secondary From 36152ec2813d2a0ba500547b6a0950b9c8b155f0 Mon Sep 17 00:00:00 2001 From: Vishal Nayak Date: Mon, 30 Sep 2019 17:52:10 -0400 Subject: [PATCH 36/46] changelog++ --- CHANGELOG.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3146686c57bd..8abd935c9638 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ FEATURES: [Stackdriver](https://cloud.google.com/stackdriver/). See the [configuration documentation](https://www.vaultproject.io/docs/config/index.html) for details. [GH-6957] - + CHANGES: * sys/seal-status now has a `storage_type` field denoting what type of storage the cluster is configured to use @@ -19,7 +19,10 @@ IMPROVEMENTS: * secrets/aws: The root config can now be read [GH-7245] * storage/cassandra: Improve storage efficiency by eliminating unnecessary copies of value data [GH-7199] - + * replication (enterprise): Write-Ahead-Log entries will not duplicate the + data belonging to the encompassing physical entries of the transaction, + thereby improving the performance and storage capacity. + BUG FIXES: * agent: Fix handling of gzipped responses [GH-7470] * auth/gcp: Fix a bug where region information in instance groups names could From a1aa5912815344f7e2b191629d0d562436fc5440 Mon Sep 17 00:00:00 2001 From: Jim Kalafut Date: Tue, 1 Oct 2019 08:03:32 -0700 Subject: [PATCH 37/46] Update vendor dir (#7539) --- .../golang.org/x/net/http/httpproxy/proxy.go | 370 ++++++++++++++++++ vendor/modules.txt | 1 + 2 files changed, 371 insertions(+) create mode 100644 vendor/golang.org/x/net/http/httpproxy/proxy.go diff --git a/vendor/golang.org/x/net/http/httpproxy/proxy.go b/vendor/golang.org/x/net/http/httpproxy/proxy.go new file mode 100644 index 000000000000..163645b86f3f --- /dev/null +++ b/vendor/golang.org/x/net/http/httpproxy/proxy.go @@ -0,0 +1,370 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package httpproxy provides support for HTTP proxy determination +// based on environment variables, as provided by net/http's +// ProxyFromEnvironment function. +// +// The API is not subject to the Go 1 compatibility promise and may change at +// any time. +package httpproxy + +import ( + "errors" + "fmt" + "net" + "net/url" + "os" + "strings" + "unicode/utf8" + + "golang.org/x/net/idna" +) + +// Config holds configuration for HTTP proxy settings. See +// FromEnvironment for details. +type Config struct { + // HTTPProxy represents the value of the HTTP_PROXY or + // http_proxy environment variable. It will be used as the proxy + // URL for HTTP requests and HTTPS requests unless overridden by + // HTTPSProxy or NoProxy. + HTTPProxy string + + // HTTPSProxy represents the HTTPS_PROXY or https_proxy + // environment variable. It will be used as the proxy URL for + // HTTPS requests unless overridden by NoProxy. + HTTPSProxy string + + // NoProxy represents the NO_PROXY or no_proxy environment + // variable. It specifies a string that contains comma-separated values + // specifying hosts that should be excluded from proxying. Each value is + // represented by an IP address prefix (1.2.3.4), an IP address prefix in + // CIDR notation (1.2.3.4/8), a domain name, or a special DNS label (*). + // An IP address prefix and domain name can also include a literal port + // number (1.2.3.4:80). + // A domain name matches that name and all subdomains. A domain name with + // a leading "." matches subdomains only. For example "foo.com" matches + // "foo.com" and "bar.foo.com"; ".y.com" matches "x.y.com" but not "y.com". + // A single asterisk (*) indicates that no proxying should be done. + // A best effort is made to parse the string and errors are + // ignored. + NoProxy string + + // CGI holds whether the current process is running + // as a CGI handler (FromEnvironment infers this from the + // presence of a REQUEST_METHOD environment variable). + // When this is set, ProxyForURL will return an error + // when HTTPProxy applies, because a client could be + // setting HTTP_PROXY maliciously. See https://golang.org/s/cgihttpproxy. + CGI bool +} + +// config holds the parsed configuration for HTTP proxy settings. +type config struct { + // Config represents the original configuration as defined above. + Config + + // httpsProxy is the parsed URL of the HTTPSProxy if defined. + httpsProxy *url.URL + + // httpProxy is the parsed URL of the HTTPProxy if defined. + httpProxy *url.URL + + // ipMatchers represent all values in the NoProxy that are IP address + // prefixes or an IP address in CIDR notation. + ipMatchers []matcher + + // domainMatchers represent all values in the NoProxy that are a domain + // name or hostname & domain name + domainMatchers []matcher +} + +// FromEnvironment returns a Config instance populated from the +// environment variables HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the +// lowercase versions thereof). HTTPS_PROXY takes precedence over +// HTTP_PROXY for https requests. +// +// The environment values may be either a complete URL or a +// "host[:port]", in which case the "http" scheme is assumed. An error +// is returned if the value is a different form. +func FromEnvironment() *Config { + return &Config{ + HTTPProxy: getEnvAny("HTTP_PROXY", "http_proxy"), + HTTPSProxy: getEnvAny("HTTPS_PROXY", "https_proxy"), + NoProxy: getEnvAny("NO_PROXY", "no_proxy"), + CGI: os.Getenv("REQUEST_METHOD") != "", + } +} + +func getEnvAny(names ...string) string { + for _, n := range names { + if val := os.Getenv(n); val != "" { + return val + } + } + return "" +} + +// ProxyFunc returns a function that determines the proxy URL to use for +// a given request URL. Changing the contents of cfg will not affect +// proxy functions created earlier. +// +// A nil URL and nil error are returned if no proxy is defined in the +// environment, or a proxy should not be used for the given request, as +// defined by NO_PROXY. +// +// As a special case, if req.URL.Host is "localhost" (with or without a +// port number), then a nil URL and nil error will be returned. +func (cfg *Config) ProxyFunc() func(reqURL *url.URL) (*url.URL, error) { + // Preprocess the Config settings for more efficient evaluation. + cfg1 := &config{ + Config: *cfg, + } + cfg1.init() + return cfg1.proxyForURL +} + +func (cfg *config) proxyForURL(reqURL *url.URL) (*url.URL, error) { + var proxy *url.URL + if reqURL.Scheme == "https" { + proxy = cfg.httpsProxy + } + if proxy == nil { + proxy = cfg.httpProxy + if proxy != nil && cfg.CGI { + return nil, errors.New("refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy") + } + } + if proxy == nil { + return nil, nil + } + if !cfg.useProxy(canonicalAddr(reqURL)) { + return nil, nil + } + + return proxy, nil +} + +func parseProxy(proxy string) (*url.URL, error) { + if proxy == "" { + return nil, nil + } + + proxyURL, err := url.Parse(proxy) + if err != nil || + (proxyURL.Scheme != "http" && + proxyURL.Scheme != "https" && + proxyURL.Scheme != "socks5") { + // proxy was bogus. Try prepending "http://" to it and + // see if that parses correctly. If not, we fall + // through and complain about the original one. + if proxyURL, err := url.Parse("http://" + proxy); err == nil { + return proxyURL, nil + } + } + if err != nil { + return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err) + } + return proxyURL, nil +} + +// useProxy reports whether requests to addr should use a proxy, +// according to the NO_PROXY or no_proxy environment variable. +// addr is always a canonicalAddr with a host and port. +func (cfg *config) useProxy(addr string) bool { + if len(addr) == 0 { + return true + } + host, port, err := net.SplitHostPort(addr) + if err != nil { + return false + } + if host == "localhost" { + return false + } + ip := net.ParseIP(host) + if ip != nil { + if ip.IsLoopback() { + return false + } + } + + addr = strings.ToLower(strings.TrimSpace(host)) + + if ip != nil { + for _, m := range cfg.ipMatchers { + if m.match(addr, port, ip) { + return false + } + } + } + for _, m := range cfg.domainMatchers { + if m.match(addr, port, ip) { + return false + } + } + return true +} + +func (c *config) init() { + if parsed, err := parseProxy(c.HTTPProxy); err == nil { + c.httpProxy = parsed + } + if parsed, err := parseProxy(c.HTTPSProxy); err == nil { + c.httpsProxy = parsed + } + + for _, p := range strings.Split(c.NoProxy, ",") { + p = strings.ToLower(strings.TrimSpace(p)) + if len(p) == 0 { + continue + } + + if p == "*" { + c.ipMatchers = []matcher{allMatch{}} + c.domainMatchers = []matcher{allMatch{}} + return + } + + // IPv4/CIDR, IPv6/CIDR + if _, pnet, err := net.ParseCIDR(p); err == nil { + c.ipMatchers = append(c.ipMatchers, cidrMatch{cidr: pnet}) + continue + } + + // IPv4:port, [IPv6]:port + phost, pport, err := net.SplitHostPort(p) + if err == nil { + if len(phost) == 0 { + // There is no host part, likely the entry is malformed; ignore. + continue + } + if phost[0] == '[' && phost[len(phost)-1] == ']' { + phost = phost[1 : len(phost)-1] + } + } else { + phost = p + } + // IPv4, IPv6 + if pip := net.ParseIP(phost); pip != nil { + c.ipMatchers = append(c.ipMatchers, ipMatch{ip: pip, port: pport}) + continue + } + + if len(phost) == 0 { + // There is no host part, likely the entry is malformed; ignore. + continue + } + + // domain.com or domain.com:80 + // foo.com matches bar.foo.com + // .domain.com or .domain.com:port + // *.domain.com or *.domain.com:port + if strings.HasPrefix(phost, "*.") { + phost = phost[1:] + } + matchHost := false + if phost[0] != '.' { + matchHost = true + phost = "." + phost + } + c.domainMatchers = append(c.domainMatchers, domainMatch{host: phost, port: pport, matchHost: matchHost}) + } +} + +var portMap = map[string]string{ + "http": "80", + "https": "443", + "socks5": "1080", +} + +// canonicalAddr returns url.Host but always with a ":port" suffix +func canonicalAddr(url *url.URL) string { + addr := url.Hostname() + if v, err := idnaASCII(addr); err == nil { + addr = v + } + port := url.Port() + if port == "" { + port = portMap[url.Scheme] + } + return net.JoinHostPort(addr, port) +} + +// Given a string of the form "host", "host:port", or "[ipv6::address]:port", +// return true if the string includes a port. +func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } + +func idnaASCII(v string) (string, error) { + // TODO: Consider removing this check after verifying performance is okay. + // Right now punycode verification, length checks, context checks, and the + // permissible character tests are all omitted. It also prevents the ToASCII + // call from salvaging an invalid IDN, when possible. As a result it may be + // possible to have two IDNs that appear identical to the user where the + // ASCII-only version causes an error downstream whereas the non-ASCII + // version does not. + // Note that for correct ASCII IDNs ToASCII will only do considerably more + // work, but it will not cause an allocation. + if isASCII(v) { + return v, nil + } + return idna.Lookup.ToASCII(v) +} + +func isASCII(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + return false + } + } + return true +} + +// matcher represents the matching rule for a given value in the NO_PROXY list +type matcher interface { + // match returns true if the host and optional port or ip and optional port + // are allowed + match(host, port string, ip net.IP) bool +} + +// allMatch matches on all possible inputs +type allMatch struct{} + +func (a allMatch) match(host, port string, ip net.IP) bool { + return true +} + +type cidrMatch struct { + cidr *net.IPNet +} + +func (m cidrMatch) match(host, port string, ip net.IP) bool { + return m.cidr.Contains(ip) +} + +type ipMatch struct { + ip net.IP + port string +} + +func (m ipMatch) match(host, port string, ip net.IP) bool { + if m.ip.Equal(ip) { + return m.port == "" || m.port == port + } + return false +} + +type domainMatch struct { + host string + port string + + matchHost bool +} + +func (m domainMatch) match(host, port string, ip net.IP) bool { + if strings.HasSuffix(host, m.host) || (m.matchHost && host == m.host[1:]) { + return m.port == "" || m.port == port + } + return false +} diff --git a/vendor/modules.txt b/vendor/modules.txt index e356d41ad552..16d16435d1b4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -667,6 +667,7 @@ golang.org/x/crypto/internal/subtle golang.org/x/crypto/pkcs12/internal/rc2 # golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 golang.org/x/net/idna +golang.org/x/net/http/httpproxy golang.org/x/net/http2 golang.org/x/net/context golang.org/x/net/http/httpguts From 0e54f829ee9194bbb29844ab4867dd7686f35878 Mon Sep 17 00:00:00 2001 From: Matthew Irish Date: Tue, 1 Oct 2019 14:57:37 -0500 Subject: [PATCH 38/46] UI CI exit 1 if there's an error (#7399) * exit 1 if there's an error * fix failing confirm tests * still need to exit the process :joy: --- ui/app/templates/components/identity/popup-alias.hbs | 4 +++- ui/lib/core/addon/templates/components/confirm/message.hbs | 4 +++- ui/scripts/start-vault.js | 3 ++- ui/tests/acceptance/aws-test.js | 2 +- ui/tests/acceptance/ssh-test.js | 2 +- ui/tests/pages/access/identity/aliases/index.js | 3 +-- ui/tests/pages/access/identity/index.js | 3 +-- 7 files changed, 12 insertions(+), 9 deletions(-) diff --git a/ui/app/templates/components/identity/popup-alias.hbs b/ui/app/templates/components/identity/popup-alias.hbs index b3ec7e22ba35..9538265e1798 100644 --- a/ui/app/templates/components/identity/popup-alias.hbs +++ b/ui/app/templates/components/identity/popup-alias.hbs @@ -26,7 +26,9 @@
  • + @onConfirm={{action "performTransaction" item}} + data-test-item-delete + />
    • {{/if}} {{/if}} diff --git a/ui/lib/core/addon/templates/components/confirm/message.hbs b/ui/lib/core/addon/templates/components/confirm/message.hbs index b5d9f5e4235b..2c7067edc467 100644 --- a/ui/lib/core/addon/templates/components/confirm/message.hbs +++ b/ui/lib/core/addon/templates/components/confirm/message.hbs @@ -35,6 +35,8 @@ class="link is-destroy" disabled={{showConfirm}} onclick={{action this.onTrigger id}} - data-test-confirm-action-trigger={{id}}> + data-test-confirm-action-trigger={{id}} + ...attributes + > {{triggerText}} diff --git a/ui/scripts/start-vault.js b/ui/scripts/start-vault.js index 2947e63fca80..f9846a86035e 100755 --- a/ui/scripts/start-vault.js +++ b/ui/scripts/start-vault.js @@ -94,11 +94,12 @@ async function processLines(input, eachLine = () => {}) { } } catch (error) { console.log(error); + process.exit(1); } finally { process.exit(0); } } catch (error) { console.log(error); - process.exit(0); + process.exit(1); } })(); diff --git a/ui/tests/acceptance/aws-test.js b/ui/tests/acceptance/aws-test.js index 1c4a3db21b77..fe9823157564 100644 --- a/ui/tests/acceptance/aws-test.js +++ b/ui/tests/acceptance/aws-test.js @@ -83,7 +83,7 @@ module('Acceptance | aws secret backend', function(hooks) { //and delete await click(`[data-test-secret-link="${roleName}"] [data-test-popup-menu-trigger]`); - await click(`[data-test-aws-role-delete="${roleName}"] button`); + await click(`[data-test-aws-role-delete="${roleName}"]`); await click(`[data-test-confirm-button]`); await settled(); diff --git a/ui/tests/acceptance/ssh-test.js b/ui/tests/acceptance/ssh-test.js index c8f386a67ba1..b50e97b8866c 100644 --- a/ui/tests/acceptance/ssh-test.js +++ b/ui/tests/acceptance/ssh-test.js @@ -124,7 +124,7 @@ module('Acceptance | ssh secret backend', function(hooks) { //and delete await click(`[data-test-secret-link="${role.name}"] [data-test-popup-menu-trigger]`); - await click(`[data-test-ssh-role-delete="${role.name}"] button`); + await click(`[data-test-ssh-role-delete="${role.name}"]`); await click(`[data-test-confirm-button]`); await settled(); diff --git a/ui/tests/pages/access/identity/aliases/index.js b/ui/tests/pages/access/identity/aliases/index.js index 82027783daa6..4cb21b1c804a 100644 --- a/ui/tests/pages/access/identity/aliases/index.js +++ b/ui/tests/pages/access/identity/aliases/index.js @@ -8,8 +8,7 @@ export default create({ menu: clickable('[data-test-popup-menu-trigger]'), name: text('[data-test-identity-link]'), }), - delete: clickable('[data-test-confirm-action-trigger]', { - scope: '[data-test-item-delete]', + delete: clickable('[data-test-item-delete]', { testContainer: '#ember-testing', }), confirmDelete: clickable('[data-test-confirm-button]'), diff --git a/ui/tests/pages/access/identity/index.js b/ui/tests/pages/access/identity/index.js index 8bb6fbdbba5d..8259ed384ec5 100644 --- a/ui/tests/pages/access/identity/index.js +++ b/ui/tests/pages/access/identity/index.js @@ -9,8 +9,7 @@ export default create({ name: text('[data-test-identity-link]'), }), - delete: clickable('[data-test-confirm-action-trigger]', { - scope: '[data-test-item-delete]', + delete: clickable('[data-test-item-delete]', { testContainer: '#ember-testing', }), confirmDelete: clickable('[data-test-confirm-button]'), From e4e69164e44989a1d3412cdc42684aa01ce09dde Mon Sep 17 00:00:00 2001 From: Matthew Irish Date: Tue, 1 Oct 2019 15:30:56 -0500 Subject: [PATCH 39/46] UI wrapped token fix (#7398) * default to token auth method * pass in selectedValue to the AuthForm * adjust when and if tasks are called so there's no race condition with wrapped_token query param * add some tests for wrapped_token * adjust redirect_to behavior so that it also works with the logout route and the wrapped_token query param * fix linting --- ui/app/components/auth-form.js | 94 +++++++++++++---------- ui/app/controllers/vault/cluster/auth.js | 2 +- ui/app/mixins/cluster-route.js | 7 +- ui/app/routes/vault/cluster/auth.js | 3 +- ui/app/routes/vault/cluster/logout.js | 2 +- ui/app/templates/components/auth-form.hbs | 3 +- ui/tests/acceptance/auth-test.js | 18 +++-- ui/tests/acceptance/redirect-to-test.js | 61 +++++++++++++-- ui/tests/acceptance/wrapped-token-test.js | 37 +++++++++ 9 files changed, 167 insertions(+), 60 deletions(-) create mode 100644 ui/tests/acceptance/wrapped-token-test.js diff --git a/ui/app/components/auth-form.js b/ui/app/components/auth-form.js index 65edd9b12757..42baadfff468 100644 --- a/ui/app/components/auth-form.js +++ b/ui/app/components/auth-form.js @@ -47,43 +47,54 @@ export default Component.extend(DEFAULTS, { wrappedToken: null, // internal oldNamespace: null, + didReceiveAttrs() { this._super(...arguments); - let token = this.get('wrappedToken'); - let newMethod = this.get('selectedAuth'); - let oldMethod = this.get('oldSelectedAuth'); + let { + wrappedToken: token, + oldWrappedToken: oldToken, + oldNamespace: oldNS, + namespace: ns, + selectedAuth: newMethod, + oldSelectedAuth: oldMethod, + } = this; - let ns = this.get('namespace'); - let oldNS = this.get('oldNamespace'); - if (oldNS === null || oldNS !== ns) { - this.get('fetchMethods').perform(); - } - this.set('oldNamespace', ns); - if (oldMethod && oldMethod !== newMethod) { - this.resetDefaults(); - } - this.set('oldSelectedAuth', newMethod); - - if (token) { - this.get('unwrapToken').perform(token); - } + next(() => { + if (!token && (oldNS === null || oldNS !== ns)) { + this.fetchMethods.perform(); + } + this.set('oldNamespace', ns); + // we only want to trigger this once + if (token && !oldToken) { + this.unwrapToken.perform(token); + this.set('oldWrappedToken', token); + } + if (oldMethod && oldMethod !== newMethod) { + this.resetDefaults(); + } + this.set('oldSelectedAuth', newMethod); + }); }, didRender() { this._super(...arguments); - let firstMethod = this.firstMethod(); // on very narrow viewports the active tab may be overflowed, so we scroll it into view here let activeEle = this.element.querySelector('li.is-active'); if (activeEle) { activeEle.scrollIntoView(); } - // set `with` to the first method - if ( - (this.get('fetchMethods.isIdle') && firstMethod && !this.get('selectedAuth')) || - (this.get('selectedAuth') && !this.get('selectedAuthBackend')) - ) { - this.set('selectedAuth', firstMethod); - } + + next(() => { + let firstMethod = this.firstMethod(); + // set `with` to the first method + if ( + !this.wrappedToken && + ((this.get('fetchMethods.isIdle') && firstMethod && !this.get('selectedAuth')) || + (this.get('selectedAuth') && !this.get('selectedAuthBackend'))) + ) { + this.set('selectedAuth', firstMethod); + } + }); }, firstMethod() { @@ -98,18 +109,23 @@ export default Component.extend(DEFAULTS, { }, selectedAuthIsPath: match('selectedAuth', /\/$/), - selectedAuthBackend: computed('methods', 'methods.[]', 'selectedAuth', 'selectedAuthIsPath', function() { - let methods = this.get('methods'); - let selectedAuth = this.get('selectedAuth'); - let keyIsPath = this.get('selectedAuthIsPath'); - if (!methods) { - return {}; - } - if (keyIsPath) { - return methods.findBy('path', selectedAuth); + selectedAuthBackend: computed( + 'wrappedToken', + 'methods', + 'methods.[]', + 'selectedAuth', + 'selectedAuthIsPath', + function() { + let { wrappedToken, methods, selectedAuth, selectedAuthIsPath: keyIsPath } = this; + if (!methods && !wrappedToken) { + return {}; + } + if (keyIsPath) { + return methods.findBy('path', selectedAuth); + } + return BACKENDS.findBy('type', selectedAuth); } - return BACKENDS.findBy('type', selectedAuth); - }), + ), providerPartialName: computed('selectedAuthBackend', function() { let type = this.get('selectedAuthBackend.type') || 'token'; @@ -146,9 +162,7 @@ export default Component.extend(DEFAULTS, { try { let response = yield adapter.toolAction('unwrap', null, { clientToken: token }); this.set('token', response.auth.client_token); - next(() => { - this.send('doSubmit'); - }); + this.send('doSubmit'); } catch (e) { this.set('error', `Token unwrap failed: ${e.errors[0]}`); } @@ -239,7 +253,7 @@ export default Component.extend(DEFAULTS, { let backendMeta = BACKENDS.find( b => (get(b, 'type') || '').toLowerCase() === (get(backend, 'type') || '').toLowerCase() ); - let attributes = get(backendMeta || {}, 'formAttributes') || {}; + let attributes = get(backendMeta || {}, 'formAttributes') || []; data = assign(data, this.getProperties(...attributes)); if (passedData) { diff --git a/ui/app/controllers/vault/cluster/auth.js b/ui/app/controllers/vault/cluster/auth.js index 8733761393e5..e4f1f7b115e4 100644 --- a/ui/app/controllers/vault/cluster/auth.js +++ b/ui/app/controllers/vault/cluster/auth.js @@ -10,7 +10,7 @@ export default Controller.extend({ namespaceQueryParam: alias('clusterController.namespaceQueryParam'), queryParams: [{ authMethod: 'with' }], wrappedToken: alias('vaultController.wrappedToken'), - authMethod: '', + authMethod: 'token', redirectTo: alias('vaultController.redirectTo'), updateNamespace: task(function*(value) { diff --git a/ui/app/mixins/cluster-route.js b/ui/app/mixins/cluster-route.js index 7ed1860728fc..7b33bc0295cf 100644 --- a/ui/app/mixins/cluster-route.js +++ b/ui/app/mixins/cluster-route.js @@ -9,6 +9,7 @@ const CLUSTER = 'vault.cluster'; const CLUSTER_INDEX = 'vault.cluster.index'; const OIDC_CALLBACK = 'vault.cluster.oidc-callback'; const DR_REPLICATION_SECONDARY = 'vault.cluster.replication-dr-promote'; +const EXCLUDED_REDIRECT_URLS = ['/vault/logout']; export { INIT, UNSEAL, AUTH, CLUSTER, CLUSTER_INDEX, DR_REPLICATION_SECONDARY }; @@ -29,13 +30,17 @@ export default Mixin.create({ if ( // only want to redirect if we're going to authenticate targetRoute === AUTH && - transition.targetName !== CLUSTER_INDEX + transition.targetName !== CLUSTER_INDEX && + !EXCLUDED_REDIRECT_URLS.includes(this.router.currentURL) ) { return this.transitionTo(targetRoute, { queryParams: { redirect_to: this.router.currentURL } }); } return this.transitionTo(targetRoute); } + if (transition.abort && targetRoute === this.router.currentRouteName) { + transition.abort(); + } return RSVP.resolve(); }, diff --git a/ui/app/routes/vault/cluster/auth.js b/ui/app/routes/vault/cluster/auth.js index 2ac997372d9a..dcb83b78dbe6 100644 --- a/ui/app/routes/vault/cluster/auth.js +++ b/ui/app/routes/vault/cluster/auth.js @@ -19,9 +19,10 @@ export default ClusterRouteBase.extend({ model() { return this._super(...arguments); }, + resetController(controller) { controller.set('wrappedToken', ''); - controller.set('authMethod', ''); + controller.set('authMethod', 'token'); }, afterModel() { diff --git a/ui/app/routes/vault/cluster/logout.js b/ui/app/routes/vault/cluster/logout.js index c2232706359b..2dd69193871c 100644 --- a/ui/app/routes/vault/cluster/logout.js +++ b/ui/app/routes/vault/cluster/logout.js @@ -22,8 +22,8 @@ export default Route.extend(ModelBoundaryRoute, { this.console.set('isOpen', false); this.console.clearLog(true); this.clearModelCache(); - this.replaceWith('vault.cluster.auth', { queryParams: { redirect_to: '' } }); this.flashMessages.clearMessages(); this.permissions.reset(); + this.replaceWith('vault.cluster.auth'); }, }); diff --git a/ui/app/templates/components/auth-form.hbs b/ui/app/templates/components/auth-form.hbs index 848b6eb5a53e..8fcf03b58eba 100644 --- a/ui/app/templates/components/auth-form.hbs +++ b/ui/app/templates/components/auth-form.hbs @@ -39,7 +39,8 @@ @valueAttribute={{'type'}} @labelAttribute={{'typeDisplay'}} @isFullwidth={{true}} - @onChange={{action (mut selectedAuth)}} + @selectedValue={{this.selectedAuth}} + @onChange={{action (mut this.selectedAuth)}} /> {{/if}} {{#if (or (eq this.selectedAuthBackend.type "jwt") (eq this.selectedAuthBackend.type "oidc"))}} diff --git a/ui/tests/acceptance/auth-test.js b/ui/tests/acceptance/auth-test.js index e6d0a0b88e4b..81ef6a171d02 100644 --- a/ui/tests/acceptance/auth-test.js +++ b/ui/tests/acceptance/auth-test.js @@ -37,20 +37,24 @@ module('Acceptance | auth', function(hooks) { let backends = supportedAuthBackends(); assert.expect(backends.length + 1); await visit('/vault/auth'); - assert.equal(currentURL(), '/vault/auth?with=token'); + assert.equal(currentURL(), '/vault/auth'); for (let backend of backends.reverse()) { await component.selectMethod(backend.type); - assert.equal( - currentURL(), - `/vault/auth?with=${backend.type}`, - `has the correct URL for ${backend.type}` - ); + if (backend.type === 'token') { + assert.equal(currentURL(), `/vault/auth`, `has the correct URL for ${backend.type}`); + } else { + assert.equal( + currentURL(), + `/vault/auth?with=${backend.type}`, + `has the correct URL for ${backend.type}` + ); + } } }); test('it clears token when changing selected auth method', async function(assert) { await visit('/vault/auth'); - assert.equal(currentURL(), '/vault/auth?with=token'); + assert.equal(currentURL(), '/vault/auth'); await component.token('token').selectMethod('github'); await component.selectMethod('token'); assert.equal(component.tokenValue, '', 'it clears the token value when toggling methods'); diff --git a/ui/tests/acceptance/redirect-to-test.js b/ui/tests/acceptance/redirect-to-test.js index 4cdec62df659..2d89732875d3 100644 --- a/ui/tests/acceptance/redirect-to-test.js +++ b/ui/tests/acceptance/redirect-to-test.js @@ -1,28 +1,62 @@ -import { currentURL, visit } from '@ember/test-helpers'; +import { currentURL, visit as _visit, settled } from '@ember/test-helpers'; import { module, test } from 'qunit'; import { setupApplicationTest } from 'ember-qunit'; -import authPage from 'vault/tests/pages/auth'; +import { create } from 'ember-cli-page-object'; +import auth from 'vault/tests/pages/auth'; +import consoleClass from 'vault/tests/pages/components/console/ui-panel'; -module('Acceptance | redirect_to functionality', function(hooks) { +const visit = async url => { + try { + await _visit(url); + } catch (e) { + if (e.message !== 'TransitionAborted') { + throw e; + } + } + + await settled(); +}; + +const consoleComponent = create(consoleClass); + +const wrappedAuth = async () => { + await consoleComponent.runCommands(`write -field=token auth/token/create policies=default -wrap-ttl=3m`); + return consoleComponent.lastLogOutput; +}; + +const setupWrapping = async () => { + await auth.logout(); + await auth.visit(); + await auth.tokenInput('root').submit(); + let wrappedToken = await wrappedAuth(); + return wrappedToken; +}; +module('Acceptance | redirect_to query param functionality', function(hooks) { setupApplicationTest(hooks); + hooks.beforeEach(function() { + // normally we'd use the auth.logout helper to visit the route and reset the app, but in this case that + // also routes us to the auth page, and then all of the transitions from the auth page get redirected back + // to the auth page resulting in no redirect_to query param being set + localStorage.clear(); + }); test('redirect to a route after authentication', async function(assert) { let url = '/vault/secrets/secret/create'; await visit(url); assert.equal( currentURL(), - `/vault/auth?redirect_to=${encodeURIComponent(url)}&with=token`, + `/vault/auth?redirect_to=${encodeURIComponent(url)}`, 'encodes url for the query param' ); // the login method on this page does another visit call that we don't want here - await authPage.tokenInput('root').submit(); + await auth.tokenInput('root').submit(); assert.equal(currentURL(), url, 'navigates to the redirect_to url after auth'); }); test('redirect from root does not include redirect_to', async function(assert) { let url = '/'; await visit(url); - assert.equal(currentURL(), `/vault/auth?with=token`, 'there is no redirect_to query param'); + assert.equal(currentURL(), `/vault/auth`, 'there is no redirect_to query param'); }); test('redirect to a route after authentication with a query param', async function(assert) { @@ -30,10 +64,21 @@ module('Acceptance | redirect_to functionality', function(hooks) { await visit(url); assert.equal( currentURL(), - `/vault/auth?redirect_to=${encodeURIComponent(url)}&with=token`, + `/vault/auth?redirect_to=${encodeURIComponent(url)}`, 'encodes url for the query param' ); - await authPage.tokenInput('root').submit(); + await auth.tokenInput('root').submit(); assert.equal(currentURL(), url, 'navigates to the redirect_to with the query param after auth'); }); + + test('redirect to logout with wrapped token authenticates you', async function(assert) { + let wrappedToken = await setupWrapping(); + let url = '/vault/secrets/cubbyhole/create'; + + await auth.logout({ + redirect_to: url, + wrapped_token: wrappedToken, + }); + assert.equal(currentURL(), url, 'authenticates then navigates to the redirect_to url after auth'); + }); }); diff --git a/ui/tests/acceptance/wrapped-token-test.js b/ui/tests/acceptance/wrapped-token-test.js new file mode 100644 index 000000000000..60db5171a29e --- /dev/null +++ b/ui/tests/acceptance/wrapped-token-test.js @@ -0,0 +1,37 @@ +import { module, test } from 'qunit'; +import { setupApplicationTest } from 'ember-qunit'; +import { currentURL } from '@ember/test-helpers'; +import { create } from 'ember-cli-page-object'; +import auth from 'vault/tests/pages/auth'; +import consoleClass from 'vault/tests/pages/components/console/ui-panel'; + +const consoleComponent = create(consoleClass); + +const wrappedAuth = async () => { + await consoleComponent.runCommands(`write -field=token auth/token/create policies=default -wrap-ttl=3m`); + return consoleComponent.lastLogOutput; +}; + +const setupWrapping = async () => { + await auth.logout(); + await auth.visit(); + await auth.tokenInput('root').submit(); + let token = await wrappedAuth(); + await auth.logout(); + return token; +}; +module('Acceptance | wrapped_token query param functionality', function(hooks) { + setupApplicationTest(hooks); + + test('it authenticates you if the query param is present', async function(assert) { + let token = await setupWrapping(); + await auth.visit({ wrapped_token: token }); + assert.equal(currentURL(), '/vault/secrets', 'authenticates and redirects to home'); + }); + + test('it authenticates when used with the with=token query param', async function(assert) { + let token = await setupWrapping(); + await auth.visit({ wrapped_token: token, with: 'token' }); + assert.equal(currentURL(), '/vault/secrets', 'authenticates and redirects to home'); + }); +}); From fe44ee073bc7bd7fa567c641acefdcd8b2ca1f4d Mon Sep 17 00:00:00 2001 From: Jim Kalafut Date: Tue, 1 Oct 2019 16:07:52 -0700 Subject: [PATCH 40/46] Add 1.2+ role parameters back to JWT API docs (#7544) This reverts 24c2f8c2ad76, which pulled the parameters while there were outstanding bugs when using them with JWT auth. --- website/source/api/auth/jwt/index.html.md.erb | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/website/source/api/auth/jwt/index.html.md.erb b/website/source/api/auth/jwt/index.html.md.erb index 1169711b5955..f4f27f021eae 100644 --- a/website/source/api/auth/jwt/index.html.md.erb +++ b/website/source/api/auth/jwt/index.html.md.erb @@ -135,18 +135,8 @@ entities attempting to login. At least one of the bound values must be set. - `verbose_oidc_logging` `(bool: false)` - Log received OIDC tokens and claims when debug-level logging is active. Not recommended in production since sensitive information may be present in OIDC responses. -- `policies` `(array: [])` - Policies to be set on tokens issued using this - role. -- `ttl` `(string: "")` - The TTL period of tokens issued using this role, - provided as "1h", where hour is the largest suffix. -- `max_ttl` `(string: "")` - The maximum allowed lifetime of tokens issued using - this role. -- `period` `(string: "")` - If set, indicates that the token generated using - this role should never expire. The token should be renewed within the duration - specified by this value. At each renewal, the token's TTL will be set to the - value of this parameter. -- `bound_cidrs` `(string: "", or list: [])` – If set, restricts usage of the - roles to client IPs falling within the range of the specified CIDR(s). + +<%= partial "partials/tokenfields" %> ### Sample Payload From d144ae8776ed9312c71b8637a114aa8bc2393b0f Mon Sep 17 00:00:00 2001 From: Vu Pham <51802661+vuhphamw@users.noreply.github.com> Date: Tue, 1 Oct 2019 16:08:34 -0700 Subject: [PATCH 41/46] Update oci-object-storage.html.md (#7543) --- .../storage/oci-object-storage.html.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/website/source/docs/configuration/storage/oci-object-storage.html.md b/website/source/docs/configuration/storage/oci-object-storage.html.md index 9f73a3f5e0c4..67900ffb0308 100644 --- a/website/source/docs/configuration/storage/oci-object-storage.html.md +++ b/website/source/docs/configuration/storage/oci-object-storage.html.md @@ -16,7 +16,7 @@ The OCI Object Storage backend is used to persist Vault's data in OCI Object Sto - **Community Supported** – the OCI Object Storage backend is supported by the community. While it has undergone review by HashiCorp employees, they may not be as knowledgeable about the technology. If you encounter problems with them, you may be referred to the original author. ```hcl -storage "oci_objectstorage" { +storage "oci" { namespace_name = "" bucket_name = "" ha_enabled = "" @@ -28,20 +28,20 @@ storage "oci_objectstorage" { For more information on OCI Object Storage, please see the Oracle's [OCI Object Storage documentation][ocios-docs]. -## `oci_objectstorage` Setup +## `oci` Setup To use the OCI Object Storage Vault storage backend, you must have a OCI account. Either using the API or web interface, create the data bucket and lock bucket if enabling high availability. The OCI Object Storage backend does not support creating the buckets automatically at this time. -## `oci_objectstorage` Authentication +## `oci` Authentication The OCI Object Storage Vault storage backend uses the official OCI Golang SDK. This means it supports the common ways of providing credentials to OCI. For more information on service accounts, please see the [OCI Identity documentation] [oci-identity]. -## `oci_objectstorage` Parameters +## `oci` Parameters - `namespace_name` `(string: )` – Specifies the name of the OCI Object Storage namespaces containing the data bucket and the lock bucket. @@ -56,15 +56,15 @@ For more information on service accounts, please see the [OCI Identity documenta - `lock_bucket_name` `(string: "")` - Specifies the name of the bucket that will be used to store the node lease data. -## `oci_objectstorage` Examples +## `oci` Examples ### Standalone Vault instance This example shows configuring OCI Object Storage as a standalone instance. ```hcl -storage "oci_objectstorage" { - namespace_name = "MyNamespace +storage "oci" { + namespace_name = "MyNamespace" bucket_name = "DataBucket" } ``` @@ -74,8 +74,8 @@ storage "oci_objectstorage" { This example shows configuring OCI Object Storage with high availability enabled. ```hcl -storage "oci_objectstorage" { - namespace_name = "MyNamespace +storage "oci" { + namespace_name = "MyNamespace" bucket_name = "DataBucket" ha_enabled = "true" lock_bucket_name = "LockBucket" From ec921940f9483502ccafbad8b30d5f7270e9498b Mon Sep 17 00:00:00 2001 From: Jim Kalafut Date: Tue, 1 Oct 2019 16:13:21 -0700 Subject: [PATCH 42/46] Fix identity token API docs (#7545) --- website/source/api/secret/identity/tokens.html.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/website/source/api/secret/identity/tokens.html.md b/website/source/api/secret/identity/tokens.html.md index d01635e0e808..6c372425da6b 100644 --- a/website/source/api/secret/identity/tokens.html.md +++ b/website/source/api/secret/identity/tokens.html.md @@ -441,7 +441,6 @@ Query this path to retrieve a set of claims about the identity tokens' configura ``` $ curl \ - --header "X-Vault-Token: ..." \ --request GET \ http://127.0.0.1:8200/v1/identity/oidc/.well-known/openid-configuration ``` @@ -474,7 +473,6 @@ Query this path to retrieve the public portion of named keys. Clients can use th ``` $ curl \ - --header "X-Vault-Token: ..." \ --request GET \ http://127.0.0.1:8200/v1/identity/oidc/.well-known/keys ``` From 6310462550443225ee511919dfa76d1f2e80672d Mon Sep 17 00:00:00 2001 From: ncabatoff Date: Wed, 2 Oct 2019 10:55:20 -0400 Subject: [PATCH 43/46] Return a useful error on attempts to renew a token via sys/leases/renew (#7298) --- vault/expiration.go | 2 +- website/source/api/system/leases.html.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/vault/expiration.go b/vault/expiration.go index a94537c71aaf..69d1098046f6 100644 --- a/vault/expiration.go +++ b/vault/expiration.go @@ -822,7 +822,7 @@ func (m *ExpirationManager) Renew(ctx context.Context, leaseID string, increment if le.Secret == nil { if le.Auth != nil { - return logical.ErrorResponse("tokens cannot be renewed through this endpoint"), logical.ErrPermissionDenied + return logical.ErrorResponse("tokens cannot be renewed through this endpoint"), nil } return logical.ErrorResponse("lease does not correspond to a secret"), nil } diff --git a/website/source/api/system/leases.html.md b/website/source/api/system/leases.html.md index 4c95ef67f35f..4e81d17e7269 100644 --- a/website/source/api/system/leases.html.md +++ b/website/source/api/system/leases.html.md @@ -90,7 +90,8 @@ $ curl \ ## Renew Lease -This endpoint renews a lease, requesting to extend the lease. +This endpoint renews a lease, requesting to extend the lease. Token leases +cannot be renewed using this endpoint, use instead the auth/token/renew endpoint. | Method | Path | | :---------------------------- | :--------------------- | From b601dfca02b41f9cf21f3ac02d480a6ef6442be2 Mon Sep 17 00:00:00 2001 From: David Adams Date: Wed, 2 Oct 2019 10:59:57 -0500 Subject: [PATCH 44/46] Add response_types_supported to OIDC configuration (#7533) The OIDC Discovery standard requires the response_types_supported field to be returned in the .well-known/openid-configuration response. Also, the AWS IAM OIDC consumer won't accept Vault as an identity provider without this field. Based on examples in the OIDC Core documentation, it appears Vault supports only the `id_token` flow, and thus that is the only value that makes sense to be set in this field. See: https://openid.net/specs/openid-connect-core-1_0.html#AuthorizationExamples --- vault/identity_store_oidc.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/vault/identity_store_oidc.go b/vault/identity_store_oidc.go index 3fd0cc9c3f98..b88d01ce583f 100644 --- a/vault/identity_store_oidc.go +++ b/vault/identity_store_oidc.go @@ -78,10 +78,11 @@ type idToken struct { // // https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderMetadata type discovery struct { - Issuer string `json:"issuer"` - Keys string `json:"jwks_uri"` - Subjects []string `json:"subject_types_supported"` - IDTokenAlgs []string `json:"id_token_signing_alg_values_supported"` + Issuer string `json:"issuer"` + Keys string `json:"jwks_uri"` + ResponseTypes []string `json:"response_types_supported"` + Subjects []string `json:"subject_types_supported"` + IDTokenAlgs []string `json:"id_token_signing_alg_values_supported"` } // oidcCache is a thin wrapper around go-cache to partition by namespace @@ -202,7 +203,7 @@ func oidcPaths(i *IdentityStore) []*framework.Path { logical.ReadOperation: i.pathOIDCDiscovery, }, HelpSynopsis: "Query OIDC configurations", - HelpDescription: "Query this path to retrieve the configured OIDC Issuer and Keys endpoints, Subjects, and signing algorithms used by the OIDC backend.", + HelpDescription: "Query this path to retrieve the configured OIDC Issuer and Keys endpoints, response types, subject types, and signing algorithms used by the OIDC backend.", }, { Pattern: "oidc/.well-known/keys/?$", @@ -1002,10 +1003,11 @@ func (i *IdentityStore) pathOIDCDiscovery(ctx context.Context, req *logical.Requ } disc := discovery{ - Issuer: c.effectiveIssuer, - Keys: c.effectiveIssuer + "/.well-known/keys", - Subjects: []string{"public"}, - IDTokenAlgs: supportedAlgs, + Issuer: c.effectiveIssuer, + Keys: c.effectiveIssuer + "/.well-known/keys", + ResponseTypes: []string{"id_token"}, + Subjects: []string{"public"}, + IDTokenAlgs: supportedAlgs, } data, err = json.Marshal(disc) From 84f808512ad3bbd26ee54827e95d5fa82e8c8d80 Mon Sep 17 00:00:00 2001 From: Jim Kalafut Date: Wed, 2 Oct 2019 09:31:07 -0700 Subject: [PATCH 45/46] changelog++ --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8abd935c9638..4bb9e62e935a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,8 @@ BUG FIXES: cause an authorization attempt to fail [GCP-74] * cli: Fix a bug where a token of an unknown format (e.g. in ~/.vault-token) could cause confusing error messages during `vault login` [GH-7508] + * identity: Add required field `response_types_supported` to identity token + `.well-known/openid-configuration` response [GH-7533] * identity (enterprise): Fixed identity case sensitive loading in secondary cluster [GH-7327] From e54c2e930d7f656209f0e572300b4a1f4957c538 Mon Sep 17 00:00:00 2001 From: Matthew Irish Date: Wed, 2 Oct 2019 11:59:47 -0500 Subject: [PATCH 46/46] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bb9e62e935a..19484112c184 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,8 @@ BUG FIXES: `.well-known/openid-configuration` response [GH-7533] * identity (enterprise): Fixed identity case sensitive loading in secondary cluster [GH-7327] + * ui: using the `wrapped_token` query param will work with `redirect_to` and + will automatically log in as intended [GH-7398] ## 1.2.3 (September 12, 2019)