Skip to content

Commit

Permalink
identity: Add random seed to tests
Browse files Browse the repository at this point in the history 
This commit introduces a random seed variable to be used with UUID
generation in Identity Store determinism tests. The seed is
automatically generated or passed during Environment Variable and
displayed during test failure. This allows for easy reproduction
of any test failures for later debugging.
  • Loading branch information
@mpalmi
mpalmi committed Feb 4, 2025
1 parent 152d486 commit 95b0db7
Show file tree
Hide file tree
Showing 3 changed files with 127 additions and 55 deletions.
139 changes: 104 additions & 35 deletions vault/identity_store_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"context"
"fmt"
"math/rand"
"os"
"regexp"
"slices"
"strconv"
Expand All @@ -18,8 +19,12 @@ import (
"github.com/go-test/deep"
"github.com/hashicorp/go-hclog"
uuid "github.com/hashicorp/go-uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"

credGithub "github.com/hashicorp/vault/builtin/credential/github"
"github.com/hashicorp/vault/builtin/credential/userpass"
credUserpass "github.com/hashicorp/vault/builtin/credential/userpass"
"github.com/hashicorp/vault/helper/activationflags"
"github.com/hashicorp/vault/helper/identity"
Expand All @@ -29,10 +34,6 @@ import (
"github.com/hashicorp/vault/sdk/logical"
"github.com/hashicorp/vault/sdk/physical"
"github.com/hashicorp/vault/sdk/physical/inmem"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
)

func TestIdentityStore_DeleteEntityAlias(t *testing.T) {
Expand Down Expand Up @@ -1416,12 +1417,65 @@ func TestIdentityStoreInvalidate_TemporaryEntity(t *testing.T) {
// the identity cleanup rename resolver to ensure that loading is deterministic
// for both.
func TestIdentityStoreLoadingIsDeterministic(t *testing.T) {
t.Run(t.Name()+"-error-resolver", func(t *testing.T) {
identityStoreLoadingIsDeterministic(t, false)
})
t.Run(t.Name()+"-identity-cleanup", func(t *testing.T) {
identityStoreLoadingIsDeterministic(t, true)
})
seedval := rand.Int63()
if os.Getenv("VAULT_TEST_IDENTITY_STORE_SEED") != "" {
var err error
seedval, err = strconv.ParseInt(os.Getenv("VAULT_TEST_IDENTITY_STORE_SEED"), 10, 64)
require.NoError(t, err)
}
seed := rand.New(rand.NewSource(seedval)) // Seed for deterministic test
defer t.Logf("Test generated with seed: %d", seedval)

tests := []struct {
name string
flags *determinismTestFlags
}{
{
name: "error-resolver-primary",
flags: &determinismTestFlags{
identityDeduplication: false,
secondary: false,
seed: seed,
},
},
{
name: "identity-cleanup-primary",
flags: &determinismTestFlags{
identityDeduplication: true,
secondary: false,
seed: seed,
},
},

{
name: "error-resolver-secondary",
flags: &determinismTestFlags{
identityDeduplication: false,
secondary: true,
seed: seed,
},
},
{
name: "identity-cleanup-secondary",
flags: &determinismTestFlags{
identityDeduplication: true,
secondary: true,
seed: seed,
},
},
}

for _, test := range tests {
t.Run(t.Name()+"-"+test.name, func(t *testing.T) {
identityStoreLoadingIsDeterministic(t, test.flags)
})
}
}

type determinismTestFlags struct {
identityDeduplication bool
secondary bool
seed *rand.Rand
}

// identityStoreLoadingIsDeterministic is a property-based test helper that
Expand All @@ -1432,7 +1486,7 @@ func TestIdentityStoreLoadingIsDeterministic(t *testing.T) {
// deterministic anyway if all data in storage was correct see comments inline
// for examples of ways storage can be corrupt with respect to the expected
// schema invariants.
func identityStoreLoadingIsDeterministic(t *testing.T, identityDeduplication bool) {
func identityStoreLoadingIsDeterministic(t *testing.T, flags *determinismTestFlags) {
// Create some state in store that could trigger non-deterministic behavior.
// The nature of the identity store schema is such that the order of loading
// entities etc shouldn't matter even if it was non-deterministic, however due
Expand All @@ -1456,7 +1510,7 @@ func identityStoreLoadingIsDeterministic(t *testing.T, identityDeduplication boo
Logger: logger,
BuiltinRegistry: corehelpers.NewMockBuiltinRegistry(),
CredentialBackends: map[string]logical.Factory{
"userpass": userpass.Factory,
"userpass": credUserpass.Factory,
},
}

Expand All @@ -1470,6 +1524,10 @@ func identityStoreLoadingIsDeterministic(t *testing.T, identityDeduplication boo

ctx := context.Background()

seed := flags.seed
identityDeduplication := flags.identityDeduplication
secondary := flags.secondary

// We create 100 entities each with 1 non-local alias and 1 local alias. We
// then randomly create duplicate alias or local alias entries with a
// probability that is unrealistic but ensures we have duplicates on every
Expand All @@ -1478,9 +1536,9 @@ func identityStoreLoadingIsDeterministic(t *testing.T, identityDeduplication boo
name := fmt.Sprintf("entity-%d", i)
alias := fmt.Sprintf("alias-%d", i)
localAlias := fmt.Sprintf("localalias-%d", i)
e := makeEntityForPacker(t, name, c.identityStore.entityPacker)
attachAlias(t, e, alias, upme)
attachAlias(t, e, localAlias, localMe)
e := makeEntityForPacker(t, name, c.identityStore.entityPacker, seed)
attachAlias(t, e, alias, upme, seed)
attachAlias(t, e, localAlias, localMe, seed)
err = TestHelperWriteToStoragePacker(ctx, c.identityStore.entityPacker, e.ID, e)
require.NoError(t, err)

Expand All @@ -1489,35 +1547,35 @@ func identityStoreLoadingIsDeterministic(t *testing.T, identityDeduplication boo
// few double and maybe triple duplicates of each type every few test runs
// and may have duplicates of both types or neither etc.
pDup := 0.3
rnd := rand.Float64()
rnd := seed.Float64()
dupeNum := 1
for rnd < pDup && dupeNum < 10 {
e := makeEntityForPacker(t, fmt.Sprintf("entity-%d-dup-%d", i, dupeNum), c.identityStore.entityPacker)
attachAlias(t, e, alias, upme)
e := makeEntityForPacker(t, fmt.Sprintf("entity-%d-dup-%d", i, dupeNum), c.identityStore.entityPacker, seed)
attachAlias(t, e, alias, upme, seed)
err = TestHelperWriteToStoragePacker(ctx, c.identityStore.entityPacker, e.ID, e)
require.NoError(t, err)
// Toss again to see if we continue
rnd = rand.Float64()
rnd = seed.Float64()
dupeNum++
}
// Toss the coin again to see if there are any local dupes
dupeNum = 1
rnd = rand.Float64()
rnd = seed.Float64()
for rnd < pDup && dupeNum < 10 {
e := makeEntityForPacker(t, fmt.Sprintf("entity-%d-localdup-%d", i, dupeNum), c.identityStore.entityPacker)
attachAlias(t, e, localAlias, localMe)
e := makeEntityForPacker(t, fmt.Sprintf("entity-%d-localdup-%d", i, dupeNum), c.identityStore.entityPacker, seed)
attachAlias(t, e, localAlias, localMe, seed)
err = TestHelperWriteToStoragePacker(ctx, c.identityStore.entityPacker, e.ID, e)
require.NoError(t, err)
rnd = rand.Float64()
rnd = seed.Float64()
dupeNum++
}
// See if we should add entity _name_ duplicates too (with no aliases)
rnd = rand.Float64()
rnd = seed.Float64()
for rnd < pDup {
e := makeEntityForPacker(t, name, c.identityStore.entityPacker)
e := makeEntityForPacker(t, name, c.identityStore.entityPacker, seed)
err = TestHelperWriteToStoragePacker(ctx, c.identityStore.entityPacker, e.ID, e)
require.NoError(t, err)
rnd = rand.Float64()
rnd = seed.Float64()
}
// One more edge case is that it's currently possible as of the time of
// writing for a failure during entity invalidation to result in a permanent
Expand All @@ -1543,26 +1601,28 @@ func identityStoreLoadingIsDeterministic(t *testing.T, identityDeduplication boo
if i%2 == 0 {
alias = fmt.Sprintf("groupalias-%d", i)
}
e := makeGroupWithNameAndAlias(t, name, alias, c.identityStore.groupPacker, upme)
e := makeGroupWithNameAndAlias(t, name, alias, c.identityStore.groupPacker, upme, seed)
err = TestHelperWriteToStoragePacker(ctx, c.identityStore.groupPacker, e.ID, e)
require.NoError(t, err)
}
// Now add 10 groups with the same alias to ensure duplicates don't cause
// non-deterministic behavior.
for i := 0; i <= 10; i++ {
name := fmt.Sprintf("group-dup-%d", i)
e := makeGroupWithNameAndAlias(t, name, "groupalias-dup", c.identityStore.groupPacker, upme)
e := makeGroupWithNameAndAlias(t, name, "groupalias-dup", c.identityStore.groupPacker, upme, seed)
err = TestHelperWriteToStoragePacker(ctx, c.identityStore.groupPacker, e.ID, e)
require.NoError(t, err)
}
// Add a second and third groups with duplicate names too.
for _, name := range []string{"group-0", "group-1", "group-1"} {
e := makeGroupWithNameAndAlias(t, name, "", c.identityStore.groupPacker, upme)
e := makeGroupWithNameAndAlias(t, name, "", c.identityStore.groupPacker, upme, seed)
err = TestHelperWriteToStoragePacker(ctx, c.identityStore.groupPacker, e.ID, e)
require.NoError(t, err)
}

entIdentityStoreDeterminismTestSetup(t, ctx, c, upme, localMe)
if secondary {
entIdentityStoreDeterminismSecondaryTestSetup(t, ctx, c, upme, localMe, seed)
}

// Storage is now primed for the test.

Expand Down Expand Up @@ -1652,7 +1712,9 @@ func identityStoreLoadingIsDeterministic(t *testing.T, identityDeduplication boo
// note `lastIDs` argument is not needed anymore but we can't change the
// signature without breaking enterprise. It's simpler to keep it unused
// for now until both parts of this merge.
entIdentityStoreDeterminismAssert(t, i, loadedNames, nil)
if secondary {
entIdentityStoreDeterminismSecondaryAssert(t, i, loadedNames, nil)
}

if i > 0 {
// Should be in the same order if we are deterministic since MemDB has strong ordering.
Expand All @@ -1676,7 +1738,7 @@ func TestIdentityStoreLoadingDuplicateReporting(t *testing.T) {
Logger: logger,
BuiltinRegistry: corehelpers.NewMockBuiltinRegistry(),
CredentialBackends: map[string]logical.Factory{
"userpass": userpass.Factory,
"userpass": credUserpass.Factory,
},
}

Expand All @@ -1690,9 +1752,16 @@ func TestIdentityStoreLoadingDuplicateReporting(t *testing.T) {

ctx := namespace.RootContext(nil)

identityCreateCaseDuplicates(t, ctx, c, upme, localMe)
seedval := rand.Int63()
if os.Getenv("VAULT_TEST_IDENTITY_STORE_DETERMINISTIC_SEED") != "" {
seedval, err = strconv.ParseInt(os.Getenv("VAULT_TEST_IDENTITY_STORE_DETERMINISTIC_SEED"), 10, 64)
require.NoError(t, err)
}
seed := rand.New(rand.NewSource(seedval)) // Seed for deterministic test
defer t.Logf("Test generated with seed %d", seedval)
identityCreateCaseDuplicates(t, ctx, c, upme, localMe, seed)

entIdentityStoreDuplicateReportTestSetup(t, ctx, c, rootToken)
entIdentityStoreDuplicateReportTestSetup(t, ctx, c, rootToken, seed)

// Storage is now primed for the test.

Expand Down
7 changes: 4 additions & 3 deletions vault/identity_store_test_stubs_oss.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,20 +7,21 @@ package vault

import (
"context"
"math/rand"
"testing"
)

//go:generate go run github.com/hashicorp/vault/tools/stubmaker

func entIdentityStoreDeterminismTestSetup(t *testing.T, ctx context.Context, c *Core, me, localme *MountEntry) {
func entIdentityStoreDeterminismSecondaryTestSetup(t *testing.T, ctx context.Context, c *Core, me, localme *MountEntry, seed *rand.Rand) {
// no op
}

func entIdentityStoreDeterminismAssert(t *testing.T, i int, loadedIDs, lastIDs []string) {
func entIdentityStoreDeterminismSecondaryAssert(t *testing.T, i int, loadedIDs, lastIDs []string) {
// no op
}

func entIdentityStoreDuplicateReportTestSetup(t *testing.T, ctx context.Context, c *Core, rootToken string) {
func entIdentityStoreDuplicateReportTestSetup(t *testing.T, ctx context.Context, c *Core, rootToken string, seed *rand.Rand) {
// no op
}

Expand Down
36 changes: 19 additions & 17 deletions vault/identity_store_util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"context"
"errors"
"fmt"
"math/rand"
"strings"
"sync"
"testing"
Expand Down Expand Up @@ -2828,14 +2829,14 @@ func (i *IdentityStore) countEntitiesByMountAccessor(ctx context.Context) (map[s
return byMountAccessor, nil
}

func makeEntityForPacker(t *testing.T, name string, p *storagepacker.StoragePacker) *identity.Entity {
func makeEntityForPacker(t *testing.T, name string, p *storagepacker.StoragePacker, seed *rand.Rand) *identity.Entity {
t.Helper()
return makeEntityForPackerWithNamespace(t, namespace.RootNamespaceID, name, p)
return makeEntityForPackerWithNamespace(t, namespace.RootNamespaceID, name, p, seed)
}

func makeEntityForPackerWithNamespace(t *testing.T, namespaceID, name string, p *storagepacker.StoragePacker) *identity.Entity {
func makeEntityForPackerWithNamespace(t *testing.T, namespaceID, name string, p *storagepacker.StoragePacker, seed *rand.Rand) *identity.Entity {
t.Helper()
id, err := uuid.GenerateUUID()
id, err := uuid.GenerateUUIDWithReader(seed)
require.NoError(t, err)
return &identity.Entity{
ID: id,
Expand All @@ -2845,9 +2846,10 @@ func makeEntityForPackerWithNamespace(t *testing.T, namespaceID, name string, p
}
}

func attachAlias(t *testing.T, e *identity.Entity, name string, me *MountEntry) *identity.Alias {
func attachAlias(t *testing.T, e *identity.Entity, name string, me *MountEntry, seed *rand.Rand) *identity.Alias {
t.Helper()
id, err := uuid.GenerateUUID()

id, err := uuid.GenerateUUIDWithReader(seed)
require.NoError(t, err)
if e.NamespaceID != me.NamespaceID {
panic("mount and entity in different namespaces")
Expand All @@ -2866,7 +2868,7 @@ func attachAlias(t *testing.T, e *identity.Entity, name string, me *MountEntry)
return a
}

func identityCreateCaseDuplicates(t *testing.T, ctx context.Context, c *Core, upme, localme *MountEntry) {
func identityCreateCaseDuplicates(t *testing.T, ctx context.Context, c *Core, upme, localme *MountEntry, seed *rand.Rand) {
t.Helper()

if upme.NamespaceID != localme.NamespaceID {
Expand All @@ -2877,31 +2879,31 @@ func identityCreateCaseDuplicates(t *testing.T, ctx context.Context, c *Core, up
// suffixes.
for i, suffix := range []string{"-case", "-case", "-cAsE"} {
// Entity duplicated by name
e := makeEntityForPackerWithNamespace(t, upme.NamespaceID, "entity"+suffix, c.identityStore.entityPacker)
e := makeEntityForPackerWithNamespace(t, upme.NamespaceID, "entity"+suffix, c.identityStore.entityPacker, seed)
err := TestHelperWriteToStoragePacker(ctx, c.identityStore.entityPacker, e.ID, e)
require.NoError(t, err)

// Entity that isn't a dupe itself but has duplicated aliases
e2 := makeEntityForPackerWithNamespace(t, upme.NamespaceID, fmt.Sprintf("entity-%d", i), c.identityStore.entityPacker)
e2 := makeEntityForPackerWithNamespace(t, upme.NamespaceID, fmt.Sprintf("entity-%d", i), c.identityStore.entityPacker, seed)
// Add local and non-local aliases for this entity (which will also be
// duplicated)
attachAlias(t, e2, "alias"+suffix, upme)
attachAlias(t, e2, "local-alias"+suffix, localme)
attachAlias(t, e2, "alias"+suffix, upme, seed)
attachAlias(t, e2, "local-alias"+suffix, localme, seed)
err = TestHelperWriteToStoragePacker(ctx, c.identityStore.entityPacker, e2.ID, e2)
require.NoError(t, err)

// Group duplicated by name
g := makeGroupWithNameAndAlias(t, "group"+suffix, "", c.identityStore.groupPacker, upme)
g := makeGroupWithNameAndAlias(t, "group"+suffix, "", c.identityStore.groupPacker, upme, seed)
err = TestHelperWriteToStoragePacker(ctx, c.identityStore.groupPacker, g.ID, g)
require.NoError(t, err)
}
}

func makeGroupWithNameAndAlias(t *testing.T, name, alias string, p *storagepacker.StoragePacker, me *MountEntry) *identity.Group {
func makeGroupWithNameAndAlias(t *testing.T, name, alias string, p *storagepacker.StoragePacker, me *MountEntry, seed *rand.Rand) *identity.Group {
t.Helper()
id, err := uuid.GenerateUUID()
id, err := uuid.GenerateUUIDWithReader(seed)
require.NoError(t, err)
id2, err := uuid.GenerateUUID()
id2, err := uuid.GenerateUUIDWithReader(seed)
require.NoError(t, err)
g := &identity.Group{
ID: id,
Expand All @@ -2922,9 +2924,9 @@ func makeGroupWithNameAndAlias(t *testing.T, name, alias string, p *storagepacke
return g
}

func makeLocalAliasWithName(t *testing.T, name, entityID string, bucketKey string, me *MountEntry) *identity.LocalAliases {
func makeLocalAliasWithName(t *testing.T, name, entityID string, bucketKey string, me *MountEntry, seed *rand.Rand) *identity.LocalAliases {
t.Helper()
id, err := uuid.GenerateUUID()
id, err := uuid.GenerateUUIDWithReader(seed)
require.NoError(t, err)
return &identity.LocalAliases{
Aliases: []*identity.Alias{
Expand Down

0 comments on commit 95b0db7

Please sign in to comment.