Reduce how often we hit the database

* Consolidate creating indices into just 3 calls. We can further refine this in getporter#1782, when we make initializing the db a single event. * Do not ping on connect. This does mean that the first query will be slower as it will establish the connection. So let's be aware of that and not assume the first call just has horrible performance. I am still looking into why the first call that establishes the connection has bad performance. Signed-off-by: Carolyn Van Slyck <[email protected]>
carolynvs · Oct 1, 2021 · 7e37220 · 7e37220
1 parent 9b87b62
commit 7e37220
Show file tree

Hide file tree

Showing 9 changed files with 102 additions and 96 deletions.
diff --git a/pkg/claims/claimstore.go b/pkg/claims/claimstore.go
@@ -2,7 +2,6 @@ package claims
 
 import (
 	"get.porter.sh/porter/pkg/storage"
-	"github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
 	"go.mongodb.org/mongo-driver/bson"
 )
@@ -34,67 +33,26 @@ func NewClaimStore(datastore storage.Store) ClaimStore {
 }
 
 func (s ClaimStore) Initialize() error {
-	var bigErr *multierror.Error
-
-	// query installations by a namespace (list) or namespace + name (get)
-	err := s.store.EnsureIndex(CollectionInstallations, storage.EnsureIndexOptions{
-		Keys:   []string{"namespace", "name"},
-		Unique: true,
-	})
-	if err != nil {
-		bigErr = multierror.Append(bigErr, err)
-	}
-
-	// query runs by installation (list)
-	err = s.store.EnsureIndex(CollectionRuns, storage.EnsureIndexOptions{
-		Keys: []string{"namespace", "installation"},
-	})
-	if err != nil {
-		bigErr = multierror.Append(bigErr, err)
-	}
-
-	// query results by installation (delete or batch get)
-	err = s.store.EnsureIndex(CollectionResults, storage.EnsureIndexOptions{
-		Keys: []string{"namespace", "installation"},
-	})
-	if err != nil {
-		bigErr = multierror.Append(bigErr, err)
-	}
-
-	// query results by run (list)
-	err = s.store.EnsureIndex(CollectionResults, storage.EnsureIndexOptions{
-		Keys: []string{"runId"},
-	})
-	if err != nil {
-		bigErr = multierror.Append(bigErr, err)
-	}
-
-	// query most recent outputs by run (porter installation run show, when we list outputs)
-	err = s.store.EnsureIndex(CollectionOutputs, storage.EnsureIndexOptions{
-		Keys: []string{"namespace", "installation", "-resultId"},
-	})
-	if err != nil {
-		bigErr = multierror.Append(bigErr, err)
-	}
-
-	// query outputs by result (list)
-	err = s.store.EnsureIndex(CollectionOutputs, storage.EnsureIndexOptions{
-		Keys:   []string{"resultId", "name"},
-		Unique: true,
-	})
-	if err != nil {
-		bigErr = multierror.Append(bigErr, err)
-	}
-
-	// query most recent outputs by name for an installation
-	err = s.store.EnsureIndex(CollectionOutputs, storage.EnsureIndexOptions{
-		Keys: []string{"namespace", "installation", "name", "-resultId"},
-	})
-	if err != nil {
-		bigErr = multierror.Append(bigErr, err)
+	opts := storage.EnsureIndexOptions{
+		Indices: []storage.Index{
+			// query installations by a namespace (list) or namespace + name (get)
+			{Collection: CollectionInstallations, Keys: []string{"namespace", "name"}, Unique: true},
+			// query runs by installation (list)
+			{Collection: CollectionRuns, Keys: []string{"namespace", "installation"}},
+			// query results by installation (delete or batch get)
+			{Collection: CollectionResults, Keys: []string{"namespace", "installation"}},
+			// query results by run (list)
+			{Collection: CollectionResults, Keys: []string{"runId"}},
+			// query most recent outputs by run (porter installation run show, when we list outputs)
+			{Collection: CollectionOutputs, Keys: []string{"namespace", "installation", "-resultId"}},
+			// query outputs by result (list)
+			{Collection: CollectionOutputs, Keys: []string{"resultId", "name"}, Unique: true},
+			// query most recent outputs by name for an installation
+			{Collection: CollectionOutputs, Keys: []string{"namespace", "installation", "name", "-resultId"}},
+		},
 	}
 
-	return bigErr.ErrorOrNil()
+	return s.store.EnsureIndex(opts)
 }
 
 func (s ClaimStore) ListInstallations(namespace string, name string, labels map[string]string) ([]Installation, error) {

diff --git a/pkg/credentials/credential_store.go b/pkg/credentials/credential_store.go
@@ -33,12 +33,13 @@ func NewCredentialStore(storage storage.Store, secrets secrets.Store) *Credentia
 
 // Initialize the underlying storage with any additional schema changes, such as indexes.
 func (s CredentialStore) Initialize() error {
-	// query credentials by namespace + name
-	err := s.Documents.EnsureIndex(CollectionCredentials, storage.EnsureIndexOptions{
-		Keys:   []string{"namespace", "name"},
-		Unique: true,
-	})
-	return err
+	indices := storage.EnsureIndexOptions{
+		Indices: []storage.Index{
+			// query credentials by namespace + name
+			{Collection: CollectionCredentials, Keys: []string{"namespace", "name"}, Unique: true},
+		},
+	}
+	return s.Documents.EnsureIndex(indices)
 }
 
 func (s CredentialStore) GetDataStore() storage.Store {

diff --git a/pkg/parameters/parameter_store.go b/pkg/parameters/parameter_store.go
@@ -33,12 +33,13 @@ func NewParameterStore(storage storage.Store, secrets secrets.Store) *ParameterS
 
 // Initialize the backend storage with any necessary schema changes, such as indexes.
 func (s ParameterStore) Initialize() error {
-	// query parameters by namespace + name
-	err := s.Documents.EnsureIndex(CollectionParameters, storage.EnsureIndexOptions{
-		Keys:   []string{"namespace", "name"},
-		Unique: true,
-	})
-	return err
+	indices := storage.EnsureIndexOptions{
+		Indices: []storage.Index{
+			// query parameters by namespace + name
+			{Collection: CollectionParameters, Keys: []string{"namespace", "name"}, Unique: true},
+		},
+	}
+	return s.Documents.EnsureIndex(indices)
 }
 
 func (s ParameterStore) GetDataStore() storage.Store {

diff --git a/pkg/storage/migrations/manager.go b/pkg/storage/migrations/manager.go
@@ -125,11 +125,11 @@ func (m *Manager) Count(collection string, opts storage.CountOptions) (int64, er
 	return m.store.Count(collection, opts)
 }
 
-func (m *Manager) EnsureIndex(collection string, opts storage.EnsureIndexOptions) error {
+func (m *Manager) EnsureIndex(opts storage.EnsureIndexOptions) error {
 	if err := m.Connect(); err != nil {
 		return err
 	}
-	return m.store.EnsureIndex(collection, opts)
+	return m.store.EnsureIndex(opts)
 }
 
 func (m *Manager) Find(collection string, opts storage.FindOptions, out interface{}) error {

diff --git a/pkg/storage/plugin_adapter.go b/pkg/storage/plugin_adapter.go
@@ -50,13 +50,13 @@ func (a PluginAdapter) Aggregate(collection string, opts AggregateOptions, out i
 	return a.unmarshalSlice(rawResults, out)
 }
 
-func (a PluginAdapter) EnsureIndex(collection string, opts EnsureIndexOptions) error {
+func (a PluginAdapter) EnsureIndex(opts EnsureIndexOptions) error {
 	err := a.Connect()
 	if err != nil {
 		return err
 	}
 
-	return a.plugin.EnsureIndex(opts.ToPluginOptions(collection))
+	return a.plugin.EnsureIndex(opts.ToPluginOptions())
 }
 
 func (a PluginAdapter) Count(collection string, opts CountOptions) (int64, error) {

diff --git a/pkg/storage/plugins/mongodb/mongodb.go b/pkg/storage/plugins/mongodb/mongodb.go
@@ -73,7 +73,7 @@ func (s *Store) Connect() error {
 	}
 
 	s.client = client
-	return s.Ping()
+	return nil
 }
 
 func (s *Store) Close() error {
@@ -119,16 +119,31 @@ func (s *Store) EnsureIndex(opts plugins.EnsureIndexOptions) error {
 	cxt, cancel := context.WithTimeout(context.Background(), s.timeout)
 	defer cancel()
 
-	c := s.getCollection(opts.Collection)
-	indexOpts := mongo.IndexModel{
-		Keys:    opts.Keys,
-		Options: options.Index(),
+	indices := make(map[string][]mongo.IndexModel, len(opts.Indices))
+	for _, index := range opts.Indices {
+		model := mongo.IndexModel{
+			Keys:    index.Keys,
+			Options: options.Index(),
+		}
+		model.Options.SetUnique(index.Unique)
+		model.Options.SetBackground(true)
+
+		c, ok := indices[index.Collection]
+		if !ok {
+			c = make([]mongo.IndexModel, 0, 1)
+		}
+		c = append(c, model)
+		indices[index.Collection] = c
 	}
-	indexOpts.Options.SetUnique(opts.Unique)
-	indexOpts.Options.SetBackground(true)
 
-	_, err := c.Indexes().CreateOne(cxt, indexOpts)
-	return err
+	for collectionName, models := range indices {
+		c := s.getCollection(collectionName)
+		if _, err := c.Indexes().CreateMany(cxt, models); err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
 func (s *Store) Count(opts plugins.CountOptions) (int64, error) {

diff --git a/pkg/storage/plugins/storage_protocol.go b/pkg/storage/plugins/storage_protocol.go
@@ -38,9 +38,21 @@ type StorageProtocol interface {
 // EnsureIndexOptions is the set of options available to the
 // StorageProtocol.EnsureIndex operation.
 type EnsureIndexOptions struct {
-	Collection string      `bson:"collection"`
-	Keys       interface{} `json:"keys"`
-	Unique     bool        `json:"unique"`
+	// Indices to create if not found.
+	Indices []Index
+}
+
+// Index on a collection.
+type Index struct {
+	// Collection name to which the index applies.
+	Collection string
+
+	// Keys describes the fields and their sort order.
+	// Example: {"namespace": 1, "name": 1}
+	Keys interface{}
+
+	// Unique specifies if the index should enforce that the indexed fields for each document are unique.
+	Unique bool
 }
 
 // AggregateOptions is the set of options available to the

diff --git a/pkg/storage/query.go b/pkg/storage/query.go
@@ -25,8 +25,21 @@ func (o AggregateOptions) ToPluginOptions(collection string) plugins.AggregateOp
 
 // EnsureIndexOptions is the set of options available to the EnsureIndex operation.
 type EnsureIndexOptions struct {
-	Keys   []string `json:"keys"`
-	Unique bool     `json:"unique"`
+	// Indices to create if not found.
+	Indices []Index
+}
+
+// Index on a collection.
+type Index struct {
+	// Collection name to which the index applies.
+	Collection string
+
+	// Keys describes the fields and their sort order.
+	// Example: ["namespace", "name", "-timestamp"]
+	Keys []string
+
+	// Unique specifies if the index should enforce that the indexed fields for each document are unique.
+	Unique bool
 }
 
 // Convert from a simplified sort specifier like []{"-key"}
@@ -52,12 +65,18 @@ func convertSortKeys(values []string) interface{} {
 	return keys
 }
 
-func (o EnsureIndexOptions) ToPluginOptions(collection string) plugins.EnsureIndexOptions {
-	return plugins.EnsureIndexOptions{
-		Collection: collection,
-		Keys:       convertSortKeys(o.Keys),
-		Unique:     o.Unique,
+func (o EnsureIndexOptions) ToPluginOptions() plugins.EnsureIndexOptions {
+	opts := plugins.EnsureIndexOptions{
+		Indices: make([]plugins.Index, len(o.Indices)),
+	}
+	for i, index := range o.Indices {
+		opts.Indices[i] = plugins.Index{
+			Collection: index.Collection,
+			Keys:       convertSortKeys(index.Keys),
+			Unique:     index.Unique,
+		}
 	}
+	return opts
 }
 
 // CountOptions is the set of options available to the Count operation on any

diff --git a/pkg/storage/store.go b/pkg/storage/store.go
@@ -18,7 +18,7 @@ type Store interface {
 
 	// EnsureIndex makes sure that the specified index exists as specified.
 	// If it does exist with a different definition, the index is recreated.
-	EnsureIndex(collection string, opts EnsureIndexOptions) error
+	EnsureIndex(opts EnsureIndexOptions) error
 
 	// Find queries a collection, optionally projecting a subset of fields, into
 	// the specified out value.