Account Service Optimisation (prebid#2440)

account/account.go

@@ -28,7 +28,7 @@ func GetAccount(ctx context.Context, cfg *config.Configuration, fetcher stored_r
 			Message: fmt.Sprintf("Prebid-server has been configured to discard requests without a valid Account ID. Please reach out to the prebid server host."),
 		}}
 	}
-	if accountJSON, accErrs := fetcher.FetchAccount(ctx, accountID); len(accErrs) > 0 || accountJSON == nil {
+	if accountJSON, accErrs := fetcher.FetchAccount(ctx, cfg.AccountDefaultsJSON(), accountID); len(accErrs) > 0 || accountJSON == nil {
 		// accountID does not reference a valid account
 		for _, e := range accErrs {
 			if _, ok := e.(stored_requests.NotFoundError); !ok {
@@ -49,25 +49,22 @@ func GetAccount(ctx context.Context, cfg *config.Configuration, fetcher stored_r
 	} else {
 		// accountID resolved to a valid account, merge with AccountDefaults for a complete config
 		account = &config.Account{}
-		completeJSON, err := jsonpatch.MergePatch(cfg.AccountDefaultsJSON(), accountJSON)
-		if err == nil {
-			err = json.Unmarshal(completeJSON, account)
+		err := json.Unmarshal(accountJSON, account)
+
+		// this logic exists for backwards compatibility. If the initial unmarshal fails above, we attempt to
+		// resolve it by converting the GDPR enforce purpose fields and then attempting an unmarshal again before
+		// declaring a malformed account error.
+		// unmarshal fetched account to determine if it is well-formed
+		if _, ok := err.(*json.UnmarshalTypeError); ok {
+			// attempt to convert deprecated GDPR enforce purpose fields to resolve issue
+			accountJSON, err = ConvertGDPREnforcePurposeFields(accountJSON)
+			// unmarshal again to check if unmarshal error still exists after GDPR field conversion
+			err = json.Unmarshal(accountJSON, account)
 
-			// this logic exists for backwards compatibility. If the initial unmarshal fails above, we attempt to
-			// resolve it by converting the GDPR enforce purpose fields and then attempting an unmarshal again before
-			// declaring a malformed account error.
-			// unmarshal fetched account to determine if it is well-formed
 			if _, ok := err.(*json.UnmarshalTypeError); ok {
-				// attempt to convert deprecated GDPR enforce purpose fields to resolve issue
-				completeJSON, err = ConvertGDPREnforcePurposeFields(completeJSON)
-				// unmarshal again to check if unmarshal error still exists after GDPR field conversion
-				err = json.Unmarshal(completeJSON, account)
-
-				if _, ok := err.(*json.UnmarshalTypeError); ok {
-					return nil, []error{&errortypes.MalformedAcct{
-						Message: fmt.Sprintf("The prebid-server account config for account id \"%s\" is malformed. Please reach out to the prebid server host.", accountID),
-					}}
-				}
+				return nil, []error{&errortypes.MalformedAcct{
+					Message: fmt.Sprintf("The prebid-server account config for account id \"%s\" is malformed. Please reach out to the prebid server host.", accountID),
+				}}
 			}
 		}
 

account/account_test.go

@@ -25,7 +25,7 @@ var mockAccountData = map[string]json.RawMessage{
 type mockAccountFetcher struct {
 }
 
-func (af mockAccountFetcher) FetchAccount(ctx context.Context, accountID string) (json.RawMessage, []error) {
+func (af mockAccountFetcher) FetchAccount(ctx context.Context, accountDefaultsJSON json.RawMessage, accountID string) (json.RawMessage, []error) {
 	if account, ok := mockAccountData[accountID]; ok {
 		return account, nil
 	}

endpoints/cookie_sync_test.go

@@ -1815,8 +1815,8 @@ type FakeAccountsFetcher struct {
 	AccountData map[string]json.RawMessage
 }
 
-func (f FakeAccountsFetcher) FetchAccount(ctx context.Context, accountID string) (json.RawMessage, []error) {
-	defaultAccountJSON := json.RawMessage(`{"disabled":false}`)
+func (f FakeAccountsFetcher) FetchAccount(ctx context.Context, defaultAccountJSON json.RawMessage, accountID string) (json.RawMessage, []error) {
+	defaultAccountJSON = json.RawMessage(`{"disabled":false}`)
 
 	if accountID == metrics.PublisherUnknown {
 		return defaultAccountJSON, nil

endpoints/events/event_test.go

@@ -83,7 +83,7 @@ type mockAccountsFetcher struct {
 	DurationMS int
 }
 
-func (maf mockAccountsFetcher) FetchAccount(ctx context.Context, accountID string) (json.RawMessage, []error) {
+func (maf mockAccountsFetcher) FetchAccount(ctx context.Context, defaultAccountJSON json.RawMessage, accountID string) (json.RawMessage, []error) {
 	if maf.DurationMS > 0 {
 		select {
 		case <-time.After(time.Duration(maf.DurationMS) * time.Millisecond):

endpoints/openrtb2/test_utils.go

@@ -1341,7 +1341,7 @@ type mockAccountFetcher struct {
 	data map[string]json.RawMessage
 }
 
-func (af *mockAccountFetcher) FetchAccount(ctx context.Context, accountID string) (json.RawMessage, []error) {
+func (af *mockAccountFetcher) FetchAccount(ctx context.Context, defaultAccountJSON json.RawMessage, accountID string) (json.RawMessage, []error) {
 	if account, ok := af.data[accountID]; ok {
 		return account, nil
 	}

endpoints/setuid_test.go

@@ -533,7 +533,7 @@ func TestSetUIDEndpointMetrics(t *testing.T) {
 					Status:  400,
 					Bidder:  "pubmatic",
 					UID:     "",
-					Errors:  []error{errors.New("Invalid JSON Patch")},
+					Errors:  []error{errors.New("unexpected end of JSON input")},
 					Success: false,
 				}
 				a.On("LogSetUIDObject", &expected).Once()
@@ -710,6 +710,7 @@ func doRequest(req *http.Request, analytics analytics.PBSAnalyticsModule, metric
 		BlacklistedAcctMap: map[string]bool{
 			"blocked_acct": true,
 		},
+		AccountDefaults: config.Account{},
 	}
 	cfg.MarshalAccountDefaults()
 

stored_requests/backends/db_fetcher/fetcher.go

@@ -151,7 +151,7 @@ func (fetcher *dbFetcher) FetchResponses(ctx context.Context, ids []string) (dat
 
 }
 
-func (fetcher *dbFetcher) FetchAccount(ctx context.Context, accountID string) (json.RawMessage, []error) {
+func (fetcher *dbFetcher) FetchAccount(ctx context.Context, accountDefaultsJSON json.RawMessage, accountID string) (json.RawMessage, []error) {
 	return nil, []error{stored_requests.NotFoundError{accountID, "Account"}}
 }
 

stored_requests/backends/empty_fetcher/fetcher.go

@@ -32,7 +32,7 @@ func (fetcher EmptyFetcher) FetchResponses(ctx context.Context, ids []string) (d
 	return nil, nil
 }
 
-func (fetcher EmptyFetcher) FetchAccount(ctx context.Context, accountID string) (json.RawMessage, []error) {
+func (fetcher EmptyFetcher) FetchAccount(ctx context.Context, accountDefaultJSON json.RawMessage, accountID string) (json.RawMessage, []error) {
 	return nil, []error{stored_requests.NotFoundError{accountID, "Account"}}
 }
 

stored_requests/backends/file_fetcher/fetcher.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/prebid/prebid-server/stored_requests"
+	jsonpatch "gopkg.in/evanphx/json-patch.v4"
 )
 
 // NewFileFetcher _immediately_ loads stored request data from local files.
@@ -38,7 +39,7 @@ func (fetcher *eagerFetcher) FetchResponses(ctx context.Context, ids []string) (
 }
 
 // FetchAccount fetches the host account configuration for a publisher
-func (fetcher *eagerFetcher) FetchAccount(ctx context.Context, accountID string) (json.RawMessage, []error) {
+func (fetcher *eagerFetcher) FetchAccount(ctx context.Context, accountDefaultsJSON json.RawMessage, accountID string) (json.RawMessage, []error) {
 	if len(accountID) == 0 {
 		return nil, []error{fmt.Errorf("Cannot look up an empty accountID")}
 	}
@@ -49,7 +50,12 @@ func (fetcher *eagerFetcher) FetchAccount(ctx context.Context, accountID string)
 			DataType: "Account",
 		}}
 	}
-	return accountJSON, nil
+
+	completeJSON, err := jsonpatch.MergePatch(accountDefaultsJSON, accountJSON)
+	if err != nil {
+		return nil, []error{err}
+	}
+	return completeJSON, nil
 }
 
 func (fetcher *eagerFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) {

stored_requests/backends/file_fetcher/fetcher_test.go

@@ -28,14 +28,20 @@ func TestAccountFetcher(t *testing.T) {
 	fetcher, err := NewFileFetcher("./test")
 	assert.NoError(t, err, "Failed to create test fetcher")
 
-	account, errs := fetcher.FetchAccount(context.Background(), "valid")
+	account, errs := fetcher.FetchAccount(context.Background(), json.RawMessage(`{"events_enabled":true}`), "valid")
 	assertErrorCount(t, 0, errs)
-	assert.JSONEq(t, `{"disabled":false, "id":"valid"}`, string(account))
+	assert.JSONEq(t, `{"disabled":false, "events_enabled":true, "id":"valid" }`, string(account))
 
-	_, errs = fetcher.FetchAccount(context.Background(), "nonexistent")
+	_, errs = fetcher.FetchAccount(context.Background(), json.RawMessage(`{"events_enabled":true}`), "nonexistent")
 	assertErrorCount(t, 1, errs)
 	assert.Error(t, errs[0])
 	assert.Equal(t, stored_requests.NotFoundError{"nonexistent", "Account"}, errs[0])
+
+	_, errs = fetcher.FetchAccount(context.Background(), json.RawMessage(`{"events_enabled"}`), "valid")
+	assertErrorCount(t, 1, errs)
+	assert.Error(t, errs[0])
+	assert.Equal(t, fmt.Errorf("Invalid JSON Document"), errs[0])
+
 }
 
 func TestInvalidDirectory(t *testing.T) {

stored_requests/backends/http_fetcher/fetcher.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/prebid/prebid-server/stored_requests"
+	jsonpatch "gopkg.in/evanphx/json-patch.v4"
 
 	"github.com/golang/glog"
 	"golang.org/x/net/context/ctxhttp"
@@ -151,7 +152,7 @@ func (fetcher *HttpFetcher) FetchAccounts(ctx context.Context, accountIDs []stri
 }
 
 // FetchAccount fetchers a single accountID and returns its corresponding json
-func (fetcher *HttpFetcher) FetchAccount(ctx context.Context, accountID string) (accountJSON json.RawMessage, errs []error) {
+func (fetcher *HttpFetcher) FetchAccount(ctx context.Context, accountDefaultsJSON json.RawMessage, accountID string) (accountJSON json.RawMessage, errs []error) {
 	accountData, errs := fetcher.FetchAccounts(ctx, []string{accountID})
 	if len(errs) > 0 {
 		return nil, errs
@@ -163,7 +164,11 @@ func (fetcher *HttpFetcher) FetchAccount(ctx context.Context, accountID string)
 			DataType: "Account",
 		}}
 	}
-	return accountJSON, nil
+	completeJSON, err := jsonpatch.MergePatch(accountDefaultsJSON, accountJSON)
+	if err != nil {
+		return nil, []error{err}
+	}
+	return completeJSON, nil
 }
 
 func (fetcher *HttpFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) {

stored_requests/backends/http_fetcher/fetcher_test.go

@@ -3,6 +3,7 @@ package http_fetcher
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -134,16 +135,25 @@ func TestFetchAccount(t *testing.T) {
 	fetcher, close := newTestAccountFetcher(t, []string{"acc-1"})
 	defer close()
 
-	account, errs := fetcher.FetchAccount(context.Background(), "acc-1")
+	account, errs := fetcher.FetchAccount(context.Background(), json.RawMessage(`{"disabled":true}`), "acc-1")
 	assert.Empty(t, errs, "Unexpected error fetching existing account")
-	assert.JSONEq(t, `"acc-1"`, string(account), "Unexpected account data fetching existing account")
+	assert.JSONEq(t, `{"disabled": true, "id":"acc-1"}`, string(account), "Unexpected account data fetching existing account")
+}
+
+func TestAccountMergeError(t *testing.T) {
+	fetcher, close := newTestAccountFetcher(t, []string{"acc-1"})
+	defer close()
+
+	_, errs := fetcher.FetchAccount(context.Background(), json.RawMessage(`{"disabled"}`), "acc-1")
+	assert.Error(t, errs[0])
+	assert.Equal(t, fmt.Errorf("Invalid JSON Document"), errs[0])
 }
 
 func TestFetchAccountNoData(t *testing.T) {
 	fetcher, close := newFetcherBrokenBackend()
 	defer close()
 
-	unknownAccount, errs := fetcher.FetchAccount(context.Background(), "unknown-acc")
+	unknownAccount, errs := fetcher.FetchAccount(context.Background(), json.RawMessage(`{disabled":true}`), "unknown-acc")
 	assert.NotEmpty(t, errs, "Retrieving unknown account should return error")
 	assert.Nil(t, unknownAccount, "Retrieving unknown account should return nil json.RawMessage")
 }
@@ -152,7 +162,7 @@ func TestFetchAccountNoIDProvided(t *testing.T) {
 	fetcher, close := newTestAccountFetcher(t, nil)
 	defer close()
 
-	account, errs := fetcher.FetchAccount(context.Background(), "")
+	account, errs := fetcher.FetchAccount(context.Background(), json.RawMessage(`{disabled":true}`), "")
 	assert.Len(t, errs, 1, "Fetching account with empty id should error")
 	assert.Nil(t, account, "Fetching account with empty id should return nil")
 }
@@ -233,12 +243,12 @@ func newHandler(t *testing.T, expectReqIDs []string, expectImpIDs []string, json
 }
 
 func newTestAccountFetcher(t *testing.T, expectAccIDs []string) (fetcher *HttpFetcher, closer func()) {
-	handler := newAccountHandler(t, expectAccIDs, jsonifyID)
+	handler := newAccountHandler(t, expectAccIDs)
 	server := httptest.NewServer(http.HandlerFunc(handler))
 	return NewFetcher(server.Client(), server.URL), server.Close
 }
 
-func newAccountHandler(t *testing.T, expectAccIDs []string, jsonifier func(string) json.RawMessage) func(w http.ResponseWriter, r *http.Request) {
+func newAccountHandler(t *testing.T, expectAccIDs []string) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		query := r.URL.Query()
 		gotAccIDs := richSplit(query.Get("account-ids"))
@@ -249,7 +259,7 @@ func newAccountHandler(t *testing.T, expectAccIDs []string, jsonifier func(strin
 
 		for _, accID := range gotAccIDs {
 			if accID != "" {
-				accIDResponse[accID] = jsonifier(accID)
+				accIDResponse[accID] = json.RawMessage(`{"id":"` + accID + `"}`)
 			}
 		}
 

stored_requests/data/by_id/stored_imps/.gitignore

@@ -1,3 +1,3 @@
 # Ignore everything in this directory, except for this file
 *
-!.gitignore
+!.gitignore

stored_requests/data/by_id/stored_requests/.gitignore

@@ -1,3 +1,3 @@
 # Ignore everything in this directory, except for this file
 *
-!.gitignore
+!.gitignore

stored_requests/fetcher.go

@@ -28,7 +28,7 @@ type Fetcher interface {
 
 type AccountFetcher interface {
 	// FetchAccount fetches the host account configuration for a publisher
-	FetchAccount(ctx context.Context, accountID string) (json.RawMessage, []error)
+	FetchAccount(ctx context.Context, accountDefaultJSON json.RawMessage, accountID string) (json.RawMessage, []error)
 }
 
 type CategoryFetcher interface {
@@ -210,7 +210,7 @@ func (f *fetcherWithCache) FetchResponses(ctx context.Context, ids []string) (da
 	return
 }
 
-func (f *fetcherWithCache) FetchAccount(ctx context.Context, accountID string) (account json.RawMessage, errs []error) {
+func (f *fetcherWithCache) FetchAccount(ctx context.Context, acccountDefaultJSON json.RawMessage, accountID string) (account json.RawMessage, errs []error) {
 	accountData := f.cache.Accounts.Get(ctx, []string{accountID})
 	// TODO: add metrics
 	if account, ok := accountData[accountID]; ok {
@@ -219,7 +219,7 @@ func (f *fetcherWithCache) FetchAccount(ctx context.Context, accountID string) (
 	} else {
 		f.metricsEngine.RecordAccountCacheResult(metrics.CacheMiss, 1)
 	}
-	account, errs = f.fetcher.FetchAccount(ctx, accountID)
+	account, errs = f.fetcher.FetchAccount(ctx, acccountDefaultJSON, accountID)
 	if len(errs) == 0 {
 		f.cache.Accounts.Save(ctx, map[string]json.RawMessage{accountID: account})
 	}

stored_requests/fetcher_test.go

@@ -243,7 +243,7 @@ func TestAccountCacheHit(t *testing.T) {
 		})
 
 	metricsEngine.On("RecordAccountCacheResult", metrics.CacheHit, 1)
-	account, errs := aFetcherWithCache.FetchAccount(ctx, "known")
+	account, errs := aFetcherWithCache.FetchAccount(ctx, json.RawMessage("{}"), "known")
 
 	accCache.AssertExpectations(t)
 	fetcher.AssertExpectations(t)
@@ -263,18 +263,17 @@ func TestAccountCacheMiss(t *testing.T) {
 	// Test read from cache
 	accCache.On("Get", ctx, uncachedAccounts).Return(map[string]json.RawMessage{})
 	accCache.On("Save", ctx, uncachedAccountsData)
-	fetcher.On("FetchAccount", ctx, "uncached").Return(uncachedAccountsData["uncached"], []error{})
+	fetcher.On("FetchAccount", ctx, json.RawMessage("{}"), "uncached").Return(uncachedAccountsData["uncached"], []error{})
 	metricsEngine.On("RecordAccountCacheResult", metrics.CacheMiss, 1)
 
-	account, errs := aFetcherWithCache.FetchAccount(ctx, "uncached")
+	account, errs := aFetcherWithCache.FetchAccount(ctx, json.RawMessage("{}"), "uncached")
 
 	accCache.AssertExpectations(t)
 	fetcher.AssertExpectations(t)
 	metricsEngine.AssertExpectations(t)
 	assert.JSONEq(t, `true`, string(account), "FetchAccount should fetch the right account data")
 	assert.Len(t, errs, 0, "FetchAccount shouldn't return any errors")
 }
-
 func TestComposedCache(t *testing.T) {
 	c1 := &mockCache{}
 	c2 := &mockCache{}
@@ -348,8 +347,8 @@ func (f *mockFetcher) FetchResponses(ctx context.Context, ids []string) (data ma
 	return args.Get(0).(map[string]json.RawMessage), args.Get(1).([]error)
 }
 
-func (a *mockFetcher) FetchAccount(ctx context.Context, accountID string) (json.RawMessage, []error) {
-	args := a.Called(ctx, accountID)
+func (a *mockFetcher) FetchAccount(ctx context.Context, defaultAccountsJSON json.RawMessage, accountID string) (json.RawMessage, []error) {
+	args := a.Called(ctx, defaultAccountsJSON, accountID)
 	return args.Get(0).(json.RawMessage), args.Get(1).([]error)
 }
 

stored_requests/multifetcher.go

@@ -40,10 +40,10 @@ func (mf MultiFetcher) FetchResponses(ctx context.Context, ids []string) (data m
 	return nil, nil
 }
 
-func (mf MultiFetcher) FetchAccount(ctx context.Context, accountID string) (account json.RawMessage, errs []error) {
+func (mf MultiFetcher) FetchAccount(ctx context.Context, accountDefaultJSON json.RawMessage, accountID string) (account json.RawMessage, errs []error) {
 	for _, f := range mf {
 		if af, ok := f.(AccountFetcher); ok {
-			if account, accErrs := af.FetchAccount(ctx, accountID); len(accErrs) == 0 {
+			if account, accErrs := af.FetchAccount(ctx, accountDefaultJSON, accountID); len(accErrs) == 0 {
 				return account, nil
 			} else {
 				accErrs = dropMissingIDs(accErrs)