Stored resp fetcher unit tests

config/stored_requests.go

@@ -341,8 +341,8 @@ func (cfg *PostgresFetcherQueries) MakeQuery(numReqs int, numImps int) (query st
 	return resolve(cfg.QueryTemplate, numReqs, numImps)
 }
 
-func (cfg *PostgresFetcherQueries) MakeQuerySingleArray(numIds int) (query string) {
-	return resolveSingleArrayQuery(cfg.QueryTemplate, numIds)
+func (cfg *PostgresFetcherQueries) MakeQueryResponses(numIds int) (query string) {
+	return resolveQueryResponses(cfg.QueryTemplate, numIds)
 }
 
 func resolve(template string, numReqs int, numImps int) (query string) {
@@ -354,7 +354,7 @@ func resolve(template string, numReqs int, numImps int) (query string) {
 	return
 }
 
-func resolveSingleArrayQuery(template string, numIds int) (query string) {
+func resolveQueryResponses(template string, numIds int) (query string) {
 	numIds = ensureNonNegative("Response", numIds)
 
 	query = strings.Replace(template, "%ID_LIST%", makeIdList(0, numIds), -1)

config/stored_requests_test.go

@@ -10,11 +10,13 @@ import (
 )
 
 const sampleQueryTemplate = "SELECT id, requestData, 'request' as type FROM stored_requests WHERE id in %REQUEST_ID_LIST% UNION ALL SELECT id, impData, 'imp' as type FROM stored_requests WHERE id in %IMP_ID_LIST%"
+const sampleResponsesQueryTemplate = "SELECT id, responseData, 'response' as type FROM stored_responses WHERE id in %ID_LIST%"
 
 func TestNormalQueryMaker(t *testing.T) {
 	madeQuery := buildQuery(sampleQueryTemplate, 1, 3)
 	assertStringsEqual(t, madeQuery, "SELECT id, requestData, 'request' as type FROM stored_requests WHERE id in ($1) UNION ALL SELECT id, impData, 'imp' as type FROM stored_requests WHERE id in ($2, $3, $4)")
 }
+
 func TestQueryMakerManyImps(t *testing.T) {
 	madeQuery := buildQuery(sampleQueryTemplate, 1, 11)
 	assertStringsEqual(t, madeQuery, "SELECT id, requestData, 'request' as type FROM stored_requests WHERE id in ($1) UNION ALL SELECT id, impData, 'imp' as type FROM stored_requests WHERE id in ($2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)")
@@ -41,6 +43,26 @@ func TestQueryMakerNegative(t *testing.T) {
 	assertStringsEqual(t, query, expected)
 }
 
+func TestNormalResponseQueryMaker(t *testing.T) {
+	madeQuery := buildQueryResponses(sampleResponsesQueryTemplate, 1)
+	assertStringsEqual(t, madeQuery, "SELECT id, responseData, 'response' as type FROM stored_responses WHERE id in ($1)")
+}
+func TestResponseQueryMakerManyIds(t *testing.T) {
+	madeQuery := buildQueryResponses(sampleResponsesQueryTemplate, 11)
+	assertStringsEqual(t, madeQuery, "SELECT id, responseData, 'response' as type FROM stored_responses WHERE id in ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)")
+}
+
+func TestResponsesQueryMakerNoIds(t *testing.T) {
+	madeQuery := buildQueryResponses(sampleResponsesQueryTemplate, 0)
+	assertStringsEqual(t, madeQuery, "SELECT id, responseData, 'response' as type FROM stored_responses WHERE id in (NULL)")
+}
+
+func TestResponseQueryMakerNegative(t *testing.T) {
+	query := buildQueryResponses(sampleResponsesQueryTemplate, -2)
+	expected := buildQueryResponses(sampleResponsesQueryTemplate, 0)
+	assertStringsEqual(t, query, expected)
+}
+
 func TestPostgressConnString(t *testing.T) {
 	db := "TestDB"
 	host := "somehost.com"
@@ -302,6 +324,13 @@ func buildQuery(template string, numReqs int, numImps int) string {
 	return cfg.MakeQuery(numReqs, numImps)
 }
 
+func buildQueryResponses(template string, numResps int) string {
+	cfg := PostgresFetcherQueries{}
+	cfg.QueryTemplate = template
+
+	return cfg.MakeQueryResponses(numResps)
+}
+
 func assertStringsEqual(t *testing.T, actual string, expected string) {
 	if actual != expected {
 		t.Errorf("Queries did not match.\n\"%s\" -- expected\n\"%s\" -- actual", expected, actual)

stored_requests/backends/db_fetcher/fetcher.go

@@ -12,25 +12,28 @@ import (
 	"github.com/prebid/prebid-server/stored_requests"
 )
 
-func NewFetcher(db *sql.DB, queryMaker func(int, int) string, queryMakerSingleArr func(int) string) stored_requests.AllFetcher {
+func NewFetcher(db *sql.DB, queryMaker func(int, int) string, responseQueryMaker func(int) string) stored_requests.AllFetcher {
 	if db == nil {
 		glog.Fatalf("The Postgres Stored Request Fetcher requires a database connection. Please report this as a bug.")
 	}
 	if queryMaker == nil {
 		glog.Fatalf("The Postgres Stored Request Fetcher requires a queryMaker function. Please report this as a bug.")
 	}
+	if responseQueryMaker == nil {
+		glog.Fatalf("The Postgres Stored Response Fetcher requires a responseQueryMaker function. Please report this as a bug.")
+	}
 	return &dbFetcher{
-		db:                    db,
-		queryMaker:            queryMaker,
-		queryMakerSingleArray: queryMakerSingleArr,
+		db:                 db,
+		queryMaker:         queryMaker,
+		responseQueryMaker: responseQueryMaker,
 	}
 }
 
 // dbFetcher fetches Stored Requests from a database. This should be instantiated through the NewFetcher() function.
 type dbFetcher struct {
-	db                    *sql.DB
-	queryMaker            func(numReqs int, numImps int) (query string)
-	queryMakerSingleArray func(numIds int) (query string)
+	db                 *sql.DB
+	queryMaker         func(numReqs int, numImps int) (query string)
+	responseQueryMaker func(numIds int) (query string)
 }
 
 func (fetcher *dbFetcher) FetchRequests(ctx context.Context, requestIDs []string, impIDs []string) (map[string]json.RawMessage, map[string]json.RawMessage, []error) {
@@ -101,7 +104,7 @@ func (fetcher *dbFetcher) FetchResponses(ctx context.Context, ids []string) (dat
 		return nil, nil
 	}
 
-	query := fetcher.queryMakerSingleArray(len(ids))
+	query := fetcher.responseQueryMaker(len(ids))
 	idInterfaces := make([]interface{}, len(ids))
 	for i := 0; i < len(ids); i++ {
 		idInterfaces[i] = ids[i]

stored_requests/backends/db_fetcher/fetcher_test.go

@@ -21,13 +21,18 @@ func TestEmptyQuery(t *testing.T) {
 	defer db.Close()
 
 	fetcher := dbFetcher{
-		db:         db,
-		queryMaker: successfulQueryMaker(""),
+		db:                 db,
+		queryMaker:         successfulQueryMaker(""),
+		responseQueryMaker: successfulResponseQueryMaker(""),
 	}
 	storedReqs, storedImps, errs := fetcher.FetchRequests(context.Background(), nil, nil)
 	assertErrorCount(t, 0, errs)
 	assertMapLength(t, 0, storedReqs)
 	assertMapLength(t, 0, storedImps)
+
+	storedResponses, errs := fetcher.FetchResponses(context.Background(), nil)
+	assertErrorCount(t, 0, errs)
+	assertMapLength(t, 0, storedResponses)
 }
 
 // TestGoodResponse makes sure we interpret DB responses properly when all the stored requests are there.
@@ -52,6 +57,24 @@ func TestGoodResponse(t *testing.T) {
 	assertHasData(t, storedImps, "imp-id-2", `{"imp":true,"value":2}`)
 }
 
+func TestGoodResponseFetchResponses(t *testing.T) {
+	mockQuery := "SELECT id, data, 'response' AS dataType FROM responses_table WHERE id IN (?, ?)"
+	mockReturn := sqlmock.NewRows([]string{"id", "data", "dataType"}).
+		AddRow("resp-id-1", `{"resp":false,"value":1}`, "response").
+		AddRow("resp-id-2", `{"resp":true,"value":2}`, "response")
+
+	mock, fetcher := newFetcher(t, mockReturn, mockQuery, "resp-id-1", "resp-id-2")
+	defer fetcher.db.Close()
+
+	storedResponses, errs := fetcher.FetchResponses(context.Background(), []string{"resp-id-1", "resp-id-2"})
+
+	assertMockExpectations(t, mock)
+	assertErrorCount(t, 0, errs)
+	assertMapLength(t, 2, storedResponses)
+	assertHasData(t, storedResponses, "resp-id-1", `{"resp":false,"value":1}`)
+	assertHasData(t, storedResponses, "resp-id-2", `{"resp":true,"value":2}`)
+}
+
 // TestPartialResponse makes sure we unpack things properly when the DB finds some of the stored requests.
 func TestPartialResponse(t *testing.T) {
 	mockQuery := "SELECT id, data, 'request' AS dataType FROM req_table WHERE id IN (?, ?) UNION ALL SELECT id, data, 'imp' as dataType FROM imp_table WHERE id IN (NULL)"
@@ -70,6 +93,22 @@ func TestPartialResponse(t *testing.T) {
 	assertHasData(t, storedReqs, "stored-req-id", "{}")
 }
 
+func TestPartialResponseFetchResponses(t *testing.T) {
+	mockQuery := "SELECT id, data, 'response' AS dataType FROM responses_table WHERE id IN (?, ?)"
+	mockReturn := sqlmock.NewRows([]string{"id", "data", "dataType"}).
+		AddRow("stored-resp-id", "{}", "response")
+
+	mock, fetcher := newFetcher(t, mockReturn, mockQuery, "stored-resp-id", "stored-resp-id-2")
+	defer fetcher.db.Close()
+
+	storedResponses, errs := fetcher.FetchResponses(context.Background(), []string{"stored-resp-id", "stored-resp-id-2"})
+
+	assertMockExpectations(t, mock)
+	assertErrorCount(t, 0, errs)
+	assertMapLength(t, 1, storedResponses)
+	assertHasData(t, storedResponses, "stored-resp-id", "{}")
+}
+
 // TestEmptyResponse makes sure we handle empty DB responses properly.
 func TestEmptyResponse(t *testing.T) {
 	mockQuery := "SELECT id, data, dataType FROM my_table WHERE id IN (?, ?)"
@@ -86,6 +125,20 @@ func TestEmptyResponse(t *testing.T) {
 	assertMapLength(t, 0, storedImps)
 }
 
+func TestEmptyResponseFetchResponses(t *testing.T) {
+	mockQuery := "SELECT id, data, dataType FROM my_table WHERE id IN (?, ?)"
+	mockReturn := sqlmock.NewRows([]string{"id", "data", "dataType"})
+
+	mock, fetcher := newFetcher(t, mockReturn, mockQuery, "stored-resp-id", "stored-resp-id-2")
+	defer fetcher.db.Close()
+
+	storedResponses, errs := fetcher.FetchResponses(context.Background(), []string{"stored-resp-id", "stored-resp-id-2"})
+
+	assertMockExpectations(t, mock)
+	assertErrorCount(t, 0, errs)
+	assertMapLength(t, 0, storedResponses)
+}
+
 // TestDatabaseError makes sure we exit with an error if the DB query fails.
 func TestDatabaseError(t *testing.T) {
 	db, mock, err := sqlmock.New()
@@ -116,16 +169,22 @@ func TestContextDeadlines(t *testing.T) {
 	mock.ExpectQuery(".*").WillDelayFor(2 * time.Minute)
 
 	fetcher := &dbFetcher{
-		db:         db,
-		queryMaker: successfulQueryMaker("SELECT id, requestData FROM my_table WHERE id IN (?, ?)"),
+		db:                 db,
+		queryMaker:         successfulQueryMaker("SELECT id, requestData FROM my_table WHERE id IN (?, ?)"),
+		responseQueryMaker: successfulResponseQueryMaker("SELECT id, responseData FROM my_table WHERE id IN (?, ?)"),
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
 	defer cancel()
+
 	_, _, errs := fetcher.FetchRequests(ctx, []string{"id"}, nil)
 	if len(errs) < 1 {
 		t.Errorf("dbFetcher should return an error when the context times out.")
 	}
+	_, errs = fetcher.FetchResponses(ctx, []string{"id"})
+	if len(errs) < 1 {
+		t.Errorf("dbFetcher should return an error when the context times out.")
+	}
 }
 
 // TestContextCancelled makes sure a hung query returns when the context is cancelled.
@@ -138,8 +197,9 @@ func TestContextCancelled(t *testing.T) {
 	mock.ExpectQuery(".*").WillDelayFor(2 * time.Minute)
 
 	fetcher := &dbFetcher{
-		db:         db,
-		queryMaker: successfulQueryMaker("SELECT id, requestData FROM my_table WHERE id IN (?, ?)"),
+		db:                 db,
+		queryMaker:         successfulQueryMaker("SELECT id, requestData FROM my_table WHERE id IN (?, ?)"),
+		responseQueryMaker: successfulResponseQueryMaker("SELECT id, responseData FROM my_table WHERE id IN (?, ?)"),
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -148,6 +208,10 @@ func TestContextCancelled(t *testing.T) {
 	if len(errs) < 1 {
 		t.Errorf("dbFetcher should return an error when the context is cancelled.")
 	}
+	_, errs = fetcher.FetchResponses(ctx, []string{"id"})
+	if len(errs) < 1 {
+		t.Errorf("dbFetcher should return an error when the context is cancelled.")
+	}
 }
 
 // Prevents #338
@@ -173,6 +237,29 @@ func TestRowErrors(t *testing.T) {
 	assertMapLength(t, 0, data)
 }
 
+func TestRowErrorsFetchResponses(t *testing.T) {
+	db, mock, err := sqlmock.New()
+	if err != nil {
+		t.Fatalf("Failed to create mock: %v", err)
+	}
+	rows := sqlmock.NewRows([]string{"id", "data", "dataType"})
+	rows.AddRow("foo", []byte(`{"data":1}`), "response")
+	rows.AddRow("bar", []byte(`{"data":2}`), "response")
+	rows.RowError(1, errors.New("Error reading from row 1"))
+	mock.ExpectQuery(".*").WillReturnRows(rows)
+	fetcher := &dbFetcher{
+		db:                 db,
+		queryMaker:         successfulQueryMaker("SELECT id, data, dataType FROM my_table WHERE id IN (?)"),
+		responseQueryMaker: successfulResponseQueryMaker("SELECT id, data, dataType FROM my_table WHERE id IN (?)"),
+	}
+	data, errs := fetcher.FetchResponses(context.Background(), []string{"foo", "bar"})
+	assertErrorCount(t, 1, errs)
+	if errs[0].Error() != "Error reading from row 1" {
+		t.Errorf("Unexpected error message: %v", errs[0].Error())
+	}
+	assertMapLength(t, 0, data)
+}
+
 func newFetcher(t *testing.T, rows *sqlmock.Rows, query string, args ...driver.Value) (sqlmock.Sqlmock, *dbFetcher) {
 	db, mock, err := sqlmock.New()
 	if err != nil {
@@ -183,8 +270,9 @@ func newFetcher(t *testing.T, rows *sqlmock.Rows, query string, args ...driver.V
 	queryRegex := fmt.Sprintf("^%s$", regexp.QuoteMeta(query))
 	mock.ExpectQuery(queryRegex).WithArgs(args...).WillReturnRows(rows)
 	fetcher := &dbFetcher{
-		db:         db,
-		queryMaker: successfulQueryMaker(query),
+		db:                 db,
+		queryMaker:         successfulQueryMaker(query),
+		responseQueryMaker: successfulResponseQueryMaker(query),
 	}
 
 	return mock, fetcher
@@ -227,3 +315,9 @@ func successfulQueryMaker(response string) func(int, int) string {
 		return response
 	}
 }
+
+func successfulResponseQueryMaker(response string) func(int) string {
+	return func(numIds int) string {
+		return response
+	}
+}

stored_requests/config/config.go

@@ -172,7 +172,7 @@ func newFetcher(cfg *config.StoredRequests, client *http.Client, db *sql.DB) (fe
 	}
 	if cfg.Postgres.FetcherQueries.QueryTemplate != "" {
 		glog.Infof("Loading Stored %s data via Postgres.\nQuery: %s", cfg.DataType(), cfg.Postgres.FetcherQueries.QueryTemplate)
-		idList = append(idList, db_fetcher.NewFetcher(db, cfg.Postgres.FetcherQueries.MakeQuery, cfg.Postgres.FetcherQueries.MakeQuerySingleArray))
+		idList = append(idList, db_fetcher.NewFetcher(db, cfg.Postgres.FetcherQueries.MakeQuery, cfg.Postgres.FetcherQueries.MakeQueryResponses))
 	} else if cfg.Postgres.CacheInitialization.Query != "" && cfg.Postgres.PollUpdates.Query != "" {
 		//in this case data will be loaded to cache via poll for updates event
 		idList = append(idList, empty_fetcher.EmptyFetcher{})

stored_requests/config/config_test.go

@@ -158,6 +158,7 @@ func TestNewEmptyCache(t *testing.T) {
 	cache := newCache(&config.StoredRequests{InMemoryCache: config.InMemoryCache{Type: "none"}})
 	assert.True(t, isEmptyCacheType(cache.Requests), "The newCache method should return an empty Request cache")
 	assert.True(t, isEmptyCacheType(cache.Imps), "The newCache method should return an empty Imp cache")
+	assert.True(t, isEmptyCacheType(cache.Responses), "The newCache method should return an empty Responses cache")
 	assert.True(t, isEmptyCacheType(cache.Accounts), "The newCache method should return an empty Account cache")
 }
 
@@ -172,6 +173,7 @@ func TestNewInMemoryCache(t *testing.T) {
 	})
 	assert.True(t, isMemoryCacheType(cache.Requests), "The newCache method should return an in-memory Request cache for StoredRequests config")
 	assert.True(t, isMemoryCacheType(cache.Imps), "The newCache method should return an in-memory Imp cache for StoredRequests config")
+	assert.True(t, isMemoryCacheType(cache.Responses), "The newCache method should return an in-memory Responses cache for StoredResponses config")
 	assert.True(t, isEmptyCacheType(cache.Accounts), "The newCache method should return an empty Account cache for StoredRequests config")
 }
 
@@ -185,6 +187,7 @@ func TestNewInMemoryAccountCache(t *testing.T) {
 	assert.True(t, isMemoryCacheType(cache.Accounts), "The newCache method should return an in-memory Account cache for Accounts config")
 	assert.True(t, isEmptyCacheType(cache.Requests), "The newCache method should return an empty Request cache for Accounts config")
 	assert.True(t, isEmptyCacheType(cache.Imps), "The newCache method should return an empty Imp cache for Accounts config")
+	assert.True(t, isEmptyCacheType(cache.Responses), "The newCache method should return an empty Responses cache for Accounts config")
 }
 
 func TestNewPostgresEventProducers(t *testing.T) {

stored_requests/events/postgres/database.go

@@ -219,18 +219,19 @@ func (e *PostgresEventProducer) sendEvents(rows *sql.Rows) (err error) {
 		return rows.Err()
 	}
 
-	if len(storedRequestData) > 0 || len(storedImpData) > 0 {
+	if len(storedRequestData) > 0 || len(storedImpData) > 0 || len(storedRespData) > 0 {
 		e.saves <- events.Save{
 			Requests:  storedRequestData,
 			Imps:      storedImpData,
 			Responses: storedRespData,
 		}
 	}
 
-	if (len(requestInvalidations) > 0 || len(impInvalidations) > 0) && !e.lastUpdate.IsZero() {
+	if (len(requestInvalidations) > 0 || len(impInvalidations) > 0 || len(respInvalidations) > 0) && !e.lastUpdate.IsZero() {
 		e.invalidations <- events.Invalidation{
-			Requests: requestInvalidations,
-			Imps:     impInvalidations,
+			Requests:  requestInvalidations,
+			Imps:      impInvalidations,
+			Responses: respInvalidations,
 		}
 	}