diff --git a/docs/endpoints/openrtb2/auction.md b/docs/endpoints/openrtb2/auction.md index cfe1978eb0b..c4df49e9cfe 100644 --- a/docs/endpoints/openrtb2/auction.md +++ b/docs/endpoints/openrtb2/auction.md @@ -110,7 +110,7 @@ The only exception here is the top-level `BidResponse`, because it's bidder-inde #### Details -##### Targeting +#### Targeting Targeting refers to strings which are sent to the adserver to [make header bidding possible](http://prebid.org/overview/intro.html#how-does-prebid-work). @@ -140,49 +140,70 @@ to set these params on the response at `response.seatbid[i].bid[j].ext.prebid.ta The winning bid for each `request.imp[i]` will also contain `hb_bidder`, `hb_size`, and `hb_pb` (with _no_ {bidderName} suffix). -#### Improving Performance +#### Bidder Aliases -`response.ext.responsetimemillis.{bidderName}` tells how long each bidder took to respond. -These can help quantify the performance impact of "the slowest bidder." - -`response.ext.errors.{bidderName}` contains messages which describe why a request may be "suboptimal". -For example, suppose a `banner` and a `video` impression are offered to a bidder -which only supports `banner`. - -In cases like these, the bidder can ignore the `video` impression and bid on the `banner` one. -However, the publisher can improve performance by only offering impressions which the bidder supports. +Requests can define Bidder aliases if they want to refer to a Bidder by a separate name. +This can be used to request bids from the same Bidder with different params. For example: -`response.ext.usersync.{bidderName}` contains user sync (aka cookie sync) status for this bidder/user. +``` +{ + "imp": [ + { + "id": "some-impression-id", + "video": { + "mimes": ["video/mp4"] + }, + "ext": { + "appnexus: { + "placementId": 123 + }, + "districtm": { + "placementId": 456 + } + } + } + ], + "ext": { + "prebid": { + "aliases": { + "districtm": "appnexus" + } + } + } +} +``` -This includes: +For all intents and purposes, the alias will be treated as another Bidder. This new Bidder will behave exactly +like the original, except that the Response will contain seprate SeatBids, and any Targeting keys +will be formed using the alias' name. -1. Whether a user sync was present for this auction. -2. URL information to initiate a usersync. +If an alias overlaps with a core Bidder's name, then the alias will take precedence. +This prevents breaking API changes as new Bidders are added to the project. -Some sample response data: +For example, if the Request defines an alias like this: ``` { - "appnexus": { - "status": "one of ['none', 'expired', 'available']", - "syncs": [ - "url": "sync.url.com", - "type": "one of ['iframe', 'redirect']" - ] - }, - "rubicon": { - "status": "available" // If a usersync is available, there are probably no syncs to run. + "aliases": { + "appnexus": "rubicon" } } ``` -A `status` of `available` means that the user was synced with this bidder for this auction. +then any `imp.ext.appnexus` params will actually go to the **rubicon** adapter. +It will become impossible to fetch bids from Appnexus within that Request. -A `status` of `expired` means that the a user was synced, but it last happened over 7 days ago and may be stale. +#### Bidder Response Times -A `status` of `none` means that no user sync existed for this bidder. +`response.ext.responsetimemillis.{bidderName}` tells how long each bidder took to respond. +These can help quantify the performance impact of "the slowest bidder." + +`response.ext.errors.{bidderName}` contains messages which describe why a request may be "suboptimal". +For example, suppose a `banner` and a `video` impression are offered to a bidder +which only supports `banner`. -PBS requests new syncs by returning the `response.ext.usersync.{bidderName}.syncs` array. +In cases like these, the bidder can ignore the `video` impression and bid on the `banner` one. +However, the publisher can improve performance by only offering impressions which the bidder supports. #### Debugging diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index 90c4ddd50d7..53a6a705828 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -191,8 +191,19 @@ func (deps *endpointDeps) validateRequest(req *openrtb.BidRequest) error { return errors.New("request.imp must contain at least one element.") } + var aliases map[string]string + if bidExt, err := deps.parseBidExt(req.Ext); err != nil { + return err + } else if bidExt != nil { + aliases = bidExt.Prebid.Aliases + } + + if err := deps.validateAliases(aliases); err != nil { + return err + } + for index, imp := range req.Imp { - if err := deps.validateImp(&imp, index); err != nil { + if err := deps.validateImp(&imp, aliases, index); err != nil { return err } } @@ -209,14 +220,10 @@ func (deps *endpointDeps) validateRequest(req *openrtb.BidRequest) error { return err } - if err := deps.validateBidRequestExt(req.Ext); err != nil { - return err - } - return nil } -func (deps *endpointDeps) validateImp(imp *openrtb.Imp, index int) error { +func (deps *endpointDeps) validateImp(imp *openrtb.Imp, aliases map[string]string, index int) error { if imp.ID == "" { return fmt.Errorf("request.imp[%d] missing required field: \"id\"", index) } @@ -255,7 +262,7 @@ func (deps *endpointDeps) validateImp(imp *openrtb.Imp, index int) error { return err } - if err := deps.validateImpExt(imp.Ext, index); err != nil { + if err := deps.validateImpExt(imp.Ext, aliases, index); err != nil { return err } @@ -321,7 +328,7 @@ func validatePmp(pmp *openrtb.PMP, impIndex int) error { return nil } -func (deps *endpointDeps) validateImpExt(ext openrtb.RawJSON, impIndex int) error { +func (deps *endpointDeps) validateImpExt(ext openrtb.RawJSON, aliases map[string]string, impIndex int) error { var bidderExts map[string]openrtb.RawJSON if err := json.Unmarshal(ext, &bidderExts); err != nil { return err @@ -332,26 +339,43 @@ func (deps *endpointDeps) validateImpExt(ext openrtb.RawJSON, impIndex int) erro } for bidder, ext := range bidderExts { - bidderName, isValid := openrtb_ext.BidderMap[bidder] - if isValid { - if err := deps.paramsValidator.Validate(bidderName, ext); err != nil { - return fmt.Errorf("request.imp[%d].ext.%s failed validation.\n%v", impIndex, bidder, err) + if bidder != "prebid" { + coreBidder := bidder + if tmp, isAlias := aliases[bidder]; isAlias { + coreBidder = tmp + } + if bidderName, isValid := openrtb_ext.BidderMap[coreBidder]; isValid { + if err := deps.paramsValidator.Validate(bidderName, ext); err != nil { + return fmt.Errorf("request.imp[%d].ext.%s failed validation.\n%v", impIndex, coreBidder, err) + } + } else { + return fmt.Errorf("request.imp[%d].ext contains unknown bidder: %s. Did you forget an alias in request.ext.prebid.aliases?", impIndex, bidder) } - } else if bidder != "prebid" { - return fmt.Errorf("request.imp[%d].ext contains unknown bidder: %s", impIndex, bidder) } } return nil } -func (deps *endpointDeps) validateBidRequestExt(ext openrtb.RawJSON) error { +func (deps *endpointDeps) parseBidExt(ext openrtb.RawJSON) (*openrtb_ext.ExtRequest, error) { if len(ext) < 1 { - return nil + return nil, nil } var tmpExt openrtb_ext.ExtRequest if err := json.Unmarshal(ext, &tmpExt); err != nil { - return fmt.Errorf("request.ext is invalid: %v", err) + return nil, fmt.Errorf("request.ext is invalid: %v", err) + } + return &tmpExt, nil +} + +func (deps *endpointDeps) validateAliases(aliases map[string]string) error { + for thisAlias, coreBidder := range aliases { + if _, isCoreBidder := openrtb_ext.BidderMap[coreBidder]; !isCoreBidder { + return fmt.Errorf("request.ext.prebid.aliases.%s refers to unknown bidder: %s", thisAlias, coreBidder) + } + if thisAlias == coreBidder { + return fmt.Errorf("request.ext.prebid.aliases.%s defines a no-op alias. Choose a different alias, or remove this entry.", thisAlias) + } } return nil } diff --git a/endpoints/openrtb2/auction_test.go b/endpoints/openrtb2/auction_test.go index 6870e9e81f1..7c35cbaa2d8 100644 --- a/endpoints/openrtb2/auction_test.go +++ b/endpoints/openrtb2/auction_test.go @@ -5,6 +5,13 @@ import ( "context" "encoding/json" "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + "github.com/evanphx/json-patch" "github.com/mxmCherry/openrtb" "github.com/prebid/prebid-server/config" @@ -13,12 +20,6 @@ import ( "github.com/prebid/prebid-server/pbsmetrics" "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" "github.com/rcrowley/go-metrics" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" ) const maxSize = 1024 * 256 @@ -34,8 +35,7 @@ func TestGoodRequests(t *testing.T) { endpoint(recorder, request, nil) if recorder.Code != http.StatusOK { - t.Errorf("Expected status %d. Got %d. Request data was %s", http.StatusOK, recorder.Code, requestData) - //t.Errorf("Response body was: %s", recorder.Body) + t.Fatalf("Expected status %d. Got %d. Request data was %s\n\nResponse body was: %s", http.StatusOK, recorder.Code, requestData, recorder.Body.String()) } var response openrtb.BidResponse @@ -666,6 +666,30 @@ var validRequests = []string{ } ] }`, + `{ + "id": "some-request-id", + "site": { + "page": "test.somepage.com" + }, + "imp": [ + { + "id": "my-imp-id", + "video": { + "mimes":["video/mp4"] + }, + "ext": { + "unknown": "good" + } + } + ], + "ext": { + "prebid": { + "aliases": { + "unknown": "appnexus" + } + } + } + }`, } var invalidRequests = []string{ @@ -873,6 +897,54 @@ var invalidRequests = []string{ } } }`, + `{ + "id": "some-request-id", + "site": { + "page": "test.somepage.com" + }, + "imp": [ + { + "id": "my-imp-id", + "video": { + "mimes":["video/mp4"] + }, + "ext": { + "unknown": "good" + } + } + ], + "ext": { + "prebid": { + "aliases": { + "unknown": "other-unknown" + } + } + } + }`, + `{ + "id": "some-request-id", + "site": { + "page": "test.somepage.com" + }, + "imp": [ + { + "id": "my-imp-id", + "video": { + "mimes":["video/mp4"] + }, + "ext": { + "appnexus": "good" + } + } + ], + "ext": { + "prebid": { + "aliases": { + "appnexus": "appnexus" + } + } + } + }`, } // StoredRequest testing diff --git a/exchange/exchange.go b/exchange/exchange.go index 6a785b04ac9..81ca45a31f8 100644 --- a/exchange/exchange.go +++ b/exchange/exchange.go @@ -28,8 +28,6 @@ type IdFetcher interface { } type exchange struct { - // The list of adapters we will consider for this auction - adapters []openrtb_ext.BidderName adapterMap map[openrtb_ext.BidderName]adaptedBidder m *pbsmetrics.Metrics cache prebid_cache_client.Client @@ -52,30 +50,26 @@ func NewExchange(client *http.Client, cache prebid_cache_client.Client, cfg *con e := new(exchange) e.adapterMap = newAdapterMap(client, cfg) - e.adapters = make([]openrtb_ext.BidderName, 0, len(e.adapterMap)) e.cache = cache e.cacheTime = time.Duration(cfg.CacheURL.ExpectedTimeMillis) * time.Millisecond - for a, _ := range e.adapterMap { - e.adapters = append(e.adapters, a) - } e.m = registry return e } func (e *exchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, usersyncs IdFetcher) (*openrtb.BidResponse, error) { // Slice of BidRequests, each a copy of the original cleaned to only contain bidder data for the named bidder - cleanRequests, errs := cleanOpenRTBRequests(bidRequest, e.adapters, usersyncs, e.m) + cleanRequests, aliases, errs := cleanOpenRTBRequests(bidRequest, usersyncs, e.m) // List of bidders we have requests for. liveAdapters := make([]openrtb_ext.BidderName, len(cleanRequests)) i := 0 - for a, _ := range cleanRequests { + for a := range cleanRequests { liveAdapters[i] = a i++ } // Randomize the list of adapters to make the auction more fair randomizeList(liveAdapters) // Process the request to check for targeting parameters. - var targData *targetData = nil + var targData *targetData shouldCacheBids := false if len(bidRequest.Ext) > 0 { var requestExt openrtb_ext.ExtRequest @@ -101,7 +95,7 @@ func (e *exchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidReque auctionCtx, cancel := e.makeAuctionContext(ctx, shouldCacheBids) defer cancel() - adapterBids, adapterExtra := e.getAllBids(auctionCtx, liveAdapters, cleanRequests, targData) + adapterBids, adapterExtra := e.getAllBids(auctionCtx, cleanRequests, aliases, targData) // Build the response return e.buildBidResponse(ctx, liveAdapters, adapterBids, bidRequest, adapterExtra, targData, errs) @@ -119,19 +113,21 @@ func (e *exchange) makeAuctionContext(ctx context.Context, needsCache bool) (auc } // This piece sends all the requests to the bidder adapters and gathers the results. -func (e *exchange) getAllBids(ctx context.Context, liveAdapters []openrtb_ext.BidderName, cleanRequests map[openrtb_ext.BidderName]*openrtb.BidRequest, targData *targetData) (map[openrtb_ext.BidderName]*pbsOrtbSeatBid, map[openrtb_ext.BidderName]*seatResponseExtra) { +func (e *exchange) getAllBids(ctx context.Context, cleanRequests map[openrtb_ext.BidderName]*openrtb.BidRequest, aliases map[string]string, targData *targetData) (map[openrtb_ext.BidderName]*pbsOrtbSeatBid, map[openrtb_ext.BidderName]*seatResponseExtra) { // Set up pointers to the bid results - adapterBids := make(map[openrtb_ext.BidderName]*pbsOrtbSeatBid, len(liveAdapters)) - adapterExtra := make(map[openrtb_ext.BidderName]*seatResponseExtra, len(liveAdapters)) - chBids := make(chan *bidResponseWrapper, len(liveAdapters)) - for _, a := range liveAdapters { + adapterBids := make(map[openrtb_ext.BidderName]*pbsOrtbSeatBid, len(cleanRequests)) + adapterExtra := make(map[openrtb_ext.BidderName]*seatResponseExtra, len(cleanRequests)) + chBids := make(chan *bidResponseWrapper, len(cleanRequests)) + + for bidderName, req := range cleanRequests { // Here we actually call the adapters and collect the bids. - go func(aName openrtb_ext.BidderName) { + go func(aName openrtb_ext.BidderName, coreBidder openrtb_ext.BidderName, request *openrtb.BidRequest) { // Passing in aName so a doesn't change out from under the go routine brw := new(bidResponseWrapper) brw.bidder = aName start := time.Now() - bids, err := e.adapterMap[aName].requestBid(ctx, cleanRequests[aName], targData, aName) + + bids, err := e.adapterMap[coreBidder].requestBid(ctx, request, targData, aName) // Add in time reporting elapsed := time.Since(start) @@ -140,7 +136,7 @@ func (e *exchange) getAllBids(ctx context.Context, liveAdapters []openrtb_ext.Bi ae := new(seatResponseExtra) ae.ResponseTimeMillis = int(elapsed / time.Millisecond) // Timing statistics - e.m.AdapterMetrics[aName].RequestTimer.UpdateSince(start) + e.m.AdapterMetrics[coreBidder].RequestTimer.UpdateSince(start) serr := make([]string, len(err)) for i := 0; i < len(err); i++ { serr[i] = err[i].Error() @@ -148,29 +144,29 @@ func (e *exchange) getAllBids(ctx context.Context, liveAdapters []openrtb_ext.Bi // in the metrics. Need to remember that in analyzing the data. switch err[i] { case context.DeadlineExceeded: - e.m.AdapterMetrics[aName].TimeoutMeter.Mark(1) + e.m.AdapterMetrics[coreBidder].TimeoutMeter.Mark(1) default: - e.m.AdapterMetrics[aName].ErrorMeter.Mark(1) + e.m.AdapterMetrics[coreBidder].ErrorMeter.Mark(1) } } ae.Errors = serr brw.adapterExtra = ae if len(err) == 0 { if bids == nil || len(bids.bids) == 0 { - // Don't want to mark no bids on error to preserve legacy behavior. - e.m.AdapterMetrics[aName].NoBidMeter.Mark(1) + // Don't want to mark no bids on error topreserve legacy behavior. + e.m.AdapterMetrics[coreBidder].NoBidMeter.Mark(1) } else { for _, bid := range bids.bids { var cpm = int64(bid.bid.Price * 1000) - e.m.AdapterMetrics[aName].PriceHistogram.Update(cpm) + e.m.AdapterMetrics[coreBidder].PriceHistogram.Update(cpm) } } } chBids <- brw - }(a) + }(bidderName, resolveBidder(string(bidderName), aliases), req) } // Wait for the bidders to do their thing - for i := 0; i < len(liveAdapters); i++ { + for i := 0; i < len(cleanRequests); i++ { brw := <-chBids adapterExtra[brw.bidder] = brw.adapterExtra adapterBids[brw.bidder] = brw.adapterBids @@ -189,7 +185,7 @@ func (e *exchange) buildBidResponse(ctx context.Context, liveAdapters []openrtb_ bidResponse.NBR = openrtb.NoBidReasonCode.Ptr(openrtb.NoBidReasonCodeInvalidRequest) } - var auc *auction = nil + var auc *auction if targData != nil { auc = newAuction(len(bidRequest.Imp)) } diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go index d471d97344d..39a16fffcb3 100644 --- a/exchange/exchange_test.go +++ b/exchange/exchange_test.go @@ -24,24 +24,23 @@ func TestNewExchange(t *testing.T) { server := httptest.NewServer(mockHandler(respStatus, "getBody", respBody)) defer server.Close() - // Just match the counts - e := NewExchange(server.Client(), nil, &config.Configuration{}, pbsmetrics.NewMetrics(metrics.NewRegistry(), AdapterList())).(*exchange) - if len(e.adapters) != len(e.adapterMap) { - t.Errorf("Exchange initialized, but adapter list doesn't match adapter map (%d - %d)", len(e.adapters), len(e.adapterMap)) - } - // Test that all adapters are in the map and not repeated - tmp := make(map[openrtb_ext.BidderName]int) - for _, a := range e.adapters { - _, ok := tmp[a] - if ok { - t.Errorf("Exchange.adapters repeats value %s", a) - } - tmp[a] = 1 - _, ok = e.adapterMap[a] - if !ok { - t.Errorf("Exchange.adapterMap missing adpater %s", a) + knownAdapters := AdapterList() + + cfg := &config.Configuration{ + CacheURL: config.Cache{ + ExpectedTimeMillis: 20, + }, + } + + e := NewExchange(server.Client(), nil, cfg, pbsmetrics.NewMetrics(metrics.NewRegistry(), knownAdapters)).(*exchange) + for _, bidderName := range knownAdapters { + if _, ok := e.adapterMap[bidderName]; !ok { + t.Errorf("NewExchange produced an Exchange without bidder %s", bidderName) } } + if e.cacheTime != time.Duration(cfg.CacheURL.ExpectedTimeMillis)*time.Millisecond { + t.Errorf("Bad cacheTime. Expected 20 ms, got %s", e.cacheTime.String()) + } } func TestHoldAuction(t *testing.T) { @@ -123,9 +122,12 @@ func TestGetAllBids(t *testing.T) { mockAdapterConfig2(e.adapterMap[BidderDummy2].(*mockAdapter), "dummy2") mockAdapterConfig3(e.adapterMap[BidderDummy3].(*mockAdapter), "dummy3") - cleanRequests := make(map[openrtb_ext.BidderName]*openrtb.BidRequest) - adapterBids, adapterExtra := e.getAllBids(ctx, e.adapters, cleanRequests, nil) - + cleanRequests := map[openrtb_ext.BidderName]*openrtb.BidRequest{ + BidderDummy: nil, + BidderDummy2: nil, + BidderDummy3: nil, + } + adapterBids, adapterExtra := e.getAllBids(ctx, cleanRequests, nil, nil) if len(adapterBids[BidderDummy].bids) != 2 { t.Errorf("GetAllBids failed to get 2 bids from BidderDummy, found %d instead", len(adapterBids[BidderDummy].bids)) } @@ -144,7 +146,7 @@ func TestGetAllBids(t *testing.T) { if len(e.adapterMap[BidderDummy2].(*mockAdapter).errs) != 2 { t.Errorf("GetAllBids, Bidder2 adapter error generation failed. Only seeing %d errors", len(e.adapterMap[BidderDummy2].(*mockAdapter).errs)) } - adapterBids, adapterExtra = e.getAllBids(ctx, e.adapters, cleanRequests, nil) + adapterBids, adapterExtra = e.getAllBids(ctx, cleanRequests, nil, nil) if len(e.adapterMap[BidderDummy2].(*mockAdapter).errs) != 2 { t.Errorf("GetAllBids, Bidder2 adapter error generation failed. Only seeing %d errors", len(e.adapterMap[BidderDummy2].(*mockAdapter).errs)) @@ -161,7 +163,7 @@ func TestGetAllBids(t *testing.T) { // Test with null pointer for bid response mockAdapterConfigErr2(e.adapterMap[BidderDummy2].(*mockAdapter)) - adapterBids, adapterExtra = e.getAllBids(ctx, e.adapters, cleanRequests, nil) + adapterBids, adapterExtra = e.getAllBids(ctx, cleanRequests, nil, nil) if len(adapterExtra[BidderDummy2].Errors) != 1 { t.Errorf("GetAllBids failed to report 1 errors on Bidder2, found %d errors", len(adapterExtra[BidderDummy2].Errors)) @@ -387,7 +389,6 @@ func runBuyerTest(t *testing.T, incoming *openrtb.BidRequest, expectBuyeridOverr bidder := &mockBidder{} ex := &exchange{ - adapters: []openrtb_ext.BidderName{openrtb_ext.BidderAppnexus}, adapterMap: map[openrtb_ext.BidderName]adaptedBidder{ openrtb_ext.BidderAppnexus: bidder, }, @@ -466,12 +467,9 @@ func NewDummyExchange(client *http.Client) *exchange { BidderDummy3: c, } - e.adapters = make([]openrtb_ext.BidderName, 0, len(e.adapterMap)) - for a, _ := range e.adapterMap { - e.adapters = append(e.adapters, a) - } - e.m = pbsmetrics.NewBlankMetrics(metrics.NewRegistry(), e.adapters) + adapterList := []openrtb_ext.BidderName{BidderDummy, BidderDummy2, BidderDummy3} + e.m = pbsmetrics.NewBlankMetrics(metrics.NewRegistry(), adapterList) e.cache = &wellBehavedCache{} return e } diff --git a/exchange/utils.go b/exchange/utils.go index 541269de9eb..34faa80365b 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -3,108 +3,109 @@ package exchange import ( "encoding/json" "fmt" + "math/rand" + + "github.com/buger/jsonparser" "github.com/mxmCherry/openrtb" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/pbsmetrics" - "math/rand" ) -// Quick little randomizer for a list of strings. Stuffing it in utils to keep other files clean -func randomizeList(list []openrtb_ext.BidderName) { - l := len(list) - perm := rand.Perm(l) - var j int - for i := 0; i < l; i++ { - j = perm[i] - list[i], list[j] = list[j], list[i] +// cleanOpenRTBRequests splits the input request into requests which are sanitized for each bidder. Intended behavior is: +// +// 1. BidRequest.Imp[].Ext will only contain the "prebid" field and a "bidder" field which has the params for the intended Bidder. +// 2. Every BidRequest.Imp[] requested Bids from the Bidder who keys it. +// 3. BidRequest.User.BuyerUID will be set to that Bidder's ID. +func cleanOpenRTBRequests(orig *openrtb.BidRequest, usersyncs IdFetcher, met *pbsmetrics.Metrics) (requestsByBidder map[openrtb_ext.BidderName]*openrtb.BidRequest, aliases map[string]string, errs []error) { + impsByBidder, errs := splitImps(orig.Imp) + if len(errs) > 0 { + return } -} - -// This will copy the openrtb BidRequest into an array of requests, where the BidRequest.Imp[].Ext field will only -// consist of the "prebid" field and the field for the appropriate bidder parameters. We will drop all extended fields -// beyond this context, so this will not be compatible with any other uses of the extension area. That is, this routine -// will work, but the adapters will not see any other extension fields. -// NOTE: the return map will only contain entries for bidders that both have the extension field in at least one Imp, -// and are listed in the adapters string. wseats and bseats can be implimented by passing the bseats list as adapters, -// or after return removing any adapters listed in wseats. Or removing all adapters in wseats from the adapters list -// before submitting. + aliases, errs = parseAliases(orig) + if len(errs) > 0 { + return + } -// Take an openrtb request, and a list of bidders, and return an openrtb request sanitized for each bidder -func cleanOpenRTBRequests(orig *openrtb.BidRequest, adapters []openrtb_ext.BidderName, usersyncs IdFetcher, met *pbsmetrics.Metrics) (map[openrtb_ext.BidderName]*openrtb.BidRequest, []error) { - // This is the clean array of openrtb requests we will be returning - cleanReqs := make(map[openrtb_ext.BidderName]*openrtb.BidRequest, len(adapters)) - errList := make([]error, 0, 1) + requestsByBidder = splitBidRequest(orig, impsByBidder, aliases, usersyncs, met) + return +} - // Decode the Imp extensions once to save time. We store the results here - imp_exts := make([]map[string]openrtb.RawJSON, len(orig.Imp)) - // Loop over every impression in the request - for i := 0; i < len(orig.Imp); i++ { - // Unpack each set of extensions found in the Imp array - err := json.Unmarshal(orig.Imp[i].Ext, &imp_exts[i]) - if err != nil { - return nil, []error{fmt.Errorf("Error unpacking extensions for Imp[%d]: %s", i, err.Error())} +func splitBidRequest(req *openrtb.BidRequest, impsByBidder map[string][]openrtb.Imp, aliases map[string]string, usersyncs IdFetcher, met *pbsmetrics.Metrics) map[openrtb_ext.BidderName]*openrtb.BidRequest { + requestsByBidder := make(map[openrtb_ext.BidderName]*openrtb.BidRequest, len(impsByBidder)) + for bidder, imps := range impsByBidder { + reqCopy := *req + coreBidder := resolveBidder(bidder, aliases) + met.AdapterMetrics[coreBidder].RequestMeter.Mark(1) + if hadSync := prepareUser(&reqCopy, coreBidder, usersyncs); !hadSync && req.App == nil { + met.AdapterMetrics[coreBidder].NoCookieMeter.Mark(1) } + reqCopy.Imp = imps + requestsByBidder[openrtb_ext.BidderName(bidder)] = &reqCopy } + return requestsByBidder +} - // Loop over every adapter we want to create a clean openrtb request for. - for i := 0; i < len(adapters); i++ { - // Go deeper into Imp array - newImps := make([]openrtb.Imp, 0, len(orig.Imp)) - bn := adapters[i].String() - - // Overwrite each extension field with a cleanly built subset - // We are looping over every impression in the Imp array - for j := 0; j < len(orig.Imp); j++ { - // Don't do anything if the current bidder's field is not present. - if val, ok := imp_exts[j][bn]; ok { - // Start with a new, empty unpacked extention - newExts := make(map[string]openrtb.RawJSON, len(orig.Imp)) - // Need to do some consistency checking to verify these fields exist. Especially the adapters one. - if pb, ok := imp_exts[j]["prebid"]; ok { - newExts["prebid"] = pb - } - newExts["bidder"] = val - // Create a "clean" byte array for this Imp's extension - // Note, if the "prebid" or "" field is missing from the source, it will be missing here as well - // The adapters should test that their field is present rather than assuming it will be there if they are - // called - b, err := json.Marshal(newExts) - if err != nil { - errList = append(errList, fmt.Errorf("Error creating sanitized bidder extents for Imp[%d], bidder %s: %s", j, bn, err.Error())) - } - // Overwrite the extention field with the new cleaned version - newImps = append(newImps, orig.Imp[j]) - newImps[len(newImps)-1].Ext = b - } - } +// splitImps takes a list of Imps and returns a map of imps which have been sanitized for each bidder. +// +// For example, suppose imps has two elements. One goes to rubicon, while the other goes to appnexus and index. +// The returned map will have three keys: rubicon, appnexus, and index--each with one Imp. +// The "imp.ext" value of the appnexus Imp will only contain the "prebid" values, and "appnexus" value at the "bidder" key. +// The "imp.ext" value of the rubicon Imp will only contain the "prebid" values, and "rubicon" value at the "bidder" key. +// +// The goal here is so that Bidders only get Imps and Imp.Ext values which are intended for them. +func splitImps(imps []openrtb.Imp) (map[string][]openrtb.Imp, []error) { + impExts, err := parseImpExts(imps) + if err != nil { + return nil, []error{err} + } - // Only add a BidRequest if there exist Imp(s) for this adapter - if len(newImps) > 0 { - // Create a new BidRequest - newReq := new(openrtb.BidRequest) - // Make a shallow copy of the original request - *newReq = *orig - prepareUser(newReq, adapters[i], usersyncs) - newReq.Imp = newImps - cleanReqs[adapters[i]] = newReq - // Grab some statistics now we know we are holding an auction - met.AdapterMetrics[adapters[i]].RequestMeter.Mark(1) - if newReq.App == nil { - _, found := usersyncs.GetId(adapters[i]) - if !found { - met.AdapterMetrics[adapters[i]].NoCookieMeter.Mark(1) - } + splitImps := make(map[string][]openrtb.Imp, len(imps)) + var errList []error + for i := 0; i < len(imps); i++ { + thisImp := imps[i] + theseBidders := impExts[i] + for intendedBidder := range theseBidders { + if intendedBidder == "prebid" { + continue } + otherImps, _ := splitImps[intendedBidder] + if impForBidder, err := sanitizedImpCopy(&thisImp, theseBidders, intendedBidder); err != nil { + errList = append(errList, err) + } else { + splitImps[intendedBidder] = append(otherImps, *impForBidder) + } } } - return cleanReqs, errList + + return splitImps, nil +} + +// sanitizedImpCopy returns a copy of imp with its ext filtered so that only "prebid" and intendedBidder exist. +// It will not mutate the input imp. +// This function expects the "ext" argument to have been unmarshalled from "imp", so we don't have to repeat that work. +func sanitizedImpCopy(imp *openrtb.Imp, ext map[string]openrtb.RawJSON, intendedBidder string) (*openrtb.Imp, error) { + impCopy := *imp + newExt := make(map[string]openrtb.RawJSON, 2) + if value, ok := ext["prebid"]; ok { + newExt["prebid"] = value + } + newExt["bidder"] = ext[intendedBidder] + extBytes, err := json.Marshal(newExt) + if err != nil { + return nil, err + } + impCopy.Ext = extBytes + return &impCopy, nil } // prepareUser changes req.User so that it's ready for the given bidder. // This *will* mutate the request, but will *not* mutate any objects nested inside it. -func prepareUser(req *openrtb.BidRequest, bidder openrtb_ext.BidderName, usersyncs IdFetcher) { +// +// This function expects bidder to be a "known" bidder name. It will not work on aliases. +// It returns true if an ID sync existed, and false otherwise. +func prepareUser(req *openrtb.BidRequest, bidder openrtb_ext.BidderName, usersyncs IdFetcher) bool { if id, ok := usersyncs.GetId(bidder); ok { if req.User == nil { req.User = &openrtb.User{ @@ -115,5 +116,54 @@ func prepareUser(req *openrtb.BidRequest, bidder openrtb_ext.BidderName, usersyn clone.BuyerUID = id req.User = &clone } + return true + } + return false +} + +// resolveBidder returns the known BidderName associated with bidder, if bidder is an alias. If it's not an alias, the bidder is returned. +func resolveBidder(bidder string, aliases map[string]string) openrtb_ext.BidderName { + if coreBidder, ok := aliases[bidder]; ok { + return openrtb_ext.BidderName(coreBidder) + } + return openrtb_ext.BidderName(bidder) +} + +// parseImpExts does a partial-unmarshal of the imp[].Ext field. +// The keys in the returned map are expected to be "prebid", core BidderNames, or Aliases for this request. +func parseImpExts(imps []openrtb.Imp) ([]map[string]openrtb.RawJSON, error) { + exts := make([]map[string]openrtb.RawJSON, len(imps)) + // Loop over every impression in the request + for i := 0; i < len(imps); i++ { + // Unpack each set of extensions found in the Imp array + err := json.Unmarshal(imps[i].Ext, &exts[i]) + if err != nil { + return nil, fmt.Errorf("Error unpacking extensions for Imp[%d]: %s", i, err.Error()) + } + } + return exts, nil +} + +// parseAliases parses the aliases from the BidRequest +func parseAliases(orig *openrtb.BidRequest) (map[string]string, []error) { + var aliases map[string]string + if value, dataType, _, err := jsonparser.Get(orig.Ext, "prebid", "aliases"); dataType == jsonparser.Object && err == nil { + if err := json.Unmarshal(value, &aliases); err != nil { + return nil, []error{err} + } + } else if dataType != jsonparser.NotExist && err != jsonparser.KeyPathNotFoundError { + return nil, []error{err} + } + return aliases, nil +} + +// Quick little randomizer for a list of strings. Stuffing it in utils to keep other files clean +func randomizeList(list []openrtb_ext.BidderName) { + l := len(list) + perm := rand.Perm(l) + var j int + for i := 0; i < l; i++ { + j = perm[i] + list[i], list[j] = list[j], list[i] } } diff --git a/exchange/utils_test.go b/exchange/utils_test.go index d60103f9039..e73224bb649 100644 --- a/exchange/utils_test.go +++ b/exchange/utils_test.go @@ -2,11 +2,12 @@ package exchange import ( "encoding/json" + "testing" + "github.com/mxmCherry/openrtb" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/pbsmetrics" "github.com/rcrowley/go-metrics" - "testing" ) func TestRandomizeList(t *testing.T) { @@ -35,6 +36,7 @@ func TestCleanOpenRTBRequests(t *testing.T) { bidRequest := openrtb.BidRequest{ ID: "This Bid", Imp: make([]openrtb.Imp, 2), + Ext: openrtb.RawJSON(`{"prebid":{"aliases":{"dummy":"appnexus","dummy2":"rubicon","dummy3":"indexExchange"}}}`), } // Need extensions for all the bidders so we know to hold auctions for them. impExt := make(map[string]interface{}) @@ -54,13 +56,7 @@ func TestCleanOpenRTBRequests(t *testing.T) { } bidRequest.Imp[0].Ext = b bidRequest.Imp[1].Ext = b - - adapters := make([]openrtb_ext.BidderName, 3) - adapters[0] = openrtb_ext.BidderName("dummy") - adapters[1] = openrtb_ext.BidderName("dummy2") - adapters[2] = openrtb_ext.BidderName("dummy3") - - cleanRequests, errList := cleanOpenRTBRequests(&bidRequest, adapters, &emptyUsersync{}, pbsmetrics.NewBlankMetrics(metrics.NewRegistry(), adapters)) + cleanRequests, _, errList := cleanOpenRTBRequests(&bidRequest, &emptyUsersync{}, pbsmetrics.NewBlankMetrics(metrics.NewRegistry(), AdapterList())) if len(errList) > 0 { for _, e := range errList { @@ -68,7 +64,7 @@ func TestCleanOpenRTBRequests(t *testing.T) { } } if len(cleanRequests) != 3 { - t.Errorf("CleanOpenRTBRequests: expected 3 requests, found %d", len(cleanRequests)) + t.Fatalf("CleanOpenRTBRequests: expected 3 requests, found %d", len(cleanRequests)) } var cleanImpExt map[string]map[string]string diff --git a/openrtb_ext/bidders.go b/openrtb_ext/bidders.go index 0bef75db535..ed68b31636d 100644 --- a/openrtb_ext/bidders.go +++ b/openrtb_ext/bidders.go @@ -14,29 +14,30 @@ import ( const schemaDirectory = "static/bidder-params" +// BidderName may refer to a bidder ID, or an Alias which is defined in the request. type BidderName string const ( BidderAppnexus BidderName = "appnexus" + BidderConversant BidderName = "conversant" BidderFacebook BidderName = "audienceNetwork" BidderIndex BidderName = "indexExchange" BidderLifestreet BidderName = "lifestreet" BidderPubmatic BidderName = "pubmatic" BidderPulsepoint BidderName = "pulsepoint" BidderRubicon BidderName = "rubicon" - BidderConversant BidderName = "conversant" ) // BidderMap stores all the valid OpenRTB 2.x Bidders in the project. This map *must not* be mutated. var BidderMap = map[string]BidderName{ "appnexus": BidderAppnexus, "audienceNetwork": BidderFacebook, + "conversant": BidderConversant, "indexExchange": BidderIndex, "lifestreet": BidderLifestreet, "pubmatic": BidderPubmatic, "pulsepoint": BidderPulsepoint, "rubicon": BidderRubicon, - "conversant": BidderConversant, } func (name BidderName) MarshalJSON() ([]byte, error) { diff --git a/openrtb_ext/request.go b/openrtb_ext/request.go index 5db599cc197..c4626fbf652 100644 --- a/openrtb_ext/request.go +++ b/openrtb_ext/request.go @@ -12,8 +12,9 @@ type ExtRequest struct { // ExtRequestPrebid defines the contract for bidrequest.ext.prebid type ExtRequestPrebid struct { - StoredRequest *ExtStoredRequest `json:"storedrequest"` + Aliases map[string]string `json:"aliases"` Cache *ExtRequestPrebidCache `json:"cache"` + StoredRequest *ExtStoredRequest `json:"storedrequest"` Targeting *ExtRequestTargeting `json:"targeting"` }