From 2023f4c64a01a01c967eadd24974823e5e34fcc8 Mon Sep 17 00:00:00 2001 From: Veronika Solovei Date: Mon, 21 Aug 2023 10:40:13 -0700 Subject: [PATCH] TID activity (#3012) * Passed activity control to video endpoint * TID activity * Minor refactoring * Added exchange end-to-end tests --- endpoints/openrtb2/video_auction.go | 11 +++ endpoints/openrtb2/video_auction_test.go | 48 ++++++++++++ exchange/exchange_test.go | 8 ++ exchange/exchangetest/activity-tid-off.json | 85 +++++++++++++++++++++ exchange/exchangetest/activity-tid-on.json | 82 ++++++++++++++++++++ exchange/utils.go | 2 + exchange/utils_test.go | 48 ++++++++++-- privacy/scrubber.go | 5 +- 8 files changed, 282 insertions(+), 7 deletions(-) create mode 100644 exchange/exchangetest/activity-tid-off.json create mode 100644 exchange/exchangetest/activity-tid-on.json diff --git a/endpoints/openrtb2/video_auction.go b/endpoints/openrtb2/video_auction.go index 49e4585d173..8fc2f844c71 100644 --- a/endpoints/openrtb2/video_auction.go +++ b/endpoints/openrtb2/video_auction.go @@ -21,6 +21,7 @@ import ( "github.com/prebid/prebid-server/hooks" "github.com/prebid/prebid-server/hooks/hookexecution" "github.com/prebid/prebid-server/ortb" + "github.com/prebid/prebid-server/privacy" jsonpatch "gopkg.in/evanphx/json-patch.v4" accountService "github.com/prebid/prebid-server/account" @@ -302,6 +303,15 @@ func (deps *endpointDeps) VideoAuctionEndpoint(w http.ResponseWriter, r *http.Re return } + activities, activitiesErr := privacy.NewActivityControl(account.Privacy) + if activitiesErr != nil { + errL = append(errL, activitiesErr) + if errortypes.ContainsFatalError(errL) { + handleError(&labels, w, errL, &vo, &debugLog) + return + } + } + secGPC := r.Header.Get("Sec-GPC") auctionRequest := &exchange.AuctionRequest{ BidRequestWrapper: bidReqWrapper, @@ -314,6 +324,7 @@ func (deps *endpointDeps) VideoAuctionEndpoint(w http.ResponseWriter, r *http.Re PubID: labels.PubID, HookExecutor: hookexecution.EmptyHookExecutor{}, TmaxAdjustments: deps.tmaxAdjustments, + Activities: activities, } auctionResponse, err := deps.ex.HoldAuction(ctx, auctionRequest, &debugLog) diff --git a/endpoints/openrtb2/video_auction_test.go b/endpoints/openrtb2/video_auction_test.go index 1f28efb8a6f..2d74bbe5c12 100644 --- a/endpoints/openrtb2/video_auction_test.go +++ b/endpoints/openrtb2/video_auction_test.go @@ -65,6 +65,20 @@ func TestVideoEndpointImpressionsNumber(t *testing.T) { assert.Equal(t, resp.AdPods[0].Targeting[0].HbDeal, "ABC_123", "If DealID exists in bid response, hb_deal targeting needs to be added to resp") } +func TestVideoEndpointInvalidPrivacyConfig(t *testing.T) { + ex := &mockExchangeVideo{} + reqBody := readVideoTestFile(t, "sample-requests/video/video_valid_sample.json") + req := httptest.NewRequest("POST", "/openrtb2/video", strings.NewReader(reqBody)) + recorder := httptest.NewRecorder() + + deps := mockDepsInvalidPrivacy(t, ex) + deps.VideoAuctionEndpoint(recorder, req, nil) + + respBytes := recorder.Body.Bytes() + expectedErrorMessage := "Critical error while running the video endpoint: unable to parse condition: bidderA.BidderB.bidderC" + assert.Equal(t, expectedErrorMessage, string(respBytes), "error message is incorrect") +} + func TestVideoEndpointImpressionsDuration(t *testing.T) { ex := &mockExchangeVideo{} reqBody := readVideoTestFile(t, "sample-requests/video/video_valid_sample_different_durations.json") @@ -1275,6 +1289,40 @@ func mockDeps(t *testing.T, ex *mockExchangeVideo) *endpointDeps { } } +func mockDepsInvalidPrivacy(t *testing.T, ex *mockExchangeVideo) *endpointDeps { + return &endpointDeps{ + fakeUUIDGenerator{}, + ex, + mockBidderParamValidator{}, + &mockVideoStoredReqFetcher{}, + &mockVideoStoredReqFetcher{}, + &mockAccountFetcher{data: mockVideoAccountData}, + &config.Configuration{MaxRequestSize: maxSize, + AccountDefaults: config.Account{ + Privacy: &config.AccountPrivacy{ + AllowActivities: config.AllowActivities{ + TransmitPreciseGeo: config.Activity{Rules: []config.ActivityRule{ + {Condition: config.ActivityCondition{ComponentName: []string{"bidderA.BidderB.bidderC"}}}, + }}, + }, + }, + }, + }, + &metricsConfig.NilMetricsEngine{}, + analyticsConf.NewPBSAnalytics(&config.Analytics{}), + map[string]string{}, + false, + []byte{}, + openrtb_ext.BuildBidderMap(), + ex.cache, + regexp.MustCompile(`[<>]`), + hardcodedResponseIPValidator{response: true}, + empty_fetcher.EmptyFetcher{}, + hooks.EmptyPlanBuilder{}, + nil, + } +} + func mockDepsAppendBidderNames(t *testing.T, ex *mockExchangeAppendBidderNames) *endpointDeps { deps := &endpointDeps{ fakeUUIDGenerator{}, diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go index 52813897142..919f8dceb54 100644 --- a/exchange/exchange_test.go +++ b/exchange/exchange_test.go @@ -37,6 +37,7 @@ import ( metricsConfig "github.com/prebid/prebid-server/metrics/config" "github.com/prebid/prebid-server/openrtb_ext" pbc "github.com/prebid/prebid-server/prebid_cache_client" + "github.com/prebid/prebid-server/privacy" "github.com/prebid/prebid-server/stored_requests" "github.com/prebid/prebid-server/stored_requests/backends/file_fetcher" "github.com/prebid/prebid-server/usersync" @@ -2351,6 +2352,11 @@ func runSpec(t *testing.T, filename string, spec *exchangeSpec) { impExtInfoMap[impID] = ImpExtInfo{} } + activityControl, err := privacy.NewActivityControl(spec.AccountPrivacy) + if err != nil { + t.Errorf("%s: Exchange returned an unexpected error. Got %s", filename, err.Error()) + } + auctionRequest := &AuctionRequest{ BidRequestWrapper: &openrtb_ext.RequestWrapper{BidRequest: &spec.IncomingRequest.OrtbRequest}, Account: config.Account{ @@ -2366,6 +2372,7 @@ func runSpec(t *testing.T, filename string, spec *exchangeSpec) { ImpExtInfoMap: impExtInfoMap, HookExecutor: &hookexecution.EmptyHookExecutor{}, TCF2Config: gdpr.NewTCF2Config(privacyConfig.GDPR.TCF2, config.AccountGDPR{}), + Activities: activityControl, } if spec.MultiBid != nil { @@ -5187,6 +5194,7 @@ type exchangeSpec struct { FledgeEnabled bool `json:"fledge_enabled,omitempty"` MultiBid *multiBidSpec `json:"multiBid,omitempty"` Server exchangeServer `json:"server,omitempty"` + AccountPrivacy *config.AccountPrivacy `json:"accountPrivacy,omitempty"` } type multiBidSpec struct { diff --git a/exchange/exchangetest/activity-tid-off.json b/exchange/exchangetest/activity-tid-off.json new file mode 100644 index 00000000000..174f09659ec --- /dev/null +++ b/exchange/exchangetest/activity-tid-off.json @@ -0,0 +1,85 @@ +{ + "accountPrivacy": { + "allowactivities": { + "transmitTid": { + "default": true, + "rules": [ + { + "allow": true, + "condition": { + "componentName": ["appnexus"], + "componentType":["bidder"] + } + } + ] + } + } + }, + "incomingRequest": { + "ortbRequest": { + "id": "some-request-id", + "site": { + "page": "test.somepage.com" + }, + "imp": [ + { + "id": "my-imp-id", + "video": { + "mimes": [ + "video/mp4" + ] + }, + "ext": { + "tid": "extTestTID", + "prebid": { + "bidder": { + "appnexus": { + "placementId": 1 + } + } + } + } + } + ], + "source": { + "tid": "sourceTestTID" + } + } + }, + "outgoingRequests": { + "appnexus": { + "expectRequest": { + "ortbRequest": { + "id": "some-request-id", + "site": { + "page": "test.somepage.com" + }, + "imp": [ + { + "id": "my-imp-id", + "video": { + "mimes": [ + "video/mp4" + ] + }, + "ext": { + "tid": "extTestTID", + "bidder": { + "placementId": 1 + } + } + } + ], + "source": { + "tid": "sourceTestTID" + } + } + }, + "mockResponse": { + "errors": [ + "appnexus-error" + ] + } + } + } +} \ No newline at end of file diff --git a/exchange/exchangetest/activity-tid-on.json b/exchange/exchangetest/activity-tid-on.json new file mode 100644 index 00000000000..98da945bff2 --- /dev/null +++ b/exchange/exchangetest/activity-tid-on.json @@ -0,0 +1,82 @@ +{ + "accountPrivacy": { + "allowactivities": { + "transmitTid": { + "default": true, + "rules": [ + { + "allow": false, + "condition": { + "componentName": ["appnexus"], + "componentType":["bidder"] + } + } + ] + } + } + }, + "incomingRequest": { + "ortbRequest": { + "id": "some-request-id", + "site": { + "page": "test.somepage.com" + }, + "imp": [ + { + "id": "my-imp-id", + "video": { + "mimes": [ + "video/mp4" + ] + }, + "ext": { + "tid": "extTestTID", + "prebid": { + "bidder": { + "appnexus": { + "placementId": 1 + } + } + } + } + } + ], + "source": { + "tid": "sourceTestTID" + } + } + }, + "outgoingRequests": { + "appnexus": { + "expectRequest": { + "ortbRequest": { + "id": "some-request-id", + "site": { + "page": "test.somepage.com" + }, + "imp": [ + { + "id": "my-imp-id", + "video": { + "mimes": [ + "video/mp4" + ] + }, + "ext": { + "bidder": { + "placementId": 1 + } + } + } + ], + "source": {} + } + }, + "mockResponse": { + "errors": [ + "appnexus-error" + ] + } + } + } +} \ No newline at end of file diff --git a/exchange/utils.go b/exchange/utils.go index 17bba334e89..907a039ae8a 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -212,6 +212,8 @@ func (rs *requestSplitter) cleanOpenRTBRequests(ctx context.Context, applyFPD(auctionReq.FirstPartyData[bidderRequest.BidderName], bidderRequest.BidRequest) } + privacyEnforcement.TID = !auctionReq.Activities.Allow(privacy.ActivityTransmitTids, scopedName) + privacyEnforcement.Apply(bidderRequest.BidRequest) allowedBidderRequests = append(allowedBidderRequests, bidderRequest) diff --git a/exchange/utils_test.go b/exchange/utils_test.go index aff1b3c09a1..63ca54c2a2e 100644 --- a/exchange/utils_test.go +++ b/exchange/utils_test.go @@ -2551,7 +2551,7 @@ func newAdapterAliasBidRequest(t *testing.T) *openrtb2.BidRequest { Language: "EN", }, Source: &openrtb2.Source{ - TID: "61018dc9-fa61-4c41-b7dc-f90b9ae80e87", + TID: "testTID", }, User: &openrtb2.User{ ID: "our-id", @@ -2595,7 +2595,7 @@ func newBidRequest(t *testing.T) *openrtb2.BidRequest { Language: "EN", }, Source: &openrtb2.Source{ - TID: "61018dc9-fa61-4c41-b7dc-f90b9ae80e87", + TID: "testTID", }, User: &openrtb2.User{ ID: "our-id", @@ -2615,7 +2615,7 @@ func newBidRequest(t *testing.T) *openrtb2.BidRequest { H: 600, }}, }, - Ext: json.RawMessage(`{"prebid":{"bidder":{"appnexus": {"placementId": 1}}}}`), + Ext: json.RawMessage(`{"prebid":{"tid":"1234567", "bidder":{"appnexus": {"placementId": 1}}}}`), }}, } } @@ -2637,7 +2637,7 @@ func newBidRequestWithBidderParams(t *testing.T) *openrtb2.BidRequest { Language: "EN", }, Source: &openrtb2.Source{ - TID: "61018dc9-fa61-4c41-b7dc-f90b9ae80e87", + TID: "testTID", }, User: &openrtb2.User{ ID: "our-id", @@ -3065,7 +3065,7 @@ func TestCleanOpenRTBRequestsSChainMultipleBidders(t *testing.T) { req := &openrtb2.BidRequest{ Site: &openrtb2.Site{}, Source: &openrtb2.Source{ - TID: "61018dc9-fa61-4c41-b7dc-f90b9ae80e87", + TID: "testTID", }, Imp: []openrtb2.Imp{{ Ext: json.RawMessage(`{"prebid":{"bidder":{"appnexus": {"placementId": 1}, "axonix": { "supplyId": "123"}}}}`), @@ -4268,7 +4268,7 @@ func TestGetMediaTypeForBid(t *testing.T) { } } -func TestCleanOpenRTBRequestsActivitiesFetchBids(t *testing.T) { +func TestCleanOpenRTBRequestsActivities(t *testing.T) { testCases := []struct { name string req *openrtb2.BidRequest @@ -4279,6 +4279,7 @@ func TestCleanOpenRTBRequestsActivitiesFetchBids(t *testing.T) { expectedUserYOB int64 expectedUserLat float64 expectedDeviceDIDMD5 string + expectedSourceTID string }{ { name: "fetch_bids_request_with_one_bidder_allowed", @@ -4288,6 +4289,7 @@ func TestCleanOpenRTBRequestsActivitiesFetchBids(t *testing.T) { expectedUserYOB: 1982, expectedUserLat: 123.456, expectedDeviceDIDMD5: "some device ID hash", + expectedSourceTID: "testTID", }, { name: "fetch_bids_request_with_one_bidder_not_allowed", @@ -4297,6 +4299,7 @@ func TestCleanOpenRTBRequestsActivitiesFetchBids(t *testing.T) { expectedUserYOB: 1982, expectedUserLat: 123.456, expectedDeviceDIDMD5: "some device ID hash", + expectedSourceTID: "testTID", }, { name: "transmit_ufpd_allowed", @@ -4306,6 +4309,7 @@ func TestCleanOpenRTBRequestsActivitiesFetchBids(t *testing.T) { expectedUserYOB: 1982, expectedUserLat: 123.456, expectedDeviceDIDMD5: "some device ID hash", + expectedSourceTID: "testTID", }, { name: "transmit_ufpd_deny", @@ -4315,6 +4319,7 @@ func TestCleanOpenRTBRequestsActivitiesFetchBids(t *testing.T) { expectedUserYOB: 0, expectedUserLat: 123.456, expectedDeviceDIDMD5: "", + expectedSourceTID: "testTID", }, { name: "transmit_precise_geo_allowed", @@ -4324,6 +4329,7 @@ func TestCleanOpenRTBRequestsActivitiesFetchBids(t *testing.T) { expectedUserYOB: 1982, expectedUserLat: 123.456, expectedDeviceDIDMD5: "some device ID hash", + expectedSourceTID: "testTID", }, { name: "transmit_precise_geo_deny", @@ -4333,6 +4339,27 @@ func TestCleanOpenRTBRequestsActivitiesFetchBids(t *testing.T) { expectedUserYOB: 1982, expectedUserLat: 123.46, expectedDeviceDIDMD5: "some device ID hash", + expectedSourceTID: "testTID", + }, + { + name: "transmit_tid_allowed", + req: newBidRequest(t), + privacyConfig: getTransmitTIDActivityConfig("appnexus", true), + expectedReqNumber: 1, + expectedUserYOB: 1982, + expectedUserLat: 123.456, + expectedDeviceDIDMD5: "some device ID hash", + expectedSourceTID: "testTID", + }, + { + name: "transmit_tid_deny", + req: newBidRequest(t), + privacyConfig: getTransmitTIDActivityConfig("appnexus", false), + expectedReqNumber: 1, + expectedUserYOB: 1982, + expectedUserLat: 123.456, + expectedDeviceDIDMD5: "some device ID hash", + expectedSourceTID: "", }, } @@ -4362,6 +4389,7 @@ func TestCleanOpenRTBRequestsActivitiesFetchBids(t *testing.T) { assert.Equal(t, test.expectedUserYOB, bidderRequests[0].BidRequest.User.Yob) assert.Equal(t, test.expectedUserLat, bidderRequests[0].BidRequest.User.Geo.Lat) assert.Equal(t, test.expectedDeviceDIDMD5, bidderRequests[0].BidRequest.Device.DIDMD5) + assert.Equal(t, test.expectedSourceTID, bidderRequests[0].BidRequest.Source.TID) } }) } @@ -4405,3 +4433,11 @@ func getTransmitPreciseGeoActivityConfig(componentName string, allow bool) *conf }, } } + +func getTransmitTIDActivityConfig(componentName string, allow bool) *config.AccountPrivacy { + return &config.AccountPrivacy{ + AllowActivities: config.AllowActivities{ + TransmitTids: buildDefaultActivityConfig(componentName, allow), + }, + } +} diff --git a/privacy/scrubber.go b/privacy/scrubber.go index 97419128952..f2fb790dc12 100644 --- a/privacy/scrubber.go +++ b/privacy/scrubber.go @@ -147,7 +147,10 @@ func (scrubber) ScrubRequest(bidRequest *openrtb2.BidRequest, enforcement Enforc if enforcement.TID { //remove source.tid and imp.ext.tid if bidRequest.Source != nil { - bidRequest.Source.TID = "" + var sourceCopy *openrtb2.Source + sourceCopy = ptrutil.Clone(bidRequest.Source) + sourceCopy.TID = "" + bidRequest.Source = sourceCopy } for ind, imp := range bidRequest.Imp { impExt := scrubExtIDs(imp.Ext, "tid")