From 9d72dbe2a505a8a2b635ac103b77c7be2c2bcd06 Mon Sep 17 00:00:00 2001 From: Veronika Solovei Date: Thu, 27 Jul 2023 12:57:46 -0700 Subject: [PATCH] TransmitUserFPD and TransmitPreciseGeo activity integration (#2906) --- exchange/utils.go | 81 ++++++++---- exchange/utils_test.go | 137 +++++++++++++++----- privacy/enforcement.go | 25 +++- privacy/enforcement_test.go | 140 ++++++++++++++++++++- privacy/enforcer.go | 4 - privacy/enforcer_test.go | 2 +- privacy/scrubber.go | 115 +++++++++++++++-- privacy/scrubber_test.go | 240 +++++++++++++++++++++++++++++++----- 8 files changed, 629 insertions(+), 115 deletions(-) diff --git a/exchange/utils.go b/exchange/utils.go index 8fd8f8c3ea4..e70f52490fc 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -112,15 +112,13 @@ func (rs *requestSplitter) cleanOpenRTBRequests(ctx context.Context, lmtEnforcer := extractLMT(req.BidRequest, rs.privacyConfig) // request level privacy policies - privacyEnforcement := privacy.Enforcement{ - COPPA: req.BidRequest.Regs != nil && req.BidRequest.Regs.COPPA == 1, - LMT: lmtEnforcer.ShouldEnforce(unknownBidder), - } + coppa := req.BidRequest.Regs != nil && req.BidRequest.Regs.COPPA == 1 + lmt := lmtEnforcer.ShouldEnforce(unknownBidder) privacyLabels.CCPAProvided = ccpaEnforcer.CanEnforce() privacyLabels.CCPAEnforced = ccpaEnforcer.ShouldEnforce(unknownBidder) - privacyLabels.COPPAEnforced = privacyEnforcement.COPPA - privacyLabels.LMTEnforced = lmtEnforcer.ShouldEnforce(unknownBidder) + privacyLabels.COPPAEnforced = coppa + privacyLabels.LMTEnforced = lmt var gdprEnforced bool var gdprPerms gdpr.Permissions = &gdpr.AlwaysAllow{} @@ -148,46 +146,75 @@ func (rs *requestSplitter) cleanOpenRTBRequests(ctx context.Context, // bidder level privacy policies for _, bidderRequest := range allBidderRequests { - bidRequestAllowed := true + privacyEnforcement := privacy.Enforcement{ + COPPA: coppa, + LMT: lmt, + } // fetchBids activity - fetchBidsActivityAllowed := auctionReq.Activities.Allow(privacy.ActivityFetchBids, - privacy.ScopedName{Scope: privacy.ScopeTypeBidder, Name: bidderRequest.BidderName.String()}) + scopedName := privacy.ScopedName{Scope: privacy.ScopeTypeBidder, Name: bidderRequest.BidderName.String()} + fetchBidsActivityAllowed := auctionReq.Activities.Allow(privacy.ActivityFetchBids, scopedName) if fetchBidsActivityAllowed == privacy.ActivityDeny { // skip the call to a bidder if fetchBids activity is not allowed // do not add this bidder to allowedBidderRequests continue } - // CCPA - privacyEnforcement.CCPA = ccpaEnforcer.ShouldEnforce(bidderRequest.BidderName.String()) + var auctionPermissions gdpr.AuctionPermissions + var gdprErr error - // GDPR if gdprEnforced { - auctionPermissions, err := gdprPerms.AuctionActivitiesAllowed(ctx, bidderRequest.BidderCoreName, bidderRequest.BidderName) - bidRequestAllowed = auctionPermissions.AllowBidRequest - - if err == nil { - privacyEnforcement.GDPRGeo = !auctionPermissions.PassGeo - privacyEnforcement.GDPRID = !auctionPermissions.PassID - } else { - privacyEnforcement.GDPRGeo = true - privacyEnforcement.GDPRID = true + auctionPermissions, gdprErr = gdprPerms.AuctionActivitiesAllowed(ctx, bidderRequest.BidderCoreName, bidderRequest.BidderName) + if !auctionPermissions.AllowBidRequest { + // auction request is not permitted by GDPR + // do not add this bidder to allowedBidderRequests + rs.me.RecordAdapterGDPRRequestBlocked(bidderRequest.BidderCoreName) + continue } + } - if !bidRequestAllowed { - rs.me.RecordAdapterGDPRRequestBlocked(bidderRequest.BidderCoreName) + passIDActivityAllowed := auctionReq.Activities.Allow(privacy.ActivityTransmitUserFPD, scopedName) + if passIDActivityAllowed == privacy.ActivityDeny { + privacyEnforcement.UFPD = true + } else { + // run existing policies (GDPR, CCPA, COPPA, LMT) + // potentially block passing IDs based on GDPR + if gdprEnforced { + if gdprErr == nil { + privacyEnforcement.GDPRID = !auctionPermissions.PassID + } else { + privacyEnforcement.GDPRID = true + } } + // potentially block passing IDs based on CCPA + privacyEnforcement.CCPA = ccpaEnforcer.ShouldEnforce(bidderRequest.BidderName.String()) + } + + passGeoActivityAllowed := auctionReq.Activities.Allow(privacy.ActivityTransmitPreciseGeo, scopedName) + if passGeoActivityAllowed == privacy.ActivityDeny { + privacyEnforcement.PreciseGeo = true + } else { + // run existing policies (GDPR, CCPA, COPPA, LMT) + // potentially block passing geo based on GDPR + if gdprEnforced { + if gdprErr == nil { + privacyEnforcement.GDPRGeo = !auctionPermissions.PassGeo + } else { + privacyEnforcement.GDPRGeo = true + } + } + // potentially block passing geo based on CCPA + privacyEnforcement.CCPA = ccpaEnforcer.ShouldEnforce(bidderRequest.BidderName.String()) + } if auctionReq.FirstPartyData != nil && auctionReq.FirstPartyData[bidderRequest.BidderName] != nil { applyFPD(auctionReq.FirstPartyData[bidderRequest.BidderName], bidderRequest.BidRequest) } - if bidRequestAllowed { - privacyEnforcement.Apply(bidderRequest.BidRequest) - allowedBidderRequests = append(allowedBidderRequests, bidderRequest) - } + privacyEnforcement.Apply(bidderRequest.BidRequest) + allowedBidderRequests = append(allowedBidderRequests, bidderRequest) + // GPP downgrade: always downgrade unless we can confirm GPP is supported if shouldSetLegacyPrivacy(rs.bidderInfo, string(bidderRequest.BidderCoreName)) { setLegacyGDPRFromGPP(bidderRequest.BidRequest, gpp) diff --git a/exchange/utils_test.go b/exchange/utils_test.go index a736dcfae08..aff1b3c09a1 100644 --- a/exchange/utils_test.go +++ b/exchange/utils_test.go @@ -2287,6 +2287,7 @@ func TestCleanOpenRTBRequestsWithOpenRTBDowngrade(t *testing.T) { bidReq.User.ID = "" bidReq.User.BuyerUID = "" bidReq.User.Yob = 0 + bidReq.User.Geo = &openrtb2.Geo{Lat: 123.46} downgradedRegs := *bidReq.Regs downgradedUser := *bidReq.User @@ -2601,6 +2602,7 @@ func newBidRequest(t *testing.T) *openrtb2.BidRequest { BuyerUID: "their-id", Yob: 1982, Ext: json.RawMessage(`{}`), + Geo: &openrtb2.Geo{Lat: 123.456}, }, Imp: []openrtb2.Imp{{ ID: "some-imp-id", @@ -4266,33 +4268,76 @@ func TestGetMediaTypeForBid(t *testing.T) { } } -func TemporarilyDisabledTestCleanOpenRTBRequestsActivitiesFetchBids(t *testing.T) { +func TestCleanOpenRTBRequestsActivitiesFetchBids(t *testing.T) { testCases := []struct { - name string - req *openrtb2.BidRequest - componentName string - allow bool - expectedReqNumber int + name string + req *openrtb2.BidRequest + privacyConfig *config.AccountPrivacy + componentName string + allow bool + expectedReqNumber int + expectedUserYOB int64 + expectedUserLat float64 + expectedDeviceDIDMD5 string }{ { - name: "request_with_one_bidder_allowed", - req: newBidRequest(t), - componentName: "appnexus", - allow: true, - expectedReqNumber: 1, - }, - { - name: "request_with_one_bidder_not_allowed", - req: newBidRequest(t), - componentName: "appnexus", - allow: false, - expectedReqNumber: 0, + name: "fetch_bids_request_with_one_bidder_allowed", + req: newBidRequest(t), + privacyConfig: getFetchBidsActivityConfig("appnexus", true), + expectedReqNumber: 1, + expectedUserYOB: 1982, + expectedUserLat: 123.456, + expectedDeviceDIDMD5: "some device ID hash", + }, + { + name: "fetch_bids_request_with_one_bidder_not_allowed", + req: newBidRequest(t), + privacyConfig: getFetchBidsActivityConfig("appnexus", false), + expectedReqNumber: 0, + expectedUserYOB: 1982, + expectedUserLat: 123.456, + expectedDeviceDIDMD5: "some device ID hash", + }, + { + name: "transmit_ufpd_allowed", + req: newBidRequest(t), + privacyConfig: getTransmitUFPDActivityConfig("appnexus", true), + expectedReqNumber: 1, + expectedUserYOB: 1982, + expectedUserLat: 123.456, + expectedDeviceDIDMD5: "some device ID hash", + }, + { + name: "transmit_ufpd_deny", + req: newBidRequest(t), + privacyConfig: getTransmitUFPDActivityConfig("appnexus", false), + expectedReqNumber: 1, + expectedUserYOB: 0, + expectedUserLat: 123.456, + expectedDeviceDIDMD5: "", + }, + { + name: "transmit_precise_geo_allowed", + req: newBidRequest(t), + privacyConfig: getTransmitPreciseGeoActivityConfig("appnexus", true), + expectedReqNumber: 1, + expectedUserYOB: 1982, + expectedUserLat: 123.456, + expectedDeviceDIDMD5: "some device ID hash", + }, + { + name: "transmit_precise_geo_deny", + req: newBidRequest(t), + privacyConfig: getTransmitPreciseGeoActivityConfig("appnexus", false), + expectedReqNumber: 1, + expectedUserYOB: 1982, + expectedUserLat: 123.46, + expectedDeviceDIDMD5: "some device ID hash", }, } for _, test := range testCases { - privacyConfig := getDefaultActivityConfig(test.componentName, test.allow) - activities, err := privacy.NewActivityControl(privacyConfig) + activities, err := privacy.NewActivityControl(test.privacyConfig) assert.NoError(t, err, "") auctionReq := AuctionRequest{ BidRequestWrapper: &openrtb_ext.RequestWrapper{BidRequest: test.req}, @@ -4312,25 +4357,51 @@ func TemporarilyDisabledTestCleanOpenRTBRequestsActivitiesFetchBids(t *testing.T bidderRequests, _, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, nil, gdpr.SignalNo) assert.Empty(t, errs) assert.Len(t, bidderRequests, test.expectedReqNumber) + + if test.expectedReqNumber == 1 { + assert.Equal(t, test.expectedUserYOB, bidderRequests[0].BidRequest.User.Yob) + assert.Equal(t, test.expectedUserLat, bidderRequests[0].BidRequest.User.Geo.Lat) + assert.Equal(t, test.expectedDeviceDIDMD5, bidderRequests[0].BidRequest.Device.DIDMD5) + } }) } } -func getDefaultActivityConfig(componentName string, allow bool) *config.AccountPrivacy { - return &config.AccountPrivacy{ - AllowActivities: config.AllowActivities{ - FetchBids: config.Activity{ - Default: ptrutil.ToPtr(true), - Rules: []config.ActivityRule{ - { - Allow: allow, - Condition: config.ActivityCondition{ - ComponentName: []string{componentName}, - ComponentType: []string{"bidder"}, - }, - }, +func buildDefaultActivityConfig(componentName string, allow bool) config.Activity { + return config.Activity{ + Default: ptrutil.ToPtr(true), + Rules: []config.ActivityRule{ + { + Allow: allow, + Condition: config.ActivityCondition{ + ComponentName: []string{componentName}, + ComponentType: []string{"bidder"}, }, }, }, } } + +func getFetchBidsActivityConfig(componentName string, allow bool) *config.AccountPrivacy { + return &config.AccountPrivacy{ + AllowActivities: config.AllowActivities{ + FetchBids: buildDefaultActivityConfig(componentName, allow), + }, + } +} + +func getTransmitUFPDActivityConfig(componentName string, allow bool) *config.AccountPrivacy { + return &config.AccountPrivacy{ + AllowActivities: config.AllowActivities{ + TransmitUserFPD: buildDefaultActivityConfig(componentName, allow), + }, + } +} + +func getTransmitPreciseGeoActivityConfig(componentName string, allow bool) *config.AccountPrivacy { + return &config.AccountPrivacy{ + AllowActivities: config.AllowActivities{ + TransmitPreciseGeo: buildDefaultActivityConfig(componentName, allow), + }, + } +} diff --git a/privacy/enforcement.go b/privacy/enforcement.go index 50d42e89435..1a66cfab929 100644 --- a/privacy/enforcement.go +++ b/privacy/enforcement.go @@ -9,22 +9,39 @@ type Enforcement struct { GDPRGeo bool GDPRID bool LMT bool + + // activities + UFPD bool + Eids bool + PreciseGeo bool + TID bool } // Any returns true if at least one privacy policy requires enforcement. -func (e Enforcement) Any() bool { +func (e Enforcement) AnyLegacy() bool { return e.CCPA || e.COPPA || e.GDPRGeo || e.GDPRID || e.LMT } +func (e Enforcement) AnyActivities() bool { + return e.UFPD || e.PreciseGeo || e.Eids || e.TID +} + // Apply cleans personally identifiable information from an OpenRTB bid request. func (e Enforcement) Apply(bidRequest *openrtb2.BidRequest) { e.apply(bidRequest, NewScrubber()) } func (e Enforcement) apply(bidRequest *openrtb2.BidRequest, scrubber Scrubber) { - if bidRequest != nil && e.Any() { - bidRequest.Device = scrubber.ScrubDevice(bidRequest.Device, e.getDeviceIDScrubStrategy(), e.getIPv4ScrubStrategy(), e.getIPv6ScrubStrategy(), e.getGeoScrubStrategy()) - bidRequest.User = scrubber.ScrubUser(bidRequest.User, e.getUserScrubStrategy(), e.getGeoScrubStrategy()) + if bidRequest != nil { + if e.AnyActivities() { + bidRequest = scrubber.ScrubRequest(bidRequest, e) + } + if e.AnyLegacy() && !(e.UFPD && e.PreciseGeo && e.Eids) { + bidRequest.User = scrubber.ScrubUser(bidRequest.User, e.getUserScrubStrategy(), e.getGeoScrubStrategy()) + } + if e.AnyLegacy() && !(e.UFPD && e.PreciseGeo) { + bidRequest.Device = scrubber.ScrubDevice(bidRequest.Device, e.getDeviceIDScrubStrategy(), e.getIPv4ScrubStrategy(), e.getIPv6ScrubStrategy(), e.getGeoScrubStrategy()) + } } } diff --git a/privacy/enforcement_test.go b/privacy/enforcement_test.go index a9d7ca9275a..be1ab0dca9f 100644 --- a/privacy/enforcement_test.go +++ b/privacy/enforcement_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/mock" ) -func TestAny(t *testing.T) { +func TestAnyLegacy(t *testing.T) { testCases := []struct { enforcement Enforcement expected bool @@ -50,12 +50,12 @@ func TestAny(t *testing.T) { } for _, test := range testCases { - result := test.enforcement.Any() + result := test.enforcement.AnyLegacy() assert.Equal(t, test.expected, result, test.description) } } -func TestApply(t *testing.T) { +func TestApplyGDPR(t *testing.T) { testCases := []struct { description string enforcement Enforcement @@ -216,6 +216,130 @@ func TestApply(t *testing.T) { } } +func TestApplyToggle(t *testing.T) { + testCases := []struct { + description string + enforcement Enforcement + expectedScrubRequestExecuted bool + expectedScrubUserExecuted bool + expectedScrubDeviceExecuted bool + }{ + { + description: "All enforced - only ScrubRequest execution expected", + enforcement: Enforcement{ + CCPA: true, + COPPA: true, + GDPRGeo: true, + GDPRID: true, + LMT: true, + UFPD: true, + Eids: true, + PreciseGeo: true, + TID: true, + }, + expectedScrubRequestExecuted: true, + expectedScrubUserExecuted: false, + expectedScrubDeviceExecuted: false, + }, + { + description: "All Legacy and no activities - ScrubUser and ScrubDevice execution expected", + enforcement: Enforcement{ + CCPA: true, + COPPA: true, + GDPRGeo: true, + GDPRID: true, + LMT: true, + UFPD: false, + Eids: false, + PreciseGeo: false, + TID: false, + }, + expectedScrubRequestExecuted: false, + expectedScrubUserExecuted: true, + expectedScrubDeviceExecuted: true, + }, + { + description: "Some Legacy and some activities - ScrubRequest, ScrubUser and ScrubDevice execution expected", + enforcement: Enforcement{ + CCPA: true, + COPPA: true, + GDPRGeo: true, + GDPRID: true, + LMT: true, + UFPD: true, + Eids: false, + PreciseGeo: false, + TID: false, + }, + expectedScrubRequestExecuted: true, + expectedScrubUserExecuted: true, + expectedScrubDeviceExecuted: true, + }, + { + description: "Some Legacy and some activities - ScrubRequest execution expected", + enforcement: Enforcement{ + CCPA: true, + COPPA: true, + GDPRGeo: true, + GDPRID: true, + LMT: true, + UFPD: true, + Eids: true, + PreciseGeo: true, + TID: false, + }, + expectedScrubRequestExecuted: true, + expectedScrubUserExecuted: false, + expectedScrubDeviceExecuted: false, + }, + { + description: "Some Legacy and some activities overlap - ScrubRequest and ScrubUser execution expected", + enforcement: Enforcement{ + CCPA: true, + COPPA: true, + GDPRGeo: true, + GDPRID: true, + LMT: true, + UFPD: true, + Eids: false, + PreciseGeo: true, + TID: false, + }, + expectedScrubRequestExecuted: true, + expectedScrubUserExecuted: true, + expectedScrubDeviceExecuted: false, + }, + } + + for _, test := range testCases { + t.Run(test.description, func(t *testing.T) { + req := &openrtb2.BidRequest{ + Device: &openrtb2.Device{}, + User: &openrtb2.User{}, + } + replacedDevice := &openrtb2.Device{} + replacedUser := &openrtb2.User{} + + m := &mockScrubber{} + + if test.expectedScrubRequestExecuted { + m.On("ScrubRequest", req, test.enforcement).Return(req).Once() + } + if test.expectedScrubUserExecuted { + m.On("ScrubUser", req.User, ScrubStrategyUserIDAndDemographic, ScrubStrategyGeoFull).Return(replacedUser).Once() + } + if test.expectedScrubDeviceExecuted { + m.On("ScrubDevice", req.Device, ScrubStrategyDeviceIDAll, ScrubStrategyIPV4Lowest8, ScrubStrategyIPV6Lowest32, ScrubStrategyGeoFull).Return(replacedDevice).Once() + } + + test.enforcement.apply(req, m) + + m.AssertExpectations(t) + + }) + } +} + func TestApplyNoneApplicable(t *testing.T) { req := &openrtb2.BidRequest{} @@ -227,6 +351,11 @@ func TestApplyNoneApplicable(t *testing.T) { GDPRGeo: false, GDPRID: false, LMT: false, + + UFPD: false, + PreciseGeo: false, + TID: false, + Eids: false, } enforcement.apply(req, m) @@ -248,6 +377,11 @@ type mockScrubber struct { mock.Mock } +func (m *mockScrubber) ScrubRequest(bidRequest *openrtb2.BidRequest, enforcement Enforcement) *openrtb2.BidRequest { + args := m.Called(bidRequest, enforcement) + return args.Get(0).(*openrtb2.BidRequest) +} + func (m *mockScrubber) ScrubDevice(device *openrtb2.Device, id ScrubStrategyDeviceID, ipv4 ScrubStrategyIPV4, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb2.Device { args := m.Called(device, id, ipv4, ipv6, geo) return args.Get(0).(*openrtb2.Device) diff --git a/privacy/enforcer.go b/privacy/enforcer.go index d63cd8de31f..a8685e29276 100644 --- a/privacy/enforcer.go +++ b/privacy/enforcer.go @@ -3,7 +3,6 @@ package privacy import ( "fmt" "github.com/prebid/prebid-server/config" - "github.com/prebid/prebid-server/errortypes" "strings" ) @@ -33,9 +32,6 @@ func NewActivityControl(privacyConf *config.AccountPrivacy) (ActivityControl, er if privacyConf == nil { return ac, err - } else { - //temporarily disable Activities if they are specified at the account level - return ac, &errortypes.Warning{Message: "account.Privacy has no effect as the feature is under development."} } plans := make(map[Activity]ActivityPlan) diff --git a/privacy/enforcer_test.go b/privacy/enforcer_test.go index e87a9eb2bff..2b9715f87f8 100644 --- a/privacy/enforcer_test.go +++ b/privacy/enforcer_test.go @@ -8,7 +8,7 @@ import ( "testing" ) -func TemporarilyDisabledTestNewActivityControl(t *testing.T) { +func TestNewActivityControl(t *testing.T) { testCases := []struct { name string diff --git a/privacy/scrubber.go b/privacy/scrubber.go index 428193bcda1..97419128952 100644 --- a/privacy/scrubber.go +++ b/privacy/scrubber.go @@ -2,6 +2,7 @@ package privacy import ( "encoding/json" + "github.com/prebid/prebid-server/util/ptrutil" "strings" "github.com/prebid/openrtb/v19/openrtb2" @@ -70,6 +71,7 @@ const ( // Scrubber removes PII from parts of an OpenRTB request. type Scrubber interface { + ScrubRequest(bidRequest *openrtb2.BidRequest, enforcement Enforcement) *openrtb2.BidRequest ScrubDevice(device *openrtb2.Device, id ScrubStrategyDeviceID, ipv4 ScrubStrategyIPV4, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb2.Device ScrubUser(user *openrtb2.User, strategy ScrubStrategyUser, geo ScrubStrategyGeo) *openrtb2.User } @@ -81,6 +83,99 @@ func NewScrubber() Scrubber { return scrubber{} } +func (scrubber) ScrubRequest(bidRequest *openrtb2.BidRequest, enforcement Enforcement) *openrtb2.BidRequest { + var userExtParsed map[string]json.RawMessage + userExtModified := false + + var userCopy *openrtb2.User + userCopy = ptrutil.Clone(bidRequest.User) + + var deviceCopy *openrtb2.Device + deviceCopy = ptrutil.Clone(bidRequest.Device) + + if userCopy != nil && (enforcement.UFPD || enforcement.Eids) { + if len(userCopy.Ext) != 0 { + json.Unmarshal(userCopy.Ext, &userExtParsed) + } + } + + if enforcement.UFPD { + // transmitUfpd covers user.ext.data, user.data, user.id, user.buyeruid, user.yob, user.gender, user.keywords, user.kwarray + // and device.{ifa, macsha1, macmd5, dpidsha1, dpidmd5, didsha1, didmd5} + if deviceCopy != nil { + deviceCopy.DIDMD5 = "" + deviceCopy.DIDSHA1 = "" + deviceCopy.DPIDMD5 = "" + deviceCopy.DPIDSHA1 = "" + deviceCopy.IFA = "" + deviceCopy.MACMD5 = "" + deviceCopy.MACSHA1 = "" + } + if userCopy != nil { + userCopy.Data = nil + userCopy.ID = "" + userCopy.BuyerUID = "" + userCopy.Yob = 0 + userCopy.Gender = "" + userCopy.Keywords = "" + userCopy.KwArray = nil + + _, hasField := userExtParsed["data"] + if hasField { + delete(userExtParsed, "data") + userExtModified = true + } + } + } + if enforcement.Eids { + //transmitEids covers user.eids and user.ext.eids + if userCopy != nil { + userCopy.EIDs = nil + _, hasField := userExtParsed["eids"] + if hasField { + delete(userExtParsed, "eids") + userExtModified = true + } + } + } + + if userExtModified { + userExt, _ := json.Marshal(userExtParsed) + userCopy.Ext = userExt + } + + if enforcement.TID { + //remove source.tid and imp.ext.tid + if bidRequest.Source != nil { + bidRequest.Source.TID = "" + } + for ind, imp := range bidRequest.Imp { + impExt := scrubExtIDs(imp.Ext, "tid") + bidRequest.Imp[ind].Ext = impExt + } + } + + if enforcement.PreciseGeo { + //round user's geographic location by rounding off IP address and lat/lng data. + //this applies to both device.geo and user.geo + if userCopy != nil && userCopy.Geo != nil { + userCopy.Geo = scrubGeoPrecision(userCopy.Geo) + } + + if deviceCopy != nil { + if deviceCopy.Geo != nil { + deviceCopy.Geo = scrubGeoPrecision(deviceCopy.Geo) + } + deviceCopy.IP = scrubIPV4Lowest8(deviceCopy.IP) + deviceCopy.IPv6 = scrubIPV6Lowest32Bits(deviceCopy.IPv6) + } + } + + bidRequest.Device = deviceCopy + bidRequest.User = userCopy + return bidRequest +} + func (scrubber) ScrubDevice(device *openrtb2.Device, id ScrubStrategyDeviceID, ipv4 ScrubStrategyIPV4, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb2.Device { if device == nil { return nil @@ -131,7 +226,7 @@ func (scrubber) ScrubUser(user *openrtb2.User, strategy ScrubStrategyUser, geo S if strategy == ScrubStrategyUserIDAndDemographic { userCopy.BuyerUID = "" userCopy.ID = "" - userCopy.Ext = scrubUserExtIDs(userCopy.Ext) + userCopy.Ext = scrubExtIDs(userCopy.Ext, "eids") userCopy.Yob = 0 userCopy.Gender = "" } @@ -205,25 +300,25 @@ func scrubGeoPrecision(geo *openrtb2.Geo) *openrtb2.Geo { return &geoCopy } -func scrubUserExtIDs(userExt json.RawMessage) json.RawMessage { - if len(userExt) == 0 { - return userExt +func scrubExtIDs(ext json.RawMessage, fieldName string) json.RawMessage { + if len(ext) == 0 { + return ext } var userExtParsed map[string]json.RawMessage - err := json.Unmarshal(userExt, &userExtParsed) + err := json.Unmarshal(ext, &userExtParsed) if err != nil { - return userExt + return ext } - _, hasEids := userExtParsed["eids"] - if hasEids { - delete(userExtParsed, "eids") + _, hasField := userExtParsed[fieldName] + if hasField { + delete(userExtParsed, fieldName) result, err := json.Marshal(userExtParsed) if err == nil { return result } } - return userExt + return ext } diff --git a/privacy/scrubber_test.go b/privacy/scrubber_test.go index 7b4afac247e..0691b2c7a2c 100644 --- a/privacy/scrubber_test.go +++ b/privacy/scrubber_test.go @@ -9,24 +9,7 @@ import ( ) func TestScrubDevice(t *testing.T) { - device := &openrtb2.Device{ - DIDMD5: "anyDIDMD5", - DIDSHA1: "anyDIDSHA1", - DPIDMD5: "anyDPIDMD5", - DPIDSHA1: "anyDPIDSHA1", - MACSHA1: "anyMACSHA1", - MACMD5: "anyMACMD5", - IFA: "anyIFA", - IP: "1.2.3.4", - IPv6: "2001:0db8:0000:0000:0000:ff00:0042:8329", - Geo: &openrtb2.Geo{ - Lat: 123.456, - Lon: 678.89, - Metro: "some metro", - City: "some city", - ZIP: "some zip", - }, - } + device := getTestDevice() testCases := []struct { description string @@ -197,20 +180,7 @@ func TestScrubDeviceNil(t *testing.T) { } func TestScrubUser(t *testing.T) { - user := &openrtb2.User{ - ID: "anyID", - BuyerUID: "anyBuyerUID", - Yob: 42, - Gender: "anyGender", - Ext: json.RawMessage(`{}`), - Geo: &openrtb2.Geo{ - Lat: 123.456, - Lon: 678.89, - Metro: "some metro", - City: "some city", - ZIP: "some zip", - }, - } + user := getTestUser() testCases := []struct { description string @@ -333,6 +303,171 @@ func TestScrubUserNil(t *testing.T) { assert.Nil(t, result) } +func TestScrubRequest(t *testing.T) { + + imps := []openrtb2.Imp{ + {ID: "testId", Ext: json.RawMessage(`{"test": 1, "tid": 2}`)}, + } + source := &openrtb2.Source{ + TID: "testTid", + } + device := getTestDevice() + user := getTestUser() + user.Ext = json.RawMessage(`{"data": 1, "eids": 2}`) + user.EIDs = []openrtb2.EID{{Source: "test"}} + + testCases := []struct { + description string + enforcement Enforcement + userExtPresent bool + expected *openrtb2.BidRequest + }{ + { + description: "enforce transmitUFPD with user.ext", + enforcement: Enforcement{UFPD: true}, + userExtPresent: true, + expected: &openrtb2.BidRequest{ + Imp: imps, + Source: source, + User: &openrtb2.User{ + EIDs: []openrtb2.EID{{Source: "test"}}, + Geo: user.Geo, + Ext: json.RawMessage(`{"eids":2}`), + }, + Device: &openrtb2.Device{ + IP: "1.2.3.4", + IPv6: "2001:0db8:0000:0000:0000:ff00:0042:8329", + Geo: device.Geo, + }, + }, + }, + { + description: "enforce transmitUFPD without user.ext", + enforcement: Enforcement{UFPD: true}, + userExtPresent: false, + expected: &openrtb2.BidRequest{ + Imp: imps, + Source: source, + User: &openrtb2.User{ + EIDs: []openrtb2.EID{{Source: "test"}}, + Geo: user.Geo, + }, + Device: &openrtb2.Device{ + IP: "1.2.3.4", + IPv6: "2001:0db8:0000:0000:0000:ff00:0042:8329", + Geo: device.Geo, + }, + }, + }, + { + description: "enforce transmitEids", + enforcement: Enforcement{Eids: true}, + userExtPresent: true, + expected: &openrtb2.BidRequest{ + Imp: imps, + Source: source, + Device: device, + User: &openrtb2.User{ + ID: "anyID", + BuyerUID: "anyBuyerUID", + Yob: 42, + Gender: "anyGender", + Geo: user.Geo, + EIDs: nil, + Ext: json.RawMessage(`{"data":1}`), + }, + }, + }, + { + description: "enforce transmitTid", + enforcement: Enforcement{TID: true}, + userExtPresent: true, + expected: &openrtb2.BidRequest{ + Imp: []openrtb2.Imp{ + {ID: "testId", Ext: json.RawMessage(`{"test":1}`)}, + }, + Source: &openrtb2.Source{ + TID: "", + }, + Device: device, + User: &openrtb2.User{ + ID: "anyID", + BuyerUID: "anyBuyerUID", + Yob: 42, + Gender: "anyGender", + Geo: user.Geo, + EIDs: []openrtb2.EID{{Source: "test"}}, + Ext: json.RawMessage(`{"data": 1, "eids": 2}`), + }, + }, + }, + { + description: "enforce precise Geo", + enforcement: Enforcement{PreciseGeo: true}, + userExtPresent: true, + expected: &openrtb2.BidRequest{ + Imp: imps, + Source: source, + User: &openrtb2.User{ + ID: "anyID", + BuyerUID: "anyBuyerUID", + Yob: 42, + Gender: "anyGender", + Geo: &openrtb2.Geo{ + Lat: 123.46, Lon: 678.89, + Metro: "some metro", + City: "some city", + ZIP: "some zip", + }, + EIDs: []openrtb2.EID{{Source: "test"}}, + Ext: json.RawMessage(`{"data": 1, "eids": 2}`), + }, + Device: &openrtb2.Device{ + IFA: "anyIFA", + DIDSHA1: "anyDIDSHA1", + DIDMD5: "anyDIDMD5", + DPIDSHA1: "anyDPIDSHA1", + DPIDMD5: "anyDPIDMD5", + MACSHA1: "anyMACSHA1", + MACMD5: "anyMACMD5", + IP: "1.2.3.0", + IPv6: "2001:0db8:0000:0000:0000:ff00:0:0", + Geo: &openrtb2.Geo{ + Lat: 123.46, Lon: 678.89, + Metro: "some metro", + City: "some city", + ZIP: "some zip", + }, + }, + }, + }, + } + + for _, test := range testCases { + t.Run(test.description, func(t *testing.T) { + bidRequest := &openrtb2.BidRequest{ + Imp: []openrtb2.Imp{ + {ID: "testId", Ext: json.RawMessage(`{"test": 1, "tid": 2}`)}, + }, + Source: &openrtb2.Source{ + TID: "testTid", + }, + User: getTestUser(), + Device: getTestDevice(), + } + if test.userExtPresent { + bidRequest.User.Ext = json.RawMessage(`{"data": 1, "eids": 2}`) + } else { + bidRequest.User.Ext = nil + } + bidRequest.User.EIDs = []openrtb2.EID{{Source: "test"}} + + result := NewScrubber().ScrubRequest(bidRequest, test.enforcement) + assert.Equal(t, test.expected, result, test.description) + }) + } +} + func TestScrubIPV4(t *testing.T) { testCases := []struct { IP string @@ -572,7 +707,46 @@ func TestScrubUserExtIDs(t *testing.T) { } for _, test := range testCases { - result := scrubUserExtIDs(test.userExt) + result := scrubExtIDs(test.userExt, "eids") assert.Equal(t, test.expected, result, test.description) } } + +func getTestUser() *openrtb2.User { + return &openrtb2.User{ + ID: "anyID", + BuyerUID: "anyBuyerUID", + Yob: 42, + Gender: "anyGender", + Ext: json.RawMessage(`{}`), + Geo: &openrtb2.Geo{ + Lat: 123.456, + Lon: 678.89, + Metro: "some metro", + City: "some city", + ZIP: "some zip", + }, + } +} + +func getTestDevice() *openrtb2.Device { + return &openrtb2.Device{ + DIDMD5: "anyDIDMD5", + DIDSHA1: "anyDIDSHA1", + DPIDMD5: "anyDPIDMD5", + DPIDSHA1: "anyDPIDSHA1", + MACSHA1: "anyMACSHA1", + MACMD5: "anyMACMD5", + IFA: "anyIFA", + IP: "1.2.3.4", + IPv6: "2001:0db8:0000:0000:0000:ff00:0042:8329", + Geo: &openrtb2.Geo{ + Lat: 123.456, + Lon: 678.89, + Metro: "some metro", + City: "some city", + ZIP: "some zip", + }, + } + +}