TransmitUserFPD and TransmitPreciseGeo activity integration

exchange/utils.go

@@ -112,15 +112,13 @@ func (rs *requestSplitter) cleanOpenRTBRequests(ctx context.Context,
 	lmtEnforcer := extractLMT(req.BidRequest, rs.privacyConfig)
 
 	// request level privacy policies
-	privacyEnforcement := privacy.Enforcement{
-		COPPA: req.BidRequest.Regs != nil && req.BidRequest.Regs.COPPA == 1,
-		LMT:   lmtEnforcer.ShouldEnforce(unknownBidder),
-	}
+	coppa := req.BidRequest.Regs != nil && req.BidRequest.Regs.COPPA == 1
+	lmt := lmtEnforcer.ShouldEnforce(unknownBidder)
 
 	privacyLabels.CCPAProvided = ccpaEnforcer.CanEnforce()
 	privacyLabels.CCPAEnforced = ccpaEnforcer.ShouldEnforce(unknownBidder)
-	privacyLabels.COPPAEnforced = privacyEnforcement.COPPA
-	privacyLabels.LMTEnforced = lmtEnforcer.ShouldEnforce(unknownBidder)
+	privacyLabels.COPPAEnforced = coppa
+	privacyLabels.LMTEnforced = lmt
 
 	var gdprEnforced bool
 	var gdprPerms gdpr.Permissions = &gdpr.AlwaysAllow{}
@@ -148,46 +146,82 @@ func (rs *requestSplitter) cleanOpenRTBRequests(ctx context.Context,
 
 	// bidder level privacy policies
 	for _, bidderRequest := range allBidderRequests {
-		bidRequestAllowed := true
+		privacyEnforcement := privacy.Enforcement{
+			COPPA: coppa,
+			LMT:   lmt,
+		}
 
 		// fetchBids activity
-		fetchBidsActivityAllowed := auctionReq.Activities.Allow(privacy.ActivityFetchBids,
-			privacy.ScopedName{Scope: privacy.ScopeTypeBidder, Name: bidderRequest.BidderName.String()})
+		scopedName := privacy.ScopedName{Scope: privacy.ScopeTypeBidder, Name: bidderRequest.BidderName.String()}
+		fetchBidsActivityAllowed := auctionReq.Activities.Allow(privacy.ActivityFetchBids, scopedName)
 		if fetchBidsActivityAllowed == privacy.ActivityDeny {
 			// skip the call to a bidder if fetchBids activity is not allowed
 			// do not add this bidder to allowedBidderRequests
 			continue
 		}
 
-		// CCPA
-		privacyEnforcement.CCPA = ccpaEnforcer.ShouldEnforce(bidderRequest.BidderName.String())
+		var auctionPermissions gdpr.AuctionPermissions
+		var gdprErr error
 
-		// GDPR
 		if gdprEnforced {
-			auctionPermissions, err := gdprPerms.AuctionActivitiesAllowed(ctx, bidderRequest.BidderCoreName, bidderRequest.BidderName)
-			bidRequestAllowed = auctionPermissions.AllowBidRequest
+			auctionPermissions, gdprErr = gdprPerms.AuctionActivitiesAllowed(ctx, bidderRequest.BidderCoreName, bidderRequest.BidderName)
+			if !auctionPermissions.AllowBidRequest {
+				// auction request is not permitted by GDPR
+				// do not add this bidder to allowedBidderRequests
+				rs.me.RecordAdapterGDPRRequestBlocked(bidderRequest.BidderCoreName)
+				continue
+			}
+		}
 
-			if err == nil {
-				privacyEnforcement.GDPRGeo = !auctionPermissions.PassGeo
+		passIDActivityAllowed := auctionReq.Activities.Allow(privacy.ActivityTransmitUserFPD, scopedName)
+		if passIDActivityAllowed == privacy.ActivityDeny {
+			privacyEnforcement.UFPD = true
+		} else {
+			// run existing policies (GDPR, CCPA, COPPA, LMT)
+			// potentially block passing IDs based on GDPR
+			if gdprEnforced && gdprErr == nil {
 				privacyEnforcement.GDPRID = !auctionPermissions.PassID
-			} else {
-				privacyEnforcement.GDPRGeo = true
+			} else if gdprEnforced && gdprErr != nil {
 				privacyEnforcement.GDPRID = true
 			}
 
-			if !bidRequestAllowed {
-				rs.me.RecordAdapterGDPRRequestBlocked(bidderRequest.BidderCoreName)
+			// potentially block passing IDs based on CCPA
+			privacyEnforcement.CCPA = ccpaEnforcer.ShouldEnforce(bidderRequest.BidderName.String())
+			// potentially block passing IDs based on COPPA
+			if req.BidRequest.Regs != nil && req.BidRequest.Regs.COPPA == 1 {
+				privacyEnforcement.COPPA = true
 			}
 		}
 
+		passGeoActivityAllowed := auctionReq.Activities.Allow(privacy.ActivityTransmitPreciseGeo, scopedName)
+		if passGeoActivityAllowed == privacy.ActivityDeny {
+			privacyEnforcement.PreciseGeo = true
+		} else {
+			// run existing policies (GDPR, CCPA, COPPA, LMT)
+			// potentially block passing geo based on GDPR
+			if gdprEnforced && gdprErr == nil {
+				privacyEnforcement.GDPRGeo = !auctionPermissions.PassGeo
+			} else if gdprEnforced && gdprErr != nil {
+				privacyEnforcement.GDPRGeo = true
+			}
+
+			// potentially block passing geo based on CCPA
+			privacyEnforcement.CCPA = ccpaEnforcer.ShouldEnforce(bidderRequest.BidderName.String())
+			// potentially block passing geo based on COPPA
+			if req.BidRequest.Regs != nil && req.BidRequest.Regs.COPPA == 1 {
+				privacyEnforcement.COPPA = true
+			}
+			// potentially block passing geo based on LMT
+			privacyEnforcement.LMT = lmtEnforcer.ShouldEnforce(unknownBidder)
+		}
+
 		if auctionReq.FirstPartyData != nil && auctionReq.FirstPartyData[bidderRequest.BidderName] != nil {
 			applyFPD(auctionReq.FirstPartyData[bidderRequest.BidderName], bidderRequest.BidRequest)
 		}
 
-		if bidRequestAllowed {
-			privacyEnforcement.Apply(bidderRequest.BidRequest)
-			allowedBidderRequests = append(allowedBidderRequests, bidderRequest)
-		}
+		privacyEnforcement.Apply(bidderRequest.BidRequest)
+		allowedBidderRequests = append(allowedBidderRequests, bidderRequest)
+
 		// GPP downgrade: always downgrade unless we can confirm GPP is supported
 		if shouldSetLegacyPrivacy(rs.bidderInfo, string(bidderRequest.BidderCoreName)) {
 			setLegacyGDPRFromGPP(bidderRequest.BidRequest, gpp)

exchange/utils_test.go

@@ -2296,6 +2296,7 @@ func TestCleanOpenRTBRequestsWithOpenRTBDowngrade(t *testing.T) {
 	bidReq.User.ID = ""
 	bidReq.User.BuyerUID = ""
 	bidReq.User.Yob = 0
+	bidReq.User.Geo = &openrtb2.Geo{Lat: 123.46}
 
 	downgradedRegs := *bidReq.Regs
 	downgradedUser := *bidReq.User
@@ -2610,6 +2611,7 @@ func newBidRequest(t *testing.T) *openrtb2.BidRequest {
 			BuyerUID: "their-id",
 			Yob:      1982,
 			Ext:      json.RawMessage(`{}`),
+			Geo:      &openrtb2.Geo{Lat: 123.456},
 		},
 		Imp: []openrtb2.Imp{{
 			ID: "some-imp-id",
@@ -4275,33 +4277,76 @@ func TestGetMediaTypeForBid(t *testing.T) {
 	}
 }
 
-func TemporarilyDisabledTestCleanOpenRTBRequestsActivitiesFetchBids(t *testing.T) {
+func TestCleanOpenRTBRequestsActivitiesFetchBids(t *testing.T) {
 	testCases := []struct {
-		name              string
-		req               *openrtb2.BidRequest
-		componentName     string
-		allow             bool
-		expectedReqNumber int
+		name                 string
+		req                  *openrtb2.BidRequest
+		privacyConfig        *config.AccountPrivacy
+		componentName        string
+		allow                bool
+		expectedReqNumber    int
+		expectedUserYOB      int64
+		expectedUserLat      float64
+		expectedDeviceDIDMD5 string
 	}{
 		{
-			name:              "request_with_one_bidder_allowed",
-			req:               newBidRequest(t),
-			componentName:     "appnexus",
-			allow:             true,
-			expectedReqNumber: 1,
-		},
-		{
-			name:              "request_with_one_bidder_not_allowed",
-			req:               newBidRequest(t),
-			componentName:     "appnexus",
-			allow:             false,
-			expectedReqNumber: 0,
+			name:                 "fetch_bids_request_with_one_bidder_allowed",
+			req:                  newBidRequest(t),
+			privacyConfig:        getFetchBidsActivityConfig("appnexus", true),
+			expectedReqNumber:    1,
+			expectedUserYOB:      1982,
+			expectedUserLat:      123.456,
+			expectedDeviceDIDMD5: "some device ID hash",
+		},
+		{
+			name:                 "fetch_bids_request_with_one_bidder_not_allowed",
+			req:                  newBidRequest(t),
+			privacyConfig:        getFetchBidsActivityConfig("appnexus", false),
+			expectedReqNumber:    0,
+			expectedUserYOB:      1982,
+			expectedUserLat:      123.456,
+			expectedDeviceDIDMD5: "some device ID hash",
+		},
+		{
+			name:                 "transmit_ufpd_allowed",
+			req:                  newBidRequest(t),
+			privacyConfig:        getTransmitUFPDActivityConfig("appnexus", true),
+			expectedReqNumber:    1,
+			expectedUserYOB:      1982,
+			expectedUserLat:      123.456,
+			expectedDeviceDIDMD5: "some device ID hash",
+		},
+		{
+			name:                 "transmit_ufpd_deny",
+			req:                  newBidRequest(t),
+			privacyConfig:        getTransmitUFPDActivityConfig("appnexus", false),
+			expectedReqNumber:    1,
+			expectedUserYOB:      0,
+			expectedUserLat:      123.456,
+			expectedDeviceDIDMD5: "",
+		},
+		{
+			name:                 "transmit_precise_geo_allowed",
+			req:                  newBidRequest(t),
+			privacyConfig:        getTransmitPreciseGeoActivityConfig("appnexus", true),
+			expectedReqNumber:    1,
+			expectedUserYOB:      1982,
+			expectedUserLat:      123.456,
+			expectedDeviceDIDMD5: "some device ID hash",
+		},
+		{
+			name:                 "transmit_precise_geo_deny",
+			req:                  newBidRequest(t),
+			privacyConfig:        getTransmitPreciseGeoActivityConfig("appnexus", false),
+			expectedReqNumber:    1,
+			expectedUserYOB:      1982,
+			expectedUserLat:      123.46,
+			expectedDeviceDIDMD5: "some device ID hash",
 		},
 	}
 
 	for _, test := range testCases {
-		privacyConfig := getDefaultActivityConfig(test.componentName, test.allow)
-		activities, err := privacy.NewActivityControl(privacyConfig)
+		activities, err := privacy.NewActivityControl(test.privacyConfig)
 		assert.NoError(t, err, "")
 		auctionReq := AuctionRequest{
 			BidRequestWrapper: &openrtb_ext.RequestWrapper{BidRequest: test.req},
@@ -4321,25 +4366,51 @@ func TemporarilyDisabledTestCleanOpenRTBRequestsActivitiesFetchBids(t *testing.T
 			bidderRequests, _, errs := reqSplitter.cleanOpenRTBRequests(context.Background(), auctionReq, nil, gdpr.SignalNo)
 			assert.Empty(t, errs)
 			assert.Len(t, bidderRequests, test.expectedReqNumber)
+
+			if test.expectedReqNumber == 1 {
+				assert.Equal(t, test.expectedUserYOB, bidderRequests[0].BidRequest.User.Yob)
+				assert.Equal(t, test.expectedUserLat, bidderRequests[0].BidRequest.User.Geo.Lat)
+				assert.Equal(t, test.expectedDeviceDIDMD5, bidderRequests[0].BidRequest.Device.DIDMD5)
+			}
 		})
 	}
 }
 
-func getDefaultActivityConfig(componentName string, allow bool) *config.AccountPrivacy {
-	return &config.AccountPrivacy{
-		AllowActivities: config.AllowActivities{
-			FetchBids: config.Activity{
-				Default: ptrutil.ToPtr(true),
-				Rules: []config.ActivityRule{
-					{
-						Allow: allow,
-						Condition: config.ActivityCondition{
-							ComponentName: []string{componentName},
-							ComponentType: []string{"bidder"},
-						},
-					},
+func buildDefaultActivityConfig(componentName string, allow bool) config.Activity {
+	return config.Activity{
+		Default: ptrutil.ToPtr(true),
+		Rules: []config.ActivityRule{
+			{
+				Allow: allow,
+				Condition: config.ActivityCondition{
+					ComponentName: []string{componentName},
+					ComponentType: []string{"bidder"},
 				},
 			},
 		},
 	}
 }
+
+func getFetchBidsActivityConfig(componentName string, allow bool) *config.AccountPrivacy {
+	return &config.AccountPrivacy{
+		AllowActivities: config.AllowActivities{
+			FetchBids: buildDefaultActivityConfig(componentName, allow),
+		},
+	}
+}
+
+func getTransmitUFPDActivityConfig(componentName string, allow bool) *config.AccountPrivacy {
+	return &config.AccountPrivacy{
+		AllowActivities: config.AllowActivities{
+			TransmitUserFPD: buildDefaultActivityConfig(componentName, allow),
+		},
+	}
+}
+
+func getTransmitPreciseGeoActivityConfig(componentName string, allow bool) *config.AccountPrivacy {
+	return &config.AccountPrivacy{
+		AllowActivities: config.AllowActivities{
+			TransmitPreciseGeo: buildDefaultActivityConfig(componentName, allow),
+		},
+	}
+}

privacy/enforcement.go

@@ -9,11 +9,17 @@ type Enforcement struct {
 	GDPRGeo bool
 	GDPRID  bool
 	LMT     bool
+
+	// activities
+	UFPD       bool
+	Eids       bool
+	PreciseGeo bool
+	TID        bool
 }
 
 // Any returns true if at least one privacy policy requires enforcement.
 func (e Enforcement) Any() bool {
-	return e.CCPA || e.COPPA || e.GDPRGeo || e.GDPRID || e.LMT
+	return e.CCPA || e.COPPA || e.GDPRGeo || e.GDPRID || e.LMT || e.UFPD || e.PreciseGeo || e.Eids || e.TID
 }
 
 // Apply cleans personally identifiable information from an OpenRTB bid request.
@@ -23,6 +29,7 @@ func (e Enforcement) Apply(bidRequest *openrtb2.BidRequest) {
 
 func (e Enforcement) apply(bidRequest *openrtb2.BidRequest, scrubber Scrubber) {
 	if bidRequest != nil && e.Any() {
+		bidRequest = scrubber.ScrubRequest(bidRequest, e)
 		bidRequest.Device = scrubber.ScrubDevice(bidRequest.Device, e.getDeviceIDScrubStrategy(), e.getIPv4ScrubStrategy(), e.getIPv6ScrubStrategy(), e.getGeoScrubStrategy())
 		bidRequest.User = scrubber.ScrubUser(bidRequest.User, e.getUserScrubStrategy(), e.getGeoScrubStrategy())
 	}

privacy/enforcement_test.go

@@ -205,6 +205,7 @@ func TestApply(t *testing.T) {
 		replacedUser := &openrtb2.User{}
 
 		m := &mockScrubber{}
+		m.On("ScrubRequest", req, test.enforcement).Return(req).Once()
 		m.On("ScrubDevice", req.Device, test.expectedDeviceID, test.expectedDeviceIPv4, test.expectedDeviceIPv6, test.expectedDeviceGeo).Return(replacedDevice).Once()
 		m.On("ScrubUser", req.User, test.expectedUser, test.expectedUserGeo).Return(replacedUser).Once()
 
@@ -227,6 +228,11 @@ func TestApplyNoneApplicable(t *testing.T) {
 		GDPRGeo: false,
 		GDPRID:  false,
 		LMT:     false,
+
+		UFPD:       false,
+		PreciseGeo: false,
+		TID:        false,
+		Eids:       false,
 	}
 	enforcement.apply(req, m)
 
@@ -248,6 +254,11 @@ type mockScrubber struct {
 	mock.Mock
 }
 
+func (m *mockScrubber) ScrubRequest(bidRequest *openrtb2.BidRequest, enforcement Enforcement) *openrtb2.BidRequest {
+	args := m.Called(bidRequest, enforcement)
+	return args.Get(0).(*openrtb2.BidRequest)
+}
+
 func (m *mockScrubber) ScrubDevice(device *openrtb2.Device, id ScrubStrategyDeviceID, ipv4 ScrubStrategyIPV4, ipv6 ScrubStrategyIPV6, geo ScrubStrategyGeo) *openrtb2.Device {
 	args := m.Called(device, id, ipv4, ipv6, geo)
 	return args.Get(0).(*openrtb2.Device)

privacy/enforcer.go

@@ -3,7 +3,6 @@ package privacy
 import (
 	"fmt"
 	"github.com/prebid/prebid-server/config"
-	"github.com/prebid/prebid-server/errortypes"
 	"strings"
 )
 
@@ -33,9 +32,6 @@ func NewActivityControl(privacyConf *config.AccountPrivacy) (ActivityControl, er
 
 	if privacyConf == nil {
 		return ac, err
-	} else {
-		//temporarily disable Activities if they are specified at the account level
-		return ac, &errortypes.Warning{Message: "account.Privacy has no effect as the feature is under development."}
 	}
 
 	plans := make(map[Activity]ActivityPlan)

privacy/enforcer_test.go

@@ -8,7 +8,7 @@ import (
 	"testing"
 )
 
-func TemporarilyDisabledTestNewActivityControl(t *testing.T) {
+func TestNewActivityControl(t *testing.T) {
 
 	testCases := []struct {
 		name            string