Emit warnings on conflicts between openrtb 2.6 pre GPP privacy and GPP (

prebid#2645)

endpoints/openrtb2/auction.go

@@ -18,6 +18,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/julienschmidt/httprouter"
 	gpplib "github.com/prebid/go-gpp"
+	"github.com/prebid/go-gpp/constants"
 	"github.com/prebid/openrtb/v17/adcom1"
 	"github.com/prebid/openrtb/v17/native1"
 	nativeRequests "github.com/prebid/openrtb/v17/native1/request"
@@ -728,18 +729,6 @@ func (deps *endpointDeps) validateRequest(req *openrtb_ext.RequestWrapper, isAmp
 		return append(errL, err)
 	}
 
-	if err := deps.validateUser(req, aliases); err != nil {
-		return append(errL, err)
-	}
-
-	if err := validateRegs(req); err != nil {
-		return append(errL, err)
-	}
-
-	if err := validateDevice(req.Device); err != nil {
-		return append(errL, err)
-	}
-
 	var gpp gpplib.GppContainer
 	if req.BidRequest.Regs != nil && len(req.BidRequest.Regs.GPP) > 0 {
 		gpp, err = gpplib.Parse(req.BidRequest.Regs.GPP)
@@ -750,8 +739,33 @@ func (deps *endpointDeps) validateRequest(req *openrtb_ext.RequestWrapper, isAmp
 		}
 	}
 
-	if ccpaPolicy, err := ccpa.ReadFromRequestWrapper(req, gpp); err != nil {
+	if errs := deps.validateUser(req, aliases, gpp); errs != nil {
+		if len(errs) > 0 {
+			errL = append(errL, errs...)
+		}
+		if errortypes.ContainsFatalError(errs) {
+			return errL
+		}
+	}
+
+	if errs := validateRegs(req, gpp); errs != nil {
+		if len(errs) > 0 {
+			errL = append(errL, errs...)
+		}
+		if errortypes.ContainsFatalError(errs) {
+			return errL
+		}
+	}
+
+	if err := validateDevice(req.Device); err != nil {
 		return append(errL, err)
+	}
+
+	if ccpaPolicy, err := ccpa.ReadFromRequestWrapper(req, gpp); err != nil {
+		errL = append(errL, err)
+		if errortypes.ContainsFatalError([]error{err}) {
+			return errL
+		}
 	} else if _, err := ccpaPolicy.Parse(exchange.GetValidBidders(aliases)); err != nil {
 		if _, invalidConsent := err.(*errortypes.Warning); invalidConsent {
 			errL = append(errL, &errortypes.Warning{
@@ -1609,29 +1623,41 @@ func (deps *endpointDeps) validateApp(req *openrtb_ext.RequestWrapper) error {
 	return err
 }
 
-func (deps *endpointDeps) validateUser(req *openrtb_ext.RequestWrapper, aliases map[string]string) error {
+func (deps *endpointDeps) validateUser(req *openrtb_ext.RequestWrapper, aliases map[string]string, gpp gpplib.GppContainer) []error {
+	var errL []error
+
+	if req == nil || req.BidRequest == nil || req.BidRequest.User == nil {
+		return nil
+	}
 	// The following fields were previously uints in the OpenRTB library we use, but have
 	// since been changed to ints. We decided to maintain the non-negative check.
-	if req != nil && req.BidRequest != nil && req.User != nil {
-		if req.User.Geo != nil && req.User.Geo.Accuracy < 0 {
-			return errors.New("request.user.geo.accuracy must be a positive number")
-		}
+	if req.User.Geo != nil && req.User.Geo.Accuracy < 0 {
+		return append(errL, errors.New("request.user.geo.accuracy must be a positive number"))
 	}
 
+	if req.User.Consent != "" {
+		for _, section := range gpp.Sections {
+			if section.GetID() == constants.SectionTCFEU2 && section.GetValue() != req.User.Consent {
+				errL = append(errL, &errortypes.Warning{
+					Message:     "user.consent GDPR string conflicts with GPP (regs.gpp) GDPR string, using regs.gpp",
+					WarningCode: errortypes.InvalidPrivacyConsentWarningCode})
+			}
+		}
+	}
 	userExt, err := req.GetUserExt()
 	if err != nil {
-		return fmt.Errorf("request.user.ext object is not valid: %v", err)
+		return append(errL, fmt.Errorf("request.user.ext object is not valid: %v", err))
 	}
 	// Check if the buyeruids are valid
 	prebid := userExt.GetPrebid()
 	if prebid != nil {
 		if len(prebid.BuyerUIDs) < 1 {
-			return errors.New(`request.user.ext.prebid requires a "buyeruids" property with at least one ID defined. If none exist, then request.user.ext.prebid should not be defined.`)
+			return append(errL, errors.New(`request.user.ext.prebid requires a "buyeruids" property with at least one ID defined. If none exist, then request.user.ext.prebid should not be defined.`))
 		}
 		for bidderName := range prebid.BuyerUIDs {
 			if _, ok := deps.bidderMap[bidderName]; !ok {
 				if _, ok := aliases[bidderName]; !ok {
-					return fmt.Errorf("request.user.ext.%s is neither a known bidder name nor an alias in request.ext.prebid.aliases.", bidderName)
+					return append(errL, fmt.Errorf("request.user.ext.%s is neither a known bidder name nor an alias in request.ext.prebid.aliases", bidderName))
 				}
 			}
 		}
@@ -1640,45 +1666,64 @@ func (deps *endpointDeps) validateUser(req *openrtb_ext.RequestWrapper, aliases
 	eids := userExt.GetEid()
 	if eids != nil {
 		if len(*eids) == 0 {
-			return errors.New("request.user.ext.eids must contain at least one element or be undefined")
+			return append(errL, errors.New("request.user.ext.eids must contain at least one element or be undefined"))
 		}
 		uniqueSources := make(map[string]struct{}, len(*eids))
 		for eidIndex, eid := range *eids {
 			if eid.Source == "" {
-				return fmt.Errorf("request.user.ext.eids[%d] missing required field: \"source\"", eidIndex)
+				return append(errL, fmt.Errorf("request.user.ext.eids[%d] missing required field: \"source\"", eidIndex))
 			}
 			if _, ok := uniqueSources[eid.Source]; ok {
-				return errors.New("request.user.ext.eids must contain unique sources")
+				return append(errL, errors.New("request.user.ext.eids must contain unique sources"))
 			}
 			uniqueSources[eid.Source] = struct{}{}
 
 			if len(eid.UIDs) == 0 {
-				return fmt.Errorf("request.user.ext.eids[%d].uids must contain at least one element or be undefined", eidIndex)
+				return append(errL, fmt.Errorf("request.user.ext.eids[%d].uids must contain at least one element or be undefined", eidIndex))
 			}
 
 			for uidIndex, uid := range eid.UIDs {
 				if uid.ID == "" {
-					return fmt.Errorf("request.user.ext.eids[%d].uids[%d] missing required field: \"id\"", eidIndex, uidIndex)
+					return append(errL, fmt.Errorf("request.user.ext.eids[%d].uids[%d] missing required field: \"id\"", eidIndex, uidIndex))
 				}
 			}
 		}
 	}
 
-	return nil
+	return errL
 }
 
-func validateRegs(req *openrtb_ext.RequestWrapper) error {
+func validateRegs(req *openrtb_ext.RequestWrapper, gpp gpplib.GppContainer) []error {
+	var errL []error
+
+	if req == nil || req.BidRequest == nil || req.BidRequest.Regs == nil {
+		return nil
+	}
+
+	if req.BidRequest.Regs.GDPR != nil && req.BidRequest.Regs.GPPSID != nil {
+		gdpr := int8(0)
+		for _, id := range req.BidRequest.Regs.GPPSID {
+			if id == int8(constants.SectionTCFEU2) {
+				gdpr = 1
+			}
+		}
+		if gdpr != *req.BidRequest.Regs.GDPR {
+			errL = append(errL, &errortypes.Warning{
+				Message:     "regs.gdpr signal conflicts with GPP (regs.gpp_sid) and will be ignored",
+				WarningCode: errortypes.InvalidPrivacyConsentWarningCode})
+		}
+	}
 	regsExt, err := req.GetRegExt()
 	if err != nil {
-		return fmt.Errorf("request.regs.ext is invalid: %v", err)
+		return append(errL, fmt.Errorf("request.regs.ext is invalid: %v", err))
 	}
 
 	gdpr := regsExt.GetGDPR()
 	if gdpr != nil && *gdpr != 0 && *gdpr != 1 {
-		return errors.New("request.regs.ext.gdpr must be either 0 or 1")
+		return append(errL, errors.New("request.regs.ext.gdpr must be either 0 or 1"))
 	}
 
-	return nil
+	return errL
 }
 
 func validateDevice(device *openrtb2.Device) error {

endpoints/openrtb2/auction_test.go

@@ -3529,15 +3529,40 @@ func TestIOS14EndToEnd(t *testing.T) {
 }
 
 func TestAuctionWarnings(t *testing.T) {
-	reqBody := validRequest(t, "us-privacy-invalid.json")
+	testCases := []struct {
+		name            string
+		file            string
+		expectedWarning string
+	}{
+		{
+			name:            "us-privacy-invalid",
+			file:            "us-privacy-invalid.json",
+			expectedWarning: "CCPA consent is invalid and will be ignored. (request.regs.ext.us_privacy must contain 4 characters)",
+		},
+		{
+			name:            "us-privacy-signals-conflict",
+			file:            "us-privacy-conflict.json",
+			expectedWarning: "regs.us_privacy consent does not match uspv1 in GPP, using regs.gpp",
+		},
+		{
+			name:            "gdpr-signals-conflict", // gdpr signals do not match
+			file:            "gdpr-conflict.json",
+			expectedWarning: "regs.gdpr signal conflicts with GPP (regs.gpp_sid) and will be ignored",
+		},
+		{
+			name:            "gdpr-signals-conflict2", // gdpr consent strings do not match
+			file:            "gdpr-conflict2.json",
+			expectedWarning: "user.consent GDPR string conflicts with GPP (regs.gpp) GDPR string, using regs.gpp",
+		},
+	}
 	deps := &endpointDeps{
 		fakeUUIDGenerator{},
 		&warningsCheckExchange{},
 		mockBidderParamValidator{},
 		&mockStoredReqFetcher{},
 		empty_fetcher.EmptyFetcher{},
 		empty_fetcher.EmptyFetcher{},
-		&config.Configuration{MaxRequestSize: int64(len(reqBody))},
+		&config.Configuration{MaxRequestSize: maxSize},
 		&metricsConfig.NilMetricsEngine{},
 		analyticsConf.NewPBSAnalytics(&config.Analytics{}),
 		map[string]string{},
@@ -3550,24 +3575,27 @@ func TestAuctionWarnings(t *testing.T) {
 		empty_fetcher.EmptyFetcher{},
 		hookexecution.NewHookExecutor(hooks.EmptyPlanBuilder{}, hookexecution.EndpointAuction, &metricsConfig.NilMetricsEngine{}),
 	}
+	for _, test := range testCases {
+		t.Run(test.name, func(t *testing.T) {
+			reqBody := validRequest(t, test.file)
+			req := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(reqBody))
+			recorder := httptest.NewRecorder()
 
-	req := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(reqBody))
-	recorder := httptest.NewRecorder()
+			deps.Auction(recorder, req, nil)
 
-	deps.Auction(recorder, req, nil)
+			if recorder.Code != http.StatusOK {
+				t.Errorf("Endpoint should return a 200")
+			}
+			warnings := deps.ex.(*warningsCheckExchange).auctionRequest.Warnings
+			if !assert.Len(t, warnings, 1, "One warning should be returned from exchange") {
+				t.FailNow()
+			}
+			actualWarning := warnings[0].(*errortypes.Warning)
+			assert.Equal(t, test.expectedWarning, actualWarning.Message, "Warning message is incorrect")
 
-	if recorder.Code != http.StatusOK {
-		t.Errorf("Endpoint should return a 200")
-	}
-	warnings := deps.ex.(*warningsCheckExchange).auctionRequest.Warnings
-	if !assert.Len(t, warnings, 1, "One warning should be returned from exchange") {
-		t.FailNow()
+			assert.Equal(t, errortypes.InvalidPrivacyConsentWarningCode, actualWarning.WarningCode, "Warning code is incorrect")
+		})
 	}
-	actualWarning := warnings[0].(*errortypes.Warning)
-	expectedMessage := "CCPA consent is invalid and will be ignored. (request.regs.ext.us_privacy must contain 4 characters)"
-	assert.Equal(t, expectedMessage, actualWarning.Message, "Warning message is incorrect")
-
-	assert.Equal(t, errortypes.InvalidPrivacyConsentWarningCode, actualWarning.WarningCode, "Warning code is incorrect")
 }
 
 func TestParseRequestParseImpInfoError(t *testing.T) {

endpoints/openrtb2/sample-requests/invalid-whole/user-ext-prebid-buyeruids-unknown.json

@@ -34,5 +34,5 @@
     }
   },
   "expectedReturnCode": 400,
-  "expectedErrorMessage": "Invalid request: request.user.ext.unknown is neither a known bidder name nor an alias in request.ext.prebid.aliases.\n"
+  "expectedErrorMessage": "Invalid request: request.user.ext.unknown is neither a known bidder name nor an alias in request.ext.prebid.aliases\n"
 }

endpoints/openrtb2/sample-requests/valid-whole/supplementary/gdpr-conflict.json

@@ -0,0 +1,56 @@
+{
+  "description": "Well formed amp request with  conflicting (2.6 gdpr consent vs. GPP) gdpr signals",
+  "mockBidRequest": {
+    "id": "b9c97a4b-cbc4-483d-b2c4-58a19ed5cfc5",
+    "site": {
+      "page": "prebid.org",
+      "publisher": {
+        "id": "a3de7af2-a86a-4043-a77b-c7e86744155e"
+      }
+    },
+    "source": {
+      "tid": "b9c97a4b-cbc4-483d-b2c4-58a19ed5cfc5"
+    },
+    "tmax": 1000,
+    "imp": [
+      {
+        "id": "/19968336/header-bid-tag-0",
+        "ext": {
+          "appnexus": {
+            "placementId": 12883451
+          }
+        },
+        "banner": {
+          "format": [
+            {
+              "w": 300,
+              "h": 250
+            },
+            {
+              "w": 300,
+              "h": 300
+            }
+          ]
+        }
+      }
+    ],
+    "regs": {
+      "gpp": "DBACNYA~CPXxRfAPXxRfAAfKABENB-CgAAAAAAAAAAYgAAAAAAAA~1NYN",
+      "gpp_sid": [6],
+      "gdpr": 1,
+      "ext": {
+        "us_privacy": "1YYY"
+      }
+    },
+    "user": {
+      "consent": "CPXxRfAPXxRfAAfKABENB-CgAAAAAAAAAAYgAAAAAAAA",
+      "ext": {}
+    }
+  },
+  "expectedBidResponse": {
+      "id":"b9c97a4b-cbc4-483d-b2c4-58a19ed5cfc5",
+      "bidid":"test bid id",
+      "nbr":0
+  },
+  "expectedReturnCode": 200
+}

endpoints/openrtb2/sample-requests/valid-whole/supplementary/gdpr-conflict2.json

@@ -0,0 +1,56 @@
+{
+  "description": "Well formed amp request with conflicting (2.6 gdpr consent vs. GPP) gdpr consent strings",
+  "mockBidRequest": {
+    "id": "b9c97a4b-cbc4-483d-b2c4-58a19ed5cfc5",
+    "site": {
+      "page": "prebid.org",
+      "publisher": {
+        "id": "a3de7af2-a86a-4043-a77b-c7e86744155e"
+      }
+    },
+    "source": {
+      "tid": "b9c97a4b-cbc4-483d-b2c4-58a19ed5cfc5"
+    },
+    "tmax": 1000,
+    "imp": [
+      {
+        "id": "/19968336/header-bid-tag-0",
+        "ext": {
+          "appnexus": {
+            "placementId": 12883451
+          }
+        },
+        "banner": {
+          "format": [
+            {
+              "w": 300,
+              "h": 250
+            },
+            {
+              "w": 300,
+              "h": 300
+            }
+          ]
+        }
+      }
+    ],
+    "regs": {
+      "gpp": "DBACNYA~CPXxRfAPXxRfAAfKABENB-CgAAAAAAAAAAYgAAAAAAAA~1NYN",
+      "gpp_sid": [2,6],
+      "gdpr": 1,
+      "ext": {
+        "us_privacy": "1YYY"
+      }
+    },
+    "user": {
+      "consent": "Invalid",
+      "ext": {}
+    }
+  },
+  "expectedBidResponse": {
+      "id":"b9c97a4b-cbc4-483d-b2c4-58a19ed5cfc5",
+      "bidid":"test bid id",
+      "nbr":0
+  },
+  "expectedReturnCode": 200
+}