From 26d9a66d775705090fdf22439190c2480f66f7b6 Mon Sep 17 00:00:00 2001 From: Sachin Survase <60501281+sachin-pubmatic@users.noreply.github.com> Date: Tue, 17 Aug 2021 21:59:12 +0530 Subject: [PATCH] UOE-6610: Upgrade prebid-server to 0.170.0 (#190) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Smaato: Add support for app (#1767) Co-authored-by: Bernhard Pickenbrock * Update sync types (#1770) * 33across: Fix Shared Memory Overwriting (#1764) This reverts commit f7df258f061788ef7e72529115aa5fd554fa9f16. * Fix race condition in Yeahmobi adapter (#1761) Co-authored-by: Gus Carreon * Pubnative: Fix Shared Memory Overwriting (#1760) * Add request for registration (#1780) * Update OpenRTB Library (#1733) * Tappx changes - Backward compatible change of version (#1777) Co-authored-by: ubuntu Co-authored-by: Albert Grandes * DMX: Enforcing w and h in imp (#1778) Co-authored-by: steve-a-districtm * Remove Authorization Headers From Debug Response (#1779) * Hide Authorization Headers In Debug Response * MakeExtHeaders Tests * Add Empty Test * Use http.Header Methods * Updates From Code Review * Fix Merge Conflict * New Adapter: Bidmachine (#1769) * New Adapter: Criteo (#1775) * Fix shared memory issue when stripping authorization header from bid requests (#1790) * RTB House: update parameters (#1785) * update parameters required by RTB House adapter * tabs to spaces Co-authored-by: Przemyslaw Iwanczak * Generate seatbid[].bid[].ext.prebid.bidid (#1772) * Basic GDPR enforcement for specific publisher-vendors. (#1782) * New Adapter: Zemanta (#1774) * add zemanta adapter * update openrtb package for zemanta * fix loop iterator reference bug * fix getMediaTypeForImp to match server behavior * Zemanta: Rename Adapter To Outbrain (#1797) * Update openrtb library to v15 (#1802) * IX: Set bidVideo when category and duration is available (#1794) * Update IX defaults (#1799) Co-authored-by: Mike Burns * Update Adyoulike endpoint to hit production servers (#1805) * Openx: use bidfloor if set - prebid.js adapter behavior (#1795) * [ORBIDDER] add gvlVendorID and set bid response currency (#1798) * New Adapter: ADXCG (#1803) * Update kidoz properties to type string (#1808) Remove definitions object from schema and define types and other parameters directly in properties objects to ensure compatibility with more downstream systems that use this schema. * Update bidmachine properties to type string (#1809) Remove definitions object from schema and define types and other parameters directly in properties objects to ensure compatibility with more downstream systems that use this schema. * transform native eventtrackers to imptrackers and jstracker (#1811) * TheMediaGrid: Added processing of imp[].ext.data (#1807) * New Adapter: adf (adformOpenRTB) (#1815) * initial adformOpenRTB adapter implementation * do not make request copy * rename adfromOpenRTB adapter to adf * fix user sync url * Set Adhese gvl id and vast modification flag (#1821) * Added gvlVendorID for mobilefuse (#1822) * AppNexus: reform bid floor handling (#1814) * PubNative: Add GVL Vendor ID (#1824) * InMobi: adding gvlVendorID to static yaml (#1826) * Epom Adapter: configure vendor id (GVL ID) (#1828) Co-authored-by: Vasyl Zarva * Update Adtarget gvlid (#1829) * Adding site to static yaml, and exemplary tests (#1827) * AdOcean adapter - add support for mobile apps (#1830) * Allow Native Ad Exchange Specific Types (#1810) * PubMatic: Fix Banner Size Assignment When No AdSlot Provided (#1825) * New Adapter: Interactive Offers (#1835) * IX: Set category in bid.cat (#1837) * New Adapter: Madvertise (#1834) * Conversant bid floor handling (#1840) * Adf adapter: banner and video mediatype support (#1841) * Test for data race conditions in adapters (#1756) * Revcontent adapter: add vendor id (GVL ID) (#1849) * Refactor: Removed unused GDPR return value (#1839) * New Adapter : Kayzen (#1838) * Add Kayzen Adapter * Beachfront: Add schain support (#1844) Co-authored-by: jim naumann * Pangle: add appid & placementid to bidder param (#1842) Co-authored-by: hcai * New Adapter: BidsCube (#1843) * add BidsCube adapter * add BidsCube adapter * add BidsCube adapter * add BidsCube adapter * add BidsCube adapter * add BidsCube adapter Co-authored-by: vlad * Add Viewdeos alias (#1846) * [Smaato] Adding TCF 2.0 vendor id (#1852) * Pass Global Privacy Control header to bidders (#1789) * Feature Request: Ability to pass Sec-GPC header to the bidder endpoints (#1712) * making Sec-GPC value check more strict * minor syntax change * gofmt fixes * updates against draft-code-review:one, more to come soon. * adding a unit test * Adding a test and request header clone update * modified one test and related logic * modifying the last test added with slight more modification of the logic * GDPR: Don't Call Bidder If It Lacks Purpose 2 Legal Basis (#1851) * Update go-gdpr package to v0.9.0 (#1856) * Marsmedia - add GVL ID to bidder config file (#1864) Co-authored-by: Vladi Izgayev * PubMatic: Added parameters dctr & pmzoneid (#1865) * Better Support For Go Modules (#1862) * IX: Update usersync default id (#1873) * AppNexus: Make Ad Pod Id Optional (#1792) * Bugfix for applyCategoryMapping (#1857) * Facebook: Drop consented providers (#1867) * Between: Fix for bid floor issue#1787 (#1870) Co-authored-by: Egor Skorokhodov * Beachfront: Fix for bid floor issue#1787 (#1878) Co-authored-by: jim naumann * Updating interactiveoffers contact info (#1881) * Docs metrics configuration (#1850) * Criteo: update maintainer email address (#1884) * New Adapter: BrightMountainMedia (#1855) New Adapter : BrightMountainMedia * New Adapter: AlgoriX (#1861) * Remove LifeStreet + Legacy Cleanup (#1883) * New Adapter: E-Volution (#1868) * [criteo] accept zoneId and networkId alternate case (#1869) * Unit test random map order fix (#1887) Co-authored-by: Veronika Solovei * Request Provided Currency Rates (#1753) * Debug override header (#1853) * Remove GDPR TCF1 (#1854) * Rename GDPR UserSyncIfAmbiguous to DefaultValue (#1858) * Accept bidfloor from impression to fix issue #1787 for sovrn adapter (#1886) * GDPR: require host specify default value (#1859) * New Adapter: Smile Wanted (#1877) * New Adapter: Smile Wanted * Implementations of changes requested by : https://github.com/prebid/prebid-server/pull/1877#pullrequestreview-676968474 * Improvement of test coverage as requested. * Implementations of changes requested by : https://github.com/prebid/prebid-server/pull/1877#pullrequestreview-683853119 * Fix a weak vendor enforcement bug where vendor does not exist (#1890) * Pubmatic: Sending GPT slotname in impression extension (#1880) * Update To Go 1.16 (#1888) * Friendlier Startup Error Messages (#1894) * Second fix for weak vendor enforcement (#1896) * Rubicon: hardcode EUR to USD for floors (#1899) Co-authored-by: Serhii Nahornyi * Outbrain adapter: overwrite tagid only if it exists (#1895) * New Adapter: BidMyAdz (#1882) Co-authored-by: BidMyAdz * Currency Conversion Utility Function (#1901) * New Adapter: SA Lunamedia (#1891) * Removed Digitrust From Prebid Server (#1892) Co-authored-by: avolcy * IX: merge eventtrackers with imptrackers for native bid responses (#1900) * Inmobi: user sync (#1911) * Rubicon: Update segtax logic (#1909) Co-authored-by: Serhii Nahornyi * New Adapter: Axonix (#1912) * New Axonix adapter * Changed endpoint * Rename adapter type * Leave in examplary only the basic test fixtures * PR comments * Rubicon: Fix Nil Reference Panic (#1918) * GDPR: host-level per-purpose vendor exceptions config (#1893) Co-authored-by: Scott Kay * Criteo - Fix fields mapping error when building bid from bidder response (#1917) * Smaato: Rework multi imp support and add adpod support (#1902) * Allowed $0.00 price bids if there are deals (#1910) * GDPR: host-level per-purpose enforce vendor signals config (#1921) * Add GDPR host-level per-purpose enforce vendor signals config * Update config defaults test with TCF2 object compare * Fix for fetcher warning at server startup (#1914) Co-authored-by: Veronika Solovei * Request Wrapper first pass (#1784) * Rubicon: Use currency conversion function (#1924) Co-authored-by: Serhii Nahornyi * New Adapter: operaads (#1916) * Fix Beachfront data race condition (#1915) Co-authored-by: Jim Naumann * Sharethrough: Add support for GPID (#1925) * Admixer: Fix for bid floor issue#1787 (#1872) * InMobi: adding native support (#1928) * Tappx: new bidder params (#1931) Co-authored-by: Albert Grandes * Fix CVE-2020-35381 (#1942) * Smaato: Split multiple media types (#1930) Co-authored-by: Bernhard Pickenbrock * New adapter: Adagio (#1907) * IX: update required site id field to be more flexible (#1934) Co-authored-by: Joshua Gross * Add SmartRTB adapter (#1071) * Adds timeout notifications for Facebook (#1182) * Add Adoppler bidder support. (#1186) * Add Adoppler bidder support. * Address code review comments. Use JSON-templates for testing. * Fix misprint; Add url.PathEscape call for adunit URL parameter. * Kidoz adapter (#1210) Co-authored-by: Ryan Haksi * AMP CCPA Fix (#1187) * Add kidoz bidder info (#1257) got this info from email communication with kidoz * populate the app ID in the FAN timeout notif url with the publisher ID (#1265) and the auction with the request ID Co-authored-by: Aadesh Patel * * Add PubMatic bidder doc file (#1255) * Add app video capability to PubMatic bidder info file * Added OpenX Bidder adapter documentation (#1291) * Restore the AMP privacy exception as an option. (#1311) * Restore the AMP privacy exception as an option. * Adds missing test case * More PR feedback * Remove unused constant * Comment tweak * Add Yieldlab Adapter (#1287) Co-authored-by: Mirko Feddern Signed-off-by: Alex Klinkert Co-authored-by: Alexander Pinnecke Co-authored-by: Alex Klinkert Co-authored-by: Mirko Feddern * Add Pubnative bidder documentation (#1340) * Add Adtarget server adapter (#1319) * Add Adtarget server adapter * Suggested changes for Adtarget * Avoid overriding AMP request original size with mutli-size (#1352) * Adds Avocet adapter (#1354) * Adding Smartadserver adapter (#1346) Co-authored-by: tadam * Metrics for TCF 2 adoption (#1360) * Add support for multiple root schain nodes (#1374) * Facebook Only Supports App Impressions (#1396) * Add Outgoing Connection Metrics (#1343) * OpenX adapter: pass optional platform (PBID-598) (#1421) * Adds keyvalue hb_format support (#1414) * feat: Add new logger module - Pubstack Analytics Module (#1331) * Pubstack Analytics V1 (#11) * V1 Pubstack (#7) * feat: Add Pubstack Logger (#6) * first version of pubstack analytics * bypass viperconfig * commit #1 * gofmt * update configuration and make the tests pass * add readme on how to configure the adapter and update the network calls * update logging and fix intake url definition * feat: Pubstack Analytics Connector * fixing go mod * fix: bad behaviour on appending path to auction url * add buffering * support bootstyrap like configuration * implement route for all the objects * supports termination signal handling for goroutines * move readme to the correct location * wording * enable configuration reload + add tests * fix logs messages * fix tests * fix log line * conclude merge * merge * update go mod Co-authored-by: Amaury Ravanel * fix duplicated channel keys Co-authored-by: Amaury Ravanel * first pass - PR reviews * rename channel* -> eventChannel * dead code * Review (#10) * use json.Decoder * update documentation * use nil instead []byte("") * clean code * do not use http.DefaultClient * fix race condition (need validation) * separate the sender and buffer logics * refactor the default configuration * remove error counter * Review GP + AR * updating default config * add more logs * remove alias fields in json * fix json serializer * close event channels Co-authored-by: Amaury Ravanel * fix race condition * first pass (pr reviews) * refactor: store enabled modules into a dedicated struct * stop goroutine * test: improve coverage * PR Review * Revert "refactor: store enabled modules into a dedicated struct" This reverts commit f57d9d61680c74244effc39a5d96d6cbb2f19f7d. # Conflicts: # analytics/config/config_test.go Co-authored-by: Amaury Ravanel * New bid adapter for Smaato (#1413) Co-authored-by: vikram Co-authored-by: Stephan * New Adprime adapter (#1418) Co-authored-by: Aiholkin * Enable geo activation of GDPR flag (#1427) * moving docs to website repo (#1443) * Add support for Account configuration (PBID-727, #1395) (#1426) * Pass Through First Party Context Data (#1479) * between adapter (#1437) Co-authored-by: Alexey Elymanov * Bidder Uniqueness Gatekeeping Test (#1506) * Smarty ads adapter (#1500) Co-authored-by: Kushneryk Pavlo Co-authored-by: user * Vtrack and event endpoints (#1467) * Add bidder name key support (#1496) * Add metrics to capture stored data fetch all/delta durations with fetch status (#1515) * Acuity ads adapter (#1537) Co-authored-by: Kushneryk Pavlo * Yieldmo app support in yaml file (#1542) Co-authored-by: Winston * Add client/AccountID support into Adoppler adapter. (#1535) * 33Across: Add video support in adapter (#1557) * Fix bug in request.imp.ext Validation (#1575) * First draft * Brian's reivew * Removed leftover comments Co-authored-by: Gus Carreon * New Adapter Initialization Framework (#1532) * Fix 33Across App Handling (#1602) * Fix adapter JSON tests to have the right test structure (#1589) * Fix JSON EMX Digital * Fix JSON Brightroll * Fix JSON Beintoo * Fix JSON Gamoshi * Fix JSON Kubient * Fix JSON Marsmedia * Fix JSON Nanointeractive * Fix JSON Telaria * Fix JSON valueimpression * Fix JSON smartyads * Fix JSON rhythmone * Fix JSON krushmedia * Fix JSON cpmstar * Fix JSON acuityads * Fix JSON avocet * Rename wrongly named acuity ads test file * Fix JSON gamma * Add expected no bid responses * Fixed indentation and asesome-markup * Added usersync support to Between SSP adapter; Major fixes and refactor (#1587) Co-authored-by: Egor Skorokhodov * 33Across: Add support for multi-imp requests (#1609) * Remove legacy GDPR AMP config flag used to prevent buyer ID scrub on AMP requests (#1565) * New Adapter: Mobfox (#1585) Co-authored-by: mobfox * New Adapter: Revcontent (#1622) * Audit beachfront tests and change some videoResponseType details (#1638) * Tappx User Syncer + Site Update (#1674) Co-authored-by: ubuntu Co-authored-by: Albert Grandes * Beachfront Additional tests (#1679) * added place holder files for all but a couple of the intended new tests. I need to grok what those couple mean before being able to name a file. * This covers most of the suggested cases and a couple more that occured to me. I'll look at the couple that I held off on next. * added the unmarshal tests and found a couple problems to address in the process. * removed my __debug_bin. should be in gitignore. * A bit of clean up and commenting. Bumped version number. Added __debug_bin to .gitignore. This is the debugging binary created by Visual Studio Code, or at least version 1.52.1. * missed a bunch of version strings * removed IP faker * If IP is not included in an AdM request, an error is now thrown for the AdM imp instead of faking it. The AdM endpoint is the only one that requires an IP. Also, added several "no-ip" test cases. * Whent back to the fake IP solution instead of the error. Removed most of the "no-ip" test cases, leaving one. * changed ip in adm-video.json to not match the faker ip * removed a debugging comment * Mobfox: Add rout to adexcange (#1702) Co-authored-by: mobfox * Add Support For SkAdN + Refactor Split Imps (#1741) * No Longer Move bid.ext To bid.ext.bidder (#1742) * No Longer Move bid.ext To bid.ext.bidder * Remove Similar Behavior From seatbid.ext * Avoid Second Bid Copy * Removed Unused seatbid.ext * Debug warnings (#1724) Co-authored-by: Veronika Solovei * FPD: Allow imp.ext.data To Passthrough To Adapters (#1765) * Basic GDPR enforcement for specific publisher-vendors. (#1782) * New Adapter: Zemanta (#1774) * add zemanta adapter * update openrtb package for zemanta * fix loop iterator reference bug * fix getMediaTypeForImp to match server behavior * Zemanta: Rename Adapter To Outbrain (#1797) * New Adapter: adf (adformOpenRTB) (#1815) * initial adformOpenRTB adapter implementation * do not make request copy * rename adfromOpenRTB adapter to adf * fix user sync url * Set Adhese gvl id and vast modification flag (#1821) * Added gvlVendorID for mobilefuse (#1822) * AppNexus: reform bid floor handling (#1814) * PubNative: Add GVL Vendor ID (#1824) * InMobi: adding gvlVendorID to static yaml (#1826) * Epom Adapter: configure vendor id (GVL ID) (#1828) Co-authored-by: Vasyl Zarva * Update Adtarget gvlid (#1829) * Adding site to static yaml, and exemplary tests (#1827) * AdOcean adapter - add support for mobile apps (#1830) * Allow Native Ad Exchange Specific Types (#1810) * PubMatic: Fix Banner Size Assignment When No AdSlot Provided (#1825) * New Adapter: Interactive Offers (#1835) * IX: Set category in bid.cat (#1837) * New Adapter: Madvertise (#1834) * Conversant bid floor handling (#1840) * Adf adapter: banner and video mediatype support (#1841) * Test for data race conditions in adapters (#1756) * Revcontent adapter: add vendor id (GVL ID) (#1849) * Refactor: Removed unused GDPR return value (#1839) * New Adapter : Kayzen (#1838) * Add Kayzen Adapter * Beachfront: Add schain support (#1844) Co-authored-by: jim naumann * Pangle: add appid & placementid to bidder param (#1842) Co-authored-by: hcai * New Adapter: BidsCube (#1843) * add BidsCube adapter * add BidsCube adapter * add BidsCube adapter * add BidsCube adapter * add BidsCube adapter * add BidsCube adapter Co-authored-by: vlad * Add Viewdeos alias (#1846) * [Smaato] Adding TCF 2.0 vendor id (#1852) * Pass Global Privacy Control header to bidders (#1789) * Feature Request: Ability to pass Sec-GPC header to the bidder endpoints (#1712) * making Sec-GPC value check more strict * minor syntax change * gofmt fixes * updates against draft-code-review:one, more to come soon. * adding a unit test * Adding a test and request header clone update * modified one test and related logic * modifying the last test added with slight more modification of the logic * GDPR: Don't Call Bidder If It Lacks Purpose 2 Legal Basis (#1851) * Update go-gdpr package to v0.9.0 (#1856) * Marsmedia - add GVL ID to bidder config file (#1864) Co-authored-by: Vladi Izgayev * PubMatic: Added parameters dctr & pmzoneid (#1865) * Better Support For Go Modules (#1862) * AppNexus: Make Ad Pod Id Optional (#1792) * Facebook: Drop consented providers (#1867) * Between: Fix for bid floor issue#1787 (#1870) Co-authored-by: Egor Skorokhodov * Beachfront: Fix for bid floor issue#1787 (#1878) Co-authored-by: jim naumann * Updating interactiveoffers contact info (#1881) * Docs metrics configuration (#1850) * Criteo: update maintainer email address (#1884) * New Adapter: BrightMountainMedia (#1855) New Adapter : BrightMountainMedia * New Adapter: AlgoriX (#1861) * Remove LifeStreet + Legacy Cleanup (#1883) * New Adapter: E-Volution (#1868) * [criteo] accept zoneId and networkId alternate case (#1869) * Request Provided Currency Rates (#1753) * Debug override header (#1853) * Remove GDPR TCF1 (#1854) * Rename GDPR UserSyncIfAmbiguous to DefaultValue (#1858) * Accept bidfloor from impression to fix issue #1787 for sovrn adapter (#1886) * GDPR: require host specify default value (#1859) * New Adapter: Smile Wanted (#1877) * New Adapter: Smile Wanted * Implementations of changes requested by : https://github.com/prebid/prebid-server/pull/1877#pullrequestreview-676968474 * Improvement of test coverage as requested. * Implementations of changes requested by : https://github.com/prebid/prebid-server/pull/1877#pullrequestreview-683853119 * Fix a weak vendor enforcement bug where vendor does not exist (#1890) * Update To Go 1.16 (#1888) * Friendlier Startup Error Messages (#1894) * Second fix for weak vendor enforcement (#1896) * Rubicon: hardcode EUR to USD for floors (#1899) Co-authored-by: Serhii Nahornyi * Outbrain adapter: overwrite tagid only if it exists (#1895) * New Adapter: BidMyAdz (#1882) Co-authored-by: BidMyAdz * Currency Conversion Utility Function (#1901) * New Adapter: SA Lunamedia (#1891) * Removed Digitrust From Prebid Server (#1892) Co-authored-by: avolcy * IX: merge eventtrackers with imptrackers for native bid responses (#1900) * Inmobi: user sync (#1911) * Rubicon: Update segtax logic (#1909) Co-authored-by: Serhii Nahornyi * New Adapter: Axonix (#1912) * New Axonix adapter * Changed endpoint * Rename adapter type * Leave in examplary only the basic test fixtures * PR comments * Rubicon: Fix Nil Reference Panic (#1918) * git rebase * Reverted some changes after prebid-server upgrade * Fixed ctv_auction.go after merging prebid-0.170.0 * Added missing gdpr.default_value * Updated usersync url for bidder Unruly Co-authored-by: el-chuck Co-authored-by: Bernhard Pickenbrock Co-authored-by: Gena Co-authored-by: guscarreon Co-authored-by: Gus Carreon Co-authored-by: bretg Co-authored-by: Scott Kay Co-authored-by: prebidtappx <77485538+prebidtappx@users.noreply.github.com> Co-authored-by: ubuntu Co-authored-by: Albert Grandes Co-authored-by: Steve Alliance Co-authored-by: steve-a-districtm Co-authored-by: Pavel Dunyashev Co-authored-by: Benjamin Co-authored-by: Brian Sardo <1168933+bsardo@users.noreply.github.com> Co-authored-by: Przemysław Iwańczak <36727380+piwanczak@users.noreply.github.com> Co-authored-by: Przemyslaw Iwanczak Co-authored-by: Veronika Solovei Co-authored-by: hhhjort <31041505+hhhjort@users.noreply.github.com> Co-authored-by: Rok Sušnik Co-authored-by: Rok Sušnik Co-authored-by: ixjohnny <75964135+ixjohnny@users.noreply.github.com> Co-authored-by: Michael Burns Co-authored-by: Mike Burns Co-authored-by: guiann Co-authored-by: Laurentiu Badea Co-authored-by: Arne Schulz Co-authored-by: adxcgcom <31470944+adxcgcom@users.noreply.github.com> Co-authored-by: agilfix Co-authored-by: TheMediaGrid <44166371+TheMediaGrid@users.noreply.github.com> Co-authored-by: Jurij Sinickij Co-authored-by: mefjush Co-authored-by: dtbarne <7635750+dtbarne@users.noreply.github.com> Co-authored-by: Pillsoo Shin Co-authored-by: Daniel Lawrence Co-authored-by: epomrnd Co-authored-by: Vasyl Zarva Co-authored-by: Marcin Muras <47107445+mmuras@users.noreply.github.com> Co-authored-by: IOTiagoFaria <76956619+IOTiagoFaria@users.noreply.github.com> Co-authored-by: notmani Co-authored-by: johnwier <49074029+johnwier@users.noreply.github.com> Co-authored-by: jcamp-revc <68560678+jcamp-revc@users.noreply.github.com> Co-authored-by: Raghu Teja <2473294+raghuteja@users.noreply.github.com> Co-authored-by: Jim Naumann Co-authored-by: jim naumann Co-authored-by: Hengsheng Cai Co-authored-by: hcai Co-authored-by: Vladyslav Laktionov Co-authored-by: vlad Co-authored-by: Ruslan Sibgatullin Co-authored-by: Vivek Narang Co-authored-by: vladi-mmg Co-authored-by: Vladi Izgayev Co-authored-by: egsk Co-authored-by: Egor Skorokhodov Co-authored-by: timoshas Co-authored-by: Léonard Labat Co-authored-by: BrightMountainMedia <69471268+BrightMountainMediaInc@users.noreply.github.com> Co-authored-by: Bugxyb Co-authored-by: e-volution-tech <61746103+e-volution-tech@users.noreply.github.com> Co-authored-by: Léonard Labat Co-authored-by: Veronika Solovei Co-authored-by: Rachel Joyce Co-authored-by: Maxime DEYMÈS <47388595+MaxSmileWanted@users.noreply.github.com> Co-authored-by: Serhii Nahornyi Co-authored-by: Serhii Nahornyi Co-authored-by: bidmyadz <82382704+bidmyadz@users.noreply.github.com> Co-authored-by: BidMyAdz Co-authored-by: lunamedia <73552749+lunamedia@users.noreply.github.com> Co-authored-by: AlexBVolcy <74930484+AlexBVolcy@users.noreply.github.com> Co-authored-by: avolcy Co-authored-by: Mani Gandham Co-authored-by: armon823 <86739148+armon823@users.noreply.github.com> Co-authored-by: César Fernández Co-authored-by: jizeyopera <70930512+jizeyopera@users.noreply.github.com> Co-authored-by: Mansi Nahar Co-authored-by: Jim Naumann Co-authored-by: Eddy Pechuzal <46331062+epechuzal@users.noreply.github.com> Co-authored-by: avolokha <84977155+avolokha@users.noreply.github.com> Co-authored-by: Olivier Co-authored-by: Joshua Gross <820727+grossjo@users.noreply.github.com> Co-authored-by: Joshua Gross Co-authored-by: evanmsmrtb Co-authored-by: Viacheslav Chimishuk Co-authored-by: rhaksi-kidoz <61601767+rhaksi-kidoz@users.noreply.github.com> Co-authored-by: Ryan Haksi Co-authored-by: Aadesh Co-authored-by: Aadesh Patel Co-authored-by: Mike Chowla Co-authored-by: Jimmy Tu Co-authored-by: Mirko Feddern <3244291+mirkorean@users.noreply.github.com> Co-authored-by: Alexander Pinnecke Co-authored-by: Alex Klinkert Co-authored-by: Mirko Feddern Co-authored-by: Artur Aleksanyan Co-authored-by: Richard Lee <14349+dlackty@users.noreply.github.com> Co-authored-by: Simon Critchley Co-authored-by: tadam75 Co-authored-by: tadam Co-authored-by: gpolaert Co-authored-by: Amaury Ravanel Co-authored-by: Vikram Co-authored-by: vikram Co-authored-by: Stephan Co-authored-by: Adprime <64427228+Adprime@users.noreply.github.com> Co-authored-by: Aiholkin Co-authored-by: Alexey Elymanov Co-authored-by: Alexey Elymanov Co-authored-by: Kushneryk Pavel Co-authored-by: Kushneryk Pavlo Co-authored-by: user Co-authored-by: Daniel Barrigas Co-authored-by: Cameron Rice <37162584+camrice@users.noreply.github.com> Co-authored-by: AcuityAdsIntegrations <72594990+AcuityAdsIntegrations@users.noreply.github.com> Co-authored-by: Winston-Yieldmo <46379634+Winston-Yieldmo@users.noreply.github.com> Co-authored-by: Winston Co-authored-by: Aparna Rao Co-authored-by: Gus Carreon Co-authored-by: mobfxoHB <74364234+mobfxoHB@users.noreply.github.com> --- README.md | 4 +- adapters/appnexus/appnexus.go | 4 +- .../video-same-adpodid-two-imps-same-pod.json | 1 - .../beachfronttest/exemplary/banner.json | 1 - .../supplemental/test_params.json | 8 +- adapters/pubmatic/pubmatic.go | 6 +- analytics/config/config_test.go | 3 +- analytics/filesystem/file_module_test.go | 3 +- config/config.go | 12 +- config/config_test.go | 2 +- endpoints/events/vtrack_test.go | 2 +- endpoints/openrtb2/auction.go | 8 - endpoints/openrtb2/ctv_auction.go | 8 +- errortypes/code.go | 1 + exchange/adapter_builders.go | 2 + exchange/events.go | 4 +- exchange/exchange.go | 54 +- exchange/exchange_test.go | 536 +++++++++++++++++- exchange/utils.go | 10 + go.sum | 11 + main.go | 2 +- metrics/config/metrics.go | 31 +- metrics/go_metrics.go | 29 + metrics/metrics.go | 34 ++ metrics/metrics_mock.go | 30 + metrics/prometheus/prometheus.go | 121 ++-- metrics/prometheus/prometheus_test.go | 1 + openrtb_ext/bidders.go | 2 + openrtb_ext/request.go | 4 + privacy/gdpr/policy_test.go | 2 +- router/router.go | 4 + usersync/usersyncers/syncer_test.go | 2 + 32 files changed, 832 insertions(+), 110 deletions(-) diff --git a/README.md b/README.md index 64486fd4393..eaadc55dfbd 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,7 @@ of exported types. Want to [add an adapter](https://docs.prebid.org/prebid-server/developers/add-new-bidder-go.html)? Found a bug? Great! -Report bugs, request features, and suggest improvements [on Github](https://github.com/prebid/prebid-server/issues). - -Or better yet, [open a pull request](https://github.com/prebid/prebid-server/compare) with the changes you'd like to see. +Or better yet, [open a pull request](https://github.com/PubMatic-OpenWrap/prebid-server/compare) with the changes you'd like to see. ## IDE Recommendations diff --git a/adapters/appnexus/appnexus.go b/adapters/appnexus/appnexus.go index 4ce46372fab..004e275cc8e 100644 --- a/adapters/appnexus/appnexus.go +++ b/adapters/appnexus/appnexus.go @@ -374,13 +374,15 @@ func (a *AppNexusAdapter) MakeRequests(request *openrtb2.BidRequest, reqInfo *ad imps := request.Imp + // Commenting out the following piece of code to avoid populating adpod_id in the Appnexus request (ref: https://inside.pubmatic.com:9443/jira/browse/UOE-6196) + // For long form requests if adpodId feature enabled, adpod_id must be sent downstream. // Adpod id is a unique identifier for pod // All impressions in the same pod must have the same pod id in request extension // For this all impressions in request should belong to the same pod // If impressions number per pod is more than maxImpsPerReq - divide those imps to several requests but keep pod id the same // If adpodId feature disabled and impressions number per pod is more than maxImpsPerReq - divide those imps to several requests but do not include ad pod id - if isVIDEO == 1 && *adPodId { + /*if isVIDEO == 1 && *adPodId { podImps := groupByPods(imps) requests := make([]*adapters.RequestData, 0, len(podImps)) diff --git a/adapters/appnexus/appnexustest/video/video-same-adpodid-two-imps-same-pod.json b/adapters/appnexus/appnexustest/video/video-same-adpodid-two-imps-same-pod.json index 5a453979f7c..d0940a2345d 100644 --- a/adapters/appnexus/appnexustest/video/video-same-adpodid-two-imps-same-pod.json +++ b/adapters/appnexus/appnexustest/video/video-same-adpodid-two-imps-same-pod.json @@ -47,7 +47,6 @@ "id": "test-request-id", "ext": { "appnexus": { - "adpod_id": "5577006791947779410", "hb_source": 6 }, "prebid": {} diff --git a/adapters/beachfront/beachfronttest/exemplary/banner.json b/adapters/beachfront/beachfronttest/exemplary/banner.json index 11db97285a7..d42a6d36181 100644 --- a/adapters/beachfront/beachfronttest/exemplary/banner.json +++ b/adapters/beachfront/beachfronttest/exemplary/banner.json @@ -24,7 +24,6 @@ } ] }, - "httpCalls": [ { "expectedRequest": { diff --git a/adapters/conversant/conversanttest/supplemental/test_params.json b/adapters/conversant/conversanttest/supplemental/test_params.json index 403bcc42226..cf71299df0f 100644 --- a/adapters/conversant/conversanttest/supplemental/test_params.json +++ b/adapters/conversant/conversanttest/supplemental/test_params.json @@ -107,7 +107,7 @@ "bidfloor": 7, "secure": 1, "tagid": "mytag", - "displaymanager": "prebid-s2s", + "displaymanager": "pubmatic-openwrap", "displaymanagerver": "2.0.0", "video": { "api": [1,2], @@ -126,7 +126,7 @@ "bidfloor": 1, "secure": 1, "tagid": "mytag", - "displaymanager": "prebid-s2s", + "displaymanager": "pubmatic-openwrap", "displaymanagerver": "2.0.0", "video": { "api": [1,2], @@ -154,7 +154,7 @@ "bidfloor": 7, "secure": 1, "tagid": "mytag", - "displaymanager": "prebid-s2s", + "displaymanager": "pubmatic-openwrap", "displaymanagerver": "2.0.0", "video": { "api": [1,2], @@ -182,7 +182,7 @@ "bidfloor": -3, "secure": 1, "tagid": "mytag", - "displaymanager": "prebid-s2s", + "displaymanager": "pubmatic-openwrap", "displaymanagerver": "2.0.0", "video": { "api": [1,2], diff --git a/adapters/pubmatic/pubmatic.go b/adapters/pubmatic/pubmatic.go index e01d3bda3bc..58123443c35 100644 --- a/adapters/pubmatic/pubmatic.go +++ b/adapters/pubmatic/pubmatic.go @@ -489,6 +489,10 @@ func assignBannerSize(banner *openrtb2.Banner) (*openrtb2.Banner, error) { return banner, nil } + if len(banner.Format) == 0 { + return nil, errors.New(fmt.Sprintf("No sizes provided for Banner %v", banner.Format)) + } + return assignBannerWidthAndHeight(banner, banner.Format[0].W, banner.Format[0].H), nil } @@ -545,7 +549,7 @@ func parseImpressionObject(imp *openrtb2.Imp, wrapExt *string, pubID *string) er imp.Banner = bannerCopy } - extMap := make(map[string]interface{}, 0) + impExtMap := make(map[string]interface{}, 0) if pubmaticExt.Keywords != nil && len(pubmaticExt.Keywords) != 0 { addKeywordsToExt(pubmaticExt.Keywords, extMap) } diff --git a/analytics/config/config_test.go b/analytics/config/config_test.go index 310dbe1a481..fe95a66ed6f 100644 --- a/analytics/config/config_test.go +++ b/analytics/config/config_test.go @@ -1,13 +1,12 @@ package config import ( + "github.com/stretchr/testify/assert" "net/http" "os" "testing" "github.com/mxmCherry/openrtb/v15/openrtb2" - "github.com/stretchr/testify/assert" - "github.com/prebid/prebid-server/analytics" "github.com/prebid/prebid-server/config" ) diff --git a/analytics/filesystem/file_module_test.go b/analytics/filesystem/file_module_test.go index 0c3d3c9e6ac..45a72266569 100644 --- a/analytics/filesystem/file_module_test.go +++ b/analytics/filesystem/file_module_test.go @@ -1,14 +1,13 @@ package filesystem import ( + "github.com/prebid/prebid-server/config" "net/http" "os" "strings" "testing" "github.com/mxmCherry/openrtb/v15/openrtb2" - "github.com/prebid/prebid-server/config" - "github.com/prebid/prebid-server/analytics" "github.com/prebid/prebid-server/usersync" ) diff --git a/config/config.go b/config/config.go index aa15c771923..cb8278678d3 100644 --- a/config/config.go +++ b/config/config.go @@ -654,7 +654,7 @@ func (cfg *Configuration) setDerivedDefaults() { setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderGumGum, "https://rtb.gumgum.com/usync/prbds2s?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dgumgum%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D") setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderImprovedigital, "https://ad.360yield.com/server_match?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&r="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dimprovedigital%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7BPUB_USER_ID%7D") setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderInMobi, "https://id5-sync.com/i/495/0.gif?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&callback="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dinmobi%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%7BID5UID%7D") - setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderIx, "https://ssum.casalemedia.com/usermatchredir?s=194962&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&cb="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dix%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D") + setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderIx, "https://ssum.casalemedia.com/usermatchredir?s=186523&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&cb="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dix%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D") // openrtb_ext.BidderInvibes doesn't have a good default. setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderJixie, "https://id.jixie.io/api/sync?pid=&gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redirect="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Djixie%26gdpr%3D{{.GDPR}}%26gdpr_consent%3D{{.GDPRConsent}}%26uid%3D%25%25JXUID%25%25") setDefaultUsersync(cfg.Adapters, openrtb_ext.BidderKrushmedia, "https://cs.krushmedia.com/4e4abdd5ecc661643458a730b1aa927d.gif?gdpr={{.GDPR}}&gdpr_consent={{.GDPRConsent}}&us_privacy={{.USPrivacy}}&redir="+url.QueryEscape(externalURL)+"%2Fsetuid%3Fbidder%3Dkrushmedia%26uid%3D%5BUID%5D") @@ -753,6 +753,10 @@ func SetupViper(v *viper.Viper, filename string) { v.SetDefault("http_client.max_idle_connections", 400) v.SetDefault("http_client.max_idle_connections_per_host", 10) v.SetDefault("http_client.idle_connection_timeout_seconds", 60) + v.SetDefault("http_client_cache.max_connections_per_host", 0) // unlimited + v.SetDefault("http_client_cache.max_idle_connections", 10) + v.SetDefault("http_client_cache.max_idle_connections_per_host", 2) + v.SetDefault("http_client_cache.idle_connection_timeout_seconds", 60) v.SetDefault("http_client.tls_handshake_timeout", 0) //no timeout v.SetDefault("http_client.response_header_timeout", 0) //unlimited v.SetDefault("http_client.dial_timeout", 0) //no timeout @@ -765,6 +769,7 @@ func SetupViper(v *viper.Viper, filename string) { v.SetDefault("metrics.disabled_metrics.account_adapter_details", false) v.SetDefault("metrics.disabled_metrics.adapter_connections_metrics", true) v.SetDefault("metrics.disabled_metrics.adapter_gdpr_request_blocked", false) + v.SetDefault("metrics.disabled_metrics.adapter_connections_metrics", true) v.SetDefault("metrics.influxdb.host", "") v.SetDefault("metrics.influxdb.database", "") v.SetDefault("metrics.influxdb.username", "") @@ -916,7 +921,8 @@ func SetupViper(v *viper.Viper, filename string) { v.SetDefault("adapters.improvedigital.endpoint", "http://ad.360yield.com/pbs") v.SetDefault("adapters.inmobi.endpoint", "https://api.w.inmobi.com/showad/openrtb/bidder/prebid") v.SetDefault("adapters.interactiveoffers.endpoint", "https://rtb.ioadx.com/bidRequest/?partnerId=d9e56d418c4825d466ee96c7a31bf1da6b62fa04") - v.SetDefault("adapters.ix.disabled", true) + v.SetDefault("adapters.ix.disabled", false) + v.SetDefault("adapters.ix.endpoint", "http://exchange.indexww.com/pbs?p=192919") v.SetDefault("adapters.jixie.endpoint", "https://hb.jixie.io/v2/hbsvrpost") v.SetDefault("adapters.kayzen.endpoint", "https://bids-{{.ZoneID}}.bidder.kayzen.io/?exchange={{.AccountID}}") v.SetDefault("adapters.krushmedia.endpoint", "http://ads4.krushmedia.com/?c=rtb&m=req&key={{.AccountID}}") @@ -996,6 +1002,8 @@ func SetupViper(v *viper.Viper, filename string) { v.BindEnv("gdpr.default_value") v.SetDefault("gdpr.enabled", true) v.SetDefault("gdpr.host_vendor_id", 0) + v.SetDefault("gdpr.default_value", "0") + v.SetDefault("gdpr.usersync_if_ambiguous", true) v.SetDefault("gdpr.timeouts_ms.init_vendorlist_fetches", 0) v.SetDefault("gdpr.timeouts_ms.active_vendorlist_fetch", 0) v.SetDefault("gdpr.non_standard_publishers", []string{""}) diff --git a/config/config_test.go b/config/config_test.go index a87d65af359..f5fbbfa341f 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -115,7 +115,7 @@ func TestExternalCacheURLValidate(t *testing.T) { } } -func TestDefaults(t *testing.T) { + func TestDefaults(t *testing.T) { cfg, _ := newDefaultConfig(t) cmpInts(t, "port", cfg.Port, 8000) diff --git a/endpoints/events/vtrack_test.go b/endpoints/events/vtrack_test.go index 6f290b22499..94a81b00d20 100644 --- a/endpoints/events/vtrack_test.go +++ b/endpoints/events/vtrack_test.go @@ -938,7 +938,7 @@ func TestGetVideoEventTracking(t *testing.T) { name: "valid_scenario", args: args{ trackerURL: "http://company.tracker.com?eventId=[EVENT_ID]&appbundle=[DOMAIN]", - bid: &openrtb2.Bid{ + bid: &openrtb2.Bid{ // AdM: vastXMLWith2Creatives, }, req: &openrtb2.BidRequest{ diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index 8757190d703..434ac180225 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -375,14 +375,6 @@ func (deps *endpointDeps) validateRequest(req *openrtb_ext.RequestWrapper) []err if err := validateCustomRates(reqPrebid.CurrencyConversions); err != nil { return []error{err} } - - if err := validateSChains(bidExt); err != nil { - return []error{err} - } - - if err := deps.validateEidPermissions(bidExt, aliases); err != nil { - return []error{err} - } } if (req.Site == nil && req.App == nil) || (req.Site != nil && req.App != nil) { diff --git a/endpoints/openrtb2/ctv_auction.go b/endpoints/openrtb2/ctv_auction.go index 067c340c964..c167641b793 100644 --- a/endpoints/openrtb2/ctv_auction.go +++ b/endpoints/openrtb2/ctv_auction.go @@ -105,8 +105,9 @@ func NewCTVEndpoint( func (deps *ctvEndpointDeps) CTVAuctionEndpoint(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { defer util.TimeTrack(time.Now(), "CTVAuctionEndpoint") - var request *openrtb.BidRequest - var response *openrtb.BidResponse + var reqWrapper *openrtb_ext.RequestWrapper + var request *openrtb2.BidRequest + var response *openrtb2.BidResponse var err error var errL []error @@ -137,10 +138,11 @@ func (deps *ctvEndpointDeps) CTVAuctionEndpoint(w http.ResponseWriter, r *http.R }() //Parse ORTB Request and do Standard Validation - request, errL = deps.parseRequest(r) + reqWrapper, errL = deps.parseRequest(r) if errortypes.ContainsFatalError(errL) && writeError(errL, w, &deps.labels) { return } + request = reqWrapper.BidRequest util.JLogf("Original BidRequest", request) //TODO: REMOVE LOG diff --git a/errortypes/code.go b/errortypes/code.go index 869e7d541a4..554357ea88a 100644 --- a/errortypes/code.go +++ b/errortypes/code.go @@ -12,6 +12,7 @@ const ( BlacklistedAcctErrorCode AcctRequiredErrorCode NoConversionRateErrorCode + NoBidPriceErrorCode ) // Defines numeric codes for well-known warnings. diff --git a/exchange/adapter_builders.go b/exchange/adapter_builders.go index a49e8161596..8606bc84b86 100755 --- a/exchange/adapter_builders.go +++ b/exchange/adapter_builders.go @@ -231,6 +231,7 @@ func newAdapterBuilders() map[openrtb_ext.BidderName]adapters.Builder { openrtb_ext.BidderSomoaudience: somoaudience.Builder, openrtb_ext.BidderSonobi: sonobi.Builder, openrtb_ext.BidderSovrn: sovrn.Builder, + openrtb_ext.BidderSpotX: spotx.Builder, openrtb_ext.BidderSynacormedia: synacormedia.Builder, openrtb_ext.BidderTappx: tappx.Builder, openrtb_ext.BidderTelaria: telaria.Builder, @@ -241,6 +242,7 @@ func newAdapterBuilders() map[openrtb_ext.BidderName]adapters.Builder { openrtb_ext.BidderUnicorn: unicorn.Builder, openrtb_ext.BidderUnruly: unruly.Builder, openrtb_ext.BidderValueImpression: valueimpression.Builder, + openrtb_ext.BidderVASTBidder: vastbidder.Builder, openrtb_ext.BidderVerizonMedia: verizonmedia.Builder, openrtb_ext.BidderViewdeos: adtelligent.Builder, openrtb_ext.BidderVisx: visx.Builder, diff --git a/exchange/events.go b/exchange/events.go index 9742e50e424..06f26b7e333 100644 --- a/exchange/events.go +++ b/exchange/events.go @@ -2,10 +2,10 @@ package exchange import ( "encoding/json" + "github.com/mxmCherry/openrtb/v15/openrtb2" "time" - jsonpatch "github.com/evanphx/json-patch" - "github.com/mxmCherry/openrtb/v15/openrtb2" + "github.com/evanphx/json-patch" "github.com/prebid/prebid-server/analytics" "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/endpoints/events" diff --git a/exchange/exchange.go b/exchange/exchange.go index 84fb8d41c36..d0dad2b73bd 100644 --- a/exchange/exchange.go +++ b/exchange/exchange.go @@ -64,6 +64,7 @@ type exchange struct { privacyConfig config.Privacy categoriesFetcher stored_requests.CategoryFetcher bidIDGenerator BidIDGenerator + trakerURL string } // Container to pass out response ext data from the GetAllBids goroutines back into the main thread @@ -228,7 +229,7 @@ func (e *exchange) HoldAuction(ctx context.Context, r AuctionRequest, debugLog * //If includebrandcategory is present in ext then CE feature is on. if requestExt.Prebid.Targeting != nil && requestExt.Prebid.Targeting.IncludeBrandCategory != nil { var rejections []string - bidCategory, adapterBids, rejections, err = applyCategoryMapping(ctx, requestExt, adapterBids, e.categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) + bidCategory, adapterBids, rejections, err = applyCategoryMapping(ctx, r.BidRequest, requestExt, adapterBids, e.categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) if err != nil { return nil, fmt.Errorf("Error in category mapping : %s", err.Error()) } @@ -517,7 +518,6 @@ func (e *exchange) getAllBids( bidsFound = true bidIDsCollision = recordAdaptorDuplicateBidIDs(e.me, adapterBids) } - } if bidIDsCollision { // record this request count this request if bid collision is detected @@ -662,7 +662,7 @@ func encodeBidResponseExt(bidResponseExt *openrtb_ext.ExtBidResponse) ([]byte, e return buffer.Bytes(), err } -func applyCategoryMapping(ctx context.Context, requestExt *openrtb_ext.ExtRequest, seatBids map[openrtb_ext.BidderName]*pbsOrtbSeatBid, categoriesFetcher stored_requests.CategoryFetcher, targData *targetData, booleanGenerator deduplicateChanceGenerator) (map[string]string, map[openrtb_ext.BidderName]*pbsOrtbSeatBid, []string, error) { +func applyCategoryMapping(ctx context.Context, bidRequest *openrtb2.BidRequest, requestExt *openrtb_ext.ExtRequest, seatBids map[openrtb_ext.BidderName]*pbsOrtbSeatBid, categoriesFetcher stored_requests.CategoryFetcher, targData *targetData, booleanGenerator deduplicateChanceGenerator) (map[string]string, map[openrtb_ext.BidderName]*pbsOrtbSeatBid, []string, error) { res := make(map[string]string) type bidDedupe struct { @@ -796,19 +796,20 @@ func applyCategoryMapping(ctx context.Context, requestExt *openrtb_ext.ExtReques if !brandCatExt.SkipDedup { if dupe, ok := dedupe[dupeKey]; ok { - dupeBidPrice, err := strconv.ParseFloat(dupe.bidPrice, 64) - if err != nil { - dupeBidPrice = 0 - } - currBidPrice, err := strconv.ParseFloat(pb, 64) - if err != nil { - currBidPrice = 0 - } - if dupeBidPrice == currBidPrice { - if booleanGenerator.Generate() { - dupeBidPrice = -1 - } else { - currBidPrice = -1 + dupeBidPrice, err := strconv.ParseFloat(dupe.bidPrice, 64) + if err != nil { + dupeBidPrice = 0 + } + currBidPrice, err := strconv.ParseFloat(pb, 64) + if err != nil { + currBidPrice = 0 + } + if dupeBidPrice == currBidPrice { + if booleanGenerator.Generate() { + dupeBidPrice = -1 + } else { + currBidPrice = -1 + } } if dupeBidPrice < currBidPrice { @@ -823,12 +824,19 @@ func applyCategoryMapping(ctx context.Context, requestExt *openrtb_ext.ExtReques if len(oldSeatBid.bids) == 1 { seatBidsToRemove = append(seatBidsToRemove, dupe.bidderName) } else { - // This is a very rare, but still possible case where bid needs to be removed from already processed bidder - // This happens when current processing bidder has a bid that has same deduplication key as a bid from already processed bidder - // and already processed bid was selected to be removed - // See example of input data in unit test `TestCategoryMappingTwoBiddersManyBidsEachNoCategorySamePrice` - // Need to remove bid by name, not index in this case - removeBidById(oldSeatBid, dupe.bidID) + // An older bid from a different seatBid we've already finished with + oldSeatBid := (seatBids)[dupe.bidderName] + rejections = updateRejections(rejections, dupe.bidID, "Bid was deduplicated") + if len(oldSeatBid.bids) == 1 { + seatBidsToRemove = append(seatBidsToRemove, dupe.bidderName) + } else { + // This is a very rare, but still possible case where bid needs to be removed from already processed bidder + // This happens when current processing bidder has a bid that has same deduplication key as a bid from already processed bidder + // and already processed bid was selected to be removed + // See example of input data in unit test `TestCategoryMappingTwoBiddersManyBidsEachNoCategorySamePrice` + // Need to remove bid by name, not index in this case + removeBidById(oldSeatBid, dupe.bidID) + } } delete(res, dupe.bidID) } else { @@ -838,9 +846,9 @@ func applyCategoryMapping(ctx context.Context, requestExt *openrtb_ext.ExtReques continue } } - dedupe[dupeKey] = bidDedupe{bidderName: bidderName, bidIndex: bidInd, bidID: bidID, bidPrice: pb} } res[bidID] = categoryDuration + dedupe[dupeKey] = bidDedupe{bidderName: bidderName, bidIndex: bidInd, bidID: bidID, bidPrice: pb} } if len(bidsToRemove) > 0 { diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go index bb05b7c0e7f..46dbc0820dc 100644 --- a/exchange/exchange_test.go +++ b/exchange/exchange_test.go @@ -2599,7 +2599,7 @@ func TestCategoryMapping(t *testing.T) { adapterBids[bidderName1] = &seatBid - bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) + bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &bidRequest, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) assert.Equal(t, nil, err, "Category mapping error should be empty") assert.Equal(t, 1, len(rejections), "There should be 1 bid rejection message") @@ -2655,7 +2655,7 @@ func TestCategoryMappingNoIncludeBrandCategory(t *testing.T) { adapterBids[bidderName1] = &seatBid - bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) + bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &bidRequest, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) assert.Equal(t, nil, err, "Category mapping error should be empty") assert.Empty(t, rejections, "There should be no bid rejection messages") @@ -2708,7 +2708,7 @@ func TestCategoryMappingTranslateCategoriesNil(t *testing.T) { adapterBids[bidderName1] = &seatBid - bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) + bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &bidRequest, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) assert.Equal(t, nil, err, "Category mapping error should be empty") assert.Equal(t, 1, len(rejections), "There should be 1 bid rejection message") @@ -2791,7 +2791,7 @@ func TestCategoryMappingTranslateCategoriesFalse(t *testing.T) { adapterBids[bidderName1] = &seatBid - bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) + bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &bidRequest, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) assert.Equal(t, nil, err, "Category mapping error should be empty") assert.Empty(t, rejections, "There should be no bid rejection messages") @@ -2862,7 +2862,7 @@ func TestCategoryDedupe(t *testing.T) { adapterBids[bidderName1] = &seatBid - bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) + bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &bidRequest, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) assert.Equal(t, nil, err, "Category mapping error should be empty") assert.Equal(t, 3, len(rejections), "There should be 2 bid rejection messages") @@ -2943,7 +2943,7 @@ func TestNoCategoryDedupe(t *testing.T) { adapterBids[bidderName1] = &seatBid - bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) + bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &bidRequest, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) assert.Equal(t, nil, err, "Category mapping error should be empty") assert.Equal(t, 2, len(rejections), "There should be 2 bid rejection messages") @@ -3009,7 +3009,7 @@ func TestCategoryMappingBidderName(t *testing.T) { adapterBids[bidderName1] = &seatBid1 adapterBids[bidderName2] = &seatBid2 - bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) + bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &bidRequest, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) assert.NoError(t, err, "Category mapping error should be empty") assert.Empty(t, rejections, "There should be 0 bid rejection messages") @@ -3064,7 +3064,7 @@ func TestCategoryMappingBidderNameNoCategories(t *testing.T) { adapterBids[bidderName1] = &seatBid1 adapterBids[bidderName2] = &seatBid2 - bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) + bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &bidRequest, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) assert.NoError(t, err, "Category mapping error should be empty") assert.Empty(t, rejections, "There should be 0 bid rejection messages") @@ -3164,8 +3164,9 @@ func TestBidRejectionErrors(t *testing.T) { seatBid := pbsOrtbSeatBid{bids: innerBids, currency: "USD"} adapterBids[bidderName] = &seatBid + bidRequest := openrtb2.BidRequest{} - bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &test.reqExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) + bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &bidRequest, &test.reqExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) if len(test.expectedCatDur) > 0 { // Bid deduplication case @@ -3229,7 +3230,7 @@ func TestCategoryMappingTwoBiddersOneBidEachNoCategorySamePrice(t *testing.T) { adapterBids[bidderNameApn1] = &seatBidApn1 adapterBids[bidderNameApn2] = &seatBidApn2 - bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) + bidCategory, adapterBids, rejections, err := applyCategoryMapping(nil, &bidRequest, &requestExt, adapterBids, categoriesFetcher, targData, &randomDeduplicateBidBooleanGenerator{}) assert.NoError(t, err, "Category mapping error should be empty") assert.Len(t, rejections, 1, "There should be 1 bid rejection message") @@ -3267,6 +3268,7 @@ func TestCategoryMappingTwoBiddersManyBidsEachNoCategorySamePrice(t *testing.T) t.Errorf("Failed to create a category Fetcher: %v", error) } + bidRequest := openrtb2.BidRequest{} requestExt := newExtRequestTranslateCategories(nil) targData := &targetData{ @@ -3313,7 +3315,7 @@ func TestCategoryMappingTwoBiddersManyBidsEachNoCategorySamePrice(t *testing.T) adapterBids[bidderNameApn1] = &seatBidApn1 adapterBids[bidderNameApn2] = &seatBidApn2 - _, adapterBids, rejections, err := applyCategoryMapping(nil, &requestExt, adapterBids, categoriesFetcher, targData, &fakeRandomDeduplicateBidBooleanGenerator{true}) + _, adapterBids, rejections, err := applyCategoryMapping(nil, &bidRequest, &requestExt, adapterBids, categoriesFetcher, targData, &fakeRandomDeduplicateBidBooleanGenerator{true}) assert.NoError(t, err, "Category mapping error should be empty") @@ -4025,3 +4027,515 @@ func (m *mockBidder) MakeBids(internalRequest *openrtb2.BidRequest, externalRequ args := m.Called(internalRequest, externalRequest, response) return args.Get(0).(*adapters.BidderResponse), args.Get(1).([]error) } + +//TestApplyAdvertiserBlocking verifies advertiser blocking +//Currently it is expected to work only with TagBidders and not woth +// normal bidders +func TestApplyAdvertiserBlocking(t *testing.T) { + type args struct { + advBlockReq *openrtb2.BidRequest + adaptorSeatBids map[*bidderAdapter]*pbsOrtbSeatBid // bidder adaptor and its dummy seat bids map + } + type want struct { + rejectedBidIds []string + validBidCountPerSeat map[string]int + } + tests := []struct { + name string + args args + want want + }{ + { + name: "reject_bid_of_blocked_adv_from_tag_bidder", + args: args{ + advBlockReq: &openrtb2.BidRequest{ + BAdv: []string{"a.com"}, // block bids returned by a.com + }, + adaptorSeatBids: map[*bidderAdapter]*pbsOrtbSeatBid{ + newTestTagAdapter("vast_tag_bidder"): { // tag bidder returning 1 bid from blocked advertiser + bids: []*pbsOrtbBid{ + { + bid: &openrtb2.Bid{ + ID: "a.com_bid", + ADomain: []string{"a.com"}, + }, + }, + { + bid: &openrtb2.Bid{ + ID: "b.com_bid", + ADomain: []string{"b.com"}, + }, + }, + { + bid: &openrtb2.Bid{ + ID: "keep_ba.com", + ADomain: []string{"ba.com"}, + }, + }, + { + bid: &openrtb2.Bid{ + ID: "keep_ba.com", + ADomain: []string{"b.a.com.shri.com"}, + }, + }, + { + bid: &openrtb2.Bid{ + ID: "reject_b.a.com.a.com.b.c.d.a.com", + ADomain: []string{"b.a.com.a.com.b.c.d.a.com"}, + }, + }, + }, + bidderCoreName: openrtb_ext.BidderVASTBidder, + }, + }, + }, + want: want{ + rejectedBidIds: []string{"a.com_bid", "reject_b.a.com.a.com.b.c.d.a.com"}, + validBidCountPerSeat: map[string]int{ + "vast_tag_bidder": 3, + }, + }, + }, + { + name: "Badv_is_not_present", // expect no advertiser blocking + args: args{ + advBlockReq: &openrtb2.BidRequest{BAdv: nil}, + adaptorSeatBids: map[*bidderAdapter]*pbsOrtbSeatBid{ + newTestTagAdapter("tab_bidder_1"): { + bids: []*pbsOrtbBid{ + {bid: &openrtb2.Bid{ID: "bid_1_adapter_1", ADomain: []string{"a.com"}}}, + {bid: &openrtb2.Bid{ID: "bid_2_adapter_1"}}, + }, + }, + }, + }, + want: want{ + rejectedBidIds: []string{}, // no bid rejection expected + validBidCountPerSeat: map[string]int{ + "tab_bidder_1": 2, + }, + }, + }, + { + name: "adomain_is_not_present_but_Badv_is_set", // reject bids without adomain as badv is set + args: args{ + advBlockReq: &openrtb2.BidRequest{BAdv: []string{"advertiser_1.com"}}, + adaptorSeatBids: map[*bidderAdapter]*pbsOrtbSeatBid{ + newTestTagAdapter("tag_bidder_1"): { + bids: []*pbsOrtbBid{ // expect all bids are rejected + {bid: &openrtb2.Bid{ID: "bid_1_adapter_1_without_adomain"}}, + {bid: &openrtb2.Bid{ID: "bid_2_adapter_1_with_empty_adomain", ADomain: []string{"", " "}}}, + }, + }, + newTestRtbAdapter("rtb_bidder_1"): { + bids: []*pbsOrtbBid{ // all bids should be present. It belongs to RTB adapator + {bid: &openrtb2.Bid{ID: "bid_1_adapter_2_without_adomain"}}, + {bid: &openrtb2.Bid{ID: "bid_2_adapter_2_with_empty_adomain", ADomain: []string{"", " "}}}, + }, + }, + }, + }, + want: want{ + rejectedBidIds: []string{"bid_1_adapter_1_without_adomain", "bid_2_adapter_1_with_empty_adomain"}, + validBidCountPerSeat: map[string]int{ + "tag_bidder_1": 0, // expect 0 bids. i.e. all bids are rejected + "rtb_bidder_1": 2, // no bid must be rejected + }, + }, + }, + { + name: "adomain_and_badv_is_not_present", // expect no advertiser blocking + args: args{ + advBlockReq: &openrtb2.BidRequest{}, + adaptorSeatBids: map[*bidderAdapter]*pbsOrtbSeatBid{ + newTestTagAdapter("tag_adaptor_1"): { + bids: []*pbsOrtbBid{ + {bid: &openrtb2.Bid{ID: "bid_without_adomain"}}, + }, + }, + }, + }, + want: want{ + rejectedBidIds: []string{}, // no rejection expected as badv not present + validBidCountPerSeat: map[string]int{ + "tag_adaptor_1": 1, + }, + }, + }, + { + name: "empty_badv", // expect no advertiser blocking + args: args{ + advBlockReq: &openrtb2.BidRequest{BAdv: []string{}}, + adaptorSeatBids: map[*bidderAdapter]*pbsOrtbSeatBid{ + newTestTagAdapter("tag_bidder_1"): { + bids: []*pbsOrtbBid{ // expect all bids are rejected + {bid: &openrtb2.Bid{ID: "bid_1_adapter_1", ADomain: []string{"a.com"}}}, + {bid: &openrtb2.Bid{ID: "bid_2_adapter_1"}}, + }, + }, + newTestRtbAdapter("rtb_bidder_1"): { + bids: []*pbsOrtbBid{ // all bids should be present. It belongs to RTB adapator + {bid: &openrtb2.Bid{ID: "bid_1_adapter_2_without_adomain"}}, + {bid: &openrtb2.Bid{ID: "bid_2_adapter_2_with_empty_adomain", ADomain: []string{"", " "}}}, + }, + }, + }, + }, + want: want{ + rejectedBidIds: []string{}, // no rejections expect as there is not badv set + validBidCountPerSeat: map[string]int{ + "tag_bidder_1": 2, + "rtb_bidder_1": 2, + }, + }, + }, + { + name: "nil_badv", // expect no advertiser blocking + args: args{ + advBlockReq: &openrtb2.BidRequest{BAdv: nil}, + adaptorSeatBids: map[*bidderAdapter]*pbsOrtbSeatBid{ + newTestTagAdapter("tag_bidder_1"): { + bids: []*pbsOrtbBid{ // expect all bids are rejected + {bid: &openrtb2.Bid{ID: "bid_1_adapter_1", ADomain: []string{"a.com"}}}, + {bid: &openrtb2.Bid{ID: "bid_2_adapter_1"}}, + }, + }, + newTestRtbAdapter("rtb_bidder_1"): { + bids: []*pbsOrtbBid{ // all bids should be present. It belongs to RTB adapator + {bid: &openrtb2.Bid{ID: "bid_1_adapter_2_without_adomain"}}, + {bid: &openrtb2.Bid{ID: "bid_2_adapter_2_with_empty_adomain", ADomain: []string{"", " "}}}, + }, + }, + }, + }, + want: want{ + rejectedBidIds: []string{}, // no rejections expect as there is not badv set + validBidCountPerSeat: map[string]int{ + "tag_bidder_1": 2, + "rtb_bidder_1": 2, + }, + }, + }, + { + name: "ad_domains_normalized_and_checked", + args: args{ + advBlockReq: &openrtb2.BidRequest{BAdv: []string{"a.com"}}, + adaptorSeatBids: map[*bidderAdapter]*pbsOrtbSeatBid{ + newTestTagAdapter("my_adapter"): { + bids: []*pbsOrtbBid{ + {bid: &openrtb2.Bid{ID: "bid_1_of_blocked_adv", ADomain: []string{"www.a.com"}}}, + // expect a.com is extracted from page url + {bid: &openrtb2.Bid{ID: "bid_2_of_blocked_adv", ADomain: []string{"http://a.com/my/page?k1=v1&k2=v2"}}}, + // invalid adomain - will be skipped and the bid will be not be rejected + {bid: &openrtb2.Bid{ID: "bid_3_with_domain_abcd1234", ADomain: []string{"abcd1234"}}}, + }, + }}, + }, + want: want{ + rejectedBidIds: []string{"bid_1_of_blocked_adv", "bid_2_of_blocked_adv"}, + validBidCountPerSeat: map[string]int{"my_adapter": 1}, + }, + }, { + name: "multiple_badv", + args: args{ + advBlockReq: &openrtb2.BidRequest{BAdv: []string{"advertiser_1.com", "advertiser_2.com", "www.advertiser_3.com"}}, + adaptorSeatBids: map[*bidderAdapter]*pbsOrtbSeatBid{ + newTestTagAdapter("tag_adapter_1"): { + bids: []*pbsOrtbBid{ + // adomain without www prefix + {bid: &openrtb2.Bid{ID: "bid_1_tag_adapter_1", ADomain: []string{"advertiser_3.com"}}}, + {bid: &openrtb2.Bid{ID: "bid_2_tag_adapter_1", ADomain: []string{"advertiser_2.com"}}}, + {bid: &openrtb2.Bid{ID: "bid_3_tag_adapter_1", ADomain: []string{"advertiser_4.com"}}}, + {bid: &openrtb2.Bid{ID: "bid_4_tag_adapter_1", ADomain: []string{"advertiser_100.com"}}}, + }, + }, + newTestTagAdapter("tag_adapter_2"): { + bids: []*pbsOrtbBid{ + // adomain has www prefix + {bid: &openrtb2.Bid{ID: "bid_1_tag_adapter_2", ADomain: []string{"www.advertiser_1.com"}}}, + }, + }, + newTestRtbAdapter("rtb_adapter_1"): { + bids: []*pbsOrtbBid{ + // should not reject following bid though its advertiser is blocked + // because this bid belongs to RTB Adaptor + {bid: &openrtb2.Bid{ID: "bid_1_rtb_adapter_2", ADomain: []string{"advertiser_1.com"}}}, + }, + }, + }, + }, + want: want{ + rejectedBidIds: []string{"bid_1_tag_adapter_1", "bid_2_tag_adapter_1", "bid_1_tag_adapter_2"}, + validBidCountPerSeat: map[string]int{ + "tag_adapter_1": 2, + "tag_adapter_2": 0, + "rtb_adapter_1": 1, + }, + }, + }, { + name: "multiple_adomain", + args: args{ + advBlockReq: &openrtb2.BidRequest{BAdv: []string{"www.advertiser_3.com"}}, + adaptorSeatBids: map[*bidderAdapter]*pbsOrtbSeatBid{ + newTestTagAdapter("tag_adapter_1"): { + bids: []*pbsOrtbBid{ + // adomain without www prefix + {bid: &openrtb2.Bid{ID: "bid_1_tag_adapter_1", ADomain: []string{"a.com", "b.com", "advertiser_3.com", "d.com"}}}, + {bid: &openrtb2.Bid{ID: "bid_2_tag_adapter_1", ADomain: []string{"a.com", "https://advertiser_3.com"}}}, + {bid: &openrtb2.Bid{ID: "bid_3_tag_adapter_1", ADomain: []string{"advertiser_4.com"}}}, + {bid: &openrtb2.Bid{ID: "bid_4_tag_adapter_1", ADomain: []string{"advertiser_100.com"}}}, + }, + }, + newTestTagAdapter("tag_adapter_2"): { + bids: []*pbsOrtbBid{ + // adomain has www prefix + {bid: &openrtb2.Bid{ID: "bid_1_tag_adapter_2", ADomain: []string{"a.com", "b.com", "www.advertiser_3.com"}}}, + }, + }, + newTestRtbAdapter("rtb_adapter_1"): { + bids: []*pbsOrtbBid{ + // should not reject following bid though its advertiser is blocked + // because this bid belongs to RTB Adaptor + {bid: &openrtb2.Bid{ID: "bid_1_rtb_adapter_2", ADomain: []string{"a.com", "b.com", "advertiser_3.com"}}}, + }, + }, + }, + }, + want: want{ + rejectedBidIds: []string{"bid_1_tag_adapter_1", "bid_2_tag_adapter_1", "bid_1_tag_adapter_2"}, + validBidCountPerSeat: map[string]int{ + "tag_adapter_1": 2, + "tag_adapter_2": 0, + "rtb_adapter_1": 1, + }, + }, + }, { + name: "case_insensitive_badv", // case of domain not matters + args: args{ + advBlockReq: &openrtb2.BidRequest{BAdv: []string{"ADVERTISER_1.COM"}}, + adaptorSeatBids: map[*bidderAdapter]*pbsOrtbSeatBid{ + newTestTagAdapter("tag_adapter_1"): { + bids: []*pbsOrtbBid{ + {bid: &openrtb2.Bid{ID: "bid_1_rtb_adapter_1", ADomain: []string{"advertiser_1.com"}}}, + {bid: &openrtb2.Bid{ID: "bid_2_rtb_adapter_1", ADomain: []string{"www.advertiser_1.com"}}}, + }, + }, + }, + }, + want: want{ + rejectedBidIds: []string{"bid_1_rtb_adapter_1", "bid_2_rtb_adapter_1"}, + validBidCountPerSeat: map[string]int{ + "tag_adapter_1": 0, // expect all bids are rejected as belongs to blocked advertiser + }, + }, + }, + { + name: "case_insensitive_adomain", + args: args{ + advBlockReq: &openrtb2.BidRequest{BAdv: []string{"advertiser_1.com"}}, + adaptorSeatBids: map[*bidderAdapter]*pbsOrtbSeatBid{ + newTestTagAdapter("tag_adapter_1"): { + bids: []*pbsOrtbBid{ + {bid: &openrtb2.Bid{ID: "bid_1_rtb_adapter_1", ADomain: []string{"advertiser_1.COM"}}}, + {bid: &openrtb2.Bid{ID: "bid_2_rtb_adapter_1", ADomain: []string{"wWw.ADVERTISER_1.com"}}}, + }, + }, + }, + }, + want: want{ + rejectedBidIds: []string{"bid_1_rtb_adapter_1", "bid_2_rtb_adapter_1"}, + validBidCountPerSeat: map[string]int{ + "tag_adapter_1": 0, // expect all bids are rejected as belongs to blocked advertiser + }, + }, + }, + { + name: "various_tld_combinations", + args: args{ + advBlockReq: &openrtb2.BidRequest{BAdv: []string{"http://blockme.shri"}}, + adaptorSeatBids: map[*bidderAdapter]*pbsOrtbSeatBid{ + newTestTagAdapter("block_bidder"): { + bids: []*pbsOrtbBid{ + {bid: &openrtb2.Bid{ADomain: []string{"www.blockme.shri"}, ID: "reject_www.blockme.shri"}}, + {bid: &openrtb2.Bid{ADomain: []string{"http://www.blockme.shri"}, ID: "rejecthttp://www.blockme.shri"}}, + {bid: &openrtb2.Bid{ADomain: []string{"https://blockme.shri"}, ID: "reject_https://blockme.shri"}}, + {bid: &openrtb2.Bid{ADomain: []string{"https://www.blockme.shri"}, ID: "reject_https://www.blockme.shri"}}, + }, + }, + newTestRtbAdapter("rtb_non_block_bidder"): { + bids: []*pbsOrtbBid{ // all below bids are eligible and should not be rejected + {bid: &openrtb2.Bid{ADomain: []string{"www.blockme.shri"}, ID: "accept_bid_www.blockme.shri"}}, + {bid: &openrtb2.Bid{ADomain: []string{"http://www.blockme.shri"}, ID: "accept_bid__http://www.blockme.shri"}}, + {bid: &openrtb2.Bid{ADomain: []string{"https://blockme.shri"}, ID: "accept_bid__https://blockme.shri"}}, + {bid: &openrtb2.Bid{ADomain: []string{"https://www.blockme.shri"}, ID: "accept_bid__https://www.blockme.shri"}}, + }, + }, + }, + }, + want: want{ + rejectedBidIds: []string{"reject_www.blockme.shri", "reject_http://www.blockme.shri", "reject_https://blockme.shri", "reject_https://www.blockme.shri"}, + validBidCountPerSeat: map[string]int{ + "block_bidder": 0, + "rtb_non_block_bidder": 4, + }, + }, + }, + { + name: "subdomain_tests", + args: args{ + advBlockReq: &openrtb2.BidRequest{BAdv: []string{"10th.college.puneunv.edu"}}, + adaptorSeatBids: map[*bidderAdapter]*pbsOrtbSeatBid{ + newTestTagAdapter("block_bidder"): { + bids: []*pbsOrtbBid{ + {bid: &openrtb2.Bid{ADomain: []string{"shri.10th.college.puneunv.edu"}, ID: "reject_shri.10th.college.puneunv.edu"}}, + {bid: &openrtb2.Bid{ADomain: []string{"puneunv.edu"}, ID: "allow_puneunv.edu"}}, + {bid: &openrtb2.Bid{ADomain: []string{"http://WWW.123.456.10th.college.PUNEUNV.edu"}, ID: "reject_123.456.10th.college.puneunv.edu"}}, + }, + }, + }, + }, + want: want{ + rejectedBidIds: []string{"reject_shri.10th.college.puneunv.edu", "reject_123.456.10th.college.puneunv.edu"}, + validBidCountPerSeat: map[string]int{ + "block_bidder": 1, + }, + }, + }, { + name: "only_domain_test", // do not expect bid rejection. edu is valid domain + args: args{ + advBlockReq: &openrtb2.BidRequest{BAdv: []string{"edu"}}, + adaptorSeatBids: map[*bidderAdapter]*pbsOrtbSeatBid{ + newTestTagAdapter("tag_bidder"): { + bids: []*pbsOrtbBid{ + {bid: &openrtb2.Bid{ADomain: []string{"school.edu"}, ID: "keep_bid_school.edu"}}, + {bid: &openrtb2.Bid{ADomain: []string{"edu"}, ID: "keep_bid_edu"}}, + {bid: &openrtb2.Bid{ADomain: []string{"..edu"}, ID: "keep_bid_..edu"}}, + }, + }, + }, + }, + want: want{ + rejectedBidIds: []string{}, + validBidCountPerSeat: map[string]int{ + "tag_bidder": 3, + }, + }, + }, + { + name: "public_suffix_in_badv", + args: args{ + advBlockReq: &openrtb2.BidRequest{BAdv: []string{"co.in"}}, // co.in is valid public suffix + adaptorSeatBids: map[*bidderAdapter]*pbsOrtbSeatBid{ + newTestTagAdapter("tag_bidder"): { + bids: []*pbsOrtbBid{ + {bid: &openrtb2.Bid{ADomain: []string{"a.co.in"}, ID: "allow_a.co.in"}}, + {bid: &openrtb2.Bid{ADomain: []string{"b.com"}, ID: "allow_b.com"}}, + }, + }, + }, + }, + want: want{ + rejectedBidIds: []string{}, + validBidCountPerSeat: map[string]int{ + "tag_bidder": 2, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.name != "reject_bid_of_blocked_adv_from_tag_bidder" { + return + } + seatBids := make(map[openrtb_ext.BidderName]*pbsOrtbSeatBid) + tagBidders := make(map[openrtb_ext.BidderName]adapters.Bidder) + adapterMap := make(map[openrtb_ext.BidderName]adaptedBidder, 0) + for adaptor, sbids := range tt.args.adaptorSeatBids { + adapterMap[adaptor.BidderName] = adaptor + if tagBidder, ok := adaptor.Bidder.(*vastbidder.TagBidder); ok { + tagBidders[adaptor.BidderName] = tagBidder + } + seatBids[adaptor.BidderName] = sbids + } + + // applyAdvertiserBlocking internally uses tagBidders from (adapter_map.go) + // not testing alias here + seatBids, rejections := applyAdvertiserBlocking(tt.args.advBlockReq, seatBids) + + re := regexp.MustCompile("bid rejected \\[bid ID:(.*?)\\] reason") + for bidder, sBid := range seatBids { + // verify only eligible bids are returned + assert.Equal(t, tt.want.validBidCountPerSeat[string(bidder)], len(sBid.bids), "Expected eligible bids are %d, but found [%d] ", tt.want.validBidCountPerSeat[string(bidder)], len(sBid.bids)) + // verify rejections + assert.Equal(t, len(tt.want.rejectedBidIds), len(rejections), "Expected bid rejections are %d, but found [%d]", len(tt.want.rejectedBidIds), len(rejections)) + // verify rejected bid ids + present := false + for _, expectRejectedBidID := range tt.want.rejectedBidIds { + for _, rejection := range rejections { + match := re.FindStringSubmatch(rejection) + rejectedBidID := strings.Trim(match[1], " ") + if expectRejectedBidID == rejectedBidID { + present = true + break + } + } + if present { + break + } + } + if len(tt.want.rejectedBidIds) > 0 && !present { + assert.Fail(t, "Expected Bid ID [%s] as rejected. But bid is not rejected", re) + } + + if sBid.bidderCoreName != openrtb_ext.BidderVASTBidder { + continue // advertiser blocking is currently enabled only for tag bidders + } + // verify eligible bids not belongs to blocked advertisers + for _, bid := range sBid.bids { + if nil != bid.bid.ADomain { + for _, adomain := range bid.bid.ADomain { + for _, blockDomain := range tt.args.advBlockReq.BAdv { + nDomain, _ := normalizeDomain(adomain) + if nDomain == blockDomain { + assert.Fail(t, "bid %s with ad domain %s is not blocked", bid.bid.ID, adomain) + } + } + } + } + + // verify this bid not belongs to rejected list + for _, rejectedBidID := range tt.want.rejectedBidIds { + if rejectedBidID == bid.bid.ID { + assert.Fail(t, "Bid ID [%s] is not expected in list of rejected bids", bid.bid.ID) + } + } + } + } + }) + } +} + +func (m *fakeCurrencyRatesHttpClient) Do(req *http.Request) (*http.Response, error) { + return &http.Response{ + Status: "200 OK", + StatusCode: http.StatusOK, + Body: ioutil.NopCloser(strings.NewReader(m.responseBody)), + }, nil +} + +type mockBidder struct { + mock.Mock + lastExtraRequestInfo *adapters.ExtraRequestInfo +} + +func (m *mockBidder) MakeRequests(request *openrtb2.BidRequest, reqInfo *adapters.ExtraRequestInfo) ([]*adapters.RequestData, []error) { + m.lastExtraRequestInfo = reqInfo + + args := m.Called(request, reqInfo) + return args.Get(0).([]*adapters.RequestData), args.Get(1).([]error) +} + +func (m *mockBidder) MakeBids(internalRequest *openrtb2.BidRequest, externalRequest *adapters.RequestData, response *adapters.ResponseData) (*adapters.BidderResponse, []error) { + args := m.Called(internalRequest, externalRequest, response) + return args.Get(0).(*adapters.BidderResponse), args.Get(1).([]error) +} diff --git a/exchange/utils.go b/exchange/utils.go index 008bbc7b6c7..af7cd12b9cd 100644 --- a/exchange/utils.go +++ b/exchange/utils.go @@ -255,6 +255,16 @@ func getAuctionBidderRequests(req AuctionRequest, } } + var sChainsByBidder map[string]*openrtb_ext.ExtRequestPrebidSChainSChain + + // Quick extra wrapper until RequestWrapper makes its way into CleanRequests + if requestExt != nil { + sChainsByBidder, err = BidderToPrebidSChains(requestExt.Prebid.SChains) + if err != nil { + return nil, []error{err} + } + } + reqExt, err := getExtJson(req.BidRequest, requestExt) if err != nil { return nil, []error{err} diff --git a/go.sum b/go.sum index 91a861677ce..d8cbf58754a 100644 --- a/go.sum +++ b/go.sum @@ -70,6 +70,13 @@ github.com/influxdata/influxdb v1.6.1 h1:OseoBlzI5ftNI/bczyxSWq6PKRCNEeiXvyWP/wS github.com/influxdata/influxdb v1.6.1/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/julienschmidt/httprouter v1.1.0 h1:7wLdtIiIpzOkC9u6sXOozpBauPdskj3ru4EI5MABq68= github.com/julienschmidt/httprouter v1.1.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= @@ -86,6 +93,8 @@ github.com/mitchellh/mapstructure v1.0.0 h1:vVpGvMXJPqSDh2VYHF7gsfQj8Ncx+Xw5Y1KH github.com/mitchellh/mapstructure v1.0.0/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE= +github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mxmCherry/openrtb/v15 v15.0.0 h1:inLuQ3Bsima9HLB2v6WjbtEFF69SWOT5Dux4QZtYdrw= github.com/mxmCherry/openrtb/v15 v15.0.0/go.mod h1:TVgncsz6MOzbL7lhun1lNuUBzVBlVDbxf9Fyy1TyhZA= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -148,6 +157,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180816142147-da425ebb7609 h1:BcMExZAULPkihVZ7UJXK7t8rwGqisXFw75tILnafhBY= github.com/xeipuuv/gojsonschema v0.0.0-20180816142147-da425ebb7609/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xorcare/pointer v1.1.0 h1:sFwXOhRF8QZ0tyVZrtxWGIoVZNEmRzBCaFWdONPQIUM= +github.com/xorcare/pointer v1.1.0/go.mod h1:6KLhkOh6YbuvZkT4YbxIbR/wzLBjyMxOiNzZhJTor2Y= github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d h1:yJIizrfO599ot2kQ6Af1enICnwBD3XoxgX3MrMwot2M= github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= diff --git a/main.go b/main.go index 9184fb4c6bd..5399ea19037 100644 --- a/main.go +++ b/main.go @@ -56,7 +56,7 @@ func InitPrebidServer(configFile string) { err = serve(Rev, cfg) if err != nil { - glog.Errorf("prebid-server failed: %v", err) + glog.Exitf("prebid-server failed: %v", err) } } diff --git a/metrics/config/metrics.go b/metrics/config/metrics.go index e6054b4562f..67548d2389c 100644 --- a/metrics/config/metrics.go +++ b/metrics/config/metrics.go @@ -147,9 +147,9 @@ func (me *MultiMetricsEngine) RecordDNSTime(dnsLookupTime time.Duration) { } } -func (me *MultiMetricsEngine) RecordTLSHandshakeTime(tlsHandshakeTime time.Duration) { +func (me *MultiMetricsEngine) RecordTLSHandshakeTime(adapterName openrtb_ext.BidderName, tlsHandshakeTime time.Duration) { for _, thisME := range *me { - thisME.RecordTLSHandshakeTime(tlsHandshakeTime) + thisME.RecordTLSHandshakeTime(adapterName, tlsHandshakeTime) } } @@ -251,9 +251,34 @@ func (me *MultiMetricsEngine) RecordAdapterGDPRRequestBlocked(adapter openrtb_ex } } +// RecordAdapterGDPRRequestBlocked across all engines +func (me *MultiMetricsEngine) RecordAdapterGDPRRequestBlocked(adapter openrtb_ext.BidderName) { + for _, thisME := range *me { + thisME.RecordAdapterGDPRRequestBlocked(adapter) + } +} + // DummyMetricsEngine is a Noop metrics engine in case no metrics are configured. (may also be useful for tests) type DummyMetricsEngine struct{} +func (me *DummyMetricsEngine) RecordAdapterDuplicateBidID(adaptor string, collisions int) { +} + +func (me *DummyMetricsEngine) RecordRequestHavingDuplicateBidID() { +} + +func (me *DummyMetricsEngine) RecordPodImpGenTime(labels metrics.PodLabels, startTime time.Time) { +} + +func (me *DummyMetricsEngine) RecordPodCombGenTime(labels metrics.PodLabels, elapsedTime time.Duration) { +} + +func (me *DummyMetricsEngine) RecordPodCompititveExclusionTime(labels metrics.PodLabels, elapsedTime time.Duration) { +} + +func (me *DummyMetricsEngine) RecordAdapterVideoBidDuration(labels metrics.AdapterLabels, videoBidDuration int) { +} + // RecordRequest as a noop func (me *DummyMetricsEngine) RecordRequest(labels metrics.Labels) { } @@ -303,7 +328,7 @@ func (me *DummyMetricsEngine) RecordDNSTime(dnsLookupTime time.Duration) { } // RecordTLSHandshakeTime as a noop -func (me *DummyMetricsEngine) RecordTLSHandshakeTime(tlsHandshakeTime time.Duration) { +func (me *DummyMetricsEngine) RecordTLSHandshakeTime(adapterName openrtb_ext.BidderName, tlsHandshakeTime time.Duration) { } // RecordAdapterBidReceived as a noop diff --git a/metrics/go_metrics.go b/metrics/go_metrics.go index 41cd9463d3f..12beb4f2052 100644 --- a/metrics/go_metrics.go +++ b/metrics/go_metrics.go @@ -102,6 +102,7 @@ type AdapterMetrics struct { ConnReused metrics.Counter ConnWaitTime metrics.Timer GDPRRequestBlocked metrics.Meter + TLSHandshakeTimer metrics.Timer } type MarkupDeliveryMetrics struct { @@ -324,6 +325,9 @@ func makeBlankAdapterMetrics(disabledMetrics config.DisabledMetrics) *AdapterMet if !disabledMetrics.AdapterGDPRRequestBlocked { newAdapter.GDPRRequestBlocked = blankMeter } + if !disabledMetrics.AdapterGDPRRequestBlocked { + newAdapter.GDPRRequestBlocked = blankMeter + } for _, err := range AdapterErrors() { newAdapter.ErrorMeters[err] = blankMeter } @@ -738,6 +742,31 @@ func (me *Metrics) RecordAdapterGDPRRequestBlocked(adapterName openrtb_ext.Bidde am.GDPRRequestBlocked.Mark(1) } +// RecordAdapterDuplicateBidID as noop +func (me *Metrics) RecordAdapterDuplicateBidID(adaptor string, collisions int) { +} + +// RecordRequestHavingDuplicateBidID as noop +func (me *Metrics) RecordRequestHavingDuplicateBidID() { +} + +// RecordPodImpGenTime as a noop +func (me *Metrics) RecordPodImpGenTime(labels PodLabels, startTime time.Time) { +} + +// RecordPodCombGenTime as a noop +func (me *Metrics) RecordPodCombGenTime(labels PodLabels, elapsedTime time.Duration) { +} + + am, ok := me.AdapterMetrics[adapterName] + if !ok { + glog.Errorf("Trying to log adapter GDPR request blocked metric for %s: adapter not found", string(adapterName)) + return + } + + am.GDPRRequestBlocked.Mark(1) +} + func doMark(bidder openrtb_ext.BidderName, meters map[openrtb_ext.BidderName]metrics.Meter) { met, ok := meters[bidder] if ok { diff --git a/metrics/metrics.go b/metrics/metrics.go index ceb1ab82976..9164dfa2c42 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -364,4 +364,38 @@ type MetricsEngine interface { RecordTimeoutNotice(sucess bool) RecordRequestPrivacy(privacy PrivacyLabels) RecordAdapterGDPRRequestBlocked(adapterName openrtb_ext.BidderName) + + // RecordAdapterDuplicateBidID captures the bid.ID collisions when adaptor + // gives the bid response with multiple bids containing same bid.ID + RecordAdapterDuplicateBidID(adaptor string, collisions int) + + // RecordRequestHavingDuplicateBidID keeps track off how many request got bid.id collision + // detected + RecordRequestHavingDuplicateBidID() + + // ad pod specific metrics + + // RecordPodImpGenTime records number of impressions generated and time taken + // by underneath algorithm to generate them + // labels accept name of the algorithm and no of impressions generated + // startTime indicates the time at which algorithm started + // This function will take care of computing the elpased time + RecordPodImpGenTime(labels PodLabels, startTime time.Time) + + // RecordPodCombGenTime records number of combinations generated and time taken + // by underneath algorithm to generate them + // labels accept name of the algorithm and no of combinations generated + // elapsedTime indicates the time taken by combination generator to compute all requested combinations + // This function will take care of computing the elpased time + RecordPodCombGenTime(labels PodLabels, elapsedTime time.Duration) + + // RecordPodCompititveExclusionTime records time take by competitive exclusion + // to compute the final Ad pod Response. + // labels accept name of the algorithm and no of combinations evaluated, total bids + // elapsedTime indicates the time taken by competitive exclusion to form final ad pod response using combinations and exclusion algorithm + // This function will take care of computing the elpased time + RecordPodCompititveExclusionTime(labels PodLabels, elapsedTime time.Duration) + + //RecordAdapterVideoBidDuration records actual ad duration returned by the bidder + RecordAdapterVideoBidDuration(labels AdapterLabels, videoBidDuration int) } diff --git a/metrics/metrics_mock.go b/metrics/metrics_mock.go index c687cbd8cca..7fba0bbd44f 100644 --- a/metrics/metrics_mock.go +++ b/metrics/metrics_mock.go @@ -145,3 +145,33 @@ func (me *MetricsEngineMock) RecordRequestPrivacy(privacy PrivacyLabels) { func (me *MetricsEngineMock) RecordAdapterGDPRRequestBlocked(adapterName openrtb_ext.BidderName) { me.Called(adapterName) } + +// RecordAdapterDuplicateBidID mock +func (me *MetricsEngineMock) RecordAdapterDuplicateBidID(adaptor string, collisions int) { + me.Called(adaptor, collisions) +} + +// RecordRequestHavingDuplicateBidID mock +func (me *MetricsEngineMock) RecordRequestHavingDuplicateBidID() { + me.Called() +} + +// RecordPodImpGenTime mock +func (me *MetricsEngineMock) RecordPodImpGenTime(labels PodLabels, startTime time.Time) { + me.Called(labels, startTime) +} + +// RecordPodCombGenTime mock +func (me *MetricsEngineMock) RecordPodCombGenTime(labels PodLabels, elapsedTime time.Duration) { + me.Called(labels, elapsedTime) +} + +// RecordPodCompititveExclusionTime mock +func (me *MetricsEngineMock) RecordPodCompititveExclusionTime(labels PodLabels, elapsedTime time.Duration) { + me.Called(labels, elapsedTime) +} + +//RecordAdapterVideoBidDuration mock +func (me *MetricsEngineMock) RecordAdapterVideoBidDuration(labels AdapterLabels, videoBidDuration int) { + me.Called(labels, videoBidDuration) +} \ No newline at end of file diff --git a/metrics/prometheus/prometheus.go b/metrics/prometheus/prometheus.go index 2e9c9f3844a..349222e4b2c 100644 --- a/metrics/prometheus/prometheus.go +++ b/metrics/prometheus/prometheus.go @@ -15,33 +15,33 @@ type Metrics struct { Registry *prometheus.Registry // General Metrics - connectionsClosed prometheus.Counter - connectionsError *prometheus.CounterVec - connectionsOpened prometheus.Counter - cookieSync prometheus.Counter - impressions *prometheus.CounterVec - impressionsLegacy prometheus.Counter - prebidCacheWriteTimer *prometheus.HistogramVec - requests *prometheus.CounterVec - requestsTimer *prometheus.HistogramVec - requestsQueueTimer *prometheus.HistogramVec - requestsWithoutCookie *prometheus.CounterVec - storedImpressionsCacheResult *prometheus.CounterVec - storedRequestCacheResult *prometheus.CounterVec - accountCacheResult *prometheus.CounterVec - storedAccountFetchTimer *prometheus.HistogramVec - storedAccountErrors *prometheus.CounterVec - storedAMPFetchTimer *prometheus.HistogramVec - storedAMPErrors *prometheus.CounterVec - storedCategoryFetchTimer *prometheus.HistogramVec - storedCategoryErrors *prometheus.CounterVec - storedRequestFetchTimer *prometheus.HistogramVec - storedRequestErrors *prometheus.CounterVec - storedVideoFetchTimer *prometheus.HistogramVec - storedVideoErrors *prometheus.CounterVec - timeoutNotifications *prometheus.CounterVec - dnsLookupTimer prometheus.Histogram - tlsHandhakeTimer prometheus.Histogram + connectionsClosed prometheus.Counter + connectionsError *prometheus.CounterVec + connectionsOpened prometheus.Counter + cookieSync prometheus.Counter + impressions *prometheus.CounterVec + impressionsLegacy prometheus.Counter + prebidCacheWriteTimer *prometheus.HistogramVec + requests *prometheus.CounterVec + requestsTimer *prometheus.HistogramVec + requestsQueueTimer *prometheus.HistogramVec + requestsWithoutCookie *prometheus.CounterVec + storedImpressionsCacheResult *prometheus.CounterVec + storedRequestCacheResult *prometheus.CounterVec + accountCacheResult *prometheus.CounterVec + storedAccountFetchTimer *prometheus.HistogramVec + storedAccountErrors *prometheus.CounterVec + storedAMPFetchTimer *prometheus.HistogramVec + storedAMPErrors *prometheus.CounterVec + storedCategoryFetchTimer *prometheus.HistogramVec + storedCategoryErrors *prometheus.CounterVec + storedRequestFetchTimer *prometheus.HistogramVec + storedRequestErrors *prometheus.CounterVec + storedVideoFetchTimer *prometheus.HistogramVec + storedVideoErrors *prometheus.CounterVec + timeoutNotifications *prometheus.CounterVec + dnsLookupTimer prometheus.Histogram + //tlsHandhakeTimer prometheus.Histogram privacyCCPA *prometheus.CounterVec privacyCOPPA *prometheus.CounterVec privacyLMT *prometheus.CounterVec @@ -49,18 +49,21 @@ type Metrics struct { requestsDuplicateBidIDCounter prometheus.Counter // total request having duplicate bid.id for given bidder // Adapter Metrics - adapterBids *prometheus.CounterVec - adapterCookieSync *prometheus.CounterVec - adapterErrors *prometheus.CounterVec - adapterPanics *prometheus.CounterVec - adapterPrices *prometheus.HistogramVec - adapterRequests *prometheus.CounterVec - adapterRequestsTimer *prometheus.HistogramVec - adapterUserSync *prometheus.CounterVec - adapterReusedConnections *prometheus.CounterVec - adapterCreatedConnections *prometheus.CounterVec - adapterConnectionWaitTime *prometheus.HistogramVec - adapterGDPRBlockedRequests *prometheus.CounterVec + adapterBids *prometheus.CounterVec + adapterCookieSync *prometheus.CounterVec + adapterErrors *prometheus.CounterVec + adapterPanics *prometheus.CounterVec + adapterPrices *prometheus.HistogramVec + adapterRequests *prometheus.CounterVec + adapterRequestsTimer *prometheus.HistogramVec + adapterUserSync *prometheus.CounterVec + adapterReusedConnections *prometheus.CounterVec + adapterCreatedConnections *prometheus.CounterVec + adapterConnectionWaitTime *prometheus.HistogramVec + adapterDuplicateBidIDCounter *prometheus.CounterVec + adapterVideoBidDuration *prometheus.HistogramVec + tlsHandhakeTimer *prometheus.HistogramVec + adapterGDPRBlockedRequests *prometheus.CounterVec // Account Metrics accountRequests *prometheus.CounterVec @@ -756,3 +759,43 @@ func (m *Metrics) RecordAdapterGDPRRequestBlocked(adapterName openrtb_ext.Bidder adapterLabel: string(adapterName), }).Inc() } + +// RecordAdapterDuplicateBidID captures the bid.ID collisions when adaptor +// gives the bid response with multiple bids containing same bid.ID +// ensure collisions value is greater than 1. This function will not give any error +// if collisions = 1 is passed +func (m *Metrics) RecordAdapterDuplicateBidID(adaptor string, collisions int) { + m.adapterDuplicateBidIDCounter.With(prometheus.Labels{ + adapterLabel: adaptor, + }).Add(float64(collisions)) +} + +// RecordRequestHavingDuplicateBidID keeps count of request when duplicate bid.id is +// detected in partner's response +func (m *Metrics) RecordRequestHavingDuplicateBidID() { + m.requestsDuplicateBidIDCounter.Inc() +} + +// pod specific metrics + +// recordAlgoTime is common method which handles algorithm time performance +func recordAlgoTime(timer *prometheus.HistogramVec, labels metrics.PodLabels, elapsedTime time.Duration) { + + pmLabels := prometheus.Labels{ + podAlgorithm: labels.AlgorithmName, + } + + if labels.NoOfImpressions != nil { + pmLabels[podNoOfImpressions] = strconv.Itoa(*labels.NoOfImpressions) + } + if labels.NoOfCombinations != nil { + pmLabels[podTotalCombinations] = strconv.Itoa(*labels.NoOfCombinations) + } + if labels.NoOfResponseBids != nil { + pmLabels[podNoOfResponseBids] = strconv.Itoa(*labels.NoOfResponseBids) + } + + m.adapterGDPRBlockedRequests.With(prometheus.Labels{ + adapterLabel: string(adapterName), + }).Inc() +} diff --git a/metrics/prometheus/prometheus_test.go b/metrics/prometheus/prometheus_test.go index e6a15061e1a..bf15f887726 100644 --- a/metrics/prometheus/prometheus_test.go +++ b/metrics/prometheus/prometheus_test.go @@ -1356,6 +1356,7 @@ func TestDisabledMetrics(t *testing.T) { assert.Nil(t, prometheusMetrics.adapterReusedConnections, "Counter Vector adapterReusedConnections should be nil") assert.Nil(t, prometheusMetrics.adapterCreatedConnections, "Counter Vector adapterCreatedConnections should be nil") assert.Nil(t, prometheusMetrics.adapterConnectionWaitTime, "Counter Vector adapterConnectionWaitTime should be nil") + assert.Nil(t, prometheusMetrics.tlsHandhakeTimer, "Counter Vector tlsHandhakeTimer should be nil") assert.Nil(t, prometheusMetrics.adapterGDPRBlockedRequests, "Counter Vector adapterGDPRBlockedRequests should be nil") } diff --git a/openrtb_ext/bidders.go b/openrtb_ext/bidders.go index 3a097b02052..71b2146f4c3 100644 --- a/openrtb_ext/bidders.go +++ b/openrtb_ext/bidders.go @@ -175,6 +175,7 @@ const ( BidderSomoaudience BidderName = "somoaudience" BidderSonobi BidderName = "sonobi" BidderSovrn BidderName = "sovrn" + BidderSpotX BidderName = "spotx" BidderSynacormedia BidderName = "synacormedia" BidderTappx BidderName = "tappx" BidderTelaria BidderName = "telaria" @@ -185,6 +186,7 @@ const ( BidderUnicorn BidderName = "unicorn" BidderUnruly BidderName = "unruly" BidderValueImpression BidderName = "valueimpression" + BidderVASTBidder BidderName = "vastbidder" BidderVerizonMedia BidderName = "verizonmedia" BidderVisx BidderName = "visx" BidderViewdeos BidderName = "viewdeos" diff --git a/openrtb_ext/request.go b/openrtb_ext/request.go index 3673196f231..24766ab1603 100644 --- a/openrtb_ext/request.go +++ b/openrtb_ext/request.go @@ -43,6 +43,10 @@ type ExtRequestPrebid struct { // The array may contain a single sstar ('*') entry to represent all bidders. NoSale []string `json:"nosale,omitempty"` + // Macros specifies list of custom macros along with the values. This is used while forming + // the tracker URLs, where PBS will replace the Custom Macro with its value with url-encoding + Macros map[string]string `json:"macros,omitempty"` + CurrencyConversions *ExtRequestCurrency `json:"currency,omitempty"` } diff --git a/privacy/gdpr/policy_test.go b/privacy/gdpr/policy_test.go index a0fa6241d72..9274c5b58be 100644 --- a/privacy/gdpr/policy_test.go +++ b/privacy/gdpr/policy_test.go @@ -28,4 +28,4 @@ func TestValidateConsent(t *testing.T) { result := ValidateConsent(test.consent) assert.Equal(t, test.expected, result, test.description) } -} +} \ No newline at end of file diff --git a/router/router.go b/router/router.go index 45688eedf59..b0de7edc0fe 100644 --- a/router/router.go +++ b/router/router.go @@ -7,6 +7,10 @@ import ( "database/sql" "encoding/json" "fmt" + "github.com/prebid/prebid-server/analytics" + "github.com/prebid/prebid-server/stored_requests" + "github.com/prebid/prebid-server/usersync" + "github.com/prometheus/client_golang/prometheus" "io/ioutil" "net" "net/http" diff --git a/usersync/usersyncers/syncer_test.go b/usersync/usersyncers/syncer_test.go index 97935e1d470..3abdae0a00c 100644 --- a/usersync/usersyncers/syncer_test.go +++ b/usersync/usersyncers/syncer_test.go @@ -137,7 +137,9 @@ func TestNewSyncerMap(t *testing.T) { openrtb_ext.BidderRevcontent: true, openrtb_ext.BidderSilverMob: true, openrtb_ext.BidderSmaato: true, + openrtb_ext.BidderSpotX: true, openrtb_ext.BidderUnicorn: true, + openrtb_ext.BidderVASTBidder: true, openrtb_ext.BidderYeahmobi: true, }