From b30a795d8fa516251654a50c0917c15526f7f910 Mon Sep 17 00:00:00 2001 From: Michael Moschovas Date: Mon, 22 Feb 2021 11:15:50 -0500 Subject: [PATCH 01/18] Creating fpd module --- modules/firstPartyData/index.js | 121 ++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 modules/firstPartyData/index.js diff --git a/modules/firstPartyData/index.js b/modules/firstPartyData/index.js new file mode 100644 index 00000000000..6ee643f42b5 --- /dev/null +++ b/modules/firstPartyData/index.js @@ -0,0 +1,121 @@ +/** + * This module adds User ID support to prebid.js + * @module modules/userId + */ + + +import find from 'core-js-pure/features/array/find.js'; +import { config } from '../../src/config.js'; +import events from '../../src/events.js'; +import * as utils from '../../src/utils.js'; +import { getRefererInfo } from '../../src/refererDetection.js' +import CONSTANTS from '../../src/constants.json'; + +const MODULE_NAME = 'First Party Data'; +let ortb2 = {}; +let shouldRun = true; + + +function setReferer() { + if (getRefererInfo().referer) utils.mergeDeep(ortb2, {site: { ref: getRefererInfo().referer}}); +} + +function setPage() { + if (getRefererInfo().canonicalUrl) utils.mergeDeep(ortb2, {site: { page: getRefererInfo().canonicalUrl}}); +} + +function setDomain() { + let parseDomain = function(url) { + if (!url || typeof url !== 'string' || url.length === 0) return; + + var match = url.match(/^(?:https?:\/\/)?(?:www\.)?(.*?(?=(\?|\#|\/|$)))/i); + + return match && match[1]; + }; + + let domain = parseDomain(getRefererInfo().canonicalUrl) + + if (domain) utils.mergeDeep(ortb2, {site: { domain: domain}}); +} + +function setDimensions() { + const width = window.innerWidth || document.documentElement.clientWidth || document.body.clientWidth; + const height = window.innerHeight|| document.documentElement.clientHeight|| document.body.clientHeight; + + utils.mergeDeep(ortb2, {device: { width: width, height: height}}); +} + +function setKeywords() { + let keywords = document.querySelector('meta[name="keywords"]'); + + if (keywords && keywords.content) utils.mergeDeep(ortb2, {site: { keywords: keywords.content.replace(/\s/g, '')}}); +} + +function validateFpd(obj) { + let validObject = Object.assign({}, Object.keys(obj).filter(key => { + if (key !== 'imp') return key; + + utils.logWarn('Filtered imp property in ortb2 data'); + }).reduce((result, key) => { + result[key] = obj[key]; + + return result; + }, {})); + + + console.log(validObject); + + return validObject; +} + +function hasChanged(config) { + Object.keys(ortb2).forEach(key => { + if (!utils.deepEqual(ortb2[key], config[key])) return false + }); + + return true; +} + +function storeValue(config) { + Object.keys(ortb2).forEach(key => { + if (!utils.deepEqual(ortb2[key],config)) ortb2[key] = config[key]; + }); +} + + +/** + * test browser support for storage config types (local storage or cookie), initializes submodules but consentManagement is required, + * so a callback is added to fire after the consentManagement module. + * @param {{getConfig:function}} config + */ +export function init() { + setReferer(); + setPage(); + setDomain(); + setDimensions(); + setKeywords(); + + config.getConfig('currency', conf => { + if (!conf.currency.adServerCurrency) return; + + utils.mergeDeep(ortb2, {cur: conf.currency.adServerCurrency}); + shouldRun = true; + config.setConfig(config.getConfig('ortb2') || {ortb2: {}}); + }); + + config.getConfig('ortb2', conf => { + if (!shouldRun || !hasChanged(conf.ortb2)) { + shouldRun = true; + return; + } + + conf.ortb2 = validateFpd(utils.mergeDeep(ortb2, conf.ortb2)); + //ortb2 = {...conf.ortb2} + shouldRun = false; + config.setConfig({ortb2: conf.ortb2}); + }); +} + +// init config update listener to start the application + +init(); From da3f25fe9841452efbfbb741ab1ebf26db7f6f03 Mon Sep 17 00:00:00 2001 From: Michael Moschovas Date: Wed, 17 Mar 2021 10:41:37 -0400 Subject: [PATCH 02/18] Continued work on FPD module. - Data validation - Pubcid optout check - Misc Fixes --- modules/firstPartyData/index.js | 127 +++++++++++++++++++++++++++++--- modules/userId/index.js | 2 +- 2 files changed, 118 insertions(+), 11 deletions(-) diff --git a/modules/firstPartyData/index.js b/modules/firstPartyData/index.js index 6ee643f42b5..08f1f8bfccd 100644 --- a/modules/firstPartyData/index.js +++ b/modules/firstPartyData/index.js @@ -10,8 +10,10 @@ import events from '../../src/events.js'; import * as utils from '../../src/utils.js'; import { getRefererInfo } from '../../src/refererDetection.js' import CONSTANTS from '../../src/constants.json'; +import { getStorageManager } from '../../src/storageManager.js'; const MODULE_NAME = 'First Party Data'; +const storage = getStorageManager(); let ortb2 = {}; let shouldRun = true; @@ -51,18 +53,128 @@ function setKeywords() { if (keywords && keywords.content) utils.mergeDeep(ortb2, {site: { keywords: keywords.content.replace(/\s/g, '')}}); } +function filterData(data, key) { + if (!Array.isArray(data)) { + utils.logWarn(`Filtered ${key} data: Must be an array of objects`); + return; + } + + let duplicate = data.filter(index => { + if (typeof index !== 'object' || !index.name || !index.segment || !Array.isArray(index.segment)) { + utils.logWarn(`Filtered ${key}.data: must be an object containing name and segment`, index); + return false; + } + + return true; + }).reduce((result, value) => { + if (value.ext && (typeof value.ext !== 'object' || Array.isArray(value.ext))) { + utils.logWarn(`Filtered ext attribute from ${key}.data: must be an object`, value); + delete value.ext; + } + + value.segment = value.segment.filter(el => { + if (!el.id || typeof el.id !== 'string') { + utils.logWarn(`Filtered ${key}.data.segment: id is required and must be a string`, el); + return false; + } + return true; + }); + + if (value.segment.length) { + result.push(value); + } else { + utils.logWarn(`Filtered ${key}.data: must contain segment data`); + } + + return result; + }, []); + + return (duplicate.length) ? duplicate : null; +} + +/** + * Retrieve an item from storage if it exists and hasn't expired. + * @param {string} key Key of the item. + * @returns {string|null} Value of the item. + */ +export function getStorageItem(key) { + let val = null; + + try { + const expVal = storage.getDataFromLocalStorage(key + '_exp'); + + if (!expVal) { + // If there is no expiry time, then just return the item + val = storage.getDataFromLocalStorage(key); + } else { + // Only return the item if it hasn't expired yet. + // Otherwise delete the item. + const expDate = new Date(expVal); + const isValid = (expDate.getTime() - Date.now()) > 0; + if (isValid) { + val = storage.getDataFromLocalStorage(key); + } else { + removeStorageItem(key); + } + } + } catch (e) { + utils.logMessage(e); + } + + return val; +} + function validateFpd(obj) { + console.log("FPD", obj); let validObject = Object.assign({}, Object.keys(obj).filter(key => { if (key !== 'imp') return key; utils.logWarn('Filtered imp property in ortb2 data'); }).reduce((result, key) => { - result[key] = obj[key]; - - return result; + let prop = obj[key]; + let modified = {}; + + let optout = (storage.cookiesAreEnabled() && storage.getCookie(name)) || + (storage.hasLocalStorage() && getStorageItem(name));console.log(output); + + if (key === 'user' && optout) { + utils.logWarn(`Filtered ${key} data: pubcid optout found`); + return result; + } + + modified = Object.keys(obj[key]).reduce((combined, keyData) => { + let data; + + if (key === 'user' && keyData === 'data') { + data = filterData(obj[key][keyData], key); + + if (data) combined[keyData] = data; + } else if(key === 'site' && keyData === 'content' && obj[key][keyData].data) { + let content = Object.keys(obj[key][keyData]).reduce((merged, contentData) => { + if (contentData === 'data') { + data = filterData(obj[key][keyData][contentData], key + '.content'); + + if (data) merged[contentData] = data; + } else { + merged[contentData] = obj[key][keyData][contentData]; + } + + return merged; + }, {}); + + if (Object.keys(content).length) combined[keyData] = content; + } else { + combined[keyData] = obj[key][keyData]; + } + + return combined; + }, {}); + + if (Object.keys(modified).length) result[key] = modified; + + return result; }, {})); - console.log(validObject); return validObject; @@ -84,9 +196,7 @@ function storeValue(config) { /** - * test browser support for storage config types (local storage or cookie), initializes submodules but consentManagement is required, - * so a callback is added to fire after the consentManagement module. - * @param {{getConfig:function}} config + * */ export function init() { setReferer(); @@ -110,12 +220,9 @@ export function init() { } conf.ortb2 = validateFpd(utils.mergeDeep(ortb2, conf.ortb2)); - //ortb2 = {...conf.ortb2} shouldRun = false; config.setConfig({ortb2: conf.ortb2}); }); } -// init config update listener to start the application - init(); diff --git a/modules/userId/index.js b/modules/userId/index.js index 3253be42a76..170b74844cf 100644 --- a/modules/userId/index.js +++ b/modules/userId/index.js @@ -762,7 +762,7 @@ function updateSubmodules() { callback: undefined, idObj: undefined } : null; - }).filter(submodule => submodule !== null); + }).filter(submodule => submodule !== null);console.log(submodules); if (!addedUserIdHook && submodules.length) { // priority value 40 will load after consentManagement with a priority of 50 From 732cb02c42daba61bd648d3f0f72ba5dab4f6cbd Mon Sep 17 00:00:00 2001 From: Michael Moschovas Date: Wed, 17 Mar 2021 10:43:56 -0400 Subject: [PATCH 03/18] Revert userId update. Committed in error --- modules/userId/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/userId/index.js b/modules/userId/index.js index 170b74844cf..3253be42a76 100644 --- a/modules/userId/index.js +++ b/modules/userId/index.js @@ -762,7 +762,7 @@ function updateSubmodules() { callback: undefined, idObj: undefined } : null; - }).filter(submodule => submodule !== null);console.log(submodules); + }).filter(submodule => submodule !== null); if (!addedUserIdHook && submodules.length) { // priority value 40 will load after consentManagement with a priority of 50 From eaaddbb9decded6d310a6503824971b2013caab7 Mon Sep 17 00:00:00 2001 From: Michael Moschovas Date: Mon, 22 Mar 2021 11:11:11 -0400 Subject: [PATCH 04/18] Added first party data unit tests and fixed bug --- modules/firstPartyData/index.js | 238 ++++----- test/spec/modules/firstPartyData_spec.js | 587 +++++++++++++++++++++++ 2 files changed, 710 insertions(+), 115 deletions(-) create mode 100644 test/spec/modules/firstPartyData_spec.js diff --git a/modules/firstPartyData/index.js b/modules/firstPartyData/index.js index 08f1f8bfccd..a49b726aa91 100644 --- a/modules/firstPartyData/index.js +++ b/modules/firstPartyData/index.js @@ -1,31 +1,35 @@ /** - * This module adds User ID support to prebid.js - * @module modules/userId + * This module sets default values and validates ortb2 first part data + * @module modules/firstPartyData */ - -import find from 'core-js-pure/features/array/find.js'; import { config } from '../../src/config.js'; -import events from '../../src/events.js'; import * as utils from '../../src/utils.js'; import { getRefererInfo } from '../../src/refererDetection.js' -import CONSTANTS from '../../src/constants.json'; import { getStorageManager } from '../../src/storageManager.js'; -const MODULE_NAME = 'First Party Data'; const storage = getStorageManager(); let ortb2 = {}; let shouldRun = true; +let win = (window === window.top) ? window : window.top; - +/** + * Checks for referer and if exists merges into ortb2 global data + */ function setReferer() { - if (getRefererInfo().referer) utils.mergeDeep(ortb2, {site: { ref: getRefererInfo().referer}}); + if (getRefererInfo().referer) utils.mergeDeep(ortb2, {site: {ref: getRefererInfo().referer}}); } +/** + * Checks for canonical url and if exists merges into ortb2 global data + */ function setPage() { - if (getRefererInfo().canonicalUrl) utils.mergeDeep(ortb2, {site: { page: getRefererInfo().canonicalUrl}}); + if (getRefererInfo().canonicalUrl) utils.mergeDeep(ortb2, {site: {page: getRefererInfo().canonicalUrl}}); } +/** + * Checks for canonical url and if exists retrieves domain and merges into ortb2 global data + */ function setDomain() { let parseDomain = function(url) { if (!url || typeof url !== 'string' || url.length === 0) return; @@ -37,49 +41,81 @@ function setDomain() { let domain = parseDomain(getRefererInfo().canonicalUrl) - if (domain) utils.mergeDeep(ortb2, {site: { domain: domain}}); + if (domain) utils.mergeDeep(ortb2, {site: {domain: domain}}); } +/** + * Checks for screen/device width and height and sets dimensions + */ function setDimensions() { - const width = window.innerWidth || document.documentElement.clientWidth || document.body.clientWidth; - const height = window.innerHeight|| document.documentElement.clientHeight|| document.body.clientHeight; + let width; + let height; - utils.mergeDeep(ortb2, {device: { width: width, height: height}}); + try { + width = win.innerWidth || win.document.documentElement.clientWidth || win.document.body.clientWidth; + height = win.innerHeight || win.document.documentElement.clientHeight || win.document.body.clientHeight; + } catch (e) { + width = window.innerWidth || window.document.documentElement.clientWidth || window.document.body.clientWidth; + height = window.innerHeight || window.document.documentElement.clientHeight || window.document.body.clientHeight; + } + + utils.mergeDeep(ortb2, {device: {width: width, height: height}}); } +/** + * Scans page for meta keywords, and if exists, merges into site.keywords + */ function setKeywords() { - let keywords = document.querySelector('meta[name="keywords"]'); + let keywords; + + try { + keywords = win.document.querySelector("meta[name='keywords']"); + } catch (e) { + keywords = window.document.querySelector("meta[name='keywords']"); + } - if (keywords && keywords.content) utils.mergeDeep(ortb2, {site: { keywords: keywords.content.replace(/\s/g, '')}}); + if (keywords && keywords.content) utils.mergeDeep(ortb2, {site: {keywords: keywords.content.replace(/\s/g, '')}}); } -function filterData(data, key) { - if (!Array.isArray(data)) { - utils.logWarn(`Filtered ${key} data: Must be an array of objects`); - return; +/** + * Filters data based on predefined requirements + * @param {Object} data object from user.data or site.content.data + * @param {String} name of data parent - user/site.content + * @returns {Object} filtered data + */ +export function filterData(data, key) { + // If data is not an array or does not exist, return null + if (!Array.isArray(data) || !data.length) { + utils.logWarn(`Filtered ${key}.data[]: Must be an array of objects`); + return null; } let duplicate = data.filter(index => { + // If index not an object, name does not exist, segment does not exist, or segment is not an array + // log warning and filter data index if (typeof index !== 'object' || !index.name || !index.segment || !Array.isArray(index.segment)) { - utils.logWarn(`Filtered ${key}.data: must be an object containing name and segment`, index); + utils.logWarn(`Filtered ${key}.data[]: must be an object containing name and segment`, index); return false; } return true; }).reduce((result, value) => { + // If ext exists and is not an object, log warning and filter data index if (value.ext && (typeof value.ext !== 'object' || Array.isArray(value.ext))) { - utils.logWarn(`Filtered ext attribute from ${key}.data: must be an object`, value); + utils.logWarn(`Filtered ext attribute from ${key}.data[]: must be an object`, value); delete value.ext; - } + } value.segment = value.segment.filter(el => { - if (!el.id || typeof el.id !== 'string') { - utils.logWarn(`Filtered ${key}.data.segment: id is required and must be a string`, el); + // For each segment index, check that id exists and is string, otherwise filter index + if (!el.id || typeof el.id !== 'string') { + utils.logWarn(`Filtered ${key}.data[].segment: id is required and must be a string`, el); return false; } return true; }); + // Check that segment data had not all been filtered out, else log warning and filter data index if (value.segment.length) { result.push(value); } else { @@ -93,111 +129,80 @@ function filterData(data, key) { } /** - * Retrieve an item from storage if it exists and hasn't expired. - * @param {string} key Key of the item. - * @returns {string|null} Value of the item. + * Validates ortb2 object and filters out invalid data + * @param {Object} ortb2 object + * @returns {Object} validated/filtered data */ -export function getStorageItem(key) { - let val = null; - - try { - const expVal = storage.getDataFromLocalStorage(key + '_exp'); - - if (!expVal) { - // If there is no expiry time, then just return the item - val = storage.getDataFromLocalStorage(key); - } else { - // Only return the item if it hasn't expired yet. - // Otherwise delete the item. - const expDate = new Date(expVal); - const isValid = (expDate.getTime() - Date.now()) > 0; - if (isValid) { - val = storage.getDataFromLocalStorage(key); - } else { - removeStorageItem(key); - } - } - } catch (e) { - utils.logMessage(e); - } - - return val; -} - -function validateFpd(obj) { - console.log("FPD", obj); - let validObject = Object.assign({}, Object.keys(obj).filter(key => { +export function validateFpd(obj) { + // Filter out imp property if exists + let validObject = Object.assign({}, Object.keys(obj).filter(key => { if (key !== 'imp') return key; utils.logWarn('Filtered imp property in ortb2 data'); }).reduce((result, key) => { - let prop = obj[key]; let modified = {}; - let optout = (storage.cookiesAreEnabled() && storage.getCookie(name)) || - (storage.hasLocalStorage() && getStorageItem(name));console.log(output); + // Checks for existsnece of pubcid optout cookie/storage + // if exists, filters user data out + let optout = (storage.cookiesAreEnabled() && storage.getCookie('_pubcid_optout')) || + (storage.hasLocalStorage() && storage.getDataFromLocalStorage('_pubcid_optout')); if (key === 'user' && optout) { utils.logWarn(`Filtered ${key} data: pubcid optout found`); return result; } - modified = Object.keys(obj[key]).reduce((combined, keyData) => { - let data; + // Create validated object by looping through ortb2 properties + modified = (typeof obj[key] === 'object' && !Array.isArray(obj[key])) + ? Object.keys(obj[key]).reduce((combined, keyData) => { + let data; - if (key === 'user' && keyData === 'data') { - data = filterData(obj[key][keyData], key); + // If key is user.data, pass into filterData to remove invalid data and return + // Else if key is site.content.data, pass into filterData to remove invalid data and return + // Else return data unfiltered + if (key === 'user' && keyData === 'data') { + data = filterData(obj[key][keyData], key); - if (data) combined[keyData] = data; - } else if(key === 'site' && keyData === 'content' && obj[key][keyData].data) { - let content = Object.keys(obj[key][keyData]).reduce((merged, contentData) => { - if (contentData === 'data') { - data = filterData(obj[key][keyData][contentData], key + '.content'); + if (data) combined[keyData] = data; + } else if (key === 'site' && keyData === 'content' && obj[key][keyData].data) { + let content = Object.keys(obj[key][keyData]).reduce((merged, contentData) => { + if (contentData === 'data') { + data = filterData(obj[key][keyData][contentData], key + '.content'); - if (data) merged[contentData] = data; - } else { - merged[contentData] = obj[key][keyData][contentData]; - } + if (data) merged[contentData] = data; + } else { + merged[contentData] = obj[key][keyData][contentData]; + } - return merged; - }, {}); + return merged; + }, {}); - if (Object.keys(content).length) combined[keyData] = content; - } else { - combined[keyData] = obj[key][keyData]; - } + if (Object.keys(content).length) combined[keyData] = content; + } else { + combined[keyData] = obj[key][keyData]; + } - return combined; - }, {}); + return combined; + }, {}) : obj[key]; + // Check if modified data has data and return if (Object.keys(modified).length) result[key] = modified; - return result; + return result; }, {})); - - console.log(validObject); + // Return validated data return validObject; } -function hasChanged(config) { - Object.keys(ortb2).forEach(key => { - if (!utils.deepEqual(ortb2[key], config[key])) return false - }); - - return true; -} - -function storeValue(config) { - Object.keys(ortb2).forEach(key => { - if (!utils.deepEqual(ortb2[key],config)) ortb2[key] = config[key]; - }); -} - +/** +* Resets global ortb2 data +*/ +export const resetOrtb2 = () => { ortb2 = {} }; /** - * - */ +* Sets default values to ortb2 if exists and adds currency and ortb2 setConfig callbacks on init +*/ export function init() { setReferer(); setPage(); @@ -205,24 +210,27 @@ export function init() { setDimensions(); setKeywords(); - config.getConfig('currency', conf => { - if (!conf.currency.adServerCurrency) return; + config.setConfig({ortb2: utils.mergeDeep({}, validateFpd(ortb2), config.getConfig('ortb2'))}); +} - utils.mergeDeep(ortb2, {cur: conf.currency.adServerCurrency}); - shouldRun = true; - config.setConfig(config.getConfig('ortb2') || {ortb2: {}}); - }); +// Set currency setConfig callback +config.getConfig('currency', conf => { + if (!conf.currency.adServerCurrency) return; - config.getConfig('ortb2', conf => { - if (!shouldRun || !hasChanged(conf.ortb2)) { - shouldRun = true; - return; - } + utils.mergeDeep(ortb2, {cur: conf.currency.adServerCurrency}); + shouldRun = true; + config.setConfig({ortb2: utils.mergeDeep({}, validateFpd(ortb2), config.getConfig('ortb2'))}); +}); - conf.ortb2 = validateFpd(utils.mergeDeep(ortb2, conf.ortb2)); +// Set ortb2 setConfig callback to pass data through validator +config.getConfig('ortb2', conf => { + if (!shouldRun) { + shouldRun = true; + } else { + conf.ortb2 = validateFpd(utils.mergeDeep({}, ortb2, conf.ortb2)); shouldRun = false; config.setConfig({ortb2: conf.ortb2}); - }); -} + } +}); init(); diff --git a/test/spec/modules/firstPartyData_spec.js b/test/spec/modules/firstPartyData_spec.js new file mode 100644 index 00000000000..a08f6950323 --- /dev/null +++ b/test/spec/modules/firstPartyData_spec.js @@ -0,0 +1,587 @@ +import {expect} from 'chai'; +import * as utils from 'src/utils.js'; +import {config} from 'src/config.js'; +import { getGlobal } from 'src/prebidGlobal.js'; +import CONSTANTS from 'src/constants.json'; +import { getRefererInfo } from 'src/refererDetection.js' +import { + filterData, + validateFpd, + init, + resetOrtb2 +} from 'modules/firstPartyData/index.js'; +import events from 'src/events.js'; + +describe('the first party data module', function () { + let sandbox, + logErrorSpy; + + let ortb2 = { + device: { + height: 911, + width: 1733 + }, + user: { + data: [{ + segment: [{ + id: 'foo' + }], + name: 'bar', + ext: 'string' + }] + }, + site: { + content: { + data: [{ + segment: [{ + id: 'test' + }], + name: 'content', + ext: { + foo: 'bar' + } + }] + } + } + }; + + let conf = { + device: { + height: 500, + width: 750 + }, + user: { + keywords: 'test1, test2', + gender: 'f', + data: [{ + segment: [{ + id: 'test' + }], + name: 'alt' + }] + }, + site: { + ref: 'domain.com', + page: 'www.domain.com/test', + ext: { + data: { + inventory: ['first'] + } + } + } + }; + + beforeEach(function () { + sandbox = sinon.sandbox.create(); + logErrorSpy = sinon.spy(utils, 'logError'); + }); + + afterEach(function () { + sandbox.restore(); + utils.logError.restore(); + config.resetConfig(); + resetOrtb2(); + }); + + describe('filtering first party data', function () { + it('returns null if empty data array should', function () { + let validated; + let duplicate = utils.deepClone(ortb2); + duplicate.user.data = []; + duplicate.site.content.data = []; + + validated = filterData(duplicate.user.data, 'user'); + expect(validated).to.equal(null); + validated = filterData(duplicate.site.content.data, 'site'); + expect(validated).to.equal(null); + }); + + it('returns null if name does not exist', function () { + let validated; + let duplicate = utils.deepClone(ortb2); + delete duplicate.user.data[0].name; + delete duplicate.site.content.data[0].name; + + validated = filterData(duplicate.user.data, 'user'); + expect(validated).to.equal(null); + validated = filterData(duplicate.site.content.data, 'site'); + expect(validated).to.equal(null); + }); + + it('returns null if segment does not exist', function () { + let validated; + let duplicate = utils.deepClone(ortb2); + delete duplicate.user.data[0].segment; + delete duplicate.site.content.data[0].segment; + + validated = filterData(duplicate.user.data, 'user'); + expect(validated).to.equal(null); + validated = filterData(duplicate.site.content.data, 'site'); + expect(validated).to.equal(null); + }); + + it('returns unfiltered data', function () { + let validated; + let duplicate = utils.deepClone(ortb2); + + delete duplicate.user.data[0].ext; + + validated = filterData(duplicate.user.data, 'user'); + expect(validated).to.deep.equal(duplicate.user.data); + validated = filterData(duplicate.site.content.data, 'site'); + expect(validated).to.deep.equal(duplicate.site.content.data); + }); + + it('returns data filtering data[0].ext for wrong type', function () { + let validated; + let duplicate = utils.deepClone(ortb2); + let expected = { + user: [{ + segment: [{ + id: 'foo' + }], + name: 'bar' + }], + site: [{ + segment: [{ + id: 'test' + }], + name: 'content', + }] + }; + + duplicate.site.content.data[0].ext = [1, 3, 5]; + + validated = filterData(duplicate.user.data, 'user'); + expect(validated).to.deep.equal(expected.user); + validated = filterData(duplicate.site.content.data, 'user'); + expect(validated).to.deep.equal(expected.site); + }); + + it('returns user data filtering data[0].segment[1] for missing id', function () { + let duplicate = utils.deepClone(ortb2); + let expected = [{ + segment: [{ + id: 'foo' + }], + name: 'bar' + }]; + duplicate.user.data[0].segment.push({foo: 'bar'}); + + let validated = filterData(duplicate.user.data, 'user'); + expect(validated).to.deep.equal(expected); + }); + + it('returns undefined for data[0].segment[0] for missing id', function () { + let duplicate = utils.deepClone(ortb2); + duplicate.user.data[0].segment[0] = [{test: 1}]; + + let validated = filterData(duplicate.user.data, 'user'); + expect(validated).to.equal(null); + }); + + it('returns data filtering data[0].segement[1] and data[0].ext', function () { + let duplicate = utils.deepClone(ortb2); + let expected = [{ + segment: [{ + id: 'foo' + }], + name: 'bar' + }, { + segment: [{ + id: '123' + }], + name: 'test-2', + ext: { + foo: 'bar' + } + }]; + + duplicate.user.data[0].segment.push({test: 3}); + duplicate.user.data.push({segment: [{id: '123'}], name: 'test-2', ext: {foo: 'bar'}}); + + let validated = filterData(duplicate.user.data, 'user'); + expect(validated).to.deep.equal(expected); + }); + }); + + describe('validating first party data', function () { + it('filters user.data[0].ext for incorrect type', function () { + let validated; + let duplicate = utils.deepClone(ortb2); + let expected = { + device: { + height: 911, + width: 1733 + }, + user: { + data: [{ + segment: [{ + id: 'foo' + }], + name: 'bar' + }] + }, + site: { + content: { + data: [{ + segment: [{ + id: 'test' + }], + name: 'content', + ext: { + foo: 'bar' + } + }] + } + } + }; + + validated = validateFpd(duplicate); + expect(validated).to.deep.equal(expected); + }); + + it('filters user and site for empty data', function () { + let validated; + let duplicate = utils.deepClone(ortb2); + let expected = { + device: { + height: 911, + width: 1733 + } + }; + + duplicate.user.data = []; + duplicate.site.content.data = []; + + validated = validateFpd(duplicate); + expect(validated).to.deep.equal(expected); + }); + + it('filters user for empty valid segment values', function () { + let validated; + let duplicate = utils.deepClone(ortb2); + let expected = { + device: { + height: 911, + width: 1733 + }, + site: { + content: { + data: [{ + segment: [{ + id: 'test' + }], + name: 'content', + ext: { + foo: 'bar' + } + }] + } + } + }; + + duplicate.user.data[0].segment.push({test: 3}); + duplicate.user.data[0].segment[0] = {foo: 'bar'}; + + validated = validateFpd(duplicate); + expect(validated).to.deep.equal(expected); + }); + + it('filters user.data[0].ext and site.content.data[0].segement[1] for invalid data', function () { + let validated; + let duplicate = utils.deepClone(ortb2); + let expected = { + device: { + height: 911, + width: 1733 + }, + user: { + data: [{ + segment: [{ + id: 'foo' + }], + name: 'bar' + }] + }, + site: { + content: { + data: [{ + segment: [{ + id: 'test' + }], + name: 'content', + ext: { + foo: 'bar' + } + }] + } + } + }; + + duplicate.site.content.data[0].segment.push({test: 3}); + + validated = validateFpd(duplicate); + expect(validated).to.deep.equal(expected); + }); + }); + + describe('first party data intitializing', function () { + let width; + let widthStub; + let height; + let heightStub; + let querySelectorStub; + let canonical; + let keywords; + + before(function() { + canonical = document.createElement('link'); + canonical.rel = 'canonical'; + keywords = document.createElement('meta'); + keywords.name = 'keywords'; + querySelectorStub = sinon.stub(window.top.document, 'querySelector'); + querySelectorStub.withArgs("link[rel='canonical']").returns(canonical); + querySelectorStub.withArgs("meta[name='keywords']").returns(keywords); + widthStub = sinon.stub(window.top, 'innerWidth').get(function () { + return width; + }); + heightStub = sinon.stub(window.top, 'innerHeight').get(function () { + return height; + }); + }); + + after(function() { + widthStub.restore(); + heightStub.restore(); + querySelectorStub.restore(); + }); + + it('sets default referer and dimension values to ortb2 data', function () { + let validated; + + width = 1120; + height = 750; + + init(); + + validated = config.getConfig('ortb2'); + expect(validated.site.ref).to.equal(getRefererInfo().referer); + expect(validated.site.page).to.be.undefined; + expect(validated.site.domain).to.be.undefined; + expect(validated.device).to.deep.equal({width: 1120, height: 750}); + expect(validated.site.keywords).to.be.undefined; + }); + + it('sets page and domain values to ortb2 data if canonical link exists', function () { + let validated; + + width = 800; + height = 400; + canonical.href = 'https://www.domain.com/path?query=12345'; + + init(); + + validated = config.getConfig('ortb2'); + expect(validated.site.ref).to.equal(getRefererInfo().referer); + expect(validated.site.page).to.equal('https://www.domain.com/path?query=12345'); + expect(validated.site.domain).to.equal('domain.com'); + expect(validated.device).to.deep.to.equal({width: 800, height: 400}); + expect(validated.site.keywords).to.be.undefined; + }); + + it('sets keyword values to ortb2 data if keywords meta exists', function () { + let validated; + + width = 1120; + height = 750; + keywords.content = 'value1,value2,value3'; + + init(); + + validated = config.getConfig('ortb2'); + expect(validated.site.ref).to.equal(getRefererInfo().referer); + expect(validated.site.page).to.equal('https://www.domain.com/path?query=12345'); + expect(validated.site.domain).to.equal('domain.com'); + expect(validated.device).to.deep.to.equal({width: 1120, height: 750}); + expect(validated.site.keywords).to.equal('value1,value2,value3'); + }); + + it('only sets values that do not exist in ortb2 config', function () { + let validated; + + config.setConfig({ortb2: {site: {ref: 'https://testpage.com', domain: 'newDomain.com'}}}); + + init(); + + validated = config.getConfig('ortb2'); + expect(validated.site.ref).to.equal('https://testpage.com'); + expect(validated.site.page).to.equal('https://www.domain.com/path?query=12345'); + expect(validated.site.domain).to.equal('newDomain.com'); + expect(validated.device).to.deep.to.equal({width: 1120, height: 750}); + expect(validated.site.keywords).to.equal('value1,value2,value3'); + }); + + it('filters ortb2 data that is set prior to init firing', function () { + let validated; + let conf = { + ortb2: { + user: { + data: {}, + gender: 'f', + age: 45 + }, + site: { + content: { + data: [{ + segment: { + test: 1 + }, + name: 'foo' + }, { + segment: [{ + id: 'test' + }, { + id: 3 + }], + name: 'bar' + }] + } + }, + device: { + width: 1, + height: 1 + } + } + }; + + config.setConfig(conf); + + init(); + + validated = config.getConfig('ortb2'); + expect(validated.site.ref).to.equal(getRefererInfo().referer); + expect(validated.site.page).to.equal('https://www.domain.com/path?query=12345'); + expect(validated.site.domain).to.equal('domain.com'); + expect(validated.site.content.data).to.deep.equal([{segment: [{id: 'test'}], name: 'bar'}]); + expect(validated.user.data).to.be.undefined; + expect(validated.device).to.deep.to.equal({width: 1, height: 1}); + expect(validated.site.keywords).to.to.equal('value1,value2,value3'); + }); + + it('filters ortb2 data that is set after init firing', function () { + let validated; + let conf = { + ortb2: { + user: { + data: {}, + gender: 'f', + age: 45 + }, + site: { + content: { + data: [{ + segment: { + test: 1 + }, + name: 'foo' + }, { + segment: [{ + id: 'test' + }, { + id: 3 + }], + name: 'bar' + }] + } + }, + device: { + width: 1, + height: 1 + } + } + }; + + init(); + + config.setConfig(conf); + + validated = config.getConfig('ortb2'); + expect(validated.site.ref).to.equal(getRefererInfo().referer); + expect(validated.site.page).to.equal('https://www.domain.com/path?query=12345'); + expect(validated.site.domain).to.equal('domain.com'); + expect(validated.site.content.data).to.deep.equal([{segment: [{id: 'test'}], name: 'bar'}]); + expect(validated.user.data).to.be.undefined; + expect(validated.device).to.deep.to.equal({width: 1, height: 1}); + expect(validated.site.keywords).to.to.equal('value1,value2,value3'); + }); + + it('should not overwrite existing data with default settings', function () { + let validated; + let conf = { + ortb2: { + site: { + ref: 'https://referer.com' + } + } + }; + + config.setConfig(conf); + + init(); + + validated = config.getConfig('ortb2'); + expect(validated.site.ref).to.equal('https://referer.com'); + }); + + it('should allow overwrite default data with setConfig', function () { + let validated; + let conf = { + ortb2: { + site: { + ref: 'https://referer.com' + } + } + }; + + init(); + + config.setConfig(conf); + + validated = config.getConfig('ortb2'); + expect(validated.site.ref).to.equal('https://referer.com'); + }); + + it('should add currency if currency config exists prior to init firing', function () { + let validated; + let conf = { + currency: { + adServerCurrency: 'USD' + } + }; + + config.setConfig(conf); + + init(); + + validated = config.getConfig('ortb2'); + expect(validated.cur).to.equal('USD'); + }); + + it('should add currency if currency config exists after init firing', function () { + let validated; + let conf = { + currency: { + adServerCurrency: 'JAP' + } + }; + + config.setConfig(conf); + + init(); + + validated = config.getConfig('ortb2'); + expect(validated.cur).to.equal('JAP'); + }); + }); +}); From 8e414712b6cf19379c8ddd62f6f62459b3dd0e06 Mon Sep 17 00:00:00 2001 From: Michael Moschovas Date: Mon, 22 Mar 2021 13:03:12 -0400 Subject: [PATCH 05/18] Added an unsubscribe for tests to run properly --- modules/firstPartyData/index.js | 10 ++++++++-- test/spec/modules/firstPartyData_spec.js | 7 ++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/modules/firstPartyData/index.js b/modules/firstPartyData/index.js index a49b726aa91..a6d354e92d1 100644 --- a/modules/firstPartyData/index.js +++ b/modules/firstPartyData/index.js @@ -204,6 +204,7 @@ export const resetOrtb2 = () => { ortb2 = {} }; * Sets default values to ortb2 if exists and adds currency and ortb2 setConfig callbacks on init */ export function init() { + // Set defaults if applicable setReferer(); setPage(); setDomain(); @@ -214,7 +215,7 @@ export function init() { } // Set currency setConfig callback -config.getConfig('currency', conf => { +const curListener = config.getConfig('currency', conf => { if (!conf.currency.adServerCurrency) return; utils.mergeDeep(ortb2, {cur: conf.currency.adServerCurrency}); @@ -223,7 +224,7 @@ config.getConfig('currency', conf => { }); // Set ortb2 setConfig callback to pass data through validator -config.getConfig('ortb2', conf => { +const ortb2Listener = config.getConfig('ortb2', conf => { if (!shouldRun) { shouldRun = true; } else { @@ -233,4 +234,9 @@ config.getConfig('ortb2', conf => { } }); +/** +* Removes listener +*/ +export const unsubscribe = () => { curListener(); ortb2Listener(); }; + init(); diff --git a/test/spec/modules/firstPartyData_spec.js b/test/spec/modules/firstPartyData_spec.js index a08f6950323..6890e59f1d9 100644 --- a/test/spec/modules/firstPartyData_spec.js +++ b/test/spec/modules/firstPartyData_spec.js @@ -8,7 +8,8 @@ import { filterData, validateFpd, init, - resetOrtb2 + resetOrtb2, + unsubscribe } from 'modules/firstPartyData/index.js'; import events from 'src/events.js'; @@ -76,6 +77,10 @@ describe('the first party data module', function () { logErrorSpy = sinon.spy(utils, 'logError'); }); + after(function () { + unsubscribe(); + }); + afterEach(function () { sandbox.restore(); utils.logError.restore(); From 5974f3f1076a6c952adf0c528548f6974815e09a Mon Sep 17 00:00:00 2001 From: Michael Moschovas Date: Tue, 23 Mar 2021 12:09:36 -0400 Subject: [PATCH 06/18] Reworked logic to use bidderRequests hook to update global/bidder configs instead of subscribing - former method was preventing tests from completing properly --- modules/firstPartyData/index.js | 74 ++++++----- test/spec/modules/firstPartyData_spec.js | 154 ++++++++++++----------- 2 files changed, 122 insertions(+), 106 deletions(-) diff --git a/modules/firstPartyData/index.js b/modules/firstPartyData/index.js index a6d354e92d1..5e67f3dfd23 100644 --- a/modules/firstPartyData/index.js +++ b/modules/firstPartyData/index.js @@ -5,12 +5,13 @@ import { config } from '../../src/config.js'; import * as utils from '../../src/utils.js'; +import { getHook } from '../../src/hook.js'; import { getRefererInfo } from '../../src/refererDetection.js' import { getStorageManager } from '../../src/storageManager.js'; -const storage = getStorageManager(); +const STORAGE = getStorageManager(); let ortb2 = {}; -let shouldRun = true; +let globalConfig = {}; let win = (window === window.top) ? window : window.top; /** @@ -77,6 +78,18 @@ function setKeywords() { if (keywords && keywords.content) utils.mergeDeep(ortb2, {site: {keywords: keywords.content.replace(/\s/g, '')}}); } +/** + * Checks for currency and if exists merges into ortb2 global data + * Sets listener for currency if changes occur or doesnt exist when run + */ +function setCurrency() { + let cur = {...config.getConfig('currency')}; + + if (cur && cur.adServerCurrency) { + utils.mergeDeep(ortb2, {cur: cur.adServerCurrency}); + } +} + /** * Filters data based on predefined requirements * @param {Object} data object from user.data or site.content.data @@ -134,6 +147,7 @@ export function filterData(data, key) { * @returns {Object} validated/filtered data */ export function validateFpd(obj) { + if (!obj) return {}; // Filter out imp property if exists let validObject = Object.assign({}, Object.keys(obj).filter(key => { if (key !== 'imp') return key; @@ -144,8 +158,8 @@ export function validateFpd(obj) { // Checks for existsnece of pubcid optout cookie/storage // if exists, filters user data out - let optout = (storage.cookiesAreEnabled() && storage.getCookie('_pubcid_optout')) || - (storage.hasLocalStorage() && storage.getDataFromLocalStorage('_pubcid_optout')); + let optout = (STORAGE.cookiesAreEnabled() && STORAGE.getCookie('_pubcid_optout')) || + (STORAGE.hasLocalStorage() && STORAGE.getDataFromLocalStorage('_pubcid_optout')); if (key === 'user' && optout) { utils.logWarn(`Filtered ${key} data: pubcid optout found`); @@ -204,39 +218,39 @@ export const resetOrtb2 = () => { ortb2 = {} }; * Sets default values to ortb2 if exists and adds currency and ortb2 setConfig callbacks on init */ export function init() { - // Set defaults if applicable setReferer(); setPage(); setDomain(); setDimensions(); setKeywords(); + setCurrency(); - config.setConfig({ortb2: utils.mergeDeep({}, validateFpd(ortb2), config.getConfig('ortb2'))}); -} + let conf = utils.mergeDeep({}, ortb2, validateFpd(config.getConfig('ortb2'))); -// Set currency setConfig callback -const curListener = config.getConfig('currency', conf => { - if (!conf.currency.adServerCurrency) return; - - utils.mergeDeep(ortb2, {cur: conf.currency.adServerCurrency}); - shouldRun = true; - config.setConfig({ortb2: utils.mergeDeep({}, validateFpd(ortb2), config.getConfig('ortb2'))}); -}); - -// Set ortb2 setConfig callback to pass data through validator -const ortb2Listener = config.getConfig('ortb2', conf => { - if (!shouldRun) { - shouldRun = true; - } else { - conf.ortb2 = validateFpd(utils.mergeDeep({}, ortb2, conf.ortb2)); - shouldRun = false; - config.setConfig({ortb2: conf.ortb2}); + if (!utils.deepEqual(conf, globalConfig)) { + config.setConfig({ortb2: conf}); + globalConfig = {...conf}; + resetOrtb2(); } -}); -/** -* Removes listener -*/ -export const unsubscribe = () => { curListener(); ortb2Listener(); }; + let bidderDuplicate = {...config.getBidderConfig()}; + + Object.keys(bidderDuplicate).forEach(bidder => { + let modConf = Object.keys(bidderDuplicate[bidder]).reduce((res, key) => { + let valid = (key !== 'ortb2') ? bidderDuplicate[bidder][key] : validateFpd(bidderDuplicate[bidder][key]); + + if (valid) res[key] = valid; + + return res; + }, {}); + + if (Object.keys(modConf).length) config.setBidderConfig({bidders: [bidder], config: modConf}); + }); +} + +function addBidderRequestHook(fn, bidderRequests) { + init(); + fn.call(this, bidderRequests); +} -init(); +getHook('addBidderRequests').before(addBidderRequestHook); diff --git a/test/spec/modules/firstPartyData_spec.js b/test/spec/modules/firstPartyData_spec.js index 6890e59f1d9..6c0ddcc6e4e 100644 --- a/test/spec/modules/firstPartyData_spec.js +++ b/test/spec/modules/firstPartyData_spec.js @@ -3,13 +3,12 @@ import * as utils from 'src/utils.js'; import {config} from 'src/config.js'; import { getGlobal } from 'src/prebidGlobal.js'; import CONSTANTS from 'src/constants.json'; -import { getRefererInfo } from 'src/refererDetection.js' +import {getRefererInfo} from 'src/refererDetection.js' import { filterData, validateFpd, init, - resetOrtb2, - unsubscribe + resetOrtb2 } from 'modules/firstPartyData/index.js'; import events from 'src/events.js'; @@ -74,16 +73,10 @@ describe('the first party data module', function () { beforeEach(function () { sandbox = sinon.sandbox.create(); - logErrorSpy = sinon.spy(utils, 'logError'); - }); - - after(function () { - unsubscribe(); }); afterEach(function () { sandbox.restore(); - utils.logError.restore(); config.resetConfig(); resetOrtb2(); }); @@ -345,6 +338,9 @@ describe('the first party data module', function () { canonical.rel = 'canonical'; keywords = document.createElement('meta'); keywords.name = 'keywords'; + }); + + beforeEach(function() { querySelectorStub = sinon.stub(window.top.document, 'querySelector'); querySelectorStub.withArgs("link[rel='canonical']").returns(canonical); querySelectorStub.withArgs("meta[name='keywords']").returns(keywords); @@ -356,10 +352,14 @@ describe('the first party data module', function () { }); }); - after(function() { + afterEach(function() { widthStub.restore(); heightStub.restore(); querySelectorStub.restore(); + canonical = document.createElement('link'); + canonical.rel = 'canonical'; + keywords = document.createElement('meta'); + keywords.name = 'keywords'; }); it('sets default referer and dimension values to ortb2 data', function () { @@ -381,8 +381,6 @@ describe('the first party data module', function () { it('sets page and domain values to ortb2 data if canonical link exists', function () { let validated; - width = 800; - height = 400; canonical.href = 'https://www.domain.com/path?query=12345'; init(); @@ -391,23 +389,21 @@ describe('the first party data module', function () { expect(validated.site.ref).to.equal(getRefererInfo().referer); expect(validated.site.page).to.equal('https://www.domain.com/path?query=12345'); expect(validated.site.domain).to.equal('domain.com'); - expect(validated.device).to.deep.to.equal({width: 800, height: 400}); + expect(validated.device).to.deep.to.equal({width: 1120, height: 750}); expect(validated.site.keywords).to.be.undefined; }); it('sets keyword values to ortb2 data if keywords meta exists', function () { let validated; - width = 1120; - height = 750; keywords.content = 'value1,value2,value3'; init(); validated = config.getConfig('ortb2'); expect(validated.site.ref).to.equal(getRefererInfo().referer); - expect(validated.site.page).to.equal('https://www.domain.com/path?query=12345'); - expect(validated.site.domain).to.equal('domain.com'); + expect(validated.site.page).to.be.undefined; + expect(validated.site.domain).to.be.undefined; expect(validated.device).to.deep.to.equal({width: 1120, height: 750}); expect(validated.site.keywords).to.equal('value1,value2,value3'); }); @@ -421,13 +417,13 @@ describe('the first party data module', function () { validated = config.getConfig('ortb2'); expect(validated.site.ref).to.equal('https://testpage.com'); - expect(validated.site.page).to.equal('https://www.domain.com/path?query=12345'); + expect(validated.site.page).to.be.undefined; expect(validated.site.domain).to.equal('newDomain.com'); expect(validated.device).to.deep.to.equal({width: 1120, height: 750}); - expect(validated.site.keywords).to.equal('value1,value2,value3'); + expect(validated.site.keywords).to.be.undefined; }); - it('filters ortb2 data that is set prior to init firing', function () { + it('filters ortb2 data that is set', function () { let validated; let conf = { ortb2: { @@ -461,6 +457,9 @@ describe('the first party data module', function () { }; config.setConfig(conf); + canonical.href = 'https://www.domain.com/path?query=12345'; + width = 1120; + height = 750; init(); @@ -471,57 +470,28 @@ describe('the first party data module', function () { expect(validated.site.content.data).to.deep.equal([{segment: [{id: 'test'}], name: 'bar'}]); expect(validated.user.data).to.be.undefined; expect(validated.device).to.deep.to.equal({width: 1, height: 1}); - expect(validated.site.keywords).to.to.equal('value1,value2,value3'); + expect(validated.site.keywords).to.be.undefined; }); - it('filters ortb2 data that is set after init firing', function () { + it('should not overwrite existing data with default settings', function () { let validated; let conf = { ortb2: { - user: { - data: {}, - gender: 'f', - age: 45 - }, site: { - content: { - data: [{ - segment: { - test: 1 - }, - name: 'foo' - }, { - segment: [{ - id: 'test' - }, { - id: 3 - }], - name: 'bar' - }] - } - }, - device: { - width: 1, - height: 1 + ref: 'https://referer.com' } } }; - init(); - config.setConfig(conf); + init(); + validated = config.getConfig('ortb2'); - expect(validated.site.ref).to.equal(getRefererInfo().referer); - expect(validated.site.page).to.equal('https://www.domain.com/path?query=12345'); - expect(validated.site.domain).to.equal('domain.com'); - expect(validated.site.content.data).to.deep.equal([{segment: [{id: 'test'}], name: 'bar'}]); - expect(validated.user.data).to.be.undefined; - expect(validated.device).to.deep.to.equal({width: 1, height: 1}); - expect(validated.site.keywords).to.to.equal('value1,value2,value3'); + expect(validated.site.ref).to.equal('https://referer.com'); }); - it('should not overwrite existing data with default settings', function () { + it('should allow overwrite default data with setConfig', function () { let validated; let conf = { ortb2: { @@ -539,54 +509,86 @@ describe('the first party data module', function () { expect(validated.site.ref).to.equal('https://referer.com'); }); - it('should allow overwrite default data with setConfig', function () { + it('should add currency if currency config exists', function () { let validated; let conf = { - ortb2: { - site: { - ref: 'https://referer.com' - } + currency: { + adServerCurrency: 'USD' } }; - init(); - config.setConfig(conf); + init(); + validated = config.getConfig('ortb2'); - expect(validated.site.ref).to.equal('https://referer.com'); + expect(validated.cur).to.equal('USD'); }); - it('should add currency if currency config exists prior to init firing', function () { + it('should filter bidderConfig data', function () { let validated; let conf = { - currency: { - adServerCurrency: 'USD' + bidders: ['bidderA', 'bidderB'], + config: { + ortb2: { + site: { + keywords: 'other', + ref: 'https://domain.com' + }, + user: { + keywords: 'test', + data: [{ + segment: [{id: 4}], + name: 't' + }] + } + } } }; - config.setConfig(conf); + config.setBidderConfig(conf); init(); - validated = config.getConfig('ortb2'); - expect(validated.cur).to.equal('USD'); + validated = config.getBidderConfig(); + expect(validated.bidderA.ortb2).to.not.be.undefined; + expect(validated.bidderA.ortb2.user.data).to.be.undefined; + expect(validated.bidderA.ortb2.user.keywords).to.equal('test'); + expect(validated.bidderA.ortb2.site.keywords).to.equal('other'); + expect(validated.bidderA.ortb2.site.ref).to.equal('https://domain.com'); }); - it('should add currency if currency config exists after init firing', function () { + it('should not filter bidderConfig data as it is valid', function () { let validated; let conf = { - currency: { - adServerCurrency: 'JAP' + bidders: ['bidderA', 'bidderB'], + config: { + ortb2: { + site: { + keywords: 'other', + ref: 'https://domain.com' + }, + user: { + keywords: 'test', + data: [{ + segment: [{id: 'data1_id'}], + name: 'data1' + }] + } + } } }; - config.setConfig(conf); + config.setBidderConfig(conf); init(); - validated = config.getConfig('ortb2'); - expect(validated.cur).to.equal('JAP'); + validated = config.getBidderConfig(); + expect(validated.bidderA.ortb2).to.not.be.undefined; + expect(validated.bidderA.ortb2.user.data).to.deep.equal([{segment: [{id: 'data1_id'}], name: 'data1'}]); + expect(validated.bidderA.ortb2.user.keywords).to.equal('test'); + expect(validated.bidderA.ortb2.site.keywords).to.equal('other'); + expect(validated.bidderA.ortb2.site.ref).to.equal('https://domain.com'); }); }); }); From f8741b4258d5653db80c6037bfacd15e3da0476e Mon Sep 17 00:00:00 2001 From: Michael Moschovas Date: Tue, 23 Mar 2021 12:33:50 -0400 Subject: [PATCH 07/18] Merge master --- test/spec/modules/firstPartyData_spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/spec/modules/firstPartyData_spec.js b/test/spec/modules/firstPartyData_spec.js index 6c0ddcc6e4e..cdbf8460810 100644 --- a/test/spec/modules/firstPartyData_spec.js +++ b/test/spec/modules/firstPartyData_spec.js @@ -3,7 +3,7 @@ import * as utils from 'src/utils.js'; import {config} from 'src/config.js'; import { getGlobal } from 'src/prebidGlobal.js'; import CONSTANTS from 'src/constants.json'; -import {getRefererInfo} from 'src/refererDetection.js' +import {getRefererInfo} from 'src/refererDetection.js'; import { filterData, validateFpd, From 11492e99860fa5351a46702ed52cd8bab9dbb1de Mon Sep 17 00:00:00 2001 From: Michael Moschovas Date: Mon, 29 Mar 2021 10:02:52 -0400 Subject: [PATCH 08/18] Removing unused references. Fixing device data to point to device.h/device.w --- modules/firstPartyData/index.js | 2 +- test/spec/modules/firstPartyData_spec.js | 44 +++++++++++------------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/modules/firstPartyData/index.js b/modules/firstPartyData/index.js index 5e67f3dfd23..3aef9ef6c34 100644 --- a/modules/firstPartyData/index.js +++ b/modules/firstPartyData/index.js @@ -60,7 +60,7 @@ function setDimensions() { height = window.innerHeight || window.document.documentElement.clientHeight || window.document.body.clientHeight; } - utils.mergeDeep(ortb2, {device: {width: width, height: height}}); + utils.mergeDeep(ortb2, {device: {w: width, h: height}}); } /** diff --git a/test/spec/modules/firstPartyData_spec.js b/test/spec/modules/firstPartyData_spec.js index cdbf8460810..68a7e92bf0d 100644 --- a/test/spec/modules/firstPartyData_spec.js +++ b/test/spec/modules/firstPartyData_spec.js @@ -1,8 +1,6 @@ import {expect} from 'chai'; import * as utils from 'src/utils.js'; import {config} from 'src/config.js'; -import { getGlobal } from 'src/prebidGlobal.js'; -import CONSTANTS from 'src/constants.json'; import {getRefererInfo} from 'src/refererDetection.js'; import { filterData, @@ -10,16 +8,14 @@ import { init, resetOrtb2 } from 'modules/firstPartyData/index.js'; -import events from 'src/events.js'; describe('the first party data module', function () { - let sandbox, - logErrorSpy; + let sandbox; let ortb2 = { device: { - height: 911, - width: 1733 + h: 911, + w: 1733 }, user: { data: [{ @@ -47,8 +43,8 @@ describe('the first party data module', function () { let conf = { device: { - height: 500, - width: 750 + h: 500, + w: 750 }, user: { keywords: 'test1, test2', @@ -209,8 +205,8 @@ describe('the first party data module', function () { let duplicate = utils.deepClone(ortb2); let expected = { device: { - height: 911, - width: 1733 + h: 911, + w: 1733 }, user: { data: [{ @@ -244,8 +240,8 @@ describe('the first party data module', function () { let duplicate = utils.deepClone(ortb2); let expected = { device: { - height: 911, - width: 1733 + h: 911, + w: 1733 } }; @@ -261,8 +257,8 @@ describe('the first party data module', function () { let duplicate = utils.deepClone(ortb2); let expected = { device: { - height: 911, - width: 1733 + h: 911, + w: 1733 }, site: { content: { @@ -291,8 +287,8 @@ describe('the first party data module', function () { let duplicate = utils.deepClone(ortb2); let expected = { device: { - height: 911, - width: 1733 + h: 911, + w: 1733 }, user: { data: [{ @@ -374,7 +370,7 @@ describe('the first party data module', function () { expect(validated.site.ref).to.equal(getRefererInfo().referer); expect(validated.site.page).to.be.undefined; expect(validated.site.domain).to.be.undefined; - expect(validated.device).to.deep.equal({width: 1120, height: 750}); + expect(validated.device).to.deep.equal({w: 1120, h: 750}); expect(validated.site.keywords).to.be.undefined; }); @@ -389,7 +385,7 @@ describe('the first party data module', function () { expect(validated.site.ref).to.equal(getRefererInfo().referer); expect(validated.site.page).to.equal('https://www.domain.com/path?query=12345'); expect(validated.site.domain).to.equal('domain.com'); - expect(validated.device).to.deep.to.equal({width: 1120, height: 750}); + expect(validated.device).to.deep.to.equal({w: 1120, h: 750}); expect(validated.site.keywords).to.be.undefined; }); @@ -404,7 +400,7 @@ describe('the first party data module', function () { expect(validated.site.ref).to.equal(getRefererInfo().referer); expect(validated.site.page).to.be.undefined; expect(validated.site.domain).to.be.undefined; - expect(validated.device).to.deep.to.equal({width: 1120, height: 750}); + expect(validated.device).to.deep.to.equal({w: 1120, h: 750}); expect(validated.site.keywords).to.equal('value1,value2,value3'); }); @@ -419,7 +415,7 @@ describe('the first party data module', function () { expect(validated.site.ref).to.equal('https://testpage.com'); expect(validated.site.page).to.be.undefined; expect(validated.site.domain).to.equal('newDomain.com'); - expect(validated.device).to.deep.to.equal({width: 1120, height: 750}); + expect(validated.device).to.deep.to.equal({w: 1120, h: 750}); expect(validated.site.keywords).to.be.undefined; }); @@ -450,8 +446,8 @@ describe('the first party data module', function () { } }, device: { - width: 1, - height: 1 + w: 1, + h: 1 } } }; @@ -469,7 +465,7 @@ describe('the first party data module', function () { expect(validated.site.domain).to.equal('domain.com'); expect(validated.site.content.data).to.deep.equal([{segment: [{id: 'test'}], name: 'bar'}]); expect(validated.user.data).to.be.undefined; - expect(validated.device).to.deep.to.equal({width: 1, height: 1}); + expect(validated.device).to.deep.to.equal({w: 1, h: 1}); expect(validated.site.keywords).to.be.undefined; }); From 9f839e24cb5de28973a9ee1c444baf006b44f5f5 Mon Sep 17 00:00:00 2001 From: Michael Moschovas Date: Tue, 30 Mar 2021 14:30:39 -0400 Subject: [PATCH 09/18] Update to include opt out configuration for enrichments/validations --- modules/firstPartyData/index.js | 21 +++++++-- modules/firstPartyData/index.md | 29 ++++++++++++ test/spec/modules/firstPartyData_spec.js | 56 ++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 modules/firstPartyData/index.md diff --git a/modules/firstPartyData/index.js b/modules/firstPartyData/index.js index 3aef9ef6c34..65d5692feaa 100644 --- a/modules/firstPartyData/index.js +++ b/modules/firstPartyData/index.js @@ -214,10 +214,7 @@ export function validateFpd(obj) { */ export const resetOrtb2 = () => { ortb2 = {} }; -/** -* Sets default values to ortb2 if exists and adds currency and ortb2 setConfig callbacks on init -*/ -export function init() { +function runEnrichments(shouldSkipValidate) { setReferer(); setPage(); setDomain(); @@ -225,6 +222,10 @@ export function init() { setKeywords(); setCurrency(); + if (shouldSkipValidate) config.setConfig({ortb2: utils.mergeDeep({}, ortb2, config.getConfig('ortb2'))}); +} + +function runValidations() { let conf = utils.mergeDeep({}, ortb2, validateFpd(config.getConfig('ortb2'))); if (!utils.deepEqual(conf, globalConfig)) { @@ -248,6 +249,18 @@ export function init() { }); } +/** +* Sets default values to ortb2 if exists and adds currency and ortb2 setConfig callbacks on init +*/ +export function init() { + let conf = config.getConfig('firstPartyData'); + let skipValidations = (conf && conf.skipValidations) || false; + let skipEnrichments = (conf && conf.skipEnrichments) || false; + + if (!skipEnrichments) runEnrichments(skipValidations); + if (!skipValidations) runValidations(); +} + function addBidderRequestHook(fn, bidderRequests) { init(); fn.call(this, bidderRequests); diff --git a/modules/firstPartyData/index.md b/modules/firstPartyData/index.md new file mode 100644 index 00000000000..16c09736fd8 --- /dev/null +++ b/modules/firstPartyData/index.md @@ -0,0 +1,29 @@ +# Overview + +``` +Module Name: First Party Data Module +``` + +# Description + +Module to perform the following functions to allow for consistent set of first party data + +- verify OpenRTB datatypes, remove/warn any that are likely to choke downstream readers +- verify that certain OpenRTB attributes are not specified: just imp for now +- optionally suppress user FPD based on a TBD opt-out signal (_pubcid_optout) +- populate available data into object: referer, meta-keywords, cur + +This module will automatically run both first party data enrichments and validations. There is no configuration required. In order to load the module and opt out of either enrichements or validations, use the below opt out configuration + +# Opt Out Configuration + +``` + +pbjs.setConfig({ + firstPartyData: { + skipValidations: true, // default to false + skipEnrichments: true // default to false + } +}); + +``` diff --git a/test/spec/modules/firstPartyData_spec.js b/test/spec/modules/firstPartyData_spec.js index 68a7e92bf0d..1bdb01e7965 100644 --- a/test/spec/modules/firstPartyData_spec.js +++ b/test/spec/modules/firstPartyData_spec.js @@ -586,5 +586,61 @@ describe('the first party data module', function () { expect(validated.bidderA.ortb2.site.keywords).to.equal('other'); expect(validated.bidderA.ortb2.site.ref).to.equal('https://domain.com'); }); + + it('should not set default values if skipEnrichments is turned on', function () { + let validated; + config.setConfig({'firstPartyData': {skipEnrichments: true}}); + + let conf = { + site: { + keywords: 'other' + }, + user: { + keywords: 'test', + data: [{ + segment: [{id: 'data1_id'}], + name: 'data1' + }] + } + } + ; + + config.setConfig({ortb2: conf}); + + init(); + + validated = config.getConfig(); + expect(validated.ortb2).to.not.be.undefined; + expect(validated.ortb2.device).to.be.undefined; + expect(validated.ortb2.site.ref).to.be.undefined; + expect(validated.ortb2.site.page).to.be.undefined; + expect(validated.ortb2.site.domain).to.be.undefined; + }); + + it('should not validate ortb2 data if skipValidations is turned on', function () { + let validated; + config.setConfig({'firstPartyData': {skipValidations: true}}); + + let conf = { + site: { + keywords: 'other' + }, + user: { + keywords: 'test', + data: [{ + segment: [{id: 'nonfiltered'}] + }] + } + } + ; + + config.setConfig({ortb2: conf}); + + init(); + + validated = config.getConfig(); + expect(validated.ortb2).to.not.be.undefined; + expect(validated.ortb2.user.data).to.deep.equal([{segment: [{id: 'nonfiltered'}]}]); + }); }); }); From ef8ed4b51528e7e32cc0be302d5ec20b65534d3d Mon Sep 17 00:00:00 2001 From: Michael Moschovas Date: Mon, 5 Apr 2021 13:26:22 -0400 Subject: [PATCH 10/18] Modified logic to use ortb2 configuration mapping. This will allow for entries to be added/removed/modified from configuration as opposed to be specifically called out in the validation functions --- modules/firstPartyData/config.js | 118 ++++++++ modules/firstPartyData/index.js | 277 +++++++++++------- test/spec/modules/firstPartyData_spec.js | 341 +++++++++++++++-------- 3 files changed, 521 insertions(+), 215 deletions(-) create mode 100644 modules/firstPartyData/config.js diff --git a/modules/firstPartyData/config.js b/modules/firstPartyData/config.js new file mode 100644 index 00000000000..ad4395af102 --- /dev/null +++ b/modules/firstPartyData/config.js @@ -0,0 +1,118 @@ +/** + * Data type map + */ +const TYPES = { + string: 'string', + object: 'object', + number: 'number', +}; + +/** + * Template to define what ortb2 attributes should be validated + * Accepted fields: + * -- invalid - {Boolean} if true, field is not valid + * -- type - {String} valid data type of field + * -- isArray - {Boolean} if true, field must be an array + * -- childType - {String} used in conjuction with isArray: true, defines valid type of array indices + * -- children - {Object} defines child properties needed to be validated (used only if type: object) + * -- required - {Array} array of strings defining any required properties for object (used only if type: object) + * -- optoutApplies - {Boolean} if true, optout logic will filter if applicable (currently only applies to user object) + */ +export const ORTB_MAP = { + imp: { + invalid: true + }, + cur: { + type: TYPES.string + }, + device: { + type: TYPES.object, + children: { + w: { type: TYPES.number }, + h: { type: TYPES.number } + } + }, + site: { + type: TYPES.object, + children: { + name: { type: TYPES.string }, + domain: { type: TYPES.string }, + page: { type: TYPES.string }, + ref: { type: TYPES.string }, + keywords: { type: TYPES.string }, + search: { type: TYPES.string }, + cat: { + type: TYPES.object, + isArray: true, + childType: TYPES.string + }, + sectioncat: { + type: TYPES.object, + isArray: true, + childType: TYPES.string + }, + pagecat: { + type: TYPES.object, + isArray: true, + childType: TYPES.string + }, + content: { + type: TYPES.object, + isArray: false, + children: { + data: { + type: TYPES.object, + isArray: true, + childType: TYPES.object, + required: ['name', 'segment'], + children: { + segment: { + type: TYPES.object, + isArray: true, + childType: TYPES.object, + required: ['id'], + children: { + id: { type: TYPES.string } + } + }, + name: { type: TYPES.string }, + ext: { type: TYPES.object }, + } + } + } + }, + publisher: { + type: TYPES.object, + isArray: false + }, + } + }, + user: { + type: TYPES.object, + optoutApplies: true, + children: { + yob: { type: TYPES.number }, + gender: { type: TYPES.string }, + keywords: { type: TYPES.string }, + data: { + type: TYPES.object, + isArray: true, + childType: TYPES.object, + required: ['name', 'segment'], + children: { + segment: { + type: TYPES.object, + isArray: true, + childType: TYPES.object, + required: ['id'], + children: { + id: { type: TYPES.string } + } + }, + name: { type: TYPES.string }, + ext: { type: TYPES.object }, + } + } + } + } +} diff --git a/modules/firstPartyData/index.js b/modules/firstPartyData/index.js index 65d5692feaa..4a3186ffb0a 100644 --- a/modules/firstPartyData/index.js +++ b/modules/firstPartyData/index.js @@ -2,30 +2,33 @@ * This module sets default values and validates ortb2 first part data * @module modules/firstPartyData */ - import { config } from '../../src/config.js'; import * as utils from '../../src/utils.js'; +import { ORTB_MAP } from './config.js'; import { getHook } from '../../src/hook.js'; +import { getGlobal } from '../../src/prebidGlobal.js'; +import { addBidderRequests } from '../../src/auction.js'; import { getRefererInfo } from '../../src/refererDetection.js' import { getStorageManager } from '../../src/storageManager.js'; const STORAGE = getStorageManager(); + let ortb2 = {}; -let globalConfig = {}; let win = (window === window.top) ? window : window.top; +let optout; /** * Checks for referer and if exists merges into ortb2 global data */ function setReferer() { - if (getRefererInfo().referer) utils.mergeDeep(ortb2, {site: {ref: getRefererInfo().referer}}); + if (getRefererInfo().referer) utils.mergeDeep(ortb2, { site: { ref: getRefererInfo().referer } }); } /** * Checks for canonical url and if exists merges into ortb2 global data */ function setPage() { - if (getRefererInfo().canonicalUrl) utils.mergeDeep(ortb2, {site: {page: getRefererInfo().canonicalUrl}}); + if (getRefererInfo().canonicalUrl) utils.mergeDeep(ortb2, { site: { page: getRefererInfo().canonicalUrl } }); } /** @@ -42,7 +45,7 @@ function setDomain() { let domain = parseDomain(getRefererInfo().canonicalUrl) - if (domain) utils.mergeDeep(ortb2, {site: {domain: domain}}); + if (domain) utils.mergeDeep(ortb2, { site: { domain: domain } }); } /** @@ -60,7 +63,7 @@ function setDimensions() { height = window.innerHeight || window.document.documentElement.clientHeight || window.document.body.clientHeight; } - utils.mergeDeep(ortb2, {device: {w: width, h: height}}); + utils.mergeDeep(ortb2, { device: { w: width, h: height } }); } /** @@ -75,7 +78,7 @@ function setKeywords() { keywords = window.document.querySelector("meta[name='keywords']"); } - if (keywords && keywords.content) utils.mergeDeep(ortb2, {site: {keywords: keywords.content.replace(/\s/g, '')}}); + if (keywords && keywords.content) utils.mergeDeep(ortb2, { site: { keywords: keywords.content.replace(/\s/g, '') } }); } /** @@ -83,124 +86,181 @@ function setKeywords() { * Sets listener for currency if changes occur or doesnt exist when run */ function setCurrency() { - let cur = {...config.getConfig('currency')}; + let cur = { ...config.getConfig('currency') }; if (cur && cur.adServerCurrency) { - utils.mergeDeep(ortb2, {cur: cur.adServerCurrency}); + utils.mergeDeep(ortb2, { cur: cur.adServerCurrency }); } } /** - * Filters data based on predefined requirements - * @param {Object} data object from user.data or site.content.data - * @param {String} name of data parent - user/site.content - * @returns {Object} filtered data + * Check if data passed is empty + * @param {*} value to test against + * @returns {Boolean} is value empty */ -export function filterData(data, key) { - // If data is not an array or does not exist, return null - if (!Array.isArray(data) || !data.length) { - utils.logWarn(`Filtered ${key}.data[]: Must be an array of objects`); - return null; +function isEmptyData(data) { + let check = true; + + if (typeof data === 'object' && !utils.isEmpty(data)) { + check = false; + } else if (typeof data !== 'object' && (utils.isNumber(data) || data)) { + check = false; } - let duplicate = data.filter(index => { - // If index not an object, name does not exist, segment does not exist, or segment is not an array - // log warning and filter data index - if (typeof index !== 'object' || !index.name || !index.segment || !Array.isArray(index.segment)) { - utils.logWarn(`Filtered ${key}.data[]: must be an object containing name and segment`, index); - return false; - } + return check; +} + +/** + * Check if required keys exist in data object + * @param {Object} data object + * @param {Array} array of required keys + * @param {String} object path (for printing warning) + * @param {Number} index of object value in the data array (for printing warning) + * @returns {Boolean} is requirements fulfilled + */ +function getRequiredData(obj, required, parent, i) { + let check = true; - return true; - }).reduce((result, value) => { - // If ext exists and is not an object, log warning and filter data index - if (value.ext && (typeof value.ext !== 'object' || Array.isArray(value.ext))) { - utils.logWarn(`Filtered ext attribute from ${key}.data[]: must be an object`, value); - delete value.ext; + required.forEach(key => { + if (!obj[key] || isEmptyData(obj[key])) { + check = false; + utils.logWarn(`Filtered ${parent}[] value at index ${i} in ortb2 data: missing required property ${key}`); } + }); - value.segment = value.segment.filter(el => { - // For each segment index, check that id exists and is string, otherwise filter index - if (!el.id || typeof el.id !== 'string') { - utils.logWarn(`Filtered ${key}.data[].segment: id is required and must be a string`, el); - return false; + return check; +} + +/** + * Check if data type is valid + * @param {*} value to test against + * @param {Object} object containing type definition and if should be array bool + * @returns {Boolean} is type fulfilled + */ +function typeValidation(data, mapping) { + let check = false; + + switch (mapping.type) { + case 'string': + if (typeof data === 'string') check = true; + break; + case 'number': + if (typeof data === 'number' && isFinite(data)) check = true; + break; + case 'object': + if (typeof data === 'object') { + if ((Array.isArray(data) && mapping.isArray) || (!Array.isArray(data) && !mapping.isArray)) check = true; } + break; + } + + return check; +} + +/** + * Validates ortb2 data arrays and filters out invalid data + * @param {Array} ortb2 data array + * @param {Object} object defining child type and if array + * @param {String} config path of data array + * @param {String} parent path for logging warnings + * @returns {Array} validated/filtered data + */ +export function filterArrayData(arr, child, path, parent) { + arr = arr.filter((index, i) => { + let check = typeValidation(index, {type: child.type, isArray: child.isArray}); + + if (check && Array.isArray(index) === Boolean(child.isArray)) { return true; - }); + } - // Check that segment data had not all been filtered out, else log warning and filter data index - if (value.segment.length) { - result.push(value); - } else { - utils.logWarn(`Filtered ${key}.data: must contain segment data`); + utils.logWarn(`Filtered ${parent}[] value at index ${i} in ortb2 data: expected type ${child.type}`); + }).filter((index, i) => { + let requiredCheck = true; + let mapping = utils.deepAccess(ORTB_MAP, path); + + if (mapping && mapping.required) requiredCheck = getRequiredData(index, mapping.required, parent, i); + + if (requiredCheck) return true; + }).reduce((result, value, i) => { + let typeBool = false; + let mapping = utils.deepAccess(ORTB_MAP, path); + + switch (child.type) { + case 'string': + result.push(value); + break; + case 'object': + if (mapping && mapping.children) { + let validObject = validateFpd(value, path + '.children.', parent + '.'); + if (Object.keys(validObject).length) { + let requiredCheck = getRequiredData(validObject, mapping.required, parent, i); + + if (requiredCheck) { + result.push(validObject); + typeBool = true; + } + } + } else { + result.push(value); + typeBool = true; + } + break; } + if (!typeBool) utils.logWarn(`Filtered ${parent}[] value at index ${i} in ortb2 data: expected type ${child.type}`); + return result; }, []); - return (duplicate.length) ? duplicate : null; + return arr; } /** * Validates ortb2 object and filters out invalid data * @param {Object} ortb2 object + * @param {String} config path of data array + * @param {String} parent path for logging warnings * @returns {Object} validated/filtered data */ -export function validateFpd(obj) { - if (!obj) return {}; - // Filter out imp property if exists - let validObject = Object.assign({}, Object.keys(obj).filter(key => { - if (key !== 'imp') return key; +export function validateFpd(fpd, path = '', parent = '') { + if (!fpd) return {}; - utils.logWarn('Filtered imp property in ortb2 data'); - }).reduce((result, key) => { - let modified = {}; + // Filter out imp property if exists + let validObject = Object.assign({}, Object.keys(fpd).filter(key => { + let mapping = utils.deepAccess(ORTB_MAP, path + key); - // Checks for existsnece of pubcid optout cookie/storage - // if exists, filters user data out - let optout = (STORAGE.cookiesAreEnabled() && STORAGE.getCookie('_pubcid_optout')) || - (STORAGE.hasLocalStorage() && STORAGE.getDataFromLocalStorage('_pubcid_optout')); + if (!mapping || !mapping.invalid) return key; - if (key === 'user' && optout) { - utils.logWarn(`Filtered ${key} data: pubcid optout found`); - return result; - } + utils.logWarn(`Filtered ${parent}${key} property in ortb2 data: invalid property`); + }).filter(key => { + let mapping = utils.deepAccess(ORTB_MAP, path + key); + // let typeBool = false; + let typeBool = (mapping) ? typeValidation(fpd[key], {type: mapping.type, isArray: mapping.isArray}) : true; - // Create validated object by looping through ortb2 properties - modified = (typeof obj[key] === 'object' && !Array.isArray(obj[key])) - ? Object.keys(obj[key]).reduce((combined, keyData) => { - let data; - - // If key is user.data, pass into filterData to remove invalid data and return - // Else if key is site.content.data, pass into filterData to remove invalid data and return - // Else return data unfiltered - if (key === 'user' && keyData === 'data') { - data = filterData(obj[key][keyData], key); - - if (data) combined[keyData] = data; - } else if (key === 'site' && keyData === 'content' && obj[key][keyData].data) { - let content = Object.keys(obj[key][keyData]).reduce((merged, contentData) => { - if (contentData === 'data') { - data = filterData(obj[key][keyData][contentData], key + '.content'); - - if (data) merged[contentData] = data; - } else { - merged[contentData] = obj[key][keyData][contentData]; - } + if (typeBool || !mapping) return key; - return merged; - }, {}); + utils.logWarn(`Filtered ${parent}${key} property in ortb2 data: expected type ${(mapping.isArray) ? 'array' : mapping.type}`); + }).reduce((result, key) => { + let mapping = utils.deepAccess(ORTB_MAP, path + key); + let modified = {}; - if (Object.keys(content).length) combined[keyData] = content; - } else { - combined[keyData] = obj[key][keyData]; - } + if (mapping) { + if (mapping.optoutApplies && optout) { + utils.logWarn(`Filtered ${parent}${key} data: pubcid optout found`); + return result; + } - return combined; - }, {}) : obj[key]; + modified = (mapping && mapping.type === 'object' && !mapping.isArray) + ? validateFpd(fpd[key], path + key + '.children.', parent + key + '.') + : (mapping && mapping.isArray && mapping.childType) + ? filterArrayData(fpd[key], { type: mapping.childType, isArray: mapping.childisArray }, path + key, parent + key) : fpd[key]; - // Check if modified data has data and return - if (Object.keys(modified).length) result[key] = modified; + // Check if modified data has data and return + (!isEmptyData(modified)) ? result[key] = modified + : utils.logWarn(`Filtered ${parent}${key} property in ortb2 data: empty data found`); + } else { + result[key] = fpd[key]; + } return result; }, {})); @@ -210,8 +270,8 @@ export function validateFpd(obj) { } /** -* Resets global ortb2 data -*/ + * Resets global ortb2 data + */ export const resetOrtb2 = () => { ortb2 = {} }; function runEnrichments(shouldSkipValidate) { @@ -222,19 +282,15 @@ function runEnrichments(shouldSkipValidate) { setKeywords(); setCurrency(); - if (shouldSkipValidate) config.setConfig({ortb2: utils.mergeDeep({}, ortb2, config.getConfig('ortb2'))}); + if (shouldSkipValidate) config.setConfig({ ortb2: utils.mergeDeep({}, ortb2, config.getConfig('ortb2') || {}) }); } function runValidations() { let conf = utils.mergeDeep({}, ortb2, validateFpd(config.getConfig('ortb2'))); - if (!utils.deepEqual(conf, globalConfig)) { - config.setConfig({ortb2: conf}); - globalConfig = {...conf}; - resetOrtb2(); - } + config.setConfig({ ortb2: conf }); - let bidderDuplicate = {...config.getBidderConfig()}; + let bidderDuplicate = { ...config.getBidderConfig() }; Object.keys(bidderDuplicate).forEach(bidder => { let modConf = Object.keys(bidderDuplicate[bidder]).reduce((res, key) => { @@ -245,14 +301,18 @@ function runValidations() { return res; }, {}); - if (Object.keys(modConf).length) config.setBidderConfig({bidders: [bidder], config: modConf}); + if (Object.keys(modConf).length) config.setBidderConfig({ bidders: [bidder], config: modConf }); }); } /** -* Sets default values to ortb2 if exists and adds currency and ortb2 setConfig callbacks on init -*/ + * Sets default values to ortb2 if exists and adds currency and ortb2 setConfig callbacks on init + */ export function init() { + // Checks for existsnece of pubcid optout cookie/storage + // if exists, filters user data out + optout = (STORAGE.cookiesAreEnabled() && STORAGE.getCookie('_pubcid_optout')) || + (STORAGE.hasLocalStorage() && STORAGE.getDataFromLocalStorage('_pubcid_optout')); let conf = config.getConfig('firstPartyData'); let skipValidations = (conf && conf.skipValidations) || false; let skipEnrichments = (conf && conf.skipEnrichments) || false; @@ -263,7 +323,18 @@ export function init() { function addBidderRequestHook(fn, bidderRequests) { init(); + resetOrtb2(); fn.call(this, bidderRequests); + addBidderRequests.getHooks({ hook: addBidderRequestHook }).remove(); } -getHook('addBidderRequests').before(addBidderRequestHook); +function initModule() { + getHook('addBidderRequests').before(addBidderRequestHook); +} + +initModule(); + +/** + * Global function to reinitiate module + */ +(getGlobal()).refreshFPD = initModule; diff --git a/test/spec/modules/firstPartyData_spec.js b/test/spec/modules/firstPartyData_spec.js index 1bdb01e7965..ebb94189e61 100644 --- a/test/spec/modules/firstPartyData_spec.js +++ b/test/spec/modules/firstPartyData_spec.js @@ -3,12 +3,79 @@ import * as utils from 'src/utils.js'; import {config} from 'src/config.js'; import {getRefererInfo} from 'src/refererDetection.js'; import { - filterData, + filterArrayData, validateFpd, init, resetOrtb2 } from 'modules/firstPartyData/index.js'; +let adapterManager = require('src/adapterManager').default; + +/** + * @param {Object} [opts] + * @returns {Bid} + */ +function mockBid(opts) { + let bidderCode = opts && opts.bidderCode; + + return { + 'ad': 'creative', + 'cpm': '1.99', + 'width': 300, + 'height': 250, + 'bidderCode': bidderCode || BIDDER_CODE, + 'requestId': utils.getUniqueIdentifierStr(), + 'creativeId': 'id', + 'currency': 'USD', + 'netRevenue': true, + 'ttl': 360, + getSize: () => '300x250' + }; +} + +/** + * @param {Bid} bid + * @param {Object} [opts] + * @returns {BidRequest} + */ +function mockBidRequest(bid, opts) { + if (!bid) { + throw new Error('bid required'); + } + let bidderCode = opts && opts.bidderCode; + let adUnitCode = opts && opts.adUnitCode; + let defaultMediaType = { + banner: { + sizes: [[300, 250], [300, 600]] + } + } + let mediaType = (opts && opts.mediaType) ? opts.mediaType : defaultMediaType; + + let requestId = utils.getUniqueIdentifierStr(); + + return { + 'bidderCode': bidderCode || bid.bidderCode, + 'auctionId': '20882439e3238c', + 'bidderRequestId': requestId, + 'bids': [ + { + 'bidder': bidderCode || bid.bidderCode, + 'params': { + 'placementId': 'id' + }, + 'adUnitCode': adUnitCode || ADUNIT_CODE, + 'sizes': [[300, 250], [300, 600]], + 'bidId': bid.requestId, + 'bidderRequestId': requestId, + 'auctionId': '20882439e3238c', + 'mediaTypes': mediaType + } + ], + 'auctionStart': 1505250713622, + 'timeout': 3000 + }; +} + describe('the first party data module', function () { let sandbox; @@ -77,125 +144,55 @@ describe('the first party data module', function () { resetOrtb2(); }); - describe('filtering first party data', function () { - it('returns null if empty data array should', function () { - let validated; - let duplicate = utils.deepClone(ortb2); - duplicate.user.data = []; - duplicate.site.content.data = []; - - validated = filterData(duplicate.user.data, 'user'); - expect(validated).to.equal(null); - validated = filterData(duplicate.site.content.data, 'site'); - expect(validated).to.equal(null); + describe('filtering first party array data', function () { + it('returns empty array if no valid data', function () { + let arr = [{}]; + let path = 'site.children.cat'; + let child = {type: 'string'}; + let parent = 'site'; + let key = 'cat'; + let validated = filterArrayData(arr, child, path, parent, key); + expect(validated).to.deep.equal([]); }); - it('returns null if name does not exist', function () { - let validated; - let duplicate = utils.deepClone(ortb2); - delete duplicate.user.data[0].name; - delete duplicate.site.content.data[0].name; - - validated = filterData(duplicate.user.data, 'user'); - expect(validated).to.equal(null); - validated = filterData(duplicate.site.content.data, 'site'); - expect(validated).to.equal(null); + it('filters invalid type of array data', function () { + let arr = ['foo', {test: 1}]; + let path = 'site.children.cat'; + let child = {type: 'string'}; + let parent = 'site'; + let key = 'cat'; + let validated = filterArrayData(arr, child, path, parent, key); + expect(validated).to.deep.equal(['foo']); }); - it('returns null if segment does not exist', function () { - let validated; - let duplicate = utils.deepClone(ortb2); - delete duplicate.user.data[0].segment; - delete duplicate.site.content.data[0].segment; - - validated = filterData(duplicate.user.data, 'user'); - expect(validated).to.equal(null); - validated = filterData(duplicate.site.content.data, 'site'); - expect(validated).to.equal(null); + it('filters all data for missing required children', function () { + let arr = [{test: 1}]; + let path = 'site.children.content.children.data'; + let child = {type: 'object'}; + let parent = 'site'; + let key = 'data'; + let validated = filterArrayData(arr, child, path, parent, key); + expect(validated).to.deep.equal([]); }); - it('returns unfiltered data', function () { - let validated; - let duplicate = utils.deepClone(ortb2); - - delete duplicate.user.data[0].ext; - - validated = filterData(duplicate.user.data, 'user'); - expect(validated).to.deep.equal(duplicate.user.data); - validated = filterData(duplicate.site.content.data, 'site'); - expect(validated).to.deep.equal(duplicate.site.content.data); + it('filters all data for invalid required children types', function () { + let arr = [{name: 'foo', segment: 1}]; + let path = 'site.children.content.children.data'; + let child = {type: 'object'}; + let parent = 'site'; + let key = 'data'; + let validated = filterArrayData(arr, child, path, parent, key); + expect(validated).to.deep.equal([]); }); - it('returns data filtering data[0].ext for wrong type', function () { - let validated; - let duplicate = utils.deepClone(ortb2); - let expected = { - user: [{ - segment: [{ - id: 'foo' - }], - name: 'bar' - }], - site: [{ - segment: [{ - id: 'test' - }], - name: 'content', - }] - }; - - duplicate.site.content.data[0].ext = [1, 3, 5]; - - validated = filterData(duplicate.user.data, 'user'); - expect(validated).to.deep.equal(expected.user); - validated = filterData(duplicate.site.content.data, 'user'); - expect(validated).to.deep.equal(expected.site); - }); - - it('returns user data filtering data[0].segment[1] for missing id', function () { - let duplicate = utils.deepClone(ortb2); - let expected = [{ - segment: [{ - id: 'foo' - }], - name: 'bar' - }]; - duplicate.user.data[0].segment.push({foo: 'bar'}); - - let validated = filterData(duplicate.user.data, 'user'); - expect(validated).to.deep.equal(expected); - }); - - it('returns undefined for data[0].segment[0] for missing id', function () { - let duplicate = utils.deepClone(ortb2); - duplicate.user.data[0].segment[0] = [{test: 1}]; - - let validated = filterData(duplicate.user.data, 'user'); - expect(validated).to.equal(null); - }); - - it('returns data filtering data[0].segement[1] and data[0].ext', function () { - let duplicate = utils.deepClone(ortb2); - let expected = [{ - segment: [{ - id: 'foo' - }], - name: 'bar' - }, { - segment: [{ - id: '123' - }], - name: 'test-2', - ext: { - foo: 'bar' - } - }]; - - duplicate.user.data[0].segment.push({test: 3}); - duplicate.user.data.push({segment: [{id: '123'}], name: 'test-2', ext: {foo: 'bar'}}); - - let validated = filterData(duplicate.user.data, 'user'); - expect(validated).to.deep.equal(expected); + it('returns only data with valid required nested children types', function () { + let arr = [{name: 'foo', segment: [{id: '1'}, {id: 2}, 'foobar']}]; + let path = 'site.children.content.children.data'; + let child = {type: 'object'}; + let parent = 'site'; + let key = 'data'; + let validated = filterArrayData(arr, child, path, parent, key); + expect(validated).to.deep.equal([{name: 'foo', segment: [{id: '1'}]}]); }); }); @@ -318,6 +315,83 @@ describe('the first party data module', function () { validated = validateFpd(duplicate); expect(validated).to.deep.equal(expected); }); + + it('filters device for invalid data types', function () { + let validated; + let duplicate = utils.deepClone(ortb2); + duplicate.device = { + h: '1', + w: '1' + } + + let expected = { + user: { + data: [{ + segment: [{ + id: 'foo' + }], + name: 'bar' + }] + }, + site: { + content: { + data: [{ + segment: [{ + id: 'test' + }], + name: 'content', + ext: { + foo: 'bar' + } + }] + } + } + }; + + duplicate.site.content.data[0].segment.push({test: 3}); + + validated = validateFpd(duplicate); + expect(validated).to.deep.equal(expected); + }); + + it('filters cur for invalid data type', function () { + let validated; + let duplicate = utils.deepClone(ortb2); + duplicate.cur = 8; + + let expected = { + device: { + h: 911, + w: 1733 + }, + user: { + data: [{ + segment: [{ + id: 'foo' + }], + name: 'bar' + }] + }, + site: { + content: { + data: [{ + segment: [{ + id: 'test' + }], + name: 'content', + ext: { + foo: 'bar' + } + }] + } + } + }; + + duplicate.site.content.data[0].segment.push({test: 3}); + + validated = validateFpd(duplicate); + expect(validated).to.deep.equal(expected); + }); }); describe('first party data intitializing', function () { @@ -521,6 +595,49 @@ describe('the first party data module', function () { expect(validated.cur).to.equal('USD'); }); + it('should filter all data', function () { + let validated; + let conf = { + imp: [], + site: { + name: 123, + domain: 456, + page: 789, + ref: 987, + keywords: ['keywords'], + search: 654, + cat: 'cat', + sectioncat: 'sectioncat', + pagecat: 'pagecat', + content: { + data: [{ + name: 1, + segment: [] + }] + } + }, + user: { + yob: 'twenty', + gender: 0, + keywords: ['foobar'], + data: ['test'] + }, + device: [800, 450], + cur: { + adServerCurrency: 'USD' + } + }; + + config.setConfig({'firstPartyData': {skipEnrichments: true}}); + + config.setConfig({ortb2: conf}); + + init(); + + validated = config.getConfig('ortb2'); + expect(validated).to.deep.equal({}); + }); + it('should filter bidderConfig data', function () { let validated; let conf = { From d9c1edab64f6a3222fe668459582e5b467db047e Mon Sep 17 00:00:00 2001 From: Michael Moschovas Date: Mon, 5 Apr 2021 13:41:56 -0400 Subject: [PATCH 11/18] Removed LGTM unneeded defensive code for check on object 'cur' --- modules/firstPartyData/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/firstPartyData/index.js b/modules/firstPartyData/index.js index 4a3186ffb0a..77eef906764 100644 --- a/modules/firstPartyData/index.js +++ b/modules/firstPartyData/index.js @@ -88,9 +88,10 @@ function setKeywords() { function setCurrency() { let cur = { ...config.getConfig('currency') }; - if (cur && cur.adServerCurrency) { + if (cur.adServerCurrency) { utils.mergeDeep(ortb2, { cur: cur.adServerCurrency }); } + } /** From ae4722389563e41d63492464c77cdff5c3935915 Mon Sep 17 00:00:00 2001 From: Michael Moschovas Date: Mon, 5 Apr 2021 13:44:11 -0400 Subject: [PATCH 12/18] Remove unused conditional --- modules/firstPartyData/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/firstPartyData/index.js b/modules/firstPartyData/index.js index 77eef906764..89583b27e52 100644 --- a/modules/firstPartyData/index.js +++ b/modules/firstPartyData/index.js @@ -251,9 +251,9 @@ export function validateFpd(fpd, path = '', parent = '') { return result; } - modified = (mapping && mapping.type === 'object' && !mapping.isArray) + modified = (mapping.type === 'object' && !mapping.isArray) ? validateFpd(fpd[key], path + key + '.children.', parent + key + '.') - : (mapping && mapping.isArray && mapping.childType) + : (mapping.isArray && mapping.childType) ? filterArrayData(fpd[key], { type: mapping.childType, isArray: mapping.childisArray }, path + key, parent + key) : fpd[key]; // Check if modified data has data and return From a63015c03233bb88a611307a59b4bef5e0e81a39 Mon Sep 17 00:00:00 2001 From: Michael Moschovas Date: Mon, 5 Apr 2021 13:53:44 -0400 Subject: [PATCH 13/18] Fix lint error --- modules/firstPartyData/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/firstPartyData/index.js b/modules/firstPartyData/index.js index 89583b27e52..52e2c6e8184 100644 --- a/modules/firstPartyData/index.js +++ b/modules/firstPartyData/index.js @@ -91,7 +91,6 @@ function setCurrency() { if (cur.adServerCurrency) { utils.mergeDeep(ortb2, { cur: cur.adServerCurrency }); } - } /** From 27457ada93f2fcfaff1a9add0e86dea5fc7ec3e8 Mon Sep 17 00:00:00 2001 From: Michael Moschovas Date: Mon, 12 Apr 2021 10:59:56 -0400 Subject: [PATCH 14/18] Updates to remove currency enrichment as well as optout for user object --- modules/firstPartyData/config.js | 1 - modules/firstPartyData/index.js | 13 ------------- test/spec/modules/firstPartyData_spec.js | 16 ---------------- 3 files changed, 30 deletions(-) diff --git a/modules/firstPartyData/config.js b/modules/firstPartyData/config.js index ad4395af102..afdbac9f263 100644 --- a/modules/firstPartyData/config.js +++ b/modules/firstPartyData/config.js @@ -89,7 +89,6 @@ export const ORTB_MAP = { }, user: { type: TYPES.object, - optoutApplies: true, children: { yob: { type: TYPES.number }, gender: { type: TYPES.string }, diff --git a/modules/firstPartyData/index.js b/modules/firstPartyData/index.js index 52e2c6e8184..3157aadeda7 100644 --- a/modules/firstPartyData/index.js +++ b/modules/firstPartyData/index.js @@ -81,18 +81,6 @@ function setKeywords() { if (keywords && keywords.content) utils.mergeDeep(ortb2, { site: { keywords: keywords.content.replace(/\s/g, '') } }); } -/** - * Checks for currency and if exists merges into ortb2 global data - * Sets listener for currency if changes occur or doesnt exist when run - */ -function setCurrency() { - let cur = { ...config.getConfig('currency') }; - - if (cur.adServerCurrency) { - utils.mergeDeep(ortb2, { cur: cur.adServerCurrency }); - } -} - /** * Check if data passed is empty * @param {*} value to test against @@ -280,7 +268,6 @@ function runEnrichments(shouldSkipValidate) { setDomain(); setDimensions(); setKeywords(); - setCurrency(); if (shouldSkipValidate) config.setConfig({ ortb2: utils.mergeDeep({}, ortb2, config.getConfig('ortb2') || {}) }); } diff --git a/test/spec/modules/firstPartyData_spec.js b/test/spec/modules/firstPartyData_spec.js index ebb94189e61..f0f73c2f61e 100644 --- a/test/spec/modules/firstPartyData_spec.js +++ b/test/spec/modules/firstPartyData_spec.js @@ -579,22 +579,6 @@ describe('the first party data module', function () { expect(validated.site.ref).to.equal('https://referer.com'); }); - it('should add currency if currency config exists', function () { - let validated; - let conf = { - currency: { - adServerCurrency: 'USD' - } - }; - - config.setConfig(conf); - - init(); - - validated = config.getConfig('ortb2'); - expect(validated.cur).to.equal('USD'); - }); - it('should filter all data', function () { let validated; let conf = { From afd2c26a09dd756c2c2fa1651987b6dfec9fc554 Mon Sep 17 00:00:00 2001 From: Michael Moschovas Date: Tue, 27 Apr 2021 11:52:40 -0400 Subject: [PATCH 15/18] Added optout flag to user.yob and user.gender fields --- modules/firstPartyData/config.js | 10 ++++++++-- modules/firstPartyData/index.js | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/modules/firstPartyData/config.js b/modules/firstPartyData/config.js index afdbac9f263..a520267630b 100644 --- a/modules/firstPartyData/config.js +++ b/modules/firstPartyData/config.js @@ -90,8 +90,14 @@ export const ORTB_MAP = { user: { type: TYPES.object, children: { - yob: { type: TYPES.number }, - gender: { type: TYPES.string }, + yob: { + type: TYPES.number, + optoutApplies: true + }, + gender: { + type: TYPES.string, + optoutApplies: true + }, keywords: { type: TYPES.string }, data: { type: TYPES.object, diff --git a/modules/firstPartyData/index.js b/modules/firstPartyData/index.js index 3157aadeda7..0e5af6f2e75 100644 --- a/modules/firstPartyData/index.js +++ b/modules/firstPartyData/index.js @@ -258,7 +258,7 @@ export function validateFpd(fpd, path = '', parent = '') { } /** - * Resets global ortb2 data + * Resets modules global ortb2 data */ export const resetOrtb2 = () => { ortb2 = {} }; From 5ed6071640a76a94b532f3ceca5b684514fd5462 Mon Sep 17 00:00:00 2001 From: Michael Moschovas Date: Tue, 27 Apr 2021 12:25:29 -0400 Subject: [PATCH 16/18] Added test for arbitrary values Added more comments --- modules/firstPartyData/index.js | 14 +++++++++--- modules/firstPartyData/index.md | 6 +++++ test/spec/modules/firstPartyData_spec.js | 29 ++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/modules/firstPartyData/index.js b/modules/firstPartyData/index.js index 0e5af6f2e75..8c5cbbdcce5 100644 --- a/modules/firstPartyData/index.js +++ b/modules/firstPartyData/index.js @@ -308,20 +308,28 @@ export function init() { if (!skipValidations) runValidations(); } +/** + * BidderRequests hook to intiate module + */ function addBidderRequestHook(fn, bidderRequests) { init(); resetOrtb2(); fn.call(this, bidderRequests); + // Removes hook after run addBidderRequests.getHooks({ hook: addBidderRequestHook }).remove(); } -function initModule() { +/** + * Sets bidderRequests hook + */ +function setupHook() { getHook('addBidderRequests').before(addBidderRequestHook); } -initModule(); +// Runs setupHook on initial load +setupHook(); /** * Global function to reinitiate module */ -(getGlobal()).refreshFPD = initModule; +(getGlobal()).refreshFPD = setupHook; diff --git a/modules/firstPartyData/index.md b/modules/firstPartyData/index.md index 16c09736fd8..32b05472798 100644 --- a/modules/firstPartyData/index.md +++ b/modules/firstPartyData/index.md @@ -13,6 +13,12 @@ Module to perform the following functions to allow for consistent set of first p - optionally suppress user FPD based on a TBD opt-out signal (_pubcid_optout) - populate available data into object: referer, meta-keywords, cur + +1. Module initializes on first load and set bidRequestHook to validate existing ortb2 global/bidder data and merge enrichments (unless opt out configured for either) +2. After hook complete, it is disabled - meaning module only runs on first auction +3. To reinitiate the module, run pbjs.refreshFPD(), which allows module to rerun as if initial load + + This module will automatically run both first party data enrichments and validations. There is no configuration required. In order to load the module and opt out of either enrichements or validations, use the below opt out configuration # Opt Out Configuration diff --git a/test/spec/modules/firstPartyData_spec.js b/test/spec/modules/firstPartyData_spec.js index f0f73c2f61e..2341754bf2d 100644 --- a/test/spec/modules/firstPartyData_spec.js +++ b/test/spec/modules/firstPartyData_spec.js @@ -622,6 +622,35 @@ describe('the first party data module', function () { expect(validated).to.deep.equal({}); }); + it('should add enrichments but not alter any arbitrary ortb2 data', function () { + let validated; + let conf = { + site: { + ext: { + data: { + inventory: ['value1'] + } + } + }, + user: { + ext: { + data: { + visitor: ['value2'] + } + } + } + }; + + config.setConfig({ortb2: conf}); + + init(); + + validated = config.getConfig('ortb2'); + expect(validated.site.ref).to.equal(getRefererInfo().referer); + expect(validated.site.ext.data).to.deep.equal({inventory: ['value1']}); + expect(validated.user.ext.data).to.deep.equal({visitor: ['value2']}); + }); + it('should filter bidderConfig data', function () { let validated; let conf = { From 707f6a5444a58e2ffcd36b8a3be2121463a7a16b Mon Sep 17 00:00:00 2001 From: Michael Moschovas Date: Wed, 28 Apr 2021 13:46:30 -0400 Subject: [PATCH 17/18] Broke module out into module and two submodules --- modules/.submodules.json | 4 + modules/enrichmentFpdModule.js | 107 ++++++ modules/firstPartyData/index.md | 35 -- modules/fpdModule/index.js | 58 +++ modules/fpdModule/index.md | 46 +++ .../config.js | 0 .../index.js | 131 +------ test/spec/modules/enrichmentFpdModule_spec.js | 97 +++++ ...rstPartyData_spec.js => fpdModule_spec.js} | 348 +----------------- test/spec/modules/validationFpdModule_spec.js | 313 ++++++++++++++++ 10 files changed, 656 insertions(+), 483 deletions(-) create mode 100644 modules/enrichmentFpdModule.js delete mode 100644 modules/firstPartyData/index.md create mode 100644 modules/fpdModule/index.js create mode 100644 modules/fpdModule/index.md rename modules/{firstPartyData => validationFpdModule}/config.js (100%) rename modules/{firstPartyData => validationFpdModule}/index.js (65%) create mode 100644 test/spec/modules/enrichmentFpdModule_spec.js rename test/spec/modules/{firstPartyData_spec.js => fpdModule_spec.js} (58%) create mode 100644 test/spec/modules/validationFpdModule_spec.js diff --git a/modules/.submodules.json b/modules/.submodules.json index 0f62627822a..6fc1c9cdf25 100644 --- a/modules/.submodules.json +++ b/modules/.submodules.json @@ -34,5 +34,9 @@ "jwplayerRtdProvider", "reconciliationRtdProvider", "geoedgeRtdProvider" + ], + "fpdModule": [ + "enrichmentFpdModule", + "validationFpdModule" ] } diff --git a/modules/enrichmentFpdModule.js b/modules/enrichmentFpdModule.js new file mode 100644 index 00000000000..a1d815917e0 --- /dev/null +++ b/modules/enrichmentFpdModule.js @@ -0,0 +1,107 @@ +/** + * This module sets default values and validates ortb2 first part data + * @module modules/firstPartyData + */ +import * as utils from '../src/utils.js'; +import { submodule } from '../src/hook.js' +import { getRefererInfo } from '../src/refererDetection.js' + +let ortb2 = {}; +let win = (window === window.top) ? window : window.top; + +/** + * Checks for referer and if exists merges into ortb2 global data + */ +function setReferer() { + if (getRefererInfo().referer) utils.mergeDeep(ortb2, { site: { ref: getRefererInfo().referer } }); +} + +/** + * Checks for canonical url and if exists merges into ortb2 global data + */ +function setPage() { + if (getRefererInfo().canonicalUrl) utils.mergeDeep(ortb2, { site: { page: getRefererInfo().canonicalUrl } }); +} + +/** + * Checks for canonical url and if exists retrieves domain and merges into ortb2 global data + */ +function setDomain() { + let parseDomain = function(url) { + if (!url || typeof url !== 'string' || url.length === 0) return; + + var match = url.match(/^(?:https?:\/\/)?(?:www\.)?(.*?(?=(\?|\#|\/|$)))/i); + + return match && match[1]; + }; + + let domain = parseDomain(getRefererInfo().canonicalUrl) + + if (domain) utils.mergeDeep(ortb2, { site: { domain: domain } }); +} + +/** + * Checks for screen/device width and height and sets dimensions + */ +function setDimensions() { + let width; + let height; + + try { + width = win.innerWidth || win.document.documentElement.clientWidth || win.document.body.clientWidth; + height = win.innerHeight || win.document.documentElement.clientHeight || win.document.body.clientHeight; + } catch (e) { + width = window.innerWidth || window.document.documentElement.clientWidth || window.document.body.clientWidth; + height = window.innerHeight || window.document.documentElement.clientHeight || window.document.body.clientHeight; + } + + utils.mergeDeep(ortb2, { device: { w: width, h: height } }); +} + +/** + * Scans page for meta keywords, and if exists, merges into site.keywords + */ +function setKeywords() { + let keywords; + + try { + keywords = win.document.querySelector("meta[name='keywords']"); + } catch (e) { + keywords = window.document.querySelector("meta[name='keywords']"); + } + + if (keywords && keywords.content) utils.mergeDeep(ortb2, { site: { keywords: keywords.content.replace(/\s/g, '') } }); +} + +/** + * Resets modules global ortb2 data + */ +const resetOrtb2 = () => { ortb2 = {} }; + +function runEnrichments() { + setReferer(); + setPage(); + setDomain(); + setDimensions(); + setKeywords(); + + return ortb2; +} + +/** + * Sets default values to ortb2 if exists and adds currency and ortb2 setConfig callbacks on init + */ +export function initSubmodule(fpdConf, data) { + resetOrtb2(); + + return (!fpdConf.skipEnrichments) ? utils.mergeDeep(runEnrichments(), data) : data; +} + +/** @type {firstPartyDataSubmodule} */ +export const enrichmentsSubmodule = { + name: 'enrichments', + queue: 2, + init: initSubmodule +} + +submodule('firstPartyData', enrichmentsSubmodule) diff --git a/modules/firstPartyData/index.md b/modules/firstPartyData/index.md deleted file mode 100644 index 32b05472798..00000000000 --- a/modules/firstPartyData/index.md +++ /dev/null @@ -1,35 +0,0 @@ -# Overview - -``` -Module Name: First Party Data Module -``` - -# Description - -Module to perform the following functions to allow for consistent set of first party data - -- verify OpenRTB datatypes, remove/warn any that are likely to choke downstream readers -- verify that certain OpenRTB attributes are not specified: just imp for now -- optionally suppress user FPD based on a TBD opt-out signal (_pubcid_optout) -- populate available data into object: referer, meta-keywords, cur - - -1. Module initializes on first load and set bidRequestHook to validate existing ortb2 global/bidder data and merge enrichments (unless opt out configured for either) -2. After hook complete, it is disabled - meaning module only runs on first auction -3. To reinitiate the module, run pbjs.refreshFPD(), which allows module to rerun as if initial load - - -This module will automatically run both first party data enrichments and validations. There is no configuration required. In order to load the module and opt out of either enrichements or validations, use the below opt out configuration - -# Opt Out Configuration - -``` - -pbjs.setConfig({ - firstPartyData: { - skipValidations: true, // default to false - skipEnrichments: true // default to false - } -}); - -``` diff --git a/modules/fpdModule/index.js b/modules/fpdModule/index.js new file mode 100644 index 00000000000..427547a4e4d --- /dev/null +++ b/modules/fpdModule/index.js @@ -0,0 +1,58 @@ +/** + * This module sets default values and validates ortb2 first part data + * @module modules/firstPartyData + */ +import { config } from '../../src/config.js'; +import { module, getHook } from '../../src/hook.js'; +import { getGlobal } from '../../src/prebidGlobal.js'; +import { addBidderRequests } from '../../src/auction.js'; + +let submodules = []; + +/** + * enable submodule in User ID + * @param {RtdSubmodule} submodule + */ +export function registerSubmodules(submodule) { + submodules.push(submodule); +} + +export function init() { + let modConf = config.getConfig('firstPartyData') || {}; + let ortb2 = config.getConfig('ortb2') || {}; + + submodules.sort((a, b) => { + return ((a.queue || 1) - (b.queue || 1)); + }).forEach(submodule => { + ortb2 = submodule.init(modConf, ortb2); + }); + + config.setConfig({ortb2}); +} + +/** + * BidderRequests hook to intiate module and reset modules ortb2 data object + */ +function addBidderRequestHook(fn, bidderRequests) { + init(); + fn.call(this, bidderRequests); + // Removes hook after run + addBidderRequests.getHooks({ hook: addBidderRequestHook }).remove(); +} + +/** + * Sets bidderRequests hook + */ +function setupHook() { + getHook('addBidderRequests').before(addBidderRequestHook); +} + +module('firstPartyData', registerSubmodules); + +// Runs setupHook on initial load +setupHook(); + +/** + * Global function to reinitiate module + */ +(getGlobal()).refreshFpd = setupHook; diff --git a/modules/fpdModule/index.md b/modules/fpdModule/index.md new file mode 100644 index 00000000000..286822c3c9e --- /dev/null +++ b/modules/fpdModule/index.md @@ -0,0 +1,46 @@ +# Overview + +``` +Module Name: First Party Data Module +``` + +# Description + +Module to perform the following functions to allow for consistent set of first party data using the following submodules. + +Enrichment Submodule: +- populate available data into object: referer, meta-keywords, cur + +Validation Submodule: +- verify OpenRTB datatypes, remove/warn any that are likely to choke downstream readers +- verify that certain OpenRTB attributes are not specified +- optionally suppress user FPD based on a TBD opt-out signal (_pubcid_optout) + + +1. Module initializes on first load and set bidRequestHook +2. When hook runs, corresponding submodule init functions are run to perform enrichments/validations dependant on submodule +3. After hook complete, it is disabled - meaning module only runs on first auction +4. To reinitiate the module, run pbjs.refreshFPD(), which allows module to rerun as if initial load + + +This module will automatically run first party data enrichments and validations dependant on which submodules are included. There is no configuration required. In order to load the module and submodule(s) and opt out of either enrichements or validations, use the below opt out configuration + +# Opt Out Configuration + +``` + +pbjs.setConfig({ + firstPartyData: { + skipValidations: true, // default to false + skipEnrichments: true // default to false + } +}); + +``` + +# Requirements + +At least one of the submodules must be included in order to successfully run the corresponding above operations. + +enrichmentFpdModule +validationFpdModule \ No newline at end of file diff --git a/modules/firstPartyData/config.js b/modules/validationFpdModule/config.js similarity index 100% rename from modules/firstPartyData/config.js rename to modules/validationFpdModule/config.js diff --git a/modules/firstPartyData/index.js b/modules/validationFpdModule/index.js similarity index 65% rename from modules/firstPartyData/index.js rename to modules/validationFpdModule/index.js index 8c5cbbdcce5..c23f7e09316 100644 --- a/modules/firstPartyData/index.js +++ b/modules/validationFpdModule/index.js @@ -5,82 +5,12 @@ import { config } from '../../src/config.js'; import * as utils from '../../src/utils.js'; import { ORTB_MAP } from './config.js'; -import { getHook } from '../../src/hook.js'; -import { getGlobal } from '../../src/prebidGlobal.js'; -import { addBidderRequests } from '../../src/auction.js'; -import { getRefererInfo } from '../../src/refererDetection.js' +import { submodule } from '../../src/hook.js'; import { getStorageManager } from '../../src/storageManager.js'; const STORAGE = getStorageManager(); - -let ortb2 = {}; -let win = (window === window.top) ? window : window.top; let optout; -/** - * Checks for referer and if exists merges into ortb2 global data - */ -function setReferer() { - if (getRefererInfo().referer) utils.mergeDeep(ortb2, { site: { ref: getRefererInfo().referer } }); -} - -/** - * Checks for canonical url and if exists merges into ortb2 global data - */ -function setPage() { - if (getRefererInfo().canonicalUrl) utils.mergeDeep(ortb2, { site: { page: getRefererInfo().canonicalUrl } }); -} - -/** - * Checks for canonical url and if exists retrieves domain and merges into ortb2 global data - */ -function setDomain() { - let parseDomain = function(url) { - if (!url || typeof url !== 'string' || url.length === 0) return; - - var match = url.match(/^(?:https?:\/\/)?(?:www\.)?(.*?(?=(\?|\#|\/|$)))/i); - - return match && match[1]; - }; - - let domain = parseDomain(getRefererInfo().canonicalUrl) - - if (domain) utils.mergeDeep(ortb2, { site: { domain: domain } }); -} - -/** - * Checks for screen/device width and height and sets dimensions - */ -function setDimensions() { - let width; - let height; - - try { - width = win.innerWidth || win.document.documentElement.clientWidth || win.document.body.clientWidth; - height = win.innerHeight || win.document.documentElement.clientHeight || win.document.body.clientHeight; - } catch (e) { - width = window.innerWidth || window.document.documentElement.clientWidth || window.document.body.clientWidth; - height = window.innerHeight || window.document.documentElement.clientHeight || window.document.body.clientHeight; - } - - utils.mergeDeep(ortb2, { device: { w: width, h: height } }); -} - -/** - * Scans page for meta keywords, and if exists, merges into site.keywords - */ -function setKeywords() { - let keywords; - - try { - keywords = win.document.querySelector("meta[name='keywords']"); - } catch (e) { - keywords = window.document.querySelector("meta[name='keywords']"); - } - - if (keywords && keywords.content) utils.mergeDeep(ortb2, { site: { keywords: keywords.content.replace(/\s/g, '') } }); -} - /** * Check if data passed is empty * @param {*} value to test against @@ -258,24 +188,10 @@ export function validateFpd(fpd, path = '', parent = '') { } /** - * Resets modules global ortb2 data + * Run validation on global and bidder config data for ortb2 */ -export const resetOrtb2 = () => { ortb2 = {} }; - -function runEnrichments(shouldSkipValidate) { - setReferer(); - setPage(); - setDomain(); - setDimensions(); - setKeywords(); - - if (shouldSkipValidate) config.setConfig({ ortb2: utils.mergeDeep({}, ortb2, config.getConfig('ortb2') || {}) }); -} - -function runValidations() { - let conf = utils.mergeDeep({}, ortb2, validateFpd(config.getConfig('ortb2'))); - - config.setConfig({ ortb2: conf }); +function runValidations(data) { + let conf = validateFpd(data); let bidderDuplicate = { ...config.getBidderConfig() }; @@ -290,46 +206,27 @@ function runValidations() { if (Object.keys(modConf).length) config.setBidderConfig({ bidders: [bidder], config: modConf }); }); + + return conf; } /** * Sets default values to ortb2 if exists and adds currency and ortb2 setConfig callbacks on init */ -export function init() { +export function initSubmodule(fpdConf, data) { // Checks for existsnece of pubcid optout cookie/storage // if exists, filters user data out optout = (STORAGE.cookiesAreEnabled() && STORAGE.getCookie('_pubcid_optout')) || (STORAGE.hasLocalStorage() && STORAGE.getDataFromLocalStorage('_pubcid_optout')); - let conf = config.getConfig('firstPartyData'); - let skipValidations = (conf && conf.skipValidations) || false; - let skipEnrichments = (conf && conf.skipEnrichments) || false; - if (!skipEnrichments) runEnrichments(skipValidations); - if (!skipValidations) runValidations(); + return (!fpdConf.skipValidations) ? runValidations(data) : data; } -/** - * BidderRequests hook to intiate module - */ -function addBidderRequestHook(fn, bidderRequests) { - init(); - resetOrtb2(); - fn.call(this, bidderRequests); - // Removes hook after run - addBidderRequests.getHooks({ hook: addBidderRequestHook }).remove(); +/** @type {firstPartyDataSubmodule} */ +export const validationSubmodule = { + name: 'validation', + queue: 1, + init: initSubmodule } -/** - * Sets bidderRequests hook - */ -function setupHook() { - getHook('addBidderRequests').before(addBidderRequestHook); -} - -// Runs setupHook on initial load -setupHook(); - -/** - * Global function to reinitiate module - */ -(getGlobal()).refreshFPD = setupHook; +submodule('firstPartyData', validationSubmodule) diff --git a/test/spec/modules/enrichmentFpdModule_spec.js b/test/spec/modules/enrichmentFpdModule_spec.js new file mode 100644 index 00000000000..e5271143f2c --- /dev/null +++ b/test/spec/modules/enrichmentFpdModule_spec.js @@ -0,0 +1,97 @@ +import { expect } from 'chai'; +import * as utils from 'src/utils.js'; +import { getRefererInfo } from 'src/refererDetection.js'; +import { initSubmodule } from 'modules/enrichmentFpdModule.js'; + +describe('the first party data enrichment module', function() { + let width; + let widthStub; + let height; + let heightStub; + let querySelectorStub; + let canonical; + let keywords; + + before(function() { + canonical = document.createElement('link'); + canonical.rel = 'canonical'; + keywords = document.createElement('meta'); + keywords.name = 'keywords'; + }); + + beforeEach(function() { + querySelectorStub = sinon.stub(window.top.document, 'querySelector'); + querySelectorStub.withArgs("link[rel='canonical']").returns(canonical); + querySelectorStub.withArgs("meta[name='keywords']").returns(keywords); + widthStub = sinon.stub(window.top, 'innerWidth').get(function() { + return width; + }); + heightStub = sinon.stub(window.top, 'innerHeight').get(function() { + return height; + }); + }); + + afterEach(function() { + widthStub.restore(); + heightStub.restore(); + querySelectorStub.restore(); + canonical = document.createElement('link'); + canonical.rel = 'canonical'; + keywords = document.createElement('meta'); + keywords.name = 'keywords'; + }); + + it('adds ref and device values', function() { + width = 800; + height = 500; + + let validated = initSubmodule({}, {}); + + expect(validated.site.ref).to.equal(getRefererInfo().referer); + expect(validated.site.page).to.be.undefined; + expect(validated.site.domain).to.be.undefined; + expect(validated.device).to.deep.equal({ w: 800, h: 500 }); + expect(validated.site.keywords).to.be.undefined; + }); + + it('adds page and domain values if canonical url exists', function() { + width = 800; + height = 500; + canonical.href = 'https://www.domain.com/path?query=12345'; + + let validated = initSubmodule({}, {}); + + expect(validated.site.ref).to.equal(getRefererInfo().referer); + expect(validated.site.page).to.equal('https://www.domain.com/path?query=12345'); + expect(validated.site.domain).to.equal('domain.com'); + expect(validated.device).to.deep.equal({ w: 800, h: 500 }); + expect(validated.site.keywords).to.be.undefined; + }); + + it('adds keyword value if keyword meta content exists', function() { + width = 800; + height = 500; + keywords.content = 'value1,value2,value3'; + + let validated = initSubmodule({}, {}); + + expect(validated.site.ref).to.equal(getRefererInfo().referer); + expect(validated.site.page).to.be.undefined; + expect(validated.site.domain).to.be.undefined; + expect(validated.device).to.deep.equal({ w: 800, h: 500 }); + expect(validated.site.keywords).to.equal('value1,value2,value3'); + }); + + it('does not overwrite existing data from getConfig ortb2', function() { + width = 800; + height = 500; + + let validated = initSubmodule({}, {device: {w: 1200, h: 700}, site: {ref: 'https://someUrl.com', page: 'test.com'}}); + + expect(validated.site.ref).to.equal('https://someUrl.com'); + expect(validated.site.page).to.equal('test.com'); + expect(validated.site.domain).to.be.undefined; + expect(validated.device).to.deep.equal({ w: 1200, h: 700 }); + expect(validated.site.keywords).to.be.undefined; + }); +}); diff --git a/test/spec/modules/firstPartyData_spec.js b/test/spec/modules/fpdModule_spec.js similarity index 58% rename from test/spec/modules/firstPartyData_spec.js rename to test/spec/modules/fpdModule_spec.js index 2341754bf2d..2f2d3bee57a 100644 --- a/test/spec/modules/firstPartyData_spec.js +++ b/test/spec/modules/fpdModule_spec.js @@ -2,83 +2,22 @@ import {expect} from 'chai'; import * as utils from 'src/utils.js'; import {config} from 'src/config.js'; import {getRefererInfo} from 'src/refererDetection.js'; -import { - filterArrayData, - validateFpd, - init, - resetOrtb2 -} from 'modules/firstPartyData/index.js'; - -let adapterManager = require('src/adapterManager').default; - -/** - * @param {Object} [opts] - * @returns {Bid} - */ -function mockBid(opts) { - let bidderCode = opts && opts.bidderCode; - - return { - 'ad': 'creative', - 'cpm': '1.99', - 'width': 300, - 'height': 250, - 'bidderCode': bidderCode || BIDDER_CODE, - 'requestId': utils.getUniqueIdentifierStr(), - 'creativeId': 'id', - 'currency': 'USD', - 'netRevenue': true, - 'ttl': 360, - getSize: () => '300x250' - }; -} - -/** - * @param {Bid} bid - * @param {Object} [opts] - * @returns {BidRequest} - */ -function mockBidRequest(bid, opts) { - if (!bid) { - throw new Error('bid required'); - } - let bidderCode = opts && opts.bidderCode; - let adUnitCode = opts && opts.adUnitCode; - let defaultMediaType = { - banner: { - sizes: [[300, 250], [300, 600]] - } - } - let mediaType = (opts && opts.mediaType) ? opts.mediaType : defaultMediaType; - - let requestId = utils.getUniqueIdentifierStr(); - - return { - 'bidderCode': bidderCode || bid.bidderCode, - 'auctionId': '20882439e3238c', - 'bidderRequestId': requestId, - 'bids': [ - { - 'bidder': bidderCode || bid.bidderCode, - 'params': { - 'placementId': 'id' - }, - 'adUnitCode': adUnitCode || ADUNIT_CODE, - 'sizes': [[300, 250], [300, 600]], - 'bidId': bid.requestId, - 'bidderRequestId': requestId, - 'auctionId': '20882439e3238c', - 'mediaTypes': mediaType - } - ], - 'auctionStart': 1505250713622, - 'timeout': 3000 - }; -} +import {init, registerSubmodules} from 'modules/fpdModule/index.js'; +import * as enrichmentModule from 'modules/enrichmentFpdModule.js'; +import * as validationModule from 'modules/validationFpdModule/index.js'; + +let enrichments = { + name: 'enrichments', + queue: 2, + init: enrichmentModule.initSubmodule +}; +let validations = { + name: 'validations', + queue: 1, + init: validationModule.initSubmodule +}; describe('the first party data module', function () { - let sandbox; - let ortb2 = { device: { h: 911, @@ -134,264 +73,8 @@ describe('the first party data module', function () { } }; - beforeEach(function () { - sandbox = sinon.sandbox.create(); - }); - afterEach(function () { - sandbox.restore(); config.resetConfig(); - resetOrtb2(); - }); - - describe('filtering first party array data', function () { - it('returns empty array if no valid data', function () { - let arr = [{}]; - let path = 'site.children.cat'; - let child = {type: 'string'}; - let parent = 'site'; - let key = 'cat'; - let validated = filterArrayData(arr, child, path, parent, key); - expect(validated).to.deep.equal([]); - }); - - it('filters invalid type of array data', function () { - let arr = ['foo', {test: 1}]; - let path = 'site.children.cat'; - let child = {type: 'string'}; - let parent = 'site'; - let key = 'cat'; - let validated = filterArrayData(arr, child, path, parent, key); - expect(validated).to.deep.equal(['foo']); - }); - - it('filters all data for missing required children', function () { - let arr = [{test: 1}]; - let path = 'site.children.content.children.data'; - let child = {type: 'object'}; - let parent = 'site'; - let key = 'data'; - let validated = filterArrayData(arr, child, path, parent, key); - expect(validated).to.deep.equal([]); - }); - - it('filters all data for invalid required children types', function () { - let arr = [{name: 'foo', segment: 1}]; - let path = 'site.children.content.children.data'; - let child = {type: 'object'}; - let parent = 'site'; - let key = 'data'; - let validated = filterArrayData(arr, child, path, parent, key); - expect(validated).to.deep.equal([]); - }); - - it('returns only data with valid required nested children types', function () { - let arr = [{name: 'foo', segment: [{id: '1'}, {id: 2}, 'foobar']}]; - let path = 'site.children.content.children.data'; - let child = {type: 'object'}; - let parent = 'site'; - let key = 'data'; - let validated = filterArrayData(arr, child, path, parent, key); - expect(validated).to.deep.equal([{name: 'foo', segment: [{id: '1'}]}]); - }); - }); - - describe('validating first party data', function () { - it('filters user.data[0].ext for incorrect type', function () { - let validated; - let duplicate = utils.deepClone(ortb2); - let expected = { - device: { - h: 911, - w: 1733 - }, - user: { - data: [{ - segment: [{ - id: 'foo' - }], - name: 'bar' - }] - }, - site: { - content: { - data: [{ - segment: [{ - id: 'test' - }], - name: 'content', - ext: { - foo: 'bar' - } - }] - } - } - }; - - validated = validateFpd(duplicate); - expect(validated).to.deep.equal(expected); - }); - - it('filters user and site for empty data', function () { - let validated; - let duplicate = utils.deepClone(ortb2); - let expected = { - device: { - h: 911, - w: 1733 - } - }; - - duplicate.user.data = []; - duplicate.site.content.data = []; - - validated = validateFpd(duplicate); - expect(validated).to.deep.equal(expected); - }); - - it('filters user for empty valid segment values', function () { - let validated; - let duplicate = utils.deepClone(ortb2); - let expected = { - device: { - h: 911, - w: 1733 - }, - site: { - content: { - data: [{ - segment: [{ - id: 'test' - }], - name: 'content', - ext: { - foo: 'bar' - } - }] - } - } - }; - - duplicate.user.data[0].segment.push({test: 3}); - duplicate.user.data[0].segment[0] = {foo: 'bar'}; - - validated = validateFpd(duplicate); - expect(validated).to.deep.equal(expected); - }); - - it('filters user.data[0].ext and site.content.data[0].segement[1] for invalid data', function () { - let validated; - let duplicate = utils.deepClone(ortb2); - let expected = { - device: { - h: 911, - w: 1733 - }, - user: { - data: [{ - segment: [{ - id: 'foo' - }], - name: 'bar' - }] - }, - site: { - content: { - data: [{ - segment: [{ - id: 'test' - }], - name: 'content', - ext: { - foo: 'bar' - } - }] - } - } - }; - - duplicate.site.content.data[0].segment.push({test: 3}); - - validated = validateFpd(duplicate); - expect(validated).to.deep.equal(expected); - }); - - it('filters device for invalid data types', function () { - let validated; - let duplicate = utils.deepClone(ortb2); - duplicate.device = { - h: '1', - w: '1' - } - - let expected = { - user: { - data: [{ - segment: [{ - id: 'foo' - }], - name: 'bar' - }] - }, - site: { - content: { - data: [{ - segment: [{ - id: 'test' - }], - name: 'content', - ext: { - foo: 'bar' - } - }] - } - } - }; - - duplicate.site.content.data[0].segment.push({test: 3}); - - validated = validateFpd(duplicate); - expect(validated).to.deep.equal(expected); - }); - - it('filters cur for invalid data type', function () { - let validated; - let duplicate = utils.deepClone(ortb2); - duplicate.cur = 8; - - let expected = { - device: { - h: 911, - w: 1733 - }, - user: { - data: [{ - segment: [{ - id: 'foo' - }], - name: 'bar' - }] - }, - site: { - content: { - data: [{ - segment: [{ - id: 'test' - }], - name: 'content', - ext: { - foo: 'bar' - } - }] - } - } - }; - - duplicate.site.content.data[0].segment.push({test: 3}); - - validated = validateFpd(duplicate); - expect(validated).to.deep.equal(expected); - }); }); describe('first party data intitializing', function () { @@ -433,6 +116,9 @@ describe('the first party data module', function () { }); it('sets default referer and dimension values to ortb2 data', function () { + registerSubmodules(enrichments); + registerSubmodules(validations); + let validated; width = 1120; diff --git a/test/spec/modules/validationFpdModule_spec.js b/test/spec/modules/validationFpdModule_spec.js new file mode 100644 index 00000000000..9e8072cb9ed --- /dev/null +++ b/test/spec/modules/validationFpdModule_spec.js @@ -0,0 +1,313 @@ +import {expect} from 'chai'; +import * as utils from 'src/utils.js'; +import { + filterArrayData, + validateFpd +} from 'modules/validationFpdModule/index.js'; + +describe('the first party data validation module', function () { + let ortb2 = { + device: { + h: 911, + w: 1733 + }, + user: { + data: [{ + segment: [{ + id: 'foo' + }], + name: 'bar', + ext: 'string' + }] + }, + site: { + content: { + data: [{ + segment: [{ + id: 'test' + }], + name: 'content', + ext: { + foo: 'bar' + } + }] + } + } + }; + + let conf = { + device: { + h: 500, + w: 750 + }, + user: { + keywords: 'test1, test2', + gender: 'f', + data: [{ + segment: [{ + id: 'test' + }], + name: 'alt' + }] + }, + site: { + ref: 'domain.com', + page: 'www.domain.com/test', + ext: { + data: { + inventory: ['first'] + } + } + } + }; + + describe('filtering first party array data', function () { + it('returns empty array if no valid data', function () { + let arr = [{}]; + let path = 'site.children.cat'; + let child = {type: 'string'}; + let parent = 'site'; + let key = 'cat'; + let validated = filterArrayData(arr, child, path, parent, key); + expect(validated).to.deep.equal([]); + }); + + it('filters invalid type of array data', function () { + let arr = ['foo', {test: 1}]; + let path = 'site.children.cat'; + let child = {type: 'string'}; + let parent = 'site'; + let key = 'cat'; + let validated = filterArrayData(arr, child, path, parent, key); + expect(validated).to.deep.equal(['foo']); + }); + + it('filters all data for missing required children', function () { + let arr = [{test: 1}]; + let path = 'site.children.content.children.data'; + let child = {type: 'object'}; + let parent = 'site'; + let key = 'data'; + let validated = filterArrayData(arr, child, path, parent, key); + expect(validated).to.deep.equal([]); + }); + + it('filters all data for invalid required children types', function () { + let arr = [{name: 'foo', segment: 1}]; + let path = 'site.children.content.children.data'; + let child = {type: 'object'}; + let parent = 'site'; + let key = 'data'; + let validated = filterArrayData(arr, child, path, parent, key); + expect(validated).to.deep.equal([]); + }); + + it('returns only data with valid required nested children types', function () { + let arr = [{name: 'foo', segment: [{id: '1'}, {id: 2}, 'foobar']}]; + let path = 'site.children.content.children.data'; + let child = {type: 'object'}; + let parent = 'site'; + let key = 'data'; + let validated = filterArrayData(arr, child, path, parent, key); + expect(validated).to.deep.equal([{name: 'foo', segment: [{id: '1'}]}]); + }); + }); + + describe('validating first party data', function () { + it('filters user.data[0].ext for incorrect type', function () { + let validated; + let duplicate = utils.deepClone(ortb2); + let expected = { + device: { + h: 911, + w: 1733 + }, + user: { + data: [{ + segment: [{ + id: 'foo' + }], + name: 'bar' + }] + }, + site: { + content: { + data: [{ + segment: [{ + id: 'test' + }], + name: 'content', + ext: { + foo: 'bar' + } + }] + } + } + }; + + validated = validateFpd(duplicate); + expect(validated).to.deep.equal(expected); + }); + + it('filters user and site for empty data', function () { + let validated; + let duplicate = utils.deepClone(ortb2); + let expected = { + device: { + h: 911, + w: 1733 + } + }; + + duplicate.user.data = []; + duplicate.site.content.data = []; + + validated = validateFpd(duplicate); + expect(validated).to.deep.equal(expected); + }); + + it('filters user for empty valid segment values', function () { + let validated; + let duplicate = utils.deepClone(ortb2); + let expected = { + device: { + h: 911, + w: 1733 + }, + site: { + content: { + data: [{ + segment: [{ + id: 'test' + }], + name: 'content', + ext: { + foo: 'bar' + } + }] + } + } + }; + + duplicate.user.data[0].segment.push({test: 3}); + duplicate.user.data[0].segment[0] = {foo: 'bar'}; + + validated = validateFpd(duplicate); + expect(validated).to.deep.equal(expected); + }); + + it('filters user.data[0].ext and site.content.data[0].segement[1] for invalid data', function () { + let validated; + let duplicate = utils.deepClone(ortb2); + let expected = { + device: { + h: 911, + w: 1733 + }, + user: { + data: [{ + segment: [{ + id: 'foo' + }], + name: 'bar' + }] + }, + site: { + content: { + data: [{ + segment: [{ + id: 'test' + }], + name: 'content', + ext: { + foo: 'bar' + } + }] + } + } + }; + + duplicate.site.content.data[0].segment.push({test: 3}); + + validated = validateFpd(duplicate); + expect(validated).to.deep.equal(expected); + }); + + it('filters device for invalid data types', function () { + let validated; + let duplicate = utils.deepClone(ortb2); + duplicate.device = { + h: '1', + w: '1' + } + + let expected = { + user: { + data: [{ + segment: [{ + id: 'foo' + }], + name: 'bar' + }] + }, + site: { + content: { + data: [{ + segment: [{ + id: 'test' + }], + name: 'content', + ext: { + foo: 'bar' + } + }] + } + } + }; + + duplicate.site.content.data[0].segment.push({test: 3}); + + validated = validateFpd(duplicate); + expect(validated).to.deep.equal(expected); + }); + + it('filters cur for invalid data type', function () { + let validated; + let duplicate = utils.deepClone(ortb2); + duplicate.cur = 8; + + let expected = { + device: { + h: 911, + w: 1733 + }, + user: { + data: [{ + segment: [{ + id: 'foo' + }], + name: 'bar' + }] + }, + site: { + content: { + data: [{ + segment: [{ + id: 'test' + }], + name: 'content', + ext: { + foo: 'bar' + } + }] + } + } + }; + + duplicate.site.content.data[0].segment.push({test: 3}); + + validated = validateFpd(duplicate); + expect(validated).to.deep.equal(expected); + }); + }); +}); From d455cd489774207135d7a0438cd55c86419a17ca Mon Sep 17 00:00:00 2001 From: Michael Moschovas Date: Wed, 12 May 2021 10:29:52 -0400 Subject: [PATCH 18/18] Updated cur to validate as an array of strings not just a string Updated comments --- modules/fpdModule/index.md | 4 ++-- modules/validationFpdModule/config.js | 4 +++- test/spec/modules/fpdModule_spec.js | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/modules/fpdModule/index.md b/modules/fpdModule/index.md index 286822c3c9e..638c966883a 100644 --- a/modules/fpdModule/index.md +++ b/modules/fpdModule/index.md @@ -14,7 +14,7 @@ Enrichment Submodule: Validation Submodule: - verify OpenRTB datatypes, remove/warn any that are likely to choke downstream readers - verify that certain OpenRTB attributes are not specified -- optionally suppress user FPD based on a TBD opt-out signal (_pubcid_optout) +- optionally suppress user FPD based on the existence of _pubcid_optout 1. Module initializes on first load and set bidRequestHook @@ -25,7 +25,7 @@ Validation Submodule: This module will automatically run first party data enrichments and validations dependant on which submodules are included. There is no configuration required. In order to load the module and submodule(s) and opt out of either enrichements or validations, use the below opt out configuration -# Opt Out Configuration +# Module Control Configuration ``` diff --git a/modules/validationFpdModule/config.js b/modules/validationFpdModule/config.js index a520267630b..f6adfea70eb 100644 --- a/modules/validationFpdModule/config.js +++ b/modules/validationFpdModule/config.js @@ -23,7 +23,9 @@ export const ORTB_MAP = { invalid: true }, cur: { - type: TYPES.string + type: TYPES.object, + isArray: true, + childType: TYPES.string }, device: { type: TYPES.object, diff --git a/test/spec/modules/fpdModule_spec.js b/test/spec/modules/fpdModule_spec.js index 2f2d3bee57a..c2a6c41835e 100644 --- a/test/spec/modules/fpdModule_spec.js +++ b/test/spec/modules/fpdModule_spec.js @@ -324,7 +324,8 @@ describe('the first party data module', function () { visitor: ['value2'] } } - } + }, + cur: ['USD'] }; config.setConfig({ortb2: conf}); @@ -335,6 +336,7 @@ describe('the first party data module', function () { expect(validated.site.ref).to.equal(getRefererInfo().referer); expect(validated.site.ext.data).to.deep.equal({inventory: ['value1']}); expect(validated.user.ext.data).to.deep.equal({visitor: ['value2']}); + expect(validated.cur).to.deep.equal(['USD']); }); it('should filter bidderConfig data', function () {