From f239a853d9ab9959e4eaa4aea7fab8eb96ff46a0 Mon Sep 17 00:00:00 2001 From: Viral Vala Date: Fri, 17 Jan 2025 19:35:46 +0530 Subject: [PATCH] UOE-11752: incorporating vastunwrap refactoring changes --- adapters/vastbidder/vastbidder.go | 6 +-- endpoints/events/vtrack_ow.go | 33 ++++------------ endpoints/events/vtrack_ow_test.go | 2 +- exchange/exchange_ow.go | 37 +++++++++++++----- exchange/exchange_ow_test.go | 12 +++--- go.mod | 13 +++--- go.sum | 21 +++++----- openrtb_ext/fastxml_ow.go | 63 ++++++++++++++++++++++-------- openrtb_ext/response.go | 7 ---- router/router_ow.go | 5 ++- 10 files changed, 109 insertions(+), 90 deletions(-) diff --git a/adapters/vastbidder/vastbidder.go b/adapters/vastbidder/vastbidder.go index 778b809ba96..a2999ef507f 100644 --- a/adapters/vastbidder/vastbidder.go +++ b/adapters/vastbidder/vastbidder.go @@ -102,9 +102,9 @@ func (a *VASTBidder) fastXMLTesting(handler *responseHandler, responseData *adap } xmlParsingMetrics := &openrtb_ext.FastXMLMetrics{ - XMLParserTime: handlerTime, - EtreeParserTime: etreeParserTime, - IsRespMismatch: isVASTMismatch, + FastXMLParserTime: handlerTime, + EtreeParserTime: etreeParserTime, + IsRespMismatch: isVASTMismatch, } responseData.FastXMLMetrics = xmlParsingMetrics diff --git a/endpoints/events/vtrack_ow.go b/endpoints/events/vtrack_ow.go index abcd86bac4e..6531632257e 100644 --- a/endpoints/events/vtrack_ow.go +++ b/endpoints/events/vtrack_ow.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/base64" "errors" - "regexp" "strings" "time" @@ -17,7 +16,6 @@ import ( var ( errEventURLNotConfigured = errors.New("event urls not configured") - tmpWSRemoverRegex = regexp.MustCompile(`>\s+<`) ) // InjectVideoEventTrackers injects the video tracking events @@ -62,20 +60,19 @@ func InjectVideoEventTrackers( //temporary if fastXMLResponse != vastXML { - fastXMLResponse, etreeXMLResponse = tmpFastXMLProcessing(fastXMLResponse, response) + fastXMLResponse, etreeXMLResponse = openrtb_ext.FastXMLPostProcessing(fastXMLResponse, response) } - isResponseMismatch := (etreeXMLResponse != fastXMLResponse) + metrics = &openrtb_ext.FastXMLMetrics{ + FastXMLParserTime: fastXMLParserTime, + EtreeParserTime: etreeParserTime, + IsRespMismatch: (etreeXMLResponse != fastXMLResponse), + } - if isResponseMismatch { + if metrics.IsRespMismatch { openrtb_ext.FastXMLLogf("\n[XML_PARSER_TEST] method:[vcr] creative:[%s]", base64.StdEncoding.EncodeToString([]byte(vastXML))) } - metrics = &openrtb_ext.FastXMLMetrics{ - XMLParserTime: fastXMLParserTime, - EtreeParserTime: etreeParserTime, - IsRespMismatch: isResponseMismatch, - } } return response, metrics, err @@ -197,9 +194,6 @@ func injectVideoEventsFastXML(vastXML string, eventURLMap map[string]string, nur return vastXML, nil } - //Add CDATA and Expand Inline Nodes - xu.ApplyXMLSettingsOperations() - var buf bytes.Buffer xu.Build(&buf) return buf.String(), nil @@ -229,16 +223,3 @@ func FindCreatives(doc *etree.Document) []*etree.Element { creatives = append(creatives, doc.FindElements("VAST/Ad/Wrapper/Creatives/Creative/NonLinearAds")...) return creatives } - -func tmpFastXMLProcessing(fastXML, etreeXML string) (string, string) { - //replace only if trackers are injected - fastXML = strings.TrimSpace(fastXML) //step1: remove heading and trailing whitespaces - fastXML = tmpWSRemoverRegex.ReplaceAllString(fastXML, "><") //step2: remove inbetween whitespaces - fastXML = strings.ReplaceAll(fastXML, " ><", "><") //step3: remove attribute endtag whitespace (this should be always before step2) - fastXML = strings.ReplaceAll(fastXML, "'", "\"") //step4: convert single quote to double quote - - etreeXML = tmpWSRemoverRegex.ReplaceAllString(etreeXML, "><") //step2: remove inbetween whitespaces - etreeXML = strings.ReplaceAll(etreeXML, " ><", "><") //step3: remove attribute endtag whitespace (this should be always before step2) - etreeXML = strings.ReplaceAll(etreeXML, "'", "\"") - return fastXML, etreeXML -} diff --git a/endpoints/events/vtrack_ow_test.go b/endpoints/events/vtrack_ow_test.go index d984c568045..4aab84145a1 100644 --- a/endpoints/events/vtrack_ow_test.go +++ b/endpoints/events/vtrack_ow_test.go @@ -544,7 +544,7 @@ func TestCompareXMLParsers(t *testing.T) { fastXML, _ := injectVideoEventsFastXML(vast, eventURLMap, false, adcom1.LinearityLinear) if vast != fastXML { - fastXML, etreeXML = tmpFastXMLProcessing(fastXML, etreeXML) + fastXML, etreeXML = openrtb_ext.FastXMLPostProcessing(fastXML, etreeXML) } if len(debugLines) > 0 { diff --git a/exchange/exchange_ow.go b/exchange/exchange_ow.go index 3549f547b64..13599733f60 100644 --- a/exchange/exchange_ow.go +++ b/exchange/exchange_ow.go @@ -2,6 +2,7 @@ package exchange import ( "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -10,7 +11,7 @@ import ( "strconv" "strings" - "git.pubmatic.com/vastunwrap/unwrap" + unwrapmodels "git.pubmatic.com/vastunwrap/unwrap/models" "github.com/golang/glog" "github.com/prebid/openrtb/v20/openrtb2" "github.com/prebid/prebid-server/v2/adapters" @@ -185,12 +186,28 @@ func recordBids(ctx context.Context, metricsEngine metrics.MetricsEngine, pubID } } -func RecordFastXMLTestMetrics(metricsEngine metrics.MetricsEngine, ctx *unwrap.UnwrapContext, etreeResp, fastxmlResp *unwrap.UnwrapResponse) { - metricsEngine.RecordXMLParserProcessingTime(metrics.XMLParserLabelFastXML, "unwrap", ctx.FastXMLTestCtx.FastXMLStats.ProcessingTime) - metricsEngine.RecordXMLParserProcessingTime(metrics.XMLParserLabelETree, "unwrap", ctx.FastXMLTestCtx.ETreeStats.ProcessingTime) +func RecordFastXMLTestMetrics(metricsEngine metrics.MetricsEngine, ctx *unwrapmodels.UnwrapContext, etreeResp, fastxmlResp *unwrapmodels.UnwrapResponse) { + fastxmlResponse, etreeResponse := openrtb_ext.FastXMLPostProcessing(string(fastxmlResp.Response), string(etreeResp.Response)) + + fastxmlMetrics := openrtb_ext.FastXMLMetrics{ + EtreeParserTime: ctx.FastXMLTestCtx.ETreeStats.ProcessingTime, + FastXMLParserTime: ctx.FastXMLTestCtx.FastXMLStats.ProcessingTime, + IsRespMismatch: etreeResponse != fastxmlResponse, + } + + if fastxmlMetrics.IsRespMismatch { + var flog openrtb_ext.UnwrapFastXMLLog + flog.InputXML = string(ctx.Vast) + for _, mock := range ctx.FastXMLTestCtx.WrapperMock { + flog.VASTXmls = append(flog.VASTXmls, string(mock.GetResponseBytes())) + } + resp, _ := json.Marshal(&flog) + openrtb_ext.FastXMLLogf("\n[XML_PARSER_TEST] method:[unwrap] response:[%s]", base64.StdEncoding.EncodeToString([]byte(resp))) + } + + recordFastXMLMetrics(metricsEngine, "unwrap", &fastxmlMetrics) metricsEngine.RecordXMLParserResponseTime(metrics.XMLParserLabelFastXML, "unwrap", ctx.FastXMLTestCtx.FastXMLStats.ResponseTime) metricsEngine.RecordXMLParserResponseTime(metrics.XMLParserLabelETree, "unwrap", ctx.FastXMLTestCtx.ETreeStats.ResponseTime) - metricsEngine.RecordXMLParserResponseMismatch("unwrap", string(etreeResp.Response) != string(fastxmlResp.Response)) } func recordVastVersion(metricsEngine metrics.MetricsEngine, adapterBids map[openrtb_ext.BidderName]*entities.PbsOrtbSeatBid) { @@ -222,17 +239,17 @@ func recordOpenWrapBidResponseMetrics(bidder *bidderAdapter, bidResponse *adapte recordFastXMLMetrics(bidder.me, "vastbidder", bidResponse.FastXMLMetrics) if bidResponse.FastXMLMetrics.IsRespMismatch { resp, _ := jsonutil.Marshal(bidResponse) - openrtb_ext.FastXMLLogf("\n[XML_PARSER_TEST] method:[vast_bidder] response:[%s]", resp) + openrtb_ext.FastXMLLogf("\n[XML_PARSER_TEST] method:[vast_bidder] response:[%s]", base64.StdEncoding.EncodeToString([]byte(resp))) } } recordVASTTagType(bidder.me, bidResponse, bidder.BidderName) } -func recordFastXMLMetrics(metricsEngine metrics.MetricsEngine, method string, vastBidderInfo *openrtb_ext.FastXMLMetrics) { - metricsEngine.RecordXMLParserProcessingTime(metrics.XMLParserLabelFastXML, method, vastBidderInfo.XMLParserTime) - metricsEngine.RecordXMLParserProcessingTime(metrics.XMLParserLabelETree, method, vastBidderInfo.EtreeParserTime) - metricsEngine.RecordXMLParserResponseMismatch(method, vastBidderInfo.IsRespMismatch) +func recordFastXMLMetrics(metricsEngine metrics.MetricsEngine, method string, fastxmlMetrics *openrtb_ext.FastXMLMetrics) { + metricsEngine.RecordXMLParserProcessingTime(metrics.XMLParserLabelFastXML, method, fastxmlMetrics.FastXMLParserTime) + metricsEngine.RecordXMLParserProcessingTime(metrics.XMLParserLabelETree, method, fastxmlMetrics.EtreeParserTime) + metricsEngine.RecordXMLParserResponseMismatch(method, fastxmlMetrics.IsRespMismatch) } func recordVASTTagType(metricsEngine metrics.MetricsEngine, adapterBids *adapters.BidderResponse, bidder openrtb_ext.BidderName) { diff --git a/exchange/exchange_ow_test.go b/exchange/exchange_ow_test.go index f8dae003c82..0a540537a6f 100644 --- a/exchange/exchange_ow_test.go +++ b/exchange/exchange_ow_test.go @@ -1829,9 +1829,9 @@ func TestRecordFastXMLMetrics(t *testing.T) { name: "Record_Fast_XML_Metrics_Respnse_matched", args: args{ vastBidderInfo: &openrtb_ext.FastXMLMetrics{ - XMLParserTime: time.Millisecond * 10, - EtreeParserTime: time.Millisecond * 20, - IsRespMismatch: false, + FastXMLParserTime: time.Millisecond * 10, + EtreeParserTime: time.Millisecond * 20, + IsRespMismatch: false, }, getMetricsEngine: func() *metrics.MetricsEngineMock { metricEngine := &metrics.MetricsEngineMock{} @@ -1846,9 +1846,9 @@ func TestRecordFastXMLMetrics(t *testing.T) { name: "Record_Fast_XML_Metrics_Respnse_mismatched", args: args{ vastBidderInfo: &openrtb_ext.FastXMLMetrics{ - XMLParserTime: time.Millisecond * 15, - EtreeParserTime: time.Millisecond * 25, - IsRespMismatch: true, + FastXMLParserTime: time.Millisecond * 15, + EtreeParserTime: time.Millisecond * 25, + IsRespMismatch: true, }, getMetricsEngine: func() *metrics.MetricsEngineMock { metricEngine := &metrics.MetricsEngineMock{} diff --git a/go.mod b/go.mod index 2d69826852f..34627b5a446 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/PubMatic-OpenWrap/prebid-server/v2 go 1.21 -replace git.pubmatic.com/vastunwrap => git.pubmatic.com/PubMatic/vastunwrap v0.0.0-20250109114230-9a6dadef48c0 +replace git.pubmatic.com/vastunwrap => git.pubmatic.com/PubMatic/vastunwrap v0.0.0-20250117134707-b97343806e58 require ( github.com/DATA-DOG/go-sqlmock v1.5.0 @@ -29,14 +29,13 @@ require ( github.com/prebid/go-gdpr v1.12.0 github.com/prebid/go-gpp v0.2.0 github.com/prebid/openrtb/v20 v20.1.0 - github.com/prebid/prebid-server/v2 v2.10.0 github.com/prometheus/client_golang v1.12.1 github.com/prometheus/client_model v0.2.0 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 github.com/rs/cors v1.11.0 github.com/sergi/go-diff v1.3.1 // indirect github.com/spf13/viper v1.15.0 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.8.1 github.com/vrischmann/go-metrics-influxdb v0.1.1 github.com/xeipuuv/gojsonschema v1.2.0 github.com/yudai/gojsondiff v1.0.0 @@ -48,12 +47,11 @@ require ( ) require ( - git.pubmatic.com/PubMatic/go-common v0.0.0-20240313090142-97ff3d63b7c3 + git.pubmatic.com/PubMatic/go-common v0.0.0-20250114170528-cb2fb632c358 git.pubmatic.com/PubMatic/go-netacuity-client v0.0.0-20240104092757-5d6f15e25fe3 git.pubmatic.com/vastunwrap v0.0.0-00010101000000-000000000000 github.com/51Degrees/device-detection-go/v4 v4.4.35 - github.com/PubMatic-OpenWrap/fastxml v0.0.0-20241125102315-0d8f851a6e52 - github.com/beevik/etree/110 v0.0.0-00010101000000-000000000000 // indirect + github.com/PubMatic-OpenWrap/fastxml v0.0.0-20250114115626-75b89b46b93d github.com/diegoholiveira/jsonlogic/v3 v3.5.3 github.com/go-sql-driver/mysql v1.7.1 github.com/golang/mock v1.6.0 @@ -64,6 +62,7 @@ require ( ) require ( + github.com/prebid/prebid-server/v2 v2.0.0-00010101000000-000000000000 github.com/spf13/cast v1.5.0 github.com/tidwall/gjson v1.17.1 github.com/tidwall/sjson v1.2.5 @@ -89,7 +88,7 @@ require ( github.com/spf13/afero v1.9.3 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/objx v0.5.2 // indirect + github.com/stretchr/objx v0.5.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect diff --git a/go.sum b/go.sum index 254675caeb9..40ca2c14345 100644 --- a/go.sum +++ b/go.sum @@ -49,12 +49,12 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.pubmatic.com/PubMatic/go-common v0.0.0-20240313090142-97ff3d63b7c3 h1:Ea8zwi1eeX4kqvi9RyyXrizIIRcstM0XBwf8U7NHrno= -git.pubmatic.com/PubMatic/go-common v0.0.0-20240313090142-97ff3d63b7c3/go.mod h1:c/I6IcDn4Mtq4mmw8wGJN3v0o10nIMX7VTuQnsalUw0= +git.pubmatic.com/PubMatic/go-common v0.0.0-20250114170528-cb2fb632c358 h1:A+ohfTqsSCfiLybo+tRQ+YXqCPTYiBp0U9lA8gv8MS8= +git.pubmatic.com/PubMatic/go-common v0.0.0-20250114170528-cb2fb632c358/go.mod h1:I6yt+Te6PaQdUW+pq7/LHNWZ6/+5SSlExknAr3Mfv58= git.pubmatic.com/PubMatic/go-netacuity-client v0.0.0-20240104092757-5d6f15e25fe3 h1:zQUpPJOjTBGu2fIydrfRWphH7EWLlBE/Qgn64BSoccI= git.pubmatic.com/PubMatic/go-netacuity-client v0.0.0-20240104092757-5d6f15e25fe3/go.mod h1:w733mqJnHt0hLR9mIFMzyDR0D94qzc7mFHsuE0tFQho= -git.pubmatic.com/PubMatic/vastunwrap v0.0.0-20240827084017-0e392d3beb8b h1:7AsXylZJDwq514L8KE0Id079VNfUsDEMUIYMlRYH+0Y= -git.pubmatic.com/PubMatic/vastunwrap v0.0.0-20240827084017-0e392d3beb8b/go.mod h1:kcoJf7k+xug8X8fLWmsiKhPnYP+k7RZkfUoUo5QF+KA= +git.pubmatic.com/PubMatic/vastunwrap v0.0.0-20250117134707-b97343806e58 h1:hAaq3Hjnyez2YQjVR2NSb3K6wVNaxvFJJQ7JcUzZtzE= +git.pubmatic.com/PubMatic/vastunwrap v0.0.0-20250117134707-b97343806e58/go.mod h1:BJJ214c6AtpmsyHVDj0Si2sQkOZivn+bKSIAF2DyZlY= github.com/51Degrees/device-detection-go/v4 v4.4.35 h1:qhP2tzoXhGE1aYY3NftMJ+ccxz0+2kM8aF4SH7fTyuA= github.com/51Degrees/device-detection-go/v4 v4.4.35/go.mod h1:dbdG1fySqdY+a5pUnZ0/G0eD03G6H3Vh8kRC+1f9qSc= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -69,8 +69,8 @@ github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMo github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PubMatic-OpenWrap/etree v1.0.2-0.20241125102329-0b5c47d99ad5 h1:uNJ9lOn3q677J2PbR9wbnHec8452lHYvUZCfqMUxk0s= github.com/PubMatic-OpenWrap/etree v1.0.2-0.20241125102329-0b5c47d99ad5/go.mod h1:5Y8qgcuDoy3XXG907UXkGnVTwihF16rXyJa4zRT7hOE= -github.com/PubMatic-OpenWrap/fastxml v0.0.0-20241231130647-0d261d31ec23 h1:Dk/teElV8EdikR6IxuqYvFRejPDCOkjy6QY1ZoIq04M= -github.com/PubMatic-OpenWrap/fastxml v0.0.0-20241231130647-0d261d31ec23/go.mod h1:TGGzSA5ziWpfLsKvqOzgdPGEZ7SJIQjHbcJw6lWoyHU= +github.com/PubMatic-OpenWrap/fastxml v0.0.0-20250114115626-75b89b46b93d h1:3GDh4j2fjVPtDoIQo4jcU+OSvU5+F9aSghX2UYm8I9Q= +github.com/PubMatic-OpenWrap/fastxml v0.0.0-20250114115626-75b89b46b93d/go.mod h1:h3KCtxPGi4/UjwtoWu2Vt07zOxPyLH46za2L74KH0NY= github.com/PubMatic-OpenWrap/prebid-openrtb/v20 v20.0.0-20240222072752-2d647d1707ef h1:CXsyYtEgZz/0++fiug6QZXrRYG6BBrl9IGveCxsnhiE= github.com/PubMatic-OpenWrap/prebid-openrtb/v20 v20.0.0-20240222072752-2d647d1707ef/go.mod h1:hLBrA/APkSrxs5MaW639l+y/EAHivDfRagO2TX/wbSc= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -91,8 +91,6 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:W github.com/aws/aws-sdk-go v1.36.29/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0= github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM= -github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= -github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -495,9 +493,8 @@ github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jH github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -506,9 +503,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= @@ -535,6 +531,7 @@ github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FB github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/openrtb_ext/fastxml_ow.go b/openrtb_ext/fastxml_ow.go index 30e55ba11b4..106da9cb100 100644 --- a/openrtb_ext/fastxml_ow.go +++ b/openrtb_ext/fastxml_ow.go @@ -3,33 +3,30 @@ package openrtb_ext import ( "bufio" "fmt" - "math/rand" "os" "path/filepath" + "regexp" "runtime/debug" "sort" + "strings" "sync" "time" "github.com/golang/glog" + "golang.org/x/exp/rand" ) -type RandomGenerator interface { - GenerateIntn(int) int -} - -type RandomNumberGenerator struct{} - -func (RandomNumberGenerator) GenerateIntn(n int) int { - return rand.Intn(n) -} - -var pid = os.Getpid() +var ( + pid = os.Getpid() + tmpWSRemoverRegex = regexp.MustCompile(`>\s+<`) +) -const maxFileSize = 1 * 1024 * 1024 * 1024 -const maxBufferSize = 256 * 1024 -const maxFiles = 10 -const flushInterval = time.Second * time.Duration(300) +const ( + maxFileSize = 1 * 1024 * 1024 * 1024 + maxBufferSize = 256 * 1024 + maxFiles = 10 + flushInterval = time.Second * time.Duration(300) +) // Writer interface can be used to define variable returned by GetWriter() method type Writer interface { @@ -232,6 +229,40 @@ func FastXMLLogf(format string, args ...any) { } } +func FastXMLPostProcessing(fastXML, etreeXML string) (string, string) { + //replace only if trackers are injected + fastXML = strings.TrimSpace(fastXML) //step1: remove heading and trailing whitespaces + fastXML = tmpWSRemoverRegex.ReplaceAllString(fastXML, "><") //step2: remove inbetween whitespaces + fastXML = strings.ReplaceAll(fastXML, " ><", "><") //step3: remove attribute endtag whitespace (this should be always before step2) + fastXML = strings.ReplaceAll(fastXML, "'", "\"") //step4: convert single quote to double quote + + etreeXML = tmpWSRemoverRegex.ReplaceAllString(etreeXML, "><") //step2: remove inbetween whitespaces + etreeXML = strings.ReplaceAll(etreeXML, " ><", "><") //step3: remove attribute endtag whitespace (this should be always before step2) + etreeXML = strings.ReplaceAll(etreeXML, "'", "\"") + return fastXML, etreeXML +} + +type UnwrapFastXMLLog struct { + InputXML string `json:"input"` + VASTXmls []string `json:"vasts"` +} + +type RandomGenerator interface { + GenerateIntn(int) int +} + +type RandomNumberGenerator struct{} + +func (RandomNumberGenerator) GenerateIntn(n int) int { + return rand.Intn(n) +} + +type FastXMLMetrics struct { + FastXMLParserTime time.Duration `json:"xmlparsertime,omitempty"` + EtreeParserTime time.Duration `json:"etreeparsertime,omitempty"` + IsRespMismatch bool `json:"isrespmismatch,omitempty"` +} + var rg RandomGenerator var bfw Writer diff --git a/openrtb_ext/response.go b/openrtb_ext/response.go index 9fc6d647539..51c0d88bdc6 100644 --- a/openrtb_ext/response.go +++ b/openrtb_ext/response.go @@ -2,7 +2,6 @@ package openrtb_ext import ( "encoding/json" - "time" "github.com/prebid/openrtb/v20/adcom1" "github.com/prebid/openrtb/v20/openrtb2" @@ -67,12 +66,6 @@ type ExtResponsePrebid struct { Floors *PriceFloorRules `json:"floors,omitempty"` } -type FastXMLMetrics struct { - XMLParserTime time.Duration `json:"xmlparsertime,omitempty"` - EtreeParserTime time.Duration `json:"etreeparsertime,omitempty"` - IsRespMismatch bool `json:"isrespmismatch,omitempty"` -} - // FledgeResponse defines the contract for bidresponse.ext.fledge type Fledge struct { AuctionConfigs []*FledgeAuctionConfig `json:"auctionconfigs,omitempty"` diff --git a/router/router_ow.go b/router/router_ow.go index 1b64c864823..229dbd1550c 100644 --- a/router/router_ow.go +++ b/router/router_ow.go @@ -3,7 +3,8 @@ package router import ( "net/http" - unwrap "git.pubmatic.com/vastunwrap/unwrap" + "git.pubmatic.com/vastunwrap/unwrap" + unwrapmodels "git.pubmatic.com/vastunwrap/unwrap/models" "github.com/julienschmidt/httprouter" "github.com/prebid/prebid-server/v2/exchange" middleware "github.com/prebid/prebid-server/v2/modules/pubmatic/openwrap/middleware/adpod" @@ -58,7 +59,7 @@ func (r *Router) registerOpenWrapEndpoints(openrtbEndpoint, ampEndpoint httprout func initFastXMLTest() { if g_metrics != nil { - unwrap.InitRecordFastXMLTestMetrics(func(ctx *unwrap.UnwrapContext, etreeResp, fastxmlResp *unwrap.UnwrapResponse) { + unwrap.InitRecordFastXMLTestMetrics(func(ctx *unwrapmodels.UnwrapContext, etreeResp, fastxmlResp *unwrapmodels.UnwrapResponse) { exchange.RecordFastXMLTestMetrics(g_metrics, ctx, etreeResp, fastxmlResp) }) }