From 247e4d8e32f5088da236fb624e2bc81152d0adb2 Mon Sep 17 00:00:00 2001 From: Markiyan Mykush <95693607+marki1an@users.noreply.github.com> Date: Tue, 19 Mar 2024 12:35:12 +0200 Subject: [PATCH 001/320] Tests: Remove flaky functional tests (#3043) --- .../functional/tests/TimeoutSpec.groovy | 73 ------------------- 1 file changed, 73 deletions(-) diff --git a/src/test/groovy/org/prebid/server/functional/tests/TimeoutSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/TimeoutSpec.groovy index d04808f4fa2..99fa3b31a0e 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/TimeoutSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/TimeoutSpec.groovy @@ -1,11 +1,9 @@ package org.prebid.server.functional.tests -import org.prebid.server.functional.model.bidder.BidderName import org.prebid.server.functional.model.db.StoredRequest import org.prebid.server.functional.model.request.amp.AmpRequest import org.prebid.server.functional.model.request.auction.BidRequest import org.prebid.server.functional.model.request.auction.PrebidStoredRequest -import org.prebid.server.functional.model.response.auction.ErrorType import org.prebid.server.functional.service.PrebidServerService import org.prebid.server.functional.testcontainers.container.PrebidServerContainer import org.prebid.server.functional.util.PBSUtils @@ -17,7 +15,6 @@ import static org.prebid.server.functional.testcontainers.container.PrebidServer class TimeoutSpec extends BaseSpec { private static final int DEFAULT_TIMEOUT = getRandomTimeout() - private static final int MIN_TIMEOUT_BIDDER_REQUEST = 5 private static final int MIN_TIMEOUT = PBSUtils.getRandomNumber(50, 150) private static final Map PBS_CONFIG = ["auction.biddertmax.max" : MAX_TIMEOUT as String, "auction.biddertmax.min" : MIN_TIMEOUT as String] @@ -284,76 +281,6 @@ class TimeoutSpec extends BaseSpec { assert isInternalProcessingTime(bidderRequest.tmax, MAX_TIMEOUT) } - def "PBS amp should return error when auction.biddertmax.min value not enough for bidder request"() { - given: "PBS config with biddertmax.min" - def prebidServerService = pbsServiceFactory.getService(["auction.biddertmax.min" : MIN_TIMEOUT_BIDDER_REQUEST as String]) - - and: "Default AMP request without timeout" - def ampRequest = AmpRequest.defaultAmpRequest.tap { - timeout = null - } - - and: "Default stored request tmax" - def minTmax = MIN_TIMEOUT_BIDDER_REQUEST - 1 - def ampStoredRequest = BidRequest.defaultStoredRequest.tap { - tmax = minTmax - } - - and: "Save storedRequest into DB" - def storedRequestModel = StoredRequest.getStoredRequest(ampRequest, ampStoredRequest) - storedRequestDao.save(storedRequestModel) - - when: "PBS processes amp request" - def bidResponse = prebidServerService.sendAmpRequest(ampRequest) - - then: "Bidder request timeout should correspond to the min from the stored request" - assert bidResponse?.ext?.debug?.resolvedRequest?.tmax == minTmax - - and: "PBS should send to bidder tmax form auction.biddertmax.min config" - assert bidResponse.ext.debug.httpcalls[BidderName.GENERIC.value]*.requestBody[0].contains("\"tmax\":${MIN_TIMEOUT_BIDDER_REQUEST}") - - and: "Bid response should shutdown by timeout from stored request" - def errors = bidResponse.ext?.errors - assert errors[ErrorType.GENERIC]*.code == [1] - assert errors[ErrorType.GENERIC]*.message == ["Timeout has been exceeded"] - } - - def "PBS auction should return error when auction.biddertmax.min value not enough for bidder request"() { - given: "PBS config with biddertmax.min" - def prebidServerService = pbsServiceFactory.getService(["auction.biddertmax.max" : MAX_TIMEOUT as String, - "auction.biddertmax.min" : MIN_TIMEOUT_BIDDER_REQUEST as String]) - - and: "Default BidRequest without timeout" - def bidRequest = BidRequest.defaultBidRequest.tap { - tmax = null - ext.prebid.storedRequest = new PrebidStoredRequest(id: PBSUtils.randomNumber) - } - - and: "Default stored request with min tmax" - def minTmax = MIN_TIMEOUT_BIDDER_REQUEST + 4 - def storedRequest = BidRequest.defaultStoredRequest.tap { - tmax = minTmax - } - - and: "Save storedRequest into DB" - def storedRequestModel = StoredRequest.getStoredRequest(bidRequest.ext.prebid.storedRequest.id, storedRequest) - storedRequestDao.save(storedRequestModel) - - when: "PBS processes auction request" - def bidResponse = prebidServerService.sendAuctionRequest(bidRequest) - - then: "Bidder request timeout should correspond to the min from the stored request" - assert bidResponse?.ext?.debug?.resolvedRequest?.tmax == minTmax - - and: "PBS should send to bidder tmax form auction.biddertmax.min config" - assert bidResponse.ext.debug.httpcalls[BidderName.GENERIC.value]*.requestBody[0].contains("\"tmax\":${MIN_TIMEOUT_BIDDER_REQUEST}") - - and: "Bid response should shutdown by timeout from stored request" - def errors = bidResponse.ext?.errors - assert errors[ErrorType.GENERIC]*.code == [1] - assert errors[ErrorType.GENERIC]*.message == ["Timeout has been exceeded"] - } - def "PBS should choose min timeout form config for bidder request when in request value lowest that in auction.biddertmax.min"() { given: "PBS config with percent" def minBidderTmax = PBSUtils.getRandomNumber(MIN_TIMEOUT, MAX_TIMEOUT) From fbdfff6411578d40396a98e8b7afb50c81dfece7 Mon Sep 17 00:00:00 2001 From: Oleksandr Zhevedenko Date: Tue, 19 Mar 2024 06:36:41 -0400 Subject: [PATCH 002/320] Docs: Documentation for transmitEids (#3058) --- docs/application-settings.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/application-settings.md b/docs/application-settings.md index 4999cd293a5..4abd9ac5bc2 100644 --- a/docs/application-settings.md +++ b/docs/application-settings.md @@ -54,6 +54,9 @@ Keep in mind following restrictions: to `sfN.enforce` value. - `privacy.gdpr.purpose-one-treatment-interpretation` - option that allows to skip the Purpose one enforcement workflow. Values: ignore, no-access-allowed, access-allowed. +- `privacy.gdpr.purposes.p4.eid.require_consent` - if equals to `true`, transmitting EIDs require P4 legal basis unless excepted. +- `privacy.gdpr.purposes.p4.eid.exceptions` - list of EID sources that are excepted from P4 enforcement and will be transmitted if any P2-P10 is consented. +- `privacy.gdpr.purposes.p4.eid.activity_transition` - defaults to `true`. If `true` and transmitEids is not specified, but transmitUfpd is specified, then the logic of transmitUfpd is used. This is to avoid breaking changes to existing configurations. The default value of the flag will be changed in a future release. - `metrics.verbosity-level` - defines verbosity level of metrics for this account, overrides `metrics.accounts` application settings configuration. - `analytics.auction-events.` - defines which channels are supported by analytics for this account - `analytics.modules..*` - space for `module-name` analytics module specific configuration, may be of any shape From 183b3b1381ff6be13387a2fa2cc06d4511348ba5 Mon Sep 17 00:00:00 2001 From: bretg Date: Tue, 19 Mar 2024 08:20:48 -0400 Subject: [PATCH 003/320] Mgidx: Update endpoint (#3053) --- src/main/resources/bidder-config/mgidx.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/resources/bidder-config/mgidx.yaml b/src/main/resources/bidder-config/mgidx.yaml index 0228aa09cc7..7c1b74aaae3 100644 --- a/src/main/resources/bidder-config/mgidx.yaml +++ b/src/main/resources/bidder-config/mgidx.yaml @@ -1,6 +1,8 @@ adapters: mgidX: - endpoint: https://us-east-x.mgid.com/pserver + # We have the following regional endpoint domains: 'us-east-x' and 'eu' + # Please deploy this config in each of your datacenters with the appropriate regional subdomain + endpoint: https://REGION.mgid.com/pserver meta-info: maintainer-email: prebid@mgid.com app-media-types: From f580dcc3999033c18276de692ff0809165f5f593 Mon Sep 17 00:00:00 2001 From: Anton Babak <76536883+AntoxaAntoxic@users.noreply.github.com> Date: Tue, 19 Mar 2024 13:24:33 +0100 Subject: [PATCH 004/320] Core: Relax VAST tag matching for `vtrack` handling (#3059) --- .../org/prebid/server/vast/VastModifier.java | 78 ++++++----- .../server/functional/tests/CacheSpec.groovy | 50 +++++++ .../prebid/server/vast/VastModifierTest.java | 127 +++++++++++++++++- 3 files changed, 218 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/prebid/server/vast/VastModifier.java b/src/main/java/org/prebid/server/vast/VastModifier.java index ce80078bfc1..d5896db41af 100644 --- a/src/main/java/org/prebid/server/vast/VastModifier.java +++ b/src/main/java/org/prebid/server/vast/VastModifier.java @@ -14,15 +14,24 @@ import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class VastModifier { - private static final String IN_LINE_TAG = ""; - private static final String IN_LINE_CLOSE_TAG = ""; - private static final String WRAPPER_TAG = ""; - private static final String WRAPPER_CLOSE_TAG = ""; - private static final String IMPRESSION_CLOSE_TAG = ""; + private static final Pattern WRAPPER_OPEN_TAG_PATTERN = + Pattern.compile("<\\s*wrapper(?:>|\\s.*?>)", Pattern.CASE_INSENSITIVE); + private static final Pattern WRAPPER_CLOSE_TAG_PATTERN = + Pattern.compile("<\\s*/\\s*wrapper(?:>|\\s.*?>)", Pattern.CASE_INSENSITIVE); + private static final Pattern INLINE_OPEN_TAG_PATTERN = + Pattern.compile("<\\s*inline(?:>|\\s.*?>)", Pattern.CASE_INSENSITIVE); + private static final Pattern INLINE_CLOSE_TAG_PATTERN = + Pattern.compile("<\\s*/\\s*inline(?:>|\\s.*?>)", Pattern.CASE_INSENSITIVE); + private static final Pattern IMPRESSION_CLOSE_TAG_PATTERN = + Pattern.compile("<\\s*/\\s*impression(?:>|\\s.*?>)", Pattern.CASE_INSENSITIVE); + private final BidderCatalog bidderCatalog; private final EventsService eventsService; private final Metrics metrics; @@ -102,45 +111,42 @@ private static String resolveVastXmlFrom(String bidAdm, String bidNurl) { : bidAdm; } - private String appendTrackingUrlToVastXml(String vastXml, String vastUrlTracking, String bidder) { - final int inLineTagIndex = StringUtils.indexOfIgnoreCase(vastXml, IN_LINE_TAG); - final int wrapperTagIndex = StringUtils.indexOfIgnoreCase(vastXml, WRAPPER_TAG); + private static String appendTrackingUrlToVastXml(String xml, String urlTracking, String bidder) { + return appendTrackingUrl(xml, urlTracking, INLINE_OPEN_TAG_PATTERN, INLINE_CLOSE_TAG_PATTERN) + .or(() -> appendTrackingUrl(xml, urlTracking, WRAPPER_OPEN_TAG_PATTERN, WRAPPER_CLOSE_TAG_PATTERN)) + .orElseThrow(() -> new PreBidException( + "VastXml does not contain neither InLine nor Wrapper for %s response".formatted(bidder))); + } + + private static Optional appendTrackingUrl(String vastXml, + String vastUrlTracking, + Pattern openTagPattern, + Pattern closeTagPattern) { - if (inLineTagIndex != -1) { - return appendTrackingUrl(vastXml, vastUrlTracking, IN_LINE_CLOSE_TAG); - } else if (wrapperTagIndex != -1) { - return appendTrackingUrl(vastXml, vastUrlTracking, WRAPPER_CLOSE_TAG); + final Matcher openTagMatcher = openTagPattern.matcher(vastXml); + if (!openTagMatcher.find()) { + return Optional.empty(); } - throw new PreBidException("VastXml does not contain neither InLine nor Wrapper for %s response" - .formatted(bidder)); - } - private static String appendTrackingUrl(String vastXml, String vastUrlTracking, String elementCloseTag) { - if (vastXml.contains(IMPRESSION_CLOSE_TAG)) { - return insertAfterExistingImpressionTag(vastXml, vastUrlTracking); + final Matcher impressionCloseTagMatcher = IMPRESSION_CLOSE_TAG_PATTERN.matcher(vastXml); + if (impressionCloseTagMatcher.find(openTagMatcher.end())) { + int replacementEnd = impressionCloseTagMatcher.end(); + while (impressionCloseTagMatcher.find(replacementEnd)) { + replacementEnd = impressionCloseTagMatcher.end(); + } + return Optional.of(insertUrlTracking(vastXml, replacementEnd, vastUrlTracking)); } - return insertBeforeElementCloseTag(vastXml, vastUrlTracking, elementCloseTag); - } - private static String insertAfterExistingImpressionTag(String vastXml, String vastUrlTracking) { - final String impressionTag = ""; - final int replacementStart = vastXml.lastIndexOf(IMPRESSION_CLOSE_TAG); + final Matcher closeTagMatcher = closeTagPattern.matcher(vastXml); + if (!closeTagMatcher.find(openTagMatcher.end())) { + return Optional.of(vastXml); + } - return vastXml.substring(0, replacementStart) + IMPRESSION_CLOSE_TAG + impressionTag - + vastXml.substring(replacementStart + IMPRESSION_CLOSE_TAG.length()); + return Optional.of(insertUrlTracking(vastXml, closeTagMatcher.start(), vastUrlTracking)); } - private static String insertBeforeElementCloseTag(String vastXml, String vastUrlTracking, String elementCloseTag) { - final int indexOfCloseTag = StringUtils.indexOfIgnoreCase(vastXml, elementCloseTag); - - if (indexOfCloseTag == -1) { - return vastXml; - } - - final String caseSpecificCloseTag = - vastXml.substring(indexOfCloseTag, indexOfCloseTag + elementCloseTag.length()); + private static String insertUrlTracking(String vastXml, int index, String vastUrlTracking) { final String impressionTag = ""; - - return vastXml.replace(caseSpecificCloseTag, impressionTag + caseSpecificCloseTag); + return vastXml.substring(0, index) + impressionTag + vastXml.substring(index); } } diff --git a/src/test/groovy/org/prebid/server/functional/tests/CacheSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/CacheSpec.groovy index 3f9ccc7ab43..ccd5f8b9cf0 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/CacheSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/CacheSpec.groovy @@ -1,5 +1,9 @@ package org.prebid.server.functional.tests +import org.prebid.server.functional.model.config.AccountAuctionConfig +import org.prebid.server.functional.model.config.AccountConfig +import org.prebid.server.functional.model.config.AccountEventsConfig +import org.prebid.server.functional.model.db.Account import org.prebid.server.functional.model.request.auction.Asset import org.prebid.server.functional.model.request.auction.BidRequest import org.prebid.server.functional.model.request.auction.Imp @@ -192,4 +196,50 @@ class CacheSpec extends BaseSpec { false | BANNER true | VIDEO } + + def "PBS should update prebid_cache.creative_size.xml metric and adding tracking xml when xml creative contain #wrapper and impression are valid xml value"() { + given: "Current value of metric prebid_cache.requests.ok" + def initialValue = getCurrentMetricValue(defaultPbsService, "prebid_cache.requests.ok") + + and: "Create and save enabled events config in account" + def accountId = PBSUtils.randomNumber.toString() + def account = new Account().tap { + uuid = accountId + config = new AccountConfig().tap { + auction = new AccountAuctionConfig(events: new AccountEventsConfig(enabled: true)) + } + } + accountDao.save(account) + + and: "Vtrack request with custom tags" + def payload = PBSUtils.randomString + def creative = "<${wrapper}>prebid.org wrapper" + + "<![CDATA[//${payload}]]>" + + "<${impression}> <![CDATA[ ]]> " + def request = VtrackRequest.getDefaultVtrackRequest(creative) + + when: "PBS processes vtrack request" + defaultPbsService.sendVtrackRequest(request, accountId) + + then: "Vast xml is modified" + def prebidCacheRequest = prebidCache.getXmlRecordedRequestsBody(payload) + assert prebidCacheRequest.size() == 1 + assert prebidCacheRequest[0].contains("/event?t=imp&b=${request.puts[0].bidid}&a=$accountId&bidder=${request.puts[0].bidder}") + + and: "prebid_cache.creative_size.xml metric should be updated" + def metrics = defaultPbsService.sendCollectedMetricsRequest() + assert metrics["prebid_cache.requests.ok"] == initialValue + 1 + + and: "account..prebid_cache.creative_size.xml should be updated" + assert metrics["account.${accountId}.prebid_cache.requests.ok" as String] == 1 + + where: + wrapper | impression + " wrapper " | " impression " + PBSUtils.getRandomCase(" wrapper ") | PBSUtils.getRandomCase(" impression ") + " wraPPer ${PBSUtils.getRandomString()} " | " imPreSSion ${PBSUtils.getRandomString()}" + " inLine " | " ImpreSSion $PBSUtils.randomNumber" + PBSUtils.getRandomCase(" inline ") | " ${PBSUtils.getRandomCase(" impression ")} $PBSUtils.randomNumber " + " inline ${PBSUtils.getRandomString()} " | " ImpreSSion " + } } diff --git a/src/test/java/org/prebid/server/vast/VastModifierTest.java b/src/test/java/org/prebid/server/vast/VastModifierTest.java index 5593a535028..89dce72913d 100644 --- a/src/test/java/org/prebid/server/vast/VastModifierTest.java +++ b/src/test/java/org/prebid/server/vast/VastModifierTest.java @@ -179,6 +179,21 @@ public void createBidVastXmlShouldBeModifiedWithNewImpressionVastUrlWhenEventsEn + ""); } + @Test + public void createBidVastXmlShouldBeModifiedWithNewImpressionVastUrlWhenEventsEnabledAndNoEmptyTag2() { + // when + final String bidAdm = "< impreSSion garbage >http:/test.com< /ImPression garbage >"; + final String result = target + .createBidVastXml(BIDDER, bidAdm, BID_NURL, BID_ID, ACCOUNT_ID, eventsContext(), emptyList(), + LINEITEM_ID); + + // then + verify(eventsService).vastUrlTracking(BID_ID, BIDDER, ACCOUNT_ID, LINEITEM_ID, eventsContext()); + + assertThat(result).isEqualTo("< impreSSion garbage >http:/test.com< /ImPression garbage >" + + ""); + } + @Test public void createBidVastXmlShouldBeModifiedWithNewImpressionAfterExistingImpressionTags() { // when @@ -196,6 +211,23 @@ public void createBidVastXmlShouldBeModifiedWithNewImpressionAfterExistingImpres + ""); } + @Test + public void createBidVastXmlShouldBeModifiedWithNewImpressionAfterExistingImpressionTags2() { + // when + final String bidAdm = "< Impression >http:/test.com< /Impression >" + + "http:/test2.com< /ImPRession garbage>"; + final String result = target + .createBidVastXml(BIDDER, bidAdm, BID_NURL, BID_ID, ACCOUNT_ID, eventsContext(), emptyList(), + LINEITEM_ID); + + // then + verify(eventsService).vastUrlTracking(BID_ID, BIDDER, ACCOUNT_ID, LINEITEM_ID, eventsContext()); + + assertThat(result).isEqualTo("< Impression >http:/test.com< /Impression >" + + "http:/test2.com< /ImPRession garbage>" + + ""); + } + @Test public void createBidVastXmlShouldInsertImpressionTagForEmptyInLine() { // when @@ -239,11 +271,27 @@ public void createBidVastXmlShouldModifyWrapperTagInCaseInsensitiveMode() { + ""); } + @Test + public void createBidVastXmlShouldModifyWrapperTagInCaseInsensitiveMode2() { + // when + final String bidAdm = "< wraPPer garbage>http:/test.com< / wraPPer garbage>"; + final String result = target + .createBidVastXml(BIDDER, bidAdm, BID_NURL, BID_ID, ACCOUNT_ID, eventsContext(), emptyList(), + LINEITEM_ID); + + // then + verify(eventsService).vastUrlTracking(BID_ID, BIDDER, ACCOUNT_ID, LINEITEM_ID, eventsContext()); + + assertThat(result).isEqualTo("< wraPPer garbage>http:/test.com" + + "< / wraPPer garbage>"); + } + @Test public void createBidVastXmlShouldInsertImpressionTagForEmptyWrapper() { // when + final String bidAdm = ""; final String result = target - .createBidVastXml(BIDDER, "", BID_NURL, + .createBidVastXml(BIDDER, bidAdm, BID_NURL, BID_ID, ACCOUNT_ID, eventsContext(), emptyList(), LINEITEM_ID); @@ -254,6 +302,23 @@ BID_ID, ACCOUNT_ID, eventsContext(), emptyList(), .isEqualTo(""); } + @Test + public void createBidVastXmlShouldInsertImpressionTagForEmptyWrapper2() { + // when + final String bidAdm = "< wraPPer garbage>< / wrapPer garbage>"; + final String result = target + .createBidVastXml(BIDDER, bidAdm, BID_NURL, + BID_ID, ACCOUNT_ID, eventsContext(), emptyList(), + LINEITEM_ID); + + // then + verify(eventsService).vastUrlTracking(BID_ID, BIDDER, ACCOUNT_ID, LINEITEM_ID, eventsContext()); + + assertThat(result) + .isEqualTo("< wraPPer garbage>< / wrapPer garbage>"); + } + @Test public void createBidVastXmlShouldNotInsertImpressionTagForNoWrapperCloseTag() { // when @@ -283,6 +348,21 @@ public void createBidVastXmlShouldModifyInlineTagInCaseInsensitiveMode() { + ""); } + @Test + public void createBidVastXmlShouldModifyInlineTagInCaseInsensitiveMode2() { + // when + final String bidAdm = "< InLIne garbage >http:/test.com"; + final String result = target + .createBidVastXml(BIDDER, bidAdm, BID_NURL, BID_ID, ACCOUNT_ID, eventsContext(), emptyList(), + LINEITEM_ID); + + // then + verify(eventsService).vastUrlTracking(BID_ID, BIDDER, ACCOUNT_ID, LINEITEM_ID, eventsContext()); + + assertThat(result).isEqualTo("< InLIne garbage >http:/test.com" + + ""); + } + @Test public void createBidVastXmlShouldBeModifiedIfInLineHasNoImpressionTags() { // when @@ -297,6 +377,21 @@ public void createBidVastXmlShouldBeModifiedIfInLineHasNoImpressionTags() { assertThat(result).isEqualTo(""); } + @Test + public void createBidVastXmlShouldBeModifiedIfInLineHasNoImpressionTags2() { + // when + final String bidAdm = "< InLIne garbage >< / InLIne garbage >"; + final String result = target + .createBidVastXml(BIDDER, bidAdm, BID_NURL, BID_ID, ACCOUNT_ID, eventsContext(), emptyList(), + LINEITEM_ID); + + // then + verify(eventsService).vastUrlTracking(BID_ID, BIDDER, ACCOUNT_ID, LINEITEM_ID, eventsContext()); + + assertThat(result).isEqualTo("< InLIne garbage >< / InLIne garbage >"); + } + @Test public void createBidVastXmlShouldNotBeModifiedIfNoParentTagsPresent() { // when @@ -312,6 +407,36 @@ public void createBidVastXmlShouldNotBeModifiedIfNoParentTagsPresent() { verify(metrics).updateAdapterRequestErrorMetric(BIDDER, MetricName.badserverresponse); } + @Test + public void createBidVastXmlShouldNotBeModifiedIfWrapperTagIsInvalid() { + // when + final String adm = ""; + final List warnings = new ArrayList<>(); + final String result = target + .createBidVastXml(BIDDER, adm, BID_NURL, BID_ID, ACCOUNT_ID, eventsContext(), warnings, LINEITEM_ID); + + // then + verify(eventsService).vastUrlTracking(BID_ID, BIDDER, ACCOUNT_ID, LINEITEM_ID, eventsContext()); + assertThat(result).isEqualTo(adm); + assertThat(warnings).containsExactly("VastXml does not contain neither InLine nor Wrapper for bidder response"); + verify(metrics).updateAdapterRequestErrorMetric(BIDDER, MetricName.badserverresponse); + } + + @Test + public void createBidVastXmlShouldNotBeModifiedIfInlineTagIsInvalid() { + // when + final String adm = ""; + final List warnings = new ArrayList<>(); + final String result = target + .createBidVastXml(BIDDER, adm, BID_NURL, BID_ID, ACCOUNT_ID, eventsContext(), warnings, LINEITEM_ID); + + // then + verify(eventsService).vastUrlTracking(BID_ID, BIDDER, ACCOUNT_ID, LINEITEM_ID, eventsContext()); + assertThat(result).isEqualTo(adm); + assertThat(warnings).containsExactly("VastXml does not contain neither InLine nor Wrapper for bidder response"); + verify(metrics).updateAdapterRequestErrorMetric(BIDDER, MetricName.badserverresponse); + } + @Test public void createBidVastXmlShouldNotModifyWhenEventsEnabledAndAdmHaveNoImpression() { // when From d87c84031d3604f7c97aa83f92e39941e854b70d Mon Sep 17 00:00:00 2001 From: Anton Babak <76536883+AntoxaAntoxic@users.noreply.github.com> Date: Wed, 20 Mar 2024 11:34:09 +0100 Subject: [PATCH 005/320] Core: Geo Lookup Updates (#3057) --- .../auction/GeoLocationServiceWrapper.java | 86 +++++++ .../server/auction/model/AuctionContext.java | 7 + .../AmpPrivacyContextFactory.java | 3 +- .../AuctionPrivacyContextFactory.java | 3 +- .../requestfactory/AmpRequestFactory.java | 16 +- .../requestfactory/AuctionRequestFactory.java | 16 +- .../requestfactory/Ortb2RequestFactory.java | 62 +++-- .../requestfactory/VideoRequestFactory.java | 16 +- .../privacy/gdpr/TcfDefinerService.java | 63 +++-- .../prebid/server/settings/model/Account.java | 2 + .../settings/model/AccountSettings.java | 12 + .../config/GeoLocationConfiguration.java | 36 +-- .../config/PrivacyServiceConfiguration.java | 7 +- .../spring/config/ServiceConfiguration.java | 19 +- .../model/config/AccountConfig.groovy | 1 + .../model/config/AccountSetting.groovy | 12 + .../model/pricefloors/Country.groovy | 19 +- .../request/auction/PublicCountryIp.groovy | 16 ++ .../server/functional/tests/GeoSpec.groovy | 222 +++++++++++++++++ .../tests/privacy/ActivityTraceLogSpec.groovy | 14 +- .../privacy/GppFetchBidActivitiesSpec.groovy | 4 +- .../privacy/GppSyncUserActivitiesSpec.groovy | 96 ++++++-- .../GppTransmitEidsActivitiesSpec.groovy | 4 +- ...GppTransmitPreciseGeoActivitiesSpec.groovy | 4 +- .../GppTransmitUfpdActivitiesSpec.groovy | 4 +- .../GeoLocationServiceWrapperTest.java | 224 ++++++++++++++++++ .../AmpPrivacyContextFactoryTest.java | 16 +- .../AuctionPrivacyContextFactoryTest.java | 12 +- .../requestfactory/AmpRequestFactoryTest.java | 18 +- .../AuctionRequestFactoryTest.java | 18 +- .../Ortb2RequestFactoryTest.java | 133 ++++++++--- .../VideoRequestFactoryTest.java | 21 +- .../privacy/gdpr/TcfDefinerServiceTest.java | 169 ++++++------- 33 files changed, 1080 insertions(+), 275 deletions(-) create mode 100644 src/main/java/org/prebid/server/auction/GeoLocationServiceWrapper.java create mode 100644 src/main/java/org/prebid/server/settings/model/AccountSettings.java create mode 100644 src/test/groovy/org/prebid/server/functional/model/config/AccountSetting.groovy create mode 100644 src/test/groovy/org/prebid/server/functional/model/request/auction/PublicCountryIp.groovy create mode 100644 src/test/groovy/org/prebid/server/functional/tests/GeoSpec.groovy create mode 100644 src/test/java/org/prebid/server/auction/GeoLocationServiceWrapperTest.java diff --git a/src/main/java/org/prebid/server/auction/GeoLocationServiceWrapper.java b/src/main/java/org/prebid/server/auction/GeoLocationServiceWrapper.java new file mode 100644 index 00000000000..7671475676d --- /dev/null +++ b/src/main/java/org/prebid/server/auction/GeoLocationServiceWrapper.java @@ -0,0 +1,86 @@ +package org.prebid.server.auction; + +import com.iab.openrtb.request.BidRequest; +import com.iab.openrtb.request.Device; +import com.iab.openrtb.request.Geo; +import io.vertx.core.Future; +import io.vertx.core.logging.Logger; +import io.vertx.core.logging.LoggerFactory; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.prebid.server.auction.model.AuctionContext; +import org.prebid.server.execution.Timeout; +import org.prebid.server.geolocation.GeoLocationService; +import org.prebid.server.geolocation.model.GeoInfo; +import org.prebid.server.metric.Metrics; +import org.prebid.server.settings.model.Account; +import org.prebid.server.settings.model.AccountSettings; + +import java.util.Objects; +import java.util.Optional; + +public class GeoLocationServiceWrapper { + + private static final Logger logger = LoggerFactory.getLogger(GeoLocationServiceWrapper.class); + + private final GeoLocationService geoLocationService; + private final Metrics metrics; + + public GeoLocationServiceWrapper(GeoLocationService geoLocationService, Metrics metrics) { + this.geoLocationService = geoLocationService; + this.metrics = Objects.requireNonNull(metrics); + } + + //todo: account settings will work as expected if the default account resolving refactoring is done + public Future lookup(AuctionContext auctionContext) { + final Account account = auctionContext.getAccount(); + final BidRequest bidRequest = auctionContext.getBidRequest(); + final Timeout timeout = auctionContext.getTimeoutContext().getTimeout(); + + final boolean isGeoLookupEnabled = Optional.ofNullable(account.getSettings()) + .map(AccountSettings::getGeoLookup) + .map(BooleanUtils::isTrue) + .orElse(false); + + final Device device = bidRequest.getDevice(); + + return isGeoLookupEnabled + ? doLookup(getIpAddress(device), getCountry(device), timeout).otherwiseEmpty() + : Future.succeededFuture(); + } + + public Future doLookup(String ipAddress, String requestCountry, Timeout timeout) { + if (geoLocationService == null || StringUtils.isNotBlank(requestCountry) || ipAddress == null) { + return Future.failedFuture("Geolocation lookup is skipped"); + } + return geoLocationService.lookup(ipAddress, timeout) + .onSuccess(geoInfo -> metrics.updateGeoLocationMetric(true)) + .onFailure(this::logError); + } + + private String getCountry(Device device) { + return Optional.ofNullable(device) + .map(Device::getGeo) + .map(Geo::getCountry) + .filter(StringUtils::isNotBlank) + .orElse(null); + } + + private String getIpAddress(Device device) { + final Optional optionalDevice = Optional.ofNullable(device); + return optionalDevice.map(Device::getIp) + .filter(StringUtils::isNotBlank) + .or(() -> optionalDevice + .map(Device::getIpv6) + .filter(StringUtils::isNotBlank)) + .orElse(null); + } + + private void logError(Throwable error) { + final String message = "Geolocation lookup failed: " + error.getMessage(); + logger.warn(message); + logger.debug(message, error); + + metrics.updateGeoLocationMetric(false); + } +} diff --git a/src/main/java/org/prebid/server/auction/model/AuctionContext.java b/src/main/java/org/prebid/server/auction/model/AuctionContext.java index 39a2b09b81c..a4c8087f0c7 100644 --- a/src/main/java/org/prebid/server/auction/model/AuctionContext.java +++ b/src/main/java/org/prebid/server/auction/model/AuctionContext.java @@ -60,6 +60,7 @@ public class AuctionContext { ActivityInfrastructure activityInfrastructure; + @JsonIgnore GeoInfo geoInfo; HookExecutionContext hookExecutionContext; @@ -123,6 +124,12 @@ public AuctionContext with(DebugContext debugContext) { .build(); } + public AuctionContext with(GeoInfo geoInfo) { + return this.toBuilder() + .geoInfo(geoInfo) + .build(); + } + public AuctionContext withRequestRejected() { return this.toBuilder() .requestRejected(true) diff --git a/src/main/java/org/prebid/server/auction/privacy/contextfactory/AmpPrivacyContextFactory.java b/src/main/java/org/prebid/server/auction/privacy/contextfactory/AmpPrivacyContextFactory.java index 1f1db91f7aa..84f9695c3ea 100644 --- a/src/main/java/org/prebid/server/auction/privacy/contextfactory/AmpPrivacyContextFactory.java +++ b/src/main/java/org/prebid/server/auction/privacy/contextfactory/AmpPrivacyContextFactory.java @@ -61,7 +61,8 @@ public Future contextFrom(AuctionContext auctionContext) { accountGdprConfig(account), requestType, requestLogInfo(requestType, bidRequest, account.getId()), - auctionContext.getTimeoutContext().getTimeout()) + auctionContext.getTimeoutContext().getTimeout(), + auctionContext.getGeoInfo()) .map(tcfContext -> logWarnings(auctionContext.getDebugWarnings(), tcfContext)) .map(tcfContext -> PrivacyContext.of(strippedPrivacy, tcfContext, tcfContext.getIpAddress())); } diff --git a/src/main/java/org/prebid/server/auction/privacy/contextfactory/AuctionPrivacyContextFactory.java b/src/main/java/org/prebid/server/auction/privacy/contextfactory/AuctionPrivacyContextFactory.java index debcefe86e4..087b72a67c6 100644 --- a/src/main/java/org/prebid/server/auction/privacy/contextfactory/AuctionPrivacyContextFactory.java +++ b/src/main/java/org/prebid/server/auction/privacy/contextfactory/AuctionPrivacyContextFactory.java @@ -57,7 +57,8 @@ public Future contextFrom(AuctionContext auctionContext) { accountGdprConfig(account), requestType, requestLogInfo(requestType, bidRequest, account.getId()), - auctionContext.getTimeoutContext().getTimeout()) + auctionContext.getTimeoutContext().getTimeout(), + auctionContext.getGeoInfo()) .map(tcfContext -> logWarnings(auctionContext.getDebugWarnings(), tcfContext)) .map(tcfContext -> PrivacyContext.of(privacy, tcfContext, tcfContext.getIpAddress())); } diff --git a/src/main/java/org/prebid/server/auction/requestfactory/AmpRequestFactory.java b/src/main/java/org/prebid/server/auction/requestfactory/AmpRequestFactory.java index ec9a3875a07..433584f8aa4 100644 --- a/src/main/java/org/prebid/server/auction/requestfactory/AmpRequestFactory.java +++ b/src/main/java/org/prebid/server/auction/requestfactory/AmpRequestFactory.java @@ -19,6 +19,7 @@ import org.apache.commons.lang3.StringUtils; import org.prebid.server.auction.DebugResolver; import org.prebid.server.auction.FpdResolver; +import org.prebid.server.auction.GeoLocationServiceWrapper; import org.prebid.server.auction.ImplicitParametersExtractor; import org.prebid.server.auction.OrtbTypesResolver; import org.prebid.server.auction.PriceGranularity; @@ -96,6 +97,7 @@ public class AmpRequestFactory { private final AmpPrivacyContextFactory ampPrivacyContextFactory; private final DebugResolver debugResolver; private final JacksonMapper mapper; + private final GeoLocationServiceWrapper geoLocationServiceWrapper; public AmpRequestFactory(Ortb2RequestFactory ortb2RequestFactory, StoredRequestProcessor storedRequestProcessor, @@ -107,7 +109,8 @@ public AmpRequestFactory(Ortb2RequestFactory ortb2RequestFactory, FpdResolver fpdResolver, AmpPrivacyContextFactory ampPrivacyContextFactory, DebugResolver debugResolver, - JacksonMapper mapper) { + JacksonMapper mapper, + GeoLocationServiceWrapper geoLocationServiceWrapper) { this.ortb2RequestFactory = Objects.requireNonNull(ortb2RequestFactory); this.storedRequestProcessor = Objects.requireNonNull(storedRequestProcessor); @@ -120,6 +123,7 @@ public AmpRequestFactory(Ortb2RequestFactory ortb2RequestFactory, this.debugResolver = Objects.requireNonNull(debugResolver); this.ampPrivacyContextFactory = Objects.requireNonNull(ampPrivacyContextFactory); this.mapper = Objects.requireNonNull(mapper); + this.geoLocationServiceWrapper = Objects.requireNonNull(geoLocationServiceWrapper); } /** @@ -142,6 +146,12 @@ public Future fromRequest(RoutingContext routingContext, long st .map(auctionContext -> auctionContext.with(debugResolver.debugContextFrom(auctionContext))) + .compose(auctionContext -> geoLocationServiceWrapper.lookup(auctionContext) + .map(auctionContext::with)) + + .compose(auctionContext -> ortb2RequestFactory.enrichBidRequestWithGeolocationData(auctionContext) + .map(auctionContext::with)) + .compose(auctionContext -> gppService.contextFrom(auctionContext) .map(auctionContext::with)) @@ -154,8 +164,8 @@ public Future fromRequest(RoutingContext routingContext, long st .compose(auctionContext -> ampPrivacyContextFactory.contextFrom(auctionContext) .map(auctionContext::with)) - .map(auctionContext -> auctionContext.with( - ortb2RequestFactory.enrichBidRequestWithAccountAndPrivacyData(auctionContext))) + .compose(auctionContext -> ortb2RequestFactory.enrichBidRequestWithAccountAndPrivacyData(auctionContext) + .map(auctionContext::with)) .compose(auctionContext -> ortb2RequestFactory.executeProcessedAuctionRequestHooks(auctionContext) .map(auctionContext::with)) diff --git a/src/main/java/org/prebid/server/auction/requestfactory/AuctionRequestFactory.java b/src/main/java/org/prebid/server/auction/requestfactory/AuctionRequestFactory.java index 128e4b2630c..95f5768c523 100644 --- a/src/main/java/org/prebid/server/auction/requestfactory/AuctionRequestFactory.java +++ b/src/main/java/org/prebid/server/auction/requestfactory/AuctionRequestFactory.java @@ -7,6 +7,7 @@ import io.vertx.core.Future; import io.vertx.ext.web.RoutingContext; import org.prebid.server.auction.DebugResolver; +import org.prebid.server.auction.GeoLocationServiceWrapper; import org.prebid.server.auction.ImplicitParametersExtractor; import org.prebid.server.auction.InterstitialProcessor; import org.prebid.server.auction.OrtbTypesResolver; @@ -48,6 +49,7 @@ public class AuctionRequestFactory { private final DebugResolver debugResolver; private final JacksonMapper mapper; private final OrtbTypesResolver ortbTypesResolver; + private final GeoLocationServiceWrapper geoLocationServiceWrapper; private static final String ENDPOINT = Endpoint.openrtb2_auction.value(); @@ -63,7 +65,8 @@ public AuctionRequestFactory(long maxRequestSize, OrtbTypesResolver ortbTypesResolver, AuctionPrivacyContextFactory auctionPrivacyContextFactory, DebugResolver debugResolver, - JacksonMapper mapper) { + JacksonMapper mapper, + GeoLocationServiceWrapper geoLocationServiceWrapper) { this.maxRequestSize = maxRequestSize; this.ortb2RequestFactory = Objects.requireNonNull(ortb2RequestFactory); @@ -78,6 +81,7 @@ public AuctionRequestFactory(long maxRequestSize, this.auctionPrivacyContextFactory = Objects.requireNonNull(auctionPrivacyContextFactory); this.debugResolver = Objects.requireNonNull(debugResolver); this.mapper = Objects.requireNonNull(mapper); + this.geoLocationServiceWrapper = Objects.requireNonNull(geoLocationServiceWrapper); } /** @@ -106,6 +110,12 @@ public Future fromRequest(RoutingContext routingContext, long st .map(auctionContext -> auctionContext.with(debugResolver.debugContextFrom(auctionContext))) + .compose(auctionContext -> geoLocationServiceWrapper.lookup(auctionContext) + .map(auctionContext::with)) + + .compose(auctionContext -> ortb2RequestFactory.enrichBidRequestWithGeolocationData(auctionContext) + .map(auctionContext::with)) + .compose(auctionContext -> gppService.contextFrom(auctionContext) .map(auctionContext::with)) @@ -121,8 +131,8 @@ public Future fromRequest(RoutingContext routingContext, long st .compose(auctionContext -> auctionPrivacyContextFactory.contextFrom(auctionContext) .map(auctionContext::with)) - .map(auctionContext -> auctionContext.with( - ortb2RequestFactory.enrichBidRequestWithAccountAndPrivacyData(auctionContext))) + .compose(auctionContext -> ortb2RequestFactory.enrichBidRequestWithAccountAndPrivacyData(auctionContext) + .map(auctionContext::with)) .compose(auctionContext -> ortb2RequestFactory.executeProcessedAuctionRequestHooks(auctionContext) .map(auctionContext::with)) diff --git a/src/main/java/org/prebid/server/auction/requestfactory/Ortb2RequestFactory.java b/src/main/java/org/prebid/server/auction/requestfactory/Ortb2RequestFactory.java index 67bd1dd38b7..c375d986482 100644 --- a/src/main/java/org/prebid/server/auction/requestfactory/Ortb2RequestFactory.java +++ b/src/main/java/org/prebid/server/auction/requestfactory/Ortb2RequestFactory.java @@ -221,7 +221,37 @@ public Future validateRequest(BidRequest bidRequest, : Future.succeededFuture(bidRequest); } - public BidRequest enrichBidRequestWithAccountAndPrivacyData(AuctionContext auctionContext) { + public Future enrichBidRequestWithGeolocationData(AuctionContext auctionContext) { + final BidRequest bidRequest = auctionContext.getBidRequest(); + final Device device = bidRequest.getDevice(); + final GeoInfo geoInfo = auctionContext.getGeoInfo(); + final Geo geo = ObjectUtil.getIfNotNull(device, Device::getGeo); + + final UpdateResult resolvedCountry = resolveCountry(geo, geoInfo); + final UpdateResult resolvedRegion = resolveRegion(geo, geoInfo); + + if (!resolvedCountry.isUpdated() && !resolvedRegion.isUpdated()) { + return Future.succeededFuture(bidRequest); + } + + final Geo updatedGeo = Optional.ofNullable(geo) + .map(Geo::toBuilder) + .orElseGet(Geo::builder) + .country(resolvedCountry.getValue()) + .region(resolvedRegion.getValue()) + .build(); + + final Device updatedDevice = Optional.ofNullable(device) + .map(Device::toBuilder) + .orElseGet(Device::builder) + .geo(updatedGeo) + .build(); + + return Future.succeededFuture(bidRequest.toBuilder().device(updatedDevice).build()); + + } + + public Future enrichBidRequestWithAccountAndPrivacyData(AuctionContext auctionContext) { final BidRequest bidRequest = auctionContext.getBidRequest(); final Account account = auctionContext.getAccount(); final PrivacyContext privacyContext = auctionContext.getPrivacyContext(); @@ -235,15 +265,16 @@ public BidRequest enrichBidRequestWithAccountAndPrivacyData(AuctionContext aucti final Regs regs = bidRequest.getRegs(); final Regs enrichedRegs = enrichRegs(regs, privacyContext, account); - if (enrichedRequestExt != null || enrichedDevice != null || enrichedRegs != null) { - return bidRequest.toBuilder() - .ext(ObjectUtils.defaultIfNull(enrichedRequestExt, requestExt)) - .device(ObjectUtils.defaultIfNull(enrichedDevice, device)) - .regs(ObjectUtils.defaultIfNull(enrichedRegs, regs)) - .build(); + if (enrichedRequestExt == null && enrichedDevice == null && enrichedRegs == null) { + return Future.succeededFuture(bidRequest); } - return bidRequest; + return Future.succeededFuture(bidRequest.toBuilder() + .ext(ObjectUtils.defaultIfNull(enrichedRequestExt, requestExt)) + .device(ObjectUtils.defaultIfNull(enrichedDevice, device)) + .regs(ObjectUtils.defaultIfNull(enrichedRegs, regs)) + .build()); + } private static Regs enrichRegs(Regs regs, PrivacyContext privacyContext, Account account) { @@ -613,9 +644,10 @@ private Device enrichDevice(Device device, PrivacyContext privacyContext) { final boolean shouldUpdateIpV6 = ipV6 != null && !Objects.equals(ipV6InRequest, ipV6); final Geo geo = ObjectUtil.getIfNotNull(device, Device::getGeo); + final GeoInfo geoInfo = privacyContext.getTcfContext().getGeoInfo(); - final UpdateResult resolvedCountry = resolveCountry(geo, privacyContext); - final UpdateResult resolvedRegion = resolveRegion(geo, privacyContext); + final UpdateResult resolvedCountry = resolveCountry(geo, geoInfo); + final UpdateResult resolvedRegion = resolveRegion(geo, geoInfo); if (shouldUpdateIpV4 || shouldUpdateIpV6 || resolvedCountry.isUpdated() || resolvedRegion.isUpdated()) { final Device.DeviceBuilder deviceBuilder = device != null ? device.toBuilder() : Device.builder(); @@ -645,10 +677,9 @@ private Device enrichDevice(Device device, PrivacyContext privacyContext) { return null; } - private UpdateResult resolveCountry(Geo geo, PrivacyContext privacyContext) { - final String countryInRequest = geo != null ? geo.getCountry() : null; + private UpdateResult resolveCountry(Geo originalGeo, GeoInfo geoInfo) { + final String countryInRequest = originalGeo != null ? originalGeo.getCountry() : null; - final GeoInfo geoInfo = privacyContext.getTcfContext().getGeoInfo(); final String alpha2CountryCode = geoInfo != null ? geoInfo.getCountry() : null; final String alpha3CountryCode = countryCodeMapper.mapToAlpha3(alpha2CountryCode); @@ -657,11 +688,10 @@ private UpdateResult resolveCountry(Geo geo, PrivacyContext privacyConte : UpdateResult.unaltered(countryInRequest); } - private static UpdateResult resolveRegion(Geo geo, PrivacyContext privacyContext) { - final String regionInRequest = geo != null ? geo.getRegion() : null; + private static UpdateResult resolveRegion(Geo originalGeo, GeoInfo geoInfo) { + final String regionInRequest = originalGeo != null ? originalGeo.getRegion() : null; final String upperCasedRegionInRequest = StringUtils.upperCase(regionInRequest); - final GeoInfo geoInfo = privacyContext.getTcfContext().getGeoInfo(); final String region = geoInfo != null ? geoInfo.getRegion() : null; final String upperCasedRegion = StringUtils.upperCase(region); diff --git a/src/main/java/org/prebid/server/auction/requestfactory/VideoRequestFactory.java b/src/main/java/org/prebid/server/auction/requestfactory/VideoRequestFactory.java index 07c256b216a..32ff1c32585 100644 --- a/src/main/java/org/prebid/server/auction/requestfactory/VideoRequestFactory.java +++ b/src/main/java/org/prebid/server/auction/requestfactory/VideoRequestFactory.java @@ -16,6 +16,7 @@ import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.prebid.server.auction.DebugResolver; +import org.prebid.server.auction.GeoLocationServiceWrapper; import org.prebid.server.auction.VideoStoredRequestProcessor; import org.prebid.server.auction.model.AuctionContext; import org.prebid.server.auction.model.CachedDebugLog; @@ -60,6 +61,7 @@ public class VideoRequestFactory { private final AuctionPrivacyContextFactory auctionPrivacyContextFactory; private final DebugResolver debugResolver; private final JacksonMapper mapper; + private final GeoLocationServiceWrapper geoLocationServiceWrapper; public VideoRequestFactory(int maxRequestSize, boolean enforceStoredRequest, @@ -70,7 +72,8 @@ public VideoRequestFactory(int maxRequestSize, Ortb2ImplicitParametersResolver paramsResolver, AuctionPrivacyContextFactory auctionPrivacyContextFactory, DebugResolver debugResolver, - JacksonMapper mapper) { + JacksonMapper mapper, + GeoLocationServiceWrapper geoLocationServiceWrapper) { this.enforceStoredRequest = enforceStoredRequest; this.maxRequestSize = maxRequestSize; @@ -81,6 +84,7 @@ public VideoRequestFactory(int maxRequestSize, this.auctionPrivacyContextFactory = Objects.requireNonNull(auctionPrivacyContextFactory); this.debugResolver = Objects.requireNonNull(debugResolver); this.mapper = Objects.requireNonNull(mapper); + this.geoLocationServiceWrapper = Objects.requireNonNull(geoLocationServiceWrapper); this.escapeLogCacheRegexPattern = StringUtils.isNotBlank(escapeLogCacheRegex) ? Pattern.compile(escapeLogCacheRegex) @@ -123,14 +127,20 @@ public Future> fromRequest(RoutingContext routingC .map(auctionContext -> auctionContext.with(debugResolver.debugContextFrom(auctionContext))) + .compose(auctionContext -> geoLocationServiceWrapper.lookup(auctionContext) + .map(auctionContext::with)) + + .compose(auctionContext -> ortb2RequestFactory.enrichBidRequestWithGeolocationData(auctionContext) + .map(auctionContext::with)) + .compose(auctionContext -> ortb2RequestFactory.activityInfrastructureFrom(auctionContext) .map(auctionContext::with)) .compose(auctionContext -> auctionPrivacyContextFactory.contextFrom(auctionContext) .map(auctionContext::with)) - .map(auctionContext -> auctionContext.with( - ortb2RequestFactory.enrichBidRequestWithAccountAndPrivacyData(auctionContext))) + .compose(auctionContext -> ortb2RequestFactory.enrichBidRequestWithAccountAndPrivacyData(auctionContext) + .map(auctionContext::with)) .compose(auctionContext -> ortb2RequestFactory.executeProcessedAuctionRequestHooks(auctionContext) .map(auctionContext::with)) diff --git a/src/main/java/org/prebid/server/privacy/gdpr/TcfDefinerService.java b/src/main/java/org/prebid/server/privacy/gdpr/TcfDefinerService.java index 0e3aba69dfa..ee9b13eb4cc 100644 --- a/src/main/java/org/prebid/server/privacy/gdpr/TcfDefinerService.java +++ b/src/main/java/org/prebid/server/privacy/gdpr/TcfDefinerService.java @@ -8,11 +8,11 @@ import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; +import org.prebid.server.auction.GeoLocationServiceWrapper; import org.prebid.server.auction.IpAddressHelper; import org.prebid.server.auction.model.IpAddress; import org.prebid.server.bidder.BidderCatalog; import org.prebid.server.execution.Timeout; -import org.prebid.server.geolocation.GeoLocationService; import org.prebid.server.geolocation.model.GeoInfo; import org.prebid.server.log.ConditionalLogger; import org.prebid.server.metric.MetricName; @@ -27,6 +27,7 @@ import org.prebid.server.settings.model.AccountGdprConfig; import org.prebid.server.settings.model.EnabledForRequestType; import org.prebid.server.settings.model.GdprConfig; +import org.prebid.server.util.ObjectUtil; import java.util.ArrayList; import java.util.Collection; @@ -59,7 +60,7 @@ public class TcfDefinerService { private final boolean consentStringMeansInScope; private final Tcf2Service tcf2Service; private final Set eeaCountries; - private final GeoLocationService geoLocationService; + private final GeoLocationServiceWrapper geoLocationServiceWrapper; private final BidderCatalog bidderCatalog; private final IpAddressHelper ipAddressHelper; private final Metrics metrics; @@ -67,7 +68,7 @@ public class TcfDefinerService { public TcfDefinerService(GdprConfig gdprConfig, Set eeaCountries, Tcf2Service tcf2Service, - GeoLocationService geoLocationService, + GeoLocationServiceWrapper geoLocationServiceWrapper, BidderCatalog bidderCatalog, IpAddressHelper ipAddressHelper, Metrics metrics) { @@ -78,7 +79,7 @@ public TcfDefinerService(GdprConfig gdprConfig, && BooleanUtils.isTrue(gdprConfig.getConsentStringMeansInScope()); this.tcf2Service = Objects.requireNonNull(tcf2Service); this.eeaCountries = Objects.requireNonNull(eeaCountries); - this.geoLocationService = geoLocationService; + this.geoLocationServiceWrapper = Objects.requireNonNull(geoLocationServiceWrapper); this.bidderCatalog = Objects.requireNonNull(bidderCatalog); this.ipAddressHelper = Objects.requireNonNull(ipAddressHelper); this.metrics = Objects.requireNonNull(metrics); @@ -93,11 +94,12 @@ public Future resolveTcfContext(Privacy privacy, AccountGdprConfig accountGdprConfig, MetricName requestType, RequestLogInfo requestLogInfo, - Timeout timeout) { + Timeout timeout, + GeoInfo geoInfo) { final Future tcfContextFuture = !isGdprEnabled(accountGdprConfig, requestType) ? Future.succeededFuture(TcfContext.empty()) - : prepareTcfContext(privacy, country, ipAddress, requestLogInfo, timeout); + : prepareTcfContext(privacy, country, ipAddress, requestLogInfo, timeout, geoInfo); return tcfContextFuture.map(this::updateTcfGeoMetrics); } @@ -112,7 +114,15 @@ public Future resolveTcfContext(Privacy privacy, RequestLogInfo requestLogInfo, Timeout timeout) { - return resolveTcfContext(privacy, null, ipAddress, accountGdprConfig, requestType, requestLogInfo, timeout); + return resolveTcfContext( + privacy, + null, + ipAddress, + accountGdprConfig, + requestType, + requestLogInfo, + timeout, + null); } public Future> resultForVendorIds(Set vendorIds, TcfContext tcfContext) { @@ -176,7 +186,8 @@ private Future prepareTcfContext(Privacy privacy, String country, String ipAddress, RequestLogInfo requestLogInfo, - Timeout timeout) { + Timeout timeout, + GeoInfo geoInfo) { final String consentString = privacy.getConsentString(); final TCStringParsingResult consentStringParsingResult = parseConsentString(consentString, requestLogInfo); @@ -205,17 +216,9 @@ private Future prepareTcfContext(Privacy privacy, return Future.succeededFuture(defaultContext.toBuilder().inGdprScope(inScopeOfGdpr(gdpr)).build()); } - if (country != null) { - return Future.succeededFuture(defaultContext.toBuilder().inGdprScope(inScopeOfGdpr(inEea)).build()); - } - - if (ipAddress != null && geoLocationService != null) { - return geoLocationService.lookup(effectiveIpAddress, timeout) - .map(geoInfo -> updateMetricsAndEnrichWithGeo(geoInfo, defaultContext)) - .recover(error -> logError(error, defaultContext)); - } - - return Future.succeededFuture(defaultContext); + return geoLocationServiceWrapper.doLookup(effectiveIpAddress, country, timeout) + .recover(ignored -> Future.succeededFuture(geoInfo)) + .map(lookupResult -> enrichWithGeoInfo(defaultContext, lookupResult, country)); } private String maybeMaskIp(String ipAddress, TCString consent) { @@ -237,28 +240,18 @@ private static boolean shouldMaskIp(TCString consent) { return isConsentValid(consent) && consent.getVersion() == 2 && !consent.getSpecialFeatureOptIns().contains(1); } - private TcfContext updateMetricsAndEnrichWithGeo(GeoInfo geoInfo, TcfContext tcfContext) { - metrics.updateGeoLocationMetric(true); - final Boolean inEea = isCountryInEea(geoInfo.getCountry()); + private TcfContext enrichWithGeoInfo(TcfContext defaultTcfContext, GeoInfo geoInfo, String defaultCountry) { + final String country = ObjectUtil.getIfNotNullOrDefault(geoInfo, GeoInfo::getCountry, () -> defaultCountry); + final Boolean inEea = isCountryInEea(country); final boolean inScope = inScopeOfGdpr(inEea); - return tcfContext.toBuilder() - .geoInfo(geoInfo) - .inGdprScope(inScope) + return defaultTcfContext.toBuilder() .inEea(inEea) + .inGdprScope(inScope) + .geoInfo(geoInfo) .build(); } - private Future logError(Throwable error, TcfContext tcfContext) { - final String message = "Geolocation lookup failed: " + error.getMessage(); - logger.warn(message); - logger.debug(message, error); - - metrics.updateGeoLocationMetric(false); - - return Future.succeededFuture(tcfContext); - } - private Boolean isCountryInEea(String country) { return country != null ? eeaCountries.contains(country) : null; } diff --git a/src/main/java/org/prebid/server/settings/model/Account.java b/src/main/java/org/prebid/server/settings/model/Account.java index b26969b87e0..8fbfa1c0752 100644 --- a/src/main/java/org/prebid/server/settings/model/Account.java +++ b/src/main/java/org/prebid/server/settings/model/Account.java @@ -26,6 +26,8 @@ public class Account { AccountHooksConfiguration hooks; + AccountSettings settings; + public static Account empty(String id) { return Account.builder().id(id).build(); } diff --git a/src/main/java/org/prebid/server/settings/model/AccountSettings.java b/src/main/java/org/prebid/server/settings/model/AccountSettings.java new file mode 100644 index 00000000000..d2a5bc611d8 --- /dev/null +++ b/src/main/java/org/prebid/server/settings/model/AccountSettings.java @@ -0,0 +1,12 @@ +package org.prebid.server.settings.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Value; + +@Value(staticConstructor = "of") +public class AccountSettings { + + @JsonProperty("geo-lookup") + Boolean geoLookup; + +} diff --git a/src/main/java/org/prebid/server/spring/config/GeoLocationConfiguration.java b/src/main/java/org/prebid/server/spring/config/GeoLocationConfiguration.java index 9d3c4e62a4b..3cff8c18d9c 100644 --- a/src/main/java/org/prebid/server/spring/config/GeoLocationConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/GeoLocationConfiguration.java @@ -4,6 +4,7 @@ import io.vertx.core.http.HttpClientOptions; import lombok.Data; import org.apache.commons.lang3.StringUtils; +import org.prebid.server.auction.GeoLocationServiceWrapper; import org.prebid.server.execution.RemoteFileSyncer; import org.prebid.server.execution.retry.FixedIntervalRetryPolicy; import org.prebid.server.geolocation.CircuitBreakerSecuredGeoLocationService; @@ -15,6 +16,7 @@ import org.prebid.server.spring.config.model.CircuitBreakerProperties; import org.prebid.server.spring.config.model.HttpClientProperties; import org.prebid.server.spring.config.model.RemoteFileSyncerProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; @@ -34,6 +36,7 @@ import java.util.ArrayList; import java.util.List; +@Configuration public class GeoLocationConfiguration { @Configuration @@ -180,22 +183,27 @@ static class GeoInfo { } } - @Configuration - static class CountryCodeMapperConfiguration { + @Bean + public CountryCodeMapper countryCodeMapper(@Value("classpath:country-codes.csv") Resource countryCodes, + @Value("classpath:mcc-country-codes.csv") Resource mccCountryCodes) + throws IOException { - @Bean - public CountryCodeMapper countryCodeMapper(@Value("classpath:country-codes.csv") Resource countryCodes, - @Value("classpath:mcc-country-codes.csv") Resource mccCountryCodes) - throws IOException { + return new CountryCodeMapper(readCsv(countryCodes), readCsv(mccCountryCodes)); + } - return new CountryCodeMapper(readCsv(countryCodes), readCsv(mccCountryCodes)); - } + private String readCsv(Resource resource) throws IOException { + final Reader reader = new InputStreamReader(resource.getInputStream(), StandardCharsets.UTF_8); + final String csv = FileCopyUtils.copyToString(reader); + reader.close(); + return csv; + } - private String readCsv(Resource resource) throws IOException { - final Reader reader = new InputStreamReader(resource.getInputStream(), StandardCharsets.UTF_8); - final String csv = FileCopyUtils.copyToString(reader); - reader.close(); - return csv; - } + @Bean + GeoLocationServiceWrapper geoLocationServiceWrapper( + @Autowired(required = false) GeoLocationService geoLocationService, + Metrics metrics) { + + return new GeoLocationServiceWrapper(geoLocationService, metrics); } + } diff --git a/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java b/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java index 1a6a78a5c42..61df84ee7bf 100644 --- a/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/PrivacyServiceConfiguration.java @@ -3,6 +3,7 @@ import io.vertx.core.Vertx; import io.vertx.core.file.FileSystem; import lombok.Data; +import org.prebid.server.auction.GeoLocationServiceWrapper; import org.prebid.server.auction.IpAddressHelper; import org.prebid.server.auction.privacy.enforcement.ActivityEnforcement; import org.prebid.server.auction.privacy.enforcement.CcpaEnforcement; @@ -13,7 +14,6 @@ import org.prebid.server.auction.privacy.enforcement.mask.UserFpdCoppaMask; import org.prebid.server.auction.privacy.enforcement.mask.UserFpdTcfMask; import org.prebid.server.bidder.BidderCatalog; -import org.prebid.server.geolocation.GeoLocationService; import org.prebid.server.json.JacksonMapper; import org.prebid.server.metric.Metrics; import org.prebid.server.privacy.HostVendorTcfDefinerService; @@ -46,7 +46,6 @@ import org.prebid.server.settings.model.SpecialFeatures; import org.prebid.server.spring.config.retry.RetryPolicyConfigurationProperties; import org.prebid.server.vertx.http.HttpClient; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Bean; @@ -162,7 +161,7 @@ TcfDefinerService tcfDefinerService( GdprConfig gdprConfig, @Value("${gdpr.eea-countries}") String eeaCountriesAsString, Tcf2Service tcf2Service, - @Autowired(required = false) GeoLocationService geoLocationService, + GeoLocationServiceWrapper geoLocationServiceWrapper, BidderCatalog bidderCatalog, IpAddressHelper ipAddressHelper, Metrics metrics) { @@ -173,7 +172,7 @@ TcfDefinerService tcfDefinerService( gdprConfig, eeaCountries, tcf2Service, - geoLocationService, + geoLocationServiceWrapper, bidderCatalog, ipAddressHelper, metrics); diff --git a/src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java b/src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java index 425396a667c..29672fd94e5 100644 --- a/src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java @@ -18,6 +18,7 @@ import org.prebid.server.auction.DsaEnforcer; import org.prebid.server.auction.ExchangeService; import org.prebid.server.auction.FpdResolver; +import org.prebid.server.auction.GeoLocationServiceWrapper; import org.prebid.server.auction.ImplicitParametersExtractor; import org.prebid.server.auction.InterstitialProcessor; import org.prebid.server.auction.IpAddressHelper; @@ -412,7 +413,8 @@ AuctionRequestFactory auctionRequestFactory( OrtbTypesResolver ortbTypesResolver, AuctionPrivacyContextFactory auctionPrivacyContextFactory, DebugResolver debugResolver, - JacksonMapper mapper) { + JacksonMapper mapper, + GeoLocationServiceWrapper geoLocationServiceWrapper) { return new AuctionRequestFactory( maxRequestSize, @@ -427,7 +429,8 @@ AuctionRequestFactory auctionRequestFactory( ortbTypesResolver, auctionPrivacyContextFactory, debugResolver, - mapper); + mapper, + geoLocationServiceWrapper); } @Bean @@ -458,7 +461,8 @@ AmpRequestFactory ampRequestFactory(Ortb2RequestFactory ortb2RequestFactory, FpdResolver fpdResolver, AmpPrivacyContextFactory ampPrivacyContextFactory, DebugResolver debugResolver, - JacksonMapper mapper) { + JacksonMapper mapper, + GeoLocationServiceWrapper geoLocationServiceWrapper) { return new AmpRequestFactory( ortb2RequestFactory, @@ -471,7 +475,8 @@ AmpRequestFactory ampRequestFactory(Ortb2RequestFactory ortb2RequestFactory, fpdResolver, ampPrivacyContextFactory, debugResolver, - mapper); + mapper, + geoLocationServiceWrapper); } @Bean @@ -485,7 +490,8 @@ VideoRequestFactory videoRequestFactory( Ortb2ImplicitParametersResolver ortb2ImplicitParametersResolver, AuctionPrivacyContextFactory auctionPrivacyContextFactory, DebugResolver debugResolver, - JacksonMapper mapper) { + JacksonMapper mapper, + GeoLocationServiceWrapper geoLocationServiceWrapper) { return new VideoRequestFactory( maxRequestSize, @@ -497,7 +503,8 @@ VideoRequestFactory videoRequestFactory( ortb2ImplicitParametersResolver, auctionPrivacyContextFactory, debugResolver, - mapper); + mapper, + geoLocationServiceWrapper); } @Bean diff --git a/src/test/groovy/org/prebid/server/functional/model/config/AccountConfig.groovy b/src/test/groovy/org/prebid/server/functional/model/config/AccountConfig.groovy index 08bb7b4d1cf..c7a3c51d6ea 100644 --- a/src/test/groovy/org/prebid/server/functional/model/config/AccountConfig.groovy +++ b/src/test/groovy/org/prebid/server/functional/model/config/AccountConfig.groovy @@ -19,6 +19,7 @@ class AccountConfig { AccountMetricsConfig metrics AccountCookieSyncConfig cookieSync AccountHooksConfiguration hooks + AccountSetting settings static getDefaultAccountConfig() { new AccountConfig(status: AccountStatus.ACTIVE) diff --git a/src/test/groovy/org/prebid/server/functional/model/config/AccountSetting.groovy b/src/test/groovy/org/prebid/server/functional/model/config/AccountSetting.groovy new file mode 100644 index 00000000000..0a056057229 --- /dev/null +++ b/src/test/groovy/org/prebid/server/functional/model/config/AccountSetting.groovy @@ -0,0 +1,12 @@ +package org.prebid.server.functional.model.config + +import com.fasterxml.jackson.databind.PropertyNamingStrategies +import com.fasterxml.jackson.databind.annotation.JsonNaming +import groovy.transform.ToString + +@ToString(includeNames = true, ignoreNulls = true) +@JsonNaming(PropertyNamingStrategies.KebabCaseStrategy) +class AccountSetting { + + Boolean geoLookup +} diff --git a/src/test/groovy/org/prebid/server/functional/model/pricefloors/Country.groovy b/src/test/groovy/org/prebid/server/functional/model/pricefloors/Country.groovy index b1d4368cf75..c9c55c129d7 100644 --- a/src/test/groovy/org/prebid/server/functional/model/pricefloors/Country.groovy +++ b/src/test/groovy/org/prebid/server/functional/model/pricefloors/Country.groovy @@ -5,23 +5,26 @@ import org.prebid.server.functional.util.privacy.model.State enum Country { - USA("USA"), - CAN("CAN"), - MULTIPLE("*") + USA("USA","US"), + CAN("CAN","CA"), + MULTIPLE("*","*") @JsonValue - final String value + final String ISOAlpha3 - Country(String value) { - this.value = value + final String ISOAlpha2 + + Country(String ISOAlpha3,String ISOAlpha2) { + this.ISOAlpha3 = ISOAlpha3 + this.ISOAlpha2 = ISOAlpha2 } @Override String toString() { - value + ISOAlpha3 } String withState(State state) { - return "${value}.${state.abbreviation}".toString() + return "${ISOAlpha3}.${state.abbreviation}".toString() } } diff --git a/src/test/groovy/org/prebid/server/functional/model/request/auction/PublicCountryIp.groovy b/src/test/groovy/org/prebid/server/functional/model/request/auction/PublicCountryIp.groovy new file mode 100644 index 00000000000..59fb0b34c25 --- /dev/null +++ b/src/test/groovy/org/prebid/server/functional/model/request/auction/PublicCountryIp.groovy @@ -0,0 +1,16 @@ +package org.prebid.server.functional.model.request.auction + +enum PublicCountryIp { + + USA_IP("209.232.44.21", "d646:2414:17b2:f371:9b62:f176:b4c0:51cd"), + UKR_IP("193.238.111.14", "3080:f30f:e4bc:0f56:41be:6aab:9d0a:58e2"), + CAN_IP("70.71.245.39", "f9b2:c742:1922:7d4b:7122:c7fc:8b75:98c8") + + final String v4 + final String v6 + + PublicCountryIp(String v4, String ipV6) { + this.v4 = v4 + this.v6 = ipV6 + } +} diff --git a/src/test/groovy/org/prebid/server/functional/tests/GeoSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/GeoSpec.groovy new file mode 100644 index 00000000000..3a38deb3ff5 --- /dev/null +++ b/src/test/groovy/org/prebid/server/functional/tests/GeoSpec.groovy @@ -0,0 +1,222 @@ +package org.prebid.server.functional.tests + +import org.prebid.server.functional.model.config.AccountAuctionConfig +import org.prebid.server.functional.model.config.AccountConfig +import org.prebid.server.functional.model.db.Account +import org.prebid.server.functional.model.pricefloors.Country +import org.prebid.server.functional.model.request.auction.BidRequest +import org.prebid.server.functional.model.config.AccountSetting +import org.prebid.server.functional.model.request.auction.Device +import org.prebid.server.functional.model.request.auction.Geo +import org.prebid.server.functional.util.PBSUtils +import java.time.Instant + +import static org.prebid.server.functional.model.AccountStatus.ACTIVE +import static org.prebid.server.functional.model.pricefloors.Country.USA +import static org.prebid.server.functional.model.request.auction.PublicCountryIp.USA_IP +import static org.prebid.server.functional.model.request.auction.TraceLevel.VERBOSE +import static org.prebid.server.functional.util.privacy.model.State.ALABAMA +import static org.prebid.server.functional.util.privacy.model.State.ONTARIO + +class GeoSpec extends BaseSpec { + + private static final String GEO_LOCATION_REQUESTS = "geolocation_requests" + private static final String GEO_LOCATION_FAIL = "geolocation_fail" + private static final String GEO_LOCATION_SUCCESSFUL = "geolocation_successful" + private static final Map GEO_LOCATION = ["geolocation.type" : "configuration", + "geolocation.configurations.[0].address-pattern" : USA_IP.v4, + "geolocation.configurations.[0].geo-info.country": USA.ISOAlpha2, + "geolocation.configurations.[0].geo-info.region" : ALABAMA.abbreviation] + + def "PBS should populate geo with country and region when geo location enabled in host and account config"() { + given: "PBS service with geolocation and default account configs" + def config = AccountConfig.defaultAccountConfig.tap { + settings = new AccountSetting(geoLookup: defaultAccountGeoLookup) + } + def defaultPbsService = pbsServiceFactory.getService( + ["settings.default-account-config": encode(config), + "geolocation.enabled" : "true"] + GEO_LOCATION) + + and: "Default bid request with device and geo data" + def bidRequest = BidRequest.defaultBidRequest.tap { + device = new Device( + ip: USA_IP.v4, + ipv6: USA_IP.v6, + geo: new Geo( + country: null, + region: null, + lat: PBSUtils.getRandomDecimal(0, 90), + lon: PBSUtils.getRandomDecimal(0, 90))) + ext.prebid.trace = VERBOSE + } + + and: "Account in the DB" + def accountConfig = new AccountConfig( + auction: new AccountAuctionConfig(debugAllow: true), + settings: new AccountSetting(geoLookup: accountGeoLookup)) + def account = new Account(status: ACTIVE, uuid: bidRequest.site.publisher.id, config: accountConfig) + accountDao.save(account) + + when: "PBS processes auction request" + defaultPbsService.sendAuctionRequest(bidRequest) + + then: "Bidder request should contain country and region" + def bidderRequests = bidder.getBidderRequest(bidRequest.id) + assert bidderRequests.device.geo.country == USA + assert bidderRequests.device.geo.region == ALABAMA.abbreviation + + and: "Metrics processed across activities should be updated" + def metrics = defaultPbsService.sendCollectedMetricsRequest() + assert metrics[GEO_LOCATION_REQUESTS] == 1 + assert metrics[GEO_LOCATION_SUCCESSFUL] == 1 + assert !metrics[GEO_LOCATION_FAIL] + + where: + defaultAccountGeoLookup | accountGeoLookup + false | true + true | true + true | null + } + + def "PBS shouldn't populate geo with country and region when geo location disable in host and account config enabled"() { + given: "PBS service with geolocation and default account configs" + def config = AccountConfig.defaultAccountConfig.tap { + settings = new AccountSetting(geoLookup: defaultAccountGeoLookupConfig) + } + def defaultPbsService = pbsServiceFactory.getService(GEO_LOCATION + + ["settings.default-account-config": encode(config), + "geolocation.enabled" : hostGeolocation]) + + and: "Default bid request with device and geo data" + def bidRequest = BidRequest.defaultBidRequest.tap { + device = new Device( + ip: USA_IP.v4, + ipv6: USA_IP.v6, + geo: new Geo( + country: null, + region: null, + lat: PBSUtils.getRandomDecimal(0, 90), + lon: PBSUtils.getRandomDecimal(0, 90))) + ext.prebid.trace = VERBOSE + } + + and: "Account in the DB" + def accountConfig = new AccountConfig( + auction: new AccountAuctionConfig(debugAllow: true), + settings: new AccountSetting(geoLookup: accountGeoLookup)) + def account = new Account(status: ACTIVE, uuid: bidRequest.site.publisher.id, config: accountConfig) + accountDao.save(account) + + when: "PBS processes auction request" + defaultPbsService.sendAuctionRequest(bidRequest) + + then: "Bidder request shouldn't contain country and region" + def bidderRequests = bidder.getBidderRequest(bidRequest.id) + assert !bidderRequests.device.geo.country + assert !bidderRequests.device.geo.region + + and: "Metrics processed across geo location shouldn't be updated" + def metrics = defaultPbsService.sendCollectedMetricsRequest() + assert !metrics[GEO_LOCATION_REQUESTS] + assert !metrics[GEO_LOCATION_SUCCESSFUL] + assert !metrics[GEO_LOCATION_FAIL] + + where: + defaultAccountGeoLookupConfig | hostGeolocation | accountGeoLookup + true | "true" | false + true | "false" | true + false | "false" | false + false | "true" | false + } + + def "PBS shouldn't populate geo with country, region and emit error in log and metric when geo look up failed"() { + given: "Test start time" + def startTime = Instant.now() + + and: "PBS service with geolocation" + def defaultPbsService = pbsServiceFactory.getService(GEO_LOCATION + + ["geolocation.configurations.[0].address-pattern": PBSUtils.randomNumber as String, + "geolocation.enabled" : "true"]) + + and: "Default bid request with device and geo data" + def bidRequest = BidRequest.defaultBidRequest.tap { + device = new Device( + ip: USA_IP.v4, + ipv6: USA_IP.v6, + geo: new Geo( + country: null, + region: null, + lat: PBSUtils.getRandomDecimal(0, 90), + lon: PBSUtils.getRandomDecimal(0, 90))) + ext.prebid.trace = VERBOSE + } + + and: "Account in the DB" + def accountConfig = new AccountConfig( + auction: new AccountAuctionConfig(debugAllow: true), + settings: new AccountSetting(geoLookup: true)) + def account = new Account(status: ACTIVE, uuid: bidRequest.site.publisher.id, config: accountConfig) + accountDao.save(account) + + when: "PBS processes auction request" + defaultPbsService.sendAuctionRequest(bidRequest) + + then: "Bidder request shouldn't contain country and region" + def bidderRequests = bidder.getBidderRequest(bidRequest.id) + assert !bidderRequests.device.geo.country + assert !bidderRequests.device.geo.region + + and: "Metrics processed across geo location should be updated" + def metrics = defaultPbsService.sendCollectedMetricsRequest() + assert metrics[GEO_LOCATION_REQUESTS] == 1 + assert metrics[GEO_LOCATION_FAIL] == 1 + assert !metrics[GEO_LOCATION_SUCCESSFUL] + + and: "PBs should emit geo failed logs" + def logs = defaultPbsService.getLogsByTime(startTime) + def getLocation = getLogsByText(logs, "GeoLocationServiceWrapper") + assert getLocation.size() == 1 + assert getLocation[0].contains("Geolocation lookup failed: " + + "ConfigurationGeoLocationService: Geo location lookup failed.") + } + + def "PBS shouldn't populate country and region via geo when geo enabled in account and country and region specified in request"() { + given: "PBS service with geolocation" + def defaultPbsService = pbsServiceFactory.getService( + ["geolocation.enabled": "true"] + GEO_LOCATION) + + and: "Default bid request with device and geo data" + def bidRequest = BidRequest.defaultBidRequest.tap { + device = new Device( + ip: USA_IP.v4, + ipv6: USA_IP.v6, + geo: new Geo( + country: Country.CAN, + region: ONTARIO.abbreviation, + lat: PBSUtils.getRandomDecimal(0, 90), + lon: PBSUtils.getRandomDecimal(0, 90))) + ext.prebid.trace = VERBOSE + } + + and: "Account in the DB" + def accountConfig = new AccountConfig( + auction: new AccountAuctionConfig(debugAllow: true), + settings: new AccountSetting(geoLookup: true)) + def account = new Account(status: ACTIVE, uuid: bidRequest.site.publisher.id, config: accountConfig) + accountDao.save(account) + + when: "PBS processes auction request" + defaultPbsService.sendAuctionRequest(bidRequest) + + then: "Bidder request should contain country and region" + def bidderRequests = bidder.getBidderRequest(bidRequest.id) + assert bidderRequests.device.geo.country == Country.CAN + assert bidderRequests.device.geo.region == ONTARIO.abbreviation + + and: "Metrics processed across activities shouldn't be updated" + def metrics = defaultPbsService.sendCollectedMetricsRequest() + assert !metrics[GEO_LOCATION_REQUESTS] + assert !metrics[GEO_LOCATION_SUCCESSFUL] + assert !metrics[GEO_LOCATION_FAIL] + } +} diff --git a/src/test/groovy/org/prebid/server/functional/tests/privacy/ActivityTraceLogSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/privacy/ActivityTraceLogSpec.groovy index 09d8c51b0e1..041738cc5ab 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/privacy/ActivityTraceLogSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/privacy/ActivityTraceLogSpec.groovy @@ -93,7 +93,7 @@ class ActivityTraceLogSpec extends PrivacyBaseSpec { componentName: GENERIC.value, componentType: BIDDER, region: ALABAMA.abbreviation, - country: USA.value)) + country: USA.ISOAlpha3)) assert fetchBidsActivity.ruleConfiguration.every { it == null } assert fetchBidsActivity.allowByDefault.contains(activity.defaultAction) assert fetchBidsActivity.result.contains("DISALLOW") @@ -133,7 +133,7 @@ class ActivityTraceLogSpec extends PrivacyBaseSpec { componentName: GENERIC.value, componentType: BIDDER, region: ALABAMA.abbreviation, - country: USA.value)) + country: USA.ISOAlpha3)) assert fetchBidsActivity.allowByDefault.contains(activity.defaultAction) assert fetchBidsActivity.ruleConfiguration.every { it == null } assert fetchBidsActivity.result.contains("ALLOW") @@ -148,7 +148,7 @@ class ActivityTraceLogSpec extends PrivacyBaseSpec { componentName: GENERIC.value, componentType: BIDDER, region: ALABAMA.abbreviation, - country: USA.value)) + country: USA.ISOAlpha3)) assert transmitUfpdActivity.allowByDefault.contains(activity.defaultAction) assert transmitUfpdActivity.ruleConfiguration.every { it == null } assert transmitUfpdActivity.result.every { it == null } @@ -163,7 +163,7 @@ class ActivityTraceLogSpec extends PrivacyBaseSpec { componentName: GENERIC.value, componentType: BIDDER, region: ALABAMA.abbreviation, - country: USA.value)) + country: USA.ISOAlpha3)) assert transmitPreciseGeoActivity.allowByDefault.contains(activity.defaultAction) assert transmitPreciseGeoActivity.ruleConfiguration.every { it == null } assert transmitPreciseGeoActivity.result.every { it == null } @@ -178,7 +178,7 @@ class ActivityTraceLogSpec extends PrivacyBaseSpec { componentName: GENERIC.value, componentType: BIDDER, region: ALABAMA.abbreviation, - country: USA.value)) + country: USA.ISOAlpha3)) assert transmitTidActivity.allowByDefault.contains(activity.defaultAction) assert transmitTidActivity.ruleConfiguration.every { it == null } assert transmitTidActivity.result.every { it == null } @@ -227,7 +227,7 @@ class ActivityTraceLogSpec extends PrivacyBaseSpec { componentType: BIDDER, gpc: bidRequest.regs.ext.gpc, region: ALABAMA.abbreviation, - country: USA.value)) + country: USA.ISOAlpha3)) assert fetchBidsActivity.ruleConfiguration.contains(new RuleConfiguration( componentNames: condition.componentName, componentTypes: condition.componentType, @@ -265,7 +265,7 @@ class ActivityTraceLogSpec extends PrivacyBaseSpec { componentType: BIDDER, gpc: bidRequest.regs.ext.gpc, region: ALABAMA.abbreviation, - country: USA.value)) + country: USA.ISOAlpha3)) assert transmitPreciseGeoActivity.ruleConfiguration.every { it == null } assert transmitPreciseGeoActivity.allowByDefault.contains(activity.defaultAction) assert transmitPreciseGeoActivity.result.every { it == null } diff --git a/src/test/groovy/org/prebid/server/functional/tests/privacy/GppFetchBidActivitiesSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/privacy/GppFetchBidActivitiesSpec.groovy index 8ba421405f7..a2683a8b7ff 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/privacy/GppFetchBidActivitiesSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/privacy/GppFetchBidActivitiesSpec.groovy @@ -372,7 +372,7 @@ class GppFetchBidActivitiesSpec extends PrivacyBaseSpec { where: deviceGeo | conditionGeo - null | [USA.value] + null | [USA.ISOAlpha3] new Geo(country: USA) | null new Geo(region: ALABAMA.abbreviation) | [USA.withState(ALABAMA)] new Geo(country: CAN, region: ALABAMA.abbreviation) | [USA.withState(ALABAMA)] @@ -419,7 +419,7 @@ class GppFetchBidActivitiesSpec extends PrivacyBaseSpec { where: deviceGeo | conditionGeo - new Geo(country: USA) | [USA.value] + new Geo(country: USA) | [USA.ISOAlpha3] new Geo(country: USA, region: ALABAMA.abbreviation) | [USA.withState(ALABAMA)] new Geo(country: USA, region: ALABAMA.abbreviation) | [CAN.withState(ONTARIO), USA.withState(ALABAMA)] } diff --git a/src/test/groovy/org/prebid/server/functional/tests/privacy/GppSyncUserActivitiesSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/privacy/GppSyncUserActivitiesSpec.groovy index 4229a356033..42a096d2c55 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/privacy/GppSyncUserActivitiesSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/privacy/GppSyncUserActivitiesSpec.groovy @@ -1,12 +1,17 @@ package org.prebid.server.functional.tests.privacy import org.prebid.server.functional.model.UidsCookie +import org.prebid.server.functional.model.config.AccountCcpaConfig +import org.prebid.server.functional.model.config.AccountConfig import org.prebid.server.functional.model.config.AccountGppConfig +import org.prebid.server.functional.model.config.AccountPrivacyConfig +import org.prebid.server.functional.model.config.AccountSetting import org.prebid.server.functional.model.config.ActivityConfig import org.prebid.server.functional.model.config.EqualityValueRule import org.prebid.server.functional.model.config.InequalityValueRule import org.prebid.server.functional.model.config.LogicalRestrictedRule import org.prebid.server.functional.model.config.GppModuleConfig +import org.prebid.server.functional.model.db.Account import org.prebid.server.functional.model.request.auction.Activity import org.prebid.server.functional.model.request.auction.ActivityRule import org.prebid.server.functional.model.request.auction.AllowActivities @@ -63,6 +68,7 @@ import static org.prebid.server.functional.model.request.auction.ActivityType.SY import static org.prebid.server.functional.model.request.auction.PrivacyModule.ALL import static org.prebid.server.functional.model.request.auction.PrivacyModule.IAB_ALL import static org.prebid.server.functional.model.request.auction.PrivacyModule.IAB_US_CUSTOM_LOGIC +import static org.prebid.server.functional.model.request.auction.PublicCountryIp.USA_IP import static org.prebid.server.functional.util.privacy.model.State.MANITOBA import static org.prebid.server.functional.util.privacy.model.State.ALABAMA import static org.prebid.server.functional.util.privacy.model.State.ALASKA @@ -75,13 +81,15 @@ class GppSyncUserActivitiesSpec extends PrivacyBaseSpec { private static final String DISALLOWED_COUNT_FOR_ACTIVITY_RULE = "requests.activity.${SYNC_USER.metricValue}.disallowed.count" private static final String DISALLOWED_COUNT_FOR_GENERIC_ADAPTER = "adapter.${GENERIC.value}.activity.${SYNC_USER.metricValue}.disallowed.count" private static final String ALERT_GENERAL = "alerts.general" + private static final String GEO_LOCATION_REQUESTS = "geolocation_requests" + private static final String GEO_LOCATION_SUCCESSFUL = "geolocation_successful" private final static int INVALID_STATUS_CODE = 451 private final static String INVALID_STATUS_MESSAGE = "Unavailable For Legal Reasons." private static final Map GEO_LOCATION = ["geolocation.enabled" : "true", "geolocation.type" : "configuration", - "geolocation.configurations.[0].address-pattern": "209."] + "geolocation.configurations.[0].address-pattern": USA_IP.v4] def "PBS cookie sync call when bidder allowed in activities should include proper responded with bidders URLs and update processed metrics"() { given: "Cookie sync request with link to account" @@ -1683,12 +1691,12 @@ class GppSyncUserActivitiesSpec extends PrivacyBaseSpec { assert metrics[ACTIVITY_RULES_PROCESSED_COUNT] == 1 where: - countyConfig | regionConfig | conditionGeo - null | null | ["$USA.value".toString()] - USA.value | ALABAMA.abbreviation | null - CAN.value | ALASKA.abbreviation | [USA.withState(ALABAMA)] - null | MANITOBA.abbreviation | [USA.withState(ALABAMA)] - CAN.value | null | [USA.withState(ALABAMA)] + countyConfig | regionConfig | conditionGeo + null | null | ["$USA.ISOAlpha3".toString()] + USA.ISOAlpha3 | ALABAMA.abbreviation | null + CAN.ISOAlpha3 | ALASKA.abbreviation | [USA.withState(ALABAMA)] + null | MANITOBA.abbreviation | [USA.withState(ALABAMA)] + CAN.ISOAlpha3 | null | [USA.withState(ALABAMA)] } def "PBS setuid should process rule when geo doesn't intersection"() { @@ -1739,11 +1747,11 @@ class GppSyncUserActivitiesSpec extends PrivacyBaseSpec { assert metrics[ACTIVITY_RULES_PROCESSED_COUNT] == 1 where: - countyConfig | regionConfig | conditionGeo - null | null | [USA.value] - CAN.value | ALASKA.abbreviation | [USA.withState(ALABAMA)] - null | MANITOBA.abbreviation | [USA.withState(ALABAMA)] - CAN.value | null | [USA.withState(ALABAMA)] + countyConfig | regionConfig | conditionGeo + null | null | [USA.ISOAlpha3] + CAN.ISOAlpha3 | ALASKA.abbreviation | [USA.withState(ALABAMA)] + null | MANITOBA.abbreviation | [USA.withState(ALABAMA)] + CAN.ISOAlpha3 | null | [USA.withState(ALABAMA)] } def "PBS cookie sync should disallowed rule when device.geo intersection"() { @@ -1792,9 +1800,9 @@ class GppSyncUserActivitiesSpec extends PrivacyBaseSpec { assert metrics[DISALLOWED_COUNT_FOR_GENERIC_ADAPTER] == 1 where: - countyConfig | regionConfig | conditionGeo - USA.value | null | [USA.value] - USA.value | ALABAMA.abbreviation | [USA.withState(ALABAMA)] + countyConfig | regionConfig | conditionGeo + USA.ISOAlpha3 | null | [USA.ISOAlpha3] + USA.ISOAlpha3 | ALABAMA.abbreviation | [USA.withState(ALABAMA)] } def "PBS setuid should disallowed rule when device.geo intersection"() { @@ -1842,8 +1850,60 @@ class GppSyncUserActivitiesSpec extends PrivacyBaseSpec { assert exception.responseBody == INVALID_STATUS_MESSAGE where: - countyConfig | regionConfig | conditionGeo - USA.value | null | [USA.value] - USA.value | ALABAMA.abbreviation | [USA.withState(ALABAMA)] + countyConfig | regionConfig | conditionGeo + USA.ISOAlpha3 | null | [USA.ISOAlpha3] + USA.ISOAlpha3 | ALABAMA.abbreviation | [USA.withState(ALABAMA)] + } + + def "PBS cookie sync should fetch geo once when gpp sync user and account require geo look up"() { + given: "Pbs config with geo location" + def prebidServerService = pbsServiceFactory.getService(PBS_CONFIG + GEO_LOCATION + + ["geolocation.configurations.[0].geo-info.country": USA.ISOAlpha3, + "geolocation.configurations.[0].geo-info.region" : ALABAMA.abbreviation]) + + and: "Cookie sync request with account connection" + def accountId = PBSUtils.randomNumber as String + def cookieSyncRequest = CookieSyncRequest.defaultCookieSyncRequest.tap { + it.account = accountId + it.gppSid = null + it.gdpr = null + } + + and: "Setup condition" + def condition = Condition.baseCondition.tap { + it.componentType = null + it.componentName = null + it.gppSid = null + it.geo = [USA.withState(ALABAMA)] + } + + and: "Set activity" + def activity = Activity.getDefaultActivity([ActivityRule.getDefaultActivityRule(condition, false)]) + def activities = AllowActivities.getDefaultAllowActivities(SYNC_USER, activity) + + and: "Flush metrics" + flushMetrics(prebidServerService) + + and: "Set up account for allow activities" + def privacy = new AccountPrivacyConfig(ccpa: new AccountCcpaConfig(enabled: true), allowActivities: activities) + def accountConfig = new AccountConfig(privacy: privacy, settings: new AccountSetting(geoLookup: true)) + def account = new Account(uuid: accountId, config: accountConfig) + accountDao.save(account) + + when: "PBS processes cookie sync request with header" + def response = prebidServerService + .sendCookieSyncRequest(cookieSyncRequest, ["X-Forwarded-For": USA_IP.v4]) + + then: "Response should not contain any URLs for bidders" + assert !response.bidderStatus.userSync.url + + and: "Metrics for disallowed activities should be updated" + def metrics = prebidServerService.sendCollectedMetricsRequest() + assert metrics[DISALLOWED_COUNT_FOR_ACTIVITY_RULE] == 1 + assert metrics[DISALLOWED_COUNT_FOR_GENERIC_ADAPTER] == 1 + + and: "Metrics processed across activities should be updated" + assert metrics[GEO_LOCATION_REQUESTS] == 1 + assert metrics[GEO_LOCATION_SUCCESSFUL] == 1 } } diff --git a/src/test/groovy/org/prebid/server/functional/tests/privacy/GppTransmitEidsActivitiesSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/privacy/GppTransmitEidsActivitiesSpec.groovy index ffecec814e2..a05f0e64b39 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/privacy/GppTransmitEidsActivitiesSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/privacy/GppTransmitEidsActivitiesSpec.groovy @@ -386,7 +386,7 @@ class GppTransmitEidsActivitiesSpec extends PrivacyBaseSpec { where: deviceGeo | conditionGeo - null | [USA.value] + null | [USA.ISOAlpha3] new Geo(country: USA) | null new Geo(region: ALABAMA.abbreviation) | [USA.withState(ALABAMA)] new Geo(country: CAN, region: ALABAMA.abbreviation) | [USA.withState(ALABAMA)] @@ -438,7 +438,7 @@ class GppTransmitEidsActivitiesSpec extends PrivacyBaseSpec { where: deviceGeo | conditionGeo - new Geo(country: USA) | [USA.value] + new Geo(country: USA) | [USA.ISOAlpha3] new Geo(country: USA, region: ALABAMA.abbreviation) | [USA.withState(ALABAMA)] new Geo(country: USA, region: ALABAMA.abbreviation) | [CAN.withState(ONTARIO), USA.withState(ALABAMA)] } diff --git a/src/test/groovy/org/prebid/server/functional/tests/privacy/GppTransmitPreciseGeoActivitiesSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/privacy/GppTransmitPreciseGeoActivitiesSpec.groovy index 431b66ab6cd..3372d8d0c15 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/privacy/GppTransmitPreciseGeoActivitiesSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/privacy/GppTransmitPreciseGeoActivitiesSpec.groovy @@ -471,7 +471,7 @@ class GppTransmitPreciseGeoActivitiesSpec extends PrivacyBaseSpec { it.componentType = null it.componentName = [PBSUtils.randomString] it.gppSid = [USP_V1.intValue] - it.geo = ["$USA.value".toString()] + it.geo = ["$USA.ISOAlpha3".toString()] } and: "Set activity" @@ -561,7 +561,7 @@ class GppTransmitPreciseGeoActivitiesSpec extends PrivacyBaseSpec { where: deviceGeo | conditionGeo - new Geo(country: USA) | [USA.value] + new Geo(country: USA) | [USA.ISOAlpha3] new Geo(country: USA, region: ALABAMA.abbreviation) | [USA.withState(ALABAMA)] new Geo(country: USA, region: ALABAMA.abbreviation) | [CAN.withState(ONTARIO), USA.withState(ALABAMA)] } diff --git a/src/test/groovy/org/prebid/server/functional/tests/privacy/GppTransmitUfpdActivitiesSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/privacy/GppTransmitUfpdActivitiesSpec.groovy index 5cd227ccef1..d056412e225 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/privacy/GppTransmitUfpdActivitiesSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/privacy/GppTransmitUfpdActivitiesSpec.groovy @@ -501,7 +501,7 @@ class GppTransmitUfpdActivitiesSpec extends PrivacyBaseSpec { where: deviceGeo | conditionGeo - null | [USA.value] + null | [USA.ISOAlpha3] new Geo(country: USA) | null new Geo(region: ALABAMA.abbreviation) | [USA.withState(ALABAMA)] new Geo(country: CAN, region: ALABAMA.abbreviation) | [USA.withState(ALABAMA)] @@ -564,7 +564,7 @@ class GppTransmitUfpdActivitiesSpec extends PrivacyBaseSpec { where: deviceGeo | conditionGeo - new Geo(country: USA) | [USA.value] + new Geo(country: USA) | [USA.ISOAlpha3] new Geo(country: USA, region: ALABAMA.abbreviation) | [USA.withState(ALABAMA)] new Geo(country: USA, region: ALABAMA.abbreviation) | [CAN.withState(ONTARIO), USA.withState(ALABAMA)] } diff --git a/src/test/java/org/prebid/server/auction/GeoLocationServiceWrapperTest.java b/src/test/java/org/prebid/server/auction/GeoLocationServiceWrapperTest.java new file mode 100644 index 00000000000..2954777d7c7 --- /dev/null +++ b/src/test/java/org/prebid/server/auction/GeoLocationServiceWrapperTest.java @@ -0,0 +1,224 @@ +package org.prebid.server.auction; + +import com.iab.openrtb.request.BidRequest; +import com.iab.openrtb.request.Device; +import com.iab.openrtb.request.Geo; +import io.vertx.core.Future; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.prebid.server.VertxTest; +import org.prebid.server.auction.model.AuctionContext; +import org.prebid.server.auction.model.TimeoutContext; +import org.prebid.server.execution.Timeout; +import org.prebid.server.execution.TimeoutFactory; +import org.prebid.server.geolocation.GeoLocationService; +import org.prebid.server.geolocation.model.GeoInfo; +import org.prebid.server.metric.Metrics; +import org.prebid.server.settings.model.Account; +import org.prebid.server.settings.model.AccountSettings; + +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneId; + +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.prebid.server.assertion.FutureAssertion.assertThat; + +public class GeoLocationServiceWrapperTest extends VertxTest { + + private static final Timeout TIMEOUT = new TimeoutFactory(Clock.fixed(Instant.now(), ZoneId.systemDefault())) + .create(1000L); + + @Rule + public final MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Mock + private GeoLocationService geoLocationService; + @Mock + private Metrics metrics; + + private GeoLocationServiceWrapper target; + + @Before + public void before() { + target = new GeoLocationServiceWrapper(geoLocationService, metrics); + } + + @Test + public void doLookupShouldFailWhenGeoLocationServiceIsNotConfigured() { + // given + target = new GeoLocationServiceWrapper(null, metrics); + + // when + final Future result = target.doLookup("ip", null, TIMEOUT); + + // then + assertThat(result).isFailed().withFailMessage("Geolocation lookup is skipped"); + verifyNoInteractions(metrics); + } + + @Test + public void doLookupShouldFailWhenRequestCountryIsProvided() { + // when + final Future result = target.doLookup("ip", "country", TIMEOUT); + + // then + assertThat(result).isFailed().withFailMessage("Geolocation lookup is skipped"); + verifyNoInteractions(geoLocationService, metrics); + } + + @Test + public void doLookupShouldFailWhenRequestIpIsNotProvided() { + // when + final Future result = target.doLookup(null, null, TIMEOUT); + + // then + assertThat(result).isFailed().withFailMessage("Geolocation lookup is skipped"); + verifyNoInteractions(geoLocationService, metrics); + } + + @Test + public void doLookupShouldFailAndUpdateMetricWhenGeoLookupFails() { + // given + given(geoLocationService.lookup("ip", TIMEOUT)).willReturn(Future.failedFuture("Bad IP")); + + // when + final Future result = target.doLookup("ip", null, TIMEOUT); + + // then + assertThat(result).isFailed().withFailMessage("Bad IP"); + verify(metrics).updateGeoLocationMetric(false); + } + + @Test + public void doLookupShouldReturnGeoInfoAndUpdateMetricWhenGeoLookupSucceeds() { + // given + final GeoInfo givenGeoInfo = GeoInfo.builder().vendor("vendor").build(); + given(geoLocationService.lookup("ip", TIMEOUT)).willReturn(Future.succeededFuture(givenGeoInfo)); + + // when + final Future result = target.doLookup("ip", null, TIMEOUT); + + // then + assertThat(result).succeededWith(givenGeoInfo); + verify(metrics).updateGeoLocationMetric(true); + } + + @Test + public void lookupShouldReturnNothingWhenLookupIsEnabledInAccountAndGeoLocationServiceIsNotConfigured() { + // given + target = new GeoLocationServiceWrapper(null, metrics); + + final AuctionContext givenContext = AuctionContext.builder() + .bidRequest(BidRequest.builder().device(Device.builder().ip("ip").build()).build()) + .timeoutContext(TimeoutContext.of(100L, TIMEOUT, 2)) + .account(Account.builder().settings(AccountSettings.of(true)).build()) + .build(); + + // when + final Future result = target.lookup(givenContext); + + // then + assertThat(result).isSucceeded().unwrap().isNull(); + verifyNoInteractions(metrics); + } + + @Test + public void lookupShouldReturnNothingWhenLookupIsEnabledInAccountAndRequestCountryIsProvided() { + // given + final AuctionContext givenContext = AuctionContext.builder() + .bidRequest(BidRequest.builder() + .device(Device.builder().ip("ip").geo(Geo.builder().country("UKR").build()).build()) + .build()) + .timeoutContext(TimeoutContext.of(100L, TIMEOUT, 2)) + .account(Account.builder().settings(AccountSettings.of(true)).build()) + .build(); + + // when + final Future result = target.lookup(givenContext); + + // then + assertThat(result).isSucceeded().unwrap().isNull(); + verifyNoInteractions(geoLocationService, metrics); + } + + @Test + public void lookupShouldReturnNothingWhenLookupIsEnabledInAccountAndRequestIpIsNotProvided() { + // given + final AuctionContext givenContext = AuctionContext.builder() + .bidRequest(BidRequest.builder().device(Device.builder().ip(null).build()).build()) + .timeoutContext(TimeoutContext.of(100L, TIMEOUT, 2)) + .account(Account.builder().settings(AccountSettings.of(true)).build()) + .build(); + + // when + final Future result = target.lookup(givenContext); + + // then + assertThat(result).isSucceeded().unwrap().isNull(); + verifyNoInteractions(geoLocationService, metrics); + } + + @Test + public void lookupShouldReturnNothingWhenLookupIsEnabledInAccountAndGeoLookupFails() { + // given + given(geoLocationService.lookup("ip", TIMEOUT)).willReturn(Future.failedFuture("Bad IP")); + + final AuctionContext givenContext = AuctionContext.builder() + .bidRequest(BidRequest.builder().device(Device.builder().ip("ip").build()).build()) + .timeoutContext(TimeoutContext.of(100L, TIMEOUT, 2)) + .account(Account.builder().settings(AccountSettings.of(true)).build()) + .build(); + + // when + final Future result = target.lookup(givenContext); + + // then + assertThat(result).isSucceeded().unwrap().isNull(); + verify(metrics).updateGeoLocationMetric(false); + } + + @Test + public void lookupShouldReturnGeoInfoAndUpdateMetricWhenGeoLookupSucceeds() { + // given + final GeoInfo givenGeoInfo = GeoInfo.builder().vendor("vendor").build(); + given(geoLocationService.lookup("ip", TIMEOUT)).willReturn(Future.succeededFuture(givenGeoInfo)); + + final AuctionContext givenContext = AuctionContext.builder() + .bidRequest(BidRequest.builder().device(Device.builder().ip("ip").build()).build()) + .timeoutContext(TimeoutContext.of(100L, TIMEOUT, 2)) + .account(Account.builder().settings(AccountSettings.of(true)).build()) + .build(); + + // when + final Future result = target.lookup(givenContext); + + // then + assertThat(result).succeededWith(givenGeoInfo); + verify(metrics).updateGeoLocationMetric(true); + } + + @Test + public void lookupShouldReturnNothingWhenGeoLookupIsDisabledInAccount() { + // given + final AuctionContext givenContext = AuctionContext.builder() + .bidRequest(BidRequest.builder().device(Device.builder().ip("ip").build()).build()) + .timeoutContext(TimeoutContext.of(100L, TIMEOUT, 2)) + .account(Account.builder().settings(AccountSettings.of(false)).build()) + .build(); + + // when + final Future result = target.lookup(givenContext); + + // then + assertThat(result).isSucceeded().unwrap().isNull(); + verifyNoInteractions(geoLocationService, metrics); + } + +} diff --git a/src/test/java/org/prebid/server/auction/privacy/contextfactory/AmpPrivacyContextFactoryTest.java b/src/test/java/org/prebid/server/auction/privacy/contextfactory/AmpPrivacyContextFactoryTest.java index f1175e02748..5410fc53237 100644 --- a/src/test/java/org/prebid/server/auction/privacy/contextfactory/AmpPrivacyContextFactoryTest.java +++ b/src/test/java/org/prebid/server/auction/privacy/contextfactory/AmpPrivacyContextFactoryTest.java @@ -77,7 +77,7 @@ public void contextFromShouldExtractInitialPrivacy() { .build(); given(privacyExtractor.validPrivacyFrom(any(), any())).willReturn(emptyPrivacy); - given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any())) + given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any(), any())) .willReturn(Future.succeededFuture(TcfContext.empty())); final AuctionContext auctionContext = givenAuctionContext( @@ -101,7 +101,7 @@ public void contextFromShouldAddTcfExtractionWarningsToAuctionDebugWarningsWhenI .build(); given(privacyExtractor.validPrivacyFrom(any(), any())).willReturn(emptyPrivacy); - given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any())) + given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any(), any())) .willReturn(Future.succeededFuture(TcfContext.builder().warnings(singletonList("Error")).build())); final AuctionContext auctionContext = givenAuctionContext( @@ -125,7 +125,7 @@ public void contextFromShouldNotRemoveConsentStringOnEmptyConsentTypeParam() { .build(); given(privacyExtractor.validPrivacyFrom(any(), any())).willReturn(privacy); - given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any())) + given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any(), any())) .willReturn(Future.succeededFuture(TcfContext.empty())); final AuctionContext auctionContext = givenAuctionContext( @@ -150,7 +150,7 @@ public void contextFromShouldRemoveConsentStringAndEmitErrorOnTcf1ConsentTypePar .build(); given(privacyExtractor.validPrivacyFrom(any(), any())).willReturn(privacy); - given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any())) + given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any(), any())) .willReturn(Future.succeededFuture(TcfContext.empty())); final AuctionContext auctionContext = givenAuctionContext( @@ -178,7 +178,7 @@ public void contextFromShouldMaskIpV4WhenCoppaEqualsToOneAndIpV4Present() { given(ipAddressHelper.maskIpv4(anyString())).willReturn("maskedIpV4"); given(ipAddressHelper.anonymizeIpv6(anyString())).willReturn("maskedIpV6"); - given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any())) + given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any(), any())) .willReturn(Future.succeededFuture(TcfContext.empty())); final BidRequest bidRequest = givenBidRequest(request -> request.device(Device.builder().ip("ip").build())); @@ -207,7 +207,7 @@ public void contextFromShouldMaskIpV6WhenCoppaEqualsToOneAndIpV6Present() { given(ipAddressHelper.maskIpv4(anyString())).willReturn("maskedIpV4"); given(ipAddressHelper.anonymizeIpv6(anyString())).willReturn("maskedIpV6"); - given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any())) + given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any(), any())) .willReturn(Future.succeededFuture(TcfContext.empty())); final BidRequest bidRequest = givenBidRequest(request -> request.device(Device.builder().ipv6("ipV6").build())); @@ -233,7 +233,7 @@ public void contextFromShouldAddRefUrlWhenPresentAndRequestTypeIsWeb() { .build(); given(privacyExtractor.validPrivacyFrom(any(), any())).willReturn(privacy); - given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any())) + given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any(), any())) .willReturn(Future.succeededFuture(TcfContext.empty())); final BidRequest bidRequest = givenBidRequest(request -> request.site(Site.builder().ref("refUrl").build())); @@ -248,7 +248,7 @@ public void contextFromShouldAddRefUrlWhenPresentAndRequestTypeIsWeb() { // then final RequestLogInfo expectedRequestLogInfo = RequestLogInfo.of(MetricName.openrtb2web, "refUrl", null); verify(tcfDefinerService) - .resolveTcfContext(any(), any(), any(), any(), any(), eq(expectedRequestLogInfo), any()); + .resolveTcfContext(any(), any(), any(), any(), any(), eq(expectedRequestLogInfo), any(), any()); } private static AuctionContext givenAuctionContext( diff --git a/src/test/java/org/prebid/server/auction/privacy/contextfactory/AuctionPrivacyContextFactoryTest.java b/src/test/java/org/prebid/server/auction/privacy/contextfactory/AuctionPrivacyContextFactoryTest.java index d656ce757cf..5449de6ce84 100644 --- a/src/test/java/org/prebid/server/auction/privacy/contextfactory/AuctionPrivacyContextFactoryTest.java +++ b/src/test/java/org/prebid/server/auction/privacy/contextfactory/AuctionPrivacyContextFactoryTest.java @@ -76,7 +76,7 @@ public void contextFromShouldExtractInitialPrivacy() { .build(); given(privacyExtractor.validPrivacyFrom(any(), any())).willReturn(emptyPrivacy); - given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any())) + given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any(), any())) .willReturn(Future.succeededFuture(TcfContext.empty())); final AuctionContext auctionContext = givenAuctionContext( @@ -100,7 +100,7 @@ public void contextFromShouldAddTcfExtractionWarningsToAuctionDebugWarningsWhenI .build(); given(privacyExtractor.validPrivacyFrom(any(), any())).willReturn(emptyPrivacy); - given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any())) + given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any(), any())) .willReturn(Future.succeededFuture(TcfContext.builder().warnings(singletonList("Error")).build())); final AuctionContext auctionContext = givenAuctionContext( @@ -127,7 +127,7 @@ public void contextFromShouldMaskIpV4WhenCoppaEqualsToOneAndIpV4Present() { given(ipAddressHelper.maskIpv4(anyString())).willReturn("maskedIpV4"); given(ipAddressHelper.anonymizeIpv6(anyString())).willReturn("maskedIpV6"); - given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any())) + given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any(), any())) .willReturn(Future.succeededFuture(TcfContext.empty())); final BidRequest bidRequest = givenBidRequest(request -> request.device(Device.builder().ip("ip").build())); @@ -156,7 +156,7 @@ public void contextFromShouldMaskIpV6WhenCoppaEqualsToOneAndIpV6Present() { given(ipAddressHelper.maskIpv4(anyString())).willReturn("maskedIpV4"); given(ipAddressHelper.anonymizeIpv6(anyString())).willReturn("maskedIpV6"); - given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any())) + given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any(), any())) .willReturn(Future.succeededFuture(TcfContext.empty())); final BidRequest bidRequest = givenBidRequest(request -> request.device(Device.builder().ipv6("ipV6").build())); @@ -182,7 +182,7 @@ public void contextFromShouldAddRefUrlWhenPresentAndRequestTypeIsWeb() { .build(); given(privacyExtractor.validPrivacyFrom(any(), any())).willReturn(privacy); - given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any())) + given(tcfDefinerService.resolveTcfContext(any(), any(), any(), any(), any(), any(), any(), any())) .willReturn(Future.succeededFuture(TcfContext.empty())); final BidRequest bidRequest = givenBidRequest(request -> request.site(Site.builder().ref("refUrl").build())); @@ -197,7 +197,7 @@ public void contextFromShouldAddRefUrlWhenPresentAndRequestTypeIsWeb() { // then final RequestLogInfo expectedRequestLogInfo = RequestLogInfo.of(MetricName.openrtb2web, "refUrl", null); verify(tcfDefinerService) - .resolveTcfContext(any(), any(), any(), any(), any(), eq(expectedRequestLogInfo), any()); + .resolveTcfContext(any(), any(), any(), any(), any(), eq(expectedRequestLogInfo), any(), any()); } private static AuctionContext givenAuctionContext( diff --git a/src/test/java/org/prebid/server/auction/requestfactory/AmpRequestFactoryTest.java b/src/test/java/org/prebid/server/auction/requestfactory/AmpRequestFactoryTest.java index 45440a5b40a..a730e9055cf 100644 --- a/src/test/java/org/prebid/server/auction/requestfactory/AmpRequestFactoryTest.java +++ b/src/test/java/org/prebid/server/auction/requestfactory/AmpRequestFactoryTest.java @@ -27,6 +27,7 @@ import org.prebid.server.VertxTest; import org.prebid.server.auction.DebugResolver; import org.prebid.server.auction.FpdResolver; +import org.prebid.server.auction.GeoLocationServiceWrapper; import org.prebid.server.auction.ImplicitParametersExtractor; import org.prebid.server.auction.OrtbTypesResolver; import org.prebid.server.auction.StoredRequestProcessor; @@ -116,6 +117,8 @@ public class AmpRequestFactoryTest extends VertxTest { private AmpPrivacyContextFactory ampPrivacyContextFactory; @Mock private DebugResolver debugResolver; + @Mock + private GeoLocationServiceWrapper geoLocationServiceWrapper; private AmpRequestFactory target; @@ -165,8 +168,11 @@ public void setUp() { .willAnswer(invocationOnMock -> Future.succeededFuture(invocationOnMock.getArgument(0))); given(ortb2RequestFactory.activityInfrastructureFrom(any())) .willReturn(Future.succeededFuture()); + given(geoLocationServiceWrapper.lookup(any())) + .willReturn(Future.succeededFuture(GeoInfo.builder().vendor("vendor").build())); given(debugResolver.debugContextFrom(any())).willReturn(DebugContext.of(true, true, null)); + final PrivacyContext defaultPrivacyContext = PrivacyContext.of( Privacy.builder() .gdpr("0") @@ -189,7 +195,8 @@ public void setUp() { fpdResolver, ampPrivacyContextFactory, debugResolver, - jacksonMapper); + jacksonMapper, + geoLocationServiceWrapper); } @Test @@ -1568,8 +1575,7 @@ public void shouldReturnPopulatedPrivacyContextAndGeoWhenPrivacyEnforcementRetur .build(), TcfContext.builder().geoInfo(geoInfo).build()); - given(ampPrivacyContextFactory.contextFrom(any())) - .willReturn(Future.succeededFuture(privacyContext)); + given(ampPrivacyContextFactory.contextFrom(any())).willReturn(Future.succeededFuture(privacyContext)); // when final AuctionContext result = target.fromRequest(routingContext, 0L).result(); @@ -1707,7 +1713,11 @@ private void givenBidRequest( .willAnswer(invocation -> Future.succeededFuture((BidRequest) invocation.getArgument(0))); given(ortb2RequestFactory.enrichBidRequestWithAccountAndPrivacyData(any())) - .willAnswer(invocation -> ((AuctionContext) invocation.getArgument(0)).getBidRequest()); + .willAnswer(invocation -> Future.succeededFuture(((AuctionContext) invocation.getArgument(0)) + .getBidRequest())); + given(ortb2RequestFactory.enrichBidRequestWithGeolocationData(any())) + .willAnswer(invocation -> Future.succeededFuture(((AuctionContext) invocation.getArgument(0)) + .getBidRequest())); given(ortb2RequestFactory.executeProcessedAuctionRequestHooks(any())) .willAnswer(invocation -> Future.succeededFuture( ((AuctionContext) invocation.getArgument(0)).getBidRequest())); diff --git a/src/test/java/org/prebid/server/auction/requestfactory/AuctionRequestFactoryTest.java b/src/test/java/org/prebid/server/auction/requestfactory/AuctionRequestFactoryTest.java index 0b6d47cee58..babe0ddaa83 100644 --- a/src/test/java/org/prebid/server/auction/requestfactory/AuctionRequestFactoryTest.java +++ b/src/test/java/org/prebid/server/auction/requestfactory/AuctionRequestFactoryTest.java @@ -26,6 +26,7 @@ import org.mockito.junit.MockitoRule; import org.prebid.server.VertxTest; import org.prebid.server.auction.DebugResolver; +import org.prebid.server.auction.GeoLocationServiceWrapper; import org.prebid.server.auction.ImplicitParametersExtractor; import org.prebid.server.auction.InterstitialProcessor; import org.prebid.server.auction.OrtbTypesResolver; @@ -101,8 +102,9 @@ public class AuctionRequestFactoryTest extends VertxTest { private AuctionPrivacyContextFactory auctionPrivacyContextFactory; @Mock private DebugResolver debugResolver; - @Mock + private GeoLocationServiceWrapper geoLocationServiceWrapper; + private AuctionRequestFactory target; @Mock @@ -171,7 +173,11 @@ public void setUp() { .willReturn(Future.succeededFuture(defaultPrivacyContext)); given(ortb2RequestFactory.enrichBidRequestWithAccountAndPrivacyData(any())) - .willAnswer(invocation -> ((AuctionContext) invocation.getArgument(0)).getBidRequest()); + .willAnswer(invocation -> Future.succeededFuture(((AuctionContext) invocation.getArgument(0)) + .getBidRequest())); + given(ortb2RequestFactory.enrichBidRequestWithGeolocationData(any())) + .willAnswer(invocation -> Future.succeededFuture(((AuctionContext) invocation.getArgument(0)) + .getBidRequest())); given(ortb2RequestFactory.executeProcessedAuctionRequestHooks(any())) .willAnswer(invocation -> Future.succeededFuture( ((AuctionContext) invocation.getArgument(0)).getBidRequest())); @@ -183,6 +189,8 @@ public void setUp() { .willReturn(Future.succeededFuture()); given(cookieDeprecationService.updateBidRequestDevice(any(), any())) .will(invocationOnMock -> invocationOnMock.getArgument(0)); + given(geoLocationServiceWrapper.lookup(any())) + .willReturn(Future.succeededFuture(GeoInfo.builder().vendor("vendor").build())); target = new AuctionRequestFactory( Integer.MAX_VALUE, @@ -197,7 +205,8 @@ public void setUp() { ortbTypesResolver, auctionPrivacyContextFactory, debugResolver, - jacksonMapper); + jacksonMapper, + geoLocationServiceWrapper); } @Test @@ -231,7 +240,8 @@ public void shouldReturnFailedFutureIfRequestBodyExceedsMaxRequestSize() { ortbTypesResolver, auctionPrivacyContextFactory, debugResolver, - jacksonMapper); + jacksonMapper, + geoLocationServiceWrapper); given(routingContext.getBodyAsString()).willReturn("body"); diff --git a/src/test/java/org/prebid/server/auction/requestfactory/Ortb2RequestFactoryTest.java b/src/test/java/org/prebid/server/auction/requestfactory/Ortb2RequestFactoryTest.java index bfb5bec8bca..05d7667d2e4 100644 --- a/src/test/java/org/prebid/server/auction/requestfactory/Ortb2RequestFactoryTest.java +++ b/src/test/java/org/prebid/server/auction/requestfactory/Ortb2RequestFactoryTest.java @@ -82,7 +82,6 @@ import java.time.Clock; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.function.UnaryOperator; @@ -93,7 +92,6 @@ import static java.util.function.UnaryOperator.identity; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; -import static org.assertj.core.groups.Tuple.tuple; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; @@ -902,10 +900,10 @@ public void enrichBidRequestWithAccountAndPrivacyDataShouldReturnSameBidRequest( .build(); // when - final BidRequest result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); + final Future result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); // then - assertThat(result) + assertThat(result).isSucceeded().unwrap() .extracting(BidRequest::getExt) .extracting(ExtRequest::getPrebid) .satisfies(extPrebid -> { @@ -956,10 +954,10 @@ public void enrichBidRequestWithAccountAndPrivacyDataShouldReturnBidRequestWithA .build(); // when - final BidRequest result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); + final Future result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); // then - assertThat(result) + assertThat(result).isSucceeded().unwrap() .extracting(BidRequest::getExt) .extracting(ExtRequest::getPrebid) .satisfies(extPrebid -> { @@ -1001,14 +999,35 @@ public void enrichBidRequestWithAccountAndPrivacyDataShouldAddCountryFromPrivacy .build(); // when - final BidRequest result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); + final Future result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); // then - assertThat(List.of(result)) + assertThat(result).isSucceeded().unwrap() .extracting(BidRequest::getDevice) .extracting(Device::getGeo) .extracting(Geo::getCountry) - .containsExactly("UKR"); + .isEqualTo("UKR"); + } + + @Test + public void enrichBidRequestWithGeolocationDataShouldAddCountryFromGeoInfo() { + // given + given(countryCodeMapper.mapToAlpha3("ua")).willReturn("UKR"); + + final AuctionContext auctionContext = AuctionContext.builder() + .bidRequest(givenBidRequest(identity())) + .geoInfo(GeoInfo.builder().vendor("v").country("ua").build()) + .build(); + + // when + final Future result = target.enrichBidRequestWithGeolocationData(auctionContext); + + // then + assertThat(result).isSucceeded().unwrap() + .extracting(BidRequest::getDevice) + .extracting(Device::getGeo) + .extracting(Geo::getCountry) + .isEqualTo("UKR"); } @Test @@ -1041,10 +1060,36 @@ public void enrichBidRequestWithAccountAndPrivacyDataShouldAddRegionFromPrivacy( .build(); // when - final BidRequest result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); + final Future result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); // then - assertThat(result) + assertThat(result).isSucceeded().unwrap() + .extracting(BidRequest::getDevice) + .extracting(Device::getGeo) + .extracting(Geo::getRegion) + .isEqualTo("REGION"); + } + + @Test + public void enrichBidRequestWithGeolocationDataShouldAddRegionFromPrivacy() { + // given + given(countryCodeMapper.mapToAlpha3(any())).willReturn(null); + + final Device device = Device.builder() + .geo(Geo.builder().region("regionInRequest").build()) + .build(); + final BidRequest bidRequest = givenBidRequest(requestCustomizer -> requestCustomizer.device(device)); + + final AuctionContext auctionContext = AuctionContext.builder() + .bidRequest(bidRequest) + .geoInfo(GeoInfo.builder().vendor("v").region("region").build()) + .build(); + + // when + final Future result = target.enrichBidRequestWithGeolocationData(auctionContext); + + // then + assertThat(result).isSucceeded().unwrap() .extracting(BidRequest::getDevice) .extracting(Device::getGeo) .extracting(Geo::getRegion) @@ -1074,10 +1119,36 @@ public void enrichBidRequestWithAccountAndPrivacyDataShouldMakeRegionUpperCasedW .build(); // when - final BidRequest result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); + final Future result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); + + // then + assertThat(result).isSucceeded().unwrap() + .extracting(BidRequest::getDevice) + .extracting(Device::getGeo) + .extracting(Geo::getRegion) + .isEqualTo("REGIONINREQUEST"); + } + + @Test + public void enrichBidRequestWithGeolocationDataShouldMakeRegionUpperCasedWhenNoGeoInfoProvided() { + // given + given(countryCodeMapper.mapToAlpha3(any())).willReturn(null); + + final Device device = Device.builder() + .geo(Geo.builder().region("regionInRequest").build()) + .build(); + final BidRequest bidRequest = givenBidRequest(requestCustomizer -> requestCustomizer.device(device)); + + final AuctionContext auctionContext = AuctionContext.builder() + .bidRequest(bidRequest) + .geoInfo(GeoInfo.builder().vendor("v").build()) + .build(); + + // when + final Future result = target.enrichBidRequestWithGeolocationData(auctionContext); // then - assertThat(result) + assertThat(result).isSucceeded().unwrap() .extracting(BidRequest::getDevice) .extracting(Device::getGeo) .extracting(Geo::getRegion) @@ -1103,7 +1174,7 @@ public void enrichBidRequestWithAccountAndPrivacyDataShouldAddIpAddressV4FromPri final Account account = Account.empty("id"); // when - final BidRequest result = target.enrichBidRequestWithAccountAndPrivacyData( + final Future result = target.enrichBidRequestWithAccountAndPrivacyData( AuctionContext.builder() .bidRequest(bidRequest) .account(account) @@ -1111,10 +1182,10 @@ public void enrichBidRequestWithAccountAndPrivacyDataShouldAddIpAddressV4FromPri .build()); // then - assertThat(Collections.singleton(result)) + assertThat(result).isSucceeded().unwrap() .extracting(BidRequest::getDevice) .extracting(Device::getIp, Device::getIpv6) - .containsOnly(tuple("ipv4", null)); + .containsOnly("ipv4", null); } @Test @@ -1136,7 +1207,7 @@ public void enrichBidRequestWithAccountAndPrivacyDataShouldAddIpAddressV6FromPri final Account account = Account.empty("id"); // when - final BidRequest result = target.enrichBidRequestWithAccountAndPrivacyData( + final Future result = target.enrichBidRequestWithAccountAndPrivacyData( AuctionContext.builder() .bidRequest(bidRequest) .account(account) @@ -1144,10 +1215,10 @@ public void enrichBidRequestWithAccountAndPrivacyDataShouldAddIpAddressV6FromPri .build()); // then - assertThat(Collections.singleton(result)) + assertThat(result).isSucceeded().unwrap() .extracting(BidRequest::getDevice) .extracting(Device::getIp, Device::getIpv6) - .containsOnly(tuple(null, "ipv6")); + .containsOnly(null, "ipv6"); } @Test @@ -1459,10 +1530,10 @@ public void enrichBidRequestWithAccountAndPrivacyDataShouldSetDsaFromAccountWhen .build(); // when - final BidRequest result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); + final Future result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); // then - assertThat(result) + assertThat(result).isSucceeded().unwrap() .extracting(BidRequest::getRegs) .extracting(Regs::getExt) .extracting(ExtRegs::getDsa) @@ -1511,14 +1582,14 @@ public void enrichBidRequestWithAccountAndPrivacyDataShouldNotSetDsaFromAccountW .build(); // when - final BidRequest result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); + final Future result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); // then - assertThat(result) + assertThat(result).isSucceeded().unwrap() .extracting(BidRequest::getRegs) .extracting(Regs::getExt) .isNull(); - assertThat(result) + assertThat(result).isSucceeded().unwrap() .extracting(BidRequest::getRegs) .isSameAs(regs); } @@ -1568,10 +1639,10 @@ public void enrichBidRequestWithAccountAndPrivacyDataShouldNotSetDsaFromAccountW .build(); // when - final BidRequest result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); + final Future result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); // then - assertThat(result) + assertThat(result).isSucceeded().unwrap() .extracting(BidRequest::getRegs) .extracting(Regs::getExt) .extracting(ExtRegs::getDsa) @@ -1621,10 +1692,10 @@ public void enrichBidRequestWithAccountAndPrivacyDataShouldSetDsaFromAccountWhen .build(); // when - final BidRequest result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); + final Future result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); // then - assertThat(result) + assertThat(result).isSucceeded().unwrap() .extracting(BidRequest::getRegs) .extracting(Regs::getExt) .extracting(ExtRegs::getDsa) @@ -1676,14 +1747,14 @@ public void enrichBidRequestWithAccountAndPrivacyDataShouldNotSetDsaFromAccountW .build(); // when - final BidRequest result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); + final Future result = target.enrichBidRequestWithAccountAndPrivacyData(auctionContext); // then - assertThat(result) + assertThat(result).isSucceeded().unwrap() .extracting(BidRequest::getRegs) .extracting(Regs::getExt) .isNull(); - assertThat(result) + assertThat(result).isSucceeded().unwrap() .extracting(BidRequest::getRegs) .isSameAs(regs); } diff --git a/src/test/java/org/prebid/server/auction/requestfactory/VideoRequestFactoryTest.java b/src/test/java/org/prebid/server/auction/requestfactory/VideoRequestFactoryTest.java index f9b4f23cc79..9d34db0f153 100644 --- a/src/test/java/org/prebid/server/auction/requestfactory/VideoRequestFactoryTest.java +++ b/src/test/java/org/prebid/server/auction/requestfactory/VideoRequestFactoryTest.java @@ -24,6 +24,7 @@ import org.mockito.stubbing.Answer; import org.prebid.server.VertxTest; import org.prebid.server.auction.DebugResolver; +import org.prebid.server.auction.GeoLocationServiceWrapper; import org.prebid.server.auction.PriceGranularity; import org.prebid.server.auction.VideoStoredRequestProcessor; import org.prebid.server.auction.model.AuctionContext; @@ -32,6 +33,7 @@ import org.prebid.server.auction.privacy.contextfactory.AuctionPrivacyContextFactory; import org.prebid.server.auction.versionconverter.BidRequestOrtbVersionConversionManager; import org.prebid.server.exception.InvalidRequestException; +import org.prebid.server.geolocation.model.GeoInfo; import org.prebid.server.metric.MetricName; import org.prebid.server.model.CaseInsensitiveMultiMap; import org.prebid.server.model.Endpoint; @@ -82,6 +84,8 @@ public class VideoRequestFactoryTest extends VertxTest { private Ortb2ImplicitParametersResolver paramsResolver; @Mock private AuctionPrivacyContextFactory auctionPrivacyContextFactory; + @Mock + private GeoLocationServiceWrapper geoLocationServiceWrapper; private VideoRequestFactory target; @@ -115,6 +119,8 @@ public void setUp() { given(routingContext.queryParams()).willReturn(MultiMap.caseInsensitiveMultiMap()); given(httpServerRequest.remoteAddress()).willReturn(new SocketAddressImpl(1234, "host")); given(httpServerRequest.headers()).willReturn(MultiMap.caseInsensitiveMultiMap()); + given(geoLocationServiceWrapper.lookup(any())) + .willReturn(Future.succeededFuture(GeoInfo.builder().vendor("vendor").build())); final PrivacyContext defaultPrivacyContext = PrivacyContext.of( Privacy.builder() @@ -140,7 +146,8 @@ public void setUp() { paramsResolver, auctionPrivacyContextFactory, debugResolver, - jacksonMapper); + jacksonMapper, + geoLocationServiceWrapper); } @Test @@ -175,7 +182,8 @@ public void shouldReturnFailedFutureIfStoredRequestIsEnforcedAndIdIsNotProvided( paramsResolver, auctionPrivacyContextFactory, debugResolver, - jacksonMapper); + jacksonMapper, + geoLocationServiceWrapper); // when final Future future = target.fromRequest(routingContext, 0L); @@ -200,7 +208,8 @@ public void shouldReturnFailedFutureIfRequestBodyExceedsMaxRequestSize() { paramsResolver, auctionPrivacyContextFactory, debugResolver, - jacksonMapper); + jacksonMapper, + geoLocationServiceWrapper); given(routingContext.getBodyAsString()).willReturn("body"); @@ -406,7 +415,11 @@ private void givenBidRequest(BidRequest bidRequest, List podErrors) { .willAnswer(answerWithFirstArgument()); given(ortb2RequestFactory.enrichBidRequestWithAccountAndPrivacyData(any())) - .willAnswer(invocation -> ((AuctionContext) invocation.getArgument(0)).getBidRequest()); + .willAnswer(invocation -> Future.succeededFuture(((AuctionContext) invocation.getArgument(0)) + .getBidRequest())); + given(ortb2RequestFactory.enrichBidRequestWithGeolocationData(any())) + .willAnswer(invocation -> Future.succeededFuture(((AuctionContext) invocation.getArgument(0)) + .getBidRequest())); given(ortb2RequestFactory.executeProcessedAuctionRequestHooks(any())) .willAnswer(invocation -> Future.succeededFuture( ((AuctionContext) invocation.getArgument(0)).getBidRequest())); diff --git a/src/test/java/org/prebid/server/privacy/gdpr/TcfDefinerServiceTest.java b/src/test/java/org/prebid/server/privacy/gdpr/TcfDefinerServiceTest.java index 450b989ab6d..c529c0026f0 100644 --- a/src/test/java/org/prebid/server/privacy/gdpr/TcfDefinerServiceTest.java +++ b/src/test/java/org/prebid/server/privacy/gdpr/TcfDefinerServiceTest.java @@ -9,10 +9,10 @@ import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; +import org.prebid.server.auction.GeoLocationServiceWrapper; import org.prebid.server.auction.IpAddressHelper; import org.prebid.server.auction.model.IpAddress; import org.prebid.server.bidder.BidderCatalog; -import org.prebid.server.geolocation.GeoLocationService; import org.prebid.server.geolocation.model.GeoInfo; import org.prebid.server.metric.MetricName; import org.prebid.server.metric.Metrics; @@ -61,7 +61,7 @@ public class TcfDefinerServiceTest { @Mock private Tcf2Service tcf2Service; @Mock - private GeoLocationService geoLocationService; + private GeoLocationServiceWrapper geoLocationServiceWrapper; @Mock private BidderCatalog bidderCatalog; @Mock @@ -69,13 +69,10 @@ public class TcfDefinerServiceTest { @Mock private Metrics metrics; - private TcfDefinerService tcfDefinerService; + private TcfDefinerService target; @Before public void setUp() { - given(geoLocationService.lookup(anyString(), any())) - .willReturn(Future.succeededFuture(GeoInfo.builder().vendor("vendor").country(EEA_COUNTRY).build())); - final GdprConfig gdprConfig = GdprConfig.builder() .defaultValue("1") .enabled(true) @@ -84,11 +81,11 @@ public void setUp() { .build()) .build(); - tcfDefinerService = new TcfDefinerService( + target = new TcfDefinerService( gdprConfig, singleton(EEA_COUNTRY), tcf2Service, - geoLocationService, + geoLocationServiceWrapper, bidderCatalog, ipAddressHelper, metrics); @@ -98,23 +95,23 @@ public void setUp() { public void resolveTcfContextShouldReturnContextWhenGdprIsDisabled() { // given final GdprConfig gdprConfig = GdprConfig.builder().enabled(false).build(); - tcfDefinerService = new TcfDefinerService( + target = new TcfDefinerService( gdprConfig, singleton(EEA_COUNTRY), tcf2Service, - geoLocationService, + geoLocationServiceWrapper, bidderCatalog, ipAddressHelper, metrics); // when - final Future result = tcfDefinerService.resolveTcfContext( + final Future result = target.resolveTcfContext( Privacy.builder().build(), null, null, MetricName.setuid, null, null); // then assertThat(result).succeededWith(TcfContext.empty()); - verifyNoInteractions(geoLocationService); + verifyNoInteractions(geoLocationServiceWrapper); verifyNoInteractions(metrics); } @@ -126,13 +123,13 @@ public void resolveTcfContextShouldReturnContextWithGdprZeroWhenGdprIsDisabledBy .build(); // when - final Future result = tcfDefinerService.resolveTcfContext( - Privacy.builder().build(), null, null, accountGdprConfig, MetricName.amp, null, null); + final Future result = target.resolveTcfContext( + Privacy.builder().build(), null, null, accountGdprConfig, MetricName.amp, null, null, null); // then assertThat(result).succeededWith(TcfContext.empty()); - verifyNoInteractions(geoLocationService); + verifyNoInteractions(geoLocationServiceWrapper); verifyNoInteractions(metrics); } @@ -142,13 +139,13 @@ public void resolveTcfContextShouldReturnContextWhenGdprIsDisabledByAccount() { final AccountGdprConfig accountGdprConfig = AccountGdprConfig.builder().enabled(false).build(); // when - final Future result = tcfDefinerService.resolveTcfContext( + final Future result = target.resolveTcfContext( Privacy.builder().build(), null, accountGdprConfig, MetricName.setuid, null, null); // then assertThat(result).succeededWith(TcfContext.empty()); - verifyNoInteractions(geoLocationService); + verifyNoInteractions(geoLocationServiceWrapper); verifyNoInteractions(metrics); } @@ -161,13 +158,13 @@ public void resolveTcfContextShouldCheckAccountConfigValueWhenRequestTypeIsUnkno .build(); // when - final Future result = tcfDefinerService.resolveTcfContext( - Privacy.builder().build(), null, null, accountGdprConfig, MetricName.setuid, null, null); + final Future result = target.resolveTcfContext( + Privacy.builder().build(), null, null, accountGdprConfig, MetricName.setuid, null, null, null); // then assertThat(result).succeededWith(TcfContext.empty()); - verifyNoInteractions(geoLocationService); + verifyNoInteractions(geoLocationServiceWrapper); verifyNoInteractions(metrics); } @@ -176,11 +173,11 @@ public void resolveTcfContextShouldCheckServiceConfigValueWhenRequestTypeIsUnkno // given final GdprConfig gdprConfig = GdprConfig.builder().enabled(false).build(); - tcfDefinerService = new TcfDefinerService( + target = new TcfDefinerService( gdprConfig, singleton(EEA_COUNTRY), tcf2Service, - geoLocationService, + geoLocationServiceWrapper, bidderCatalog, ipAddressHelper, metrics); @@ -190,13 +187,13 @@ public void resolveTcfContextShouldCheckServiceConfigValueWhenRequestTypeIsUnkno .build(); // when - final Future result = tcfDefinerService.resolveTcfContext( - Privacy.builder().build(), null, null, accountGdprConfig, MetricName.setuid, null, null); + final Future result = target.resolveTcfContext( + Privacy.builder().build(), null, null, accountGdprConfig, MetricName.setuid, null, null, null); // then assertThat(result).succeededWith(TcfContext.empty()); - verifyNoInteractions(geoLocationService); + verifyNoInteractions(geoLocationServiceWrapper); verifyNoInteractions(metrics); } @@ -208,11 +205,11 @@ public void resolveTcfContextShouldConsiderTcfVersionOneAsCorruptedVersionTwo() .consentStringMeansInScope(true) .build(); - tcfDefinerService = new TcfDefinerService( + target = new TcfDefinerService( gdprConfig, singleton(EEA_COUNTRY), tcf2Service, - geoLocationService, + geoLocationServiceWrapper, bidderCatalog, ipAddressHelper, metrics); @@ -220,9 +217,9 @@ public void resolveTcfContextShouldConsiderTcfVersionOneAsCorruptedVersionTwo() final String vendorConsent = "BOEFEAyOEFEAyAHABDENAI4AAAB9vABAASA"; // when - final Future result = tcfDefinerService.resolveTcfContext( + final Future result = target.resolveTcfContext( Privacy.builder().gdpr("1").consentString(vendorConsent).ccpa(null).coppa(null).build(), "london", null, - null, MetricName.setuid, null, null); + null, MetricName.setuid, null, null, null); // then assertThat(result).isSucceeded(); @@ -240,11 +237,11 @@ public void resolveTcfContextShouldTreatTcfConsentWithTcfPolicyVersionGreaterTha .consentStringMeansInScope(true) .build(); - tcfDefinerService = new TcfDefinerService( + target = new TcfDefinerService( gdprConfig, singleton(EEA_COUNTRY), tcf2Service, - geoLocationService, + geoLocationServiceWrapper, bidderCatalog, ipAddressHelper, metrics); @@ -255,13 +252,14 @@ public void resolveTcfContextShouldTreatTcfConsentWithTcfPolicyVersionGreaterTha .encode(); // when - final Future result = tcfDefinerService.resolveTcfContext( + final Future result = target.resolveTcfContext( Privacy.builder().gdpr("1").consentString(vendorConsent).ccpa(null).coppa(null).build(), "london", null, null, MetricName.setuid, null, + null, null); // then @@ -280,11 +278,11 @@ public void resolveTcfContextShouldConsiderPresenceOfConsentStringAsInScope() { .consentStringMeansInScope(true) .build(); - tcfDefinerService = new TcfDefinerService( + target = new TcfDefinerService( gdprConfig, singleton(EEA_COUNTRY), tcf2Service, - geoLocationService, + geoLocationServiceWrapper, bidderCatalog, ipAddressHelper, metrics); @@ -292,7 +290,7 @@ public void resolveTcfContextShouldConsiderPresenceOfConsentStringAsInScope() { final String vendorConsent = "CPBCa-mPBCa-mAAAAAENA0CAAEAAAAAAACiQAaQAwAAgAgABoAAAAAA"; // when - final Future result = tcfDefinerService.resolveTcfContext( + final Future result = target.resolveTcfContext( Privacy.builder().consentString(vendorConsent).build(), null, null, null, null, null); // then @@ -308,19 +306,21 @@ public void resolveTcfContextShouldConsiderPresenceOfConsentStringAsInScope() { true, null, null, null); assertThat(result.result().getConsent()).isNotNull(); - verifyNoInteractions(geoLocationService); + verifyNoInteractions(geoLocationServiceWrapper); verify(metrics).updatePrivacyTcfRequestsMetric(2); verify(metrics).updatePrivacyTcfGeoMetric(2, null); } @Test - public void resolveTcfContextShouldReturnGdprFromCountryWhenGdprFromRequestIsNotValid() { + public void resolveTcfContextShouldReturnGdprFromCountryWhenGdprFromRequestIsNotValidAndGeoLookupSkipped() { // given final Privacy privacy = Privacy.builder().gdpr(EMPTY).consentString("consent").build(); + given(geoLocationServiceWrapper.doLookup(anyString(), eq(EEA_COUNTRY), any())) + .willReturn(Future.succeededFuture()); // when - final Future result = tcfDefinerService.resolveTcfContext( - privacy, EEA_COUNTRY, "ip", null, null, null, null); + final Future result = target.resolveTcfContext( + privacy, EEA_COUNTRY, "ip", null, null, null, null, null); // then assertThat(result).isSucceeded(); @@ -332,25 +332,21 @@ public void resolveTcfContextShouldReturnGdprFromCountryWhenGdprFromRequestIsNot TcfContext::getIpAddress) .containsExactly(true, "consent", null, true, "ip"); - verifyNoInteractions(geoLocationService); verify(metrics).updatePrivacyTcfGeoMetric(2, true); } @Test - public void resolveTcfContextShouldReturnGdprFromGeoLocationServiceWhenGdprFromRequestIsNotValid() { + public void resolveTcfContextShouldReturnGdprFromExistingGeoInfoWhenGdprFromRequestIsNotValidAndGeoLookupFailed() { // given - given(ipAddressHelper.toIpAddress(anyString())).willReturn(IpAddress.of("ip", IpAddress.IP.v4)); - given(ipAddressHelper.maskIpv4(anyString())).willReturn("ip-masked"); - - final GeoInfo geoInfo = GeoInfo.builder().vendor("vendor").country("ua").build(); - given(geoLocationService.lookup(eq("ip"), any())).willReturn(Future.succeededFuture(geoInfo)); + final Privacy privacy = Privacy.builder().gdpr(EMPTY).consentString("consent").build(); + given(geoLocationServiceWrapper.doLookup(anyString(), eq(EEA_COUNTRY), any())) + .willReturn(Future.failedFuture("Bad ip")); - final String consentString = "COwayg7OwaybYN6AAAENAPCgAIAAAAAAAAAAASkAAAAAAAAAAA"; - final Privacy privacy = Privacy.builder().gdpr(EMPTY).consentString(consentString).build(); + final GeoInfo geoInfo = GeoInfo.builder().vendor("vendor").country(EEA_COUNTRY).build(); // when - final Future result = tcfDefinerService.resolveTcfContext( - privacy, "ip", null, MetricName.setuid, null, null); + final Future result = target.resolveTcfContext( + privacy, EEA_COUNTRY, "ip", null, null, null, null, geoInfo); // then assertThat(result).isSucceeded(); @@ -360,35 +356,27 @@ public void resolveTcfContextShouldReturnGdprFromGeoLocationServiceWhenGdprFromR TcfContext::getGeoInfo, TcfContext::getInEea, TcfContext::getIpAddress) - .containsExactly(true, consentString, geoInfo, true, "ip-masked"); + .containsExactly(true, "consent", geoInfo, true, "ip"); - verify(ipAddressHelper).maskIpv4(eq("ip")); - verify(geoLocationService).lookup(eq("ip-masked"), any()); - verify(metrics).updateGeoLocationMetric(true); verify(metrics).updatePrivacyTcfGeoMetric(2, true); } @Test - public void resolveTcfContextShouldConsultDefaultValueWhenGeoLookupFailed() { + public void resolveTcfContextShouldReturnGdprFromGeoLocationServiceWhenGdprFromRequestIsNotValid() { // given - final GdprConfig gdprConfig = GdprConfig.builder() - .enabled(true) - .defaultValue("0") - .build(); - tcfDefinerService = new TcfDefinerService( - gdprConfig, - singleton(EEA_COUNTRY), - tcf2Service, - geoLocationService, - bidderCatalog, - ipAddressHelper, - metrics); + given(ipAddressHelper.toIpAddress(anyString())).willReturn(IpAddress.of("ip", IpAddress.IP.v4)); + given(ipAddressHelper.maskIpv4(anyString())).willReturn("ip-masked"); + + final GeoInfo geoInfo = GeoInfo.builder().vendor("vendor").country("ua").build(); + given(geoLocationServiceWrapper.doLookup(eq("ip-masked"), any(), any())) + .willReturn(Future.succeededFuture(geoInfo)); - given(geoLocationService.lookup(anyString(), any())).willReturn(Future.failedFuture("Bad ip")); + final String consentString = "COwayg7OwaybYN6AAAENAPCgAIAAAAAAAAAAASkAAAAAAAAAAA"; + final Privacy privacy = Privacy.builder().gdpr(EMPTY).consentString(consentString).build(); // when - final Future result = tcfDefinerService.resolveTcfContext( - Privacy.builder().build(), "ip", null, MetricName.setuid, null, null); + final Future result = target.resolveTcfContext( + privacy, "ip", null, MetricName.setuid, null, null); // then assertThat(result).isSucceeded(); @@ -398,32 +386,33 @@ public void resolveTcfContextShouldConsultDefaultValueWhenGeoLookupFailed() { TcfContext::getGeoInfo, TcfContext::getInEea, TcfContext::getIpAddress) - .containsExactly(false, null, null, null, "ip"); + .containsExactly(true, consentString, geoInfo, true, "ip-masked"); - verify(metrics).updateGeoLocationMetric(false); + verify(ipAddressHelper).maskIpv4(eq("ip")); + verify(geoLocationServiceWrapper).doLookup(eq("ip-masked"), any(), any()); } @Test - public void resolveTcfContextShouldConsultDefaultValueAndSkipGeoLookupWhenIpIsNull() { + public void resolveTcfContextShouldConsultDefaultValueWhenGeoLookupFailed() { // given final GdprConfig gdprConfig = GdprConfig.builder() .enabled(true) .defaultValue("0") .build(); - tcfDefinerService = new TcfDefinerService( + target = new TcfDefinerService( gdprConfig, singleton(EEA_COUNTRY), tcf2Service, - geoLocationService, + geoLocationServiceWrapper, bidderCatalog, ipAddressHelper, metrics); - given(geoLocationService.lookup(anyString(), any())).willReturn(Future.failedFuture("Bad ip")); + given(geoLocationServiceWrapper.doLookup(anyString(), any(), any())).willReturn(Future.failedFuture("Bad ip")); // when - final Future result = tcfDefinerService.resolveTcfContext( - Privacy.builder().build(), null, null, MetricName.setuid, null, null); + final Future result = target.resolveTcfContext( + Privacy.builder().build(), "ip", null, MetricName.setuid, null, null); // then assertThat(result).isSucceeded(); @@ -433,9 +422,7 @@ public void resolveTcfContextShouldConsultDefaultValueAndSkipGeoLookupWhenIpIsNu TcfContext::getGeoInfo, TcfContext::getInEea, TcfContext::getIpAddress) - .containsExactly(false, null, null, null, null); - - verifyNoInteractions(geoLocationService); + .containsExactly(false, null, null, null, "ip"); } @Test @@ -446,7 +433,7 @@ public void resolveTcfContextShouldReturnTcfContextWithConsentValidAsTrue() { .consentString("CPBCa-mPBCa-mAAAAAENA0CAAEAAAAAAACiQAaQAwAAgAgABoAAAAAA") .build(); // when - final Future result = tcfDefinerService.resolveTcfContext( + final Future result = target.resolveTcfContext( privacy, null, null, MetricName.setuid, null, null); // then @@ -464,7 +451,7 @@ public void resolveTcfContextShouldReturnTcfContextWithConsentValidAsFalse() { final Privacy privacy = Privacy.builder().gdpr("1").consentString("invalid").build(); // when - final Future result = tcfDefinerService.resolveTcfContext( + final Future result = target.resolveTcfContext( privacy, null, null, MetricName.setuid, null, null); // then @@ -482,7 +469,7 @@ public void resolveTcfContextShouldIncrementMissingConsentStringMetric() { final Privacy privacy = Privacy.builder().gdpr("1").consentString(EMPTY).build(); // when - tcfDefinerService.resolveTcfContext( + target.resolveTcfContext( privacy, null, null, MetricName.setuid, null, null); // then @@ -494,7 +481,7 @@ public void resolveTcfContextShouldIncrementInvalidConsentStringMetric() { final Privacy privacy = Privacy.builder().gdpr("1").consentString("abc").build(); // when - tcfDefinerService.resolveTcfContext( + target.resolveTcfContext( privacy, null, null, MetricName.setuid, null, null); // then @@ -507,7 +494,7 @@ public void resultForVendorIdsShouldNotSetTcfRequestsAndTcfGeoMetricsWhenConsent given(tcf2Service.permissionsFor(any(), any())).willReturn(Future.succeededFuture()); // when - tcfDefinerService.resultForVendorIds(singleton(1), TcfContext.builder() + target.resultForVendorIds(singleton(1), TcfContext.builder() .inGdprScope(true) .consent(TCStringEmpty.create()) .ipAddress("ip") @@ -521,7 +508,7 @@ public void resultForVendorIdsShouldNotSetTcfRequestsAndTcfGeoMetricsWhenConsent @Test public void resultForVendorIdsShouldAllowAllWhenGdprIsZero() { // when - final Future> result = tcfDefinerService.resultForVendorIds( + final Future> result = target.resultForVendorIds( singleton(1), TcfContext.builder().inGdprScope(false).build()); // then @@ -537,7 +524,7 @@ public void resultForVendorIdsShouldReturnRestrictAllWhenConsentIsMissing() { given(tcf2Service.permissionsFor(any(), any())).willReturn(Future.succeededFuture()); // when - tcfDefinerService.resultForVendorIds(singleton(1), TcfContext.builder() + target.resultForVendorIds(singleton(1), TcfContext.builder() .inGdprScope(true) .consent(TCStringEmpty.create()) .build()); @@ -549,7 +536,7 @@ public void resultForVendorIdsShouldReturnRestrictAllWhenConsentIsMissing() { @Test public void resultForBidderNamesShouldReturnAllowAllWhenGdprIsZero() { // when - final Future> result = tcfDefinerService.resultForBidderNames( + final Future> result = target.resultForBidderNames( singleton("b1"), TcfContext.builder().inGdprScope(false).build(), null); @@ -569,7 +556,7 @@ public void resultForVendorIdsShouldReturnTcfResponseFromTcf2ServiceWhenConsentS VendorPermission.of(2, null, PrivacyEnforcementAction.allowAll())))); // when - final Future> result = tcfDefinerService.resultForVendorIds( + final Future> result = target.resultForVendorIds( new HashSet<>(asList(1, 2)), TcfContext.builder() .inGdprScope(true) @@ -593,7 +580,7 @@ public void resultForBidderNamesShouldReturnTcfResponseFromTcf2ServiceWhenConsen // when final Set bidderNames = new HashSet<>(asList("b1", "b2")); final String consentString = "COwayg7OwaybYN6AAAENAPCgAIAAAAAAAAAAASkAAAAAAAAAAA"; - final Future> result = tcfDefinerService.resultForBidderNames( + final Future> result = target.resultForBidderNames( bidderNames, TcfContext.builder() .inGdprScope(true) From b62026f296c4984c84f0e9df173808e7315e4bf2 Mon Sep 17 00:00:00 2001 From: sullis Date: Wed, 20 Mar 2024 03:36:53 -0700 Subject: [PATCH 006/320] Core: Add Netty epoll arm64 dependency (#3064) --- pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pom.xml b/pom.xml index 4cca09abf73..b50bf802584 100644 --- a/pom.xml +++ b/pom.xml @@ -189,6 +189,11 @@ netty-transport-native-epoll linux-x86_64 + + io.netty + netty-transport-native-epoll + linux-aarch_64 + org.apache.commons commons-lang3 From d0a9a49c3234695742c5cb62e45d9c8d68d37dca Mon Sep 17 00:00:00 2001 From: serhiinahornyi Date: Fri, 22 Mar 2024 02:11:55 +0200 Subject: [PATCH 007/320] Prebid Server prepare release 2.13.0 --- extra/bundle/pom.xml | 2 +- extra/modules/confiant-ad-quality/pom.xml | 2 +- extra/modules/ortb2-blocking/pom.xml | 2 +- extra/modules/pb-richmedia-filter/pom.xml | 2 +- extra/modules/pom.xml | 4 ++-- extra/pom.xml | 4 ++-- pom.xml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/extra/bundle/pom.xml b/extra/bundle/pom.xml index b48eaf7780d..5eecd9b6eae 100644 --- a/extra/bundle/pom.xml +++ b/extra/bundle/pom.xml @@ -5,7 +5,7 @@ org.prebid prebid-server-aggregator - 2.13.0-SNAPSHOT + 2.13.0 ../../extra/pom.xml diff --git a/extra/modules/confiant-ad-quality/pom.xml b/extra/modules/confiant-ad-quality/pom.xml index e04ca09ea57..f338a7b2744 100644 --- a/extra/modules/confiant-ad-quality/pom.xml +++ b/extra/modules/confiant-ad-quality/pom.xml @@ -5,7 +5,7 @@ org.prebid.server.hooks.modules all-modules - 2.13.0-SNAPSHOT + 2.13.0 confiant-ad-quality diff --git a/extra/modules/ortb2-blocking/pom.xml b/extra/modules/ortb2-blocking/pom.xml index 27bdb434d58..126658db4e6 100644 --- a/extra/modules/ortb2-blocking/pom.xml +++ b/extra/modules/ortb2-blocking/pom.xml @@ -5,7 +5,7 @@ org.prebid.server.hooks.modules all-modules - 2.13.0-SNAPSHOT + 2.13.0 ortb2-blocking diff --git a/extra/modules/pb-richmedia-filter/pom.xml b/extra/modules/pb-richmedia-filter/pom.xml index d5f8b4059b4..12ce4e5020a 100644 --- a/extra/modules/pb-richmedia-filter/pom.xml +++ b/extra/modules/pb-richmedia-filter/pom.xml @@ -5,7 +5,7 @@ org.prebid.server.hooks.modules all-modules - 2.13.0-SNAPSHOT + 2.13.0 pb-richmedia-filter diff --git a/extra/modules/pom.xml b/extra/modules/pom.xml index 32f53d9ae05..5e705c9ca0a 100644 --- a/extra/modules/pom.xml +++ b/extra/modules/pom.xml @@ -5,7 +5,7 @@ org.prebid prebid-server-aggregator - 2.13.0-SNAPSHOT + 2.13.0 ../../extra/pom.xml @@ -43,7 +43,7 @@ org.prebid prebid-server - 2.13.0-SNAPSHOT + 2.13.0 org.projectlombok diff --git a/extra/pom.xml b/extra/pom.xml index dd0ac70cacc..1ebd4339bef 100644 --- a/extra/pom.xml +++ b/extra/pom.xml @@ -4,14 +4,14 @@ org.prebid prebid-server-aggregator - 2.13.0-SNAPSHOT + 2.13.0 pom https://github.com/prebid/prebid-server-java scm:git:git@github.com:prebid/prebid-server-java.git scm:git:git@github.com:prebid/prebid-server-java.git - HEAD + 2.13.0 diff --git a/pom.xml b/pom.xml index b50bf802584..ddd7c83ded4 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.prebid prebid-server-aggregator - 2.13.0-SNAPSHOT + 2.13.0 extra/pom.xml From de76e955da8ebb02fe0ff803555e3ebacef98ca0 Mon Sep 17 00:00:00 2001 From: serhiinahornyi Date: Fri, 22 Mar 2024 02:14:25 +0200 Subject: [PATCH 008/320] Prebid Server prepare for next development iteration --- extra/bundle/pom.xml | 2 +- extra/modules/confiant-ad-quality/pom.xml | 2 +- extra/modules/ortb2-blocking/pom.xml | 2 +- extra/modules/pb-richmedia-filter/pom.xml | 2 +- extra/modules/pom.xml | 4 ++-- extra/pom.xml | 4 ++-- pom.xml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/extra/bundle/pom.xml b/extra/bundle/pom.xml index 5eecd9b6eae..1ca9ddf5047 100644 --- a/extra/bundle/pom.xml +++ b/extra/bundle/pom.xml @@ -5,7 +5,7 @@ org.prebid prebid-server-aggregator - 2.13.0 + 2.14.0-SNAPSHOT ../../extra/pom.xml diff --git a/extra/modules/confiant-ad-quality/pom.xml b/extra/modules/confiant-ad-quality/pom.xml index f338a7b2744..bb41e6fbdf0 100644 --- a/extra/modules/confiant-ad-quality/pom.xml +++ b/extra/modules/confiant-ad-quality/pom.xml @@ -5,7 +5,7 @@ org.prebid.server.hooks.modules all-modules - 2.13.0 + 2.14.0-SNAPSHOT confiant-ad-quality diff --git a/extra/modules/ortb2-blocking/pom.xml b/extra/modules/ortb2-blocking/pom.xml index 126658db4e6..c61cb2f4075 100644 --- a/extra/modules/ortb2-blocking/pom.xml +++ b/extra/modules/ortb2-blocking/pom.xml @@ -5,7 +5,7 @@ org.prebid.server.hooks.modules all-modules - 2.13.0 + 2.14.0-SNAPSHOT ortb2-blocking diff --git a/extra/modules/pb-richmedia-filter/pom.xml b/extra/modules/pb-richmedia-filter/pom.xml index 12ce4e5020a..ef47f0e6e85 100644 --- a/extra/modules/pb-richmedia-filter/pom.xml +++ b/extra/modules/pb-richmedia-filter/pom.xml @@ -5,7 +5,7 @@ org.prebid.server.hooks.modules all-modules - 2.13.0 + 2.14.0-SNAPSHOT pb-richmedia-filter diff --git a/extra/modules/pom.xml b/extra/modules/pom.xml index 5e705c9ca0a..03ce84b0956 100644 --- a/extra/modules/pom.xml +++ b/extra/modules/pom.xml @@ -5,7 +5,7 @@ org.prebid prebid-server-aggregator - 2.13.0 + 2.14.0-SNAPSHOT ../../extra/pom.xml @@ -43,7 +43,7 @@ org.prebid prebid-server - 2.13.0 + 2.14.0-SNAPSHOT org.projectlombok diff --git a/extra/pom.xml b/extra/pom.xml index 1ebd4339bef..6d132feb5c6 100644 --- a/extra/pom.xml +++ b/extra/pom.xml @@ -4,14 +4,14 @@ org.prebid prebid-server-aggregator - 2.13.0 + 2.14.0-SNAPSHOT pom https://github.com/prebid/prebid-server-java scm:git:git@github.com:prebid/prebid-server-java.git scm:git:git@github.com:prebid/prebid-server-java.git - 2.13.0 + HEAD diff --git a/pom.xml b/pom.xml index ddd7c83ded4..4364981ab09 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.prebid prebid-server-aggregator - 2.13.0 + 2.14.0-SNAPSHOT extra/pom.xml From 9b842c5ac73ad22b621acd0326b98badc0127fa4 Mon Sep 17 00:00:00 2001 From: Markiyan Mykush <95693607+marki1an@users.noreply.github.com> Date: Mon, 25 Mar 2024 16:09:11 +0200 Subject: [PATCH 009/320] Iqzone: Add mType support (#3044) --- .../server/bidder/iqzone/IqzoneBidder.java | 69 ++++++++++--------- .../bidder/iqzone/IqzoneBidderTest.java | 54 +++++++++------ .../iqzone/test-auction-iqzone-response.json | 1 + .../iqzone/test-iqzone-bid-response.json | 1 + 4 files changed, 72 insertions(+), 53 deletions(-) diff --git a/src/main/java/org/prebid/server/bidder/iqzone/IqzoneBidder.java b/src/main/java/org/prebid/server/bidder/iqzone/IqzoneBidder.java index 7785cd476b8..c6f473bc52a 100644 --- a/src/main/java/org/prebid/server/bidder/iqzone/IqzoneBidder.java +++ b/src/main/java/org/prebid/server/bidder/iqzone/IqzoneBidder.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.node.TextNode; import com.iab.openrtb.request.BidRequest; import com.iab.openrtb.request.Imp; +import com.iab.openrtb.response.Bid; import com.iab.openrtb.response.BidResponse; import com.iab.openrtb.response.SeatBid; import org.apache.commons.collections4.CollectionUtils; @@ -49,32 +50,43 @@ public Result>> makeHttpRequests(BidRequest request final List> httpRequests = new ArrayList<>(); for (Imp imp : request.getImp()) { + final ExtImpIqzone extImpIqzone; try { - final ExtImpIqzone extImpIqzone = parseImpExt(imp); - final Imp modifiedImp = modifyImp(imp, extImpIqzone); - - httpRequests.add(makeHttpRequest(request, modifiedImp)); + extImpIqzone = parseImpExt(imp); } catch (IllegalArgumentException e) { return Result.withError(BidderError.badInput(e.getMessage())); } + + final Imp modifiedImp = modifyImp(imp, extImpIqzone); + httpRequests.add(makeHttpRequest(request, modifiedImp)); } return Result.withValues(httpRequests); } private ExtImpIqzone parseImpExt(Imp imp) { - return mapper.mapper().convertValue(imp.getExt(), IQZONE_EXT_TYPE_REFERENCE).getBidder(); + try { + return mapper.mapper().convertValue(imp.getExt(), IQZONE_EXT_TYPE_REFERENCE).getBidder(); + } catch (IllegalArgumentException e) { + throw new PreBidException(e.getMessage()); + } } private Imp modifyImp(Imp imp, ExtImpIqzone impExt) { final String placementId = impExt.getPlacementId(); - final ObjectNode modifiedImpExtBidder = mapper.mapper().createObjectNode(); + final String endpointId = impExt.getEndpointId(); + + final boolean isPlacementIdEmpty = StringUtils.isEmpty(placementId); + if (isPlacementIdEmpty && StringUtils.isEmpty(endpointId)) { + return imp; + } - if (StringUtils.isNotEmpty(placementId)) { + final ObjectNode modifiedImpExtBidder = mapper.mapper().createObjectNode(); + if (!isPlacementIdEmpty) { modifiedImpExtBidder.set("placementId", TextNode.valueOf(placementId)); modifiedImpExtBidder.set("type", TextNode.valueOf("publisher")); } else { - modifiedImpExtBidder.set("endpointId", TextNode.valueOf(impExt.getEndpointId())); + modifiedImpExtBidder.set("endpointId", TextNode.valueOf(endpointId)); modifiedImpExtBidder.set("type", TextNode.valueOf("network")); } @@ -84,8 +96,7 @@ private Imp modifyImp(Imp imp, ExtImpIqzone impExt) { } private HttpRequest makeHttpRequest(BidRequest request, Imp imp) { - final BidRequest outgoingRequest = request.toBuilder().imp(List.of(imp)).build(); - + final BidRequest outgoingRequest = request.toBuilder().imp(Collections.singletonList(imp)).build(); return BidderUtil.defaultRequest(outgoingRequest, endpointUrl, mapper); } @@ -93,45 +104,39 @@ private HttpRequest makeHttpRequest(BidRequest request, Imp imp) { public Result> makeBids(BidderCall httpCall, BidRequest bidRequest) { try { final BidResponse bidResponse = mapper.decodeValue(httpCall.getResponse().getBody(), BidResponse.class); - return Result.of(extractBids(httpCall.getRequest().getPayload(), bidResponse), Collections.emptyList()); + return Result.withValues(extractBids(bidResponse)); } catch (DecodeException | PreBidException e) { return Result.withError(BidderError.badServerResponse(e.getMessage())); } } - private List extractBids(BidRequest bidRequest, BidResponse bidResponse) { + private List extractBids(BidResponse bidResponse) { if (bidResponse == null || CollectionUtils.isEmpty(bidResponse.getSeatbid())) { return Collections.emptyList(); } - return bidsFromResponse(bidRequest, bidResponse); - } - - private List bidsFromResponse(BidRequest bidRequest, BidResponse bidResponse) { return bidResponse.getSeatbid().stream() .filter(Objects::nonNull) .map(SeatBid::getBid) .filter(Objects::nonNull) .flatMap(Collection::stream) - .map(bid -> BidderBid.of(bid, getBidType(bid.getImpid(), bidRequest.getImp()), bidResponse.getCur())) + .filter(Objects::nonNull) + .map(bid -> BidderBid.of(bid, getBidMediaType(bid), bidResponse.getCur())) .toList(); } - private static BidType getBidType(String impId, List imps) { - for (Imp imp : imps) { - if (imp.getId().equals(impId)) { - if (imp.getBanner() != null) { - return BidType.banner; - } - if (imp.getVideo() != null) { - return BidType.video; - } - if (imp.getXNative() != null) { - return BidType.xNative; - } - throw new PreBidException("Unknown impression type for ID: \"%s\"".formatted(impId)); - } + private static BidType getBidMediaType(Bid bid) { + final Integer markupType = bid.getMtype(); + if (markupType == null) { + throw new PreBidException("Missing MType for bid: " + bid.getId()); } - throw new PreBidException("Failed to find impression for ID: \"%s\"".formatted(impId)); + + return switch (markupType) { + case 1 -> BidType.banner; + case 2 -> BidType.video; + case 4 -> BidType.xNative; + default -> throw new PreBidException( + "Unable to fetch mediaType " + bid.getMtype() + " in multi-format: " + bid.getImpid()); + }; } } diff --git a/src/test/java/org/prebid/server/bidder/iqzone/IqzoneBidderTest.java b/src/test/java/org/prebid/server/bidder/iqzone/IqzoneBidderTest.java index 94e31912408..5cb1f7f43fa 100644 --- a/src/test/java/org/prebid/server/bidder/iqzone/IqzoneBidderTest.java +++ b/src/test/java/org/prebid/server/bidder/iqzone/IqzoneBidderTest.java @@ -3,11 +3,8 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.TextNode; -import com.iab.openrtb.request.Banner; import com.iab.openrtb.request.BidRequest; import com.iab.openrtb.request.Imp; -import com.iab.openrtb.request.Native; -import com.iab.openrtb.request.Video; import com.iab.openrtb.response.Bid; import com.iab.openrtb.response.BidResponse; import com.iab.openrtb.response.SeatBid; @@ -116,6 +113,26 @@ public void makeHttpRequestsShouldModifyImpExtWithEndpointIdAndTypeIfEndpointIdP .set("bidder", mapper.valueToTree(expectedImpExtBidder))); } + @Test + public void makeHttpRequestsShouldNotModifyImpExt() { + // given + final BidRequest bidRequest = givenBidRequest(impBuilder -> + impBuilder.ext(mapper.valueToTree(ExtPrebid.of(null, ExtImpIqzone.of(null, null))))); + + // when + final Result>> result = target.makeHttpRequests(bidRequest); + + // then + assertThat(result.getErrors()).isEmpty(); + assertThat(result.getValue()) + .extracting(HttpRequest::getPayload) + .flatExtracting(BidRequest::getImp) + .extracting(Imp::getExt) + .extracting(ext -> ext.get("bidder")) + .map(JsonNode::isEmpty) + .containsExactly(true); + } + @Test public void makeBidsShouldReturnErrorIfResponseBodyCouldNotBeParsed() { // given @@ -163,9 +180,8 @@ public void makeBidsShouldReturnEmptyResponseIfBidResponseSeatBidIsNull() throws @Test public void makeBidsShouldCorrectlyProceedWithVideo() throws JsonProcessingException { // given - final BidderCall httpCall = givenHttpCall(givenBidRequest(impBuilder -> impBuilder - .id("someId").video(Video.builder().build())), - mapper.writeValueAsString(givenBidResponse(bidBuilder -> bidBuilder.impid("someId")))); + final BidderCall httpCall = givenHttpCall(givenBidRequest(), + mapper.writeValueAsString(givenBidResponse(bidBuilder -> bidBuilder.mtype(2)))); // when final Result> result = target.makeBids(httpCall, null); @@ -179,9 +195,8 @@ public void makeBidsShouldCorrectlyProceedWithVideo() throws JsonProcessingExcep @Test public void makeBidsShouldCorrectlyProceedWithNative() throws JsonProcessingException { // given - final BidderCall httpCall = givenHttpCall(givenBidRequest(impBuilder -> impBuilder - .id("someId").xNative(Native.builder().build())), - mapper.writeValueAsString(givenBidResponse(bidBuilder -> bidBuilder.impid("someId")))); + final BidderCall httpCall = givenHttpCall(givenBidRequest(), + mapper.writeValueAsString(givenBidResponse(bidBuilder -> bidBuilder.mtype(4)))); // when final Result> result = target.makeBids(httpCall, null); @@ -195,9 +210,8 @@ public void makeBidsShouldCorrectlyProceedWithNative() throws JsonProcessingExce @Test public void makeBidsShouldCorrectlyProceedWithBanner() throws JsonProcessingException { // given - final BidderCall httpCall = givenHttpCall(givenBidRequest(impBuilder -> impBuilder - .id("someId").banner(Banner.builder().build())), - mapper.writeValueAsString(givenBidResponse(bidBuilder -> bidBuilder.impid("someId")))); + final BidderCall httpCall = givenHttpCall(givenBidRequest(), + mapper.writeValueAsString(givenBidResponse(bidBuilder -> bidBuilder.mtype(1)))); // when final Result> result = target.makeBids(httpCall, null); @@ -211,9 +225,8 @@ public void makeBidsShouldCorrectlyProceedWithBanner() throws JsonProcessingExce @Test public void makeBidsShouldReturnErrorIfImpIdDoesNotMatchImpIdInBid() throws JsonProcessingException { // given - final BidderCall httpCall = givenHttpCall(givenBidRequest(impBuilder -> impBuilder - .id("someIdThatIsDifferentFromIDInBid").xNative(Native.builder().build())), - mapper.writeValueAsString(givenBidResponse(bidBuilder -> bidBuilder.impid("someId")))); + final BidderCall httpCall = givenHttpCall(givenBidRequest(identity()), + mapper.writeValueAsString(givenBidResponse(identity()))); // when final Result> result = target.makeBids(httpCall, null); @@ -222,17 +235,16 @@ public void makeBidsShouldReturnErrorIfImpIdDoesNotMatchImpIdInBid() throws Json assertThat(result.getErrors()).hasSize(1) .allSatisfy(error -> { assertThat(error.getType()).isEqualTo(BidderError.Type.bad_server_response); - assertThat(error.getMessage()).startsWith("Failed to find impression for ID:"); + assertThat(error.getMessage()).startsWith("Missing MType for bid: null"); }); assertThat(result.getValue()).isEmpty(); } @Test - public void makeBidsShouldReturnErrorWhenMissingType() throws JsonProcessingException { + public void makeBidsShouldReturnErrorWhenMissingMType() throws JsonProcessingException { // given - final BidderCall httpCall = givenHttpCall( - givenBidRequest(impBuilder -> impBuilder.id("someId")), - mapper.writeValueAsString(givenBidResponse(bidBuilder -> bidBuilder.impid("someId")))); + final BidderCall httpCall = givenHttpCall(givenBidRequest(), + mapper.writeValueAsString(givenBidResponse(bidBuilder -> bidBuilder.mtype(null)))); // when final Result> result = target.makeBids(httpCall, null); @@ -241,7 +253,7 @@ public void makeBidsShouldReturnErrorWhenMissingType() throws JsonProcessingExce assertThat(result.getErrors()).hasSize(1) .allSatisfy(error -> { assertThat(error.getType()).isEqualTo(BidderError.Type.bad_server_response); - assertThat(error.getMessage()).startsWith("Unknown impression type for ID"); + assertThat(error.getMessage()).startsWith("Missing MType for bid: null"); }); } diff --git a/src/test/resources/org/prebid/server/it/openrtb2/iqzone/test-auction-iqzone-response.json b/src/test/resources/org/prebid/server/it/openrtb2/iqzone/test-auction-iqzone-response.json index ec5eadf42d0..4a6e48aac57 100644 --- a/src/test/resources/org/prebid/server/it/openrtb2/iqzone/test-auction-iqzone-response.json +++ b/src/test/resources/org/prebid/server/it/openrtb2/iqzone/test-auction-iqzone-response.json @@ -8,6 +8,7 @@ "impid": "imp_id", "price": 3.33, "adm": "adm001", + "mtype": 1, "adid": "adid001", "cid": "cid001", "crid": "crid001", diff --git a/src/test/resources/org/prebid/server/it/openrtb2/iqzone/test-iqzone-bid-response.json b/src/test/resources/org/prebid/server/it/openrtb2/iqzone/test-iqzone-bid-response.json index 31a6f4419e3..26af1edc5a3 100644 --- a/src/test/resources/org/prebid/server/it/openrtb2/iqzone/test-iqzone-bid-response.json +++ b/src/test/resources/org/prebid/server/it/openrtb2/iqzone/test-iqzone-bid-response.json @@ -7,6 +7,7 @@ "id": "bid_id", "impid": "imp_id", "price": 3.33, + "mtype": 1, "adid": "adid001", "crid": "crid001", "cid": "cid001", From 26100b0d2e59940dfc0cbbefe9591cb0b260a2c0 Mon Sep 17 00:00:00 2001 From: Serhii Kolomiiets <46626223+VeryExtraordinaryUsername@users.noreply.github.com> Date: Mon, 25 Mar 2024 16:49:50 +0200 Subject: [PATCH 010/320] Bizzclick: host parameter update (#2962) --- .../bidder/bizzclick/BizzclickBidder.java | 13 +++- .../request/bizzclick/ExtImpBizzclick.java | 6 ++ .../resources/bidder-config/bizzclick.yaml | 2 +- .../bidder/bizzclick/BizzclickBidderTest.java | 67 +++++++++++++++---- .../org/prebid/server/it/BizzclickTest.java | 4 ++ .../test-auction-bizzclick-request.json | 1 + .../server/it/test-application.properties | 2 +- 7 files changed, 76 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/prebid/server/bidder/bizzclick/BizzclickBidder.java b/src/main/java/org/prebid/server/bidder/bizzclick/BizzclickBidder.java index cf0278d4219..19e077624c6 100644 --- a/src/main/java/org/prebid/server/bidder/bizzclick/BizzclickBidder.java +++ b/src/main/java/org/prebid/server/bidder/bizzclick/BizzclickBidder.java @@ -9,6 +9,7 @@ import io.vertx.core.MultiMap; import io.vertx.core.http.HttpMethod; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.prebid.server.bidder.Bidder; import org.prebid.server.bidder.model.BidderBid; import org.prebid.server.bidder.model.BidderCall; @@ -33,8 +34,10 @@ public class BizzclickBidder implements Bidder { private static final TypeReference> BIZZCLICK_EXT_TYPE_REFERENCE = new TypeReference<>() { }; - private static final String URL_SOURCE_ID_MACRO = "{{.SourceId}}"; - private static final String URL_ACCOUNT_ID_MACRO = "{{.AccountID}}"; + private static final String DEFAULT_HOST = "us-e-node1"; + private static final String URL_HOST_MACRO = "{{Host}}"; + private static final String URL_SOURCE_ID_MACRO = "{{SourceId}}"; + private static final String URL_ACCOUNT_ID_MACRO = "{{AccountID}}"; private static final String DEFAULT_CURRENCY = "USD"; private final String endpointUrl; @@ -100,7 +103,11 @@ private static MultiMap headers(Device device) { } private String buildEndpointUrl(ExtImpBizzclick ext) { - return endpointUrl.replace(URL_SOURCE_ID_MACRO, HttpUtil.encodeUrl(ext.getPlacementId())) + final String host = StringUtils.isBlank(ext.getHost()) ? DEFAULT_HOST : ext.getHost(); + final String sourceId = StringUtils.isBlank(ext.getSourceId()) ? ext.getPlacementId() : ext.getSourceId(); + return endpointUrl + .replace(URL_HOST_MACRO, HttpUtil.encodeUrl(host)) + .replace(URL_SOURCE_ID_MACRO, HttpUtil.encodeUrl(sourceId)) .replace(URL_ACCOUNT_ID_MACRO, HttpUtil.encodeUrl(ext.getAccountId())); } diff --git a/src/main/java/org/prebid/server/proto/openrtb/ext/request/bizzclick/ExtImpBizzclick.java b/src/main/java/org/prebid/server/proto/openrtb/ext/request/bizzclick/ExtImpBizzclick.java index 588137321e3..dae1cd49c62 100644 --- a/src/main/java/org/prebid/server/proto/openrtb/ext/request/bizzclick/ExtImpBizzclick.java +++ b/src/main/java/org/prebid/server/proto/openrtb/ext/request/bizzclick/ExtImpBizzclick.java @@ -6,9 +6,15 @@ @Value(staticConstructor = "of") public class ExtImpBizzclick { + @JsonProperty("host") + String host; + @JsonProperty("accountId") String accountId; @JsonProperty("placementId") String placementId; + + @JsonProperty("sourceId") + String sourceId; } diff --git a/src/main/resources/bidder-config/bizzclick.yaml b/src/main/resources/bidder-config/bizzclick.yaml index afb208477ae..f5037c1014a 100644 --- a/src/main/resources/bidder-config/bizzclick.yaml +++ b/src/main/resources/bidder-config/bizzclick.yaml @@ -1,6 +1,6 @@ adapters: bizzclick: - endpoint: http://us-e-node1.bizzclick.com/bid?rtb_seat_id={{.SourceId}}&secret_key={{.AccountID}} + endpoint: http://{{Host}}.bizzclick.com/bid?rtb_seat_id={{SourceId}}&secret_key={{AccountID}} meta-info: maintainer-email: support@bizzclick.com app-media-types: diff --git a/src/test/java/org/prebid/server/bidder/bizzclick/BizzclickBidderTest.java b/src/test/java/org/prebid/server/bidder/bizzclick/BizzclickBidderTest.java index 45e0b9b968c..18d5a44a250 100644 --- a/src/test/java/org/prebid/server/bidder/bizzclick/BizzclickBidderTest.java +++ b/src/test/java/org/prebid/server/bidder/bizzclick/BizzclickBidderTest.java @@ -36,7 +36,11 @@ public class BizzclickBidderTest extends VertxTest { - private static final String ENDPOINT = "https://test.domain.dm/uri?source={{.SourceId}}&account={{.AccountID}}"; + private static final String ENDPOINT = "https://{{Host}}/uri?source={{SourceId}}&account={{AccountID}}"; + private static final String DEFAULT_HOST = "host"; + private static final String DEFAULT_ACCOUNT_ID = "accountId"; + private static final String DEFAULT_SOURCE_ID = "sourceId"; + private static final String DEFAULT_PLACEMENT_ID = "placementId"; private final BizzclickBidder target = new BizzclickBidder(ENDPOINT, jacksonMapper); @@ -65,7 +69,7 @@ public void makeHttpRequestsShouldReturnErrorWhenImpExtCouldNotBeParsed() { public void makeHttpRequestsShouldRelyOnlyOnFirstImpExt() { // given final BidRequest bidRequest = givenBidRequest( - givenImp("accountId", "placementId"), + givenImp(), givenImp(imp -> imp.ext(mapper.valueToTree(ExtPrebid.of(null, mapper.createArrayNode()))))); // when @@ -83,8 +87,8 @@ public void makeHttpRequestsShouldRelyOnlyOnFirstImpExt() { public void makeHttpRequestsShouldRemoveExtFromEachImp() { // given final BidRequest bidRequest = givenBidRequest( - givenImp("accountId1", "placementId1"), - givenImp("accountId2", "placementId2")); + givenImp("host", "accountId1", "placementId1", "sourceId1"), + givenImp("host", "accountId2", "placementId2", "sourceId2")); // when final Result>> result = target.makeHttpRequests(bidRequest); @@ -101,7 +105,7 @@ public void makeHttpRequestsShouldRemoveExtFromEachImp() { @Test public void makeHttpRequestsShouldCreateRequestWithXOpenRtbVersionHeader() { // given - final BidRequest bidRequest = givenBidRequest(givenImp("accountId", "placementId")); + final BidRequest bidRequest = givenBidRequest(givenImp()); // when final Result>> result = target.makeHttpRequests(bidRequest); @@ -120,7 +124,7 @@ public void makeHttpRequestsShouldCreateRequestWithUserAgentHeaderIfDeviceUaPres // given final BidRequest bidRequest = givenBidRequest( device -> device.ua("ua"), - givenImp("accountId", "placementId")); + givenImp()); // when final Result>> result = target.makeHttpRequests(bidRequest); @@ -139,7 +143,7 @@ public void makeHttpRequestsShouldCreateRequestWithXForwardedForHeaderIfDeviceIp // given final BidRequest bidRequest = givenBidRequest( device -> device.ipv6("ipv6"), - givenImp("accountId", "placementId")); + givenImp()); // when final Result>> result = target.makeHttpRequests(bidRequest); @@ -158,7 +162,7 @@ public void makeHttpRequestsShouldCreateRequestWithXForwardedForHeaderIfDeviceIp // given final BidRequest bidRequest = givenBidRequest( device -> device.ip("ip"), - givenImp("accountId", "placementId")); + givenImp()); // when final Result>> result = target.makeHttpRequests(bidRequest); @@ -177,7 +181,7 @@ public void makeHttpRequestsShouldCreateRequestWithXForwardedForHeaderWithDevice // given final BidRequest bidRequest = givenBidRequest( device -> device.ip("ip").ipv6("ipv6"), - givenImp("accountId", "placementId")); + givenImp()); // when final Result>> result = target.makeHttpRequests(bidRequest); @@ -194,7 +198,7 @@ public void makeHttpRequestsShouldCreateRequestWithXForwardedForHeaderWithDevice @Test public void makeHttpRequestsShouldCreateSingleRequestWithExpectedUri() { // given - final BidRequest bidRequest = givenBidRequest(givenImp("account id", "placement id")); + final BidRequest bidRequest = givenBidRequest(givenImp()); // when final Result>> result = target.makeHttpRequests(bidRequest); @@ -202,7 +206,33 @@ public void makeHttpRequestsShouldCreateSingleRequestWithExpectedUri() { // then assertThat(result.getValue()) .extracting(HttpRequest::getUri) - .containsExactly("https://test.domain.dm/uri?source=placement+id&account=account+id"); + .containsExactly( + String.format("https://%s/uri?source=%s&account=%s", + DEFAULT_HOST, + DEFAULT_SOURCE_ID, + DEFAULT_ACCOUNT_ID)); + assertThat(result.getErrors()).isEmpty(); + } + + @Test + public void makeHttpRequestsShouldCreateSingleRequestWithExpectedAlternativeUri() { + // given + final String expectedDefaultHost = "us-e-node1"; + final BidRequest bidRequest = givenBidRequest( + givenImp(expectedDefaultHost, DEFAULT_ACCOUNT_ID, DEFAULT_PLACEMENT_ID, null) + ); + + // when + final Result>> result = target.makeHttpRequests(bidRequest); + + // then + assertThat(result.getValue()) + .extracting(HttpRequest::getUri) + .containsExactly( + String.format("https://%s/uri?source=%s&account=%s", + expectedDefaultHost, + DEFAULT_PLACEMENT_ID, + DEFAULT_ACCOUNT_ID)); assertThat(result.getErrors()).isEmpty(); } @@ -210,7 +240,7 @@ public void makeHttpRequestsShouldCreateSingleRequestWithExpectedUri() { public void makeHttpRequestsShouldCreateSingleRequest() { // given final BidRequest bidRequest = givenBidRequest( - givenImp("accountId", "placementId"), + givenImp(), givenImp(identity())); // when @@ -417,8 +447,17 @@ private Imp givenImp(UnaryOperator impCustomizer) { return impCustomizer.apply(Imp.builder()).build(); } - private Imp givenImp(String accountId, String placementId) { - final ExtPrebid ext = ExtPrebid.of(null, ExtImpBizzclick.of(accountId, placementId)); + private Imp givenImp() { + final ExtPrebid ext = ExtPrebid.of(null, ExtImpBizzclick.of( + DEFAULT_HOST, DEFAULT_ACCOUNT_ID, DEFAULT_PLACEMENT_ID, DEFAULT_SOURCE_ID + )); + return givenImp(imp -> imp.ext(mapper.valueToTree(ext))); + } + + private Imp givenImp(String host, String accountId, String placementId, String sourceId) { + final ExtPrebid ext = ExtPrebid.of( + null, ExtImpBizzclick.of(host, accountId, placementId, sourceId) + ); return givenImp(imp -> imp.ext(mapper.valueToTree(ext))); } diff --git a/src/test/java/org/prebid/server/it/BizzclickTest.java b/src/test/java/org/prebid/server/it/BizzclickTest.java index 665ddfc75df..4d98d48a52f 100644 --- a/src/test/java/org/prebid/server/it/BizzclickTest.java +++ b/src/test/java/org/prebid/server/it/BizzclickTest.java @@ -10,6 +10,7 @@ import java.io.IOException; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; import static com.github.tomakehurst.wiremock.client.WireMock.equalToJson; import static com.github.tomakehurst.wiremock.client.WireMock.post; import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; @@ -22,6 +23,9 @@ public class BizzclickTest extends IntegrationTest { public void openrtb2AuctionShouldRespondWithBidsFromBizzclick() throws IOException, JSONException { // given WIRE_MOCK_RULE.stubFor(post(urlPathEqualTo("/bizzclick-exchange")) + .withQueryParam("host", equalTo("host")) + .withQueryParam("source", equalTo("placementId")) + .withQueryParam("account", equalTo("accountId")) .withRequestBody(equalToJson(jsonFrom("openrtb2/bizzclick/test-bizzclick-bid-request.json"))) .willReturn(aResponse().withBody(jsonFrom("openrtb2/bizzclick/test-bizzclick-bid-response.json")))); diff --git a/src/test/resources/org/prebid/server/it/openrtb2/bizzclick/test-auction-bizzclick-request.json b/src/test/resources/org/prebid/server/it/openrtb2/bizzclick/test-auction-bizzclick-request.json index 2fae1241d83..bfbeccf737f 100644 --- a/src/test/resources/org/prebid/server/it/openrtb2/bizzclick/test-auction-bizzclick-request.json +++ b/src/test/resources/org/prebid/server/it/openrtb2/bizzclick/test-auction-bizzclick-request.json @@ -9,6 +9,7 @@ }, "ext": { "bizzclick": { + "host": "host", "accountId": "accountId", "placementId": "placementId" } diff --git a/src/test/resources/org/prebid/server/it/test-application.properties b/src/test/resources/org/prebid/server/it/test-application.properties index 1f0a9689ece..856a4f06c6b 100644 --- a/src/test/resources/org/prebid/server/it/test-application.properties +++ b/src/test/resources/org/prebid/server/it/test-application.properties @@ -112,7 +112,7 @@ adapters.bidscube.endpoint=http://localhost:8090/bidscube-exchange adapters.bidstack.enabled=true adapters.bidstack.endpoint=http://localhost:8090/bidstack-exchange adapters.bizzclick.enabled=true -adapters.bizzclick.endpoint=http://localhost:8090/bizzclick-exchange +adapters.bizzclick.endpoint=http://localhost:8090/bizzclick-exchange?host={{Host}}&source={{SourceId}}&account={{AccountID}} adapters.bliink.enabled=true adapters.bliink.endpoint=http://localhost:8090/bliink-exchange adapters.bluesea.enabled=true From 044205dd8757a373f5fdc601d5af6ab6208afce3 Mon Sep 17 00:00:00 2001 From: Anton Babak <76536883+AntoxaAntoxic@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:53:47 +0100 Subject: [PATCH 011/320] MediaGrid: Add GPP Macros (#3042) --- src/main/resources/bidder-config/grid.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/resources/bidder-config/grid.yaml b/src/main/resources/bidder-config/grid.yaml index b9365394326..e735a5159f7 100644 --- a/src/main/resources/bidder-config/grid.yaml +++ b/src/main/resources/bidder-config/grid.yaml @@ -6,14 +6,16 @@ adapters: app-media-types: - banner - video + - native site-media-types: - banner - video + - native supported-vendors: vendor-id: 686 usersync: cookie-family-name: grid redirect: - url: https://x.bidswitch.net/check_uuid/{{redirect_url}} + url: https://x.bidswitch.net/check_uuid/{{redirect_url}}?gdpr={{gdpr}}&gdpr_consent={{gdpr_consent}}&gpp={{gpp}}&gpp_sid={{gpp_sid}}&us_privacy={{us_privacy}} support-cors: false uid-macro: '${BSW_UUID}' From f353f4dad400b3645115757409994c2de76bdf67 Mon Sep 17 00:00:00 2001 From: Piotr Jaworski <109736938+piotrj-rtbh@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:55:37 +0100 Subject: [PATCH 012/320] RTB House: Regional endpoints (#3023) --- src/main/resources/bidder-config/rtbhouse.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/resources/bidder-config/rtbhouse.yaml b/src/main/resources/bidder-config/rtbhouse.yaml index fd2f73da770..774ed55715c 100644 --- a/src/main/resources/bidder-config/rtbhouse.yaml +++ b/src/main/resources/bidder-config/rtbhouse.yaml @@ -1,6 +1,17 @@ adapters: rtbhouse: - endpoint: https://prebidserver-s2s-ams.creativecdn.com/bidder/prebidserver/bids + # Contact prebid@rtbhouse.com to ask about enabling a connection to the bidder. + # Please configure the following endpoints for your datacenter + # EMEA + endpoint: http://prebidserver-s2s-ams.creativecdn.com/bidder/prebidserver/bids + # US East + # endpoint: http://prebidserver-s2s-ash.creativecdn.com/bidder/prebidserver/bids + # US West + # endpoint: http://prebidserver-s2s-phx.creativecdn.com/bidder/prebidserver/bids + # APAC + # endpoint: http://prebidserver-s2s-sin.creativecdn.com/bidder/prebidserver/bids + geoscope: + - global endpoint-compression: gzip meta-info: maintainer-email: prebid@rtbhouse.com From 6c933d4ca4deb12a0a7c6fb074a5f52246de1783 Mon Sep 17 00:00:00 2001 From: osulzhenko <125548596+osulzhenko@users.noreply.github.com> Date: Tue, 26 Mar 2024 14:15:47 +0200 Subject: [PATCH 013/320] Remove unused allure report (#3073) --- .gitignore | 1 - pom.xml | 25 -- .../service/PrebidServerService.groovy | 26 -- .../functional/tests/BidderParamsSpec.groovy | 2 - .../functional/util/AllureReporter.groovy | 314 ------------------ src/test/resources/META-INF/aop-ajc.xml | 7 - ...amework.runtime.extension.IGlobalExtension | 1 - 7 files changed, 376 deletions(-) delete mode 100644 src/test/groovy/org/prebid/server/functional/util/AllureReporter.groovy delete mode 100644 src/test/resources/META-INF/aop-ajc.xml diff --git a/.gitignore b/.gitignore index 3c057d9bda4..5f0817bd269 100644 --- a/.gitignore +++ b/.gitignore @@ -13,5 +13,4 @@ target/ .DS_Store -.allure/ src/main/proto/ diff --git a/pom.xml b/pom.xml index 4364981ab09..a7190cb3a90 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,6 @@ 3.0.14 1.17.4 5.14.0 - 2.19.0 1.9.9.1 1.12.14 @@ -84,7 +83,6 @@ ${maven-surefire-plugin.version} 0.40.2 1.13.1 - 2.10.0 false true false @@ -592,12 +590,6 @@ ${mockserver.version} test - - io.qameta.allure - allure-java-commons - ${allure.version} - test - @@ -917,7 +909,6 @@ -javaagent:"${settings.localRepository}/org/aspectj/aspectjweaver/${aspectj.version}/aspectjweaver-${aspectj.version}.jar" - target/allure-results ${mockserver.version} ${project.version} 2 @@ -961,11 +952,6 @@ - - io.qameta.allure - allure-maven - ${allure-maven.version} - org.apache.maven.plugins maven-compiler-plugin @@ -990,17 +976,6 @@ - - org.jacoco - jacoco-maven-plugin - - - - report - - - - diff --git a/src/test/groovy/org/prebid/server/functional/service/PrebidServerService.groovy b/src/test/groovy/org/prebid/server/functional/service/PrebidServerService.groovy index 048c81c0c27..8a1eceb6f87 100644 --- a/src/test/groovy/org/prebid/server/functional/service/PrebidServerService.groovy +++ b/src/test/groovy/org/prebid/server/functional/service/PrebidServerService.groovy @@ -1,7 +1,6 @@ package org.prebid.server.functional.service import com.fasterxml.jackson.core.type.TypeReference -import io.qameta.allure.Step import io.restassured.authentication.AuthenticationScheme import io.restassured.authentication.BasicAuthScheme import io.restassured.builder.RequestSpecBuilder @@ -82,7 +81,6 @@ class PrebidServerService implements ObjectMapperWrapper { prometheusRequestSpecification = buildAndGetRequestSpecification(pbsContainer.prometheusRootUri, authenticationScheme) } - @Step("[POST] /openrtb2/auction") BidResponse sendAuctionRequest(BidRequest bidRequest, Map headers = [:]) { def response = postAuction(bidRequest, headers) @@ -90,7 +88,6 @@ class PrebidServerService implements ObjectMapperWrapper { decode(response.body.asString(), BidResponse) } - @Step("[POST RAW] /openrtb2/auction") RawAuctionResponse sendAuctionRequestRaw(BidRequest bidRequest, Map headers = [:]) { def response = postAuction(bidRequest, headers) @@ -100,7 +97,6 @@ class PrebidServerService implements ObjectMapperWrapper { } } - @Step("[GET] /openrtb2/amp") AmpResponse sendAmpRequest(AmpRequest ampRequest, Map headers = [:]) { def response = getAmp(ampRequest, headers) @@ -108,7 +104,6 @@ class PrebidServerService implements ObjectMapperWrapper { decode(response.body.asString(), AmpResponse) } - @Step("[GET RAW] /openrtb2/amp") RawAmpResponse sendAmpRequestRaw(AmpRequest ampRequest, Map headers = [:]) { def response = getAmp(ampRequest, headers) @@ -118,7 +113,6 @@ class PrebidServerService implements ObjectMapperWrapper { } } - @Step("[POST] /cookie_sync without cookie") CookieSyncResponse sendCookieSyncRequest(CookieSyncRequest request) { def response = postCookieSync(request) @@ -126,7 +120,6 @@ class PrebidServerService implements ObjectMapperWrapper { decode(response.body.asString(), CookieSyncResponse) } - @Step("[POST] /cookie_sync with headers") CookieSyncResponse sendCookieSyncRequest(CookieSyncRequest request, Map headers) { def response = postCookieSync(request, null, headers) @@ -134,7 +127,6 @@ class PrebidServerService implements ObjectMapperWrapper { decode(response.body.asString(), CookieSyncResponse) } - @Step("[POST] /cookie_sync with uids cookie") CookieSyncResponse sendCookieSyncRequest(CookieSyncRequest request, UidsCookie uidsCookie) { def response = postCookieSync(request, uidsCookie) @@ -142,7 +134,6 @@ class PrebidServerService implements ObjectMapperWrapper { decode(response.body.asString(), CookieSyncResponse) } - @Step("[POST] /cookie_sync with uids and additional cookies") CookieSyncResponse sendCookieSyncRequest(CookieSyncRequest request, UidsCookie uidsCookie, Map additionalCookies) { @@ -152,7 +143,6 @@ class PrebidServerService implements ObjectMapperWrapper { decode(response.body.asString(), CookieSyncResponse) } - @Step("[POST RAW] /cookie_sync with uids cookies") RawCookieSyncResponse sendCookieSyncRequestRaw(CookieSyncRequest request, UidsCookie uidsCookie) { def response = postCookieSync(request, uidsCookie) @@ -162,7 +152,6 @@ class PrebidServerService implements ObjectMapperWrapper { } } - @Step("[POST RAW] /cookie_sync with uids and additional cookies") RawCookieSyncResponse sendCookieSyncRequestRaw(CookieSyncRequest request, UidsCookie uidsCookie, Map additionalCookies) { @@ -174,7 +163,6 @@ class PrebidServerService implements ObjectMapperWrapper { } } - @Step("[GET] /setuid") SetuidResponse sendSetUidRequest(SetuidRequest request, UidsCookie uidsCookie, Map header = [:]) { def uidsCookieAsJson = encode(uidsCookie) def uidsCookieAsEncodedJson = Base64.urlEncoder.encodeToString(uidsCookieAsJson.bytes) @@ -192,7 +180,6 @@ class PrebidServerService implements ObjectMapperWrapper { setuidResponse } - @Step("[GET] /getuids") GetuidResponse sendGetUidRequest(UidsCookie uidsCookie) { def uidsCookieAsJson = encode(uidsCookie) def uidsCookieAsEncodedJson = Base64.urlEncoder.encodeToString(uidsCookieAsJson.bytes) @@ -204,7 +191,6 @@ class PrebidServerService implements ObjectMapperWrapper { decode(response.body.asString(), GetuidResponse) } - @Step("[GET] /event") byte[] sendEventRequest(EventRequest eventRequest, Map headers = [:]) { def response = given(requestSpecification).headers(headers) .queryParams(toMap(eventRequest)) @@ -214,7 +200,6 @@ class PrebidServerService implements ObjectMapperWrapper { response.body.asByteArray() } - @Step("[POST] /vtrack") PrebidCacheResponse sendVtrackRequest(VtrackRequest request, String account) { def response = given(requestSpecification).queryParam("a", account) .body(request) @@ -224,7 +209,6 @@ class PrebidServerService implements ObjectMapperWrapper { decode(response.body.asString(), PrebidCacheResponse) } - @Step("[GET] /status") StatusResponse sendStatusRequest() { def response = given(requestSpecification).get(STATUS_ENDPOINT) @@ -232,7 +216,6 @@ class PrebidServerService implements ObjectMapperWrapper { decode(response.body.asString(), StatusResponse) } - @Step("[GET] /info/bidders") String sendInfoBiddersRequest() { def response = given(requestSpecification).get(INFO_BIDDERS_ENDPOINT) @@ -240,7 +223,6 @@ class PrebidServerService implements ObjectMapperWrapper { response.body().asString() } - @Step("[GET] /info/bidders with params={queryParam}") List sendInfoBiddersRequest(Map queryParam) { def response = given(requestSpecification).queryParams(queryParam).get(INFO_BIDDERS_ENDPOINT) @@ -248,7 +230,6 @@ class PrebidServerService implements ObjectMapperWrapper { decode(response.asString(), new TypeReference>() {}) } - @Step("[GET] /info/bidders/{bidderName}") BidderInfoResponse sendBidderInfoRequest(BidderName bidderName) { def response = given(requestSpecification).get("$INFO_BIDDERS_ENDPOINT/$bidderName.value") @@ -257,7 +238,6 @@ class PrebidServerService implements ObjectMapperWrapper { decode(response.body.asString(), BidderInfoResponse) } - @Step("[GET] /bidders/params") BiddersParamsResponse sendBiddersParamsRequest() { def response = given(requestSpecification).get(BIDDERS_PARAMS_ENDPOINT) @@ -265,7 +245,6 @@ class PrebidServerService implements ObjectMapperWrapper { decode(response.body.asString(), BiddersParamsResponse) } - @Step("[GET] /currency/rates") CurrencyRatesResponse sendCurrencyRatesRequest() { def response = given(adminRequestSpecification).get(CURRENCY_RATES_ENDPOINT) @@ -273,7 +252,6 @@ class PrebidServerService implements ObjectMapperWrapper { decode(response.body.asString(), CurrencyRatesResponse) } - @Step("[GET] /logging/httpinteraction") String sendLoggingHttpInteractionRequest(HttpInteractionRequest httpInteractionRequest) { def response = given(adminRequestSpecification).queryParams(toMap(httpInteractionRequest)) .get(HTTP_INTERACTION_ENDPOINT) @@ -282,7 +260,6 @@ class PrebidServerService implements ObjectMapperWrapper { response.body().asString() } - @Step("[GET] /collected-metrics") Map sendCollectedMetricsRequest() { def response = given(adminRequestSpecification).get(COLLECTED_METRICS_ENDPOINT) @@ -290,7 +267,6 @@ class PrebidServerService implements ObjectMapperWrapper { decode(response.asString(), new TypeReference>() {}) } - @Step("[GET] /pbs-admin/force-deals-update") void sendForceDealsUpdateRequest(ForceDealsUpdateRequest forceDealsUpdateRequest) { def response = given(adminRequestSpecification).queryParams(toMap(forceDealsUpdateRequest)) .get(FORCE_DEALS_UPDATE_ENDPOINT) @@ -298,7 +274,6 @@ class PrebidServerService implements ObjectMapperWrapper { checkResponseStatusCode(response, 204) } - @Step("[GET] /pbs-admin/lineitem-status") LineItemStatusReport sendLineItemStatusRequest(String lineItemId) { def request = given(adminRequestSpecification) if (lineItemId != null) { @@ -311,7 +286,6 @@ class PrebidServerService implements ObjectMapperWrapper { decode(response.body.asString(), LineItemStatusReport) } - @Step("[GET] /metrics") String sendPrometheusMetricsRequest() { def response = given(prometheusRequestSpecification).get(PROMETHEUS_METRICS_ENDPOINT) diff --git a/src/test/groovy/org/prebid/server/functional/tests/BidderParamsSpec.groovy b/src/test/groovy/org/prebid/server/functional/tests/BidderParamsSpec.groovy index 14467df1967..0e5328dc3ae 100644 --- a/src/test/groovy/org/prebid/server/functional/tests/BidderParamsSpec.groovy +++ b/src/test/groovy/org/prebid/server/functional/tests/BidderParamsSpec.groovy @@ -1,6 +1,5 @@ package org.prebid.server.functional.tests -import io.qameta.allure.Issue import org.prebid.server.functional.model.bidder.Generic import org.prebid.server.functional.model.db.Account import org.prebid.server.functional.model.db.StoredImp @@ -271,7 +270,6 @@ class BidderParamsSpec extends BaseSpec { } // TODO: create same test for enabled circuit breaker - @Issue("https://github.com/prebid/prebid-server-java/issues/1478") def "PBS should emit warning when bidder endpoint is invalid"() { given: "Pbs config" def pbsService = pbsServiceFactory.getService(["adapters.generic.enabled" : "true", diff --git a/src/test/groovy/org/prebid/server/functional/util/AllureReporter.groovy b/src/test/groovy/org/prebid/server/functional/util/AllureReporter.groovy deleted file mode 100644 index 6ada5fe95c9..00000000000 --- a/src/test/groovy/org/prebid/server/functional/util/AllureReporter.groovy +++ /dev/null @@ -1,314 +0,0 @@ -package org.prebid.server.functional.util - -import io.qameta.allure.Allure -import io.qameta.allure.AllureLifecycle -import io.qameta.allure.Description -import io.qameta.allure.Flaky -import io.qameta.allure.Muted -import io.qameta.allure.model.Label -import io.qameta.allure.model.Link -import io.qameta.allure.model.Parameter -import io.qameta.allure.model.Status -import io.qameta.allure.model.StatusDetails -import io.qameta.allure.model.TestResult -import io.qameta.allure.util.AnnotationUtils -import org.spockframework.runtime.AbstractRunListener -import org.spockframework.runtime.extension.IGlobalExtension -import org.spockframework.runtime.extension.builtin.UnrollIterationNameProvider -import org.spockframework.runtime.model.ErrorInfo -import org.spockframework.runtime.model.FeatureInfo -import org.spockframework.runtime.model.IterationInfo -import org.spockframework.runtime.model.MethodInfo -import org.spockframework.runtime.model.SpecInfo - -import java.lang.annotation.Annotation -import java.lang.annotation.Repeatable -import java.lang.reflect.Method -import java.security.MessageDigest -import java.security.NoSuchAlgorithmException -import java.util.stream.Collectors -import java.util.stream.Stream - -import static io.qameta.allure.util.ResultsUtils.createFrameworkLabel -import static io.qameta.allure.util.ResultsUtils.createHostLabel -import static io.qameta.allure.util.ResultsUtils.createLanguageLabel -import static io.qameta.allure.util.ResultsUtils.createPackageLabel -import static io.qameta.allure.util.ResultsUtils.createParameter -import static io.qameta.allure.util.ResultsUtils.createParentSuiteLabel -import static io.qameta.allure.util.ResultsUtils.createSubSuiteLabel -import static io.qameta.allure.util.ResultsUtils.createSuiteLabel -import static io.qameta.allure.util.ResultsUtils.createTestClassLabel -import static io.qameta.allure.util.ResultsUtils.createTestMethodLabel -import static io.qameta.allure.util.ResultsUtils.createThreadLabel -import static io.qameta.allure.util.ResultsUtils.firstNonEmpty -import static io.qameta.allure.util.ResultsUtils.getProvidedLabels -import static io.qameta.allure.util.ResultsUtils.getStatus -import static io.qameta.allure.util.ResultsUtils.getStatusDetails -import static java.nio.charset.StandardCharsets.UTF_8 -import static java.util.Comparator.comparing -import static org.apache.commons.lang3.StringUtils.EMPTY - -/** - * This is a temporary port of https://github.com/allure-framework/allure-java/tree/master/allure-spock to add support - * for Spock 2.0. - * **/ -class AllureReporter extends AbstractRunListener implements IGlobalExtension { - - private static final String FRAMEWORK = "spock" - private static final String LANGUAGE = "groovy" - private static final String MD5 = "md5" - private static final String GIVEN = "Given:" - private static final String WHEN = "When:" - private static final String THEN = "Then:" - private static final String EXPECT = "Expect:" - private static final String WHERE = "Where:" - private static final String AND = "And:" - private static final String CLEANUP = "Cleanup:" - - private final Map stepSpockMap = new HashMap<>() - private final ThreadLocal testUuid - = InheritableThreadLocal.withInitial({ UUID.randomUUID().toString() }) - - private final AllureLifecycle lifecycle - - AllureReporter() { - this(Allure.getLifecycle()) - - this.stepSpockMap.put("SETUP", GIVEN) - this.stepSpockMap.put("WHEN", WHEN) - this.stepSpockMap.put("THEN", THEN) - this.stepSpockMap.put("EXPECT", EXPECT) - this.stepSpockMap.put("WHERE", WHERE) - this.stepSpockMap.put("AND", AND) - this.stepSpockMap.put("CLEANUP", CLEANUP) - } - - AllureReporter(AllureLifecycle lifecycle) { - this.lifecycle = lifecycle - } - - @Override - void visitSpec(SpecInfo spec) { - spec.addListener(this) - } - - @Override - void beforeIteration(IterationInfo iteration) { - String uuid = testUuid.get() - FeatureInfo feature = iteration.feature - SpecInfo spec = feature.spec - List parameters = getParameters(iteration.dataVariables) - SpecInfo subSpec = spec.subSpec - SpecInfo superSpec = spec.superSpec - String packageName = spec.package - String specName = spec.name - String testClassName = spec.reflection.name - String testMethodName = iteration.displayName - - List