From 0f41d3f856ea6f805514c9252764851ccbad7576 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Fri, 13 Dec 2024 08:54:54 -0800 Subject: [PATCH 01/36] Fix geolookup: mock dbReader TU --- ...meDataProcessedAuctionRequestHookTest.java | 33 ++++++++++++------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java index 157b70b9474..29170dc01f5 100644 --- a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java +++ b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java @@ -12,13 +12,16 @@ import com.iab.openrtb.request.Device; import com.iab.openrtb.request.Imp; import com.maxmind.geoip2.DatabaseReader; +import com.maxmind.geoip2.exception.GeoIp2Exception; +import com.maxmind.geoip2.model.CountryResponse; +import com.maxmind.geoip2.record.Country; import io.vertx.core.Future; import io.vertx.core.Vertx; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.prebid.server.analytics.reporter.greenbids.model.ExplorationResult; import org.prebid.server.analytics.reporter.greenbids.model.Ortb2ImpExtResult; @@ -52,12 +55,9 @@ import org.prebid.server.proto.openrtb.ext.request.ExtRequest; import org.prebid.server.proto.openrtb.ext.request.ExtRequestPrebid; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.net.URL; +import java.net.InetAddress; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.Collections; import java.util.List; @@ -88,15 +88,27 @@ public class GreenbidsRealTimeDataProcessedAuctionRequestHookTest { @Mock(strictness = LENIENT) private DatabaseReaderFactory databaseReaderFactory; - @Mock - private DatabaseReader dbReader; + @Mock(strictness = LENIENT) + private DatabaseReader databaseReader; + + @Mock(strictness = LENIENT) + private CountryResponse countryResponse; + + @Mock(strictness = LENIENT) + private Country country; private GreenbidsRealTimeDataProcessedAuctionRequestHook target; @BeforeEach - public void setUp() throws IOException { + public void setUp() throws IOException, GeoIp2Exception { final Storage storage = StorageOptions.newBuilder() .setProjectId("test_project").build().getService(); + + Mockito.when(country.getName()).thenReturn("United States"); + Mockito.when(countryResponse.getCountry()).thenReturn(country); + Mockito.when(databaseReader.country(Mockito.any(InetAddress.class))).thenReturn(countryResponse); + Mockito.when(databaseReaderFactory.getDatabaseReader()).thenReturn(databaseReader); + final FilterService filterService = new FilterService(); final OnnxModelRunnerFactory onnxModelRunnerFactory = new OnnxModelRunnerFactory(); final ThrottlingThresholdsFactory throttlingThresholdsFactory = new ThrottlingThresholdsFactory(); @@ -118,7 +130,7 @@ public void setUp() throws IOException { final OnnxModelRunnerWithThresholds onnxModelRunnerWithThresholds = new OnnxModelRunnerWithThresholds( modelCache, thresholdCache); - when(databaseReaderFactory.getDatabaseReader()).thenReturn(dbReader); + final GreenbidsInferenceDataService greenbidsInferenceDataService = new GreenbidsInferenceDataService( databaseReaderFactory, TestBidRequestProvider.MAPPER); @@ -162,7 +174,6 @@ public void callShouldExitEarlyWhenPartnerNotActivatedInBidRequest() { assertThat(result.analyticsTags()).isNull(); } - @Disabled("Broken until dbReader is mocked") @Test public void callShouldNotFilterBiddersAndReturnAnalyticsTagWhenExploration() throws OrtException, IOException { // given @@ -217,7 +228,6 @@ public void callShouldNotFilterBiddersAndReturnAnalyticsTagWhenExploration() thr assertThat(fingerprint).isNotNull(); } - @Disabled("Broken until dbReader is mocked") @Test public void callShouldFilterBiddersBasedOnModelWhenAnyFeatureNotAvailable() throws OrtException, IOException { // given @@ -278,7 +288,6 @@ public void callShouldFilterBiddersBasedOnModelWhenAnyFeatureNotAvailable() thro assertThat(resultBidRequest).usingRecursiveComparison().isEqualTo(expectedBidRequest); } - @Disabled("Broken until dbReader is mocked") @Test public void callShouldFilterBiddersBasedOnModelResults() throws OrtException, IOException { // given From 12e072bff8901682778acfc842f01523aed17b06 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Fri, 13 Dec 2024 10:28:59 -0800 Subject: [PATCH 02/36] fetch mmdb from GCS --- .../data/config/DatabaseReaderFactory.java | 40 +++++++++++++++---- .../GreenbidsRealTimeDataConfiguration.java | 5 ++- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index a40c98ebb25..f5e68c20443 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -1,5 +1,8 @@ package org.prebid.server.hooks.modules.greenbids.real.time.data.config; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; import io.vertx.core.Promise; import io.vertx.core.Vertx; import com.maxmind.geoip2.DatabaseReader; @@ -12,35 +15,50 @@ import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; public class DatabaseReaderFactory implements Initializable { - private final String geoLiteCountryUrl; + private final String gcsBucketName; + + private final String geoLiteCountryPath; private final Vertx vertx; + private final Storage storage; + private final AtomicReference databaseReaderRef = new AtomicReference<>(); - public DatabaseReaderFactory(String geoLitCountryUrl, Vertx vertx) { - this.geoLiteCountryUrl = geoLitCountryUrl; + public DatabaseReaderFactory(String gcsBucketName, String geoLiteCountryPath, Vertx vertx, Storage storage) { + this.gcsBucketName = gcsBucketName; + this.geoLiteCountryPath = geoLiteCountryPath; this.vertx = vertx; + this.storage = storage; } @Override public void initialize(Promise initializePromise) { - vertx.executeBlocking(() -> { try { - final URL url = new URL(geoLiteCountryUrl); + final Blob blob = getBlob(geoLiteCountryPath); final Path databasePath = Files.createTempFile("GeoLite2-Country", ".mmdb"); - try (InputStream inputStream = url.openStream(); - FileOutputStream outputStream = new FileOutputStream(databasePath.toFile())) { - inputStream.transferTo(outputStream); + try (FileOutputStream outputStream = new FileOutputStream(databasePath.toFile())) { + outputStream.write(blob.getContent()); } databaseReaderRef.set(new DatabaseReader.Builder(databasePath.toFile()).build()); + + System.out.println( + "DatabaseReaderFactory/initialize: \n" + + " gcsBucketName: " + gcsBucketName + "\n" + + " geoLiteCountryPath: " + geoLiteCountryPath + "\n" + + " blob: " + blob + "\n" + + " databasePath: " + databasePath + "\n" + + " gcsBucketName: " + gcsBucketName + ); + } catch (IOException e) { throw new PreBidException("Failed to initialize DatabaseReader from URL", e); } @@ -49,6 +67,12 @@ public void initialize(Promise initializePromise) { .onComplete(initializePromise); } + private Blob getBlob(String blobName) { + return Optional.ofNullable(storage.get(gcsBucketName)) + .map(bucket -> bucket.get(blobName)) + .orElseThrow(() -> new PreBidException("Bucket not found: " + gcsBucketName)); + } + public DatabaseReader getDatabaseReader() { return databaseReaderRef.get(); } diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java index 959352d1908..66caeef2ed1 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java @@ -31,8 +31,9 @@ public class GreenbidsRealTimeDataConfiguration { @Bean - DatabaseReaderFactory databaseReaderFactory(GreenbidsRealTimeDataProperties properties, Vertx vertx) { - return new DatabaseReaderFactory(properties.getGeoLiteCountryPath(), vertx); + DatabaseReaderFactory databaseReaderFactory( + GreenbidsRealTimeDataProperties properties, Vertx vertx, Storage storage) { + return new DatabaseReaderFactory(properties.gcsBucketName, properties.getGeoLiteCountryPath(), vertx, storage); } @Bean From 4a8d2a35d0ddb3e1e984ff589febce602354fd43 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Fri, 13 Dec 2024 10:47:29 -0800 Subject: [PATCH 03/36] local run dbReader from GCS --- .../time/data/config/DatabaseReaderFactory.java | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index f5e68c20443..b0d19d2d229 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -2,7 +2,6 @@ import com.google.cloud.storage.Blob; import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageOptions; import io.vertx.core.Promise; import io.vertx.core.Vertx; import com.maxmind.geoip2.DatabaseReader; @@ -11,8 +10,6 @@ import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.util.Optional; @@ -49,16 +46,6 @@ public void initialize(Promise initializePromise) { } databaseReaderRef.set(new DatabaseReader.Builder(databasePath.toFile()).build()); - - System.out.println( - "DatabaseReaderFactory/initialize: \n" + - " gcsBucketName: " + gcsBucketName + "\n" + - " geoLiteCountryPath: " + geoLiteCountryPath + "\n" + - " blob: " + blob + "\n" + - " databasePath: " + databasePath + "\n" + - " gcsBucketName: " + gcsBucketName - ); - } catch (IOException e) { throw new PreBidException("Failed to initialize DatabaseReader from URL", e); } @@ -67,9 +54,9 @@ public void initialize(Promise initializePromise) { .onComplete(initializePromise); } - private Blob getBlob(String blobName) { + private Blob getBlob(String geoLiteCountryPath) { return Optional.ofNullable(storage.get(gcsBucketName)) - .map(bucket -> bucket.get(blobName)) + .map(bucket -> bucket.get(geoLiteCountryPath)) .orElseThrow(() -> new PreBidException("Bucket not found: " + gcsBucketName)); } From 2eb7a4c7810c7c71cc9d42eb9025197fe0351cf4 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Fri, 13 Dec 2024 10:52:03 -0800 Subject: [PATCH 04/36] fallback to RTD maxmind to get country --- .../time/data/core/GreenbidsInferenceDataService.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java index 3bd3e37b859..b0303f16d69 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.iab.openrtb.request.BidRequest; import com.iab.openrtb.request.Device; +import com.iab.openrtb.request.Geo; import com.iab.openrtb.request.Imp; import com.maxmind.geoip2.DatabaseReader; import com.maxmind.geoip2.exception.GeoIp2Exception; @@ -86,12 +87,16 @@ private List extractMessagesForImp( final String ip = Optional.ofNullable(bidRequest.getDevice()) .map(Device::getIp) .orElse(null); - final String countryFromIp = getCountry(ip); + final String country = Optional.ofNullable(bidRequest.getDevice()) + .map(Device::getGeo) + .map(Geo::getCountry) + .orElse(getCountry(ip)); + return createThrottlingMessages( bidderNode, impId, greenbidsUserAgent, - countryFromIp, + country, hostname, hourBucket, minuteQuadrant); From 00e7255af51aa1e7b00d4ae01bd9e8d9d0dad03b Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Mon, 16 Dec 2024 02:05:40 -0800 Subject: [PATCH 05/36] fixes review --- .../time/data/config/DatabaseReaderFactory.java | 15 ++++++++++----- .../GreenbidsRealTimeDataConfiguration.java | 3 ++- .../data/core/GreenbidsInferenceDataService.java | 2 +- ...alTimeDataProcessedAuctionRequestHookTest.java | 10 +++++----- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index b0d19d2d229..620b56d499d 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -2,6 +2,7 @@ import com.google.cloud.storage.Blob; import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; import io.vertx.core.Promise; import io.vertx.core.Vertx; import com.maxmind.geoip2.DatabaseReader; @@ -38,7 +39,7 @@ public DatabaseReaderFactory(String gcsBucketName, String geoLiteCountryPath, Ve public void initialize(Promise initializePromise) { vertx.executeBlocking(() -> { try { - final Blob blob = getBlob(geoLiteCountryPath); + final Blob blob = getBlob(); final Path databasePath = Files.createTempFile("GeoLite2-Country", ".mmdb"); try (FileOutputStream outputStream = new FileOutputStream(databasePath.toFile())) { @@ -54,10 +55,14 @@ public void initialize(Promise initializePromise) { .onComplete(initializePromise); } - private Blob getBlob(String geoLiteCountryPath) { - return Optional.ofNullable(storage.get(gcsBucketName)) - .map(bucket -> bucket.get(geoLiteCountryPath)) - .orElseThrow(() -> new PreBidException("Bucket not found: " + gcsBucketName)); + private Blob getBlob() { + try { + return Optional.ofNullable(storage.get(gcsBucketName)) + .map(bucket -> bucket.get(geoLiteCountryPath)) + .orElseThrow(() -> new PreBidException("Bucket not found: " + gcsBucketName)); + } catch (StorageException e) { + throw new PreBidException("Error accessing GCS artefact for model: ", e); + } } public DatabaseReader getDatabaseReader() { diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java index 66caeef2ed1..4e2da2cc5ce 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java @@ -33,7 +33,8 @@ public class GreenbidsRealTimeDataConfiguration { @Bean DatabaseReaderFactory databaseReaderFactory( GreenbidsRealTimeDataProperties properties, Vertx vertx, Storage storage) { - return new DatabaseReaderFactory(properties.gcsBucketName, properties.getGeoLiteCountryPath(), vertx, storage); + return new DatabaseReaderFactory( + properties.getGcsBucketName(), properties.getGeoLiteCountryPath(), vertx, storage); } @Bean diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java index b0303f16d69..e3a2988b112 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java @@ -90,7 +90,7 @@ private List extractMessagesForImp( final String country = Optional.ofNullable(bidRequest.getDevice()) .map(Device::getGeo) .map(Geo::getCountry) - .orElse(getCountry(ip)); + .orElseGet(() -> getCountry(ip)); return createThrottlingMessages( bidderNode, diff --git a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java index 29170dc01f5..b7cbac4f46a 100644 --- a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java +++ b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java @@ -21,7 +21,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.prebid.server.analytics.reporter.greenbids.model.ExplorationResult; import org.prebid.server.analytics.reporter.greenbids.model.Ortb2ImpExtResult; @@ -66,6 +65,7 @@ import static java.util.function.UnaryOperator.identity; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mock.Strictness.LENIENT; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -104,10 +104,10 @@ public void setUp() throws IOException, GeoIp2Exception { final Storage storage = StorageOptions.newBuilder() .setProjectId("test_project").build().getService(); - Mockito.when(country.getName()).thenReturn("United States"); - Mockito.when(countryResponse.getCountry()).thenReturn(country); - Mockito.when(databaseReader.country(Mockito.any(InetAddress.class))).thenReturn(countryResponse); - Mockito.when(databaseReaderFactory.getDatabaseReader()).thenReturn(databaseReader); + when(country.getName()).thenReturn("United States"); + when(countryResponse.getCountry()).thenReturn(country); + when(databaseReader.country(any(InetAddress.class))).thenReturn(countryResponse); + when(databaseReaderFactory.getDatabaseReader()).thenReturn(databaseReader); final FilterService filterService = new FilterService(); final OnnxModelRunnerFactory onnxModelRunnerFactory = new OnnxModelRunnerFactory(); From 6217ac4c91079e2c2cd3a6323244906b720d2c3d Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Mon, 16 Dec 2024 02:51:23 -0800 Subject: [PATCH 06/36] empty commit check flaky TI From 5e685d0b032acb7df310985ace681dac5125fbbc Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Mon, 16 Dec 2024 12:41:31 +0100 Subject: [PATCH 07/36] empty commit check flaky TI From 972cfd47ae2f37d4397ac5d189c6efbced21a50f Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Wed, 18 Dec 2024 11:32:37 +0100 Subject: [PATCH 08/36] load mmdb from official URL with accountId + licenseKey --- .../data/config/DatabaseReaderFactory.java | 77 ++++++++++++------- .../GreenbidsRealTimeDataConfiguration.java | 3 +- .../GreenbidsRealTimeDataProperties.java | 4 + 3 files changed, 54 insertions(+), 30 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index 620b56d499d..cb602a2d4f4 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -1,52 +1,48 @@ package org.prebid.server.hooks.modules.greenbids.real.time.data.config; -import com.google.cloud.storage.Blob; import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageException; +import com.maxmind.db.Reader; import io.vertx.core.Promise; import io.vertx.core.Vertx; import com.maxmind.geoip2.DatabaseReader; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.prebid.server.exception.PreBidException; import org.prebid.server.vertx.Initializable; -import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Optional; +import java.nio.file.StandardCopyOption; +import java.util.Base64; import java.util.concurrent.atomic.AtomicReference; +import java.util.zip.GZIPInputStream; public class DatabaseReaderFactory implements Initializable { - private final String gcsBucketName; - - private final String geoLiteCountryPath; + private final GreenbidsRealTimeDataProperties properties; private final Vertx vertx; - private final Storage storage; - private final AtomicReference databaseReaderRef = new AtomicReference<>(); - public DatabaseReaderFactory(String gcsBucketName, String geoLiteCountryPath, Vertx vertx, Storage storage) { - this.gcsBucketName = gcsBucketName; - this.geoLiteCountryPath = geoLiteCountryPath; + public DatabaseReaderFactory(GreenbidsRealTimeDataProperties properties, Vertx vertx, Storage storage) { + this.properties = properties; this.vertx = vertx; - this.storage = storage; } @Override public void initialize(Promise initializePromise) { vertx.executeBlocking(() -> { try { - final Blob blob = getBlob(); - final Path databasePath = Files.createTempFile("GeoLite2-Country", ".mmdb"); - - try (FileOutputStream outputStream = new FileOutputStream(databasePath.toFile())) { - outputStream.write(blob.getContent()); - } - - databaseReaderRef.set(new DatabaseReader.Builder(databasePath.toFile()).build()); + final Path tarGzPath = downloadFile( + properties.geoLiteCountryPath, + properties.maxMindAccountId, + properties.maxMindLicenseKey); + databaseReaderRef.set(extractMMDB(tarGzPath)); } catch (IOException e) { throw new PreBidException("Failed to initialize DatabaseReader from URL", e); } @@ -55,13 +51,38 @@ public void initialize(Promise initializePromise) { .onComplete(initializePromise); } - private Blob getBlob() { - try { - return Optional.ofNullable(storage.get(gcsBucketName)) - .map(bucket -> bucket.get(geoLiteCountryPath)) - .orElseThrow(() -> new PreBidException("Bucket not found: " + gcsBucketName)); - } catch (StorageException e) { - throw new PreBidException("Error accessing GCS artefact for model: ", e); + private Path downloadFile(String url, String accountId, String licenseKey) throws IOException { + final URL downloadUrl = new URL(url + "&account_id=" + accountId + "&license_key=" + licenseKey); + final HttpURLConnection connection = (HttpURLConnection) downloadUrl.openConnection(); + connection.setRequestMethod("GET"); + + final String auth = accountId + ":" + licenseKey; + final String encodedAuth = Base64.getEncoder().encodeToString(auth.getBytes()); + connection.setRequestProperty("Authorization", "Basic " + encodedAuth); + + final Path tempFile = Files.createTempFile("geolite2", ".tar.gz"); + try (InputStream inputStream = connection.getInputStream()) { + Files.copy(inputStream, tempFile, StandardCopyOption.REPLACE_EXISTING); + } + return tempFile; + } + + private DatabaseReader extractMMDB(Path tarGzPath) throws IOException { + try (GZIPInputStream gis = new GZIPInputStream(Files.newInputStream(tarGzPath)); + TarArchiveInputStream tarInput = new TarArchiveInputStream(gis)) { + + TarArchiveEntry currentEntry; + while ((currentEntry = tarInput.getNextTarEntry()) != null) { + if (currentEntry.getName().contains("GeoLite2-Country.mmdb")) { + break; + } + } + + final DatabaseReader databaseReader = new DatabaseReader.Builder(tarInput) + .fileMode(Reader.FileMode.MEMORY).build(); + return databaseReader; + } catch (IOException e) { + throw new RuntimeException("Failed to extract MMDB file", e); } } diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java index 4e2da2cc5ce..15112edc10f 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java @@ -33,8 +33,7 @@ public class GreenbidsRealTimeDataConfiguration { @Bean DatabaseReaderFactory databaseReaderFactory( GreenbidsRealTimeDataProperties properties, Vertx vertx, Storage storage) { - return new DatabaseReaderFactory( - properties.getGcsBucketName(), properties.getGeoLiteCountryPath(), vertx, storage); + return new DatabaseReaderFactory(properties, vertx, storage); } @Bean diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java index 86736a6011f..ccd4efbf01d 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java @@ -11,6 +11,10 @@ public class GreenbidsRealTimeDataProperties { String geoLiteCountryPath; + String maxMindAccountId; + + String maxMindLicenseKey; + String gcsBucketName; Integer cacheExpirationMinutes; From 587d1ba38a8a9a9cd85d3aeabd4c9b4acdeb9e0e Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Wed, 18 Dec 2024 12:38:33 +0100 Subject: [PATCH 09/36] add Locale to convert alpha3 to countryName --- .../real/time/data/core/GreenbidsInferenceDataService.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java index e3a2988b112..223fee98884 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java @@ -26,6 +26,7 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -90,6 +91,7 @@ private List extractMessagesForImp( final String country = Optional.ofNullable(bidRequest.getDevice()) .map(Device::getGeo) .map(Geo::getCountry) + .map(GreenbidsInferenceDataService::getCountryNameFromAlpha3) .orElseGet(() -> getCountry(ip)); return createThrottlingMessages( @@ -102,6 +104,11 @@ private List extractMessagesForImp( minuteQuadrant); } + private static String getCountryNameFromAlpha3(String isoCode) { + final Locale local = new Locale("", isoCode); + return local.getDisplayCountry(); + } + private String getCountry(String ip) { if (ip == null) { return null; From 8e39877505f529cd037c98a96a4d5163b8b2f2c8 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Thu, 19 Dec 2024 18:05:15 +0100 Subject: [PATCH 10/36] httpClient loadFile + extractMMDB debug --- .../data/config/DatabaseReaderFactory.java | 78 +++++++++++-------- .../GreenbidsRealTimeDataConfiguration.java | 5 +- .../GreenbidsRealTimeDataProperties.java | 2 + 3 files changed, 51 insertions(+), 34 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index cb602a2d4f4..3abc864f0ca 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -1,23 +1,23 @@ package org.prebid.server.hooks.modules.greenbids.real.time.data.config; -import com.google.cloud.storage.Storage; import com.maxmind.db.Reader; +import io.vertx.core.Future; +import io.vertx.core.MultiMap; import io.vertx.core.Promise; import io.vertx.core.Vertx; import com.maxmind.geoip2.DatabaseReader; +import io.vertx.core.buffer.Buffer; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.prebid.server.exception.PreBidException; import org.prebid.server.vertx.Initializable; +import org.prebid.server.vertx.httpclient.HttpClient; +import org.prebid.server.vertx.httpclient.model.HttpClientResponse; import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.util.Base64; import java.util.concurrent.atomic.AtomicReference; import java.util.zip.GZIPInputStream; @@ -27,44 +27,52 @@ public class DatabaseReaderFactory implements Initializable { private final Vertx vertx; + private final HttpClient httpClient; + private final AtomicReference databaseReaderRef = new AtomicReference<>(); - public DatabaseReaderFactory(GreenbidsRealTimeDataProperties properties, Vertx vertx, Storage storage) { + public DatabaseReaderFactory(GreenbidsRealTimeDataProperties properties, Vertx vertx, HttpClient httpClient) { this.properties = properties; this.vertx = vertx; + this.httpClient = httpClient; } @Override public void initialize(Promise initializePromise) { - vertx.executeBlocking(() -> { - try { - final Path tarGzPath = downloadFile( - properties.geoLiteCountryPath, - properties.maxMindAccountId, - properties.maxMindLicenseKey); - databaseReaderRef.set(extractMMDB(tarGzPath)); - } catch (IOException e) { - throw new PreBidException("Failed to initialize DatabaseReader from URL", e); - } - return null; - }).mapEmpty() - .onComplete(initializePromise); + vertx.executeBlocking(promise -> downloadFile(properties) + .onSuccess(tarGzPath -> { + try { + databaseReaderRef.set(extractMMDB(tarGzPath)); + promise.complete(); + } catch (IOException e) { + promise.fail(new PreBidException("Failed to extract MMDB file", e)); + } + }).onFailure(promise::fail)); } - private Path downloadFile(String url, String accountId, String licenseKey) throws IOException { - final URL downloadUrl = new URL(url + "&account_id=" + accountId + "&license_key=" + licenseKey); - final HttpURLConnection connection = (HttpURLConnection) downloadUrl.openConnection(); - connection.setRequestMethod("GET"); + private Future downloadFile(GreenbidsRealTimeDataProperties properties) { + final String downloadUrl = properties.geoLiteCountryPath + "&account_id=" + properties.maxMindAccountId + + "&license_key=" + properties.maxMindLicenseKey; - final String auth = accountId + ":" + licenseKey; - final String encodedAuth = Base64.getEncoder().encodeToString(auth.getBytes()); - connection.setRequestProperty("Authorization", "Basic " + encodedAuth); + final Future responseFuture = httpClient.get(downloadUrl, + MultiMap.caseInsensitiveMultiMap(), + properties.getTimeoutMs()); - final Path tempFile = Files.createTempFile("geolite2", ".tar.gz"); - try (InputStream inputStream = connection.getInputStream()) { - Files.copy(inputStream, tempFile, StandardCopyOption.REPLACE_EXISTING); - } - return tempFile; + return responseFuture + .compose(response -> { + if (response.getStatusCode() != 200) { + return Future.failedFuture( + new PreBidException("Failed to download DB from URL: " + downloadUrl + + " Status: " + response.getStatusCode())); + } + + return vertx.fileSystem() + .createTempFile("geolite2", ".tar.gz"); + }) + .compose(tempFilePath -> vertx.fileSystem() + .writeFile(tempFilePath, Buffer.buffer( + responseFuture.result().getBody().getBytes(StandardCharsets.ISO_8859_1))) + .map(v -> Path.of(tempFilePath))); } private DatabaseReader extractMMDB(Path tarGzPath) throws IOException { @@ -72,12 +80,18 @@ private DatabaseReader extractMMDB(Path tarGzPath) throws IOException { TarArchiveInputStream tarInput = new TarArchiveInputStream(gis)) { TarArchiveEntry currentEntry; + boolean hasDatabaseFile = false; while ((currentEntry = tarInput.getNextTarEntry()) != null) { if (currentEntry.getName().contains("GeoLite2-Country.mmdb")) { + hasDatabaseFile = true; break; } } + if (!hasDatabaseFile) { + throw new RuntimeException("GeoLite2-Country.mmdb not found in the archive"); + } + final DatabaseReader databaseReader = new DatabaseReader.Builder(tarInput) .fileMode(Reader.FileMode.MEMORY).build(); return databaseReader; diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java index 15112edc10f..8e92ea8095d 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java @@ -17,6 +17,7 @@ import org.prebid.server.hooks.modules.greenbids.real.time.data.core.GreenbidsInvocationService; import org.prebid.server.hooks.modules.greenbids.real.time.data.v1.GreenbidsRealTimeDataProcessedAuctionRequestHook; import org.prebid.server.json.ObjectMapperProvider; +import org.prebid.server.vertx.httpclient.HttpClient; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; @@ -32,8 +33,8 @@ public class GreenbidsRealTimeDataConfiguration { @Bean DatabaseReaderFactory databaseReaderFactory( - GreenbidsRealTimeDataProperties properties, Vertx vertx, Storage storage) { - return new DatabaseReaderFactory(properties, vertx, storage); + GreenbidsRealTimeDataProperties properties, Vertx vertx, HttpClient httpClient) { + return new DatabaseReaderFactory(properties, vertx, httpClient); } @Bean diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java index ccd4efbf01d..a72f5849823 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java @@ -22,4 +22,6 @@ public class GreenbidsRealTimeDataProperties { String onnxModelCacheKeyPrefix; String thresholdsCacheKeyPrefix; + + Long timeoutMs; } From b2f3c8ab91dd9c8c545c50bb65fb4d26922c459e Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Mon, 23 Dec 2024 12:14:15 +0100 Subject: [PATCH 11/36] downloadFile debug --- .../data/config/DatabaseReaderFactory.java | 127 +++++++++++++----- .../GreenbidsRealTimeDataConfiguration.java | 5 +- .../GreenbidsRealTimeDataProperties.java | 4 - 3 files changed, 93 insertions(+), 43 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index 3abc864f0ca..68034b83d13 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -1,21 +1,21 @@ package org.prebid.server.hooks.modules.greenbids.real.time.data.config; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.http.HttpClientResponse; + import com.maxmind.db.Reader; import io.vertx.core.Future; -import io.vertx.core.MultiMap; import io.vertx.core.Promise; import io.vertx.core.Vertx; import com.maxmind.geoip2.DatabaseReader; -import io.vertx.core.buffer.Buffer; +import io.vertx.core.file.OpenOptions; +import io.vertx.core.http.HttpMethod; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.prebid.server.exception.PreBidException; import org.prebid.server.vertx.Initializable; -import org.prebid.server.vertx.httpclient.HttpClient; -import org.prebid.server.vertx.httpclient.model.HttpClientResponse; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.concurrent.atomic.AtomicReference; @@ -27,58 +27,113 @@ public class DatabaseReaderFactory implements Initializable { private final Vertx vertx; - private final HttpClient httpClient; - private final AtomicReference databaseReaderRef = new AtomicReference<>(); - public DatabaseReaderFactory(GreenbidsRealTimeDataProperties properties, Vertx vertx, HttpClient httpClient) { + public DatabaseReaderFactory( + GreenbidsRealTimeDataProperties properties, Vertx vertx) { this.properties = properties; this.vertx = vertx; - this.httpClient = httpClient; } @Override public void initialize(Promise initializePromise) { - vertx.executeBlocking(promise -> downloadFile(properties) - .onSuccess(tarGzPath -> { - try { - databaseReaderRef.set(extractMMDB(tarGzPath)); + + System.out.println( + "DatabaseReaderFactory/initialize/ \n" + + "properties: " + properties + "\n" + + "vertx: " + vertx + "\n" + ); + + vertx.executeBlocking(promise -> { + downloadAndExtract() + .onSuccess(databaseReader -> { + databaseReaderRef.set(databaseReader); promise.complete(); - } catch (IOException e) { - promise.fail(new PreBidException("Failed to extract MMDB file", e)); - } - }).onFailure(promise::fail)); + }) + .onFailure(promise::fail); + }); + } + + private Future downloadAndExtract() { + final String downloadUrl = properties.geoLiteCountryPath; + + System.out.println( + "DatabaseReaderFactory/downloadAndExtract/ \n" + + "downloadUrl: " + downloadUrl + "\n" + ); + + final Path tmpPath = Path.of("geolite2.tar.gz"); + + return downloadFile(downloadUrl, tmpPath) + .compose(v -> { + try { + return Future.succeededFuture(extractMMDB(tmpPath)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + private Future downloadFile(String downloadUrl, Path tmpPath) { + + System.out.println( + "DatabaseReaderFactory/downloadFile \n" + + "tmpPath: " + tmpPath + "\n" + ); + + return vertx.fileSystem().open(tmpPath.toString(), new OpenOptions()) + .compose(tmpFile -> sendHttpRequest(downloadUrl) + .compose(response -> response.pipeTo(tmpFile)) + .onComplete(result -> tmpFile.close())); + } - private Future downloadFile(GreenbidsRealTimeDataProperties properties) { - final String downloadUrl = properties.geoLiteCountryPath + "&account_id=" + properties.maxMindAccountId - + "&license_key=" + properties.maxMindLicenseKey; + private Future sendHttpRequest(String url) { + return vertx.createHttpClient().request(HttpMethod.GET, url) + .compose(request -> { + System.out.println( + "DatabaseReaderFactory/sendHttpRequest/ before send \n" + + "request: " + request + "\n" + ); - final Future responseFuture = httpClient.get(downloadUrl, - MultiMap.caseInsensitiveMultiMap(), - properties.getTimeoutMs()); + Future responseFuture = request.send(); - return responseFuture - .compose(response -> { - if (response.getStatusCode() != 200) { - return Future.failedFuture( - new PreBidException("Failed to download DB from URL: " + downloadUrl - + " Status: " + response.getStatusCode())); - } + System.out.println( + "DatabaseReaderFactory/sendHttpRequest/ after sent \n" + + "response: " + responseFuture + "\n" + ); - return vertx.fileSystem() - .createTempFile("geolite2", ".tar.gz"); + return responseFuture; }) - .compose(tempFilePath -> vertx.fileSystem() - .writeFile(tempFilePath, Buffer.buffer( - responseFuture.result().getBody().getBytes(StandardCharsets.ISO_8859_1))) - .map(v -> Path.of(tempFilePath))); + .map(this::validateResponse); + } + + private HttpClientResponse validateResponse(HttpClientResponse response) { + + System.out.println( + "DatabaseReaderFactory/validateResponse/ \n" + + "response: " + response + "\n" + ); + + final int statusCode = response.statusCode(); + if (statusCode != HttpResponseStatus.OK.code()) { + throw new PreBidException("Got unexpected response from server with status code %s and message %s" + .formatted(statusCode, response.statusMessage())); + } + return response; } private DatabaseReader extractMMDB(Path tarGzPath) throws IOException { try (GZIPInputStream gis = new GZIPInputStream(Files.newInputStream(tarGzPath)); TarArchiveInputStream tarInput = new TarArchiveInputStream(gis)) { + System.out.println( + "DatabaseReaderFactory/extractMMDB/ \n" + + "tarGzPath: " + tarGzPath + "\n" + + "gis: " + gis + "\n" + + "tarInput: " + tarInput + "\n" + ); + TarArchiveEntry currentEntry; boolean hasDatabaseFile = false; while ((currentEntry = tarInput.getNextTarEntry()) != null) { diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java index 8e92ea8095d..0bf89b7ec8a 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java @@ -17,7 +17,6 @@ import org.prebid.server.hooks.modules.greenbids.real.time.data.core.GreenbidsInvocationService; import org.prebid.server.hooks.modules.greenbids.real.time.data.v1.GreenbidsRealTimeDataProcessedAuctionRequestHook; import org.prebid.server.json.ObjectMapperProvider; -import org.prebid.server.vertx.httpclient.HttpClient; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; @@ -33,8 +32,8 @@ public class GreenbidsRealTimeDataConfiguration { @Bean DatabaseReaderFactory databaseReaderFactory( - GreenbidsRealTimeDataProperties properties, Vertx vertx, HttpClient httpClient) { - return new DatabaseReaderFactory(properties, vertx, httpClient); + GreenbidsRealTimeDataProperties properties, Vertx vertx) { + return new DatabaseReaderFactory(properties, vertx); } @Bean diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java index a72f5849823..e807b21d4d7 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java @@ -11,10 +11,6 @@ public class GreenbidsRealTimeDataProperties { String geoLiteCountryPath; - String maxMindAccountId; - - String maxMindLicenseKey; - String gcsBucketName; Integer cacheExpirationMinutes; From e56b002e045974eaa84f5a6867684a087881dd60 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Tue, 24 Dec 2024 12:43:29 +0100 Subject: [PATCH 12/36] module config for debug --- .../data/config/DatabaseReaderFactory.java | 5 +- sample/configs/prebid-config-with-module.yaml | 62 +++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index 68034b83d13..01734995725 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -62,7 +62,7 @@ private Future downloadAndExtract() { "downloadUrl: " + downloadUrl + "\n" ); - final Path tmpPath = Path.of("geolite2.tar.gz"); + final Path tmpPath = Path.of("/var/tmp/prebid/tmp/GeoLite2-Country.tar.gz"); return downloadFile(downloadUrl, tmpPath) .compose(v -> { @@ -78,7 +78,8 @@ private Future downloadFile(String downloadUrl, Path tmpPath) { System.out.println( "DatabaseReaderFactory/downloadFile \n" + - "tmpPath: " + tmpPath + "\n" + "tmpPath: " + tmpPath + "\n" + + "tmpPath.toString(): " + tmpPath.toString() + "\n" ); return vertx.fileSystem().open(tmpPath.toString(), new OpenOptions()) diff --git a/sample/configs/prebid-config-with-module.yaml b/sample/configs/prebid-config-with-module.yaml index 06fc01fe3e2..aeadb959eca 100644 --- a/sample/configs/prebid-config-with-module.yaml +++ b/sample/configs/prebid-config-with-module.yaml @@ -26,6 +26,45 @@ settings: stored-imps-dir: sample stored-responses-dir: sample categories-dir: + default-account-config: > + { + "auction": { + "price-floors": { + "enabled": true, + "fetch": { + "enabled": false, + "timeout-ms": 5000, + "max-rules": 0, + "max-file-size-kb": 200, + "max-age-sec": 86400, + "period-sec": 3600 + }, + "enforce-floors-rate": 100, + "adjust-for-bid-adjustment": true, + "enforce-deal-floors": true, + "use-dynamic-data": true + } + }, + "hooks": { + "modules": { + "greenbids": { + "enabled": true, + "pbuid": "PBUID_FROM_GREENBIDS", + "targetTpr": 0.95, + "explorationRate": 0.001 + } + } + }, + "analytics": { + "modules": { + "greenbids": { + "enabled": true, + "pbuid": "PBUID_FROM_GREENBIDS", + "greenbidsSampling": 0.001 + } + } + } + } gdpr: default-value: 1 vendorlist: @@ -40,6 +79,8 @@ admin-endpoints: on-application-port: true protected: false hooks: + greenbids-real-time-data: + enabled: true confiant-ad-quality: enabled: true host-execution-plan: > @@ -47,6 +88,19 @@ hooks: "endpoints": { "/openrtb2/auction": { "stages": { + "processed-auction-request": { + "groups": [ + { + "timeout": 100, + "hook-sequence": [ + { + "module-code": "greenbids-real-time-data", + "hook-impl-code": "greenbids-real-time-data-processed-auction-request" + } + ] + } + ] + }, "all-processed-bid-responses": { "groups": [ { @@ -83,3 +137,11 @@ hooks: long-interval: 1800000 scan-state-check-interval: 2900 bidders-to-exclude-from-scan: + greenbids-real-time-data: + google-cloud-greenbids-project: "greenbids-357713" + gcs-bucket-name: "greenbids-europe-west1-prebid-server-staging" + cache-expiration-minutes: 15 + geo-lite-country-path: "TOREPLACEWITHURL" + onnx-model-cache-key-prefix: "onnxModelRunner_" + thresholds-cache-key-prefix: "throttlingThresholds_" + timeout-ms: 10000 From 37c7be71dce01748e0e0545906b30e1618ac9616 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Tue, 24 Dec 2024 12:44:18 +0100 Subject: [PATCH 13/36] application.yaml with increased timeout 20 sec --- src/main/resources/application.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index eb73cc16478..3585d7a7507 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -4,7 +4,7 @@ spring: vertx: worker-pool-size: 20 uploads-dir: file-uploads - init-timeout-ms: 5000 + init-timeout-ms: 20000 enable-per-client-endpoint-metrics: false server: max-initial-line-length: 8092 @@ -296,6 +296,7 @@ analytics: count: 100 report-ttl-ms: 900000 greenbids: + enabled: true analytics-server-version: "2.2.0" analytics-server: http://localhost:8090 exploratory-sampling-split: 0.9 From af3c1232f3f01d8d5a163a5d97151fc3612e528e Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Tue, 24 Dec 2024 15:04:13 +0100 Subject: [PATCH 14/36] changes v8 --- .../data/config/DatabaseReaderFactory.java | 70 ++++++------------- .../core/GreenbidsInferenceDataService.java | 7 ++ src/main/resources/application.yaml | 2 +- 3 files changed, 29 insertions(+), 50 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index 01734995725..9cfa85e2d6f 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -1,6 +1,8 @@ package org.prebid.server.hooks.modules.greenbids.real.time.data.config; +import com.iab.openrtb.request.Request; import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.http.HttpClientRequest; import io.vertx.core.http.HttpClientResponse; import com.maxmind.db.Reader; @@ -10,6 +12,7 @@ import com.maxmind.geoip2.DatabaseReader; import io.vertx.core.file.OpenOptions; import io.vertx.core.http.HttpMethod; +import io.vertx.core.http.RequestOptions; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.prebid.server.exception.PreBidException; @@ -44,68 +47,37 @@ public void initialize(Promise initializePromise) { "vertx: " + vertx + "\n" ); - vertx.executeBlocking(promise -> { - downloadAndExtract() - .onSuccess(databaseReader -> { - databaseReaderRef.set(databaseReader); - promise.complete(); - }) - .onFailure(promise::fail); - }); + vertx.executeBlocking(() -> downloadAndExtract().onSuccess(databaseReaderRef::set)) + .mapEmpty() + .onComplete(initializePromise); } private Future downloadAndExtract() { final String downloadUrl = properties.geoLiteCountryPath; + final String tmpPath = "/var/tmp/prebid/tmp/GeoLite2-Country.tar.gz"; - System.out.println( - "DatabaseReaderFactory/downloadAndExtract/ \n" + - "downloadUrl: " + downloadUrl + "\n" - ); - - final Path tmpPath = Path.of("/var/tmp/prebid/tmp/GeoLite2-Country.tar.gz"); + //final String downloadUrl = "https://github.com/prebid/prebid-server-java/blob/master/checkstyle.xml"; + //final String tmpPath = "/var/tmp/prebid/tmp/checkstyle.xml"; return downloadFile(downloadUrl, tmpPath) - .compose(v -> { - try { - return Future.succeededFuture(extractMMDB(tmpPath)); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + .map(v -> extractMMDB(tmpPath)); } - private Future downloadFile(String downloadUrl, Path tmpPath) { - - System.out.println( - "DatabaseReaderFactory/downloadFile \n" + - "tmpPath: " + tmpPath + "\n" + - "tmpPath.toString(): " + tmpPath.toString() + "\n" - ); - - return vertx.fileSystem().open(tmpPath.toString(), new OpenOptions()) + private Future downloadFile(String downloadUrl, String tmpPath) { + return vertx.fileSystem().open(tmpPath, new OpenOptions()) .compose(tmpFile -> sendHttpRequest(downloadUrl) .compose(response -> response.pipeTo(tmpFile)) .onComplete(result -> tmpFile.close())); - } private Future sendHttpRequest(String url) { - return vertx.createHttpClient().request(HttpMethod.GET, url) - .compose(request -> { - System.out.println( - "DatabaseReaderFactory/sendHttpRequest/ before send \n" + - "request: " + request + "\n" - ); - - Future responseFuture = request.send(); - - System.out.println( - "DatabaseReaderFactory/sendHttpRequest/ after sent \n" + - "response: " + responseFuture + "\n" - ); - - return responseFuture; - }) + final RequestOptions options = new RequestOptions() + .setFollowRedirects(true) + .setMethod(HttpMethod.GET) + .setAbsoluteURI(url); + + return vertx.createHttpClient().request(options) + .compose(HttpClientRequest::send) .map(this::validateResponse); } @@ -124,8 +96,8 @@ private HttpClientResponse validateResponse(HttpClientResponse response) { return response; } - private DatabaseReader extractMMDB(Path tarGzPath) throws IOException { - try (GZIPInputStream gis = new GZIPInputStream(Files.newInputStream(tarGzPath)); + private DatabaseReader extractMMDB(String tarGzPath) { + try (GZIPInputStream gis = new GZIPInputStream(Files.newInputStream(Path.of(tarGzPath))); TarArchiveInputStream tarInput = new TarArchiveInputStream(gis)) { System.out.println( diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java index 223fee98884..dc998dce569 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java @@ -115,6 +115,13 @@ private String getCountry(String ip) { } final DatabaseReader databaseReader = databaseReaderFactory.getDatabaseReader(); + + + System.out.println( + "GreenbidsInferenceDataService/getCountry/ \n" + + "databaseReader: " + databaseReader + ); + try { final InetAddress inetAddress = InetAddress.getByName(ip); final CountryResponse response = databaseReader.country(inetAddress); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 3585d7a7507..a0255391f8e 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -4,7 +4,7 @@ spring: vertx: worker-pool-size: 20 uploads-dir: file-uploads - init-timeout-ms: 20000 + init-timeout-ms: 5000 enable-per-client-endpoint-metrics: false server: max-initial-line-length: 8092 From 98bf147252eca7abf3cf127e05281e73858a8e50 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Tue, 24 Dec 2024 15:06:51 +0100 Subject: [PATCH 15/36] loggers request send --- .../time/data/config/DatabaseReaderFactory.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index 9cfa85e2d6f..b72ad9e1412 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -77,7 +77,21 @@ private Future sendHttpRequest(String url) { .setAbsoluteURI(url); return vertx.createHttpClient().request(options) - .compose(HttpClientRequest::send) + .compose(request -> { + System.out.println( + "DatabaseReaderFactory/sendHttpRequest/ before send \n" + + "request: " + request + "\n" + ); + + Future responseFuture = request.send(); + + System.out.println( + "DatabaseReaderFactory/sendHttpRequest/ after sent \n" + + "response: " + responseFuture + "\n" + ); + + return responseFuture; + }) .map(this::validateResponse); } From 2cc7d99bb92b74be9aad56f89760ad5dfd137488 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Fri, 27 Dec 2024 11:31:01 +0100 Subject: [PATCH 16/36] debugged load tar.gz + extract dbReader --- .../data/config/DatabaseReaderFactory.java | 25 +++++++++++++------ .../greenbids/GreenbidsAnalyticsReporter.java | 2 ++ .../GreenbidsAnalyticsReporterTest.java | 2 ++ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index b72ad9e1412..54f9fe53354 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -1,8 +1,6 @@ package org.prebid.server.hooks.modules.greenbids.real.time.data.config; -import com.iab.openrtb.request.Request; import io.netty.handler.codec.http.HttpResponseStatus; -import io.vertx.core.http.HttpClientRequest; import io.vertx.core.http.HttpClientResponse; import com.maxmind.db.Reader; @@ -16,6 +14,8 @@ import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.prebid.server.exception.PreBidException; +import org.prebid.server.log.Logger; +import org.prebid.server.log.LoggerFactory; import org.prebid.server.vertx.Initializable; import java.io.IOException; @@ -32,6 +32,8 @@ public class DatabaseReaderFactory implements Initializable { private final AtomicReference databaseReaderRef = new AtomicReference<>(); + private static final Logger logger = LoggerFactory.getLogger(DatabaseReaderFactory.class); + public DatabaseReaderFactory( GreenbidsRealTimeDataProperties properties, Vertx vertx) { this.properties = properties; @@ -55,19 +57,20 @@ public void initialize(Promise initializePromise) { private Future downloadAndExtract() { final String downloadUrl = properties.geoLiteCountryPath; final String tmpPath = "/var/tmp/prebid/tmp/GeoLite2-Country.tar.gz"; - - //final String downloadUrl = "https://github.com/prebid/prebid-server-java/blob/master/checkstyle.xml"; - //final String tmpPath = "/var/tmp/prebid/tmp/checkstyle.xml"; - return downloadFile(downloadUrl, tmpPath) .map(v -> extractMMDB(tmpPath)); } private Future downloadFile(String downloadUrl, String tmpPath) { - return vertx.fileSystem().open(tmpPath, new OpenOptions()) + logger.info("downloadFile(): URL={}, tmpPath={}", downloadUrl, tmpPath); + + // Open the file for writing + return vertx.fileSystem().mkdirs("/var/tmp/prebid/tmp") + .compose(v -> vertx.fileSystem().open(tmpPath, new OpenOptions())) .compose(tmpFile -> sendHttpRequest(downloadUrl) .compose(response -> response.pipeTo(tmpFile)) - .onComplete(result -> tmpFile.close())); + .onSuccess(ignored -> logger.info("File downloaded successfully to {}", tmpPath)) + .onFailure(error -> logger.error("Failed to download file from {} to {}.", downloadUrl, tmpPath, error))); } private Future sendHttpRequest(String url) { @@ -136,6 +139,12 @@ private DatabaseReader extractMMDB(String tarGzPath) { final DatabaseReader databaseReader = new DatabaseReader.Builder(tarInput) .fileMode(Reader.FileMode.MEMORY).build(); + + System.out.println( + "DatabaseReaderFactory/extractMMDB/ \n" + + "databaseReader: " + databaseReader + "\n" + ); + return databaseReader; } catch (IOException e) { throw new RuntimeException("Failed to extract MMDB file", e); diff --git a/src/main/java/org/prebid/server/analytics/reporter/greenbids/GreenbidsAnalyticsReporter.java b/src/main/java/org/prebid/server/analytics/reporter/greenbids/GreenbidsAnalyticsReporter.java index 65de3f02ebc..daf51c9d74f 100644 --- a/src/main/java/org/prebid/server/analytics/reporter/greenbids/GreenbidsAnalyticsReporter.java +++ b/src/main/java/org/prebid/server/analytics/reporter/greenbids/GreenbidsAnalyticsReporter.java @@ -35,6 +35,7 @@ import org.prebid.server.auction.model.AuctionContext; import org.prebid.server.auction.model.BidRejectionTracker; import org.prebid.server.exception.PreBidException; +import org.prebid.server.hooks.execution.model.ExecutionStatus; import org.prebid.server.hooks.execution.model.GroupExecutionOutcome; import org.prebid.server.hooks.execution.model.HookExecutionContext; import org.prebid.server.hooks.execution.model.HookExecutionOutcome; @@ -205,6 +206,7 @@ private Map extractAnalyticsResultFromAnalyticsTag(Au .map(GroupExecutionOutcome::getHooks) .flatMap(Collection::stream) .filter(hook -> "greenbids-real-time-data".equals(hook.getHookId().getModuleCode())) + .filter(hook -> hook.getStatus() == ExecutionStatus.success) .map(HookExecutionOutcome::getAnalyticsTags) .map(Tags::activities) .flatMap(Collection::stream) diff --git a/src/test/java/org/prebid/server/analytics/reporter/greenbids/GreenbidsAnalyticsReporterTest.java b/src/test/java/org/prebid/server/analytics/reporter/greenbids/GreenbidsAnalyticsReporterTest.java index cccdbb4e149..3e5e8678c2c 100644 --- a/src/test/java/org/prebid/server/analytics/reporter/greenbids/GreenbidsAnalyticsReporterTest.java +++ b/src/test/java/org/prebid/server/analytics/reporter/greenbids/GreenbidsAnalyticsReporterTest.java @@ -39,6 +39,7 @@ import org.prebid.server.auction.model.AuctionContext; import org.prebid.server.auction.model.BidRejectionReason; import org.prebid.server.auction.model.BidRejectionTracker; +import org.prebid.server.hooks.execution.model.ExecutionStatus; import org.prebid.server.hooks.execution.model.GroupExecutionOutcome; import org.prebid.server.hooks.execution.model.HookExecutionContext; import org.prebid.server.hooks.execution.model.HookExecutionOutcome; @@ -656,6 +657,7 @@ private static HookExecutionContext givenHookExecutionContextWithAnalyticsTag() final HookExecutionOutcome hookExecutionOutcome = HookExecutionOutcome.builder() .hookId(HookId.of("greenbids-real-time-data", null)) .analyticsTags(tags) + .status(ExecutionStatus.success) .build(); final GroupExecutionOutcome groupExecutionOutcome = GroupExecutionOutcome.of( From 62068e5c9270214d9327319e17358d2d9d06e3af Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Fri, 27 Dec 2024 15:32:19 +0100 Subject: [PATCH 17/36] configs rollback to master --- sample/configs/prebid-config-with-module.yaml | 62 ------------------- src/main/resources/application.yaml | 1 - 2 files changed, 63 deletions(-) diff --git a/sample/configs/prebid-config-with-module.yaml b/sample/configs/prebid-config-with-module.yaml index aeadb959eca..06fc01fe3e2 100644 --- a/sample/configs/prebid-config-with-module.yaml +++ b/sample/configs/prebid-config-with-module.yaml @@ -26,45 +26,6 @@ settings: stored-imps-dir: sample stored-responses-dir: sample categories-dir: - default-account-config: > - { - "auction": { - "price-floors": { - "enabled": true, - "fetch": { - "enabled": false, - "timeout-ms": 5000, - "max-rules": 0, - "max-file-size-kb": 200, - "max-age-sec": 86400, - "period-sec": 3600 - }, - "enforce-floors-rate": 100, - "adjust-for-bid-adjustment": true, - "enforce-deal-floors": true, - "use-dynamic-data": true - } - }, - "hooks": { - "modules": { - "greenbids": { - "enabled": true, - "pbuid": "PBUID_FROM_GREENBIDS", - "targetTpr": 0.95, - "explorationRate": 0.001 - } - } - }, - "analytics": { - "modules": { - "greenbids": { - "enabled": true, - "pbuid": "PBUID_FROM_GREENBIDS", - "greenbidsSampling": 0.001 - } - } - } - } gdpr: default-value: 1 vendorlist: @@ -79,8 +40,6 @@ admin-endpoints: on-application-port: true protected: false hooks: - greenbids-real-time-data: - enabled: true confiant-ad-quality: enabled: true host-execution-plan: > @@ -88,19 +47,6 @@ hooks: "endpoints": { "/openrtb2/auction": { "stages": { - "processed-auction-request": { - "groups": [ - { - "timeout": 100, - "hook-sequence": [ - { - "module-code": "greenbids-real-time-data", - "hook-impl-code": "greenbids-real-time-data-processed-auction-request" - } - ] - } - ] - }, "all-processed-bid-responses": { "groups": [ { @@ -137,11 +83,3 @@ hooks: long-interval: 1800000 scan-state-check-interval: 2900 bidders-to-exclude-from-scan: - greenbids-real-time-data: - google-cloud-greenbids-project: "greenbids-357713" - gcs-bucket-name: "greenbids-europe-west1-prebid-server-staging" - cache-expiration-minutes: 15 - geo-lite-country-path: "TOREPLACEWITHURL" - onnx-model-cache-key-prefix: "onnxModelRunner_" - thresholds-cache-key-prefix: "throttlingThresholds_" - timeout-ms: 10000 diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index a0255391f8e..eb73cc16478 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -296,7 +296,6 @@ analytics: count: 100 report-ttl-ms: 900000 greenbids: - enabled: true analytics-server-version: "2.2.0" analytics-server: http://localhost:8090 exploratory-sampling-split: 0.9 From 1055606c16b56ea00c9d49430a378d7f728bdc28 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Fri, 27 Dec 2024 15:38:45 +0100 Subject: [PATCH 18/36] inference data service rollback --- .../real/time/data/core/GreenbidsInferenceDataService.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java index dc998dce569..f7b67d1cabb 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java @@ -116,12 +116,6 @@ private String getCountry(String ip) { final DatabaseReader databaseReader = databaseReaderFactory.getDatabaseReader(); - - System.out.println( - "GreenbidsInferenceDataService/getCountry/ \n" + - "databaseReader: " + databaseReader - ); - try { final InetAddress inetAddress = InetAddress.getByName(ip); final CountryResponse response = databaseReader.country(inetAddress); From 2d5ba29fecb57145e2a8688ac5e0c357a48ec9b9 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Fri, 27 Dec 2024 15:40:07 +0100 Subject: [PATCH 19/36] properties: add tmpDir, tmpPath --- .../time/data/config/GreenbidsRealTimeDataProperties.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java index e807b21d4d7..3dba5dd0080 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java @@ -11,6 +11,10 @@ public class GreenbidsRealTimeDataProperties { String geoLiteCountryPath; + String tmpDir; + + String tmpPath; + String gcsBucketName; Integer cacheExpirationMinutes; From b4ae0d721435859e9f5af0d7d555173997339c1e Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Fri, 27 Dec 2024 15:54:43 +0100 Subject: [PATCH 20/36] dbReaderFactory: add eventually (to discuss) IllegalStateException --- .../data/config/DatabaseReaderFactory.java | 76 +++++++------------ 1 file changed, 26 insertions(+), 50 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index 54f9fe53354..5e226a6eed9 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -1,6 +1,7 @@ package org.prebid.server.hooks.modules.greenbids.real.time.data.config; import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.http.HttpClientRequest; import io.vertx.core.http.HttpClientResponse; import com.maxmind.db.Reader; @@ -42,13 +43,6 @@ public DatabaseReaderFactory( @Override public void initialize(Promise initializePromise) { - - System.out.println( - "DatabaseReaderFactory/initialize/ \n" + - "properties: " + properties + "\n" + - "vertx: " + vertx + "\n" - ); - vertx.executeBlocking(() -> downloadAndExtract().onSuccess(databaseReaderRef::set)) .mapEmpty() .onComplete(initializePromise); @@ -56,55 +50,45 @@ public void initialize(Promise initializePromise) { private Future downloadAndExtract() { final String downloadUrl = properties.geoLiteCountryPath; - final String tmpPath = "/var/tmp/prebid/tmp/GeoLite2-Country.tar.gz"; + final String tmpPath = properties.tmpPath; return downloadFile(downloadUrl, tmpPath) - .map(v -> extractMMDB(tmpPath)); + .compose(unused -> { + DatabaseReader databaseReader; + try { + databaseReader = extractMMDB(tmpPath); + } catch (RuntimeException e) { + removeFile(tmpPath); + throw new PreBidException("Failed to extract MMDB file. Removed file.", e); + } + removeFile(tmpPath); + return Future.succeededFuture(databaseReader); + }); } private Future downloadFile(String downloadUrl, String tmpPath) { - logger.info("downloadFile(): URL={}, tmpPath={}", downloadUrl, tmpPath); - - // Open the file for writing - return vertx.fileSystem().mkdirs("/var/tmp/prebid/tmp") + return vertx.fileSystem().mkdirs(properties.tmpDir) .compose(v -> vertx.fileSystem().open(tmpPath, new OpenOptions())) .compose(tmpFile -> sendHttpRequest(downloadUrl) .compose(response -> response.pipeTo(tmpFile)) + .eventually(v -> tmpFile.close()) .onSuccess(ignored -> logger.info("File downloaded successfully to {}", tmpPath)) - .onFailure(error -> logger.error("Failed to download file from {} to {}.", downloadUrl, tmpPath, error))); + .onFailure(error -> logger.error( + "Failed to download file from {} to {}.", downloadUrl, tmpPath, error))); } private Future sendHttpRequest(String url) { final RequestOptions options = new RequestOptions() .setFollowRedirects(true) .setMethod(HttpMethod.GET) + .setTimeout(properties.timeoutMs) .setAbsoluteURI(url); return vertx.createHttpClient().request(options) - .compose(request -> { - System.out.println( - "DatabaseReaderFactory/sendHttpRequest/ before send \n" + - "request: " + request + "\n" - ); - - Future responseFuture = request.send(); - - System.out.println( - "DatabaseReaderFactory/sendHttpRequest/ after sent \n" + - "response: " + responseFuture + "\n" - ); - - return responseFuture; - }) + .compose(HttpClientRequest::send) .map(this::validateResponse); } private HttpClientResponse validateResponse(HttpClientResponse response) { - - System.out.println( - "DatabaseReaderFactory/validateResponse/ \n" + - "response: " + response + "\n" - ); - final int statusCode = response.statusCode(); if (statusCode != HttpResponseStatus.OK.code()) { throw new PreBidException("Got unexpected response from server with status code %s and message %s" @@ -117,13 +101,6 @@ private DatabaseReader extractMMDB(String tarGzPath) { try (GZIPInputStream gis = new GZIPInputStream(Files.newInputStream(Path.of(tarGzPath))); TarArchiveInputStream tarInput = new TarArchiveInputStream(gis)) { - System.out.println( - "DatabaseReaderFactory/extractMMDB/ \n" + - "tarGzPath: " + tarGzPath + "\n" + - "gis: " + gis + "\n" + - "tarInput: " + tarInput + "\n" - ); - TarArchiveEntry currentEntry; boolean hasDatabaseFile = false; while ((currentEntry = tarInput.getNextTarEntry()) != null) { @@ -137,20 +114,19 @@ private DatabaseReader extractMMDB(String tarGzPath) { throw new RuntimeException("GeoLite2-Country.mmdb not found in the archive"); } - final DatabaseReader databaseReader = new DatabaseReader.Builder(tarInput) + return new DatabaseReader.Builder(tarInput) .fileMode(Reader.FileMode.MEMORY).build(); - - System.out.println( - "DatabaseReaderFactory/extractMMDB/ \n" + - "databaseReader: " + databaseReader + "\n" - ); - - return databaseReader; } catch (IOException e) { throw new RuntimeException("Failed to extract MMDB file", e); } } + private Future removeFile(String filePath) { + return vertx.fileSystem().delete(filePath) + .onSuccess(ignored -> logger.info("File {} removed successfully", filePath)) + .onFailure(err -> logger.error("Failed to remove file {}", filePath, err)); + } + public DatabaseReader getDatabaseReader() { return databaseReaderRef.get(); } From 397215165fdcd53a5298d0956dacad8d01607fa5 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Fri, 27 Dec 2024 16:04:38 +0100 Subject: [PATCH 21/36] fmt --- .../real/time/data/config/DatabaseReaderFactory.java | 3 +-- .../greenbids/real/time/data/core/OnnxModelRunnerTest.java | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index 5e226a6eed9..aaeae030ed1 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -3,7 +3,6 @@ import io.netty.handler.codec.http.HttpResponseStatus; import io.vertx.core.http.HttpClientRequest; import io.vertx.core.http.HttpClientResponse; - import com.maxmind.db.Reader; import io.vertx.core.Future; import io.vertx.core.Promise; @@ -53,7 +52,7 @@ private Future downloadAndExtract() { final String tmpPath = properties.tmpPath; return downloadFile(downloadUrl, tmpPath) .compose(unused -> { - DatabaseReader databaseReader; + final DatabaseReader databaseReader; try { databaseReader = extractMMDB(tmpPath); } catch (RuntimeException e) { diff --git a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/OnnxModelRunnerTest.java b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/OnnxModelRunnerTest.java index 4a18b0a0fc0..51c4b34ff91 100644 --- a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/OnnxModelRunnerTest.java +++ b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/OnnxModelRunnerTest.java @@ -29,7 +29,7 @@ public void setUp() throws OrtException, IOException { public void runModelShouldReturnProbabilitiesWhenValidThrottlingInferenceRow() throws OrtException { // given final String[][] throttlingInferenceRow = {{ - "Chrome 59", "rubicon", "adunitcodevalue", "US", "www.leparisien.fr", "PC", "10", "1"}}; + "Chrome 59", "rubicon", "adunitcodevalue", "US", "www.leparisien.fr", "PC", "10", "1"}}; // when final OrtSession.Result actualResult = target.runModel(throttlingInferenceRow); @@ -58,7 +58,7 @@ public void runModelShouldReturnProbabilitiesWhenValidThrottlingInferenceRow() t public void runModelShouldThrowOrtExceptionWhenNonValidThrottlingInferenceRow() { // given final String[][] throttlingInferenceRowWithMissingColumn = {{ - "Chrome 59", "adunitcodevalue", "US", "www.leparisien.fr", "PC", "10", "1"}}; + "Chrome 59", "adunitcodevalue", "US", "www.leparisien.fr", "PC", "10", "1"}}; // when & then assertThatThrownBy(() -> target.runModel(throttlingInferenceRowWithMissingColumn)) From 2f991c3256cb97db41b93623f0b09da422367153 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Mon, 30 Dec 2024 12:58:45 +0100 Subject: [PATCH 22/36] fix review --- .../data/config/DatabaseReaderFactory.java | 22 +++++-------------- .../GreenbidsRealTimeDataProperties.java | 2 -- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index aaeae030ed1..88bf78521c6 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -51,26 +51,15 @@ private Future downloadAndExtract() { final String downloadUrl = properties.geoLiteCountryPath; final String tmpPath = properties.tmpPath; return downloadFile(downloadUrl, tmpPath) - .compose(unused -> { - final DatabaseReader databaseReader; - try { - databaseReader = extractMMDB(tmpPath); - } catch (RuntimeException e) { - removeFile(tmpPath); - throw new PreBidException("Failed to extract MMDB file. Removed file.", e); - } - removeFile(tmpPath); - return Future.succeededFuture(databaseReader); - }); + .flatMap(unused -> Future.succeededFuture(extractMMDB(tmpPath))) + .onComplete(ar -> removeFile(tmpPath)); } private Future downloadFile(String downloadUrl, String tmpPath) { - return vertx.fileSystem().mkdirs(properties.tmpDir) - .compose(v -> vertx.fileSystem().open(tmpPath, new OpenOptions())) + return vertx.fileSystem().open(tmpPath, new OpenOptions()) .compose(tmpFile -> sendHttpRequest(downloadUrl) .compose(response -> response.pipeTo(tmpFile)) .eventually(v -> tmpFile.close()) - .onSuccess(ignored -> logger.info("File downloaded successfully to {}", tmpPath)) .onFailure(error -> logger.error( "Failed to download file from {} to {}.", downloadUrl, tmpPath, error))); } @@ -110,19 +99,18 @@ private DatabaseReader extractMMDB(String tarGzPath) { } if (!hasDatabaseFile) { - throw new RuntimeException("GeoLite2-Country.mmdb not found in the archive"); + throw new PreBidException("GeoLite2-Country.mmdb not found in the archive"); } return new DatabaseReader.Builder(tarInput) .fileMode(Reader.FileMode.MEMORY).build(); } catch (IOException e) { - throw new RuntimeException("Failed to extract MMDB file", e); + throw new PreBidException("Failed to extract MMDB file", e); } } private Future removeFile(String filePath) { return vertx.fileSystem().delete(filePath) - .onSuccess(ignored -> logger.info("File {} removed successfully", filePath)) .onFailure(err -> logger.error("Failed to remove file {}", filePath, err)); } diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java index 3dba5dd0080..e86776d155e 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java @@ -11,8 +11,6 @@ public class GreenbidsRealTimeDataProperties { String geoLiteCountryPath; - String tmpDir; - String tmpPath; String gcsBucketName; From a694373fa56f84100e70deff13ed09ab95f75f8a Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Tue, 31 Dec 2024 10:59:24 +0100 Subject: [PATCH 23/36] add TU geo defined --- .../GreenbidsRealTimeDataConfiguration.java | 7 ++- .../core/GreenbidsInferenceDataService.java | 14 ++++-- .../GreenbidsInferenceDataServiceTest.java | 50 +++++++++++++++++-- .../core/GreenbidsInvocationServiceTest.java | 4 +- .../data/util/TestBidRequestProvider.java | 10 +++- ...meDataProcessedAuctionRequestHookTest.java | 13 +++-- 6 files changed, 81 insertions(+), 17 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java index 0bf89b7ec8a..596d41bc939 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java @@ -5,6 +5,7 @@ import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; import io.vertx.core.Vertx; +import org.prebid.server.geolocation.CountryCodeMapper; import org.prebid.server.hooks.modules.greenbids.real.time.data.model.filter.ThrottlingThresholds; import org.prebid.server.hooks.modules.greenbids.real.time.data.core.ThrottlingThresholdsFactory; import org.prebid.server.hooks.modules.greenbids.real.time.data.core.GreenbidsInferenceDataService; @@ -37,9 +38,11 @@ DatabaseReaderFactory databaseReaderFactory( } @Bean - GreenbidsInferenceDataService greenbidsInferenceDataService(DatabaseReaderFactory databaseReaderFactory) { + GreenbidsInferenceDataService greenbidsInferenceDataService( + DatabaseReaderFactory databaseReaderFactory, + CountryCodeMapper countryCodeMapper) { return new GreenbidsInferenceDataService( - databaseReaderFactory, ObjectMapperProvider.mapper()); + databaseReaderFactory, ObjectMapperProvider.mapper(), countryCodeMapper); } @Bean diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java index f7b67d1cabb..97a37d4d4a0 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java @@ -14,6 +14,7 @@ import com.maxmind.geoip2.record.Country; import org.apache.commons.lang3.StringUtils; import org.prebid.server.exception.PreBidException; +import org.prebid.server.geolocation.CountryCodeMapper; import org.prebid.server.hooks.modules.greenbids.real.time.data.config.DatabaseReaderFactory; import org.prebid.server.hooks.modules.greenbids.real.time.data.model.data.ThrottlingMessage; import org.prebid.server.proto.openrtb.ext.request.ExtImpPrebid; @@ -37,9 +38,15 @@ public class GreenbidsInferenceDataService { private final ObjectMapper mapper; - public GreenbidsInferenceDataService(DatabaseReaderFactory dbReaderFactory, ObjectMapper mapper) { + private final CountryCodeMapper countryCodeMapper; + + public GreenbidsInferenceDataService( + DatabaseReaderFactory dbReaderFactory, + ObjectMapper mapper, + CountryCodeMapper countryCodeMapper) { this.databaseReaderFactory = Objects.requireNonNull(dbReaderFactory); this.mapper = Objects.requireNonNull(mapper); + this.countryCodeMapper = Objects.requireNonNull(countryCodeMapper); } public List extractThrottlingMessagesFromBidRequest(BidRequest bidRequest) { @@ -91,7 +98,8 @@ private List extractMessagesForImp( final String country = Optional.ofNullable(bidRequest.getDevice()) .map(Device::getGeo) .map(Geo::getCountry) - .map(GreenbidsInferenceDataService::getCountryNameFromAlpha3) + .map(countryCodeMapper::mapToAlpha2) + .map(GreenbidsInferenceDataService::getCountryNameFromAlpha2) .orElseGet(() -> getCountry(ip)); return createThrottlingMessages( @@ -104,7 +112,7 @@ private List extractMessagesForImp( minuteQuadrant); } - private static String getCountryNameFromAlpha3(String isoCode) { + private static String getCountryNameFromAlpha2(String isoCode) { final Locale local = new Locale("", isoCode); return local.getDisplayCountry(); } diff --git a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataServiceTest.java b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataServiceTest.java index 1ac1bcc5cb1..6453fc5cef1 100644 --- a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataServiceTest.java +++ b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataServiceTest.java @@ -15,6 +15,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.prebid.server.exception.PreBidException; +import org.prebid.server.geolocation.CountryCodeMapper; import org.prebid.server.hooks.modules.greenbids.real.time.data.config.DatabaseReaderFactory; import org.prebid.server.hooks.modules.greenbids.real.time.data.model.data.ThrottlingMessage; import org.prebid.server.hooks.modules.greenbids.real.time.data.util.TestBidRequestProvider; @@ -50,16 +51,20 @@ public class GreenbidsInferenceDataServiceTest { @Mock private Country country; + @Mock + private CountryCodeMapper countryCodeMapper; + private GreenbidsInferenceDataService target; @BeforeEach public void setUp() { when(databaseReaderFactory.getDatabaseReader()).thenReturn(databaseReader); - target = new GreenbidsInferenceDataService(databaseReaderFactory, TestBidRequestProvider.MAPPER); + target = new GreenbidsInferenceDataService( + databaseReaderFactory, TestBidRequestProvider.MAPPER, countryCodeMapper); } @Test - public void extractThrottlingMessagesFromBidRequestShouldReturnValidThrottlingMessages() + public void extractThrottlingMessagesFromBidRequestShouldReturnValidThrottlingMessagesWhenGeoIsNull() throws IOException, GeoIp2Exception { // given final Banner banner = givenBanner(); @@ -68,7 +73,7 @@ public void extractThrottlingMessagesFromBidRequestShouldReturnValidThrottlingMe .ext(givenImpExt()) .banner(banner) .build(); - final Device device = givenDevice(identity()); + final Device device = givenDevice(identity(), null); final BidRequest bidRequest = givenBidRequest(request -> request, List.of(imp), device, null); final CountryResponse countryResponse = mock(CountryResponse.class); @@ -100,6 +105,43 @@ public void extractThrottlingMessagesFromBidRequestShouldReturnValidThrottlingMe }); } + @Test + public void extractThrottlingMessagesFromBidRequestShouldReturnValidThrottlingMessagesWhenGeoDefined() { + // given + final Banner banner = givenBanner(); + final Imp imp = Imp.builder() + .id("adunitcodevalue") + .ext(givenImpExt()) + .banner(banner) + .build(); + final Device device = givenDevice(identity(), "FRA"); + final BidRequest bidRequest = givenBidRequest(request -> request, List.of(imp), device, null); + + final ZonedDateTime timestamp = ZonedDateTime.now(ZoneId.of("UTC")); + final Integer expectedHourBucket = timestamp.getHour(); + final Integer expectedMinuteQuadrant = (timestamp.getMinute() / 15) + 1; + + when(countryCodeMapper.mapToAlpha2("FRA")).thenReturn("FR"); + + // when + final List throttlingMessages = target.extractThrottlingMessagesFromBidRequest(bidRequest); + + // then + assertThat(throttlingMessages).isNotEmpty(); + assertThat(throttlingMessages.getFirst().getBidder()).isEqualTo("rubicon"); + assertThat(throttlingMessages.get(1).getBidder()).isEqualTo("appnexus"); + assertThat(throttlingMessages.getLast().getBidder()).isEqualTo("pubmatic"); + + throttlingMessages.forEach(message -> { + assertThat(message.getAdUnitCode()).isEqualTo("adunitcodevalue"); + assertThat(message.getCountry()).isEqualTo("France"); + assertThat(message.getHostname()).isEqualTo("www.leparisien.fr"); + assertThat(message.getDevice()).isEqualTo("PC"); + assertThat(message.getHourBucket()).isEqualTo(String.valueOf(expectedHourBucket)); + assertThat(message.getMinuteQuadrant()).isEqualTo(String.valueOf(expectedMinuteQuadrant)); + }); + } + @Test public void extractThrottlingMessagesFromBidRequestShouldHandleMissingIp() { // given @@ -146,7 +188,7 @@ public void extractThrottlingMessagesFromBidRequestShouldThrowPreBidExceptionWhe .ext(givenImpExt()) .banner(banner) .build(); - final Device device = givenDevice(identity()); + final Device device = givenDevice(identity(), null); final BidRequest bidRequest = givenBidRequest(request -> request, List.of(imp), device, null); when(databaseReader.country(any(InetAddress.class))).thenThrow(new GeoIp2Exception("GeoIP failure")); diff --git a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInvocationServiceTest.java b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInvocationServiceTest.java index 1bf0f5409e4..bae2e38bc78 100644 --- a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInvocationServiceTest.java +++ b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInvocationServiceTest.java @@ -44,7 +44,7 @@ public void createGreenbidsInvocationResultShouldReturnUpdateBidRequestWhenNotEx .ext(givenImpExt()) .banner(banner) .build(); - final Device device = givenDevice(identity()); + final Device device = givenDevice(identity(), null); final BidRequest bidRequest = givenBidRequest(request -> request, List.of(imp), device, null); final Map> impsBiddersFilterMap = givenImpsBiddersFilterMap(); final Partner partner = givenPartner(0.0); @@ -81,7 +81,7 @@ public void createGreenbidsInvocationResultShouldReturnNoActionWhenExploration() .ext(givenImpExt()) .banner(banner) .build(); - final Device device = givenDevice(identity()); + final Device device = givenDevice(identity(), null); final BidRequest bidRequest = givenBidRequest(request -> request, List.of(imp), device, null); final Map> impsBiddersFilterMap = givenImpsBiddersFilterMap(); final Partner partner = givenPartner(1.0); diff --git a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/util/TestBidRequestProvider.java b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/util/TestBidRequestProvider.java index 11ca069e447..f7cf9ca698d 100644 --- a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/util/TestBidRequestProvider.java +++ b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/util/TestBidRequestProvider.java @@ -7,6 +7,7 @@ import com.iab.openrtb.request.BidRequest; import com.iab.openrtb.request.Device; import com.iab.openrtb.request.Format; +import com.iab.openrtb.request.Geo; import com.iab.openrtb.request.Imp; import com.iab.openrtb.request.Site; import org.prebid.server.json.ObjectMapperProvider; @@ -68,10 +69,15 @@ public static ObjectNode givenImpExt() { return extNode; } - public static Device givenDevice(UnaryOperator deviceCustomizer) { + public static Device givenDevice(UnaryOperator deviceCustomizer, String countryAlpha3) { final String userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36" + " (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36"; - return deviceCustomizer.apply(Device.builder().ua(userAgent).ip("151.101.194.216")).build(); + final Geo geo = givenGeoWithCountry(countryAlpha3); + return deviceCustomizer.apply(Device.builder().ua(userAgent).ip("151.101.194.216").geo(geo)).build(); + } + + public static Geo givenGeoWithCountry(String countryAlpha3) { + return Geo.builder().country(countryAlpha3).build(); } public static Device givenDeviceWithoutUserAgent(UnaryOperator deviceCustomizer) { diff --git a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java index b7cbac4f46a..a0e8ecc10a1 100644 --- a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java +++ b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java @@ -25,6 +25,7 @@ import org.prebid.server.analytics.reporter.greenbids.model.ExplorationResult; import org.prebid.server.analytics.reporter.greenbids.model.Ortb2ImpExtResult; import org.prebid.server.auction.model.AuctionContext; +import org.prebid.server.geolocation.CountryCodeMapper; import org.prebid.server.hooks.execution.v1.analytics.ActivityImpl; import org.prebid.server.hooks.execution.v1.analytics.AppliedToImpl; import org.prebid.server.hooks.execution.v1.analytics.ResultImpl; @@ -97,6 +98,9 @@ public class GreenbidsRealTimeDataProcessedAuctionRequestHookTest { @Mock(strictness = LENIENT) private Country country; + @Mock + private CountryCodeMapper countryCodeMapper; + private GreenbidsRealTimeDataProcessedAuctionRequestHook target; @BeforeEach @@ -133,7 +137,8 @@ public void setUp() throws IOException, GeoIp2Exception { final GreenbidsInferenceDataService greenbidsInferenceDataService = new GreenbidsInferenceDataService( databaseReaderFactory, - TestBidRequestProvider.MAPPER); + TestBidRequestProvider.MAPPER, + countryCodeMapper); final GreenbidsInvocationService greenbidsInvocationService = new GreenbidsInvocationService(); target = new GreenbidsRealTimeDataProcessedAuctionRequestHook( TestBidRequestProvider.MAPPER, @@ -154,7 +159,7 @@ public void callShouldExitEarlyWhenPartnerNotActivatedInBidRequest() { .banner(banner) .build(); - final Device device = givenDevice(identity()); + final Device device = givenDevice(identity(), null); final BidRequest bidRequest = givenBidRequest(request -> request, List.of(imp), device, null); final AuctionContext auctionContext = givenAuctionContext(bidRequest, context -> context); final AuctionInvocationContext invocationContext = givenAuctionInvocationContext(auctionContext); @@ -186,7 +191,7 @@ public void callShouldNotFilterBiddersAndReturnAnalyticsTagWhenExploration() thr .build(); final Double explorationRate = 1.0; - final Device device = givenDevice(identity()); + final Device device = givenDevice(identity(), null); final ExtRequest extRequest = givenExtRequest(explorationRate); final BidRequest bidRequest = givenBidRequest(request -> request, List.of(imp), device, extRequest); final AuctionContext auctionContext = givenAuctionContext(bidRequest, context -> context); @@ -300,7 +305,7 @@ public void callShouldFilterBiddersBasedOnModelResults() throws OrtException, IO .build(); final Double explorationRate = 0.0001; - final Device device = givenDevice(identity()); + final Device device = givenDevice(identity(), null); final ExtRequest extRequest = givenExtRequest(explorationRate); final BidRequest bidRequest = givenBidRequest(request -> request, List.of(imp), device, extRequest); final AuctionContext auctionContext = givenAuctionContext(bidRequest, context -> context); From 2a7878bc9b682fcf786bd766fd528ccca1c10a15 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Tue, 31 Dec 2024 11:08:00 +0100 Subject: [PATCH 24/36] fix review: getters --- .../real/time/data/config/DatabaseReaderFactory.java | 4 ++-- .../time/data/core/GreenbidsInferenceDataServiceTest.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index 88bf78521c6..5ca0925a111 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -48,8 +48,8 @@ public void initialize(Promise initializePromise) { } private Future downloadAndExtract() { - final String downloadUrl = properties.geoLiteCountryPath; - final String tmpPath = properties.tmpPath; + final String downloadUrl = properties.getGeoLiteCountryPath(); + final String tmpPath = properties.getTmpPath(); return downloadFile(downloadUrl, tmpPath) .flatMap(unused -> Future.succeededFuture(extractMMDB(tmpPath))) .onComplete(ar -> removeFile(tmpPath)); diff --git a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataServiceTest.java b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataServiceTest.java index 6453fc5cef1..58accddcad2 100644 --- a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataServiceTest.java +++ b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataServiceTest.java @@ -84,7 +84,7 @@ public void extractThrottlingMessagesFromBidRequestShouldReturnValidThrottlingMe when(databaseReader.country(any(InetAddress.class))).thenReturn(countryResponse); when(countryResponse.getCountry()).thenReturn(country); - when(country.getName()).thenReturn("US"); + when(country.getName()).thenReturn("United States"); // when final List throttlingMessages = target.extractThrottlingMessagesFromBidRequest(bidRequest); @@ -97,7 +97,7 @@ public void extractThrottlingMessagesFromBidRequestShouldReturnValidThrottlingMe throttlingMessages.forEach(message -> { assertThat(message.getAdUnitCode()).isEqualTo("adunitcodevalue"); - assertThat(message.getCountry()).isEqualTo("US"); + assertThat(message.getCountry()).isEqualTo("United States"); assertThat(message.getHostname()).isEqualTo("www.leparisien.fr"); assertThat(message.getDevice()).isEqualTo("PC"); assertThat(message.getHourBucket()).isEqualTo(String.valueOf(expectedHourBucket)); From 760cbc6f306b1cfbbe07fad1a05104f1680772ac Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Thu, 2 Jan 2025 11:25:06 +0100 Subject: [PATCH 25/36] fix review2: small fixes --- .../time/data/config/DatabaseReaderFactory.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index 5ca0925a111..7da3846122a 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -1,6 +1,7 @@ package org.prebid.server.hooks.modules.greenbids.real.time.data.config; import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.file.FileSystem; import io.vertx.core.http.HttpClientRequest; import io.vertx.core.http.HttpClientResponse; import com.maxmind.db.Reader; @@ -26,18 +27,21 @@ public class DatabaseReaderFactory implements Initializable { + private static final Logger logger = LoggerFactory.getLogger(DatabaseReaderFactory.class); + private final GreenbidsRealTimeDataProperties properties; private final Vertx vertx; private final AtomicReference databaseReaderRef = new AtomicReference<>(); - private static final Logger logger = LoggerFactory.getLogger(DatabaseReaderFactory.class); + private final FileSystem fileSystem; public DatabaseReaderFactory( GreenbidsRealTimeDataProperties properties, Vertx vertx) { this.properties = properties; this.vertx = vertx; + this.fileSystem = vertx.fileSystem(); } @Override @@ -56,10 +60,9 @@ private Future downloadAndExtract() { } private Future downloadFile(String downloadUrl, String tmpPath) { - return vertx.fileSystem().open(tmpPath, new OpenOptions()) + return fileSystem.open(tmpPath, new OpenOptions()) .compose(tmpFile -> sendHttpRequest(downloadUrl) .compose(response -> response.pipeTo(tmpFile)) - .eventually(v -> tmpFile.close()) .onFailure(error -> logger.error( "Failed to download file from {} to {}.", downloadUrl, tmpPath, error))); } @@ -68,7 +71,7 @@ private Future sendHttpRequest(String url) { final RequestOptions options = new RequestOptions() .setFollowRedirects(true) .setMethod(HttpMethod.GET) - .setTimeout(properties.timeoutMs) + .setTimeout(properties.getTimeoutMs()) .setAbsoluteURI(url); return vertx.createHttpClient().request(options) @@ -109,8 +112,8 @@ private DatabaseReader extractMMDB(String tarGzPath) { } } - private Future removeFile(String filePath) { - return vertx.fileSystem().delete(filePath) + private void removeFile(String filePath) { + fileSystem.delete(filePath) .onFailure(err -> logger.error("Failed to remove file {}", filePath, err)); } From 0c372f962c36fa2bf6b3fe895ddb6d186b7e83b5 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Thu, 2 Jan 2025 11:50:26 +0100 Subject: [PATCH 26/36] fix review2: small fixes --- .../data/config/DatabaseReaderFactory.java | 2 +- .../core/GreenbidsInferenceDataService.java | 4 +++- .../GreenbidsInferenceDataServiceTest.java | 23 +++++++++---------- .../core/GreenbidsInvocationServiceTest.java | 4 ++-- .../data/util/TestBidRequestProvider.java | 6 +++++ ...meDataProcessedAuctionRequestHookTest.java | 6 ++--- 6 files changed, 26 insertions(+), 19 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index 7da3846122a..4f2d36a130a 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -55,7 +55,7 @@ private Future downloadAndExtract() { final String downloadUrl = properties.getGeoLiteCountryPath(); final String tmpPath = properties.getTmpPath(); return downloadFile(downloadUrl, tmpPath) - .flatMap(unused -> Future.succeededFuture(extractMMDB(tmpPath))) + .map(unused -> extractMMDB(tmpPath)) .onComplete(ar -> removeFile(tmpPath)); } diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java index 97a37d4d4a0..2d8a7475e96 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java @@ -114,7 +114,9 @@ private List extractMessagesForImp( private static String getCountryNameFromAlpha2(String isoCode) { final Locale local = new Locale("", isoCode); - return local.getDisplayCountry(); + return Optional.of(local) + .map(Locale::getDisplayCountry) + .orElse(""); } private String getCountry(String ip) { diff --git a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataServiceTest.java b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataServiceTest.java index 58accddcad2..fafc759ca11 100644 --- a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataServiceTest.java +++ b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataServiceTest.java @@ -73,7 +73,7 @@ public void extractThrottlingMessagesFromBidRequestShouldReturnValidThrottlingMe .ext(givenImpExt()) .banner(banner) .build(); - final Device device = givenDevice(identity(), null); + final Device device = givenDevice(identity()); final BidRequest bidRequest = givenBidRequest(request -> request, List.of(imp), device, null); final CountryResponse countryResponse = mock(CountryResponse.class); @@ -91,9 +91,9 @@ public void extractThrottlingMessagesFromBidRequestShouldReturnValidThrottlingMe // then assertThat(throttlingMessages).isNotEmpty(); - assertThat(throttlingMessages.getFirst().getBidder()).isEqualTo("rubicon"); - assertThat(throttlingMessages.get(1).getBidder()).isEqualTo("appnexus"); - assertThat(throttlingMessages.getLast().getBidder()).isEqualTo("pubmatic"); + assertThat(throttlingMessages) + .extracting(ThrottlingMessage::getBidder) + .containsExactly("rubicon", "appnexus", "pubmatic"); throttlingMessages.forEach(message -> { assertThat(message.getAdUnitCode()).isEqualTo("adunitcodevalue"); @@ -128,9 +128,9 @@ public void extractThrottlingMessagesFromBidRequestShouldReturnValidThrottlingMe // then assertThat(throttlingMessages).isNotEmpty(); - assertThat(throttlingMessages.getFirst().getBidder()).isEqualTo("rubicon"); - assertThat(throttlingMessages.get(1).getBidder()).isEqualTo("appnexus"); - assertThat(throttlingMessages.getLast().getBidder()).isEqualTo("pubmatic"); + assertThat(throttlingMessages) + .extracting(ThrottlingMessage::getBidder) + .containsExactly("rubicon", "appnexus", "pubmatic"); throttlingMessages.forEach(message -> { assertThat(message.getAdUnitCode()).isEqualTo("adunitcodevalue"); @@ -163,10 +163,9 @@ public void extractThrottlingMessagesFromBidRequestShouldHandleMissingIp() { // then assertThat(throttlingMessages).isNotEmpty(); - - assertThat(throttlingMessages.getFirst().getBidder()).isEqualTo("rubicon"); - assertThat(throttlingMessages.get(1).getBidder()).isEqualTo("appnexus"); - assertThat(throttlingMessages.getLast().getBidder()).isEqualTo("pubmatic"); + assertThat(throttlingMessages) + .extracting(ThrottlingMessage::getBidder) + .containsExactly("rubicon", "appnexus", "pubmatic"); throttlingMessages.forEach(message -> { assertThat(message.getAdUnitCode()).isEqualTo("adunitcodevalue"); @@ -188,7 +187,7 @@ public void extractThrottlingMessagesFromBidRequestShouldThrowPreBidExceptionWhe .ext(givenImpExt()) .banner(banner) .build(); - final Device device = givenDevice(identity(), null); + final Device device = givenDevice(identity()); final BidRequest bidRequest = givenBidRequest(request -> request, List.of(imp), device, null); when(databaseReader.country(any(InetAddress.class))).thenThrow(new GeoIp2Exception("GeoIP failure")); diff --git a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInvocationServiceTest.java b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInvocationServiceTest.java index bae2e38bc78..1bf0f5409e4 100644 --- a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInvocationServiceTest.java +++ b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInvocationServiceTest.java @@ -44,7 +44,7 @@ public void createGreenbidsInvocationResultShouldReturnUpdateBidRequestWhenNotEx .ext(givenImpExt()) .banner(banner) .build(); - final Device device = givenDevice(identity(), null); + final Device device = givenDevice(identity()); final BidRequest bidRequest = givenBidRequest(request -> request, List.of(imp), device, null); final Map> impsBiddersFilterMap = givenImpsBiddersFilterMap(); final Partner partner = givenPartner(0.0); @@ -81,7 +81,7 @@ public void createGreenbidsInvocationResultShouldReturnNoActionWhenExploration() .ext(givenImpExt()) .banner(banner) .build(); - final Device device = givenDevice(identity(), null); + final Device device = givenDevice(identity()); final BidRequest bidRequest = givenBidRequest(request -> request, List.of(imp), device, null); final Map> impsBiddersFilterMap = givenImpsBiddersFilterMap(); final Partner partner = givenPartner(1.0); diff --git a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/util/TestBidRequestProvider.java b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/util/TestBidRequestProvider.java index f7cf9ca698d..c66ba26d953 100644 --- a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/util/TestBidRequestProvider.java +++ b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/util/TestBidRequestProvider.java @@ -76,6 +76,12 @@ public static Device givenDevice(UnaryOperator deviceCusto return deviceCustomizer.apply(Device.builder().ua(userAgent).ip("151.101.194.216").geo(geo)).build(); } + public static Device givenDevice(UnaryOperator deviceCustomizer) { + final String userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36" + + " (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36"; + return deviceCustomizer.apply(Device.builder().ua(userAgent).ip("151.101.194.216")).build(); + } + public static Geo givenGeoWithCountry(String countryAlpha3) { return Geo.builder().country(countryAlpha3).build(); } diff --git a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java index a0e8ecc10a1..284741a1b68 100644 --- a/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java +++ b/extra/modules/greenbids-real-time-data/src/test/java/org/prebid/server/hooks/modules/greenbids/real/time/data/v1/GreenbidsRealTimeDataProcessedAuctionRequestHookTest.java @@ -159,7 +159,7 @@ public void callShouldExitEarlyWhenPartnerNotActivatedInBidRequest() { .banner(banner) .build(); - final Device device = givenDevice(identity(), null); + final Device device = givenDevice(identity()); final BidRequest bidRequest = givenBidRequest(request -> request, List.of(imp), device, null); final AuctionContext auctionContext = givenAuctionContext(bidRequest, context -> context); final AuctionInvocationContext invocationContext = givenAuctionInvocationContext(auctionContext); @@ -191,7 +191,7 @@ public void callShouldNotFilterBiddersAndReturnAnalyticsTagWhenExploration() thr .build(); final Double explorationRate = 1.0; - final Device device = givenDevice(identity(), null); + final Device device = givenDevice(identity()); final ExtRequest extRequest = givenExtRequest(explorationRate); final BidRequest bidRequest = givenBidRequest(request -> request, List.of(imp), device, extRequest); final AuctionContext auctionContext = givenAuctionContext(bidRequest, context -> context); @@ -305,7 +305,7 @@ public void callShouldFilterBiddersBasedOnModelResults() throws OrtException, IO .build(); final Double explorationRate = 0.0001; - final Device device = givenDevice(identity(), null); + final Device device = givenDevice(identity()); final ExtRequest extRequest = givenExtRequest(explorationRate); final BidRequest bidRequest = givenBidRequest(request -> request, List.of(imp), device, extRequest); final AuctionContext auctionContext = givenAuctionContext(bidRequest, context -> context); From 3bbc02d6b0d2454cde00195cfd4dd1526c980c18 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Tue, 7 Jan 2025 14:37:53 +0100 Subject: [PATCH 27/36] fix review: Locale nullable --- .../real/time/data/core/GreenbidsInferenceDataService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java index 2d8a7475e96..450ad124807 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java @@ -113,8 +113,9 @@ private List extractMessagesForImp( } private static String getCountryNameFromAlpha2(String isoCode) { - final Locale local = new Locale("", isoCode); - return Optional.of(local) + return Optional.ofNullable(isoCode) + .filter(code -> !code.isBlank()) + .map(code -> new Locale("", code)) .map(Locale::getDisplayCountry) .orElse(""); } From 599d3325216bc132a33286f8ef8875a08bd9a463 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Tue, 7 Jan 2025 15:44:41 +0100 Subject: [PATCH 28/36] fix review: Locale nullable --- .../real/time/data/core/GreenbidsInferenceDataService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java index 450ad124807..973c6f33eac 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java @@ -115,9 +115,9 @@ private List extractMessagesForImp( private static String getCountryNameFromAlpha2(String isoCode) { return Optional.ofNullable(isoCode) .filter(code -> !code.isBlank()) - .map(code -> new Locale("", code)) + .map(code -> new Locale(StringUtils.EMPTY, code)) .map(Locale::getDisplayCountry) - .orElse(""); + .orElse(StringUtils.EMPTY); } private String getCountry(String ip) { From 40113b401ade8f3dd7de294121e49d79273e5673 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Tue, 7 Jan 2025 15:53:02 +0100 Subject: [PATCH 29/36] fix review: Locale nullable --- .../time/data/core/GreenbidsInferenceDataService.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java index 973c6f33eac..3f8a58671c3 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java @@ -113,11 +113,9 @@ private List extractMessagesForImp( } private static String getCountryNameFromAlpha2(String isoCode) { - return Optional.ofNullable(isoCode) - .filter(code -> !code.isBlank()) - .map(code -> new Locale(StringUtils.EMPTY, code)) - .map(Locale::getDisplayCountry) - .orElse(StringUtils.EMPTY); + return StringUtils.isBlank(isoCode) + ? StringUtils.EMPTY + : new Locale(StringUtils.EMPTY, isoCode).getDisplayCountry(); } private String getCountry(String ip) { From cc4d6a676581134cd6ed54d581581dfb932b836d Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Tue, 7 Jan 2025 15:58:33 +0100 Subject: [PATCH 30/36] empty commit retest CI From 8cdeae9ab4ab740871f552636a170ba49617e6f8 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Tue, 7 Jan 2025 17:21:05 +0100 Subject: [PATCH 31/36] fix review: handle dbreader null --- .../real/time/data/core/GreenbidsInferenceDataService.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java index 3f8a58671c3..6bc599a0b85 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java @@ -123,8 +123,12 @@ private String getCountry(String ip) { return null; } - final DatabaseReader databaseReader = databaseReaderFactory.getDatabaseReader(); + return Optional.ofNullable(databaseReaderFactory.getDatabaseReader()) + .map(dbReader -> getCountryFromIpUsingDatabase(dbReader, ip)) + .orElse(null); + } + private String getCountryFromIpUsingDatabase(DatabaseReader databaseReader, String ip) { try { final InetAddress inetAddress = InetAddress.getByName(ip); final CountryResponse response = databaseReader.country(inetAddress); From 806e48c307bbff6e88362e1d0bb106e186bd490a Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Wed, 8 Jan 2025 12:27:02 +0100 Subject: [PATCH 32/36] empty commit retest CI From 97946436b6b65080af46f09171f696b5274f5fb4 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Thu, 16 Jan 2025 01:24:49 +0100 Subject: [PATCH 33/36] fix review: dbreader + config + inferencedata --- .../data/config/DatabaseReaderFactory.java | 23 +++++++++++++------ .../GreenbidsRealTimeDataConfiguration.java | 9 ++++---- .../GreenbidsRealTimeDataProperties.java | 2 ++ .../core/GreenbidsInferenceDataService.java | 19 +++++++-------- 4 files changed, 30 insertions(+), 23 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index 4f2d36a130a..5ff95b347a7 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -2,6 +2,7 @@ import io.netty.handler.codec.http.HttpResponseStatus; import io.vertx.core.file.FileSystem; +import io.vertx.core.http.HttpClientOptions; import io.vertx.core.http.HttpClientRequest; import io.vertx.core.http.HttpClientResponse; import com.maxmind.db.Reader; @@ -37,8 +38,7 @@ public class DatabaseReaderFactory implements Initializable { private final FileSystem fileSystem; - public DatabaseReaderFactory( - GreenbidsRealTimeDataProperties properties, Vertx vertx) { + public DatabaseReaderFactory(GreenbidsRealTimeDataProperties properties, Vertx vertx) { this.properties = properties; this.vertx = vertx; this.fileSystem = vertx.fileSystem(); @@ -46,7 +46,8 @@ public DatabaseReaderFactory( @Override public void initialize(Promise initializePromise) { - vertx.executeBlocking(() -> downloadAndExtract().onSuccess(databaseReaderRef::set)) + downloadAndExtract() + .onSuccess(databaseReaderRef::set) .mapEmpty() .onComplete(initializePromise); } @@ -55,7 +56,7 @@ private Future downloadAndExtract() { final String downloadUrl = properties.getGeoLiteCountryPath(); final String tmpPath = properties.getTmpPath(); return downloadFile(downloadUrl, tmpPath) - .map(unused -> extractMMDB(tmpPath)) + .compose(ignored -> vertx.executeBlocking(() -> extractMMDB(tmpPath))) .onComplete(ar -> removeFile(tmpPath)); } @@ -74,7 +75,11 @@ private Future sendHttpRequest(String url) { .setTimeout(properties.getTimeoutMs()) .setAbsoluteURI(url); - return vertx.createHttpClient().request(options) + final HttpClientOptions httpClientOptions = new HttpClientOptions() + .setConnectTimeout(properties.getTimeoutMs().intValue()) + .setMaxRedirects(properties.getMaxRedirects()); + + return vertx.createHttpClient(httpClientOptions).request(options) .compose(HttpClientRequest::send) .map(this::validateResponse); } @@ -113,8 +118,12 @@ private DatabaseReader extractMMDB(String tarGzPath) { } private void removeFile(String filePath) { - fileSystem.delete(filePath) - .onFailure(err -> logger.error("Failed to remove file {}", filePath, err)); + fileSystem.exists(filePath).onSuccess(exists -> { + if (exists) { + fileSystem.delete(filePath) + .onFailure(err -> logger.error("Failed to remove file {}", filePath, err)); + } + }); } public DatabaseReader getDatabaseReader() { diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java index 596d41bc939..a6b47f7d6ad 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataConfiguration.java @@ -32,15 +32,14 @@ public class GreenbidsRealTimeDataConfiguration { @Bean - DatabaseReaderFactory databaseReaderFactory( - GreenbidsRealTimeDataProperties properties, Vertx vertx) { + DatabaseReaderFactory databaseReaderFactory(GreenbidsRealTimeDataProperties properties, Vertx vertx) { return new DatabaseReaderFactory(properties, vertx); } @Bean - GreenbidsInferenceDataService greenbidsInferenceDataService( - DatabaseReaderFactory databaseReaderFactory, - CountryCodeMapper countryCodeMapper) { + GreenbidsInferenceDataService greenbidsInferenceDataService(DatabaseReaderFactory databaseReaderFactory, + CountryCodeMapper countryCodeMapper) { + return new GreenbidsInferenceDataService( databaseReaderFactory, ObjectMapperProvider.mapper(), countryCodeMapper); } diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java index e86776d155e..4752a2e8840 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/GreenbidsRealTimeDataProperties.java @@ -22,4 +22,6 @@ public class GreenbidsRealTimeDataProperties { String thresholdsCacheKeyPrefix; Long timeoutMs; + + Integer maxRedirects; } diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java index 6bc599a0b85..c16a8de917f 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/core/GreenbidsInferenceDataService.java @@ -40,10 +40,9 @@ public class GreenbidsInferenceDataService { private final CountryCodeMapper countryCodeMapper; - public GreenbidsInferenceDataService( - DatabaseReaderFactory dbReaderFactory, - ObjectMapper mapper, - CountryCodeMapper countryCodeMapper) { + public GreenbidsInferenceDataService(DatabaseReaderFactory dbReaderFactory, + ObjectMapper mapper, + CountryCodeMapper countryCodeMapper) { this.databaseReaderFactory = Objects.requireNonNull(dbReaderFactory); this.mapper = Objects.requireNonNull(mapper); this.countryCodeMapper = Objects.requireNonNull(countryCodeMapper); @@ -100,6 +99,7 @@ private List extractMessagesForImp( .map(Geo::getCountry) .map(countryCodeMapper::mapToAlpha2) .map(GreenbidsInferenceDataService::getCountryNameFromAlpha2) + .filter(c -> !c.isEmpty()) .orElseGet(() -> getCountry(ip)); return createThrottlingMessages( @@ -119,13 +119,10 @@ private static String getCountryNameFromAlpha2(String isoCode) { } private String getCountry(String ip) { - if (ip == null) { - return null; - } - - return Optional.ofNullable(databaseReaderFactory.getDatabaseReader()) - .map(dbReader -> getCountryFromIpUsingDatabase(dbReader, ip)) - .orElse(null); + final DatabaseReader databaseReader = databaseReaderFactory.getDatabaseReader(); + return ip != null && databaseReader != null + ? getCountryFromIpUsingDatabase(databaseReader, ip) + : null; } private String getCountryFromIpUsingDatabase(DatabaseReader databaseReader, String ip) { From a7cbc337464df06e9342470d57490b6260c7368f Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Thu, 16 Jan 2025 10:22:35 +0100 Subject: [PATCH 34/36] fix tmpFile closing --- .../real/time/data/config/DatabaseReaderFactory.java | 5 ++--- .../server/execution/file/supplier/RemoteFileSupplier.java | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java index 5ff95b347a7..322cdf48ff4 100644 --- a/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java +++ b/extra/modules/greenbids-real-time-data/src/main/java/org/prebid/server/hooks/modules/greenbids/real/time/data/config/DatabaseReaderFactory.java @@ -63,9 +63,8 @@ private Future downloadAndExtract() { private Future downloadFile(String downloadUrl, String tmpPath) { return fileSystem.open(tmpPath, new OpenOptions()) .compose(tmpFile -> sendHttpRequest(downloadUrl) - .compose(response -> response.pipeTo(tmpFile)) - .onFailure(error -> logger.error( - "Failed to download file from {} to {}.", downloadUrl, tmpPath, error))); + .onFailure(ignore -> tmpFile.close()) + .compose(response -> response.pipeTo(tmpFile))); } private Future sendHttpRequest(String url) { diff --git a/src/main/java/org/prebid/server/execution/file/supplier/RemoteFileSupplier.java b/src/main/java/org/prebid/server/execution/file/supplier/RemoteFileSupplier.java index e8b8f313c54..ab2c6314f09 100644 --- a/src/main/java/org/prebid/server/execution/file/supplier/RemoteFileSupplier.java +++ b/src/main/java/org/prebid/server/execution/file/supplier/RemoteFileSupplier.java @@ -96,6 +96,7 @@ private Future isSizeChanged() { private Future downloadFile() { return fileSystem.open(tmpPath, new OpenOptions()) .compose(tmpFile -> sendHttpRequest(getRequestOptions) + .onFailure(ignore -> tmpFile.close()) .compose(response -> response.pipeTo(tmpFile)) .onComplete(result -> tmpFile.close())); } From ea1b65ca030c8a42a1c55aeb800a50b5239f4043 Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Thu, 16 Jan 2025 12:45:19 +0100 Subject: [PATCH 35/36] fix tmpFile closing --- .../server/execution/file/supplier/RemoteFileSupplier.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/prebid/server/execution/file/supplier/RemoteFileSupplier.java b/src/main/java/org/prebid/server/execution/file/supplier/RemoteFileSupplier.java index ab2c6314f09..ca929739de5 100644 --- a/src/main/java/org/prebid/server/execution/file/supplier/RemoteFileSupplier.java +++ b/src/main/java/org/prebid/server/execution/file/supplier/RemoteFileSupplier.java @@ -97,8 +97,7 @@ private Future downloadFile() { return fileSystem.open(tmpPath, new OpenOptions()) .compose(tmpFile -> sendHttpRequest(getRequestOptions) .onFailure(ignore -> tmpFile.close()) - .compose(response -> response.pipeTo(tmpFile)) - .onComplete(result -> tmpFile.close())); + .compose(response -> response.pipeTo(tmpFile))); } private Future sendHttpRequest(RequestOptions requestOptions) { From 829b83ade721512a2a70cf0f6b559cc245a8933c Mon Sep 17 00:00:00 2001 From: EvgeniiMunin Date: Fri, 17 Jan 2025 01:00:42 +0100 Subject: [PATCH 36/36] rollback modif on RemoteFileSupplier --- .../server/execution/file/supplier/RemoteFileSupplier.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/prebid/server/execution/file/supplier/RemoteFileSupplier.java b/src/main/java/org/prebid/server/execution/file/supplier/RemoteFileSupplier.java index ca929739de5..e8b8f313c54 100644 --- a/src/main/java/org/prebid/server/execution/file/supplier/RemoteFileSupplier.java +++ b/src/main/java/org/prebid/server/execution/file/supplier/RemoteFileSupplier.java @@ -96,8 +96,8 @@ private Future isSizeChanged() { private Future downloadFile() { return fileSystem.open(tmpPath, new OpenOptions()) .compose(tmpFile -> sendHttpRequest(getRequestOptions) - .onFailure(ignore -> tmpFile.close()) - .compose(response -> response.pipeTo(tmpFile))); + .compose(response -> response.pipeTo(tmpFile)) + .onComplete(result -> tmpFile.close())); } private Future sendHttpRequest(RequestOptions requestOptions) {