Agma: Fix race condition (#3453)

.github/workflows/docker-image-publish.yml

@@ -55,11 +55,18 @@ jobs:
         with:
           images: ${{ matrix.package-name }}
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
       - name: Build and push Docker image
         uses: docker/build-push-action@v5
         with:
           context: .
           file: ${{ matrix.dockerfile-path }}
           push: true
+          platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

README.md

@@ -100,12 +100,30 @@ There are a couple of 'hello world' test requests described in sample/requests/R
 
 ## Running Docker image
 
-Starting from PBS Java v2.9, you can download prebuilt Docker images from [GitHub Packages](https://github.com/orgs/prebid/packages?repo_name=prebid-server-java) page,
-and use them instead of plain .jar files. This prebuilt images are delivered with or without extra modules.
+Starting from PBS Java v3.11.0, you can download prebuilt Docker images from [GitHub Packages](https://github.com/orgs/prebid/packages?repo_name=prebid-server-java) page,
+and use them instead of plain .jar files. These prebuilt images are delivered in 2 flavors:
+- https://github.com/prebid/prebid-server-java/pkgs/container/prebid-server-java is a bare PBS and doesn't contain modules.
+- https://github.com/prebid/prebid-server-java/pkgs/container/prebid-server-java-bundle is a "bundle" that contains PBS and all the modules.
 
-In order to run such image correctly, you should attach PBS config file. Easiest way is to mount config file into container,
+To run PBS from image correctly, you should provide the PBS config file. The easiest way is to mount the config file into the container,
 using [--mount or --volume (-v) Docker CLI arguments](https://docs.docker.com/engine/reference/commandline/run/).
-Keep in mind, that config file should be mounted into specific location: ```/app/prebid-server/``` or ```/app/prebid-server/conf/```.
+Keep in mind that the config file should be mounted into a specific location: ```/app/prebid-server/conf/``` or ```/app/prebid-server/```.
+
+PBS follows the regular Spring Boot config load hierarchy and type.
+For simple configuration, a single `application.yaml` mounted to `/app/prebid-server/conf/` will be enough.
+Please consult [Spring Externalized Configuration](https://docs.spring.io/spring-boot/reference/features/external-config.html) for all possible ways to configure PBS.
+
+You can also supply command-line parameters through `JAVA_OPTS` environment variable which will be appended to the `java` command before the `-jar ...` parameter.
+Please pay attention to line breaks and escape them if needed.
+
+Example execution using sample configuration:
+```shell
+docker run --rm -v ./sample:/app/prebid-server/sample:ro -p 8060:8060 -p 8080:8080 ghcr.io/prebid/prebid-server-java:latest --spring.config.additional-location=sample/configs/prebid-config.yaml
+```
+or
+```shell
+docker run --rm -v ./sample:/app/prebid-server/sample:ro -p 8060:8060 -p 8080:8080 -e JAVA_OPTS=-Dspring.config.additional-location=sample/configs/prebid-config.yaml ghcr.io/prebid/prebid-server-java:latest
+```
 
 # Documentation
 

docs/application-settings.md

@@ -259,6 +259,51 @@ Here's an example YAML file containing account-specific settings:
           default: true
 ```
 
+## Setting Account Configuration in S3
+
+This is identical to the account configuration in a file system, with the main difference that your file system is
+[AWS S3](https://aws.amazon.com/de/s3/) or any S3 compatible storage, such as [MinIO](https://min.io/).
+
+
+The general idea is that you'll place all the account-specific settings in a separate YAML file and point to that file.
+
+```yaml
+settings:
+  s3:
+      accessKeyId: <S3 access key>
+      secretAccessKey: <S3 access key>
+      endpoint: <endpoint> # http://s3.storage.com
+      bucket: <bucket name> # prebid-application-settings
+      region: <region name> # if not provided AWS_GLOBAL will be used. Example value: 'eu-central-1'
+      accounts-dir: accounts
+      stored-imps-dir: stored-impressions
+      stored-requests-dir: stored-requests
+      stored-responses-dir: stored-responses
+
+  # recommended to configure an in memory cache, but this is optional
+  in-memory-cache:
+      # example settings, tailor to your needs
+      cache-size: 100000
+      ttl-seconds: 1200 # 20 minutes
+      # recommended to configure
+      s3-update:
+          refresh-rate: 900000 # Refresh every 15 minutes
+          timeout: 5000
+```
+
+### File format
+
+We recommend using the `json` format for your account configuration. A minimal configuration may look like this.
+
+```json
+{
+  "id" : "979c7116-1f5a-43d4-9a87-5da3ccc4f52c",
+  "status" : "active"
+}
+```
+
+This pairs nicely if you have a default configuration defined in your prebid server config under `settings.default-account-config`.
+
 ## Setting Account Configuration in the Database
 
 In database approach account properties are stored in database table(s).

docs/developers/code-reviews.md

@@ -43,3 +43,4 @@ explaining it. Are there better ways to achieve those goals?
 - Does the code use any global, mutable state? [Inject dependencies](https://en.wikipedia.org/wiki/Dependency_injection) instead!
 - Can the code be organized into smaller, more modular pieces?
 - Is there dead code which can be deleted? Or TODO comments which should be resolved?
+- Look for code used by other adapters. Encourage adapter submitter to utilize common code. 

extra/bundle/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.prebid</groupId>
         <artifactId>prebid-server-aggregator</artifactId>
-        <version>3.12.0-SNAPSHOT</version>
+        <version>3.13.0-SNAPSHOT</version>
         <relativePath>../../extra/pom.xml</relativePath>
     </parent>
 
@@ -40,6 +40,11 @@
             <artifactId>pb-richmedia-filter</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.prebid.server.hooks.modules</groupId>
+            <artifactId>pb-response-correction</artifactId>
+            <version>${project.version}</version>
+        </dependency>
     </dependencies>
 
     <build>

extra/modules/confiant-ad-quality/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.prebid.server.hooks.modules</groupId>
         <artifactId>all-modules</artifactId>
-        <version>3.12.0-SNAPSHOT</version>
+        <version>3.13.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>confiant-ad-quality</artifactId>

extra/modules/confiant-ad-quality/src/test/java/org/prebid/server/hooks/modules/com/confiant/adquality/v1/ConfiantAdQualityBidResponsesScanHookTest.java

@@ -71,11 +71,7 @@ public void setUp() {
 
     @Test
     public void codeShouldHaveValidConfigsWhenInitialized() {
-        // given
-
-        // when
-
-        // then
+        // when and then
         assertThat(target.code()).isEqualTo("confiant-ad-quality-bid-responses-scan-hook");
     }
 

extra/modules/confiant-ad-quality/src/test/java/org/prebid/server/hooks/modules/com/confiant/adquality/v1/ConfiantAdQualityModuleTest.java

@@ -8,11 +8,7 @@ public class ConfiantAdQualityModuleTest {
 
     @Test
     public void shouldHaveValidInitialConfigs() {
-        // given
-
-        // when
-
-        // then
+        // when and then
         assertThat(ConfiantAdQualityModule.CODE).isEqualTo("confiant-ad-quality");
     }
 }

extra/modules/fiftyone-devicedetection/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.prebid.server.hooks.modules</groupId>
         <artifactId>all-modules</artifactId>
-        <version>3.12.0-SNAPSHOT</version>
+        <version>3.13.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>fiftyone-devicedetection</artifactId>

extra/modules/fiftyone-devicedetection/src/test/java/org/prebid/server/hooks/modules/fiftyone/devicedetection/v1/hooks/FiftyOneDeviceDetectionRawAuctionRequestHookTest.java

@@ -402,7 +402,6 @@ public void callShouldReturnUpdateActionWhenFilterIsNull() {
     @Test
     public void callShouldReturnUpdateActionWhenNoWhitelistAndNoAuctionContext() {
         // given
-
         final AuctionInvocationContext context = AuctionInvocationContextImpl.of(
                 null,
                 null,
@@ -470,7 +469,6 @@ public void callShouldReturnNoUpdateActionWhenWhitelistFilledAndNoAuctionContext
     @Test
     public void callShouldReturnUpdateActionWhenNoWhitelistAndNoAccount() {
         // given
-
         final AuctionContext auctionContext = AuctionContext.builder().build();
         final AuctionInvocationContext context = AuctionInvocationContextImpl.of(
                 null,
@@ -493,7 +491,6 @@ public void callShouldReturnUpdateActionWhenNoWhitelistAndNoAccount() {
     @Test
     public void callShouldReturnNoUpdateActionWhenNoWhitelistAndNoAccountButDeviceIdIsSet() {
         // given
-
         final AuctionContext auctionContext = AuctionContext.builder().build();
         final AuctionInvocationContext context = AuctionInvocationContextImpl.of(
                 null,
@@ -568,7 +565,6 @@ public void callShouldReturnNoUpdateActionWhenWhitelistFilledAndNoAccount() {
     @Test
     public void callShouldReturnUpdateActionWhenNoWhitelistAndNoAccountID() {
         // given
-
         final AuctionContext auctionContext = AuctionContext.builder()
                 .account(Account.builder()
                         .build())
@@ -648,7 +644,6 @@ public void callShouldReturnNoUpdateActionWhenWhitelistFilledAndNoAccountID() {
     @Test
     public void callShouldReturnUpdateActionWhenNoWhitelistAndEmptyAccountID() {
         // given
-
         final AuctionContext auctionContext = AuctionContext.builder()
                 .account(Account.builder()
                         .id("")
@@ -731,7 +726,6 @@ public void callShouldReturnNoUpdateActionWhenWhitelistFilledAndEmptyAccountID()
     @Test
     public void callShouldReturnUpdateActionWhenNoWhitelistAndAllowedAccountID() {
         // given
-
         final AuctionContext auctionContext = AuctionContext.builder()
                 .account(Account.builder()
                         .id("42")
@@ -814,7 +808,6 @@ public void callShouldReturnUpdateActionWhenWhitelistFilledAndAllowedAccountID()
     @Test
     public void callShouldReturnUpdateActionWhenNoWhitelistAndNotAllowedAccountID() {
         // given
-
         final AuctionContext auctionContext = AuctionContext.builder()
                 .account(Account.builder()
                         .id("29")

extra/modules/ortb2-blocking/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.prebid.server.hooks.modules</groupId>
         <artifactId>all-modules</artifactId>
-        <version>3.12.0-SNAPSHOT</version>
+        <version>3.13.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>ortb2-blocking</artifactId>

extra/modules/ortb2-blocking/src/main/java/org/prebid/server/hooks/modules/ortb2/blocking/v1/Ortb2BlockingBidderRequestHook.java

@@ -42,7 +42,8 @@ public Future<InvocationResult<BidderRequestPayload>> call(BidderRequestPayload
         final BidRequest bidRequest = bidderRequestPayload.bidRequest();
 
         final ModuleContext moduleContext = moduleContext(invocationContext)
-                .with(bidder, bidderSupportedOrtbVersion(bidder, aliases(bidRequest)));
+                .with(bidder, bidderSupportedOrtbVersion(
+                        bidder, aliases(invocationContext.auctionContext().getBidRequest())));
 
         final ExecutionResult<BlockedAttributes> blockedAttributesResult = BlockedAttributesResolver
                 .create(

extra/modules/ortb2-blocking/src/test/java/org/prebid/server/hooks/modules/ortb2/blocking/v1/Ortb2BlockingBidderRequestHookTest.java

@@ -36,11 +36,14 @@
 import org.prebid.server.hooks.v1.bidder.BidderRequestPayload;
 import org.prebid.server.spring.config.bidder.model.Ortb;
 
+import java.util.Map;
+
 import static java.util.Arrays.asList;
 import static java.util.Collections.singletonList;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.SoftAssertions.assertSoftly;
 import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.BDDMockito.given;
 
 @ExtendWith(MockitoExtension.class)
@@ -50,10 +53,10 @@ public class Ortb2BlockingBidderRequestHookTest {
             .setPropertyNamingStrategy(PropertyNamingStrategies.KEBAB_CASE)
             .setSerializationInclusion(JsonInclude.Include.NON_NULL);
 
-    @Mock
+    @Mock(strictness = Mock.Strictness.LENIENT)
     private BidderCatalog bidderCatalog;
 
-    @Mock
+    @Mock(strictness = Mock.Strictness.LENIENT)
     private BidRejectionTracker bidRejectionTracker;
 
     private Ortb2BlockingBidderRequestHook hook;
@@ -68,10 +71,21 @@ public void setUp() {
 
     @Test
     public void shouldReturnResultWithNoActionWhenNoBlockingAttributes() {
+        // given
+        given(bidderCatalog.bidderInfoByName(anyString()))
+                .willReturn(bidderInfo(OrtbVersion.ORTB_2_6));
+        given(bidderCatalog.bidderInfoByName(eq("bidder1Base")))
+                .willReturn(bidderInfo(OrtbVersion.ORTB_2_5));
+
         // when
         final Future<InvocationResult<BidderRequestPayload>> result = hook.call(
                 BidderRequestPayloadImpl.of(emptyRequest()),
-                BidderInvocationContextImpl.of("bidder1", bidRejectionTracker, null, true));
+                BidderInvocationContextImpl.of(
+                        "bidder1",
+                        Map.of("bidder1", "bidder1Base"),
+                        bidRejectionTracker,
+                        null,
+                        true));
 
         // then
         assertThat(result.succeeded()).isTrue();
@@ -258,6 +272,7 @@ private static BidderInfo bidderInfo(OrtbVersion ortbVersion) {
                 null,
                 null,
                 0,
+                null,
                 false,
                 false,
                 null,

extra/modules/ortb2-blocking/src/test/java/org/prebid/server/hooks/modules/ortb2/blocking/v1/model/BidderInvocationContextImpl.java

@@ -1,6 +1,7 @@
 package org.prebid.server.hooks.modules.ortb2.blocking.v1.model;
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.iab.openrtb.request.BidRequest;
 import lombok.Builder;
 import lombok.Value;
 import lombok.experimental.Accessors;
@@ -9,6 +10,8 @@
 import org.prebid.server.execution.Timeout;
 import org.prebid.server.hooks.v1.bidder.BidderInvocationContext;
 import org.prebid.server.model.Endpoint;
+import org.prebid.server.proto.openrtb.ext.request.ExtRequest;
+import org.prebid.server.proto.openrtb.ext.request.ExtRequestPrebid;
 
 import java.util.Map;
 
@@ -39,6 +42,28 @@ public static BidderInvocationContext of(String bidder,
         return BidderInvocationContextImpl.builder()
                 .bidder(bidder)
                 .auctionContext(AuctionContext.builder()
+                        .bidRequest(BidRequest.builder().build())
+                        .bidRejectionTrackers(Map.of(bidder, bidRejectionTracker))
+                        .build())
+                .accountConfig(accountConfig)
+                .debugEnabled(debugEnabled)
+                .build();
+    }
+
+    public static BidderInvocationContext of(String bidder,
+                                             Map<String, String> aliases,
+                                             BidRejectionTracker bidRejectionTracker,
+                                             ObjectNode accountConfig,
+                                             boolean debugEnabled) {
+
+        return BidderInvocationContextImpl.builder()
+                .bidder(bidder)
+                .auctionContext(AuctionContext.builder()
+                        .bidRequest(BidRequest.builder()
+                                .ext(ExtRequest.of(ExtRequestPrebid.builder()
+                                        .aliases(aliases)
+                                        .build()))
+                                .build())
                         .bidRejectionTrackers(Map.of(bidder, bidRejectionTracker))
                         .build())
                 .accountConfig(accountConfig)

extra/modules/pb-response-correction/pom.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.prebid.server.hooks.modules</groupId>
+        <artifactId>all-modules</artifactId>
+        <version>3.13.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>pb-response-correction</artifactId>
+
+    <name>pb-response-correction</name>
+    <description>Response correction module</description>
+</project>

extra/modules/pb-response-correction/src/lombok.config

@@ -0,0 +1 @@
+lombok.anyConstructor.addConstructorProperties = true