Merge remote-tracking branch 'origin/master' into functional-tests-fo…

…r-exponential-backoff

# Conflicts:
#	src/test/groovy/org/prebid/server/functional/testcontainers/scaffolding/VendorList.groovy
#	src/test/groovy/org/prebid/server/functional/tests/privacy/PrivacyBaseSpec.groovy

.github/workflows/pr-functional-tests.yml

@@ -20,7 +20,7 @@ jobs:
         java: [ 17 ]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up JDK
         uses: actions/setup-java@v3

.github/workflows/pr-java-ci.yml

@@ -20,7 +20,7 @@ jobs:
         java: [ 17 ]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up JDK
         uses: actions/setup-java@v3

.github/workflows/pr-module-functional-tests.yml

@@ -20,7 +20,7 @@ jobs:
         java: [ 17 ]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up JDK
         uses: actions/setup-java@v3
@@ -33,4 +33,4 @@ jobs:
         run: mvn package -DskipUnitTests=true --file extra/pom.xml
 
       - name: Run module tests
-        run: mvn -B verify -DskipUnitTests=true -DskipFunctionalTests=true -Dtests.max-container-count=5 -DdockerfileName=Dockerfile-modules --file extra/pom.xml
+        run: mvn -B verify -DskipUnitTests=true -DskipFunctionalTests=true -DskipModuleFunctionalTests=false -Dtests.max-container-count=5 -DdockerfileName=Dockerfile-modules --file extra/pom.xml

.github/workflows/release-asset-publish.yml

@@ -14,7 +14,7 @@ jobs:
       matrix:
         java: [ 17 ]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK
         uses: actions/setup-java@v3
         with:

.github/workflows/trivy-security-check.yml

@@ -0,0 +1,27 @@
+name: Security Check
+
+on:
+  pull_request:
+    branches: [master]
+
+jobs:
+  build:
+    name: Trivy security check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: 'fs'
+          ignore-unfixed: true
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+          severity: 'CRITICAL,HIGH'
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: 'trivy-results.sarif'

docs/application-settings.md

@@ -54,6 +54,9 @@ Keep in mind following restrictions:
   to `sfN.enforce` value.
 - `privacy.gdpr.purpose-one-treatment-interpretation` - option that allows to skip the Purpose one enforcement workflow.
   Values: ignore, no-access-allowed, access-allowed.
+- `privacy.gdpr.purposes.p4.eid.require_consent` - if equals to `true`, transmitting EIDs require P4 legal basis unless excepted.
+- `privacy.gdpr.purposes.p4.eid.exceptions` - list of EID sources that are excepted from P4 enforcement and will be transmitted if any P2-P10 is consented.
+- `privacy.gdpr.purposes.p4.eid.activity_transition` - defaults to `true`. If `true` and transmitEids is not specified, but transmitUfpd is specified, then the logic of transmitUfpd is used. This is to avoid breaking changes to existing configurations. The default value of the flag will be changed in a future release. 
 - `metrics.verbosity-level` - defines verbosity level of metrics for this account, overrides `metrics.accounts` application settings configuration. 
 - `analytics.auction-events.<channel>` - defines which channels are supported by analytics for this account
 - `analytics.modules.<module-name>.*` - space for `module-name` analytics module specific configuration, may be of any shape

docs/developers/functional-tests.md

@@ -46,7 +46,7 @@ You have two options for running modular tests:
 1. Use `mvn verify -DdockerfileName=Dockerfile-modules` to include all previous steps (including Java tests and functional tests) because Groovy runs in the `failsafe:integration-test` phase.
 2. For modular tests only, use a more granular command:
 
-`mvn -B verify -DskipUnitTests=true -DskipFunctionalTests=true -DdockerfileName=Dockerfile-modules`
+`mvn -B verify -DskipUnitTests=true -DskipFunctionalTests=true -DskipModuleFunctionalTests=false -DdockerfileName=Dockerfile-modules`
 
 ## Developing
 

docs/metrics.md

@@ -106,8 +106,6 @@ Following metrics are collected and submitted if account is configured with `det
 
 ## /cookie_sync endpoint metrics
 - `cookie_sync_requests` - number of requests received
-- `cookie_sync.<bidder-name>.gen` - number of times cookies was synced per bidder 
-- `cookie_sync.<bidder-name>.matches` - number of times cookie was already matched when synced per bidder 
 - `cookie_sync.<bidder-name>.tcf.blocked` - number of times cookie sync was prevented by TCF per bidder
 
 ## /setuid endpoint metrics

extra/bundle/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.prebid</groupId>
         <artifactId>prebid-server-aggregator</artifactId>
-        <version>2.12.0-SNAPSHOT</version>
+        <version>2.13.0-SNAPSHOT</version>
         <relativePath>../../extra/pom.xml</relativePath>
     </parent>
 

extra/modules/confiant-ad-quality/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.prebid.server.hooks.modules</groupId>
         <artifactId>all-modules</artifactId>
-        <version>2.12.0-SNAPSHOT</version>
+        <version>2.13.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>confiant-ad-quality</artifactId>

extra/modules/confiant-ad-quality/src/main/java/org/prebid/server/hooks/modules/com/confiant/adquality/config/ConfiantAdQualityModuleConfiguration.java

@@ -3,7 +3,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import io.vertx.core.Promise;
 import io.vertx.core.Vertx;
-import org.prebid.server.auction.PrivacyEnforcementService;
+import org.prebid.server.auction.privacy.enforcement.mask.UserFpdActivityMask;
 import org.prebid.server.hooks.modules.com.confiant.adquality.core.BidsScanner;
 import org.prebid.server.hooks.modules.com.confiant.adquality.core.RedisClient;
 import org.prebid.server.hooks.modules.com.confiant.adquality.core.RedisScanStateChecker;
@@ -20,6 +20,7 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.PropertySource;
 
+import java.util.Collections;
 import java.util.List;
 
 @ConditionalOnProperty(prefix = "hooks." + ConfiantAdQualityModule.CODE, name = "enabled", havingValue = "true")
@@ -37,7 +38,7 @@ ConfiantAdQualityModule confiantAdQualityModule(
             RedisConfig redisConfig,
             RedisRetryConfig retryConfig,
             Vertx vertx,
-            PrivacyEnforcementService privacyEnforcementService,
+            UserFpdActivityMask userFpdActivityMask,
             ObjectMapper objectMapper) {
 
         final RedisConnectionConfig writeNodeConfig = redisConfig.getWriteNode();
@@ -55,14 +56,14 @@ ConfiantAdQualityModule confiantAdQualityModule(
 
         bidsScanner.start(scannerPromise);
 
-        return new ConfiantAdQualityModule(List.of(
-                new ConfiantAdQualityBidResponsesScanHook(bidsScanner, biddersToExcludeFromScan, privacyEnforcementService)));
+        return new ConfiantAdQualityModule(Collections.singletonList(
+                new ConfiantAdQualityBidResponsesScanHook(bidsScanner, biddersToExcludeFromScan, userFpdActivityMask)));
     }
 
     @Bean
     ObjectMapper objectMapper() {
         return new ObjectMapper();
-    };
+    }
 
     @Bean
     @ConfigurationProperties(prefix = "hooks.modules.confiant-ad-quality.redis-config")

extra/modules/confiant-ad-quality/src/main/java/org/prebid/server/hooks/modules/com/confiant/adquality/core/RedisParser.java

@@ -41,7 +41,7 @@ public BidsScanResult parseBidsScanResult(String redisResponse) {
             } catch (JsonProcessingException errorParse) {
                 message = String.format("Error during parse redis response: %s", redisResponse);
             }
-            logger.info(message);
+            logger.warn(message);
             return BidsScanResult.builder()
                     .bidScanResults(Collections.emptyList())
                     .debugMessages(List.of(message))

extra/modules/confiant-ad-quality/src/main/java/org/prebid/server/hooks/modules/com/confiant/adquality/v1/ConfiantAdQualityBidResponsesScanHook.java

@@ -9,9 +9,9 @@
 import org.prebid.server.activity.infrastructure.payload.ActivityInvocationPayload;
 import org.prebid.server.activity.infrastructure.payload.impl.ActivityInvocationPayloadImpl;
 import org.prebid.server.activity.infrastructure.payload.impl.BidRequestActivityInvocationPayload;
-import org.prebid.server.auction.PrivacyEnforcementService;
 import org.prebid.server.auction.model.AuctionContext;
 import org.prebid.server.auction.model.BidderResponse;
+import org.prebid.server.auction.privacy.enforcement.mask.UserFpdActivityMask;
 import org.prebid.server.hooks.execution.v1.bidder.AllProcessedBidResponsesPayloadImpl;
 import org.prebid.server.hooks.modules.com.confiant.adquality.core.AnalyticsMapper;
 import org.prebid.server.hooks.modules.com.confiant.adquality.core.BidsMapper;
@@ -29,6 +29,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -37,19 +38,16 @@ public class ConfiantAdQualityBidResponsesScanHook implements AllProcessedBidRes
     private static final String CODE = "confiant-ad-quality-bid-responses-scan-hook";
 
     private final BidsScanner bidsScanner;
-
     private final List<String> biddersToExcludeFromScan;
+    private final UserFpdActivityMask userFpdActivityMask;
 
-    private final PrivacyEnforcementService privacyEnforcementService;
-
-    public ConfiantAdQualityBidResponsesScanHook(
-            BidsScanner bidsScanner,
-            List<String> biddersToExcludeFromScan,
-            PrivacyEnforcementService privacyEnforcementService) {
+    public ConfiantAdQualityBidResponsesScanHook(BidsScanner bidsScanner,
+                                                 List<String> biddersToExcludeFromScan,
+                                                 UserFpdActivityMask userFpdActivityMask) {
 
-        this.bidsScanner = bidsScanner;
-        this.biddersToExcludeFromScan = biddersToExcludeFromScan;
-        this.privacyEnforcementService = privacyEnforcementService;
+        this.bidsScanner = Objects.requireNonNull(bidsScanner);
+        this.biddersToExcludeFromScan = Objects.requireNonNull(biddersToExcludeFromScan);
+        this.userFpdActivityMask = Objects.requireNonNull(userFpdActivityMask);
     }
 
     @Override
@@ -60,7 +58,7 @@ public Future<InvocationResult<AllProcessedBidResponsesPayload>> call(
         final BidRequest bidRequest = getBidRequest(auctionInvocationContext);
         final List<BidderResponse> responses = allProcessedBidResponsesPayload.bidResponses();
         final Map<Boolean, List<BidderResponse>> needScanMap = responses.stream()
-                .collect(Collectors.groupingBy(bidderResponse -> !biddersToExcludeFromScan.contains(bidderResponse.getBidder())));
+                .collect(Collectors.groupingBy(this::isScanRequired));
 
         final List<BidderResponse> toScan = needScanMap.getOrDefault(true, Collections.emptyList());
         final List<BidderResponse> avoidScan = needScanMap.getOrDefault(false, Collections.emptyList());
@@ -69,6 +67,11 @@ public Future<InvocationResult<AllProcessedBidResponsesPayload>> call(
                 .map(scanResult -> toInvocationResult(scanResult, toScan, avoidScan, auctionInvocationContext));
     }
 
+    private boolean isScanRequired(BidderResponse bidderResponse) {
+        return !biddersToExcludeFromScan.contains(bidderResponse.getBidder())
+                && !bidderResponse.getSeatBid().getBids().isEmpty();
+    }
+
     private BidRequest getBidRequest(AuctionInvocationContext auctionInvocationContext) {
         final AuctionContext auctionContext = auctionInvocationContext.auctionContext();
         final BidRequest bidRequest = auctionContext.getBidRequest();
@@ -78,10 +81,8 @@ private BidRequest getBidRequest(AuctionInvocationContext auctionInvocationConte
         final boolean disallowTransmitGeo = !auctionContext.getActivityInfrastructure()
                 .isAllowed(Activity.TRANSMIT_GEO, activityInvocationPayload);
 
-        final User maskedUser = privacyEnforcementService
-                .maskUserConsideringActivityRestrictions(bidRequest.getUser(), true, disallowTransmitGeo);
-        final Device maskedDevice = privacyEnforcementService
-                .maskDeviceConsideringActivityRestrictions(bidRequest.getDevice(), true, disallowTransmitGeo);
+        final User maskedUser = userFpdActivityMask.maskUser(bidRequest.getUser(), true, true, disallowTransmitGeo);
+        final Device maskedDevice = userFpdActivityMask.maskDevice(bidRequest.getDevice(), true, disallowTransmitGeo);
 
         return bidRequest.toBuilder()
                 .user(maskedUser)