Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add TransmitEids activity #2901

Merged
merged 31 commits into from
Mar 6, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
6596893
Split `TRANSMIT_UFPD` to `TRANSMIT_UFPD` and `TRANSMIT_EIDS`
CTMBNara Jan 3, 2024
905bdc6
Add `p4.eid.activity_transition` field.
CTMBNara Jan 3, 2024
34f147a
Remove unnecessary logic.
CTMBNara Jan 3, 2024
4803586
Refactor `PrivacyEnforcementService`.
CTMBNara Jan 5, 2024
42bbbaa
Merge remote-tracking branch 'origin/master' into add-transmit-eids-a…
CTMBNara Jan 5, 2024
a420683
Refactor `Tcf2Service`.
CTMBNara Jan 9, 2024
d69125e
Merge remote-tracking branch 'origin/master' into add-transmit-eids-a…
CTMBNara Jan 9, 2024
2ea1489
Implement TCF2.2 Purpose4 logic.
CTMBNara Jan 10, 2024
1deee5f
Merge remote-tracking branch 'origin/master' into add-transmit-eids-a…
CTMBNara Jan 11, 2024
7d3a713
Refactoring.
CTMBNara Jan 12, 2024
d7ff32c
Add natural permissions for Purpose4.
CTMBNara Jan 12, 2024
de5c045
Add missing feature.
CTMBNara Jan 15, 2024
94c0c47
Merge remote-tracking branch 'origin/master' into add-transmit-eids-a…
CTMBNara Jan 15, 2024
a68f527
Refactoring.
CTMBNara Jan 15, 2024
cd1a803
Refactoring.
CTMBNara Jan 15, 2024
ebb9d14
Refactoring.
CTMBNara Jan 15, 2024
06866c1
Merge remote-tracking branch 'origin/master' into add-transmit-eids-a…
CTMBNara Jan 30, 2024
ad8611f
Fix functional tests.
CTMBNara Jan 30, 2024
d8f7404
Merge remote-tracking branch 'origin/master' into add-transmit-eids-a…
CTMBNara Feb 13, 2024
f3cda35
Resolve merge conflicts.
CTMBNara Feb 13, 2024
2a2b7a0
Fix eidExceptions usage.
CTMBNara Feb 15, 2024
0af9d24
Refactor.
CTMBNara Feb 15, 2024
959f43a
Merge remote-tracking branch 'origin/master' into add-transmit-eids-a…
CTMBNara Feb 27, 2024
ca963c1
Resolve merge conflicts.
CTMBNara Feb 27, 2024
4eaf393
Tests: Add functional tests for aligning TCF and tests for support tr…
osulzhenko Feb 29, 2024
e99b19b
Merge remote-tracking branch 'origin/master' into add-transmit-eids-a…
Feb 29, 2024
0f05f39
Resolve merge conflicts.
Feb 29, 2024
556167a
update failed tests
osulzhenko Feb 29, 2024
790a486
Optimize memory usage for tcf functional tests
osulzhenko Mar 1, 2024
21587b5
Tests: Fix memory leak issue (#3033)
osulzhenko Mar 6, 2024
7262bff
tests covered ratio fix
osulzhenko Mar 6, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import io.vertx.core.Promise;
import io.vertx.core.Vertx;
import org.prebid.server.auction.PrivacyEnforcementService;
import org.prebid.server.auction.privacy.enforcement.mask.UserFpdActivityMask;
import org.prebid.server.hooks.modules.com.confiant.adquality.core.BidsScanner;
import org.prebid.server.hooks.modules.com.confiant.adquality.core.RedisClient;
import org.prebid.server.hooks.modules.com.confiant.adquality.core.RedisScanStateChecker;
Expand All @@ -20,6 +20,7 @@
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.PropertySource;

import java.util.Collections;
import java.util.List;

@ConditionalOnProperty(prefix = "hooks." + ConfiantAdQualityModule.CODE, name = "enabled", havingValue = "true")
Expand All @@ -37,7 +38,7 @@ ConfiantAdQualityModule confiantAdQualityModule(
RedisConfig redisConfig,
RedisRetryConfig retryConfig,
Vertx vertx,
PrivacyEnforcementService privacyEnforcementService,
UserFpdActivityMask userFpdActivityMask,
ObjectMapper objectMapper) {

final RedisConnectionConfig writeNodeConfig = redisConfig.getWriteNode();
Expand All @@ -55,14 +56,14 @@ ConfiantAdQualityModule confiantAdQualityModule(

bidsScanner.start(scannerPromise);

return new ConfiantAdQualityModule(List.of(
new ConfiantAdQualityBidResponsesScanHook(bidsScanner, biddersToExcludeFromScan, privacyEnforcementService)));
return new ConfiantAdQualityModule(Collections.singletonList(
new ConfiantAdQualityBidResponsesScanHook(bidsScanner, biddersToExcludeFromScan, userFpdActivityMask)));
}

@Bean
ObjectMapper objectMapper() {
return new ObjectMapper();
};
}

@Bean
@ConfigurationProperties(prefix = "hooks.modules.confiant-ad-quality.redis-config")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@
import org.prebid.server.activity.infrastructure.payload.ActivityInvocationPayload;
import org.prebid.server.activity.infrastructure.payload.impl.ActivityInvocationPayloadImpl;
import org.prebid.server.activity.infrastructure.payload.impl.BidRequestActivityInvocationPayload;
import org.prebid.server.auction.PrivacyEnforcementService;
import org.prebid.server.auction.model.AuctionContext;
import org.prebid.server.auction.model.BidderResponse;
import org.prebid.server.auction.privacy.enforcement.mask.UserFpdActivityMask;
import org.prebid.server.hooks.execution.v1.bidder.AllProcessedBidResponsesPayloadImpl;
import org.prebid.server.hooks.modules.com.confiant.adquality.core.AnalyticsMapper;
import org.prebid.server.hooks.modules.com.confiant.adquality.core.BidsMapper;
Expand All @@ -29,6 +29,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.Stream;

Expand All @@ -37,19 +38,16 @@ public class ConfiantAdQualityBidResponsesScanHook implements AllProcessedBidRes
private static final String CODE = "confiant-ad-quality-bid-responses-scan-hook";

private final BidsScanner bidsScanner;

private final List<String> biddersToExcludeFromScan;
private final UserFpdActivityMask userFpdActivityMask;

private final PrivacyEnforcementService privacyEnforcementService;

public ConfiantAdQualityBidResponsesScanHook(
BidsScanner bidsScanner,
List<String> biddersToExcludeFromScan,
PrivacyEnforcementService privacyEnforcementService) {
public ConfiantAdQualityBidResponsesScanHook(BidsScanner bidsScanner,
List<String> biddersToExcludeFromScan,
UserFpdActivityMask userFpdActivityMask) {

this.bidsScanner = bidsScanner;
this.biddersToExcludeFromScan = biddersToExcludeFromScan;
this.privacyEnforcementService = privacyEnforcementService;
this.bidsScanner = Objects.requireNonNull(bidsScanner);
this.biddersToExcludeFromScan = Objects.requireNonNull(biddersToExcludeFromScan);
this.userFpdActivityMask = Objects.requireNonNull(userFpdActivityMask);
}

@Override
Expand Down Expand Up @@ -83,10 +81,8 @@ private BidRequest getBidRequest(AuctionInvocationContext auctionInvocationConte
final boolean disallowTransmitGeo = !auctionContext.getActivityInfrastructure()
.isAllowed(Activity.TRANSMIT_GEO, activityInvocationPayload);

final User maskedUser = privacyEnforcementService
.maskUserConsideringActivityRestrictions(bidRequest.getUser(), true, disallowTransmitGeo);
final Device maskedDevice = privacyEnforcementService
.maskDeviceConsideringActivityRestrictions(bidRequest.getDevice(), true, disallowTransmitGeo);
final User maskedUser = userFpdActivityMask.maskUser(bidRequest.getUser(), true, true, disallowTransmitGeo);
final Device maskedDevice = userFpdActivityMask.maskDevice(bidRequest.getDevice(), true, disallowTransmitGeo);

return bidRequest.toBuilder()
.user(maskedUser)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@
import org.mockito.junit.MockitoJUnit;
import org.mockito.junit.MockitoRule;
import org.prebid.server.activity.infrastructure.ActivityInfrastructure;
import org.prebid.server.auction.PrivacyEnforcementService;
import org.prebid.server.auction.model.AuctionContext;
import org.prebid.server.auction.model.BidderResponse;
import org.prebid.server.auction.privacy.enforcement.mask.UserFpdActivityMask;
import org.prebid.server.bidder.model.BidderSeatBid;
import org.prebid.server.hooks.execution.v1.bidder.AllProcessedBidResponsesPayloadImpl;
import org.prebid.server.hooks.modules.com.confiant.adquality.core.BidsMapper;
Expand Down Expand Up @@ -61,15 +61,15 @@ public class ConfiantAdQualityBidResponsesScanHookTest {
private ActivityInfrastructure activityInfrastructure;

@Mock
private PrivacyEnforcementService privacyEnforcementService;
private UserFpdActivityMask userFpdActivityMask;

private ConfiantAdQualityBidResponsesScanHook target;

private final RedisParser redisParser = new RedisParser(new ObjectMapper());

@Before
public void setUp() {
target = new ConfiantAdQualityBidResponsesScanHook(bidsScanner, List.of(), privacyEnforcementService);
target = new ConfiantAdQualityBidResponsesScanHook(bidsScanner, List.of(), userFpdActivityMask);
}

@Test
Expand Down Expand Up @@ -176,7 +176,7 @@ public void callShouldSubmitToScanBidsWhichAreNotPartOfTheExcludeToScanListWhenH
.bidRequest(BidRequest.builder().cur(List.of("USD")).build())
.build();

target = new ConfiantAdQualityBidResponsesScanHook(bidsScanner, List.of(secureBidderName), privacyEnforcementService);
target = new ConfiantAdQualityBidResponsesScanHook(bidsScanner, List.of(secureBidderName), userFpdActivityMask);

doReturn(List.of(secureBidderResponse, notSecureBadBidderResponse, notSecureGoodBidderResponse)).when(allProcessedBidResponsesPayload).bidResponses();
doReturn(Future.succeededFuture(bidsScanResult)).when(bidsScanner).submitBids(any());
Expand Down Expand Up @@ -234,7 +234,7 @@ public void callShouldSubmitToScanOnlyBidsWithDataWhenSomeBiddersRespondWithEmpt
.bidRequest(BidRequest.builder().cur(List.of("USD")).build())
.build();

target = new ConfiantAdQualityBidResponsesScanHook(bidsScanner, List.of(secureBidderName), privacyEnforcementService);
target = new ConfiantAdQualityBidResponsesScanHook(bidsScanner, List.of(secureBidderName), userFpdActivityMask);

doReturn(List.of(secureBidderResponse, notSecureBadBidderResponse, emptyBidderResponse)).when(allProcessedBidResponsesPayload).bidResponses();
doReturn(Future.succeededFuture(bidsScanResult)).when(bidsScanner).submitBids(any());
Expand Down Expand Up @@ -277,9 +277,9 @@ public void callShouldSubmitBidsWithoutMaskedGeoInfoWhenTransmitGeoIsAllowed() {
final Boolean transmitGeoIsAllowed = true;
final BidsScanResult bidsScanResult = redisParser.parseBidsScanResult(
"[[[{\"tag_key\": \"tag\", \"issues\":[{\"spec_name\":\"malicious_domain\",\"value\":\"ads.deceivenetworks.net\",\"first_adinstance\":\"e91e8da982bb8b7f80100426\"}]}]]]");
final User user = privacyEnforcementService.maskUserConsideringActivityRestrictions(
getUser(), true, !transmitGeoIsAllowed);
final Device device = privacyEnforcementService.maskDeviceConsideringActivityRestrictions(
final User user = userFpdActivityMask.maskUser(
getUser(), true, true, !transmitGeoIsAllowed);
final Device device = userFpdActivityMask.maskDevice(
getDevice(), true, !transmitGeoIsAllowed);

bidsScanner.enableScan();
Expand All @@ -306,9 +306,9 @@ public void callShouldSubmitBidsWithMaskedGeoInfoWhenTransmitGeoIsNotAllowed() {
final Boolean transmitGeoIsAllowed = false;
final BidsScanResult bidsScanResult = redisParser.parseBidsScanResult(
"[[[{\"tag_key\": \"tag\", \"issues\":[{\"spec_name\":\"malicious_domain\",\"value\":\"ads.deceivenetworks.net\",\"first_adinstance\":\"e91e8da982bb8b7f80100426\"}]}]]]");
final User user = privacyEnforcementService.maskUserConsideringActivityRestrictions(
getUser(), true, !transmitGeoIsAllowed);
final Device device = privacyEnforcementService.maskDeviceConsideringActivityRestrictions(
final User user = userFpdActivityMask.maskUser(
getUser(), true, true, !transmitGeoIsAllowed);
final Device device = userFpdActivityMask.maskDevice(
getDevice(), true, !transmitGeoIsAllowed);

bidsScanner.enableScan();
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/com/iab/openrtb/request/User.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
@Value
public class User {

public static final User EMPTY = User.builder().build();

/**
* Exchange-specific ID for the user.
*/
Expand Down
3 changes: 3 additions & 0 deletions src/main/java/org/prebid/server/activity/Activity.java
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ public enum Activity {
@JsonProperty("transmitUfpd")
TRANSMIT_UFPD,

@JsonProperty("transmitEids")
TRANSMIT_EIDS,

@JsonProperty("transmitPreciseGeo")
TRANSMIT_GEO,

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,12 @@
import org.prebid.server.metric.Metrics;
import org.prebid.server.proto.openrtb.ext.request.TraceLevel;
import org.prebid.server.settings.model.Account;
import org.prebid.server.settings.model.AccountGdprConfig;
import org.prebid.server.settings.model.AccountPrivacyConfig;
import org.prebid.server.settings.model.GdprConfig;
import org.prebid.server.settings.model.Purpose;
import org.prebid.server.settings.model.PurposeEid;
import org.prebid.server.settings.model.Purposes;
import org.prebid.server.settings.model.activity.AccountActivityConfiguration;
import org.prebid.server.settings.model.activity.privacy.AccountPrivacyModuleConfig;

Expand All @@ -27,6 +32,7 @@
import java.util.Objects;
import java.util.Optional;
import java.util.function.BinaryOperator;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.function.UnaryOperator;
import java.util.stream.Collectors;
Expand All @@ -36,14 +42,20 @@ public class ActivityInfrastructureCreator {
private static final Logger logger = LoggerFactory.getLogger(ActivityInfrastructureCreator.class);

private final ActivityRuleFactory activityRuleFactory;
private final Purpose defaultPurpose4;
private final Metrics metrics;
private final JacksonMapper jacksonMapper;

public ActivityInfrastructureCreator(ActivityRuleFactory activityRuleFactory,
GdprConfig gdprConfig,
Metrics metrics,
JacksonMapper jacksonMapper) {

this.activityRuleFactory = Objects.requireNonNull(activityRuleFactory);
this.defaultPurpose4 = Optional.ofNullable(gdprConfig)
.map(GdprConfig::getPurposes)
.map(Purposes::getP4)
.orElse(null);
this.metrics = Objects.requireNonNull(metrics);
this.jacksonMapper = Objects.requireNonNull(jacksonMapper);
}
Expand Down Expand Up @@ -74,12 +86,15 @@ Map<Activity, ActivityController> parse(Account account, GppContext gppContext,

return Arrays.stream(Activity.values()).collect(Collectors.toMap(
UnaryOperator.identity(),
activity -> from(
activity,
activitiesConfiguration.get(activity),
modulesConfigs,
gppContext,
debug),
fallbackActivity(
activitiesConfiguration,
accountPrivacyConfig,
activity -> from(
activity,
activitiesConfiguration.get(activity),
modulesConfigs,
gppContext,
debug)),
(oldValue, newValue) -> oldValue,
enumMapFactory()));
}
Expand All @@ -94,6 +109,28 @@ private BinaryOperator<AccountPrivacyModuleConfig> takeFirstAndLogDuplicates(Str
};
}

// TODO: remove this wrapper after transition period
private Function<Activity, ActivityController> fallbackActivity(
Map<Activity, AccountActivityConfiguration> activitiesConfiguration,
Optional<AccountPrivacyConfig> accountPrivacyConfig,
Function<Activity, ActivityController> activityControllerCreator) {

final boolean imitateTransmitEids = !activitiesConfiguration.containsKey(Activity.TRANSMIT_EIDS)
&& activitiesConfiguration.containsKey(Activity.TRANSMIT_UFPD)
&& accountPrivacyConfig
.map(AccountPrivacyConfig::getGdpr)
.map(AccountGdprConfig::getPurposes)
.map(Purposes::getP4)
.or(() -> Optional.ofNullable(defaultPurpose4))
.map(Purpose::getEid)
.map(PurposeEid::getActivityTransition)
.orElse(true);

return originalActivity -> originalActivity == Activity.TRANSMIT_EIDS && imitateTransmitEids
? activityControllerCreator.apply(Activity.TRANSMIT_UFPD)
: activityControllerCreator.apply(originalActivity);
}

private ActivityController from(Activity activity,
AccountActivityConfiguration activityConfiguration,
Map<PrivacyModuleQualifier, AccountPrivacyModuleConfig> modulesConfigs,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public USNatModule(Activity activity, USNatGppReader gppReader) {
private static PrivacyModule innerModule(Activity activity, USNatGppReader gppReader) {
return switch (activity) {
case SYNC_USER, MODIFY_UFDP -> new USNatSyncUser(gppReader);
case TRANSMIT_UFPD -> new USNatTransmitUfpd(gppReader);
case TRANSMIT_UFPD, TRANSMIT_EIDS -> new USNatTransmitUfpd(gppReader);
case TRANSMIT_GEO -> new USNatTransmitGeo(gppReader);
case CALL_BIDDER, TRANSMIT_TID, REPORT_ANALYTICS -> USNatDefault.instance();
};
Expand Down
Loading
Loading