From a475313e8a529215925b4975ac827f8393e0bee4 Mon Sep 17 00:00:00 2001 From: william-harris Date: Tue, 26 Mar 2024 15:54:37 +0100 Subject: [PATCH] Yieldlab: Add Digital Service Act (DSA) support (#3011) --- .../bidder/yieldlab/YieldlabBidder.java | 94 +++++++- .../YieldlabDigitalServicesActResponse.java | 26 +++ .../yieldlab/model/YieldlabResponse.java | 2 + .../bidder/yieldlab/YieldlabBidderTest.java | 203 +++++++++++++++++- 4 files changed, 322 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/prebid/server/bidder/yieldlab/model/YieldlabDigitalServicesActResponse.java diff --git a/src/main/java/org/prebid/server/bidder/yieldlab/YieldlabBidder.java b/src/main/java/org/prebid/server/bidder/yieldlab/YieldlabBidder.java index bd957cd0ee0..6a547009892 100644 --- a/src/main/java/org/prebid/server/bidder/yieldlab/YieldlabBidder.java +++ b/src/main/java/org/prebid/server/bidder/yieldlab/YieldlabBidder.java @@ -2,6 +2,8 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.iab.openrtb.request.App; import com.iab.openrtb.request.BidRequest; import com.iab.openrtb.request.Device; @@ -14,6 +16,9 @@ import io.netty.handler.codec.http.HttpHeaderValues; import io.vertx.core.MultiMap; import io.vertx.core.http.HttpMethod; +import io.vertx.core.logging.Logger; +import io.vertx.core.logging.LoggerFactory; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.http.client.utils.URIBuilder; @@ -23,11 +28,15 @@ import org.prebid.server.bidder.model.BidderError; import org.prebid.server.bidder.model.HttpRequest; import org.prebid.server.bidder.model.Result; +import org.prebid.server.bidder.yieldlab.model.YieldlabDigitalServicesActResponse; import org.prebid.server.bidder.yieldlab.model.YieldlabResponse; import org.prebid.server.exception.PreBidException; import org.prebid.server.json.DecodeException; import org.prebid.server.json.JacksonMapper; import org.prebid.server.proto.openrtb.ext.ExtPrebid; +import org.prebid.server.proto.openrtb.ext.request.ExtRegs; +import org.prebid.server.proto.openrtb.ext.request.ExtRegsDsa; +import org.prebid.server.proto.openrtb.ext.request.ExtRegsDsaTransparency; import org.prebid.server.proto.openrtb.ext.request.ExtRequest; import org.prebid.server.proto.openrtb.ext.request.ExtRequestPrebid; import org.prebid.server.proto.openrtb.ext.request.ExtUser; @@ -41,13 +50,16 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.stream.Collectors; public class YieldlabBidder implements Bidder { + private static final Logger logger = LoggerFactory.getLogger(YieldlabBidder.class); private static final TypeReference> YIELDLAB_EXT_TYPE_REFERENCE = new TypeReference<>() { }; @@ -58,6 +70,9 @@ public class YieldlabBidder implements Bidder { private static final String CREATIVE_ID = "%s%s%s"; private static final String AD_SOURCE_BANNER = ""; private static final String AD_SOURCE_URL = "https://ad.yieldlab.net/d/%s/%s/%s?%s"; + private static final String TRANSPARENCY_TEMPLATE = "%s~%s"; + private static final String TRANSPARENCY_TEMPLATE_PARAMS_DELIMITER = "_"; + private static final String TRANSPARENCY_TEMPLATE_DELIMITER = "~~"; private static final String VAST_MARKUP = """ Yieldlab @@ -189,6 +204,8 @@ private String makeUrl(ExtImpYieldlab extImpYieldlab, BidRequest request) { uriBuilder.addParameter("consent", consent); } + extractDsaRequestParamsFromBidRequest(request).forEach(uriBuilder::addParameter); + return uriBuilder.toString(); } @@ -231,6 +248,63 @@ private static String getConsentParameter(User user) { return ObjectUtils.defaultIfNull(consent, ""); } + private static Map extractDsaRequestParamsFromBidRequest(BidRequest request) { + return Optional.ofNullable(request.getRegs()) + .map(Regs::getExt) + .map(ExtRegs::getDsa) + .map(YieldlabBidder::extractDsaRequestParamsFromDsaRegsExtension) + .orElse(Collections.emptyMap()); + } + + private static Map extractDsaRequestParamsFromDsaRegsExtension(final ExtRegsDsa dsa) { + final Map dsaRequestParams = new HashMap<>(); + + if (dsa.getDsaRequired() != null) { + dsaRequestParams.put("dsarequired", dsa.getDsaRequired().toString()); + } + + if (dsa.getPubRender() != null) { + dsaRequestParams.put("dsapubrender", dsa.getPubRender().toString()); + } + + if (dsa.getDataToPub() != null) { + dsaRequestParams.put("dsadatatopub", dsa.getDataToPub().toString()); + } + + final List dsaTransparency = dsa.getTransparency(); + if (CollectionUtils.isNotEmpty(dsaTransparency)) { + final String encodedTransparencies = encodeTransparenciesAsString(dsaTransparency); + if (StringUtils.isNotBlank(encodedTransparencies)) { + dsaRequestParams.put("dsatransparency", encodedTransparencies); + } + } + + return dsaRequestParams; + } + + private static String encodeTransparenciesAsString(List transparencies) { + return transparencies.stream() + .filter(YieldlabBidder::isTransparencyValid) + .map(YieldlabBidder::encodeTransparency) + .collect(Collectors.joining(TRANSPARENCY_TEMPLATE_DELIMITER)); + } + + private static boolean isTransparencyValid(ExtRegsDsaTransparency transparency) { + return StringUtils.isNotBlank(transparency.getDomain()) + && transparency.getDsaParams() != null + && CollectionUtils.isNotEmpty(transparency.getDsaParams()); + } + + private static String encodeTransparency(ExtRegsDsaTransparency transparency) { + return TRANSPARENCY_TEMPLATE.formatted(transparency.getDomain(), + encodeTransparencyParams(transparency.getDsaParams())); + } + + private static String encodeTransparencyParams(List dsaParams) { + return dsaParams.stream().map(Objects::toString).collect(Collectors.joining( + TRANSPARENCY_TEMPLATE_PARAMS_DELIMITER)); + } + private static MultiMap resolveHeaders(Site site, Device device, User user) { final MultiMap headers = MultiMap.caseInsensitiveMultiMap() .add(HttpUtil.ACCEPT_HEADER, HttpHeaderValues.APPLICATION_JSON); @@ -339,7 +413,8 @@ private Bid.BidBuilder addBidParams(YieldlabResponse yieldlabResponse, BidReques .dealid(resolveNumberParameter(yieldlabResponse.getPid())) .crid(makeCreativeId(bidRequest, yieldlabResponse, matchedExtImp)) .w(resolveSizeParameter(yieldlabResponse.getAdSize(), true)) - .h(resolveSizeParameter(yieldlabResponse.getAdSize(), false)); + .h(resolveSizeParameter(yieldlabResponse.getAdSize(), false)) + .ext(resolveExtParameter(yieldlabResponse)); return updatedBid; } @@ -408,4 +483,21 @@ private String makeNurl(BidRequest bidRequest, ExtImpYieldlab extImpYieldlab, Yi yieldlabResponse.getAdSize(), uriBuilder.toString().replace("?", "")); } + + private ObjectNode resolveExtParameter(YieldlabResponse yieldlabResponse) { + final YieldlabDigitalServicesActResponse dsa = yieldlabResponse.getDsa(); + if (dsa == null) { + return null; + } + final ObjectNode ext = mapper.mapper().createObjectNode(); + final JsonNode dsaNode; + try { + dsaNode = mapper.mapper().valueToTree(dsa); + } catch (IllegalArgumentException e) { + logger.error("Failed to serialize DSA object for adslot {}", yieldlabResponse.getId(), e); + return null; + } + ext.set("dsa", dsaNode); + return ext; + } } diff --git a/src/main/java/org/prebid/server/bidder/yieldlab/model/YieldlabDigitalServicesActResponse.java b/src/main/java/org/prebid/server/bidder/yieldlab/model/YieldlabDigitalServicesActResponse.java new file mode 100644 index 00000000000..4fd18d813df --- /dev/null +++ b/src/main/java/org/prebid/server/bidder/yieldlab/model/YieldlabDigitalServicesActResponse.java @@ -0,0 +1,26 @@ +package org.prebid.server.bidder.yieldlab.model; + +import lombok.AllArgsConstructor; +import lombok.Value; + +import java.util.List; + +@AllArgsConstructor(staticName = "of") +@Value(staticConstructor = "of") +public class YieldlabDigitalServicesActResponse { + + String behalf; + + String paid; + + Integer adrender; + + List transparency; + + @AllArgsConstructor(staticName = "of") + @Value(staticConstructor = "of") + public static class Transparency { + String domain; + List dsaparams; + } +} diff --git a/src/main/java/org/prebid/server/bidder/yieldlab/model/YieldlabResponse.java b/src/main/java/org/prebid/server/bidder/yieldlab/model/YieldlabResponse.java index 4cd1dbf7294..9a2b54652ad 100644 --- a/src/main/java/org/prebid/server/bidder/yieldlab/model/YieldlabResponse.java +++ b/src/main/java/org/prebid/server/bidder/yieldlab/model/YieldlabResponse.java @@ -22,4 +22,6 @@ public class YieldlabResponse { Integer did; String pvid; + + YieldlabDigitalServicesActResponse dsa; } diff --git a/src/test/java/org/prebid/server/bidder/yieldlab/YieldlabBidderTest.java b/src/test/java/org/prebid/server/bidder/yieldlab/YieldlabBidderTest.java index 969f4b3605d..b16852a80a0 100644 --- a/src/test/java/org/prebid/server/bidder/yieldlab/YieldlabBidderTest.java +++ b/src/test/java/org/prebid/server/bidder/yieldlab/YieldlabBidderTest.java @@ -1,6 +1,9 @@ package org.prebid.server.bidder.yieldlab; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.iab.openrtb.request.Banner; import com.iab.openrtb.request.BidRequest; import com.iab.openrtb.request.Device; @@ -19,9 +22,12 @@ import org.prebid.server.bidder.model.HttpRequest; import org.prebid.server.bidder.model.HttpResponse; import org.prebid.server.bidder.model.Result; +import org.prebid.server.bidder.yieldlab.model.YieldlabDigitalServicesActResponse; import org.prebid.server.bidder.yieldlab.model.YieldlabResponse; import org.prebid.server.proto.openrtb.ext.ExtPrebid; import org.prebid.server.proto.openrtb.ext.request.ExtRegs; +import org.prebid.server.proto.openrtb.ext.request.ExtRegsDsa; +import org.prebid.server.proto.openrtb.ext.request.ExtRegsDsaTransparency; import org.prebid.server.proto.openrtb.ext.request.ExtUser; import org.prebid.server.proto.openrtb.ext.request.yieldlab.ExtImpYieldlab; import org.prebid.server.proto.openrtb.ext.response.BidType; @@ -226,7 +232,7 @@ public void makeBidsShouldReturnCorrectBidderBid() throws JsonProcessingExceptio .build(); final YieldlabResponse yieldlabResponse = YieldlabResponse.of(1, 201d, "yieldlab", - "728x90", 1234, 5678, "40cb3251-1e1e-4cfd-8edc-7d32dc1a21e5"); + "728x90", 1234, 5678, "40cb3251-1e1e-4cfd-8edc-7d32dc1a21e5", null); final BidderCall httpCall = givenHttpCall(mapper.writeValueAsString(yieldlabResponse)); @@ -274,7 +280,7 @@ public void makeBidsShouldReturnCorrectAdm() throws JsonProcessingException { .build(); final YieldlabResponse yieldlabResponse = YieldlabResponse.of(12345, 201d, "yieldlab", - "728x90", 1234, 5678, "40cb3251-1e1e-4cfd-8edc-7d32dc1a21e5"); + "728x90", 1234, 5678, "40cb3251-1e1e-4cfd-8edc-7d32dc1a21e5", null); final BidderCall httpCall = givenHttpCall(mapper.writeValueAsString(yieldlabResponse)); @@ -301,6 +307,199 @@ public void makeBidsShouldReturnCorrectAdm() throws JsonProcessingException { .containsExactly(expectedAdm); } + @Test + public void makeHttpRequestsShouldAddDsaRequestParamsToRequestWhenDsaIsPresent() { + //given + final ExtRegsDsa dsa = ExtRegsDsa.of( + 1, 2, 3, List.of(ExtRegsDsaTransparency.of("testDomain", List.of(1, 2, 3))) + ); + final Regs regs = Regs.builder() + .ext(ExtRegs.of(null, null, null, dsa)) + .build(); + final BidRequest bidRequest = BidRequest.builder() + .imp(singletonList(Imp.builder() + .ext(mapper.valueToTree(ExtPrebid.of(null, + ExtImpYieldlab.builder() + .adslotId("123") + .build()))) + .build())) + .regs(regs) + .build(); + + //when + final Result>> result = target.makeHttpRequests(bidRequest); + + //then + final List expectations = List.of( + "&dsarequired=1", + "&dsapubrender=2", + "&dsadatatopub=3", + "&dsatransparency=testDomain%7E1_2_3" + ); + assertThat(result.getErrors()).isEmpty(); + assertThat(result.getValue().get(0).getUri()) + .contains(expectations); + } + + @Test + public void makeHttpRequestsEncodesDsaTransparencyCorrectlyWhenBidRequestContainsDsaTransparencyInformation() { + //given + final ExtRegsDsa dsa = ExtRegsDsa.of( + 1, 2, 3, List.of( + ExtRegsDsaTransparency.of("testDomain", List.of(1, 2, 3)), + ExtRegsDsaTransparency.of("testDomain2", List.of(4, 5, 6)) + ) + ); + final Regs regs = Regs.builder() + .ext(ExtRegs.of(null, null, null, dsa)) + .build(); + final BidRequest bidRequest = BidRequest.builder() + .imp(singletonList(Imp.builder() + .ext(mapper.valueToTree(ExtPrebid.of(null, + ExtImpYieldlab.builder() + .adslotId("123") + .build()))) + .build())) + .regs(regs) + .build(); + + //when + final Result>> result = target.makeHttpRequests(bidRequest); + + //then + final List expectations = List.of( + "&dsarequired=1", + "&dsapubrender=2", + "&dsadatatopub=3", + "&dsatransparency=testDomain%7E1_2_3%7E%7EtestDomain2%7E4_5_6" + ); + assertThat(result.getErrors()).isEmpty(); + assertThat(result.getValue().get(0).getUri()) + .contains(expectations); + } + + @Test + public void makeHttpRequestsShouldNotSendDsaInfoInBidRequestWhenDsaIsMissing() { + //given + final ExtRegsDsa dsa = ExtRegsDsa.of( + 2, null, 3, null + ); + final Regs regs = Regs.builder() + .ext(ExtRegs.of(null, null, null, dsa)) + .build(); + final BidRequest bidRequest = BidRequest.builder() + .imp(singletonList(Imp.builder() + .ext(mapper.valueToTree(ExtPrebid.of(null, + ExtImpYieldlab.builder() + .adslotId("123") + .build()))) + .build())) + .regs(regs) + .build(); + + //when + final Result>> result = target.makeHttpRequests(bidRequest); + + //then + final List expectations = List.of( + "dsarequired=2", + "dsadatatopub=3" + ); + assertThat(result.getErrors()).isEmpty(); + assertThat(result.getValue().get(0).getUri()) + .contains(expectations); + } + + @Test + public void makeHttpRequestsShouldNotAddDsaTransparencyParamsToBidRequestWhenParamsAreMissing() { + //given + final ExtRegsDsa dsa = ExtRegsDsa.of( + 2, null, 3, List.of(ExtRegsDsaTransparency.of("domain", null)) + ); + final Regs regs = Regs.builder() + .ext(ExtRegs.of(null, null, null, dsa)) + .build(); + final BidRequest bidRequest = BidRequest.builder() + .imp(singletonList(Imp.builder() + .ext(mapper.valueToTree(ExtPrebid.of(null, + ExtImpYieldlab.builder() + .adslotId("123") + .build()))) + .build())) + .regs(regs) + .build(); + + //when + final Result>> result = target.makeHttpRequests(bidRequest); + + //then + final List expectations = List.of( + "dsarequired=2", + "dsadatatopub=3" + ); + assertThat(result.getErrors()).isEmpty(); + assertThat(result.getValue().get(0).getUri()) + .contains(expectations); + } + + @Test + public void makeBidsShouldAddDsaParamsWhenDsaIsPresentInResponse() throws JsonProcessingException { + // given + final BidRequest bidRequest = BidRequest.builder() + .imp(singletonList(Imp.builder() + .id("test-imp-id") + .banner(Banner.builder().w(1).h(1).build()) + .ext(mapper.valueToTree(ExtPrebid.of(null, ExtImpYieldlab.builder() + .adslotId("1") + .supplyId("2") + .adSize("adSize") + .targeting(singletonMap("key", "value")) + .extId("extId") + .build()))) + .build())) + .build(); + + final YieldlabDigitalServicesActResponse dsaResponse = YieldlabDigitalServicesActResponse.of( + "yieldlab", + "yieldlab", + 2, + List.of( + YieldlabDigitalServicesActResponse.Transparency.of( + "yieldlab.de", + List.of(1, 2, 3) + ) + ) + ); + + final YieldlabResponse yieldlabResponse = YieldlabResponse.of(1, 201d, "yieldlab", + "728x90", 1234, 5678, "40cb3251-1e1e-4cfd-8edc-7d32dc1a21e5", dsaResponse + ); + + final BidderCall httpCall = givenHttpCall(mapper.writeValueAsString(yieldlabResponse)); + + // when + final Result> result = target.makeBids(httpCall, bidRequest); + + // then + final ObjectNode expectedTransparency = mapper.createObjectNode(); + expectedTransparency.put("domain", "yieldlab.de"); + expectedTransparency.set("dsaparams", mapper.convertValue(List.of(1, 2, 3), JsonNode.class)); + + final ArrayNode transparencies = mapper.createArrayNode(); + transparencies.add(expectedTransparency); + + final ObjectNode expectedDsa = mapper.createObjectNode(); + expectedDsa.put("paid", "yieldlab"); + expectedDsa.put("behalf", "yieldlab"); + expectedDsa.put("adrender", 2); + expectedDsa.set("transparency", transparencies); + + final JsonNode actualDsa = result.getValue().get(0).getBid().getExt().get("dsa"); + + assertThat(result.getErrors()).isEmpty(); + assertThat(actualDsa).isEqualTo(expectedDsa); + } + private static BidderCall givenHttpCall(String body) { return BidderCall.succeededHttp( HttpRequest.builder().build(),