Merge branch 'main' into rel

.github/workflows/pip-tools.yaml

@@ -8,8 +8,8 @@ name: Update dependencies with pip-tools
 
 on:
   schedule:
-    # Run weekly on day 0 at 00:00 UTC
-    - cron: "0 0 * * 0"
+    # Run monthly
+    - cron: "0 0 1 * *"
 
 permissions:
   contents: read

CHANGELOG.rst

@@ -6,6 +6,20 @@ CHANGELOG
 .. This is included by docs/developer/changelog.rst
 
 
+Version v5.11.0
+---------------
+
+Begin collecting domain on Offer creation (migration).
+Add additional UTM substitution variables.
+
+:Date: December 4, 2024
+
+ * @davidfischer: Collect the domain on offer creation (#954)
+ * @davidfischer: Add two additional flight substitution variables (#953)
+ * @davidfischer: Link parameter substitutions (#950)
+ * @ericholscher: Run pip-tools monthly (#949)
+
+
 Version v5.10.1
 ---------------
 

adserver/api/mixins.py

@@ -57,5 +57,8 @@ def finalize_response(self, request, response, *args, **kwargs):
             response["X-Adserver-Country"] = str(request.geo.country)
             response["X-Adserver-Region"] = str(request.geo.region)
             response["X-Adserver-Metro"] = str(request.geo.metro)
+            response["X-Adserver-Continent"] = str(request.geo.continent)
+            response["X-Adserver-Latitude"] = str(request.geo.lat)
+            response["X-Adserver-Longitude"] = str(request.geo.lng)
 
         return response

adserver/middleware.py

@@ -180,9 +180,12 @@ class CloudflareGeoIpMiddleware(GeoIpMiddleware):
     COUNTRY_HEADER = "CF-IPCountry"
 
     # These fields will require a custom transform rule
-    # https://developers.cloudflare.com/rules/transform/
+    # https://developers.cloudflare.com/rules/transform/managed-transforms/reference/#add-visitor-location-headers
     REGION_HEADER = "X-Cloudflare-Geo-Region"  # ip.src.region_code
     METRO_HEADER = "X-Cloudflare-Geo-Metro"  # ip.src.metro_code
+    CONTINENT_HEADER = "X-Cloudflare-Geo-Continent"  # ip.src.continent
+    LATITUDE_HEADER = "X-Cloudflare-Geo-Lat"  # ip.src.lat
+    LONGITUDE_HEADER = "X-Cloudflare-Geo-Lon"  # ip.src.lon
 
     def get_geoip(self, request):
         geo = super().get_geoip(request)
@@ -194,8 +197,13 @@ def get_geoip(self, request):
             country_code = None
 
         geo.country = country_code
+        # Region is the state/province within a country (not wider region like EU)
+        # See "continent"
         geo.region = request.headers.get(self.REGION_HEADER, None)
         geo.metro = request.headers.get(self.METRO_HEADER, None)
+        geo.continent = request.headers.get(self.CONTINENT_HEADER, None)
+        geo.lat = request.headers.get(self.LATITUDE_HEADER, None)
+        geo.lng = request.headers.get(self.LONGITUDE_HEADER, None)
 
         return geo
 

adserver/migrations/0099_link_advertiser_guide.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.0.9 on 2024-11-27 00:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('adserver', '0098_rotation_aggregation'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='advertisement',
+            name='link',
+            field=models.URLField(help_text="URL of your landing page. This may contain UTM parameters so you know the traffic came from us. The publisher will be added in the 'ea-publisher' query parameter. Additional variable substitutions are available. See the <a href='https://www.ethicalads.io/advertiser-guide/#measuring-conversions'>advertiser guide</a>. ", max_length=1024, verbose_name='Link URL'),
+        ),
+        migrations.AlterField(
+            model_name='historicaladvertisement',
+            name='link',
+            field=models.URLField(help_text="URL of your landing page. This may contain UTM parameters so you know the traffic came from us. The publisher will be added in the 'ea-publisher' query parameter. Additional variable substitutions are available. See the <a href='https://www.ethicalads.io/advertiser-guide/#measuring-conversions'>advertiser guide</a>. ", max_length=1024, verbose_name='Link URL'),
+        ),
+    ]

adserver/migrations/0100_add_offer_domain.py

@@ -0,0 +1,28 @@
+# Generated by Django 5.0.9 on 2024-12-02 23:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('adserver', '0099_link_advertiser_guide'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='click',
+            name='domain',
+            field=models.CharField(blank=True, max_length=10000, null=True, verbose_name='Domain'),
+        ),
+        migrations.AddField(
+            model_name='offer',
+            name='domain',
+            field=models.CharField(blank=True, max_length=10000, null=True, verbose_name='Domain'),
+        ),
+        migrations.AddField(
+            model_name='view',
+            name='domain',
+            field=models.CharField(blank=True, max_length=10000, null=True, verbose_name='Domain'),
+        ),
+    ]

adserver/models.py

@@ -1613,7 +1613,9 @@ class Advertisement(TimeStampedModel, IndestructibleModel):
         help_text=_(
             "URL of your landing page. "
             "This may contain UTM parameters so you know the traffic came from us. "
-            "The publisher will be added in the 'ea-publisher' query parameter."
+            "The publisher will be added in the 'ea-publisher' query parameter. "
+            "Additional variable substitutions are available. "
+            "See the <a href='https://www.ethicalads.io/advertiser-guide/#measuring-conversions'>advertiser guide</a>. "
         ),
     )
     image = models.ImageField(
@@ -1773,6 +1775,7 @@ def _record_base(
         parsed_ua = parse(user_agent)
         country = get_client_country(request)
         url = url or request.headers.get("referer")
+        domain = get_domain_from_url(url)
 
         if (
             model != Click
@@ -1797,6 +1800,7 @@ def _record_base(
             client_id=client_id,
             country=country,
             url=url,
+            domain=domain,
             paid_eligible=paid_eligible,
             rotations=rotations,
             # Derived user agent data
@@ -2587,6 +2591,10 @@ class AdBase(TimeStampedModel, IndestructibleModel):
     country = CountryField(null=True)
     url = models.CharField(_("Page URL"), max_length=10000, blank=True, null=True)
 
+    # Domain of the URL or None if the URL is not available
+    # Should not include http or any characters after the TLD
+    domain = models.CharField(_("Domain"), max_length=10000, blank=True, null=True)
+
     # Fields derived from the user agent - these should not be user identifiable
     browser_family = models.CharField(
         _("Browser Family"), max_length=1000, blank=True, null=True, default=None

adserver/tests/test_api.py

@@ -1790,7 +1790,7 @@ def test_view_tracking_ratelimit(self):
         self.assertEqual(resp["X-Adserver-Reason"], "Ratelimited view impression")
 
     def test_click_tracking_variable_expansion(self):
-        self.ad.link = "http://example.com?utm_source=${publisher}"
+        self.ad.link = "http://example.com?utm_source=${publisher}&ad=${advertisement}"
         self.ad.save()
 
         Offer.objects.filter(id=self.offer["nonce"]).update(viewed=True)
@@ -1799,7 +1799,7 @@ def test_click_tracking_variable_expansion(self):
         self.assertEqual(resp.status_code, 302)
         self.assertEqual(
             resp["Location"],
-            "http://example.com?utm_source=test-publisher&ea-publisher=test-publisher",
+            "http://example.com?utm_source=test-publisher&ad=ad-slug&ea-publisher=test-publisher",
         )
 
         # invalid string replacement template

adserver/tests/test_models.py

@@ -633,6 +633,7 @@ def test_offer_ad(self):
         self.assertEqual(offer.advertisement, self.ad1)
         self.assertEqual(offer.div_id, div_id)
         self.assertEqual(offer.url, url)
+        self.assertEqual(offer.domain, "example.com")
         self.assertEqual(offer.ip, "1.1.0.0")  # anonymized
         self.assertEqual(offer.os_family, "Linux")
         self.assertEqual(offer.browser_family, "Chrome")

adserver/utils.py

@@ -53,6 +53,7 @@ class GeolocationData:
     metro: int = None
     lat: float = None
     lng: float = None
+    continent: str = None
 
 
 def get_ad_day():

adserver/views.py

@@ -1207,6 +1207,13 @@ def get_response(self, request, advertisement, publisher):
             advertisement=advertisement.slug,
             advertisement_slug=advertisement.slug,
             advertisement_name=advertisement.name,
+            flight=advertisement.flight.slug,
+            flight_slug=advertisement.flight.slug,
+            flight_name=advertisement.flight.name,
+            # For privacy, don't reveal more than country/continent to advertisers
+            country=str(request.geo.country) if request.geo else "None",
+            # request.geo.region is a state/province/region inside a country
+            continent=str(request.geo.continent) if request.geo else "None",
         )
 
         # Append a query string param ?ea-publisher=${publisher}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ethical-ad-server",
-  "version": "5.10.1",
+  "version": "5.11.0",
   "description": "",
   "main": "index.js",
   "engines": {