Deprecation: notification and feature flag for build.image config

Define a weekly task to communicate our users about the deprecation of
`build.image` using the deprecation plan we used for the configuration file v2
as well.

- 3 brownout days
- final removal date on October 2nd
- weekly onsite/email notification on Wednesday at 11:15 CEST (around ~22k projects affected)
- allow to opt-out from these emails
- feature flag for brownout days
- build detail's page notification

Related:
* readthedocs/meta#48
* #10354
* #10587

readthedocs/builds/models.py

@@ -1104,6 +1104,21 @@ def deprecated_config_used(self):
 
         return int(self.config.get("version", "1")) != LATEST_CONFIGURATION_VERSION
 
+    def deprecated_build_image_used(self):
+        """
+        Check whether this particular build is using the deprecated "build" config.
+
+        Note we are using this to communicate deprecation of "build.image".
+        See https://github.com/readthedocs/meta/discussions/48
+        """
+        if not self.config:
+            # Don't notify users without a config file.
+            # We hope they will migrate to `build.os` in the process of adding a `.readthedocs.yaml`
+            return False
+
+        build_config_key = self.config.get("build", {})
+        return "image" in build_config_key or "os" not in build_config_key
+
     def reset(self):
         """
         Reset the build so it can be re-used when re-trying.

readthedocs/core/forms.py

@@ -23,7 +23,10 @@ class Meta:
         model = UserProfile
         # Don't allow users edit someone else's user page
         profile_fields = ["first_name", "last_name", "homepage"]
-        optout_email_fields = ["optout_email_config_file_deprecation"]
+        optout_email_fields = [
+            "optout_email_config_file_deprecation",
+            "optout_email_build_image_deprecation",
+        ]
         fields = (
             *profile_fields,
             *optout_email_fields,

readthedocs/core/migrations/0014_optout_email_build_image_deprecation.py

@@ -0,0 +1,30 @@
+# Generated by Django 3.2.20 on 2023-08-01 13:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0013_add_optout_email_config_file_deprecation"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="historicaluserprofile",
+            name="optout_email_build_image_deprecation",
+            field=models.BooleanField(
+                default=False,
+                null=True,
+                verbose_name="Opt-out from email about '\"build.image\" config key deprecation'",
+            ),
+        ),
+        migrations.AddField(
+            model_name="userprofile",
+            name="optout_email_build_image_deprecation",
+            field=models.BooleanField(
+                default=False,
+                null=True,
+                verbose_name="Opt-out from email about '\"build.image\" config key deprecation'",
+            ),
+        ),
+    ]

readthedocs/core/models.py

@@ -51,6 +51,13 @@ class UserProfile(TimeStampedModel):
         default=False,
         null=True,
     )
+    # NOTE: this is a temporary field that we can remove after October 16, 2023
+    # See https://blog.readthedocs.com/build-image-config-deprecated/
+    optout_email_build_image_deprecation = models.BooleanField(
+        _("Opt-out from email about '\"build.image\" config key deprecation'"),
+        default=False,
+        null=True,
+    )
 
     # Model history
     history = ExtraHistoricalRecords()

readthedocs/doc_builder/director.py

@@ -255,6 +255,16 @@ def checkout(self):
         ) and self.data.config.version not in ("2", 2):
             raise BuildUserError(BuildUserError.NO_CONFIG_FILE_DEPRECATED)
 
+        # Raise a build error if the project is using "build.image" on their config file
+        if self.data.project.has_feature(Feature.BUILD_IMAGE_CONFIG_KEY_DEPRECATED):
+            build_config_key = self.data.config.source_config.get("build", {})
+            if "image" in build_config_key:
+                raise BuildUserError(BuildUserError.BUILD_IMAGE_CONFIG_KEY_DEPRECATED)
+
+            # TODO: move this validation to the Config object once we are settled here
+            if "image" not in build_config_key and "os" not in build_config_key:
+                raise BuildUserError(BuildUserError.BUILD_OS_REQUIRED)
+
         if self.vcs_repository.supports_submodules:
             self.vcs_repository.update_submodules(self.data.config)
 

readthedocs/doc_builder/exceptions.py

@@ -66,6 +66,15 @@ class BuildUserError(BuildBaseException):
         "Add a configuration file to your project to make it build successfully. "
         "Read more at https://docs.readthedocs.io/en/stable/config-file/v2.html"
     )
+    BUILD_IMAGE_CONFIG_KEY_DEPRECATED = gettext_noop(
+        'The configuration key "build.image" is deprecated. '
+        'Please, use "build.os" instead to make it build successfully. '
+        "Read more at https://docs.readthedocs.io/en/stable/config-file/v2.html"
+    )
+    BUILD_OS_REQUIRED = gettext_noop(
+        'The configuration key "build.os" is required to build your documentation. '
+        "Read more at https://docs.readthedocs.io/en/stable/config-file/v2.html"
+    )
 
 
 class BuildUserSkip(BuildUserError):

readthedocs/projects/models.py

@@ -1950,6 +1950,7 @@ def add_features(sender, **kwargs):
     GIT_CLONE_FETCH_CHECKOUT_PATTERN = "git_clone_fetch_checkout_pattern"
     HOSTING_INTEGRATIONS = "hosting_integrations"
     NO_CONFIG_FILE_DEPRECATED = "no_config_file"
+    BUILD_IMAGE_CONFIG_KEY_DEPRECATED = "build_image_config_key_deprecated"
     SCALE_IN_PROTECTION = "scale_in_prtection"
 
     FEATURES = (
@@ -2093,6 +2094,12 @@ def add_features(sender, **kwargs):
             NO_CONFIG_FILE_DEPRECATED,
             _("Build: Building without a configuration file is deprecated."),
         ),
+        (
+            BUILD_IMAGE_CONFIG_KEY_DEPRECATED,
+            _(
+                'Build: Building using "build.image" in the configuration file is deprecated.'
+            ),
+        ),
         (
             SCALE_IN_PROTECTION,
             _("Build: Set scale-in protection before/after building."),

readthedocs/projects/tasks/utils.py

@@ -323,6 +323,162 @@ def deprecated_config_file_used_notification():
     )
 
 
+class DeprecatedBuildImageSiteNotification(SiteNotification):
+    failure_message = _(
+        'Your project(s) "{{ project_slugs }}" are using the deprecated "build.image" '
+        'config on their ".readthedocs.yaml" file. '
+        'This config is deprecated in favor of "build.os" and <strong>will be removed on October 16, 2023</strong>. '  # noqa
+        '<a href="https://blog.readthedocs.com/build-image-config-deprecated/">Read our blog post to migrate to "build.os"</a> '  # noqa
+        "and ensure your project continues building successfully."
+    )
+    failure_level = WARNING_PERSISTENT
+
+
+class DeprecatedBuildImageEmailNotification(Notification):
+    app_templates = "projects"
+    name = "deprecated_build_image_used"
+    subject = '[Action required] Update your ".readthedocs.yaml" file to use "build.os"'
+    level = REQUIREMENT
+
+    def send(self):
+        """Method overwritten to remove on-site backend."""
+        backend = EmailBackend(self.request)
+        backend.send(self)
+
+
+@app.task(queue="web")
+def deprecated_build_image_notification():
+    """
+    Send an email notification about using "build.image" to all maintainers of the project.
+
+    This is a scheduled task to be executed on the webs.
+    Note the code uses `.iterator` and `.only` to avoid killing the db with this query.
+    Besdies, it excludes projects with enough spam score to be skipped.
+    """
+    # Skip projects with a spam score bigger than this value.
+    # Currently, this gives us ~250k in total (from ~550k we have in our database)
+    spam_score = 300
+
+    projects = set()
+    start_datetime = datetime.datetime.now()
+    queryset = Project.objects.exclude(users__profile__banned=True)
+    if settings.ALLOW_PRIVATE_REPOS:
+        # Only send emails to active customers
+        queryset = queryset.filter(
+            organizations__stripe_subscription__status=SubscriptionStatus.active
+        )
+    else:
+        # Take into account spam score on community
+        queryset = queryset.annotate(spam_score=Sum("spam_rules__value")).filter(
+            Q(spam_score__lt=spam_score) | Q(is_spam=False)
+        )
+    queryset = queryset.only("slug", "default_version").order_by("id")
+    n_projects = queryset.count()
+
+    for i, project in enumerate(queryset.iterator()):
+        if i % 500 == 0:
+            log.info(
+                'Finding projects using "build.image" config key.',
+                progress=f"{i}/{n_projects}",
+                current_project_pk=project.pk,
+                current_project_slug=project.slug,
+                projects_found=len(projects),
+                time_elapsed=(datetime.datetime.now() - start_datetime).seconds,
+            )
+
+        # Only check for the default version because if the project is using tags
+        # they won't be able to update those and we will send them emails forever.
+        # We can update this query if we consider later.
+        version = (
+            project.versions.filter(slug=project.default_version).only("id").first()
+        )
+        if version:
+            # Use a fixed date here to avoid changing the date on each run
+            years_ago = timezone.datetime(2022, 8, 1)
+            build = (
+                version.builds.filter(success=True, date__gt=years_ago)
+                .only("_config")
+                .order_by("-date")
+                .first()
+            )
+            # TODO: uncomment this line before merging
+            # if build and build.deprecated_build_image_used():
+            if build and "image" in build.config.get("build", {}):
+                projects.add(project.slug)
+
+    # Store all the users we want to contact
+    users = set()
+
+    n_projects = len(projects)
+    queryset = Project.objects.filter(slug__in=projects).order_by("id")
+    for i, project in enumerate(queryset.iterator()):
+        if i % 500 == 0:
+            log.info(
+                'Querying all the users we want to contact about "build.image" deprecation.',
+                progress=f"{i}/{n_projects}",
+                current_project_pk=project.pk,
+                current_project_slug=project.slug,
+                users_found=len(users),
+                time_elapsed=(datetime.datetime.now() - start_datetime).seconds,
+            )
+
+        users.update(AdminPermission.owners(project).values_list("username", flat=True))
+
+    # Only send 1 email per user,
+    # even if that user has multiple projects using "build.image".
+    # The notification will mention all the projects.
+    queryset = User.objects.filter(
+        username__in=users,
+        profile__banned=False,
+        profile__optout_email_build_image_deprecation=False,
+    ).order_by("id")
+
+    n_users = queryset.count()
+    for i, user in enumerate(queryset.iterator()):
+        if i % 500 == 0:
+            log.info(
+                'Sending deprecated "build.image" config key notification to users.',
+                progress=f"{i}/{n_users}",
+                current_user_pk=user.pk,
+                current_user_username=user.username,
+                time_elapsed=(datetime.datetime.now() - start_datetime).seconds,
+            )
+
+        # All the projects for this user that are using "build.image".
+        # Use set() intersection in Python that's pretty quick since we only need the slugs.
+        # Otherwise we have to pass 82k slugs to the DB query, which makes it pretty slow.
+        user_projects = AdminPermission.projects(user, admin=True).values_list(
+            "slug", flat=True
+        )
+        user_projects_slugs = list(set(user_projects) & projects)
+        user_projects = Project.objects.filter(slug__in=user_projects_slugs)
+
+        # Create slug string for onsite notification
+        user_project_slugs = ", ".join(user_projects_slugs[:5])
+        if len(user_projects) > 5:
+            user_project_slugs += " and others..."
+
+        n_site = DeprecatedBuildImageSiteNotification(
+            user=user,
+            context_object=user,
+            extra_context={"project_slugs": user_project_slugs},
+            success=False,
+        )
+        n_site.send()
+
+        n_email = DeprecatedBuildImageEmailNotification(
+            user=user,
+            context_object=user,
+            extra_context={"projects": user_projects},
+        )
+        n_email.send()
+
+    log.info(
+        'Finish sending deprecated "build.image" config key notifications.',
+        time_elapsed=(datetime.datetime.now() - start_datetime).seconds,
+    )
+
+
 @app.task(queue="web")
 def set_builder_scale_in_protection(instance, protected_from_scale_in):
     """

readthedocs/settings/base.py

@@ -536,6 +536,11 @@ def TEMPLATES(self):
             'schedule': crontab(day_of_week='wednesday', hour=11, minute=15),
             'options': {'queue': 'web'},
         },
+        'weekly-build-image-notification': {
+            'task': 'readthedocs.projects.tasks.utils.deprecated_build_image_notification',
+            'schedule': crontab(day_of_week='wednesday', hour=9, minute=15),
+            'options': {'queue': 'web'},
+        },
     }
 
     # Sentry

readthedocs/templates/builds/build_detail.html

@@ -163,17 +163,29 @@
       {% endif %}
 
       {# This message is not dynamic and only appears when loading the page after the build has finished #}
-      {% if build.finished and build.deprecated_config_used %}
-        <div class="build-ideas">
+      {% if build.finished and build.deprecated_build_image_used %}
+      <div class="build-ideas">
           <p>
-            {% blocktrans trimmed with config_file_link="https://blog.readthedocs.com/migrate-configuration-v2/" %}
+              {% blocktrans trimmed with config_file_link="https://blog.readthedocs.com/build-image-config-deprecated/" %}
               <strong>Your builds will stop working soon!</strong><br/>
-              Configuration files will <strong>soon be required</strong> by projects, and will no longer be optional.
-              <a href="{{ config_file_link }}">Read our blog post to create one</a>
+              "build.image" config key is deprecated and it will be removed soon.
+              <a href="{{ config_file_link }}">Read our blog post to know how to migrate to new key "build.os"</a>
               and ensure your project continues building successfully.
-            {% endblocktrans %}
+              {% endblocktrans %}
           </p>
-        </div>
+      </div>
+      {% endif %}
+      {% if build.finished and build.deprecated_config_used %}
+      <div class="build-ideas">
+        <p>
+          {% blocktrans trimmed with config_file_link="https://blog.readthedocs.com/migrate-configuration-v2/" %}
+            <strong>Your builds will stop working soon!</strong><br/>
+            Configuration files will <strong>soon be required</strong> by projects, and will no longer be optional.
+            <a href="{{ config_file_link }}">Read our blog post to create one</a>
+            and ensure your project continues building successfully.
+          {% endblocktrans %}
+        </p>
+      </div>
       {% endif %}
 
     {% endif %}

readthedocs/templates/projects/notifications/deprecated_build_image_used_email.html

@@ -0,0 +1 @@
+{# TODO: copy the text from the TXT version once we agree on its content #}

readthedocs/templates/projects/notifications/deprecated_build_image_used_email.txt

@@ -0,0 +1,31 @@
+{% extends "core/email/common.txt" %}
+{% block content %}
+The Read the Docs build system is deprecating "build.image" config key on ".readthedocs.yaml" starting on October 16, 2023.
+We are sending weekly notifications about this issue to all impacted users,
+as well as temporary build failures (brownouts) as the date approaches for those who haven't migrated their projects.
+
+The timeline for this deprecation is as follows:
+
+* Monday, August 28, 2023: Do the first brownout (temporarily enforce this deprecation) for 12 hours: 00:01 PST to 11:59 PST (noon)
+* Monday, September 18, 2023: Do a second brownout (temporarily enforce this deprecation) for 24 hours: 00:01 PST to 23:59 PST (midnight)
+* Monday, October 2, 2023: Do a third and final brownout (temporarily enforce this deprecation) for 48 hours: 00:01 PST to October 3, 2023 23:59 PST (midnight)
+* Monday, October 16, 2023: Fully remove support for building documentation using "build.image" on the configuration file
+
+We have identified that the following projects which you maintain, and were built in the last year, are impacted by this deprecation:
+
+{% for project in projects|slice:":15" %}
+* {{ project.slug }} ({{ production_uri }}{{ project.get_absolute_url }})
+{% endfor %}
+{% if projects.count > 15 %}
+* ... and {{ projects.count|add:"-15" }} more projects.
+{% endif %}
+
+Please use "build.os" on your configuration file to ensure that they continue building successfully and to stop receiving these notifications.
+If you want to opt-out from these emails, you can edit your preferences in your account settings, at https://readthedocs.org/accounts/edit/.
+
+For more information on how to use "build.os",
+read our blog post at https://blog.readthedocs.com/build-image-config-deprecated/
+
+Get in touch with us via our support ({{ production_uri }}{% url 'support' %})
+and let us know if you are unable to use a configuration file for any reason.
+{% endblock %}