Config: deprecated notification for projects without config file

readthedocs/builds/models.py

@@ -1069,10 +1069,20 @@ def can_rebuild(self):
     def external_version_name(self):
         return external_version_name(self)
 
-    def using_latest_config(self):
-        if self.config:
-            return int(self.config.get('version', '1')) == LATEST_CONFIGURATION_VERSION
-        return False
+    def deprecated_config_used(self):
+        """
+        Check whether this particular build is using a deprecated config file.
+
+        When using v1 or not having a config file at all, it returns ``True``.
+        Returns ``False`` only when it has a config file and it is using v2.
+
+        Note we are using this to communicate deprecation of v1 file and not using a config file.
+        See https://github.com/readthedocs/readthedocs.org/issues/10342
+        """
+        if not self.config:
+            return True
+
+        return int(self.config.get("version", "1")) != LATEST_CONFIGURATION_VERSION
 
     def reset(self):
         """

readthedocs/projects/tasks/utils.py

@@ -3,9 +3,13 @@
 
 import structlog
 from celery.worker.request import Request
-from django.db.models import Q
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.db.models import Q, Sum
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
+from djstripe.enums import SubscriptionStatus
+from messages_extends.constants import WARNING_PERSISTENT
 
 from readthedocs.builds.constants import (
     BUILD_FINAL_STATES,
@@ -14,7 +18,12 @@
 )
 from readthedocs.builds.models import Build
 from readthedocs.builds.tasks import send_build_status
+from readthedocs.core.permissions import AdminPermission
 from readthedocs.core.utils.filesystem import safe_rmtree
+from readthedocs.notifications import Notification, SiteNotification
+from readthedocs.notifications.backends import EmailBackend
+from readthedocs.notifications.constants import REQUIREMENT
+from readthedocs.projects.models import Project
 from readthedocs.storage import build_media_storage
 from readthedocs.worker import app
 
@@ -154,6 +163,158 @@ def send_external_build_status(version_type, build_pk, commit, status):
         send_build_status.delay(build_pk, commit, status)
 
 
+class DeprecatedConfigFileSiteNotification(SiteNotification):
+
+    # TODO: mention all the project slugs here
+    # Maybe trim them to up to 5 projects to avoid sending a huge blob of text
+    failure_message = _(
+        'Your project(s) "{{ project_slugs }}" don\'t have a configuration file. '
+        "Configuration files will <strong>soon be required</strong> by projects, "
+        "and will no longer be optional. "
+        '<a href="https://blog.readthedocs.com/migrate-configuration-v2/">Read our blog post to create one</a> '  # noqa
+        "and ensure your project continues building successfully."
+    )
+    failure_level = WARNING_PERSISTENT
+
+
+class DeprecatedConfigFileEmailNotification(Notification):
+
+    app_templates = "projects"
+    name = "deprecated_config_file_used"
+    context_object_name = "project"
+    subject = "[Action required] Add a configuration file to your project to prevent build failure"
+    level = REQUIREMENT
+
+    def send(self):
+        """Method overwritten to remove on-site backend."""
+        backend = EmailBackend(self.request)
+        backend.send(self)
+
+
+@app.task(queue="web")
+def deprecated_config_file_used_notification():
+    """
+    Create a notification about not using a config file for all the maintainers of the project.
+
+    This is a scheduled task to be executed on the webs.
+    Note the code uses `.iterator` and `.only` to avoid killing the db with this query.
+    Besdies, it excludes projects with enough spam score to be skipped.
+    """
+    # Skip projects with a spam score bigger than this value.
+    # Currently, this gives us ~250k in total (from ~550k we have in our database)
+    spam_score = 300
+
+    projects = set()
+    start_datetime = datetime.datetime.now()
+    queryset = Project.objects.exclude(users__profile__banned=True)
+    if settings.ALLOW_PRIVATE_REPOS:
+        # Only send emails to active customers
+        queryset = queryset.filter(
+            organizations__stripe_subscription__status=SubscriptionStatus.active
+        )
+    else:
+        # Take into account spam score on community
+        queryset = queryset.annotate(spam_score=Sum("spam_rules__value")).filter(
+            Q(spam_score__lt=spam_score) | Q(is_spam=False)
+        )
+    queryset = queryset.only("slug", "default_version").order_by("id")
+    n_projects = queryset.count()
+
+    for i, project in enumerate(queryset.iterator()):
+        if i % 500 == 0:
+            log.info(
+                "Finding projects without a configuration file.",
+                progress=f"{i}/{n_projects}",
+                current_project_pk=project.pk,
+                current_project_slug=project.slug,
+                projects_found=len(projects),
+                time_elapsed=(datetime.datetime.now() - start_datetime).seconds,
+            )
+
+        # Only check for the default version because if the project is using tags
+        # they won't be able to update those and we will send them emails forever.
+        # We can update this query if we consider later.
+        version = (
+            project.versions.filter(slug=project.default_version).only("id").first()
+        )
+        if version:
+            build = (
+                version.builds.filter(success=True)
+                .only("_config")
+                .order_by("-date")
+                .first()
+            )
+            if build and build.deprecated_config_used():
+                projects.add(project.slug)
+
+    # Store all the users we want to contact
+    users = set()
+
+    n_projects = len(projects)
+    queryset = Project.objects.filter(slug__in=projects).order_by("id")
+    for i, project in enumerate(queryset.iterator()):
+        if i % 500 == 0:
+            log.info(
+                "Querying all the users we want to contact.",
+                progress=f"{i}/{n_projects}",
+                current_project_pk=project.pk,
+                current_project_slug=project.slug,
+                users_found=len(users),
+                time_elapsed=(datetime.datetime.now() - start_datetime).seconds,
+            )
+
+        users.update(AdminPermission.owners(project).values_list("username", flat=True))
+
+    # Only send 1 email per user,
+    # even if that user has multiple projects without a configuration file.
+    # The notification will mention all the projects.
+    queryset = User.objects.filter(username__in=users, profile__banned=False).order_by(
+        "id"
+    )
+    n_users = queryset.count()
+    for i, user in enumerate(queryset.iterator()):
+        if i % 500 == 0:
+            log.info(
+                "Sending deprecated config file notification to users.",
+                progress=f"{i}/{n_users}",
+                current_user_pk=user.pk,
+                current_user_username=user.username,
+                time_elapsed=(datetime.datetime.now() - start_datetime).seconds,
+            )
+
+        # All the projects for this user that don't have a configuration file
+        user_projects = (
+            AdminPermission.projects(user, admin=True)
+            .filter(slug__in=projects)
+            .only("slug")
+        )
+
+        user_project_slugs = ", ".join([p.slug for p in user_projects[:5]])
+        if user_projects.count() > 5:
+            user_project_slugs += " and others..."
+
+        n_site = DeprecatedConfigFileSiteNotification(
+            user=user,
+            context_object=user_projects,
+            extra_context={"project_slugs": user_project_slugs},
+            success=False,
+        )
+        n_site.send()
+
+        # TODO: uncomment this code when we are ready to send email notifications
+        # n_email = DeprecatedConfigFileEmailNotification(
+        #     user=user,
+        #     context_object=user_projects,
+        #     extra_context={"project_slugs": user_project_slugs},
+        # )
+        # n_email.send()
+
+    log.info(
+        "Finish sending deprecated config file notifications.",
+        time_elapsed=(datetime.datetime.now() - start_datetime).seconds,
+    )
+
+
 class BuildRequest(Request):
 
     def on_timeout(self, soft, timeout):

readthedocs/rtd_tests/tests/test_builds.py

@@ -249,7 +249,7 @@ def test_build_is_stale(self):
         self.assertTrue(build_two.is_stale)
         self.assertFalse(build_three.is_stale)
 
-    def test_using_latest_config(self):
+    def test_deprecated_config_used(self):
         now = timezone.now()
 
         build = get(
@@ -260,12 +260,12 @@ def test_using_latest_config(self):
             state='finished',
         )
 
-        self.assertFalse(build.using_latest_config())
+        self.assertTrue(build.deprecated_config_used())
 
         build.config = {'version': 2}
         build.save()
 
-        self.assertTrue(build.using_latest_config())
+        self.assertFalse(build.deprecated_config_used())
 
     def test_build_is_external(self):
         # Turn the build version to EXTERNAL type.

readthedocs/settings/base.py

@@ -532,6 +532,11 @@ def TEMPLATES(self):
             'schedule': crontab(minute='*/15'),
             'options': {'queue': 'web'},
         },
+        'weekly-config-file-notification': {
+            'task': 'readthedocs.projects.tasks.utils.deprecated_config_file_used_notification',
+            'schedule': crontab(day_of_week='wednesday', hour=11, minute=15),
+            'options': {'queue': 'web'},
+        },
     }
 
     MULTIPLE_BUILD_SERVERS = [CELERY_DEFAULT_QUEUE]

readthedocs/templates/builds/build_detail.html

@@ -161,18 +161,21 @@
           </p>
         </div>
       {% endif %}
-      {% if build.finished and not build.using_latest_config %}
+
+      {# This message is not dynamic and only appears when loading the page after the build has finished #}
+      {% if build.finished and build.deprecated_config_used %}
         <div class="build-ideas">
           <p>
-            {% blocktrans trimmed with config_file_link="https://docs.readthedocs.io/page/config-file/v2.html" %}
-              <strong>Configure your documentation builds!</strong>
-              Adding a <a href="{{ config_file_link }}">.readthedocs.yaml</a> file to your project
-              is the recommended way to configure your documentation builds.
-              You can declare dependencies, set up submodules, and many other great features.
+            {% blocktrans trimmed with config_file_link="https://blog.readthedocs.com/migrate-configuration-v2/" %}
+              <strong>Your builds will stop working soon!</strong><br/>
+              Configuration files will <strong>soon be required</strong> by projects, and will no longer be optional.
+              <a href="{{ config_file_link }}">Read our blog post to create one</a>
+              and ensure your project continues building successfully.
             {% endblocktrans %}
           </p>
         </div>
       {% endif %}
+
     {% endif %}
 
     {% if build.finished and build.config.build.commands %}

readthedocs/templates/projects/notifications/deprecated_config_file_used_email.html

@@ -0,0 +1,29 @@
+{% extends "core/email/common.html" %}
+{% block content %}
+The Read the Docs build system will start requiring a configuration file v2 (<code>.readthedocs.yaml</code>) starting on <strong>September 25, 2023</strong>.
+We are scheduling brownout days to provide extra reminders by failing build without a configuration file v2 during some hours before the final day.
+Keep these dates in mind to avoid unexpected behaviours:
+
+<ul>
+    <li><strong>Monday, July 24, 2023</strong>: Do the first brownout (temporarily enforce this deprecation) for 12 hours: 00:01 PST to 11:59 PST (noon)</li>
+    <li><strong>Monday, August 14, 2023</strong>: Do a second brownout (temporarily enforce this deprecation) for 24 hours: 00:01 PST to 23:59 PST (midnight)</li>
+    <li><strong>Monday, September 4, 2023</strong> Do a third and final brownout (temporarily enforce this deprecation) for 48 hours: 00:01 PST to 23:59 PST (midnight)</li>
+    <li><strong>Monday, September 25, 2023</strong>: Fully remove support for building documentation without configuration file v2.</li>
+</ul>
+
+We have identified the following projects where you are admin are impacted by this deprecation:
+
+<ul>
+{% for project in object %}
+    <li>{{ project.slug }}</li>
+{% endfor %}
+</ul>
+
+You require to add a configuration file to your projects to ensure they continues building successfully and stop receiving these notifications.
+
+For more information on how to create a required configuration file,
+<a href="https://blog.readthedocs.com/migrate-configuration-v2/">read our blog post</a>
+
+Get in touch with us <a href="{{ production_uri }}{% url 'support' %}">via our support</a>
+and let us know if you are unable to use a configuration file for any reason.
+{% endblock %}

readthedocs/templates/projects/notifications/deprecated_config_file_used_email.txt

@@ -0,0 +1,25 @@
+{% extends "core/email/common.txt" %}
+{% block content %}
+The Read the Docs build system will start requiring a configuration file v2 (.readthedocs.yaml) starting on September 25, 2023.
+We are scheduling brownout days to provide extra reminders by failing build without a configuration file v2 during some hours before the final day.
+Keep these dates in mind to avoid unexpected behaviours:
+
+* Monday, July 24, 2023: Do the first brownout (temporarily enforce this deprecation) for 12 hours: 00:01 PST to 11:59 PST (noon)
+* Monday, August 14, 2023: Do a second brownout (temporarily enforce this deprecation) for 24 hours: 00:01 PST to 23:59 PST (midnight)
+* Monday, September 4, 2023: Do a third and final brownout (temporarily enforce this deprecation) for 48 hours: 00:01 PST to 23:59 PST (midnight)
+* Monday, September 25, 2023: Fully remove support for building documentation without configuration file v2.
+
+We have identified the following projects where you are admin are impacted by this deprecation:
+
+{% for project in object %}
+* {{ project.slug }}
+{% endfor %}
+
+You require to add a configuration file to your projects to ensure they continues building successfully and stop receiving these notifications.
+
+For more information on how to create a required configuration file, see:
+https://blog.readthedocs.com/migrate-configuration-v2/
+
+Get in touch with us at {{ production_uri }}{% url 'support' %}
+and let us know if you are unable to use a configuration file for any reason.
+{% endblock %}