feat: allow section-based Surrogate-Key to purge chunks (#2851)

Author: JacobCoffee

.github/workflows/purge-cache.yml

@@ -0,0 +1,28 @@
+name: Purge Fastly Cache
+
+on:
+  push:
+    branches: [main]
+    paths: ['static/**', 'templates/**', '**/templates/**']
+  workflow_dispatch:
+    inputs:
+      target:
+        description: 'Surrogate key to purge'
+        required: true
+        default: 'pydotorg-app'
+        type: choice
+        options: [pydotorg-app, downloads, events, sponsors, jobs]
+
+permissions: {}
+
+jobs:
+  purge:
+    runs-on: ubuntu-latest
+    env:
+      KEY: ${{ inputs.target || 'pydotorg-app' }}
+    steps:
+      - name: Purge ${{ env.KEY }}
+        run: |
+          curl -fsS -X POST \
+            "https://api.fastly.com/service/${{ secrets.FASTLY_SERVICE_ID }}/purge/${{ env.KEY }}" \
+            -H "Fastly-Key: ${{ secrets.FASTLY_API_KEY }}"

apps/downloads/models.py

@@ -16,7 +16,7 @@
 from apps.cms.models import ContentManageable, NameSlugModel
 from apps.downloads.managers import ReleaseManager
 from apps.pages.models import Page
-from fastly.utils import purge_url
+from fastly.utils import purge_surrogate_key, purge_url
 
 DEFAULT_MARKUP_TYPE = getattr(settings, "DEFAULT_MARKUP_TYPE", "markdown")
 
@@ -288,38 +288,35 @@ def promote_latest_release(sender, instance, **kwargs):
 
 @receiver(post_save, sender=Release)
 def purge_fastly_download_pages(sender, instance, **kwargs):
-    """Purge Fastly caches so new Downloads show up more quickly."""
+    """Purge Fastly caches so new Downloads show up more quickly.
+
+    Uses surrogate key purging to attempt to clear ALL pages under /downloads/
+    in one request, including dynamically added pages like /downloads/android/,
+    /downloads/ios/, etc. Independently purges a set of specific non-/downloads/
+    URLs via individual URL purges.
+    """
     # Don't purge on fixture loads
     if kwargs.get("raw", False):
         return
 
     # Only purge on published instances
     if instance.is_published:
-        # Purge our common pages
-        purge_url("/downloads/")
-        purge_url("/downloads/feed.rss")
-        purge_url("/downloads/latest/python2/")
-        purge_url("/downloads/latest/python3/")
-        # Purge minor version specific URLs (like /downloads/latest/python3.14/)
-        version = instance.get_version()
-        if instance.version == Release.PYTHON3 and version:
-            match = re.match(r"^3\.(\d+)", version)
-            if match:
-                purge_url(f"/downloads/latest/python3.{match.group(1)}/")
-        purge_url("/downloads/latest/prerelease/")
-        purge_url("/downloads/latest/pymanager/")
-        purge_url("/downloads/macos/")
-        purge_url("/downloads/source/")
-        purge_url("/downloads/windows/")
+        # Purge all /downloads/* pages via surrogate key (preferred method)
+        # This catches everything: /downloads/android/, /downloads/release/*, etc.
+        # Falls back to purge_url if FASTLY_SERVICE_ID is not configured.
+        if getattr(settings, "FASTLY_SERVICE_ID", None):
+            purge_surrogate_key("downloads")
+        else:
+            purge_url("/downloads/")
+
+        # Also purge related pages outside /downloads/
         purge_url("/ftp/python/")
         if instance.get_version():
             purge_url(f"/ftp/python/{instance.get_version()}/")
-        # See issue #584 for details
+        # See issue #584 for details - these are under /box/, not /downloads/
         purge_url("/box/supernav-python-downloads/")
         purge_url("/box/homepage-downloads/")
         purge_url("/box/download-sources/")
-        # Purge the release page itself
-        purge_url(instance.get_absolute_url())
 
 
 @receiver(post_save, sender=Release)

fastly/utils.py

@@ -19,3 +19,30 @@ def purge_url(path):
         )
 
     return None
+
+
+def purge_surrogate_key(key):
+    """Purge all Fastly cached content tagged with a surrogate key.
+
+    Common keys (set by GlobalSurrogateKey middleware):
+        - 'pydotorg-app': Purges entire site
+        - 'downloads': Purges all /downloads/* pages
+        - 'events': Purges all /events/* pages
+        - 'sponsors': Purges all /sponsors/* pages
+        - etc. (first path segment becomes the surrogate key)
+
+    Returns the response from Fastly API, or None if not configured.
+    """
+    if settings.DEBUG:
+        return None
+
+    api_key = getattr(settings, "FASTLY_API_KEY", None)
+    service_id = getattr(settings, "FASTLY_SERVICE_ID", None)
+    if not api_key or not service_id:
+        return None
+
+    return requests.post(
+        f"https://api.fastly.com/service/{service_id}/purge/{key}",
+        headers={"Fastly-Key": api_key},
+        timeout=30,
+    )

pydotorg/middleware.py

@@ -19,17 +19,47 @@ def __call__(self, request):
 
 
 class GlobalSurrogateKey:
-    """Middleware to insert a Surrogate-Key for purging in Fastly or other caches."""
+    """Middleware to insert a Surrogate-Key for purging in Fastly or other caches.
+
+    Adds both a global key (for full site purges) and section-based keys
+    derived from the URL path (for targeted purges like /downloads/).
+    """
 
     def __init__(self, get_response):
         """Store the get_response callable."""
         self.get_response = get_response
 
+    def get_section_key(self, path):
+        """Extract section surrogate key from URL path.
+
+        Examples:
+            /downloads/ -> downloads
+            /downloads/release/python-3141/ -> downloads
+            /events/python-events/ -> events
+            / -> None
+
+        """
+        parts = path.strip("/").split("/")
+        if parts and parts[0]:
+            return parts[0]
+        return None
+
     def __call__(self, request):
-        """Append the global surrogate key to the response header."""
+        """Append the global and section surrogate keys to the response header."""
         response = self.get_response(request)
+        keys = []
         if hasattr(settings, "GLOBAL_SURROGATE_KEY"):
-            response["Surrogate-Key"] = " ".join(
-                filter(None, [settings.GLOBAL_SURROGATE_KEY, response.get("Surrogate-Key")])
-            )
+            keys.append(settings.GLOBAL_SURROGATE_KEY)
+
+        section_key = self.get_section_key(request.path)
+        if section_key:
+            keys.append(section_key)
+
+        existing = response.get("Surrogate-Key")
+        if existing:
+            keys.append(existing)
+
+        if keys:
+            response["Surrogate-Key"] = " ".join(keys)
+
         return response

pydotorg/settings/base.py

@@ -265,6 +265,7 @@
 ### Fastly ###
 FASTLY_API_KEY = False  # Set to Fastly API key in production to allow pages to
 # be purged on save
+FASTLY_SERVICE_ID = config("FASTLY_SERVICE_ID", default=None)  # Required for surrogate key purging
 
 # Jobs
 JOB_THRESHOLD_DAYS = 90

pydotorg/tests/test_middleware.py

@@ -1,6 +1,8 @@
 from django.contrib.redirects.models import Redirect
 from django.contrib.sites.models import Site
-from django.test import TestCase
+from django.test import TestCase, override_settings
+
+from pydotorg.middleware import GlobalSurrogateKey
 
 
 class MiddlewareTests(TestCase):
@@ -21,3 +23,39 @@ def test_redirects(self):
         response = self.client.get(url)
         self.assertEqual(response.status_code, 301)
         self.assertEqual(response["Location"], redirect.new_path)
+
+
+class GlobalSurrogateKeyTests(TestCase):
+    def test_get_section_key(self):
+        """Test section key extraction from URL paths."""
+        middleware = GlobalSurrogateKey(lambda _: None)
+
+        self.assertEqual(middleware.get_section_key("/downloads/"), "downloads")
+        self.assertEqual(middleware.get_section_key("/downloads/release/python-3141/"), "downloads")
+        self.assertEqual(middleware.get_section_key("/events/"), "events")
+        self.assertEqual(middleware.get_section_key("/events/python-events/123/"), "events")
+        self.assertEqual(middleware.get_section_key("/sponsors/"), "sponsors")
+
+        # returns None
+        self.assertIsNone(middleware.get_section_key("/"))
+
+        self.assertEqual(middleware.get_section_key("/downloads"), "downloads")
+        self.assertEqual(middleware.get_section_key("downloads/"), "downloads")
+
+    @override_settings(GLOBAL_SURROGATE_KEY="pydotorg-app")
+    def test_surrogate_key_header_includes_section(self):
+        """Test that Surrogate-Key header includes both global and section keys."""
+        response = self.client.get("/downloads/")
+        self.assertTrue(response.has_header("Surrogate-Key"))
+        surrogate_key = response["Surrogate-Key"]
+
+        self.assertIn("pydotorg-app", surrogate_key)
+        self.assertIn("downloads", surrogate_key)
+
+    @override_settings(GLOBAL_SURROGATE_KEY="pydotorg-app")
+    def test_surrogate_key_header_homepage(self):
+        """Test that homepage only has global surrogate key."""
+        response = self.client.get("/")
+        self.assertTrue(response.has_header("Surrogate-Key"))
+        surrogate_key = response["Surrogate-Key"]
+        self.assertEqual(surrogate_key, "pydotorg-app")