Bump workflow actions to current majors; retry flaky registry checks once

.github/dependabot.yml

@@ -5,3 +5,8 @@ updates:
     directory: /
     schedule:
       interval: monthly
+    # One PR for all action bumps, so they merge (and deploy) together; separate
+    # PRs also queue behind Dependabot's five-open-PR limit.
+    groups:
+      github-actions:
+        patterns: ["*"]

.github/workflows/pages.yml

@@ -22,8 +22,8 @@ jobs:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
     steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.12"
       # Network-free gate: a malformed registry (duplicate id, missing image,
@@ -37,8 +37,8 @@ jobs:
         run: |
           mkdir _site
           cp -r index.html css js images assets vendor devices.json _site/
-      - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1
+      - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0
         with:
           path: _site
       - id: deployment
-        uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5
+        uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1

.github/workflows/validate.yml

@@ -14,12 +14,15 @@ permissions:
 jobs:
   registry:
     runs-on: ubuntu-latest
+    # A normal run takes well under a minute; this bounds a total upstream outage,
+    # where every request waits out its timeout and its one retry.
+    timeout-minutes: 20
     permissions:
       contents: read
       issues: write   # the nightly opens an issue when upstream firmware breaks
     steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.12"
       - name: Validator unit tests (offline)
@@ -56,11 +59,11 @@ jobs:
       run:
         working-directory: tests
     steps:
-      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: "3.12"
-      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
         with:
           node-version: "22"
           cache: npm
@@ -70,7 +73,7 @@ jobs:
       - run: npx playwright test
       - name: Keep traces of failed tests
         if: failure()
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
           name: playwright-results
           path: tests/test-results/

scripts/test_validate_registry.py

@@ -16,6 +16,29 @@
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 import validate_registry as vr  # noqa: E402
 
+vr.RETRY_DELAY = 0  # the retry tests below must not sleep
+
+
+class FakeResponse:
+    def __init__(self, body=b"", status=200, headers=None, url="https://example.com/"):
+        self.status, self.headers, self._body, self._url = status, headers or {}, body, url
+
+    def read(self):
+        return self._body
+
+    def geturl(self):
+        return self._url
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        return False
+
+
+def http_error(code):
+    return vr.urllib.error.HTTPError("https://example.com/x", code, "err", {}, None)
+
 
 class ShapeChecks(unittest.TestCase):
     def test_good_manifest_has_no_errors(self):
@@ -100,6 +123,60 @@ def test_dropped_connection_returns_false(self):
             with mock.patch.object(vr.urllib.request, "urlopen", side_effect=exc):
                 self.assertFalse(vr.head_ok("https://example.com/x.bin"), exc)
 
+    def test_transient_failure_is_retried_once(self):
+        # One blip (the 2026-10-23 nightly: AIR-1's classic page) must not fail the run.
+        ok = FakeResponse(headers={"Access-Control-Allow-Origin": "*"})
+        for blip in (vr.urllib.error.URLError("reset"), TimeoutError(), http_error(503)):
+            urlopen = mock.Mock(side_effect=[blip, ok])
+            with mock.patch.object(vr.urllib.request, "urlopen", urlopen):
+                self.assertTrue(vr.head_ok("https://example.com/x.bin"), blip)
+            self.assertEqual(urlopen.call_count, 2, blip)
+
+    def test_head_never_reads_a_body(self):
+        # On Python 3.12 a redirected HEAD becomes a GET; reading it would pull
+        # down a whole firmware image for every part checked.
+        ok = FakeResponse()
+        ok.read = mock.Mock(side_effect=AssertionError("HEAD read a body"))
+        with mock.patch.object(vr.urllib.request, "urlopen", return_value=ok):
+            self.assertTrue(vr.head_ok("https://example.com/x.bin"))
+
+    def test_client_error_is_not_retried(self):
+        urlopen = mock.Mock(side_effect=http_error(404))
+        with mock.patch.object(vr.urllib.request, "urlopen", urlopen):
+            self.assertFalse(vr.head_ok("https://example.com/x.bin"))
+        self.assertEqual(urlopen.call_count, 1)
+
+    def test_gives_up_after_one_retry(self):
+        urlopen = mock.Mock(side_effect=vr.urllib.error.URLError("down"))
+        with mock.patch.object(vr.urllib.request, "urlopen", urlopen):
+            self.assertFalse(vr.head_ok("https://example.com/x.bin"))
+        self.assertEqual(urlopen.call_count, 2)
+
+    def test_classic_page_fetch_is_retried_and_resolves_against_the_final_url(self):
+        # Without the trailing slash, Pages redirects to .../AIR-1/; relative
+        # manifests must resolve there, not at the site root.
+        page = "https://apolloautomation.github.io/AIR-1"
+        html = b'<esp-web-install-button manifest="./firmware-factory/manifest.json">'
+        urlopen = mock.Mock(side_effect=[TimeoutError(), FakeResponse(html, url=page + "/")])
+        vr._classic_pages.clear()
+        with mock.patch.object(vr.urllib.request, "urlopen", urlopen):
+            self.assertEqual(vr.classic_manifests(page), {page + "/firmware-factory/manifest.json"})
+        self.assertEqual(urlopen.call_count, 2)
+        vr._classic_pages.clear()
+
+    def test_manifest_fetch_is_retried(self):
+        body = b'{"builds": [{"chipFamily": "ESP32", "parts": [{"path": "a.bin"}]}]}'
+        cors = {"Access-Control-Allow-Origin": "*"}
+        urlopen = mock.Mock(side_effect=[
+            vr.urllib.error.URLError("reset"), FakeResponse(body, headers=cors),  # manifest
+            FakeResponse(headers=cors),                                            # part HEAD
+        ])
+        vr.errors.clear()
+        with mock.patch.object(vr.urllib.request, "urlopen", urlopen):
+            vr.check_manifest("dev", "stable", "v", "https://example.com/m/manifest.json")
+        self.assertEqual(vr.errors, [])
+        self.assertEqual(urlopen.call_count, 3)
+
     def test_cors_ok(self):
         self.assertTrue(vr.cors_ok({"Access-Control-Allow-Origin": "*"}))
         self.assertTrue(vr.cors_ok({"Access-Control-Allow-Origin": vr.ORIGIN}))

scripts/validate_registry.py

@@ -11,15 +11,19 @@
 reachable; and each manifest is one the variant's classic installer page offers.
 (That can't tell two of a page's builds apart, so two variants swapped with each
 other still pass; a variant duplicating another's manifest is caught offline.)
+A request that fails in a way that may be transient (connection error, timeout,
+5xx) is retried once, so one network blip doesn't fail the nightly run.
 Exit 1 on any failure.
 
 Run: python scripts/validate_registry.py [--offline]
 """
+import collections
 import http.client
 import json
 import os
 import re
 import sys
+import time
 import urllib.error
 import urllib.parse
 import urllib.request
@@ -37,21 +41,44 @@
 # Channel and variant keys are written into HTML attributes and CSS selectors.
 UNSAFE_KEY_CHARS = set('"<>&`\\')
 MANIFEST_ATTR = re.compile(r"""\bmanifest\s*=\s*["']([^"']+)["']""")
+RETRY_DELAY = 5  # seconds before the one retry of a request that may have failed transiently
 errors = []
 
+Response = collections.namedtuple("Response", "status headers body url")
+
+def fetch(url, method="GET", headers=None):
+    """One request, read in full (the body is skipped for HEAD).
+
+    Raises like urlopen on failure. A failure that may be transient (connection
+    error, timeout, 5xx) is retried once after RETRY_DELAY seconds; a 4xx is not,
+    because it won't fix itself. URLError and a read-phase TimeoutError are both
+    OSErrors; a dropped connection can also surface unwrapped (RemoteDisconnected).
+    """
+    req = urllib.request.Request(url, headers=dict(UA, **(headers or {})), method=method)
+    for attempt in (1, 2):
+        try:
+            with urllib.request.urlopen(req, timeout=30) as r:
+                body = b"" if method == "HEAD" else r.read()
+                return Response(r.status, r.headers, body, r.geturl())
+        except urllib.error.HTTPError as e:
+            if e.code < 500 or attempt == 2:
+                raise
+        except (OSError, http.client.HTTPException):
+            if attempt == 2:
+                raise
+        time.sleep(RETRY_DELAY)
+
 def head(url):
     """HEAD url with the installer's Origin; return the headers on a 200, else None.
 
-    Any connection, timeout or protocol failure counts as unreachable rather than
-    crashing the run. URLError and a read-phase TimeoutError are both OSErrors;
-    a dropped connection can also surface unwrapped (RemoteDisconnected).
+    Any failure (after fetch()'s retry) counts as unreachable rather than
+    crashing the run.
     """
-    req = urllib.request.Request(url, headers=dict(UA, Origin=ORIGIN), method="HEAD")
     try:
-        with urllib.request.urlopen(req, timeout=30) as r:
-            return r.headers if r.status == 200 else None
+        r = fetch(url, "HEAD", {"Origin": ORIGIN})
     except (OSError, http.client.HTTPException):
         return None
+    return r.headers if r.status == 200 else None
 
 def head_ok(url):
     return head(url) is not None
@@ -344,10 +371,9 @@ def check_platforms_shape(platforms, firmware, dev_id):
 def check_manifest(dev_id, channel, variant, murl):
     where = f"{dev_id} {channel}/{variant}"
     try:
-        req = urllib.request.Request(murl, headers=dict(UA, Origin=ORIGIN))
-        with urllib.request.urlopen(req, timeout=30) as r:
-            readable = cors_ok(r.headers)
-            manifest = json.load(r)
+        r = fetch(murl, headers={"Origin": ORIGIN})
+        readable = cors_ok(r.headers)
+        manifest = json.loads(r.body)
     except Exception as e:
         errors.append(f"{where}: manifest fetch/parse failed: {e}")
         return
@@ -374,10 +400,8 @@ def classic_manifests(page):
     """Fetch a classic installer page once; None when it can't be reached."""
     if page not in _classic_pages:
         try:
-            req = urllib.request.Request(page, headers=UA)
-            with urllib.request.urlopen(req, timeout=30) as r:
-                html = r.read().decode("utf-8", "replace")
-                _classic_pages[page] = manifests_on_page(html, r.geturl())
+            r = fetch(page)
+            _classic_pages[page] = manifests_on_page(r.body.decode("utf-8", "replace"), r.url)
         except (OSError, http.client.HTTPException, ValueError):
             _classic_pages[page] = None
     return _classic_pages[page]