diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65aa668..5e19510 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,5 +21,7 @@ jobs: node-version: 22 - run: node scripts/check-docs-site.mjs - run: bash scripts/tests/check-docs-site.test.sh + - name: Check the FernDesk write-path retry policy + run: python3 scripts/tests/ferndesk-sync-retry.test.py - name: Validate the Mintlify build run: npm exec --yes --package=mint@4.2.876 -- mint validate diff --git a/.gitignore b/.gitignore index f25e155..627aaa5 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ .mintlify/ node_modules/ +# Python bytecode from scripts/tests +__pycache__/ + # FernDesk sync local caches (never commit API-derived IDs casually) .ferndesk-slug-cache*.json ferndesk-sync-summary.json diff --git a/scripts/FERNDESK.md b/scripts/FERNDESK.md index f814665..a6edad6 100644 --- a/scripts/FERNDESK.md +++ b/scripts/FERNDESK.md @@ -27,6 +27,37 @@ GitHub Action: `.github/workflows/ferndesk-sync.yml` Repo secret required: `FERNDESK_API_KEY`. +## Write-path rate limits (429) + +`POST /articles` answers `429` / `{"code":"rate_limited"}` under load. The write +path (article create, update, publish, and collection create) backs off +exponentially — 5s doubling to a 180s cap, the slower ladder 429 needs; 5xx and +CF 1010 keep 3s doubling to 90s — and waits for `Retry-After` (delta-seconds or +HTTP-date) whenever the server sends it and it is larger. + +Retries stay bounded on three axes, so a rate-limited run finishes and reports +instead of hanging: + +| Env | Default | Meaning | +|-----|---------|---------| +| `FERNDESK_WRITE_RETRIES` | `12` | attempts per write | +| `FERNDESK_WRITE_DEADLINE` | `1800` | seconds of retrying for one write | +| `FERNDESK_WRITE_BUDGET` | `5400` | seconds of retry time for the whole run (`0` disables) | + +Reads share the 12-attempt default. A `Retry-After` above 300s is treated as +unusable and the exponential ladder is used instead. Hard 4xx (400/401/404) +fail immediately without retrying. + +One page that exhausts its retries does not abort the rest of the run: the sync +continues, then logs a `FAILURES` line, records `failed` / `failed_slugs` in the +SUMMARY, and exits `1`. Nothing is cached for a failed write, so the next run +retries that page cleanly. Re-run the sync (or the workflow) once the limit +clears. + +A create that trips a slug conflict (409/422, or a message naming the slug) is +recovered by looking the article up and PATCHing it, so a stale cache or a +pagination miss does not surface as a failure. + ## Factory Droid path (agent) When Manager Deploy lands **prod** and content needs judgment (rewrites, gap fill, migration QA), launch Factory Droid only: @@ -41,6 +72,8 @@ Prompt the Droid to run `scripts/ferndesk_sync.py` against the tip of `CortexLM/ - Sync **upserts** by slug; it does **not** delete FernDesk-only articles. - Mintlify remains the source of truth in git until cutover is complete. +- `python3 scripts/tests/ferndesk-sync-retry.test.py` covers the write-path + retry policy offline (faked transport, virtual clock). ## Cloudflare / GitHub Actions diff --git a/scripts/ferndesk_sync.py b/scripts/ferndesk_sync.py index e9aca31..73c0f1b 100755 --- a/scripts/ferndesk_sync.py +++ b/scripts/ferndesk_sync.py @@ -8,12 +8,17 @@ FERNDESK_DRY_LOCAL 1 = discover pages only, no API FERNDESK_FULL_SCAN 1 = rebuild slug cache by listing all articles FERNDESK_SLUG_CACHE path to slug→id cache (default .ferndesk-slug-cache.json) + FERNDESK_WRITE_RETRIES attempts per article/collection write (default 12) + FERNDESK_WRITE_DEADLINE seconds of retrying allowed per write (default 1800) + FERNDESK_WRITE_BUDGET seconds of retry time for the whole run (default 5400) DOCS_ROOT docs repo root (default: cwd) Idempotent upsert by slug. Never deletes FernDesk-only articles (safe migration). """ from __future__ import annotations +import datetime +import email.utils import hashlib import json import os @@ -88,7 +93,111 @@ def _is_cf_1010(status: int, body: str) -> bool: return "1010" in b or "browser_signature_banned" in b or "error 1010" in b -def api(key: str, path: str, method: str = "GET", body: dict | None = None, retries: int = 12): +# Transient failures worth retrying. 429 is the article-write rate limit +# (`{"code":"rate_limited"}`); 5xx are origin hiccups behind Cloudflare. +_RETRY_STATUSES = (429, 502, 503, 504) +_WRITE_METHODS = {"POST", "PATCH", "PUT", "DELETE"} +_BACKOFF_BASE = 3.0 +_BACKOFF_CAP = 90.0 +# 429 needs a longer cool-down than 5xx/CF 1010 — FernDesk rate limits recover +# slowly once the article writes start tripping them. +_RATE_LIMIT_BACKOFF_BASE = 5.0 +_RATE_LIMIT_BACKOFF_CAP = 180.0 +# A server-supplied Retry-After past this cap is treated as unusable and the +# exponential ladder is used instead — an unbounded header must not park CI. +_RETRY_AFTER_CAP = 300.0 +# Attempts per request. Writes additionally carry a per-write deadline and a +# run-wide budget so a rate-limited run always finishes and reports. +_DEFAULT_RETRIES = 12 +_DEFAULT_WRITE_RETRIES = 12 + + +def _env_number(name: str, default: float, minimum: float = 0.0) -> float: + raw = os.environ.get(name) + if raw is None or not raw.strip(): + return default + try: + value = float(raw) + except ValueError: + log(f"WARN {name}={raw!r} is not a number; using {default:g}") + return default + if value < minimum: + log(f"WARN {name}={raw!r} below minimum {minimum:g}; using {minimum:g}") + return minimum + return value + + +def _header(headers, name: str) -> str | None: + """Read a response header from curl_cffi/urllib without trusting either API.""" + if headers is None: + return None + try: + value = headers.get(name) + except Exception: + return None + return value if isinstance(value, str) else None + + +def _retry_after_seconds(value: str | None) -> float | None: + """Retry-After as delta-seconds or HTTP-date; None when absent or malformed.""" + if not value or not value.strip(): + return None + raw = value.strip() + try: + return max(0.0, float(raw)) + except ValueError: + pass + try: + when = email.utils.parsedate_to_datetime(raw) + except (TypeError, ValueError): + return None + if when is None: + return None + if when.tzinfo is None: + when = when.replace(tzinfo=datetime.timezone.utc) + return max(0.0, (when - datetime.datetime.now(datetime.timezone.utc)).total_seconds()) + + +def _retry_delay(attempt: int, retry_after: float | None, status: int | None = None) -> tuple[float, str]: + """Wait before the next attempt: server's Retry-After when usable, else backoff. + + 429 gets the slower ladder main introduced (5s doubling to 180s); other + transient statuses and CF 1010 keep the shorter one (3s doubling to 90s). + """ + if status == 429: + backoff = min(_RATE_LIMIT_BACKOFF_CAP, _RATE_LIMIT_BACKOFF_BASE * (2**attempt)) + else: + backoff = min(_BACKOFF_CAP, _BACKOFF_BASE * (2**attempt)) + if retry_after is None: + return backoff, "backoff" + if retry_after > _RETRY_AFTER_CAP: + return backoff, f"backoff (Retry-After {retry_after:.0f}s over {_RETRY_AFTER_CAP:.0f}s cap)" + return max(backoff, retry_after), "Retry-After" + + +# Retry time spent on writes so far. A sustained 429 storm across ~100 pages +# could otherwise run past the job timeout and lose the SUMMARY entirely, so +# the run stops retrying once the budget is gone and reports what landed. +_write_budget_spent = 0.0 + + +def _write_budget_left() -> float | None: + """Seconds of write retry time left, or None when the budget is disabled.""" + limit = _env_number("FERNDESK_WRITE_BUDGET", 5400.0, 0.0) + if limit <= 0: + return None + return max(0.0, limit - _write_budget_spent) + + +def api( + key: str, + path: str, + method: str = "GET", + body: dict | None = None, + retries: int | None = None, + deadline: float | None = None, + label: str | None = None, +): headers = { "Authorization": f"Bearer {key}", "Accept": _ACCEPT, @@ -102,8 +211,46 @@ def api(key: str, path: str, method: str = "GET", body: dict | None = None, retr client_kind, cffi_requests = _http_client() last = None url = API + path + tag = f" [{label}]" if label else "" + + # Writes are the constrained path (POST /articles 429 `rate_limited`), so + # they additionally get a wall-clock deadline and share a run-wide budget; + # reads just get the attempt count. + if retries is None: + retries = int( + _env_number("FERNDESK_WRITE_RETRIES", _DEFAULT_WRITE_RETRIES, 1.0) + if method in _WRITE_METHODS + else _DEFAULT_RETRIES + ) + if deadline is None and method in _WRITE_METHODS: + deadline = _env_number("FERNDESK_WRITE_DEADLINE", 1800.0, 1.0) + started = time.monotonic() + retry_after: float | None = None + retry_status: int | None = None + status_note = "transient failure" + write = method in _WRITE_METHODS + budget_left = _write_budget_left() if write else None for i in range(retries): + if i: + wait, why = _retry_delay(i - 1, retry_after, retry_status) + if deadline is not None and time.monotonic() - started + wait > deadline: + last = f"{last} (deadline {deadline:.0f}s exceeded after {i} attempts)" + break + if budget_left is not None: + if wait > budget_left: + last = ( + f"{last} (write retry budget exhausted after {i} attempts; " + "rerun the sync once the rate limit clears)" + ) + break + budget_left -= wait + globals()["_write_budget_spent"] = _write_budget_spent + wait + log( + f"{method} {path}{tag} retry {i}/{retries - 1} " + f"{status_note}; sleep {wait:.1f}s ({why})" + ) + time.sleep(wait) try: if client_kind == "curl_cffi": resp = cffi_requests.request( @@ -117,19 +264,16 @@ def api(key: str, path: str, method: str = "GET", body: dict | None = None, retr txt = (resp.text or "")[:400] if resp.status_code >= 400: last = f"{method} {path} -> {resp.status_code} {txt}" - if resp.status_code in (429, 502, 503, 504) or _is_cf_1010( + if resp.status_code in _RETRY_STATUSES or _is_cf_1010( resp.status_code, txt ): - # 429 needs longer cool-down than 5xx/1010 - cap = 180 if resp.status_code == 429 else 90 - wait = min(cap, (5 if resp.status_code == 429 else 3) * (2**i)) - kind = ( + retry_after = _retry_after_seconds(_header(resp.headers, "Retry-After")) + retry_status = resp.status_code + status_note = ( "cf1010/403" if _is_cf_1010(resp.status_code, txt) - else f"rate/limit {resp.status_code}" + else f"HTTP {resp.status_code}" ) - log(f"{kind}; sleep {wait}s") - time.sleep(wait) continue raise RuntimeError(last) raw = resp.content or b"" @@ -143,27 +287,25 @@ def api(key: str, path: str, method: str = "GET", body: dict | None = None, retr except urllib.error.HTTPError as e: txt = e.read().decode("utf-8", "replace")[:400] last = f"{method} {path} -> {e.code} {txt}" - if e.code in (429, 502, 503, 504) or _is_cf_1010(e.code, txt): - cap = 180 if e.code == 429 else 90 - wait = min(cap, (5 if e.code == 429 else 3) * (2**i)) - kind = "cf1010/403" if _is_cf_1010(e.code, txt) else f"rate/limit {e.code}" - log(f"{kind}; sleep {wait}s") - time.sleep(wait) + if e.code in _RETRY_STATUSES or _is_cf_1010(e.code, txt): + retry_after = _retry_after_seconds(_header(e.headers, "Retry-After")) + retry_status = e.code + status_note = "cf1010/403" if _is_cf_1010(e.code, txt) else f"HTTP {e.code}" continue raise RuntimeError(last) from e except urllib.error.URLError as e: last = f"{method} {path} -> URLError {e}" - wait = min(90, 3 * (2**i)) - log(f"transport error; sleep {wait}s ({e})") - time.sleep(wait) + retry_after = None + retry_status = None + status_note = f"transport error ({e})" continue except RuntimeError: raise except Exception as e: last = f"{method} {path} -> {type(e).__name__} {e}" - wait = min(90, 3 * (2**i)) - log(f"transport error; sleep {wait}s ({e})") - time.sleep(wait) + retry_after = None + retry_status = None + status_note = f"transport error ({e})" continue raise RuntimeError(last or "retries exhausted") @@ -371,7 +513,8 @@ def main() -> int: cache_path.write_text(json.dumps(by_slug, indent=2) + "\n") log(f"wrote slug cache {len(by_slug)}") - created = updated = skipped = 0 + created = updated = skipped = failed = 0 + failures: list[dict] = [] for page in pages: page["keywords"] = page["keywords"].replace("{ENV}", target) coll = ensure_collection(key, page["collection"], section["id"], colls) @@ -392,10 +535,18 @@ def main() -> int: log(f"DRY update {page['slug']} -> {eid}") skipped += 1 continue - api(key, f"/articles/{eid}", "PATCH", body_common) - if publish and estatus != "published": - time.sleep(0.15) - api(key, f"/articles/{eid}/publish", "POST", {}) + try: + api(key, f"/articles/{eid}", "PATCH", body_common, label=page["slug"]) + if publish and estatus != "published": + time.sleep(0.15) + api(key, f"/articles/{eid}/publish", "POST", {}, label=page["slug"]) + except RuntimeError as e: + # One stuck article must not abort the remaining pages; the + # failure still fails the run and lands in SUMMARY. + failed += 1 + failures.append({"slug": page["slug"], "op": "update", "error": str(e)}) + log(f"ERROR update {page['slug']} failed after retries: {e}") + continue updated += 1 log(f"updated {page['slug']}") time.sleep(1.2) @@ -406,34 +557,47 @@ def main() -> int: created += 1 continue try: - art = api(key, "/articles", "POST", {**body_common, "publish": publish}) + art = api( + key, + "/articles", + "POST", + {**body_common, "publish": publish}, + label=page["slug"], + ) except RuntimeError as e: # Slug may already exist (pagination/cache miss) — recover via lookup + PATCH. msg = str(e) - if "409" not in msg and "already" not in msg.lower() and "slug" not in msg.lower() and "422" not in msg: - raise - log(f"create conflict for {page['slug']}; looking up existing…") - found = None - for a in list_all( - key, - f"/articles?sectionId={urllib.parse.quote(str(section['id']))}&slug={urllib.parse.quote(page['slug'])}", - max_pages=5, - ): - if a.get("slug") == page["slug"]: - found = a - break - if not found: - raise - art = found - api(key, f"/articles/{art['id']}", "PATCH", body_common) - if publish and art.get("status") != "published": - time.sleep(0.3) - api(key, f"/articles/{art['id']}/publish", "POST", {}) - by_slug[page["slug"]] = {"id": art["id"], "status": art.get("status") or "published"} - cache_path.write_text(json.dumps(by_slug, indent=2) + "\n") - updated += 1 - log(f"recovered-update {page['slug']}") - time.sleep(1.5) + if "409" in msg or "already" in msg.lower() or "slug" in msg.lower() or "422" in msg: + log(f"create conflict for {page['slug']}; looking up existing…") + found = None + for a in list_all( + key, + f"/articles?sectionId={urllib.parse.quote(str(section['id']))}&slug={urllib.parse.quote(page['slug'])}", + max_pages=5, + ): + if a.get("slug") == page["slug"]: + found = a + break + if found: + art = found + api(key, f"/articles/{art['id']}", "PATCH", body_common, label=page["slug"]) + if publish and art.get("status") != "published": + time.sleep(0.3) + api(key, f"/articles/{art['id']}/publish", "POST", {}, label=page["slug"]) + by_slug[page["slug"]] = { + "id": art["id"], + "status": art.get("status") or "published", + } + cache_path.write_text(json.dumps(by_slug, indent=2) + "\n") + updated += 1 + log(f"recovered-update {page['slug']}") + time.sleep(1.5) + continue + # Still stuck: one page must not abort the remaining pages, but the + # failure is recorded and fails the run. + failed += 1 + failures.append({"slug": page["slug"], "op": "create", "error": str(e)}) + log(f"ERROR create {page['slug']} failed after retries: {e}") continue by_slug[page["slug"]] = { "id": art.get("id"), @@ -451,11 +615,21 @@ def main() -> int: "created": created, "updated": updated, "skipped": skipped, + "failed": failed, } + if failures: + summary["failed_slugs"] = [f["slug"] for f in failures] + log("FAILURES " + json.dumps(failures)) log("SUMMARY " + json.dumps(summary)) Path(os.environ.get("FERNDESK_SUMMARY_PATH") or "ferndesk-sync-summary.json").write_text( json.dumps(summary, indent=2) + "\n" ) + if failures: + log( + f"ERROR {failed} of {len(pages)} pages failed after retries " + "(see FAILURES above); rerun the sync once the rate limit clears" + ) + return 1 return 0 diff --git a/scripts/tests/ferndesk-sync-retry.test.py b/scripts/tests/ferndesk-sync-retry.test.py new file mode 100644 index 0000000..d3539a9 --- /dev/null +++ b/scripts/tests/ferndesk-sync-retry.test.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +"""Guards the FernDesk sync write-path retry policy (COR-444 residual). + +`POST /articles` answers 429 `rate_limited` under load, so the write path has to +back off, honour `Retry-After`, and stay bounded. Runs offline: the HTTP client +is faked and `time.sleep` is virtual, so no network and no real waits. +""" +from __future__ import annotations + +import importlib.util +import json +import os +import sys +import tempfile +from contextlib import contextmanager +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +_spec = importlib.util.spec_from_file_location( + "ferndesk_sync", ROOT / "scripts" / "ferndesk_sync.py" +) +fs = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(fs) + +failures: list[str] = [] + + +def check(name: str, cond: bool, detail: object = "") -> None: + if cond: + print(f"ok {name}") + else: + failures.append(name) + print(f"FAIL {name} {detail}") + + +@contextmanager +def env(**pairs: str): + saved = {k: os.environ.get(k) for k in pairs} + os.environ.update({k: str(v) for k, v in pairs.items()}) + try: + yield + finally: + for k, v in saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + +class FakeTime: + """Virtual clock: `sleep` advances `monotonic` so deadlines are testable.""" + + def __init__(self) -> None: + self.now = 0.0 + self.slept: list[float] = [] + + def sleep(self, seconds: float) -> None: + self.slept.append(seconds) + self.now += seconds + + def monotonic(self) -> float: + return self.now + + +class FakeResponse: + def __init__(self, status_code: int, payload: object = None, headers=None) -> None: + self.status_code = status_code + self.headers = headers or {} + body = b"" if payload is None else json.dumps(payload).encode() + self.content = body + self.text = body.decode() + + +class FakeClient: + """Scripted transport; records every request it is asked to make.""" + + def __init__(self, script) -> None: + self.script = script + self.calls: list[tuple[str, str]] = [] + + def request(self, method, url, **kwargs): + self.calls.append((method, url)) + return self.script(method, url, len(self.calls)) + + +def install(script, monkeypatch_time: bool = True): + """Point the sync module at a fake transport and (optionally) a virtual clock.""" + client = FakeClient(script) + logs: list[str] = [] + saved = (fs._http_client, fs.log, fs.time if monkeypatch_time else None) + fake_time = FakeTime() + fs._http_client = lambda: ("curl_cffi", client) + fs.log = lambda msg: logs.append(str(msg)) + if monkeypatch_time: + fs.time = fake_time + return client, logs, fake_time, saved + + +def restore(saved) -> None: + fs._http_client, fs.log, saved_time = saved + if saved_time is not None: + fs.time = saved_time + fs._write_budget_spent = 0.0 + + +# --- Retry-After parsing ----------------------------------------------------- +check("retry-after absent is None", fs._retry_after_seconds(None) is None) +check("retry-after blank is None", fs._retry_after_seconds(" ") is None) +check("retry-after garbage is None", fs._retry_after_seconds("soon") is None) +check("retry-after seconds parsed", fs._retry_after_seconds("45") == 45.0) +check("retry-after negative clamps to 0", fs._retry_after_seconds("-3") == 0.0) +_date = fs.email.utils.formatdate(fs.time.time() + 120, usegmt=True) +_parsed = fs._retry_after_seconds(_date) +check( + "retry-after HTTP-date parsed", + _parsed is not None and 100 <= _parsed <= 130, + _parsed, +) +check("header helper tolerates None", fs._header(None, "Retry-After") is None) +check("header helper rejects non-str", fs._header({"Retry-After": 5}, "Retry-After") is None) + +# --- Delay ladder ------------------------------------------------------------ +_wait, why = fs._retry_delay(0, None, 503) +check("first backoff is the base", _wait == fs._BACKOFF_BASE and why == "backoff", (_wait, why)) +_wait, _ = fs._retry_delay(1, None, 503) +check("backoff grows exponentially", _wait == fs._BACKOFF_BASE * 2, _wait) +_wait, _ = fs._retry_delay(9, None, 503) +check("backoff is capped", _wait == fs._BACKOFF_CAP, _wait) +_wait, why = fs._retry_delay(0, None, 429) +check("429 uses the slower ladder", _wait == fs._RATE_LIMIT_BACKOFF_BASE, (_wait, why)) +_wait, _ = fs._retry_delay(6, None, 429) +check("429 ladder reaches its higher cap", _wait == fs._RATE_LIMIT_BACKOFF_CAP, _wait) +_wait, why = fs._retry_delay(0, 45.0, 429) +check("retry-after wins when larger", _wait == 45.0 and why == "Retry-After", (_wait, why)) +_wait, why = fs._retry_delay(0, 1.0, 429) +check("server floor never undercut", _wait == fs._RATE_LIMIT_BACKOFF_BASE, _wait) +_wait, why = fs._retry_delay(0, fs._RETRY_AFTER_CAP + 1, 429) +check("absurd retry-after falls back to backoff", _wait == fs._RATE_LIMIT_BACKOFF_BASE and "cap" in why, (_wait, why)) + +# --- 429 then success on the write path ------------------------------------- +def flaky_then_ok(method, url, n): + if n < 3: + return FakeResponse(429, {"error": "Too many requests", "code": "rate_limited"}, + {"Retry-After": "20"}) + return FakeResponse(201, {"id": "art-1", "status": "published"}) + + +client, logs, fake_time, saved = install(flaky_then_ok) +try: + with env(FERNDESK_WRITE_RETRIES="5", FERNDESK_WRITE_DEADLINE="600"): + art = fs.api("k", "/articles", "POST", {"title": "t"}, label="getting-started-quickstart") + check("write retries through 429 and succeeds", art.get("id") == "art-1", art) + check("write made three attempts", len(client.calls) == 3, client.calls) + check("retry-after honoured on every wait", fake_time.slept == [20.0, 20.0], fake_time.slept) + check("write method is POST throughout", {m for m, _ in client.calls} == {"POST"}, client.calls) + check("logs name the article slug", any("getting-started-quickstart" in line for line in logs), logs) + check("logs report the rate limit", any("HTTP 429" in line for line in logs), logs) + check("logs report the wait reason", any("Retry-After" in line for line in logs), logs) +finally: + restore(saved) + +# --- exhausted retries raise with the body ---------------------------------- +client, logs, fake_time, saved = install(lambda m, u, n: FakeResponse(429, {"code": "rate_limited"}, {})) +try: + with env(FERNDESK_WRITE_RETRIES="4"): + raised = None + try: + fs.api("k", "/articles", "POST", {}, label="slug-x") + except RuntimeError as e: + raised = str(e) + check("exhausted retries raise", raised is not None, raised) + check("error keeps the last status", raised and "429" in raised, raised) + check("error keeps the api code", raised and "rate_limited" in raised, raised) + check("attempts match the configured budget", len(client.calls) == 4, len(client.calls)) +finally: + restore(saved) + +# --- deadline stops the ladder ---------------------------------------------- +client, logs, fake_time, saved = install(lambda m, u, n: FakeResponse(503, {"code": "unavailable"}, {})) +try: + with env(FERNDESK_WRITE_RETRIES="20"): + raised = None + try: + fs.api("k", "/articles", "POST", {}, deadline=10) + except RuntimeError as e: + raised = str(e) + check("deadline aborts a long ladder", raised is not None and "deadline" in raised, raised) + check("deadline stops before the cap", len(client.calls) == 3, len(client.calls)) + check("virtual clock stayed within the deadline", fake_time.now <= 10, fake_time.now) +finally: + restore(saved) + +# --- 5xx still retried, hard 4xx is not ------------------------------------- +client, logs, fake_time, saved = install( + lambda m, u, n: FakeResponse(400, {"error": "bad request", "code": "invalid_request"}, {}) +) +try: + with env(FERNDESK_WRITE_RETRIES="5"): + raised = None + try: + fs.api("k", "/articles", "POST", {}) + except RuntimeError as e: + raised = str(e) + check("400 fails fast without retrying", len(client.calls) == 1, len(client.calls)) + check("400 surfaces its code", raised and "invalid_request" in raised, raised) +finally: + restore(saved) + +# --- reads keep the lighter budget ------------------------------------------ +client, logs, fake_time, saved = install(lambda m, u, n: FakeResponse(429, {"code": "rate_limited"}, {})) +try: + with env(FERNDESK_WRITE_RETRIES="99"): + try: + fs.api("k", "/articles") + except RuntimeError: + pass + check("reads keep the shared default budget", len(client.calls) == fs._DEFAULT_RETRIES, len(client.calls)) +finally: + restore(saved) + +# --- run-wide write budget -------------------------------------------------- +client, logs, fake_time, saved = install( + lambda m, u, n: FakeResponse(429, {"code": "rate_limited"}, {"Retry-After": "30"}) +) +try: + with env(FERNDESK_WRITE_RETRIES="50", FERNDESK_WRITE_BUDGET="100"): + raised = None + try: + fs.api("k", "/articles", "POST", {}, label="budget-a") + except RuntimeError as e: + raised = str(e) + check("run budget stops a rate-limit storm", raised is not None and "budget" in raised, raised) + check("run budget spends no more than allowed", fake_time.now <= 100, fake_time.now) + spent = fs._write_budget_spent + check("run budget is charged for waits", 0 < spent <= 100, spent) + check("run budget is shared across writes", spent > 0, spent) +finally: + restore(saved) + +# --- budget disabled means retry until the per-write deadline ---------------- +client, logs, fake_time, saved = install( + lambda m, u, n: FakeResponse(429, {"code": "rate_limited"}, {"Retry-After": "30"}) +) +try: + with env(FERNDESK_WRITE_RETRIES="50", FERNDESK_WRITE_BUDGET="0"): + raised = None + try: + fs.api("k", "/articles", "POST", {}, deadline=120) + except RuntimeError as e: + raised = str(e) + check("budget 0 disables the run-wide cap", raised is not None and "deadline" in raised, raised) +finally: + restore(saved) + +# --- shared fixtures for the end-to-end runs --------------------------------- +docs_root = Path(tempfile.mkdtemp(prefix="ferndesk-docs-")) +(docs_root / "getting-started").mkdir(parents=True) +(docs_root / "getting-started" / "quickstart.mdx").write_text( + "---\ntitle: Quickstart\n---\n\nHello.\n", encoding="utf-8" +) + +SECTIONS = [{"id": "sec-1", "name": "Staging"}] +COLLECTIONS = [{"id": "col-1", "sectionId": "sec-1", "title": "Getting Started"}] + +# --- create conflict recovery (main's slug lookup + PATCH) still works ------- +def conflict_routes(method, url, n): + if "/sections" in url: + return FakeResponse(200, SECTIONS) + if "/collections" in url: + return FakeResponse(200, COLLECTIONS) + if "/articles" in url and method == "POST": + return FakeResponse(409, {"error": "slug already exists", "code": "conflict"}) + if "/articles" in url and "slug=" in url: + return FakeResponse(200, {"results": [{"id": "art-9", "slug": "getting-started-quickstart", + "status": "draft", "sectionId": "sec-1"}], + "has_more": False}) + if "/articles" in url and method == "PATCH": + return FakeResponse(200, {"id": "art-9", "status": "draft"}) + if "/articles" in url: + return FakeResponse(200, {"results": [], "has_more": False}) + return FakeResponse(404, {"code": "not_found"}) + + +summary_path = docs_root / "summary-conflict.json" +cache_path = docs_root / "cache-conflict.json" +client, logs, fake_time, saved = install(conflict_routes) +try: + with env( + FERNDESK_API_KEY="test-key", + FERNDESK_TARGET="staging", + FERNDESK_SLUG_CACHE=str(cache_path), + FERNDESK_SUMMARY_PATH=str(summary_path), + DOCS_ROOT=str(docs_root), + FERNDESK_FULL_SCAN="1", + ): + code = fs.main() + summary = json.loads(summary_path.read_text()) + check("conflict recovery keeps the run green", code == 0, code) + check("conflict recovery counts an update", summary.get("updated") == 1, summary) + check("conflict recovery reports no failures", summary.get("failed") == 0, summary) + check("conflict recovery caches the found id", json.loads(cache_path.read_text()).get( + "getting-started-quickstart", {}).get("id") == "art-9", cache_path.read_text()) + check("conflict recovery is logged", any("recovered-update" in line for line in logs), logs) +finally: + restore(saved) + +# --- end-to-end: a permanently rate-limited page fails the run honestly ----- +def sync_routes(method, url, n): + if "/sections" in url: + return FakeResponse(200, SECTIONS) + if "/collections" in url: + return FakeResponse(200, COLLECTIONS) + if "/articles" in url and method == "POST": + # The residual COR-444 case: article creation stays rate limited. + return FakeResponse(429, {"error": "Too many requests", "code": "rate_limited"}, + {"Retry-After": "5"}) + if "/articles" in url: + # Slug scan lists the section before the write path runs. + return FakeResponse(200, {"results": [], "has_more": False}) + return FakeResponse(404, {"code": "not_found"}) + + +summary_path = docs_root / "summary.json" +cache_path = docs_root / "cache.json" +client, logs, fake_time, saved = install(sync_routes) +try: + with env( + FERNDESK_API_KEY="test-key", + FERNDESK_TARGET="staging", + FERNDESK_WRITE_RETRIES="3", + FERNDESK_SLUG_CACHE=str(cache_path), + FERNDESK_SUMMARY_PATH=str(summary_path), + DOCS_ROOT=str(docs_root), + FERNDESK_FULL_SCAN="1", + ): + code = fs.main() + summary = json.loads(summary_path.read_text()) + check("a stuck page fails the run", code == 1, code) + check("summary counts the failure", summary.get("failed") == 1, summary) + check("summary names the stuck slug", summary.get("failed_slugs") == ["getting-started-quickstart"], summary) + check("summary still reports the page total", summary.get("pages") == 1, summary) + check("no article id cached for a failed write", json.loads(cache_path.read_text()) == {}, cache_path.read_text()) + check("FAILURES line is logged", any(line.startswith("FAILURES ") for line in logs), logs) +finally: + restore(saved) + +if failures: + print(f"ferndesk-sync-retry: {len(failures)} check(s) failed", file=sys.stderr) + sys.exit(1) +print("ferndesk-sync-retry: ok")