Handle documentation entry queries across the classic and Astro cutover

Author: zayenz

docs/deployment-runbook.md

@@ -243,8 +243,15 @@ Worker route remains sufficient rollback.
    The narrow canary route is more specific than `/doc/*` and would otherwise
    keep intercepting that version.
 
-The production documentation routes are `/doc`, `/doc/*`, `/doc-latest`,
-`/doc-latest/*` and `/robots.txt`. The Worker serves the same checked-in robots
+The production route patterns are `www.gecode.dev/doc*` and
+`www.gecode.dev/robots.txt*`. Cloudflare matches query strings too, so the
+trailing wildcards cover bare entry points with queries. The Worker handles
+only `/doc`, `/doc/...`, `/doc-latest`, `/doc-latest/...` and `/robots.txt`;
+it passes other matching website paths through to the origin unchanged.
+`/doc` redirects to `/documentation.html`, which works during the classic-site
+soak and through the Astro fallback afterward.
+
+The Worker serves the same checked-in robots
 file as Astro so the indexing policy takes effect before the website cutover.
 It selects aliases through `LATEST_DOC_VERSION`; it does not copy alias objects.
 Only `https://www.gecode.dev/doc/latest/...` is indexable. Versioned URLs remain

docs/static-documentation-hosting.md

@@ -29,8 +29,8 @@ The active Astro site remains free on GitHub Pages.
 
 ```text
 www.gecode.dev/*                  GitHub Pages (Astro)
-www.gecode.dev/doc/*              Cloudflare Worker route
-www.gecode.dev/doc-latest/*       Cloudflare Worker route
+www.gecode.dev/doc*               Cloudflare Worker route
+www.gecode.dev/robots.txt*        Cloudflare Worker route
                                       |
                                       v
                               private R2 bucket
@@ -221,7 +221,9 @@ versioned key in R2.
 - Proxy the `www` record while retaining its GitHub Pages CNAME origin.
 - Test the staging custom domain, then deploy the checked-in canary environment.
 - After the canary passes, install the production routes for
-  `www.gecode.dev/doc/*` and `www.gecode.dev/doc-latest/*`.
+  `www.gecode.dev/doc*` and `www.gecode.dev/robots.txt*`. The trailing wildcard
+  includes query strings; the Worker passes similarly prefixed website paths
+  outside its documentation namespace to the origin unchanged.
 - Remove the canary after the production smoke test so its more-specific route
   no longer intercepts the selected version.
 - Compare status, body digest, MIME type, cache headers, and range behavior

scripts/docs/smoke-worker.mjs

@@ -46,8 +46,20 @@ for (const prefix of immutableOnly ? [`/doc/${version}`] : [`/doc/${version}`, "
 
 if (!immutableOnly) {
   await check("/doc?smoke=1", 308, {}, (response) => {
-    assert.equal(response.headers.get("location"), `${origin}/documentation/?smoke=1`);
+    assert.equal(response.headers.get("location"), `${origin}/documentation.html?smoke=1`);
   });
+  await check("/robots.txt?smoke=1", 200, {}, async (response) => {
+    assert.match(response.headers.get("content-type"), /text\/plain/);
+    const body = await response.text();
+    assert.doesNotMatch(body, /^Disallow:\s*\/doc(?:\/latest|-latest)/m);
+    assert.match(body, /^Sitemap: https:\/\/www\.gecode\.dev\/doc\/sitemap\.xml$/m);
+  });
+  if (production) {
+    await check("/documentation.html?smoke=1", 200, { redirect: "follow" }, (response) => {
+      assertNoindex(response, false);
+      assert.equal(response.headers.get("x-gecode-documentation-version"), null);
+    });
+  }
 }
 await check(`/doc/${version}/reference/doxygen.css`, 200, {}, (response) => {
   assert.match(response.headers.get("content-type"), /text\/css/);

workers/docs/README.md

@@ -21,6 +21,13 @@ version URLs, including PDFs, carry `X-Robots-Tag: noindex`. The
 `noindex`; staging documentation is also `noindex`. These paths remain
 crawlable so search engines can read the indexing headers.
 
+Production routes use `doc*` and `robots.txt*` because Cloudflare matches query
+strings against route patterns. Requests outside the exact documentation
+namespaces, such as `/documentation.html`, pass through to the production
+origin without documentation indexing headers. Unknown staging paths return
+404, avoiding a fetch back into the custom-domain Worker. The `/doc` landing
+redirect uses `/documentation.html`, which exists before and after Astro.
+
 ## Local validation
 
 Run all Worker integration tests and compile the production configuration:

workers/docs/src/index.test.ts

@@ -317,15 +317,61 @@ describe("documentation worker", () => {
       expect(response.headers.get("location")).toBe(`${base}${path}/?view=1`);
     }
     expect((await request("/doc/6.4.0/missing-directory")).status).toBe(404);
-    expect((await request("/doc")).headers.get("location")).toBe(`${base}/documentation/`);
+    for (const path of ["/doc", "/doc/", "/doc?smoke=1", "/doc/?smoke=1"]) {
+      const response = await request(path);
+      expect(response.status).toBe(308);
+      expect(response.headers.get("location")).toBe(`${base}/documentation.html${path.includes("?") ? "?smoke=1" : ""}`);
+    }
+  });
+
+  it("passes neighboring website paths through without changing requests or responses", async () => {
+    const originFetch = vi.spyOn(globalThis, "fetch");
+    try {
+      for (const [path, method, status] of [
+        ["/documentation.html?smoke=1", "GET", 200],
+        ["/documentation/", "HEAD", 404],
+        ["/documents/submit?draft=1", "POST", 201],
+        ["/doc-latest-news.html", "GET", 200],
+        ["/robots.txt.bak?download=1", "GET", 404],
+      ] as const) {
+        const incoming = new Request(`${base}${path}`, {
+          method, headers: { "X-Request-Test": "preserved" },
+          body: method === "POST" ? "submission bytes" : undefined,
+        });
+        const upstream = new Response(method === "HEAD" ? null : "origin content", {
+          status,
+          headers: {
+            "Content-Type": "text/html",
+            "X-Robots-Tag": "index, follow",
+            Link: '<https://www.gecode.dev/documentation/>; rel="canonical"',
+          },
+        });
+        originFetch.mockResolvedValueOnce(upstream);
+        const context = createExecutionContext();
+        const response = await worker.fetch(incoming, env, context);
+        await waitOnExecutionContext(context);
+        expect(originFetch).toHaveBeenLastCalledWith(incoming);
+        expect(response).toBe(upstream);
+        expect(response.status).toBe(status);
+        expect(response.headers.get("x-robots-tag")).toBe("index, follow");
+        expect(response.headers.get("link")).toBe('<https://www.gecode.dev/documentation/>; rel="canonical"');
+        if (method === "POST") expect(await incoming.text()).toBe("submission bytes");
+      }
+      const staging = await request("https://docs-staging.gecode.dev/documentation.html");
+      expect(staging.status).toBe(404);
+      expect(staging.headers.get("x-robots-tag")).toBe("noindex");
+      expect(originFetch).toHaveBeenCalledTimes(5);
+    } finally {
+      originFetch.mockRestore();
+    }
   });
 
   it("returns explicit errors", async () => {
     expect((await request("/doc/6.4.0/missing.html")).status).toBe(404);
     const method = await request("/doc/6.4.0/index.html", { method: "POST" });
     expect(method.status).toBe(405);
     expect(method.headers.get("allow")).toBe("GET, HEAD");
-    expect((await request("/doc/%2e%2e/secret")).status).toBe(400);
+    expect((await request("/doc/6.4.0/%")).status).toBe(400);
     expect((await request("/doc/6.4.0/%252e%252e/secret")).status).toBe(400);
     expect((await request("/doc/6.4.0/reference%2fPageChange.html")).status).toBe(400);
   });

workers/docs/src/index.ts

@@ -185,7 +185,7 @@ async function serve(request: Request, env: Env, context: ExecutionContext): Pro
     destination.pathname = pathname;
     return Response.redirect(destination.href, 308);
   };
-  if (url.pathname === "/doc" || url.pathname === "/doc/") return redirect("/documentation/");
+  if (url.pathname === "/doc" || url.pathname === "/doc/") return redirect("/documentation.html");
   const resolved = resolvePath(url.pathname, env.LATEST_DOC_VERSION);
   if (!resolved) return errorResponse(400, "Invalid documentation path");
 
@@ -285,6 +285,18 @@ async function serve(request: Request, env: Env, context: ExecutionContext): Pro
 
 export default {
   async fetch(request: Request, env: Env, context: ExecutionContext): Promise<Response> {
+    const url = new URL(request.url);
+    const pathname = url.pathname;
+    const ownsPath = pathname === "/robots.txt"
+      || pathname === "/doc" || pathname.startsWith("/doc/")
+      || pathname === "/doc-latest" || pathname.startsWith("/doc-latest/");
+    // Wildcard routes also receive /documentation.html and similarly named
+    // website paths. Leave their origin response and indexing headers intact.
+    if (!ownsPath) {
+      if (url.hostname === "www.gecode.dev") return fetch(request);
+      return applyIndexingPolicy(request, errorResponse(404, "Page not found"), env);
+    }
+
     let response: Response;
     try {
       response = await serve(request, env, context);

workers/docs/wrangler.jsonc

@@ -51,23 +51,11 @@
       "observability": { "enabled": true },
       "routes": [
         {
-          "pattern": "www.gecode.dev/robots.txt",
+          "pattern": "www.gecode.dev/robots.txt*",
           "zone_name": "gecode.dev"
         },
         {
-          "pattern": "www.gecode.dev/doc",
-          "zone_name": "gecode.dev"
-        },
-        {
-          "pattern": "www.gecode.dev/doc/*",
-          "zone_name": "gecode.dev"
-        },
-        {
-          "pattern": "www.gecode.dev/doc-latest",
-          "zone_name": "gecode.dev"
-        },
-        {
-          "pattern": "www.gecode.dev/doc-latest/*",
+          "pattern": "www.gecode.dev/doc*",
           "zone_name": "gecode.dev"
         }
       ],