fix(game): keep the API, the service worker and the offline page out of search results

apps/backend/src/http/api.ts

@@ -5,14 +5,16 @@ import { errorCatcher } from "@/http/middlewares/error-catcher"
 import { requestLog } from "@/http/middlewares/request-log"
 import { buildShaPlugin, versionRoute } from "@/http/plugins/build-sha"
 import { corsPlugin } from "@/http/plugins/cors"
+import { noIndexPlugin } from "@/http/plugins/no-index"
 import { v1Routes } from "@/http/routes"
 
-//composed transport api: build sha, cors, global error catcher, health,
+//composed transport api: build sha, noindex, cors, global error catcher, health,
 //version, v1 routes, not-found. the worker entrypoint just bootstraps env and
 //delegates to api.fetch.
 export const api = newEndpoint<Env>()
   .use("*", requestLog()) //first: request id + one request log line
   .use("*", buildShaPlugin()) //x-build-sha on every response
+  .use("*", noIndexPlugin()) //x-robots-tag: noindex on every response
   .use("*", corsPlugin()) //setup cors plugin
   .onError(errorCatcher) //catch + envelope unhandled throws
 

apps/backend/src/http/plugins/no-index.test.ts

@@ -0,0 +1,35 @@
+import { Hono } from "hono"
+import { describe, expect, it } from "vitest"
+import { noIndexPlugin } from "@/http/plugins/no-index"
+
+function app() {
+  const server = new Hono()
+  server.use("*", noIndexPlugin())
+  server.get("/", (c) => c.json({ ok: true }))
+  server.get("/boom", () => {
+    throw new Error("boom")
+  })
+  //a fetch() response has immutable headers (Response.redirect does too)
+  server.get("/proxied", () =>
+    Response.redirect("https://example.com/", 302),
+  )
+  server.onError((_error, c) => c.json({ error: true }, 500))
+  return server
+}
+
+describe("no index", () => {
+  it("marks every response noindex, errors and not-found included", async () => {
+    for (const path of ["/", "/boom", "/missing"]) {
+      const response = await app().request(path)
+
+      expect(response.headers.get("x-robots-tag"), path).toBe("noindex")
+    }
+  })
+
+  it("keeps the status of a response with immutable headers", async () => {
+    const response = await app().request("/proxied")
+
+    expect(response.status).toBe(302)
+    expect(response.headers.get("x-robots-tag")).toBe("noindex")
+  })
+})

apps/backend/src/http/plugins/no-index.ts

@@ -0,0 +1,14 @@
+import type { MiddlewareHandler } from "hono"
+
+//x-robots-tag: noindex on every response, errors included. the api answers
+//json to machines and none of it belongs in a search result; robots.txt must
+//not block /api instead, or a crawler that already found a link never reads
+//this header. c.header() re-wraps a finalized response, as in the build-sha
+//plugin; a websocket upgrade passes through untouched, for the same reason
+export function noIndexPlugin(): MiddlewareHandler {
+  return async (c, next) => {
+    await next()
+    if (c.req.header("upgrade")?.toLowerCase() === "websocket") return
+    c.header("x-robots-tag", "noindex")
+  }
+}

apps/game/public/_headers

@@ -19,3 +19,12 @@
 # this header.
 /_shell
   X-Robots-Tag: noindex
+
+# The service worker script and the offline fallback page are for the browser
+# to fetch, not for people to land on. Both answer 200, so a crawler that
+# follows the registration can list them. Same rule as /_shell: robots.txt must
+# not block them, or the crawler never reads this header.
+/sw.js
+  X-Robots-Tag: noindex
+/_pwa/*
+  X-Robots-Tag: noindex

apps/game/public/_pwa/offline-fallback.html

@@ -10,6 +10,7 @@
     <meta name="apple-mobile-web-app-capable" content="yes" />
     <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent" />
     <meta name="mobile-web-app-capable" content="yes" />
+    <meta name="robots" content="noindex" />
     <title>Offline — Abalone</title>
     <link rel="shortcut icon" href="/favicons/favicon.ico" type="image/x-icon" />
     <style>

apps/game/src/server/security-headers.test.ts

@@ -125,4 +125,10 @@ describe("public/_headers", () => {
     })
     expect(CACHE_HASHED_ASSET).toBe("public, max-age=31536000, immutable")
   })
+
+  it("keeps the shell, the service worker and the offline page out of search", () => {
+    for (const path of ["/_shell", "/sw.js", "/_pwa/*"]) {
+      expect(rules[path], path).toEqual({ "X-Robots-Tag": "noindex" })
+    }
+  })
 })

e2e/seo.spec.ts

@@ -150,6 +150,17 @@ test("/_shell is served with noindex", async ({ request }) => {
   expect(response.headers()["x-robots-tag"]).toBe("noindex")
 })
 
+// Fetched by the browser, never by a visitor, yet each answers 200 and a crawler
+// that follows the service worker registration can list them
+for (const path of ["/sw.js", "/_pwa/offline-fallback"]) {
+  test(`${path} is served with noindex`, async ({ request }) => {
+    const response = await request.get(path)
+
+    expect(response.status()).toBe(200)
+    expect(response.headers()["x-robots-tag"]).toBe("noindex")
+  })
+}
+
 // Crawlers ask the root for it whatever the head says; a 404 there leaves a
 // search result with a generic globe
 test("/favicon.ico is served at the root, and the home head links an icon", async ({