Remediate August SEO audit findings

.github/workflows/ci.yml

@@ -38,6 +38,11 @@ jobs:
       - name: Build
         run: pnpm build
 
+      - name: Check built SEO
+        run: pnpm check:seo
+        env:
+          CHECK_REMOTE_ASSETS: "1"
+
   vendor-guard:
     # vendor/ is written only by the sync-docs action. A human edit here is
     # silently overwritten on the next sync, so reject it at review time.

astro.config.ts

@@ -8,6 +8,7 @@ import sentry from "@sentry/astro";
 // generation, inlined from the former @rivet-dev/docs-theme package.
 import { docsPipeline } from './src/integrations/docs-pipeline';
 import { redirects } from './redirects.mjs';
+import { getRouteSeoPolicy } from './src/lib/routeSeoPolicy';
 
 // Wildcard sub-path redirects (`wildcardRedirects` in redirects.mjs) are applied
 // only at the Caddy layer in production. Astro's static output treats a redirect
@@ -44,19 +45,9 @@ export default defineConfig({
 			applyBaseStyles: false,
 		}),
 		sitemap({
-			// Cookbooks and comparison guides are intentionally hidden from the site
-			// and kept out of SEO, so exclude them from the sitemap.
-			//
-			// Self-host guides are generated for every product from one source and
-			// all canonicalize to the Actors variant, so only that variant belongs
-			// in the sitemap. Submitting the copies asks Google to index pages the
-			// canonical tag then excludes.
-			filter: (page) =>
-				!page.includes('/api/') &&
-				!page.includes('/internal/') &&
-				!page.includes('/cookbook') &&
-				!page.includes('/compare') &&
-				!/\/(?!actors\/)[^/]+\/self-host\//.test(page),
+			// The same policy emits page-level robots directives and decides sitemap
+			// membership, so a hidden route cannot accidentally be submitted.
+			filter: (page) => getRouteSeoPolicy(page).sitemap,
 		}),
 		sentry({
       		project: "website",

package.json

@@ -18,6 +18,7 @@
 		"gen:markdown": "tsx ./scripts/generate-markdown.ts",
 		"gen:skills": "tsx ./scripts/generate-skills.ts",
 		"check:sitemap": "tsx ./scripts/check-sitemap.ts",
+		"check:seo": "tsx ./scripts/check-seo.ts",
 		"assemble": "tsx ./scripts/assemble.ts",
 		"predev": "pnpm assemble"
 	},
@@ -82,6 +83,7 @@
 		"@types/escape-html": "^1.0.4",
 		"@types/lodash": "^4.17.21",
 		"eslint": "9.39.1",
+		"parse5": "^8.0.1",
 		"playwright": "^1.57.0",
 		"prettier": "^3.6.2",
 		"prettier-plugin-tailwindcss": "^0.7.1",

redirects.caddy

@@ -71,8 +71,6 @@ redir /docs/general/skill /actors/docs/general/skill/{http.request.uri.prefixed_
 redir /docs/general/skill/ /actors/docs/general/skill/{http.request.uri.prefixed_query} 301
 redir /docs/general/wasm-vs-native-sdk /actors/docs/general/wasm-vs-native-sdk/{http.request.uri.prefixed_query} 301
 redir /docs/general/wasm-vs-native-sdk/ /actors/docs/general/wasm-vs-native-sdk/{http.request.uri.prefixed_query} 301
-redir /docs/actors/helper-types /actors/docs/helper-types/{http.request.uri.prefixed_query} 301
-redir /docs/actors/helper-types/ /actors/docs/helper-types/{http.request.uri.prefixed_query} 301
 redir /docs/actors/http-api /actors/docs/http-api/{http.request.uri.prefixed_query} 301
 redir /docs/actors/http-api/ /actors/docs/http-api/{http.request.uri.prefixed_query} 301
 redir /docs/actors /actors/docs/{http.request.uri.prefixed_query} 301
@@ -109,12 +107,8 @@ redir /docs/actors/quickstart/supabase /actors/docs/quickstart/supabase/{http.re
 redir /docs/actors/quickstart/supabase/ /actors/docs/quickstart/supabase/{http.request.uri.prefixed_query} 301
 redir /docs/actors/request-handler /actors/docs/request-handler/{http.request.uri.prefixed_query} 301
 redir /docs/actors/request-handler/ /actors/docs/request-handler/{http.request.uri.prefixed_query} 301
-redir /docs/actors/scaling /actors/docs/scaling/{http.request.uri.prefixed_query} 301
-redir /docs/actors/scaling/ /actors/docs/scaling/{http.request.uri.prefixed_query} 301
 redir /docs/actors/schedule /actors/docs/schedule/{http.request.uri.prefixed_query} 301
 redir /docs/actors/schedule/ /actors/docs/schedule/{http.request.uri.prefixed_query} 301
-redir /docs/actors/sharing-and-joining-state /actors/docs/sharing-and-joining-state/{http.request.uri.prefixed_query} 301
-redir /docs/actors/sharing-and-joining-state/ /actors/docs/sharing-and-joining-state/{http.request.uri.prefixed_query} 301
 redir /docs/actors/sqlite-drizzle /actors/docs/sqlite-drizzle/{http.request.uri.prefixed_query} 301
 redir /docs/actors/sqlite-drizzle/ /actors/docs/sqlite-drizzle/{http.request.uri.prefixed_query} 301
 redir /docs/actors/sqlite /actors/docs/sqlite/{http.request.uri.prefixed_query} 301
@@ -239,6 +233,18 @@ redir /docs/actors/external-sql /actors/docs/state/{http.request.uri.prefixed_qu
 redir /docs/actors/external-sql/ /actors/docs/state/{http.request.uri.prefixed_query} 301
 redir /docs/actors/raw-sql /actors/docs/sqlite/{http.request.uri.prefixed_query} 301
 redir /docs/actors/raw-sql/ /actors/docs/sqlite/{http.request.uri.prefixed_query} 301
+redir /actors/docs/helper-types /actors/docs/types/{http.request.uri.prefixed_query} 301
+redir /actors/docs/helper-types/ /actors/docs/types/{http.request.uri.prefixed_query} 301
+redir /docs/actors/helper-types /actors/docs/types/{http.request.uri.prefixed_query} 301
+redir /docs/actors/helper-types/ /actors/docs/types/{http.request.uri.prefixed_query} 301
+redir /actors/docs/scaling /actors/docs/design-patterns/{http.request.uri.prefixed_query} 301
+redir /actors/docs/scaling/ /actors/docs/design-patterns/{http.request.uri.prefixed_query} 301
+redir /docs/actors/scaling /actors/docs/design-patterns/{http.request.uri.prefixed_query} 301
+redir /docs/actors/scaling/ /actors/docs/design-patterns/{http.request.uri.prefixed_query} 301
+redir /actors/docs/sharing-and-joining-state /actors/docs/design-patterns/{http.request.uri.prefixed_query} 301
+redir /actors/docs/sharing-and-joining-state/ /actors/docs/design-patterns/{http.request.uri.prefixed_query} 301
+redir /docs/actors/sharing-and-joining-state /actors/docs/design-patterns/{http.request.uri.prefixed_query} 301
+redir /docs/actors/sharing-and-joining-state/ /actors/docs/design-patterns/{http.request.uri.prefixed_query} 301
 redir /docs/actors/ephemeral-variables /actors/docs/state/{http.request.uri.prefixed_query} 301
 redir /docs/actors/ephemeral-variables/ /actors/docs/state/{http.request.uri.prefixed_query} 301
 redir /docs/actors/persistence /actors/docs/state/{http.request.uri.prefixed_query} 301
@@ -397,9 +403,27 @@ redir /agent-os/docs/apps /dynamic-apps/docs/{http.request.uri.prefixed_query} 3
 redir /agent-os/docs/apps/ /dynamic-apps/docs/{http.request.uri.prefixed_query} 301
 redir /sales /talk-to-an-engineer/{http.request.uri.prefixed_query} 301
 redir /sales/ /talk-to-an-engineer/{http.request.uri.prefixed_query} 301
-redir /registry/* /agentos{http.request.uri.path}{http.request.uri.prefixed_query} 301
-redir /typedoc/* /actors/docs{http.request.uri.prefixed_query} 301
-redir /learn/* /actors{http.request.uri.path}{http.request.uri.prefixed_query} 301
-redir /compare/* /actors{http.request.uri.path}{http.request.uri.prefixed_query} 301
-redir /docs/agent-os/* /agentos/docs{http.request.uri.prefixed_query} 301
-redir /agent-os/* /agentos{http.request.uri.prefixed_query} 301
+@wildcard0Trailing path /registry/*/
+redir @wildcard0Trailing /agentos{http.request.uri.path}{http.request.uri.prefixed_query} 301
+@wildcard0Bare {
+	path /registry/*
+	not path */
+}
+redir @wildcard0Bare /agentos{http.request.uri.path}/{http.request.uri.prefixed_query} 301
+redir /typedoc/* /actors/docs/{http.request.uri.prefixed_query} 301
+@wildcard2Trailing path /learn/*/
+redir @wildcard2Trailing /actors{http.request.uri.path}{http.request.uri.prefixed_query} 301
+@wildcard2Bare {
+	path /learn/*
+	not path */
+}
+redir @wildcard2Bare /actors{http.request.uri.path}/{http.request.uri.prefixed_query} 301
+@wildcard3Trailing path /compare/*/
+redir @wildcard3Trailing /actors{http.request.uri.path}{http.request.uri.prefixed_query} 301
+@wildcard3Bare {
+	path /compare/*
+	not path */
+}
+redir @wildcard3Bare /actors{http.request.uri.path}/{http.request.uri.prefixed_query} 301
+redir /docs/agent-os/* /agentos/docs/{http.request.uri.prefixed_query} 301
+redir /agent-os/* /agentos/{http.request.uri.prefixed_query} 301

redirects.mjs

@@ -55,6 +55,14 @@ const explicitRedirects = {
 	'/docs/actors/scheduling': '/actors/docs/schedule/',
 	'/docs/actors/external-sql': '/actors/docs/state/',
 	'/docs/actors/raw-sql': '/actors/docs/sqlite/',
+	// Empty moved-page stubs were retired. Preserve both their current product
+	// URLs and legacy flat-docs spellings as one-hop redirects.
+	'/actors/docs/helper-types': '/actors/docs/types/',
+	'/docs/actors/helper-types': '/actors/docs/types/',
+	'/actors/docs/scaling': '/actors/docs/design-patterns/',
+	'/docs/actors/scaling': '/actors/docs/design-patterns/',
+	'/actors/docs/sharing-and-joining-state': '/actors/docs/design-patterns/',
+	'/docs/actors/sharing-and-joining-state': '/actors/docs/design-patterns/',
 	'/docs/actors/ephemeral-variables': '/actors/docs/state/',
 	'/docs/actors/persistence': '/actors/docs/state/',
 	'/docs/actors/postgres': '/actors/docs/state/',