From 1b680ed7829e8007d89ae3fcb7007420d46f1d31 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 11 Nov 2026 12:41:49 +0000 Subject: [PATCH] fix(stats.js): serve the WebMCP plugin same-origin; tracker scripts skip the crawl gate - /npm/@profullstack/webmcp@0/src/.js mirrors jsDelivr @0 (allowlist, 15 min cache, CORS *). 12 fleet sites load stats.js but their CSP refuses cdn.jsdelivr.net; all of them allow crawlproof.com. - /stats.js, /careers.js, the mirror and /api/tracker/v1/webmcp bypass the crawl gateway: public static embeds that cloud-hosted agent browsers (the visitors WebMCP is for) were getting 402s on. Co-Authored-By: Claude Opus 5.5 --- app/npm/[...path]/route.ts | 35 +++++++++++++++++++++++++++++++++ app/stats.js/route.ts | 9 ++++++--- lib/crawl-policy.ts | 11 +++++++++++ proxy.ts | 5 ++++- tests/contract/stats-js.test.ts | 4 +++- 5 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 app/npm/[...path]/route.ts diff --git a/app/npm/[...path]/route.ts b/app/npm/[...path]/route.ts new file mode 100644 index 00000000..974fd218 --- /dev/null +++ b/app/npm/[...path]/route.ts @@ -0,0 +1,35 @@ +// Same-origin mirror of @profullstack/webmcp for the stats.js WebMCP plugin. +// GET /npm/@profullstack/webmcp@0/src/.js +// +// Sites that run stats.js allow crawlproof.com in their CSP but often not +// cdn.jsdelivr.net, so the plugin loads the embed from here. Only the +// package's own modules are served, fetched from jsDelivr's @0 alias so a +// new 0.x release reaches every site without a crawlproof deploy. The embed +// imports its siblings relatively, so they resolve back to this route. + +import { NextRequest, NextResponse } from "next/server"; + +export const runtime = "nodejs"; + +const MODULES = new Set(["embed.js", "core.js", "site.js", "forms.js", "mcp-bridge.js", "index.js"]); +const PREFIX = ["@profullstack", "webmcp@0", "src"]; +const UPSTREAM = "https://cdn.jsdelivr.net/npm/@profullstack/webmcp@0/src/"; + +export async function GET(_request: NextRequest, { params }: { params: Promise<{ path: string[] }> }) { + const { path } = await params; + const file = path?.[3]; + if (!path || path.length !== 4 || PREFIX.some((p, i) => path[i] !== p) || !file || !MODULES.has(file)) { + return new NextResponse("not found", { status: 404 }); + } + const upstream = await fetch(UPSTREAM + file, { next: { revalidate: 900 } }).catch(() => null); + if (!upstream?.ok) return new NextResponse("upstream unavailable", { status: 502 }); + return new NextResponse(await upstream.text(), { + headers: { + "content-type": "application/javascript; charset=utf-8", + // Module scripts are fetched in CORS mode from the customer's origin. + "access-control-allow-origin": "*", + "cache-control": "public, max-age=900, s-maxage=900", + "x-webmcp-version": upstream.headers.get("x-jsd-version") ?? "", + }, + }); +} diff --git a/app/stats.js/route.ts b/app/stats.js/route.ts index 4b08c099..f72e0389 100644 --- a/app/stats.js/route.ts +++ b/app/stats.js/route.ts @@ -7,9 +7,12 @@ import { env } from "@/lib/env"; import { VISITOR_SNIPPET } from "@/lib/tracker/visitorSnippet"; -// The opt-in WebMCP plugin (data-webmcp) loads this. @0 follows every 0.x -// release of github.com/profullstack/webmcp, the same way sites tag it. -const WEBMCP_EMBED = "https://cdn.jsdelivr.net/npm/@profullstack/webmcp@0/src/embed.js"; +// The WebMCP plugin loads this. It is served from crawlproof.com itself +// (app/npm/[...path], proxying jsDelivr @0) because every site that runs +// stats.js already allows crawlproof.com in its CSP, while many do not allow +// cdn.jsdelivr.net. The path keeps "@profullstack/webmcp" so the load-once +// check below still recognises it. +const WEBMCP_EMBED = `${env.siteUrl}/npm/@profullstack/webmcp@0/src/embed.js`; const snippet = `(function(){ try { diff --git a/lib/crawl-policy.ts b/lib/crawl-policy.ts index a6da3efa..2931805b 100644 --- a/lib/crawl-policy.ts +++ b/lib/crawl-policy.ts @@ -23,3 +23,14 @@ export const isAdClickPath = (path: string) => path.startsWith("/a/") || path == /** Email tracking endpoints: /t//(o.png|c|u) and /api/v1/tracking//events. */ export const isEmailTrackingPath = (path: string) => path.startsWith("/t/") || path.startsWith("/api/v1/tracking/"); +/** + * The tracker's own embeddable scripts and the WebMCP plugin they load: + * /stats.js, /careers.js, /npm/@profullstack/webmcp@…, /api/tracker/v1/webmcp. + * Static, public, and fetched by customers' pages (including cloud-hosted + * agent browsers, the visitors WebMCP is for), so never the crawl gateway's 402. + */ +export const isTrackerAssetPath = (path: string) => + path === "/stats.js" || + path === "/careers.js" || + path.startsWith("/npm/@profullstack/webmcp@") || + path === "/api/tracker/v1/webmcp"; diff --git a/proxy.ts b/proxy.ts index 2446ba80..b6069aba 100644 --- a/proxy.ts +++ b/proxy.ts @@ -1,5 +1,5 @@ import { gate } from "@/lib/crawl-gateway"; -import { isAdClickPath, isEmailTrackingPath } from "@/lib/crawl-policy"; +import { isAdClickPath, isEmailTrackingPath, isTrackerAssetPath } from "@/lib/crawl-policy"; import { NextResponse, type NextRequest } from "next/server"; import { createServerClient, type CookieOptions } from "@supabase/ssr"; import { trackReferralCode } from "@profullstack/stack/referrals"; @@ -17,6 +17,9 @@ export async function proxy(request: NextRequest) { // proxies, link scanners and the sender's CLI. None of them should meet the // crawl gateway's 402, and none of them carry a session worth refreshing. if (isEmailTrackingPath(request.nextUrl.pathname)) return NextResponse.next(); + // Tracker scripts embedded on customers' pages (stats.js, careers.js, the + // WebMCP plugin): public static assets, no session, never a 402. + if (isTrackerAssetPath(request.nextUrl.pathname)) return NextResponse.next(); // Crawl gateway first: AI training crawlers get 402 Payment Required (or the // sales page at /crawl) unless they present a paid pass. People, Googlebot // and retrieval crawlers fall through to everything below. diff --git a/tests/contract/stats-js.test.ts b/tests/contract/stats-js.test.ts index 75759686..8e12e608 100644 --- a/tests/contract/stats-js.test.ts +++ b/tests/contract/stats-js.test.ts @@ -53,7 +53,9 @@ describe("/stats.js", () => { // Dashboard opt-in is only asked for when the tag says nothing. expect(script).toContain("/api/tracker/v1/webmcp?site="); expect(script).toContain("window.__crawlproofWebMCP"); - expect(script).toContain("https://cdn.jsdelivr.net/npm/@profullstack/webmcp@0/src/embed.js"); + // Same-origin mirror: sites already allow crawlproof.com in their CSP. + expect(script).toContain("/npm/@profullstack/webmcp@0/src/embed.js"); + expect(script).not.toContain("cdn.jsdelivr.net"); expect(script).toContain("ws.type = 'module'"); // Never loaded twice when the page carries its own embed. expect(script).toContain("window.ProfullstackWebMCP");