diff --git a/package.json b/package.json index bb55ef1..a6813a8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "p0dcasters", - "version": "0.6.2", + "version": "0.6.3", "private": true, "type": "module", "scripts": { diff --git a/src/lib/crawl-gateway.ts b/src/lib/crawl-gateway.ts index f4ffd8e..daf0ef7 100644 --- a/src/lib/crawl-gateway.ts +++ b/src/lib/crawl-gateway.ts @@ -73,7 +73,14 @@ export const gateway = createGateway({ contact: "mailto:hello@p0dcasters.com", // A refused crawler may still read the map of the site, so the refusal is // legible to it rather than a wall. robots.txt and /crawl are always open. - openPaths: ["/llms.txt", "/skill.md"], + // + // /api/mcp is how an agent uses the directory, not how it copies it, so no + // user-agent list or spoofed-browser check ever answers it 402. Connector + // clients (claude.ai's custom-connector verifier among them) read any non-2xx + // without WWW-Authenticate as "needs sign-in", find no OAuth metadata and + // fail with "couldn't register with the sign-in service" (2026-11-11). The + // endpoint keeps its own per-caller budget in lib/throttle.ts. + openPaths: ["/llms.txt", "/skill.md", "/api/mcp"], training: TRAINING, retrieval: RETRIEVAL, denyCidrs: OVH_VPS_RANGES, diff --git a/test/crawl-gateway.test.mjs b/test/crawl-gateway.test.mjs index 58d435f..c5e995a 100644 --- a/test/crawl-gateway.test.mjs +++ b/test/crawl-gateway.test.mjs @@ -33,3 +33,21 @@ test('Lightpanda may still read robots.txt', async () => { const res = await gate(req('/robots.txt', { 'user-agent': 'Lightpanda/1.0' })); assert.notEqual(res?.status, 402); }); + +test('the MCP endpoint is never sold, whatever the client calls itself', async () => { + for (const ua of ['anthropic-ai/1.0', 'ClaudeBot/1.0', CHROME, 'Lightpanda/1.0']) { + const res = await gate( + new Request('https://p0dcasters.com/api/mcp', { + method: 'POST', + headers: { 'x-forwarded-for': '37.231.157.22', 'user-agent': ua, 'content-type': 'application/json' }, + body: '{}', + }), + ); + assert.equal(res ?? undefined, undefined, ua); + } +}); + +test('OAuth discovery under /.well-known is never sold', async () => { + const res = await gate(req('/.well-known/oauth-protected-resource', { 'user-agent': 'anthropic-ai/1.0' })); + assert.notEqual(res?.status, 402); +});