diff --git a/README.md b/README.md index c8e5ab2..fb831d1 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,7 @@ anything is sent, created, or deleted. Claude Code and Copilot CLI include telemetry adapters, but **`TELEMETRY_ENABLED` is `false`**: this package sends no usage events and generates no telemetry hooks in client manifests. [docs/telemetry.md](docs/telemetry.md) describes -the event contract, what would be stored locally when enabled, opt-outs, and +the event contract, what would be stored locally when enabled, the opt-out, and the separate approval required before collection can be enabled. The Arcade MCP gateway remains the canonical source for request and tool-call telemetry. Adapter behavior is covered by in-process tests; @@ -203,7 +203,7 @@ Configure paths in `release-please-config.json`. The workflow lives at Each release also gets `arcade-chatgpt-{VERSION}.zip`, the package to upload in OpenAI's plugin portal for the ChatGPT and Codex directory. It holds the portable plugin only (`plugin.json` at the ZIP root, `mcp.json`, `skills/`, -the logo, and `LICENSE`), with no hooks or other clients' folders. Build it +the listing images, and `LICENSE`), with no hooks or other clients' folders. Build it locally with `npm run package:chatgpt` in `dev/` (writes `dist/` at the repo root); the build fails if the `extensions.com.openai.interface` listing breaks the portal's limits, such as the 30-character display name and subtitle diff --git a/dev/scripts/generate-manifests.mjs b/dev/scripts/generate-manifests.mjs index 878cc7a..13daf71 100644 --- a/dev/scripts/generate-manifests.mjs +++ b/dev/scripts/generate-manifests.mjs @@ -24,10 +24,20 @@ const inPlugin = (file) => `${PLUGIN_DIR}/${file}`; // One policy URL for both clients. The other OpenAI directory fields below are // written into plugin/plugin.json from here too. Identity text (name, descriptions, -// category, website, logo) stays in plugin/plugin.json. +// category, website) stays in plugin/plugin.json. Listing images and brandColor +// are written from the constants below. Cursor's manifest takes only `logo` +// (without the "./" prefix). Claude has no logo field. Dark variants and +// brandColor stay on the OpenAI listing; the other clients' schemas have no +// fields for them. const privacyPolicyUrl = "https://www.arcade.dev/privacy-policy/"; const openaiSupportUrl = "https://support.arcade.dev"; const openaiTermsUrl = "https://www.arcade.dev/terms-of-service/"; +const OPENAI_LOGO = "./assets/logo.png"; +const OPENAI_LOGO_DARK = "./assets/logo-dark.png"; +const OPENAI_COMPOSER_ICON = "./assets/composer-icon.png"; +const OPENAI_COMPOSER_ICON_DARK = "./assets/composer-icon-dark.png"; +const OPENAI_BRAND_COLOR = "#4708F9"; +const OPENAI_IMAGE_KEYS = ["logo", "logoDark", "composerIcon", "composerIconDark", "brandColor", "brandColorDark"]; const OPENAI_CAPABILITIES = [ "Read and search email", @@ -43,7 +53,13 @@ const OPENAI_DEFAULT_PROMPTS = [ "Which GitHub issues are assigned to me?", ]; -// Tool names are the MCP forms of Arcade toolkit tools (dots become underscores). +// Reviewers sign up for their own free Arcade account. Positive prompts call +// Arcade_SelectTools, then Arcade_UseTool for the toolkit tool named in +// parentheses (Toolkit.ToolName). Flights, Finance, and YouTube need no user +// OAuth; fares, prices, and video lists change, so expected_behavior checks +// the shape of the result. The Math average is the one fixed number. Negative +// prompts do not mention Arcade, so the app should not be called. Positive +// descriptions stay within OpenAI's 4000-character limit. // demo_recording_url is a real review field and is required before MCP review, // but a ZIP may omit it. No recording URL is published here; omitting the field // leaves any value already saved in the dashboard. @@ -51,48 +67,48 @@ const OPENAI_REVIEW = { test_cases: { positive: [ { - description: "Summarize unread email in the connected Gmail account", - prompt: "Summarize my unread email from the last 7 days.", - tools_triggered: "Gmail_ListEmails", - expected_behavior: "Lists the seeded unread test emails (sender, subject, one-line summary each) from the reviewer Gmail account. No email is changed or marked read.", + description: "Find trip-planning tools without running them", + prompt: "Use Arcade to find tools that could help me plan a weekend trip to Chicago: flights, hotels and driving directions. Just list what's available, don't run anything yet.", + tools_triggered: "Arcade_SelectTools", + expected_behavior: "Lists tools for flights, hotels, and driving directions, with a short description of each, and runs none of them. Returns no account names, emails, or connection status. Which tools are offered can vary with the catalog.", }, { - description: "Read tomorrow's calendar", - prompt: "What's on my calendar tomorrow?", - tools_triggered: "GoogleCalendar_ListEvents", - expected_behavior: "Returns the seeded recurring test events for tomorrow with times and titles from the reviewer Google Calendar. Nothing is created or changed.", + description: "Find a one-way fare and total it for 3 travelers", + prompt: "Use Arcade to find the cheapest one-way economy flight from SFO to JFK two weeks from today, then use Arcade's Math tools to calculate the total fare for 3 travelers.", + tools_triggered: "Arcade_SelectTools, Arcade_UseTool (GoogleFlights.SearchFlights), Arcade_UseTool (Math.Multiply)", + expected_behavior: "Reports the cheapest one-way economy fare it finds from SFO to JFK about two weeks out, with airline, flight number, and times, then the total for 3 travelers. Fares change; the total should equal the reported fare times 3. Nothing is booked.", }, { - description: "Find GitHub issues assigned to the user", - prompt: "Which open GitHub issues are assigned to me in arcade-review/sample-tracker?", - tools_triggered: "Github_ListIssues", - expected_behavior: "Lists the seeded open issues assigned to the reviewer GitHub account in the sample repo, with titles and links.", + description: "Get the Google Finance summary for NVDA", + prompt: "Use Arcade to get the Google Finance stock summary for NVDA.", + tools_triggered: "Arcade_SelectTools, Arcade_UseTool (GoogleFinance.GetStockSummary)", + expected_behavior: "Returns the Google Finance summary for NVDA, including NVIDIA's latest price in USD and the day's move in value and percent. Prices change; check the ticker and those fields. No order is placed.", }, { - description: "Read recent messages in a Slack channel", - prompt: "What did people say in #launch-plan on Slack this week?", - tools_triggered: "Slack_GetMessages", - expected_behavior: "Summarizes the seeded messages in #launch-plan in the review Slack workspace and names who said what. Nothing is posted.", + description: "Search YouTube for videos about MCP", + prompt: "Use Arcade to search YouTube for videos explaining the Model Context Protocol.", + tools_triggered: "Arcade_SelectTools, Arcade_UseTool (Youtube.SearchForVideos)", + expected_behavior: "Returns YouTube videos about the Model Context Protocol, each with a title, channel, and link. The list changes; check that the videos are about MCP.", }, { - description: "Draft an email and stop for confirmation", - prompt: "Draft a reply to the 'Q4 venue options' email saying Thursday works, but don't send it.", - tools_triggered: "Gmail_ListEmails, Gmail_WriteDraftReplyEmail", - expected_behavior: "Shows the draft text and recipient, asks before saving, then saves it as a Gmail draft. The email is not sent.", + description: "Average a list of amounts", + prompt: "Use Arcade to average my monthly software bills: $1,249.50, $389.99, $72.25 and $418.26.", + tools_triggered: "Arcade_SelectTools, Arcade_UseTool (Math.Avg)", + expected_behavior: "Answers $532.50.", }, ], negative: [ { - description: "General knowledge question that needs no connected app", + description: "General knowledge question that needs no tool", prompt: "What's the difference between TCP and UDP?", }, { - description: "Request outside any connected app (payments)", + description: "Payment request; Arcade connects no payment app and does not move money", prompt: "Send $50 to my landlord on Venmo.", }, { - description: "Irreversible bulk action without a confirmed target", - prompt: "Delete every email in my inbox.", + description: "Concept question about a stock ratio, with no quote to fetch", + prompt: "Explain what a P/E ratio means when evaluating a stock.", }, ], }, @@ -117,11 +133,9 @@ const applyOpenAiDirectory = (openai) => { developerName, category, websiteURL, - logo, - composerIcon, ...rest } = openai.interface; - for (const key of GENERATED_INTERFACE_KEYS) delete rest[key]; + for (const key of [...GENERATED_INTERFACE_KEYS, ...OPENAI_IMAGE_KEYS]) delete rest[key]; openai.interface = { displayName, shortDescription, @@ -134,8 +148,11 @@ const applyOpenAiDirectory = (openai) => { privacyPolicyURL: privacyPolicyUrl, termsOfServiceURL: openaiTermsUrl, defaultPrompt: OPENAI_DEFAULT_PROMPTS, - logo, - composerIcon, + logo: OPENAI_LOGO, + logoDark: OPENAI_LOGO_DARK, + composerIcon: OPENAI_COMPOSER_ICON, + composerIconDark: OPENAI_COMPOSER_ICON_DARK, + brandColor: OPENAI_BRAND_COLOR, ...rest, }; openai.review = OPENAI_REVIEW; @@ -182,7 +199,7 @@ export const COPIED_FILES = { * entry here — buildFiles throws if one is missing. */ export const FILE_SOURCES = { - // extensions.com.openai listing URLs, capabilities, defaultPrompt, review, and publication. + // extensions.com.openai listing URLs, images, brandColor, capabilities, defaultPrompt, review, and publication. [PLUGIN_JSON]: ["dev/scripts/generate-manifests.mjs"], // plugin.json and mcp.json supply identity and the gateway URL; VERSION // supplies the version; hook-hosts.mjs supplies the hooks manifest path; the @@ -343,18 +360,15 @@ const buildFiles = (root) => { throw new Error(`plugin.json version ${plugin.version} does not match VERSION ${version}`); } const { url } = readJson(root, MCP_JSON).mcpServers.arcade; - const { interface: openaiListing } = plugin.extensions["com.openai"]; // The ChatGPT directory already lists a plugin named "Arcade" from another // company, so the OpenAI listing uses "Arcade.dev". The other clients keep // "Arcade". const displayName = "Arcade"; - if (!openaiListing.logo) { - throw new Error("plugin.json extensions.com.openai.interface.logo is missing"); - } - // Codex wants the logo path with a leading "./"; Cursor takes it without one. - // The Claude marketplace has no logo field. `logo` is relative to the plugin - // directory; the repo-root Cursor manifest prefixes it with plugin/. - const logo = openaiListing.logo.replace(/^\.\//, ""); + // Codex and ChatGPT want the logo path with a leading "./" (OPENAI_LOGO). + // Cursor takes it without one. The Claude marketplace has no logo field. + // `logo` is relative to the plugin directory; the repo-root Cursor manifest + // prefixes it with plugin/. Dark icons are not copied onto Cursor or Claude. + const logo = OPENAI_LOGO.replace(/^\.\//, ""); const { name, description, author, homepage, license, keywords, repository } = plugin; const identity = { name, description, author, homepage, license, keywords, version }; const owner = { ...author, email: ACCEPTED_LISTING_OWNER_EMAIL }; diff --git a/dev/scripts/package-chatgpt.mjs b/dev/scripts/package-chatgpt.mjs index d8fb7e6..61da9ab 100644 --- a/dev/scripts/package-chatgpt.mjs +++ b/dev/scripts/package-chatgpt.mjs @@ -45,6 +45,21 @@ export const MAX_DEFAULT_PROMPT_LENGTH = 128; const isMultiline = (value) => /[\r\n\u2028\u2029]/.test(value); +// sRGB relative luminance. OpenAI requires brandColor contrast of at least 2:1 +// against white (https://developers.openai.com/plugins/deploy/submission). +const linearChannel = (byte) => { + const s = byte / 255; + return s <= 0.04045 ? s / 12.92 : ((s + 0.055) / 1.055) ** 2.4; +}; + +const contrastAgainstWhite = (hex) => { + const n = Number.parseInt(hex.slice(1), 16); + const luminance = 0.2126 * linearChannel((n >> 16) & 255) + + 0.7152 * linearChannel((n >> 8) & 255) + + 0.0722 * linearChannel(n & 255); + return 1.05 / (luminance + 0.05); +}; + /** Returns every problem the portal would reject in the OpenAI listing. */ export const listingProblems = (listing) => { const problems = []; @@ -61,6 +76,18 @@ export const listingProblems = (listing) => { problems.push(`category must be one of: ${LISTING_CATEGORIES.join(", ")}`); } if (!listing.logo) problems.push("logo is required"); + if (listing.brandColor !== undefined && !/^#[0-9A-F]{6}$/i.test(listing.brandColor)) { + problems.push("brandColor must be a #RRGGBB color"); + } else if (typeof listing.brandColor === "string" && contrastAgainstWhite(listing.brandColor) < 2) { + problems.push("brandColor must have at least 2:1 contrast against white"); + } + for (const key of ["logo", "logoDark", "composerIcon", "composerIconDark"]) { + const value = listing[key]; + if (value === undefined) continue; + if (typeof value !== "string" || !value.startsWith("./") || value.split("/").includes("..")) { + problems.push(`${key} must be a relative path starting with ./`); + } + } // A remote MCP submission ("With MCP") needs all four listing URLs; only a // skills-only upload can omit them. This ZIP always ships mcp.json. for (const key of LISTING_URLS) { @@ -114,8 +141,11 @@ const listFiles = (root, dir) => export const packageFiles = (root = ROOT) => { const pluginRoot = pluginRootOf(root); const plugin = JSON.parse(readFileSync(join(pluginRoot, "plugin.json"), "utf8")); - const { logo, composerIcon } = plugin.extensions["com.openai"].interface; - const images = [logo, composerIcon].filter(Boolean).map((path) => path.replace(/^\.\//, "")); + const listing = plugin.extensions["com.openai"].interface; + const images = ["logo", "logoDark", "composerIcon", "composerIconDark"] + .map((key) => listing[key]) + .filter(Boolean) + .map((path) => path.replace(/^\.\//, "")); return [...new Set(["plugin.json", "mcp.json", "LICENSE", ...listFiles(pluginRoot, "skills"), ...images])]; }; diff --git a/dev/test/check.test.mjs b/dev/test/check.test.mjs index 22439fe..fb86d31 100644 --- a/dev/test/check.test.mjs +++ b/dev/test/check.test.mjs @@ -20,12 +20,13 @@ test("plugin.json and mcp.json match the vendored Agent Plugins 1.0 schemas", () } const openaiListing = plugin.extensions?.["com.openai"]?.interface; assert.ok(openaiListing?.displayName, "plugin.json: Codex displayName"); - for (const key of ["logo", "composerIcon"]) { + for (const key of ["logo", "logoDark", "composerIcon", "composerIconDark"]) { const file = openaiListing?.[key]; assert.ok(file && existsSync(path.join(ROOT, "plugin", file)), `plugin.json: Codex ${key} must point at a committed file`); // Codex's own plugins all write these paths with a leading "./". assert.match(file, /^\.\//, `plugin.json: Codex ${key} must start with ./`); } + assert.match(openaiListing.brandColor, /^#[0-9A-F]{6}$/i); }); test("hand-written files name the gateway from mcp.json", () => { diff --git a/dev/test/directory-policy.test.mjs b/dev/test/directory-policy.test.mjs index 8e19180..af35a95 100644 --- a/dev/test/directory-policy.test.mjs +++ b/dev/test/directory-policy.test.mjs @@ -55,7 +55,9 @@ const COMPONENT_FOLDERS = ["agents/", "commands/", "skills/"]; // listing logo. const ALLOWED_MEDIA_PATH_FIELDS = [ { file: "plugin.json", field: ["extensions", "com.openai", "interface", "logo"] }, + { file: "plugin.json", field: ["extensions", "com.openai", "interface", "logoDark"] }, { file: "plugin.json", field: ["extensions", "com.openai", "interface", "composerIcon"] }, + { file: "plugin.json", field: ["extensions", "com.openai", "interface", "composerIconDark"] }, { file: ".cursor-plugin/plugin.json", field: ["logo"] }, ]; @@ -449,25 +451,87 @@ test("hook commands use ${CLAUDE_PLUGIN_ROOT} paths and run no package launcher ); }); +// The only process environment reads allowed in plugin scripts: the one +// telemetry opt-out switch and the client data-folder variables, each read by +// name inside readOptOutEnv in this one file. +const ENV_READ_FILE = "hooks/telemetry-run.mjs"; +const ENV_READ_ALLOWLIST = [ + "ARCADE_PLUGIN_TELEMETRY", + "CLAUDE_PLUGIN_DATA", + "COPILOT_PLUGIN_DATA", +]; +const CHILD_PROCESS = /\bchild_process\b|\b(?:spawn|spawnSync|exec|execSync|execFile|execFileSync|fork)\(/; +const OTHER_RUNTIME_ENV = /\bDeno\.env\b|\bBun\.env\b|\bimport\.meta\.env\b/; +// Indirect ways to reach the environment object: process["env"], process?.env, +// or destructuring `env` out of process. +const INDIRECT_ENV = /\bprocess\s*(?:\?\.)?\s*\[|\bprocess\s*\?\.\s*env\b|\{[^}]*\benv\b[^}]*\}\s*=\s*(?:globalThis\.)?process\b/; + +/** + * Lists the lines of one plugin script that download code, start a child + * process, or read the process environment outside the allowlist. + * @param {string} file Path relative to plugin/. + * @param {string} text + */ +const scriptOffenders = (file, text) => { + const offenders = []; + text.split("\n").forEach((line, index) => { + const at = `${file}:${index + 1}`; + if (/\bfetch\(/.test(line)) offenders.push(`${at}: fetch(`); + const child = line.match(CHILD_PROCESS); + if (child) offenders.push(`${at}: ${child[0]}`); + const other = line.match(OTHER_RUNTIME_ENV); + if (other) offenders.push(`${at}: ${other[0]}`); + if (INDIRECT_ENV.test(line)) offenders.push(`${at}: indirect process.env access`); + for (const match of line.matchAll(/\bprocess\s*\.\s*env\b(?:\s*\.\s*([A-Za-z_$][\w$]*))?/g)) { + const name = match[1]; + const allowed = file === ENV_READ_FILE && name !== undefined && ENV_READ_ALLOWLIST.includes(name); + if (!allowed) offenders.push(`${at}: ${match[0]}`); + } + }); + return offenders; +}; + // Scripts in the plugin directory. dev/ sits outside it. -test("no script downloads code or gives a child process the whole environment", () => { +test("no script downloads code, starts a child process, or reads the environment outside the opt-out allowlist", () => { const offenders = []; for (const file of REGULAR_FILES) { if (!/\.(?:mjs|cjs|js|ts|sh)$/.test(file)) continue; - read(file) - .toString("utf8") - .split("\n") - .forEach((line, index) => { - if (/\bfetch\(/.test(line)) offenders.push(`${file}:${index + 1}: fetch(`); - if (/\.\.\.process\.env\b/.test(line)) offenders.push(`${file}:${index + 1}: ...process.env`); - }); + offenders.push(...scriptOffenders(file, read(file).toString("utf8"))); } assertNone( offenders, - "Keep network downloads out of plugin scripts, and pass child processes only the variables they need instead of ...process.env.", + `Keep network downloads and child processes out of plugin scripts. Read the process environment only by name, for ${ENV_READ_ALLOWLIST.join(", ")}, inside readOptOutEnv in ${ENV_READ_FILE}.`, ); }); +test("the environment check catches disallowed reads and child processes", () => { + const disallowed = [ + [ENV_READ_FILE, "const { spawn } = await import(\"node:child_process\");"], + [ENV_READ_FILE, "spawn(process.execPath, [SENDER]);"], + [ENV_READ_FILE, "fork(\"./send.mjs\");"], + [ENV_READ_FILE, "const env = process.env;"], + [ENV_READ_FILE, "const childEnv = { ...process.env };"], + [ENV_READ_FILE, "Object.keys(process.env);"], + [ENV_READ_FILE, "Object.entries(process.env).forEach(send);"], + [ENV_READ_FILE, "const token = process.env.GITHUB_TOKEN;"], + [ENV_READ_FILE, "const home = process.env[\"HOME\"];"], + [ENV_READ_FILE, "const value = process.env[name];"], + [ENV_READ_FILE, "const { env } = process;"], + [ENV_READ_FILE, "const path = process?.env?.PATH;"], + [ENV_READ_FILE, "Deno.env.get(\"HOME\");"], + [ENV_READ_FILE, "Bun.env.HOME;"], + [ENV_READ_FILE, "import.meta.env.MODE;"], + [ENV_READ_FILE, "const off = process.env.DO_NOT_TRACK;"], + ["hooks/session-start.mjs", "const off = process.env.ARCADE_PLUGIN_TELEMETRY;"], + ]; + for (const [file, line] of disallowed) { + assert.notDeepEqual(scriptOffenders(file, line), [], `expected an offender for ${file}: ${line}`); + } + for (const name of ENV_READ_ALLOWLIST) { + assert.deepEqual(scriptOffenders(ENV_READ_FILE, ` ${name}: process.env.${name},`), []); + } +}); + test("no text file names a bundled image or font path", () => { const offenders = []; for (const file of REGULAR_FILES) { diff --git a/dev/test/fixtures/telemetry-fake-adapter.mjs b/dev/test/fixtures/telemetry-fake-adapter.mjs index 1c73ad7..7fc878a 100644 --- a/dev/test/fixtures/telemetry-fake-adapter.mjs +++ b/dev/test/fixtures/telemetry-fake-adapter.mjs @@ -15,10 +15,6 @@ export const FAKE_OTHER_PREFIX = "mcp__fake_other__"; const fakeAdapter = { host: "claude-code", dataVariable: "ARCADE_TEST_PLUGIN_DATA", - optOutSwitches: [ - { name: "FAKE_DISABLE_TELEMETRY", anyValue: true }, - { name: "FAKE_OFFLINE", anyValue: false }, - ], requiresTurn: true, promptReminder: true, subagentSession: false, diff --git a/dev/test/manifests.test.mjs b/dev/test/manifests.test.mjs index d72269b..1b036de 100644 --- a/dev/test/manifests.test.mjs +++ b/dev/test/manifests.test.mjs @@ -95,6 +95,24 @@ test("Claude manifest displays Arcade without changing the install identifier", assert.equal(manifest.name, "arcade"); }); +test("OpenAI image variants and brandColor stay off the other client manifests", () => { + for (const file of [ + "plugin/.cursor-plugin/plugin.json", + ".cursor-plugin/plugin.json", + "plugin/.claude-plugin/plugin.json", + ".claude-plugin/marketplace.json", + ".cursor-plugin/marketplace.json", + ]) { + const text = readRepoFile(file); + for (const field of ["logoDark", "composerIcon", "composerIconDark", "brandColor", "logo-dark.png", "composer-icon"]) { + assert.equal(text.includes(field), false, `${file} contains ${field}`); + } + } + const cursor = JSON.parse(readRepoFile("plugin/.cursor-plugin/plugin.json")); + assert.equal(cursor.logo, "assets/logo.png"); + assert.deepEqual(Object.keys(cursor).includes("brandColor"), false); +}); + test("listing contact and privacy metadata agree across clients", () => { const portable = JSON.parse(readRepoFile("plugin/plugin.json")); const claude = JSON.parse(readRepoFile("plugin/.claude-plugin/plugin.json")); diff --git a/dev/test/package-chatgpt.test.mjs b/dev/test/package-chatgpt.test.mjs index cdb312a..6872076 100644 --- a/dev/test/package-chatgpt.test.mjs +++ b/dev/test/package-chatgpt.test.mjs @@ -8,11 +8,14 @@ import { mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { test } from "node:test"; +import { inflateSync } from "node:zlib"; import { archiveProblems, listingProblems, packageChatGPT } from "../scripts/package-chatgpt.mjs"; import { readRepoFile, ROOT } from "./helpers.mjs"; const plugin = JSON.parse(readRepoFile("plugin/plugin.json")); -const logo = plugin.extensions["com.openai"].interface.logo.replace(/^\.\//, ""); +const listingImages = ["logo", "logoDark", "composerIcon", "composerIconDark"].map((key) => + plugin.extensions["com.openai"].interface[key].replace(/^\.\//, ""), +); test("the OpenAI listing fits the portal's limits", () => { assert.deepEqual(listingProblems(plugin.extensions["com.openai"].interface), []); @@ -30,6 +33,13 @@ test("listingProblems reports what final directory submission rejects", () => { assert.deepEqual(listingProblems({ ...listing, defaultPrompt: ["Hi there", "Hi there"] }), ["defaultPrompt entries must be unique"]); assert.deepEqual(listingProblems({ ...listing, supportURL: undefined }), ["supportURL is required for a remote MCP submission"]); assert.deepEqual(listingProblems({ ...listing, termsOfServiceURL: undefined }), ["termsOfServiceURL is required for a remote MCP submission"]); + assert.deepEqual(listingProblems({ ...listing, brandColor: "#FFFFFF" }), [ + "brandColor must have at least 2:1 contrast against white", + ]); + assert.deepEqual(listingProblems({ ...listing, brandColor: "purple" }), ["brandColor must be a #RRGGBB color"]); + assert.deepEqual(listingProblems({ ...listing, logoDark: "assets/logo-dark.png" }), [ + "logoDark must be a relative path starting with ./", + ]); }); test("archiveProblems rejects hooks and apps on the manifest OpenAI reads", () => { @@ -60,13 +70,41 @@ test("the directory listing has the four MCP URLs and five positive review cases assert.equal("demo_recording_url" in openai.review, false); assert.deepEqual(openai.publication.countries, ["US"]); assert.doesNotMatch(openai.publication.release_notes, /\d+\.\d+\.\d+/); + assert.equal(openai.interface.logo, "./assets/logo.png"); + assert.equal(openai.interface.logoDark, "./assets/logo-dark.png"); + assert.equal(openai.interface.composerIcon, "./assets/composer-icon.png"); + assert.equal(openai.interface.composerIconDark, "./assets/composer-icon-dark.png"); + assert.equal(openai.interface.brandColor, "#4708F9"); assert.deepEqual(openai.review.test_cases.positive.map((item) => item.tools_triggered), [ - "Gmail_ListEmails", - "GoogleCalendar_ListEvents", - "Github_ListIssues", - "Slack_GetMessages", - "Gmail_ListEmails, Gmail_WriteDraftReplyEmail", + "Arcade_SelectTools", + "Arcade_SelectTools, Arcade_UseTool (GoogleFlights.SearchFlights), Arcade_UseTool (Math.Multiply)", + "Arcade_SelectTools, Arcade_UseTool (GoogleFinance.GetStockSummary)", + "Arcade_SelectTools, Arcade_UseTool (Youtube.SearchForVideos)", + "Arcade_SelectTools, Arcade_UseTool (Math.Avg)", + ]); + assert.deepEqual(openai.review.test_cases.positive.map((item) => item.prompt), [ + "Use Arcade to find tools that could help me plan a weekend trip to Chicago: flights, hotels and driving directions. Just list what's available, don't run anything yet.", + "Use Arcade to find the cheapest one-way economy flight from SFO to JFK two weeks from today, then use Arcade's Math tools to calculate the total fare for 3 travelers.", + "Use Arcade to get the Google Finance stock summary for NVDA.", + "Use Arcade to search YouTube for videos explaining the Model Context Protocol.", + "Use Arcade to average my monthly software bills: $1,249.50, $389.99, $72.25 and $418.26.", + ]); + assert.deepEqual(openai.review.test_cases.negative.map((item) => item.prompt), [ + "What's the difference between TCP and UDP?", + "Send $50 to my landlord on Venmo.", + "Explain what a P/E ratio means when evaluating a stock.", ]); + for (const item of openai.review.test_cases.positive) { + assert.deepEqual(Object.keys(item), ["description", "prompt", "tools_triggered", "expected_behavior"]); + assert.ok(item.description.length <= 4000, item.description); + assert.ok(item.expected_behavior.length > 0 && item.expected_behavior.length <= 4000); + assert.equal(item.prompt.includes("\n"), false); + } + for (const item of openai.review.test_cases.negative) { + assert.deepEqual(Object.keys(item), ["description", "prompt"]); + assert.equal(item.prompt.includes("\n"), false); + assert.doesNotMatch(item.prompt, /^Use Arcade to /); + } }); const hasUnzip = !spawnSync("unzip", ["-v"]).error; @@ -83,7 +121,7 @@ test("the ZIP has plugin.json at its root and only the portable plugin", { skip: const listed = spawnSync("unzip", ["-Z1", zipPath], { encoding: "utf8" }).stdout.trim().split("\n"); assert.deepEqual(listed, files); - for (const required of ["plugin.json", "mcp.json", "skills/try-arcade/SKILL.md", "skills/scale-arcade/SKILL.md", logo]) { + for (const required of ["plugin.json", "mcp.json", "skills/try-arcade/SKILL.md", "skills/scale-arcade/SKILL.md", ...listingImages]) { assert.ok(listed.includes(required), `missing ${required}`); } for (const name of listed) { @@ -99,8 +137,89 @@ test("the ZIP has plugin.json at its root and only the portable plugin", { skip: const validate = ajv.compile(JSON.parse(readRepoFile(`schemas/agent-plugins/1.0.0/${file.replace(".json", ".schema.json")}`))); assert.ok(validate(JSON.parse(readFileSync(path.join(out, file), "utf8"))), `${file}: ${ajv.errorsText(validate.errors)}`); } - assert.deepEqual(readFileSync(path.join(out, logo)), readFileSync(path.join(ROOT, "plugin", logo))); + for (const image of listingImages) { + const bytes = readFileSync(path.join(out, image)); + assert.deepEqual(bytes, readFileSync(path.join(ROOT, "plugin", image))); + assertOpaqueSquarePng(bytes, image, image.startsWith("assets/composer-icon") ? 256 : 512); + } } finally { rmSync(work, { recursive: true, force: true }); } }); + +const paeth = (left, up, upLeft) => { + const estimate = left + up - upLeft; + const leftDistance = Math.abs(estimate - left); + const upDistance = Math.abs(estimate - up); + const upLeftDistance = Math.abs(estimate - upLeft); + if (leftDistance <= upDistance && leftDistance <= upLeftDistance) return left; + if (upDistance <= upLeftDistance) return up; + return upLeft; +}; + +/** Square, RGBA or RGB, no tRNS, and no pixel with alpha below 255. */ +const assertOpaqueSquarePng = (buf, label, pixels) => { + assert.equal(buf.subarray(0, 8).toString("hex"), "89504e470d0a1a0a", label); + assert.ok(buf.length <= 5 * 1024 * 1024, `${label} is over 5 MiB`); + let offset = 8; + let width = 0; + let height = 0; + let bitDepth = 0; + let colorType = 0; + const idat = []; + let transparentChunk = false; + while (offset + 12 <= buf.length) { + const length = buf.readUInt32BE(offset); + const type = buf.subarray(offset + 4, offset + 8).toString("ascii"); + const data = buf.subarray(offset + 8, offset + 8 + length); + if (type === "IHDR") { + width = data.readUInt32BE(0); + height = data.readUInt32BE(4); + bitDepth = data[8]; + colorType = data[9]; + } else if (type === "IDAT") { + idat.push(data); + } else if (type === "tRNS") { + transparentChunk = true; + } else if (type === "IEND") { + break; + } + offset += 12 + length; + } + assert.equal(width, pixels, `${label} width`); + assert.equal(height, pixels, `${label} height`); + assert.equal(bitDepth, 8, `${label} bit depth`); + assert.equal(transparentChunk, false, `${label} has a tRNS chunk`); + // 2 = RGB, 6 = RGBA. Palette PNGs (color type 3) are what left a transparent + // strip in the previous logo. + assert.ok(colorType === 2 || colorType === 6, `${label} color type ${colorType}`); + if (colorType === 2) return; + const channels = 4; + const raw = inflateSync(Buffer.concat(idat)); + const stride = width * channels; + const rows = []; + let pos = 0; + for (let y = 0; y < height; y++) { + const filter = raw[pos]; + pos += 1; + const row = Buffer.alloc(stride); + const previous = y > 0 ? rows[y - 1] : null; + for (let i = 0; i < stride; i++) { + const left = i >= channels ? row[i - channels] : 0; + const up = previous ? previous[i] : 0; + const upLeft = previous && i >= channels ? previous[i - channels] : 0; + const sample = raw[pos + i]; + if (filter === 0) row[i] = sample; + else if (filter === 1) row[i] = (sample + left) & 255; + else if (filter === 2) row[i] = (sample + up) & 255; + else if (filter === 3) row[i] = (sample + Math.floor((left + up) / 2)) & 255; + else if (filter === 4) row[i] = (sample + paeth(left, up, upLeft)) & 255; + else assert.fail(`${label} uses PNG filter ${filter}`); + } + pos += stride; + rows.push(row); + for (let x = 3; x < stride; x += 4) { + assert.equal(row[x], 255, `${label} has a transparent pixel on row ${y}`); + } + } +}; diff --git a/dev/test/telemetry-claude.test.mjs b/dev/test/telemetry-claude.test.mjs index 3ed55a6..f2b77d1 100644 --- a/dev/test/telemetry-claude.test.mjs +++ b/dev/test/telemetry-claude.test.mjs @@ -254,16 +254,12 @@ test("arcade-used is set only by a successful Arcade gateway call", async () => assert.deepEqual([attempt.sent[0], arcade.sent[0]].map((e) => e.properties.arcade_used_before), [false, false]); }); -test("Claude opt-out switches send nothing and write no data dir files", async () => { +test("Claude opt-out switch sends nothing and write no data dir files", async () => { const base = hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar" }); const sessionStart = hookInput({ hook_event_name: "SessionStart", source: "startup" }); const cases = [ - ["DISABLE_TELEMETRY=1", { DISABLE_TELEMETRY: "1" }], - ["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1", { CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1" }], - ["DISABLE_TELEMETRY=0", { DISABLE_TELEMETRY: "0" }], - ["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=false", { CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "false" }], ["ARCADE_PLUGIN_TELEMETRY=0", { ARCADE_PLUGIN_TELEMETRY: "0" }], - ["DO_NOT_TRACK=1", { DO_NOT_TRACK: "1" }], + ["ARCADE_PLUGIN_TELEMETRY=Off", { ARCADE_PLUGIN_TELEMETRY: "Off" }], ["missing CLAUDE_PLUGIN_DATA", { CLAUDE_PLUGIN_DATA: "" }], ["relative CLAUDE_PLUGIN_DATA", { CLAUDE_PLUGIN_DATA: "relative/data" }], ]; diff --git a/dev/test/telemetry-copilot.test.mjs b/dev/test/telemetry-copilot.test.mjs index b66e587..64a7791 100644 --- a/dev/test/telemetry-copilot.test.mjs +++ b/dev/test/telemetry-copilot.test.mjs @@ -9,7 +9,6 @@ import { HOOKS, HOSTS, telemetryHookRows } from "../../plugin/hooks/hook-hosts.m import { SCOPE_DIRECTORY, SCOPE_TTL_MS } from "../../plugin/hooks/hook-scope.mjs"; import { ARCADE_USED_FILE } from "../../plugin/hooks/telemetry-config.mjs"; import { buildEvent, isArcadeCall } from "../../plugin/hooks/telemetry-events.mjs"; -import { isOptedOut } from "../../plugin/hooks/telemetry-run.mjs"; import copilotAdapter, { COPILOT_ARCADE_SERVER } from "../../plugin/hooks/telemetry-adapters/copilot-cli.mjs"; import { buildHookManifest } from "../scripts/generate-manifests.mjs"; import { @@ -301,9 +300,7 @@ test("opt-out and missing data dir send nothing", async () => { }); const cases = [ ["ARCADE_PLUGIN_TELEMETRY=0", { ARCADE_PLUGIN_TELEMETRY: "0" }], - ["DO_NOT_TRACK=1", { DO_NOT_TRACK: "1" }], - ["COPILOT_OFFLINE=true", { COPILOT_OFFLINE: "true" }], - ["COPILOT_OFFLINE=1", { COPILOT_OFFLINE: "1" }], + ["ARCADE_PLUGIN_TELEMETRY=off", { ARCADE_PLUGIN_TELEMETRY: "off" }], ["no COPILOT_PLUGIN_DATA", { COPILOT_PLUGIN_DATA: "" }], ["relative COPILOT_PLUGIN_DATA", { COPILOT_PLUGIN_DATA: "relative/data" }], ]; @@ -313,7 +310,7 @@ test("opt-out and missing data dir send nothing", async () => { assert.equal(sent.length, 0, label); assert.deepEqual(readdirSync(dataDir), [], label); } - for (const extra of [{ COPILOT_OFFLINE: "false" }, { COPILOT_OFFLINE: "0" }, { DISABLE_TELEMETRY: "1" }]) { + for (const extra of [{ ARCADE_PLUGIN_TELEMETRY: "1" }, { ARCADE_PLUGIN_TELEMETRY: "on" }]) { const dataDir = tempDataDir(); await captureTelemetry({ adapter: copilotAdapter, @@ -324,8 +321,6 @@ test("opt-out and missing data dir send nothing", async () => { const { sent } = await captureTelemetry({ adapter: copilotAdapter, input, dataDir, env: extra }); assert.ok(sent.length > 0, JSON.stringify(extra)); } - assert.equal(isOptedOut(copilotAdapter.optOutSwitches, { COPILOT_OFFLINE: "true" }), true); - assert.equal(isOptedOut(copilotAdapter.optOutSwitches, { COPILOT_OFFLINE: "false" }), false); }); test("malformed hook input without a prompt sends nothing", async () => { diff --git a/dev/test/telemetry-foundation.test.mjs b/dev/test/telemetry-foundation.test.mjs index ad865b7..50529bc 100644 --- a/dev/test/telemetry-foundation.test.mjs +++ b/dev/test/telemetry-foundation.test.mjs @@ -163,11 +163,8 @@ test("runTelemetry opt-out matrix sends nothing and writes no files", async () = const cases = [ ["ARCADE_PLUGIN_TELEMETRY=0", { ARCADE_PLUGIN_TELEMETRY: "0" }], ["ARCADE_PLUGIN_TELEMETRY=OFF", { ARCADE_PLUGIN_TELEMETRY: "OFF" }], - ["DO_NOT_TRACK=1", { DO_NOT_TRACK: "1" }], - ["FAKE_DISABLE_TELEMETRY=1", { FAKE_DISABLE_TELEMETRY: "1" }], - ["FAKE_DISABLE_TELEMETRY=0", { FAKE_DISABLE_TELEMETRY: "0" }], - ["FAKE_OFFLINE=true", { FAKE_OFFLINE: "true" }], - ["FAKE_OFFLINE=1", { FAKE_OFFLINE: "1" }], + ["ARCADE_PLUGIN_TELEMETRY=false", { ARCADE_PLUGIN_TELEMETRY: "false" }], + ["ARCADE_PLUGIN_TELEMETRY=no", { ARCADE_PLUGIN_TELEMETRY: "no" }], ["missing data dir", { ARCADE_TEST_PLUGIN_DATA: "" }], ["relative data dir", { ARCADE_TEST_PLUGIN_DATA: "relative/data" }], ]; @@ -245,9 +242,11 @@ test("runTelemetry survives malformed input and send failures", async () => { }); test("isOptedOut respects case on ARCADE_PLUGIN_TELEMETRY", () => { - assert.equal(isOptedOut([], { ARCADE_PLUGIN_TELEMETRY: "OFF" }), true); - assert.equal(isOptedOut([], { ARCADE_PLUGIN_TELEMETRY: "off" }), true); - assert.equal(isOptedOut(fakeAdapter.optOutSwitches, { FAKE_OFFLINE: "FALSE" }), false); + assert.equal(isOptedOut({ ARCADE_PLUGIN_TELEMETRY: "OFF" }), true); + assert.equal(isOptedOut({ ARCADE_PLUGIN_TELEMETRY: "off" }), true); + assert.equal(isOptedOut({ ARCADE_PLUGIN_TELEMETRY: "1" }), false); + assert.equal(isOptedOut({ ARCADE_PLUGIN_TELEMETRY: "" }), false); + assert.equal(isOptedOut({}), false); }); test("telemetry-send exits without reading the environment or contacting a host", async () => { diff --git a/dev/test/telemetry-helpers.mjs b/dev/test/telemetry-helpers.mjs index e1eb760..b18a203 100644 --- a/dev/test/telemetry-helpers.mjs +++ b/dev/test/telemetry-helpers.mjs @@ -23,10 +23,6 @@ import { ROOT } from "./helpers.mjs"; const TELEMETRY_ENV_BLOCKLIST = [ "ARCADE_PLUGIN_TELEMETRY", "ARCADE_PLUGIN_TELEMETRY_HOST", - "DO_NOT_TRACK", - "DISABLE_TELEMETRY", - "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", - "COPILOT_OFFLINE", "CLAUDE_PLUGIN_DATA", "COPILOT_PLUGIN_DATA", "PLUGIN_ROOT", @@ -151,9 +147,6 @@ export const telemetryEnv = (adapter, dataDir, serverUrl, extra = {}) => ({ [adapter.dataVariable]: dataDir, ARCADE_PLUGIN_TELEMETRY_HOST: serverUrl, ARCADE_PLUGIN_TELEMETRY: "", - DO_NOT_TRACK: "", - FAKE_DISABLE_TELEMETRY: "", - FAKE_OFFLINE: "", ...extra, }); diff --git a/dev/test/telemetry-off.test.mjs b/dev/test/telemetry-off.test.mjs index 380c12c..14916b8 100644 --- a/dev/test/telemetry-off.test.mjs +++ b/dev/test/telemetry-off.test.mjs @@ -1,7 +1,10 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { HOOKS, HOSTS } from "../../plugin/hooks/hook-hosts.mjs"; -import { TELEMETRY_ENABLED } from "../../plugin/hooks/telemetry-config.mjs"; +import claudeCodeAdapter from "../../plugin/hooks/telemetry-adapters/claude-code.mjs"; +import copilotAdapter from "../../plugin/hooks/telemetry-adapters/copilot-cli.mjs"; +import { OPT_OUT_ENV, TELEMETRY_ENABLED } from "../../plugin/hooks/telemetry-config.mjs"; +import { readOptOutEnv } from "../../plugin/hooks/telemetry-run.mjs"; import { buildHookManifest } from "../scripts/generate-manifests.mjs"; import { pluginPath, readRepoFile } from "./helpers.mjs"; @@ -20,7 +23,7 @@ test("this build ships no telemetry key, sender, or copy of the environment", () const hook = (file) => readRepoFile(`plugin/hooks/${file}`); assert.doesNotMatch(hook("telemetry-config.mjs"), /process\.env|phc_/); assert.doesNotMatch(hook("telemetry-send.mjs"), /process\.env|\bfetch\b|\bimport\b/); - assert.doesNotMatch(hook("telemetry-run.mjs"), /\.\.\.(?:process\.)?env\b/); + assert.doesNotMatch(hook("telemetry-run.mjs"), /\.\.\.(?:process\.)?env\b|child_process|\bspawn\(/); assert.doesNotMatch(hook("session-start.mjs"), /telemetry/i); }); @@ -54,3 +57,8 @@ test("flat manifests keep matchers and reject if and extra args", () => { /flat format does not support/, ); }); + +test("readOptOutEnv picks only the opt-out switch and each client's data-folder name", () => { + const names = [OPT_OUT_ENV, claudeCodeAdapter.dataVariable, copilotAdapter.dataVariable]; + assert.deepEqual(Object.keys(readOptOutEnv()).sort(), names.sort()); +}); diff --git a/docs/install/claude-code.md b/docs/install/claude-code.md index 4c78718..3ad990b 100644 --- a/docs/install/claude-code.md +++ b/docs/install/claude-code.md @@ -48,8 +48,7 @@ events are sent and generated manifests include no telemetry hooks. When telemetry is enabled, the adapter in `plugin/hooks/telemetry-adapters/claude-code.mjs` classifies prompts, records `PreToolUse` on the plugin and claude.ai Arcade prefixes, and stores scope under `CLAUDE_PLUGIN_DATA`. Opt out with -`ARCADE_PLUGIN_TELEMETRY=0`, `DISABLE_TELEMETRY`, or -`CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC`. See +`ARCADE_PLUGIN_TELEMETRY=0`. See [telemetry.md](../telemetry.md) and [what CI exercises](../telemetry.md#what-ci-exercises). diff --git a/docs/install/copilot.md b/docs/install/copilot.md index 3a93444..36135c3 100644 --- a/docs/install/copilot.md +++ b/docs/install/copilot.md @@ -39,8 +39,7 @@ hooks and nothing is sent. When enabled, `plugin/hooks/telemetry-adapters/copilo records MCP tools as `-`, operator stops with `subagent_session`, and session-scoped prompt state under `COPILOT_PLUGIN_DATA` (no `turn`, no `PreToolUse`, no built-in CLI/web events). See -[telemetry.md](../telemetry.md) for opt-outs (`ARCADE_PLUGIN_TELEMETRY=0` and -`COPILOT_OFFLINE`) and +[telemetry.md](../telemetry.md) for the opt-out (`ARCADE_PLUGIN_TELEMETRY=0`) and [what CI exercises](../telemetry.md#what-ci-exercises). Shared `com.github.copilot/hooks/hooks.json` uses `runOnlyIfScriptExists` to diff --git a/docs/telemetry.md b/docs/telemetry.md index 151e0c1..68bc6bf 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -5,7 +5,7 @@ build also ships no PostHog key, and `plugin/hooks/telemetry-send.mjs` exits without reading the environment or contacting a server. `TELEMETRY_ENABLED` in `plugin/hooks/telemetry-config.mjs` stays `false` until -collection is separately approved after ingestion, opt-outs, privacy, and +collection is separately approved after ingestion, the opt-out, privacy, and distribution requirements are verified. The sections below describe what **would** be sent when telemetry is turned on, @@ -44,11 +44,15 @@ except short acknowledgements and background task results, whether or not telemetry is on. Copilot CLI has no routing reminder hook, so `reminder_sent` stays false there. -## Opt-outs when telemetry is on +## Opt-out when telemetry is on -While `TELEMETRY_ENABLED` is `false`, these switches are inert but documented -for a future enabled build. They are implemented in `plugin/hooks/telemetry-run.mjs` -and each client's adapter in `plugin/hooks/telemetry-adapters/`. +While `TELEMETRY_ENABLED` is `false`, this switch is inert but documented for a +future enabled build. `ARCADE_PLUGIN_TELEMETRY` is the only opt-out, on every +client. It is implemented in `plugin/hooks/telemetry-run.mjs`. +The hooks read it and each client's data-folder variable +(`CLAUDE_PLUGIN_DATA`, `COPILOT_PLUGIN_DATA`) by name in `readOptOutEnv`, and +only after the `TELEMETRY_ENABLED` check, so this build reads no environment +variables. They read no other variable and pass the environment to nothing. Set `ARCADE_PLUGIN_TELEMETRY=0` in your environment, or in Claude Code's `settings.json`: @@ -57,11 +61,8 @@ Set `ARCADE_PLUGIN_TELEMETRY=0` in your environment, or in Claude Code's { "env": { "ARCADE_PLUGIN_TELEMETRY": "0" } } ``` -`false`, `off`, and `no` also work. Telemetry is also off when `DO_NOT_TRACK` -is set to any non-empty value other than `0`, `false`, `off`, or `no`. In -Claude Code, `DISABLE_TELEMETRY` and `CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC` -opt out when set to **any** non-empty value (including `0` and `false`), matching -Claude's own telemetry switches. +`false`, `off`, and `no` also work, in any case. Any other value, including an +unset or empty variable, leaves telemetry on. With telemetry opted out, the client still invokes its configured Node hooks. The telemetry hook exits before classifying the prompt or storing state. Claude @@ -69,11 +70,7 @@ Code's routing hook still adds its reminder. An environment variable cannot remove hooks from the manifest. In Copilot CLI, set `ARCADE_PLUGIN_TELEMETRY` to `0`, `false`, `off`, or `no` -in your shell before starting `copilot`. Any other value, including an unset -variable, leaves that switch inactive. `COPILOT_OFFLINE` turns telemetry off -when it is set to any non-empty value other than `0`, `false`, `off`, or `no`. -An unset or empty `COPILOT_OFFLINE`, and those four values, leave telemetry on -through this switch. +in your shell before starting `copilot`. ## Where it runs @@ -213,7 +210,7 @@ was needed or whether a user's task succeeded. `could_use_arcade` and excluded. Neither host sends a task ID. Claude Code supplies a hashed `turn`; Copilot CLI supplies only a hashed -`session`, which can contain multiple prompts. Detached delivery can change +`session`, which can contain multiple prompts. Delivery can change event arrival order. Tool attempts, completions, and failures are separate observations; timestamps alone cannot pair calls or establish a task outcome. @@ -249,7 +246,7 @@ shipping telemetry hooks. Confirm, outside this repository: - user-facing disclosure and privacy policy coverage for plugin usage events - analytics retention and processor handling for the PostHog project - verified ingestion per client and `telemetry_version` for the build under review -- opt-out switches verified on each supported client surface +- the `ARCADE_PLUGIN_TELEMETRY` opt-out verified on each supported client surface - applicable store or distribution decisions for builds that transmit data The change that turns telemetry on also has to add an event sender to @@ -260,6 +257,12 @@ The change that turns telemetry on also has to add an event sender to Tests can't point the sender at a local server until it reads a host override again. +The detached child process that used to run `telemetry-send.mjs` is gone, and +`defaultSend` in `plugin/hooks/telemetry-run.mjs` does nothing. Plugin scripts +may not start child processes or call `fetch(`, which +`dev/test/directory-policy.test.mjs` enforces, so the enabling change also has +to choose a sender that works within or deliberately changes that rule. + These are release acceptance requirements; the current code and CI do not enforce them. A repository install can consume a branch before a GitHub release is tagged, so a branch install can transmit data before any release is tagged. diff --git a/plugin/assets/composer-icon-dark.png b/plugin/assets/composer-icon-dark.png new file mode 100644 index 0000000..748e3f0 Binary files /dev/null and b/plugin/assets/composer-icon-dark.png differ diff --git a/plugin/assets/composer-icon.png b/plugin/assets/composer-icon.png new file mode 100644 index 0000000..748e3f0 Binary files /dev/null and b/plugin/assets/composer-icon.png differ diff --git a/plugin/assets/logo-dark.png b/plugin/assets/logo-dark.png new file mode 100644 index 0000000..6e9793e Binary files /dev/null and b/plugin/assets/logo-dark.png differ diff --git a/plugin/assets/logo.png b/plugin/assets/logo.png index d006146..6e9793e 100644 Binary files a/plugin/assets/logo.png and b/plugin/assets/logo.png differ diff --git a/plugin/hooks/telemetry-adapter.mjs b/plugin/hooks/telemetry-adapter.mjs index 657579f..31c2fc1 100644 --- a/plugin/hooks/telemetry-adapter.mjs +++ b/plugin/hooks/telemetry-adapter.mjs @@ -39,13 +39,6 @@ import { TELEMETRY_HOSTS } from "./telemetry-contract.mjs"; * @property {string[]} [extraArgs] */ -/** - * @typedef {{ name: string, anyValue: boolean }} OptOutSwitch - * A client's own off switch. `anyValue: true`: any non-empty value turns - * telemetry off. `anyValue: false`: any value except empty, 0, false, off, - * or no does. - */ - /** * @typedef {Record} ToolProperties * Contract properties for an MCP tool event (`server`, `tool`, `service`), or @@ -56,7 +49,6 @@ import { TELEMETRY_HOSTS } from "./telemetry-contract.mjs"; * @typedef {object} TelemetryAdapter * @property {typeof TELEMETRY_HOSTS[number]} host The `host` property on every event. * @property {string} dataVariable Environment variable naming the plugin's data folder. - * @property {OptOutSwitch[]} optOutSwitches * @property {boolean} requiresTurn Whether tool events count as app work only * with the same prompt ID as the prompt that opened scope. Without it, a * tool event inherits its session's scope. diff --git a/plugin/hooks/telemetry-adapters/claude-code.mjs b/plugin/hooks/telemetry-adapters/claude-code.mjs index e9eabd6..3558f7c 100644 --- a/plugin/hooks/telemetry-adapters/claude-code.mjs +++ b/plugin/hooks/telemetry-adapters/claude-code.mjs @@ -85,11 +85,6 @@ const bashHookRows = (event) => const claudeCodeAdapter = { host: "claude-code", dataVariable: "CLAUDE_PLUGIN_DATA", - // Claude Code reads these as set for any non-empty value, even "0" or "false". - optOutSwitches: [ - { name: "DISABLE_TELEMETRY", anyValue: true }, - { name: "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", anyValue: true }, - ], requiresTurn: true, promptReminder: true, subagentSession: false, diff --git a/plugin/hooks/telemetry-adapters/copilot-cli.mjs b/plugin/hooks/telemetry-adapters/copilot-cli.mjs index ab461a1..63f194f 100644 --- a/plugin/hooks/telemetry-adapters/copilot-cli.mjs +++ b/plugin/hooks/telemetry-adapters/copilot-cli.mjs @@ -73,9 +73,6 @@ const toolProperties = (toolName, toolInput) => { const copilotCliAdapter = { host: "copilot-cli", dataVariable: "COPILOT_PLUGIN_DATA", - // Copilot documents "true" and doesn't say which other values it - // accepts, so any value that isn't clearly off counts. - optOutSwitches: [{ name: "COPILOT_OFFLINE", anyValue: false }], requiresTurn: false, promptReminder: false, subagentSession: true, diff --git a/plugin/hooks/telemetry-run.mjs b/plugin/hooks/telemetry-run.mjs index 1d8b32d..d917f82 100644 --- a/plugin/hooks/telemetry-run.mjs +++ b/plugin/hooks/telemetry-run.mjs @@ -1,39 +1,35 @@ // @ts-check /** Opt-out checks and the shared telemetry hook body. Never throws. */ -import { spawn } from "node:child_process"; import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import path from "node:path"; -import { fileURLToPath } from "node:url"; import { scopeForInput } from "./hook-scope.mjs"; -import { ARCADE_USED_FILE, EVENT_ENV, OPT_OUT_ENV, TELEMETRY_ENABLED } from "./telemetry-config.mjs"; +import { ARCADE_USED_FILE, OPT_OUT_ENV, TELEMETRY_ENABLED } from "./telemetry-config.mjs"; import { buildEvent, isArcadeCall } from "./telemetry-events.mjs"; /** @typedef {import("./telemetry-adapter.mjs").TelemetryAdapter} TelemetryAdapter */ -const SENDER = path.join( - path.dirname(fileURLToPath(import.meta.url)), - "telemetry-send.mjs", -); +/** + * The only place shipped plugin code reads the process environment. It picks + * the one opt-out switch and the client data-folder variables by name into a + * plain object, and is called only after the TELEMETRY_ENABLED check, so a + * build with telemetry off reads no environment at all. + * @returns {NodeJS.ProcessEnv} + */ +export const readOptOutEnv = () => ({ + ARCADE_PLUGIN_TELEMETRY: process.env.ARCADE_PLUGIN_TELEMETRY, + CLAUDE_PLUGIN_DATA: process.env.CLAUDE_PLUGIN_DATA, + COPILOT_PLUGIN_DATA: process.env.COPILOT_PLUGIN_DATA, +}); const OFF_VALUES = ["0", "false", "off", "no"]; -const isSet = (/** @type {string} */ value) => value !== "" && !OFF_VALUES.includes(value.toLowerCase()); - /** - * A switch with `anyValue` is on for any non-empty value, even "0" or "false", - * because that is how Claude Code reads its own switches. - * @param {{ name: string, anyValue: boolean }[]} optOutSwitches - * @param {NodeJS.ProcessEnv} env + * ARCADE_PLUGIN_TELEMETRY set to 0, false, off, or no (any case) is the only opt-out. + * @param {NodeJS.ProcessEnv} [env] Defaults to readOptOutEnv(). */ -export const isOptedOut = (optOutSwitches, env = process.env) => { - if (OFF_VALUES.includes((env[OPT_OUT_ENV] ?? "").toLowerCase())) return true; - if (isSet(env.DO_NOT_TRACK ?? "")) return true; - return optOutSwitches.some(({ name, anyValue }) => { - const value = env[name] ?? ""; - return anyValue ? value !== "" : isSet(value); - }); -}; +export const isOptedOut = (env = readOptOutEnv()) => + OFF_VALUES.includes((env[OPT_OUT_ENV] ?? "").toLowerCase()); const readArcadeUsed = (/** @type {string} */ dir) => { try { @@ -64,13 +60,14 @@ const markArcadeUsed = (/** @type {string} */ dir) => { */ export const clearSessionScope = ({ adapter, - env = process.env, + env: givenEnv, input = {}, enabled = false, }) => { try { if (!enabled && !TELEMETRY_ENABLED) return; - if (isOptedOut(adapter.optOutSwitches, env)) return; + const env = givenEnv ?? readOptOutEnv(); + if (isOptedOut(env)) return; const dir = env[adapter.dataVariable]; if (!dir || !path.isAbsolute(dir)) return; scopeForInput( @@ -82,26 +79,9 @@ export const clearSessionScope = ({ } }; -const defaultSend = (/** @type {object} */ event, env = process.env) => { - // Claude Code kills hook processes when the session exits, so the network - // call runs in a detached child that can outlive this hook. The event goes - // in an environment variable, not an argument, because other users on the - // machine can read a process's arguments but not its environment. Pass only - // PATH, SystemRoot (Windows needs it for network calls), and the event. Do - // not copy the parent environment. - /** @type {Record} */ - const childEnv = { [EVENT_ENV]: JSON.stringify(event) }; - if (env.PATH) childEnv.PATH = env.PATH; - if (env.SystemRoot) childEnv.SystemRoot = env.SystemRoot; - const child = spawn(process.execPath, [SENDER], { - detached: true, - stdio: "ignore", - windowsHide: true, - env: childEnv, - }); - child.on("error", () => {}); - child.unref(); -}; +// No sender yet. The earlier detached child process is gone, so an enabled +// build needs a sender that does not start a child process. +const defaultSend = () => {}; /** * @param {object} options @@ -117,18 +97,19 @@ export const runTelemetry = async ({ input, adapter, argv = process.argv, - env = process.env, + env: givenEnv, now = Date.now(), enabled = false, - send = (event) => defaultSend(event, env), + send = defaultSend, }) => { try { if (!enabled && !TELEMETRY_ENABLED) return; + const env = givenEnv ?? readOptOutEnv(); // The client always sets this. Without an absolute data folder there is // nowhere to keep the arcade-used flag, so nothing is sent. const dir = env[adapter.dataVariable]; if (!dir || !path.isAbsolute(dir)) return; - if (isOptedOut(adapter.optOutSwitches, env)) return; + if (isOptedOut(env)) return; const normalized = adapter.normalize(input ?? {}); const { appWork } = scopeForInput(normalized, { host: adapter.host, diff --git a/plugin/plugin.json b/plugin/plugin.json index e2dcd8b..59b020c 100644 --- a/plugin/plugin.json +++ b/plugin/plugin.json @@ -41,54 +41,57 @@ "Which GitHub issues are assigned to me?" ], "logo": "./assets/logo.png", - "composerIcon": "./assets/logo.png" + "logoDark": "./assets/logo-dark.png", + "composerIcon": "./assets/composer-icon.png", + "composerIconDark": "./assets/composer-icon-dark.png", + "brandColor": "#4708F9" }, "review": { "test_cases": { "positive": [ { - "description": "Summarize unread email in the connected Gmail account", - "prompt": "Summarize my unread email from the last 7 days.", - "tools_triggered": "Gmail_ListEmails", - "expected_behavior": "Lists the seeded unread test emails (sender, subject, one-line summary each) from the reviewer Gmail account. No email is changed or marked read." + "description": "Find trip-planning tools without running them", + "prompt": "Use Arcade to find tools that could help me plan a weekend trip to Chicago: flights, hotels and driving directions. Just list what's available, don't run anything yet.", + "tools_triggered": "Arcade_SelectTools", + "expected_behavior": "Lists tools for flights, hotels, and driving directions, with a short description of each, and runs none of them. Returns no account names, emails, or connection status. Which tools are offered can vary with the catalog." }, { - "description": "Read tomorrow's calendar", - "prompt": "What's on my calendar tomorrow?", - "tools_triggered": "GoogleCalendar_ListEvents", - "expected_behavior": "Returns the seeded recurring test events for tomorrow with times and titles from the reviewer Google Calendar. Nothing is created or changed." + "description": "Find a one-way fare and total it for 3 travelers", + "prompt": "Use Arcade to find the cheapest one-way economy flight from SFO to JFK two weeks from today, then use Arcade's Math tools to calculate the total fare for 3 travelers.", + "tools_triggered": "Arcade_SelectTools, Arcade_UseTool (GoogleFlights.SearchFlights), Arcade_UseTool (Math.Multiply)", + "expected_behavior": "Reports the cheapest one-way economy fare it finds from SFO to JFK about two weeks out, with airline, flight number, and times, then the total for 3 travelers. Fares change; the total should equal the reported fare times 3. Nothing is booked." }, { - "description": "Find GitHub issues assigned to the user", - "prompt": "Which open GitHub issues are assigned to me in arcade-review/sample-tracker?", - "tools_triggered": "Github_ListIssues", - "expected_behavior": "Lists the seeded open issues assigned to the reviewer GitHub account in the sample repo, with titles and links." + "description": "Get the Google Finance summary for NVDA", + "prompt": "Use Arcade to get the Google Finance stock summary for NVDA.", + "tools_triggered": "Arcade_SelectTools, Arcade_UseTool (GoogleFinance.GetStockSummary)", + "expected_behavior": "Returns the Google Finance summary for NVDA, including NVIDIA's latest price in USD and the day's move in value and percent. Prices change; check the ticker and those fields. No order is placed." }, { - "description": "Read recent messages in a Slack channel", - "prompt": "What did people say in #launch-plan on Slack this week?", - "tools_triggered": "Slack_GetMessages", - "expected_behavior": "Summarizes the seeded messages in #launch-plan in the review Slack workspace and names who said what. Nothing is posted." + "description": "Search YouTube for videos about MCP", + "prompt": "Use Arcade to search YouTube for videos explaining the Model Context Protocol.", + "tools_triggered": "Arcade_SelectTools, Arcade_UseTool (Youtube.SearchForVideos)", + "expected_behavior": "Returns YouTube videos about the Model Context Protocol, each with a title, channel, and link. The list changes; check that the videos are about MCP." }, { - "description": "Draft an email and stop for confirmation", - "prompt": "Draft a reply to the 'Q4 venue options' email saying Thursday works, but don't send it.", - "tools_triggered": "Gmail_ListEmails, Gmail_WriteDraftReplyEmail", - "expected_behavior": "Shows the draft text and recipient, asks before saving, then saves it as a Gmail draft. The email is not sent." + "description": "Average a list of amounts", + "prompt": "Use Arcade to average my monthly software bills: $1,249.50, $389.99, $72.25 and $418.26.", + "tools_triggered": "Arcade_SelectTools, Arcade_UseTool (Math.Avg)", + "expected_behavior": "Answers $532.50." } ], "negative": [ { - "description": "General knowledge question that needs no connected app", + "description": "General knowledge question that needs no tool", "prompt": "What's the difference between TCP and UDP?" }, { - "description": "Request outside any connected app (payments)", + "description": "Payment request; Arcade connects no payment app and does not move money", "prompt": "Send $50 to my landlord on Venmo." }, { - "description": "Irreversible bulk action without a confirmed target", - "prompt": "Delete every email in my inbox." + "description": "Concept question about a stock ratio, with no quote to fetch", + "prompt": "Explain what a P/E ratio means when evaluating a stock." } ] },