From 06bd9015cd0df6d09fd5928488c4daaad65046da Mon Sep 17 00:00:00 2001 From: Jianghan <58623049+SuperstructureJH@users.noreply.github.com> Date: Thu, 10 Sep 2026 10:49:06 +0800 Subject: [PATCH 1/8] feat: add shared OpenAI and ByteDance image generation plugin --- build/dsh-desktop.patch.yml | 4 + docs/STATUS.md | 14 ++ package-lock.json | 57 +++-- package.json | 3 +- packages/dsh-image-generation/LICENSE | 21 ++ packages/dsh-image-generation/README.md | 56 +++++ packages/dsh-image-generation/client.js | 138 +++++++++++++ .../dsh-image-generation/cordis.patch.yml | 3 + packages/dsh-image-generation/index.js | 92 +++++++++ packages/dsh-image-generation/lib/assets.js | 50 +++++ packages/dsh-image-generation/lib/commit.js | 31 +++ packages/dsh-image-generation/lib/provider.js | 130 ++++++++++++ packages/dsh-image-generation/lib/settings.js | 67 ++++++ packages/dsh-image-generation/lib/storage.js | 30 +++ packages/dsh-image-generation/lib/writer.js | 13 ++ packages/dsh-image-generation/package.json | 30 +++ .../skills/generate-image/SKILL.md | 14 ++ patches/@deepseek-ai+dsh+0.1.2-rc.1.patch | 3 +- scripts/verify-image-generation.mjs | 99 +++++++++ test/image-generation.test.mjs | 194 ++++++++++++++++++ 20 files changed, 1031 insertions(+), 18 deletions(-) create mode 100644 docs/STATUS.md create mode 100644 packages/dsh-image-generation/LICENSE create mode 100644 packages/dsh-image-generation/README.md create mode 100644 packages/dsh-image-generation/client.js create mode 100644 packages/dsh-image-generation/cordis.patch.yml create mode 100644 packages/dsh-image-generation/index.js create mode 100644 packages/dsh-image-generation/lib/assets.js create mode 100644 packages/dsh-image-generation/lib/commit.js create mode 100644 packages/dsh-image-generation/lib/provider.js create mode 100644 packages/dsh-image-generation/lib/settings.js create mode 100644 packages/dsh-image-generation/lib/storage.js create mode 100644 packages/dsh-image-generation/lib/writer.js create mode 100644 packages/dsh-image-generation/package.json create mode 100644 packages/dsh-image-generation/skills/generate-image/SKILL.md create mode 100644 scripts/verify-image-generation.mjs create mode 100644 test/image-generation.test.mjs diff --git a/build/dsh-desktop.patch.yml b/build/dsh-desktop.patch.yml index 735dc568f..e8301e6b8 100644 --- a/build/dsh-desktop.patch.yml +++ b/build/dsh-desktop.patch.yml @@ -21,6 +21,10 @@ - id: dsh-desktop-market-installer name: dsh-desktop-market-installer + # One shared image tool for conversations and Office workflows. + - id: dsh-image-generation + name: dsh-image-generation + # DSH PPT owns one direct PPTD workflow and one standard Composer client. # The package contains no Tencent editor runtime or SlideP route. - id: dsh-ppt-composer diff --git a/docs/STATUS.md b/docs/STATUS.md new file mode 100644 index 000000000..ebc9f9173 --- /dev/null +++ b/docs/STATUS.md @@ -0,0 +1,14 @@ +# Implementation and verification status + +## Image generation plugin — 2026-09-10 + +- **Implemented:** public `dsh-image-generation` package, default Desktop composition, shared `image_generate` tool and `generate-image` Skill, OpenAI / ByteDance adapters, plugin configuration card, automatic validation on save, host-only atomic credential storage, governed PNG output. +- **PASS:** 19 focused integration tests plus 2 Desktop dependency-closure checks; full Desktop suite 90 files / 769 tests; TypeScript check and production build. The final ByteDance non-generating parameter probe and real macOS sandboxed PNG writer pass the focused suite. +- **PASS:** actual Host composition/authentication/origin and configuration save smoke against a loopback provider. Browser inspection confirmed the correct plugin settings entry, automatic save success/error, masked input retained on error, and no test-image action. This is a simulated-provider acceptance, not a paid-model result. +- **PASS:** macOS arm64 development DMG/ZIP build and strict deep code-signature verification. Package-level smoke starts Node and Harness from inside the built `.app`, validates default plugin/client composition, authentication and save flows. Browser inspection confirms both light/dark form layout and automatic save success. A final package-level check exercises both adapters, bundled Sharp and actual macOS sandboxed PNG output against a loopback provider. Build provenance and SHA-256 sums accompany the local artifacts. +- **PASS:** standalone tarball installation using the actual `dsh plugin --profile web add` command into an isolated DSH home, followed by authenticated Host startup and plugin/client discovery without the Desktop overlay. +- **Pending:** user acceptance. The development package uses the local Apple Development identity; Apple notarization is not run. +- **NOT_RUN:** current valid-key requests to the real OpenAI/ByteDance services, real generated-image visual review, insertion/render/edit/save/reopen in PowerPoint or Word, Windows and Intel Mac package acceptance. No paid image generation was used for configuration validation. +- **Distribution:** source and tarball are prepared for public distribution. npm publishing and upstream PR merge remain separate actions. + +Validation checks connection/model metadata or required-parameter handling; actual image generation permissions, quota and visual quality are established during real generation. The Desktop package is based on upstream `main` (`c8c33c4`) and adds the shared image capability; independent Word/Excel feature branches are outside this PR. diff --git a/package-lock.json b/package-lock.json index 16f2c92ea..571cb1857 100644 --- a/package-lock.json +++ b/package-lock.json @@ -240,6 +240,9 @@ "dsh-desktop-hmr-fallback": "file:packages/dsh-desktop-hmr-fallback", "dsh-desktop-market-installer": "file:packages/dsh-desktop-market-installer", "dsh-desktop-preset-transfer": "file:packages/dsh-desktop-preset-transfer", + "dsh-image-generation": "file:packages/dsh-image-generation", + "dsh-ppt": "file:packages/ppt-bundles/dsh-ppt-0.1.1-rc.2-desktop-20260906.tgz", + "dsh-ppt-composer": "file:packages/ppt-bundles/dsh-ppt-composer-0.1.1-rc.2-desktop-20260906.tgz", "dshmarket": "file:packages/dshmarket", "electron-updater": "^6.8.9", "node": "24.9.0", @@ -247,9 +250,7 @@ "qrcode": "^1.5.4", "react": "18.3.1", "react-dom": "18.3.1", - "ws": "^8.21.3", - "dsh-ppt": "file:packages/ppt-bundles/dsh-ppt-0.1.1-rc.2-desktop-20260906.tgz", - "dsh-ppt-composer": "file:packages/ppt-bundles/dsh-ppt-composer-0.1.1-rc.2-desktop-20260906.tgz" + "ws": "^8.21.3" }, "devDependencies": { "@types/node": "24.10.1", @@ -9972,6 +9973,10 @@ "resolved": "packages/dsh-desktop-preset-transfer", "link": true }, + "node_modules/dsh-image-generation": { + "resolved": "packages/dsh-image-generation", + "link": true + }, "node_modules/dsh-ppt": { "version": "0.1.1-rc.2", "resolved": "file:packages/ppt-bundles/dsh-ppt-0.1.1-rc.2-desktop-20260906.tgz", @@ -10018,8 +10023,8 @@ "integrity": "sha512-xGbky3/woNS9h0lNC17B1axE7POaYd2maHzx0hP6ZZt199XoRMSrvfJlFS3zCTNmX++LfSZ+zRXfe9+q7vb6MA==", "license": "MIT", "dependencies": { - "dsh-ppt": "0.1.1-rc.2", - "@deepseek-ai/schemastery": "3.18.2" + "@deepseek-ai/schemastery": "3.18.2", + "dsh-ppt": "0.1.1-rc.2" }, "peerDependencies": { "@deepseek-ai/cordis": "4.0.2", @@ -14856,6 +14861,17 @@ "dev": true, "license": "MIT" }, + "node_modules/string_decoder": { + "version": "1.1.1", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/string_decoder/node_modules/safe-buffer": { + "version": "5.1.2", + "license": "MIT" + }, "node_modules/string-width": { "version": "4.2.3", "license": "MIT", @@ -14868,17 +14884,6 @@ "node": ">=8" } }, - "node_modules/string_decoder": { - "version": "1.1.1", - "license": "MIT", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, - "node_modules/string_decoder/node_modules/safe-buffer": { - "version": "5.1.2", - "license": "MIT" - }, "node_modules/stringify-entities": { "version": "4.0.4", "license": "MIT", @@ -16423,6 +16428,26 @@ "fflate": "^0.8.2" } }, + "packages/dsh-image-generation": { + "version": "0.1.0", + "license": "MIT", + "dependencies": { + "@deepseek-ai/schemastery": "^3.18.2", + "sharp": "^0.35.4" + }, + "peerDependencies": { + "@deepseek-ai/cordis": "^4.0.2", + "@deepseek-ai/dsh-client-connection": "^0.1.2-rc.1", + "@deepseek-ai/dsh-credentials": "^0.1.2-rc.1", + "@deepseek-ai/dsh-sandbox": "^0.1.2-rc.1", + "@deepseek-ai/dsh-sandbox-policy": "^0.1.2-rc.1", + "@deepseek-ai/dsh-settings": "^0.1.2-rc.1", + "@deepseek-ai/dsh-skill": "^0.1.2-rc.1", + "@deepseek-ai/dsh-subprocess": "^0.1.2-rc.1", + "@deepseek-ai/dsh-system-prompt": "^0.1.2-rc.1", + "@deepseek-ai/dsh-tools": "^0.1.2-rc.1" + } + }, "packages/dshmarket": { "version": "1.40.0", "license": "MIT", diff --git a/package.json b/package.json index a92089fe7..fa5c59cf4 100644 --- a/package.json +++ b/package.json @@ -283,7 +283,8 @@ "react-dom": "18.3.1", "ws": "^8.21.3", "dsh-ppt": "file:packages/ppt-bundles/dsh-ppt-0.1.1-rc.2-desktop-20260906.tgz", - "dsh-ppt-composer": "file:packages/ppt-bundles/dsh-ppt-composer-0.1.1-rc.2-desktop-20260906.tgz" + "dsh-ppt-composer": "file:packages/ppt-bundles/dsh-ppt-composer-0.1.1-rc.2-desktop-20260906.tgz", + "dsh-image-generation": "file:packages/dsh-image-generation" }, "devDependencies": { "@types/node": "24.10.1", diff --git a/packages/dsh-image-generation/LICENSE b/packages/dsh-image-generation/LICENSE new file mode 100644 index 000000000..f37235ec4 --- /dev/null +++ b/packages/dsh-image-generation/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 DataElement + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/dsh-image-generation/README.md b/packages/dsh-image-generation/README.md new file mode 100644 index 000000000..b2591d3d0 --- /dev/null +++ b/packages/dsh-image-generation/README.md @@ -0,0 +1,56 @@ +# dsh-image-generation + +DSH/Cordis 公共生图插件，提供 `image_generate` 工具、`generate-image` Skill 和「设置 → 插件 → 插件配置 → 生图工具」卡片。PPT、Word 和普通对话共用同一份配置与图片资产。 + +## 使用 + +DSH Desktop 默认装载本插件。在生图工具卡片选择字节或 OpenAI，填写该平台的 API Key 并保存。模型和地址已有默认值，可在高级设置中修改。 + +| 服务商 | 默认模型 | 默认 API 地址 | +| --- | --- | --- | +| 字节 / 火山方舟 | `doubao-seedream-4-5-251128` | `https://ark.cn-beijing.volces.com/api/v3` | +| OpenAI | `gpt-image-1.5` | `https://api.openai.com/v1` | + +保存自动发起一次非生图校验请求，界面显示校验中、保存成功或具体错误。失败保留输入及之前的有效配置。Key 更换后立即生效；两个服务商分别保存配置和 Key。更换 API 地址的域名时需要重新填写 Key。 + +- OpenAI：`GET /models/{model}`，检查凭据和模型元数据访问。Key 需要允许读取该模型信息。 +- 字节：`POST /images/generations`，请求体仅含 `model`，省略必填 `prompt`。只将明确的 prompt 缺参错误识别为连接校验成功，其他错误正常报错。该请求进入参数检查，不提交生图任务。采用此方式是因为 Ark 官方运行时 SDK 没有可依赖的模型列表接口。 + +校验结果与实际生图成功分别记录。生图权限、余额、内容审核和输出效果在真正调用时确认。本插件没有生成测试图按钮。 + +提示词示例：「为这份 PPT 生成一张留出左侧标题空间的科技插画，风格沿用当前模板。」Agent 加载 Skill 后调用 `image_generate`，经过当前 Host 审批流程，将图片保存到工作区 `.workbuddy/generated-images/.png`。工具返回路径、尺寸、字节数、哈希、服务商和模型。图片可保留透明通道，标题、表格和简单图表继续使用 Office 原生对象。 + +本期支持文生图，每次生成一张图片。OpenAI 的 16:9 / 4:3 请求使用 1536×1024 画布，竖图使用 1024×1536；文档按实际返回尺寸等比放置或裁剪。字节按目标比例选择支持的画布。编辑图、参考图、批量生图和本地模型留待后续版本。 + +## 独立分发 + +该目录是可公开分发的 MIT npm 包，包含 Host、Client、Skill 和 `dsh.bundle.patch`。本 PR 提供源码与 tarball；npm registry 发布为单独操作。 + +在仓库根目录打包： + +```sh +npm pack ./packages/dsh-image-generation --pack-destination /absolute/output/directory +``` + +在具有匹配 Harness 服务的独立 DSH 安装中，可通过 `dsh plugin --profile web add /absolute/path/dsh-image-generation-0.1.0.tgz` 安装。要求 Harness `0.1.2-rc.1` 对应的 settings、credentials、connection、tools、skills、systemPrompt、sandboxPolicy、sandbox 和 subprocess 服务。Desktop 已默认装载时直接配置即可。 + +## 凭据与执行边界 + +配置通过经过 Host 鉴权和 Origin 检查的 `/api/image-generation.settings`、`/api/image-generation.save` 路由读写。API 响应使用 `no-store`，读取接口仅返回 Key 是否已配置。 + +服务商、模型、地址和 Key 在 `ctx.credentials` 的 `dsh-image-generation/configuration` 私有记录中一次提交。凭据服务的跨进程锁和修订检查防止并发保存覆盖；失败保存保持整个旧记录。使用同一记录使一次工具调用读到一致的配置。通用 settings 仅注册卡片命名空间。Agent 参数、图片、日志和工作区均不携带 Key。 + +生图通过标准工具执行和审计流程，并默认要求 Host 审批。请求前及落盘前读取当前会话的 sandboxPolicy，要求工作区写权限；输出目录和目标文件拒绝符号链接。实际写入由 `ctx.sandbox` 包装的 `ctx.subprocess` 子进程执行，使用工作区写权限；仅通过 stdin 接收图片字节，清除继承环境，Key 留在 Host。PNG 使用临时文件、原子创建和内容哈希校验，避免并发生图产生半个文件。HTTP 禁止重定向，只接受 HTTPS 地址（本机回环开发服务允许 HTTP），约束超时、响应大小和解码像素；不下载厂商返回的任意远程 URL。系统沙箱的跨平台执行能力以部署环境为准，本期本地验收环境为 macOS arm64。 + +## 验证 + +```sh +npx vitest run test/image-generation.test.mjs test/desktop-plugin-closure.test.ts +node scripts/verify-image-generation.mjs +npm run typecheck +npm test +``` + +测试使用本机模拟服务和真实 Harness 凭据存储、工具执行管线，覆盖两个厂商、单请求保存、成功/失败、修订冲突、凭据隔离、PNG 落盘、取消、体积限制和目录越界。Host smoke 启动隔离实例，验证默认装载、Client 入口、鉴权、Origin 及保存结果。真实付费模型与 Office 文档视觉验收在 `docs/STATUS.md` 单独记录。 + +接口参考：[OpenAI Images API](https://developers.openai.com/api/reference/resources/images/methods/generate)、[OpenAI Models API](https://developers.openai.com/api/reference/resources/models/methods/retrieve)、[火山方舟生图 API](https://www.volcengine.com/docs/82379/1541523)、[火山方舟官方 Python SDK](https://github.com/volcengine/volcengine-python-sdk/tree/master/volcenginesdkarkruntime/resources/images)。 diff --git a/packages/dsh-image-generation/client.js b/packages/dsh-image-generation/client.js new file mode 100644 index 000000000..9bf566595 --- /dev/null +++ b/packages/dsh-image-generation/client.js @@ -0,0 +1,138 @@ +window.__ModuleLoader__.load({ + id: 'dsh-image-generation', + factory: require => { + const React = require('react') + const h = React.createElement + const NS = 'settings.imageGeneration' + const zh = { + title: '生图工具', description: '为 PPT、Word 和对话生成图片。', provider: '服务商', bytedance: '字节 · 豆包 Seedream', openai: 'OpenAI', + apiKey: 'API Key', keyPlaceholder: '输入服务商的 API Key', savedKey: '已配置，输入新 Key 可替换', keyLink: '获取 API Key', + advanced: '高级设置', model: '模型 ID', baseUrl: 'API 地址', defaults: '已提供默认模型和地址，通常无需修改。', + save: '保存', saving: '正在校验…', saved: '已保存，连接校验通过', hint: '保存时自动校验连接；模型的生图权限和额度在实际调用时确认。', + configured: '已配置', loading: '正在读取配置…', reload: '重新读取配置', readOnly: '当前配置由管理员管理。', + AUTH: 'API Key 无效或已过期，请检查后重新保存。', PERMISSION: '当前 Key 无权访问，请确认模型已开通及账号已完成所需认证。', + MODEL: '无法访问所选模型，请检查模型 ID、接入点或模型开通状态。', KEY_REQUIRED: '请填写 API Key；更换服务地址后需要重新填写。', + QUOTA: '服务商额度不足或请求受限，请检查账户后重试。', ENDPOINT: '请填写有效的 API 基础地址，例如以 /v1 或 /api/v3 结尾的 HTTPS 地址。', + TIMEOUT: '连接超时，请检查网络后重新保存。', CONFLICT: '配置已在其他窗口更新，请重新打开此卡片后保存。', + READ_ONLY: '当前凭据存储为只读，请联系管理员。', RESPONSE: '服务商返回了无法识别的结果，请检查 API 地址。', + PARAMETERS: '服务商未通过连接检查，请确认 API 地址、模型 ID 及接入点配置。', + UNAVAILABLE: '连接失败，请检查网络和 API 地址后重新保存。', CANCELLED: '校验已取消，可以重新保存。', PROVIDER_ERROR: '服务商暂时不可用，请稍后重新保存。', + } + const en = { + title: 'Image generation', description: 'Generate images for PPT, Word and conversations.', provider: 'Provider', bytedance: 'ByteDance · Seedream', openai: 'OpenAI', + apiKey: 'API Key', keyPlaceholder: 'Enter your provider API key', savedKey: 'Configured; enter a new key to replace it', keyLink: 'Get an API key', + advanced: 'Advanced settings', model: 'Model ID', baseUrl: 'API base URL', defaults: 'The default model and URL are ready to use.', + save: 'Save', saving: 'Validating…', saved: 'Saved. Connection validated.', hint: 'Saving validates the connection automatically. Image permissions and quota are confirmed during generation.', + configured: 'Configured', loading: 'Loading settings…', reload: 'Reload settings', readOnly: 'These settings are managed by your administrator.', + AUTH: 'The API key is invalid or expired.', PERMISSION: 'Check model access and account verification.', MODEL: 'Check the model ID, inference endpoint and model access.', + KEY_REQUIRED: 'Enter an API key. A new API origin requires you to enter the key again.', QUOTA: 'Check provider quota and rate limits.', + ENDPOINT: 'Enter an HTTPS API base URL, normally ending in /v1 or /api/v3.', TIMEOUT: 'The connection timed out. Check your network and save again.', + CONFLICT: 'Settings changed in another window. Reopen this card before saving.', READ_ONLY: 'The credential store is read-only.', + RESPONSE: 'The provider returned an invalid response. Check the API URL.', UNAVAILABLE: 'Connection failed. Check your network and API URL.', + PARAMETERS: 'The provider did not confirm the connection check. Verify the API URL and model ID.', + CANCELLED: 'Validation was cancelled. Save again to retry.', PROVIDER_ERROR: 'The provider is temporarily unavailable. Save again later.', + } + const css = ` + .dshImageCard{list-style:none;border:1px solid var(--dsw-alias-border-l2);border-radius:14px;color:var(--dsw-alias-label-primary);overflow:hidden} + .dshImageHeader{display:flex;align-items:center;gap:12px;width:100%;padding:18px 20px;background:transparent;border:0;text-align:left;color:inherit;font:inherit;cursor:pointer} + .dshImageHeading{display:flex;flex:1;flex-direction:column;gap:6px;min-width:0}.dshImageTitle{font-size:16px;font-weight:600;line-height:24px} + .dshImageDescription{color:var(--dsw-alias-label-tertiary);font-size:14px;line-height:22px}.dshImageBadge{font-size:12px;color:var(--dsw-alias-label-secondary)} + .dshImageChevron{width:8px;height:8px;border-right:1.5px solid var(--dsw-alias-label-tertiary);border-bottom:1.5px solid var(--dsw-alias-label-tertiary);transform:rotate(45deg);margin-right:4px}.dshImageHeader[aria-expanded=true] .dshImageChevron{transform:rotate(225deg)} + .dshImageBody{padding:0 20px 20px;display:flex;flex-direction:column;gap:16px}.dshImageFields{border:0;margin:0;padding:0;display:flex;flex-direction:column;gap:16px;min-width:0} + .dshImageField{display:flex;flex-direction:column;gap:7px;font-size:13px}.dshImageField input,.dshImageField select{box-sizing:border-box;width:100%;min-width:0;height:36px;padding:0 11px;border:1px solid var(--dsw-alias-border-l3);border-radius:8px;background:var(--dsw-alias-bg-layer-1);color:inherit;font:inherit} + .dshImageField input::placeholder{color:var(--dsw-alias-label-tertiary)}.dshImageLink{align-self:flex-start;font-size:12px;color:var(--dsw-alias-label-secondary);text-decoration:underline} + .dshImageAdvanced summary{cursor:pointer;font-size:13px;color:var(--dsw-alias-label-secondary)}.dshImageAdvanced[open]{display:flex;flex-direction:column}.dshImageAdvanced .dshImageField{margin-top:14px} + .dshImageHint{font-size:12px;line-height:19px;color:var(--dsw-alias-label-tertiary);margin:0}.dshImageActions{display:flex;align-items:center;gap:12px;flex-wrap:wrap} + .dshImageSave{font:inherit;font-size:13px;font-weight:500;border:0;border-radius:18px;padding:8px 20px;background:var(--dsw-alias-button-primary-fill);color:var(--dsw-alias-label-primary-foreground);cursor:pointer}.dshImageSave:disabled{opacity:.5;cursor:default} + .dshImageStatus{margin:0;font-size:13px;line-height:20px}.dshImageError{color:var(--dsw-alias-state-error-primary)} + .dshImageCard :focus-visible{outline:2px solid var(--dsw-alias-brand-primary);outline-offset:2px} + ` + async function api(path, options = {}) { + const response = await fetch(`/api/image-generation.${path}`, { ...options, cache: 'no-store' }) + let data + try { data = await response.json() } catch { throw { code: 'RESPONSE' } } + if (!response.ok) throw data + return data + } + function ImageCard({ t }) { + const [expanded, setExpanded] = React.useState(false) + const [saved, setSaved] = React.useState(null) + const [drafts, setDrafts] = React.useState({}) + const [provider, setProvider] = React.useState('bytedance') + const [status, setStatus] = React.useState('') + const [error, setError] = React.useState('') + const [loading, setLoading] = React.useState(true) + const [busy, setBusy] = React.useState(false) + const inFlight = React.useRef(false) + const lifetime = React.useRef(null) + const id = React.useId() + const load = React.useCallback(async signal => { + setLoading(true); setError('') + try { + const result = await api('settings', { signal }) + setSaved(result); setProvider(result.provider) + setDrafts(Object.fromEntries(Object.entries(result.profiles).map(([key, value]) => [key, { ...value, apiKey: '' }]))) + } catch (error) { if (!signal.aborted) setError(error.code || 'UNAVAILABLE') } + finally { if (!signal.aborted) setLoading(false) } + }, []) + React.useEffect(() => { + const controller = new AbortController(); lifetime.current = controller + void load(controller.signal) + return () => controller.abort() + }, [load]) + const draft = drafts[provider] + const edit = (key, value) => { + setDrafts(previous => ({ ...previous, [provider]: { ...previous[provider], [key]: value } })) + setStatus(''); setError('') + } + const save = async event => { + event.preventDefault() + if (inFlight.current || !saved) return + inFlight.current = true; setBusy(true); setError(''); setStatus('') + try { + const result = await api('save', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, signal: lifetime.current.signal, + body: JSON.stringify({ revision: saved.revision, provider, model: draft.model, baseUrl: draft.baseUrl, apiKey: draft.apiKey }), + }) + setSaved(result) + setDrafts(previous => ({ ...previous, [provider]: { ...result.profiles[provider], apiKey: '' } })) + setStatus('saved') + } catch (error) { if (!lifetime.current.signal.aborted) setError(error.code || 'UNAVAILABLE') } + finally { inFlight.current = false; if (!lifetime.current.signal.aborted) setBusy(false) } + } + const field = (key, label, type = 'text', placeholder) => h('label', { className: 'dshImageField', key }, + t(label), h('input', { name: key, type, value: draft[key], placeholder, autoComplete: 'off', spellCheck: false, onChange: event => edit(key, event.target.value) })) + return h('li', { className: 'dshImageCard', 'data-testid': 'image-generation-card' }, + h('button', { className: 'dshImageHeader', type: 'button', 'aria-expanded': expanded, 'aria-controls': `${id}-body`, onClick: () => setExpanded(value => !value) }, + h('span', { className: 'dshImageHeading' }, h('span', { className: 'dshImageTitle' }, t('title')), h('span', { className: 'dshImageDescription' }, t('description'))), + saved?.profiles[saved.provider]?.configured && h('span', { className: 'dshImageBadge' }, t('configured')), + h('span', { className: 'dshImageChevron', 'aria-hidden': true })), + expanded && h('form', { id: `${id}-body`, className: 'dshImageBody', onSubmit: save, 'aria-busy': busy || loading }, + loading ? h('p', { className: 'dshImageHint' }, t('loading')) : draft && h(React.Fragment, null, + h('fieldset', { className: 'dshImageFields', disabled: busy || !saved.writable }, + h('label', { className: 'dshImageField' }, t('provider'), h('select', { name: 'provider', value: provider, onChange: event => { setProvider(event.target.value); setStatus(''); setError('') } }, + h('option', { value: 'bytedance' }, t('bytedance')), h('option', { value: 'openai' }, t('openai')))), + field('apiKey', 'apiKey', 'password', t(draft.configured ? 'savedKey' : 'keyPlaceholder')), + h('a', { className: 'dshImageLink', href: provider === 'openai' ? 'https://platform.openai.com/api-keys' : 'https://console.volcengine.com/ark/apiKey', target: '_blank', rel: 'noreferrer' }, t('keyLink')), + h('details', { className: 'dshImageAdvanced' }, h('summary', null, t('advanced')), field('model', 'model'), field('baseUrl', 'baseUrl'))), + h('p', { className: 'dshImageHint' }, t('defaults')), + h('div', { className: 'dshImageActions' }, h('button', { className: 'dshImageSave', type: 'submit', disabled: busy || !saved.writable || (!draft.apiKey.trim() && !draft.configured) }, t(busy ? 'saving' : 'save')), + status && h('p', { className: 'dshImageStatus', role: 'status' }, t(status))), + h('p', { className: 'dshImageHint' }, t(saved.writable ? 'hint' : 'readOnly'))), + error && h('p', { className: 'dshImageStatus dshImageError', role: 'alert' }, t(Object.hasOwn(en, error) ? error : 'UNAVAILABLE')), + !loading && !saved && h('button', { className: 'dshImageSave', type: 'button', onClick: () => load(lifetime.current.signal) }, t('reload')))) + } + return { + inject: ['slots', 'locale'], + apply(ctx) { + ctx.effect(() => ctx.locale.register(NS, { zh, en }), 'image-generation locale') + ctx.effect(() => { + const style = document.createElement('style'); style.dataset.pluginCss = 'dsh-image-generation'; style.textContent = css + document.head.appendChild(style) + return () => style.remove() + }, 'image-generation styles') + ctx.slots.inject('settings.plugin.item', () => ctx.slots.register({ name: 'settings.plugin.item', key: 'image-generation', order: 100, locale: NS }, ImageCard)) + }, + } + }, +}) diff --git a/packages/dsh-image-generation/cordis.patch.yml b/packages/dsh-image-generation/cordis.patch.yml new file mode 100644 index 000000000..88b9c1a9d --- /dev/null +++ b/packages/dsh-image-generation/cordis.patch.yml @@ -0,0 +1,3 @@ +- insert: + - id: dsh-image-generation + name: dsh-image-generation diff --git a/packages/dsh-image-generation/index.js b/packages/dsh-image-generation/index.js new file mode 100644 index 000000000..494017856 --- /dev/null +++ b/packages/dsh-image-generation/index.js @@ -0,0 +1,92 @@ +import { readFile } from 'node:fs/promises' +import { fileURLToPath } from 'node:url' +import z from '@deepseek-ai/schemastery' +import { defineTool } from '@deepseek-ai/dsh-tools' +import { BUNDLED_SKILL_RANK } from '@deepseek-ai/dsh-skill' +import { createSettings } from './lib/settings.js' +import { generate, readBounded, safeError } from './lib/provider.js' +import { assetDirectory, normalizeImage, workspaceFor } from './lib/assets.js' +import { commitImage, writerPlan } from './lib/commit.js' + +export const name = 'dsh-image-generation' +export const inject = ['settings', 'credentials', 'connection', 'tools', 'skills', 'systemPrompt', 'sandboxPolicy', 'sandbox', 'subprocess'] +export const Config = z.object({}) + +export function imageTool(ctx, settings) { + return defineTool({ + name: 'image_generate', + description: 'Generate one photo, illustration or background through the configured image provider and save an Office-compatible PNG in the session workspace. Load the generate-image Skill first. Keep charts, tables and simple diagrams editable.', + parameters: { + prompt: { type: 'string', required: true, description: 'Visual subject, composition, palette, lighting, and space for document or slide text.' }, + style_context: { type: 'string', description: 'Shared brand, template and illustration style for consistency across the document.' }, + purpose: { type: 'string', enum: ['general', 'presentation', 'document', 'web', 'marketing'] }, + aspect_ratio: { type: 'string', enum: ['1:1', '16:9', '9:16', '4:3', '3:4'], description: 'Target composition; actual dimensions are returned. OpenAI uses its closest supported canvas.' }, + }, + output: { + schema: { type: 'object', additionalProperties: false, properties: { + asset_id: { type: 'string', required: true }, workspace_path: { type: 'string', required: true }, + media_type: { type: 'string', required: true }, width: { type: 'integer', required: true }, height: { type: 'integer', required: true }, + bytes: { type: 'integer', required: true }, sha256: { type: 'string', required: true }, + provider: { type: 'string', required: true }, model: { type: 'string', required: true }, + } }, + render: (_args, value) => [{ type: 'text', text: JSON.stringify(value) }], + }, + timeoutMs: 240_000, + isConcurrencySafe: () => true, + presentCall: args => ({ card: 'generic', kind: 'execute', title: 'Generate image', rawInput: args }), + async execute(args, exec) { + try { + const root = await workspaceFor(ctx, exec) + await assetDirectory(root) + writerPlan(ctx, root, exec) + const spec = await settings.active() + ctx.logger.info('image-generation: generation started; provider=%s', spec.provider) + const raw = await generate(spec.provider, spec, spec.key, args, { signal: exec.signal }) + const image = await normalizeImage(raw) + // Recheck live session permissions after a potentially long provider call. + if (await workspaceFor(ctx, exec) !== root) throw new Error('workspace changed') + const asset = await commitImage(ctx, root, image.data, exec) + ctx.logger.info('image-generation: generation saved; provider=%s sha256=%s', spec.provider, asset.sha256) + return { ...asset, asset_id: `sha256:${asset.sha256}`, media_type: 'image/png', width: image.width, height: image.height, bytes: image.data.length, provider: spec.provider, model: spec.model } + } catch (error) { + const safe = safeError(error) + ctx.logger.info('image-generation: generation failed; code=%s', safe.code) + throw safe + } + }, + }) +} + +export async function apply(ctx) { + // The card is discovered through the standard settings namespace ledger. + // Endpoint/model/key are committed together by the credential provider, so a + // failed save cannot pair an old key with a newly persisted endpoint. + ctx.settings.register('image-generation', Config, { applies: 'live' }) + const settings = createSettings(ctx) + for (const [suffix, method] of [['settings', 'GET'], ['save', 'POST']]) { + ctx.connection.fetch.register({ path: `/api/image-generation.${suffix}`, methods: [method], async fetch(request) { + try { + const value = method === 'GET' ? await settings.describe() : await settings.save( + JSON.parse((await readBounded(request, 16_384, request.signal)).toString('utf8')), request.signal, + ) + return Response.json(value, { headers: { 'Cache-Control': 'no-store' } }) + } catch (error) { + const safe = safeError(error) + return Response.json({ code: safe.code, error: safe.message }, { status: safe.status, headers: { 'Cache-Control': 'no-store' } }) + } + } }) + } + ctx.tools.register(imageTool(ctx, settings)) + ctx.on('tools/pre-execute', (exec, next) => exec.name === 'image_generate' + ? Promise.resolve({ kind: 'ask', reason: 'Generate an image with the configured provider and save it in this workspace. Provider usage may be billed.' }) + : next()) + const locator = new URL('./skills/generate-image/SKILL.md', import.meta.url) + const candidate = { + name: 'generate-image', description: 'Create reusable photos, illustrations and backgrounds for presentations, documents and other image requests with the configured image_generate tool.', + invocation: { modelInvocable: true, userInvocable: true }, provider: name, source: 'bundled', rank: BUNDLED_SKILL_RANK, + locator, resourceBase: { kind: 'directory', path: fileURLToPath(new URL('.', locator)) }, + } + ctx.skills.registerProvider(() => ({ name, list: async () => [candidate], get: async selected => selected.name === candidate.name + ? { ...candidate, content: (await readFile(locator, 'utf8')).replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/u, '').trim() } : undefined })) + ctx.systemPrompt.section({ name: 'tool:image-generation', order: 114, text: () => 'For image creation, load the generate-image Skill and call image_generate. Reuse its PNG workspace path in PPT and Word. The user configures this shared capability in Settings > Plugins > Image generation.' }) +} diff --git a/packages/dsh-image-generation/lib/assets.js b/packages/dsh-image-generation/lib/assets.js new file mode 100644 index 000000000..1d41c9631 --- /dev/null +++ b/packages/dsh-image-generation/lib/assets.js @@ -0,0 +1,50 @@ +import { lstat, mkdir, realpath } from 'node:fs/promises' +import path from 'node:path' +import sharp from 'sharp' +import { ImageError, MAX_IMAGE_BYTES } from './provider.js' + +export async function workspaceFor(ctx, exec) { + const session = exec.agent?.session + if (!session?.header.cwd) throw new ImageError('WORKSPACE', 'Choose a workspace for the generated image.') + const policy = ctx.sandboxPolicy.resolve({ session }) + if (!['workspace-write', 'danger-full-access'].includes(policy.mode)) throw new ImageError('POLICY', 'Image generation requires workspace write permission.') + const root = await realpath(session.header.cwd) + if (root !== await realpath(policy.workspaceRoot) || !(await lstat(root)).isDirectory()) throw new ImageError('WORKSPACE', 'The session workspace does not match its write policy.') + exec.signal?.throwIfAborted() + return root +} + +/** A predictable subdirectory, checked before the paid request and again before commit. */ +export async function assetDirectory(root, create = false) { + let directory = root + for (const part of ['.workbuddy', 'generated-images']) { + directory = path.join(directory, part) + try { + const info = await lstat(directory) + if (info.isSymbolicLink() || !info.isDirectory() || await realpath(directory) !== directory) throw new ImageError('ASSET_PATH', 'The generated image directory must be a regular workspace directory.') + } catch (error) { + if (error.code !== 'ENOENT') throw error + if (!create) continue + try { await mkdir(directory, { mode: 0o700 }) } catch (error) { if (error.code !== 'EEXIST') throw error } + const info = await lstat(directory) + if (!info.isDirectory() || info.isSymbolicLink() || await realpath(directory) !== directory) throw new ImageError('ASSET_PATH', 'The generated image directory must be a regular workspace directory.') + } + } + return directory +} + +export async function normalizeImage(raw) { + try { + const image = sharp(raw, { limitInputPixels: 64_000_000, failOn: 'warning', animated: false }) + const meta = await image.metadata() + if (!['png', 'jpeg', 'webp'].includes(meta.format) || (meta.pages ?? 1) > 1 || meta.width > 8192 || meta.height > 8192) throw new Error('unsupported image') + // The Office asset preserves alpha and full resolution; attachment storage may + // independently compress the preview. Never label a normalized WebP as PNG. + const { data, info } = await image.rotate().toColourspace('srgb').png().toBuffer({ resolveWithObject: true }) + if (data.length > MAX_IMAGE_BYTES) throw new ImageError('TOO_LARGE', 'The normalized image exceeds 20 MB.') + return { data, width: info.width, height: info.height } + } catch (error) { + if (error instanceof ImageError) throw error + throw new ImageError('IMAGE', 'The provider returned an unsupported or damaged image.', 502) + } +} diff --git a/packages/dsh-image-generation/lib/commit.js b/packages/dsh-image-generation/lib/commit.js new file mode 100644 index 000000000..0c72b4180 --- /dev/null +++ b/packages/dsh-image-generation/lib/commit.js @@ -0,0 +1,31 @@ +import { createHash } from 'node:crypto' +import { fileURLToPath } from 'node:url' +import { ImageError } from './provider.js' + +export function writerPlan(ctx, root, exec) { + const policy = ctx.sandboxPolicy.resolve({ session: exec.agent.session }) + return ctx.sandbox.confine([process.execPath, fileURLToPath(new URL('./writer.js', import.meta.url))], { + ...policy, mode: 'workspace-write', workspaceRoot: root, + }) +} + +export async function commitImage(ctx, root, data, exec) { + const plan = writerPlan(ctx, root, exec) + const signal = AbortSignal.any([AbortSignal.timeout(30_000), ...(exec.signal ? [exec.signal] : [])]) + // Tombstones clear ordinary parent entries too: no launch credentials or + // NODE_OPTIONS enter the asset writer, even if the host inherited them. + const env = Object.fromEntries(Object.keys(process.env).map(key => [key, undefined])) + if (process.platform === 'win32') env.SystemRoot = process.env.SystemRoot + const child = ctx.subprocess.spawn({ + argv: plan.argv, cwd: root, env, signal, graceMs: 1000, + stdio: { stdin: { data: data.toString('base64') }, stdout: { maxBytes: 1024 }, stderr: { maxBytes: 2048 } }, + }) + const outcome = await child.done + signal.throwIfAborted() + if (outcome.exitCode !== 0) throw new ImageError('ASSET_PATH', 'The sandbox could not save the generated image. Check workspace permissions and sandbox availability.') + let result + try { result = JSON.parse(child.collected.stdout.readFrom(0).text) } catch { throw new ImageError('ASSET_PATH', 'The image writer returned an invalid result.') } + const hash = createHash('sha256').update(data).digest('hex') + if (result.sha256 !== hash || result.workspace_path !== `.workbuddy/generated-images/${hash}.png`) throw new ImageError('ASSET_PATH', 'The generated asset identity does not match its image.') + return result +} diff --git a/packages/dsh-image-generation/lib/provider.js b/packages/dsh-image-generation/lib/provider.js new file mode 100644 index 000000000..dddad33c5 --- /dev/null +++ b/packages/dsh-image-generation/lib/provider.js @@ -0,0 +1,130 @@ +export const DEFAULTS = Object.freeze({ + bytedance: { baseUrl: 'https://ark.cn-beijing.volces.com/api/v3', model: 'doubao-seedream-4-5-251128' }, + openai: { baseUrl: 'https://api.openai.com/v1', model: 'gpt-image-1.5' }, +}) +export const MAX_IMAGE_BYTES = 20 * 1024 * 1024 + +export class ImageError extends Error { + constructor(code, message, status = 400) { super(message); this.code = code; this.status = status } +} +export function safeError(error) { + if (error instanceof ImageError) return error + if (error?.name === 'TimeoutError') return new ImageError('TIMEOUT', 'The image provider timed out. Save again to retry.', 504) + if (error?.name === 'AbortError') return new ImageError('CANCELLED', 'Image operation cancelled.', 499) + return new ImageError('UNAVAILABLE', 'The image service is unavailable. Check the connection and try again.', 502) +} +export function profile(provider, input = {}) { + if (!Object.hasOwn(DEFAULTS, provider)) throw new ImageError('PROVIDER', 'Choose OpenAI or ByteDance.') + const model = String(input.model ?? DEFAULTS[provider].model).trim() + if (!/^[A-Za-z0-9][A-Za-z0-9._:/-]{0,199}$/.test(model)) throw new ImageError('MODEL', 'Enter a valid model ID.') + let url + try { url = new URL(String(input.baseUrl ?? DEFAULTS[provider].baseUrl).trim()) } catch { throw new ImageError('ENDPOINT', 'Enter a valid API base URL.') } + const loopback = ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname) + if ((url.protocol !== 'https:' && !(url.protocol === 'http:' && loopback)) || url.username || url.password || url.search || url.hash) { + throw new ImageError('ENDPOINT', 'Use an HTTPS API base URL without credentials, query parameters or fragments.') + } + if (/\/images\/generations\/?$/.test(url.pathname)) throw new ImageError('ENDPOINT', 'Enter the API base URL ending in /v1 or /api/v3.') + return { baseUrl: url.href.replace(/\/+$/, ''), model } +} + +// Bound the stream before parsing; a Content-Length header alone is insufficient. +export async function readBounded(response, maxBytes, signal) { + if (!response.body) throw new ImageError('RESPONSE', 'The provider returned an empty response.', 502) + const reader = response.body.getReader() + const chunks = []; let bytes = 0 + try { + while (true) { + signal?.throwIfAborted() + const part = await reader.read() + if (part.done) break + bytes += part.value.byteLength + if (bytes > maxBytes) throw new ImageError('TOO_LARGE', 'The provider response exceeds the image size limit.', 502) + chunks.push(part.value) + } + return Buffer.concat(chunks, bytes) + } finally { await reader.cancel().catch(() => {}); reader.releaseLock() } +} + +async function request(url, key, { signal, body, maxBytes = 2 * 1024 * 1024, fetchImpl = fetch, parameterProbe = false } = {}) { + try { + const response = await fetchImpl(url, { + method: body ? 'POST' : 'GET', redirect: 'error', signal, + headers: { Authorization: `Bearer ${key}`, ...(body ? { 'Content-Type': 'application/json' } : {}) }, + ...(body ? { body: JSON.stringify(body) } : {}), + }) + if (!response.ok) { + if (parameterProbe && response.status === 400) { + const raw = await readBounded(response, 64 * 1024, signal) + let error + try { error = JSON.parse(raw.toString('utf8')).error } catch { /* fail closed below */ } + // Ark has no documented runtime Models API. A request with the required + // prompt omitted reaches authentication/parameter validation without + // submitting an inference job. Accept only the prompt-required error. + const code = error?.code + const message = String(error?.message ?? '') + if (/^(?:MissingParameter|InvalidParameter)(?:\.[A-Za-z]+)?$/.test(code ?? '') + && (error?.param === 'prompt' || /\bprompt\b/i.test(message)) + && /required|missing|empty|not provided|not set|不能为空|必填|缺少/i.test(message)) return { probe: 'connection' } + throw new ImageError('PARAMETERS', 'The provider did not confirm the expected parameter check. Verify the API URL and model.', 502) + } + await response.body?.cancel() + const errors = { + 401: ['AUTH', 'The API key is invalid or expired.'], + 403: ['PERMISSION', 'This API key lacks access. Enable the model and check account verification.'], + 404: ['MODEL', 'The model or API endpoint is unavailable.'], + 429: ['QUOTA', 'The provider quota or rate limit has been reached.'], + 400: ['PARAMETERS', 'The provider rejected the request. Check the model and supported image parameters.'], + } + const [code, message] = errors[response.status] ?? ['PROVIDER_ERROR', 'The image provider could not complete the request.'] + throw new ImageError(code, message, 502) + } + const data = await readBounded(response, maxBytes, signal) + try { return JSON.parse(data.toString('utf8')) } catch { throw new ImageError('RESPONSE', 'The provider returned invalid JSON.', 502) } + } catch (error) { throw safeError(error) } +} + +/** One non-generating request: OpenAI model metadata or Ark required-prompt validation. */ +export async function validateConnection(provider, spec, key, options = {}) { + const signal = AbortSignal.any([AbortSignal.timeout(15_000), ...(options.signal ? [options.signal] : [])]) + if (provider === 'openai') { + const result = await request(`${spec.baseUrl}/models/${encodeURIComponent(spec.model)}`, key, { ...options, signal }) + if (result.id !== spec.model) throw new ImageError('MODEL', 'The requested model was not returned by the provider.') + return 'model' + } + const result = await request(`${spec.baseUrl}/images/generations`, key, { ...options, signal, body: { model: spec.model }, parameterProbe: true }) + if (result.probe === 'connection') return 'connection' + throw new ImageError('RESPONSE', 'The provider returned an unexpected validation response.', 502) +} + +export function generationBody(provider, spec, args) { + const prompt = typeof args.prompt === 'string' ? args.prompt.trim() : '' + const style = typeof args.style_context === 'string' ? args.style_context.trim() : '' + if (!prompt || prompt.length + style.length > 8000) throw new ImageError('PROMPT', 'Enter an image description of up to 8000 characters including style context.') + const ratio = args.aspect_ratio ?? '1:1' + const sizes = provider === 'openai' + ? { '1:1': '1024x1024', '16:9': '1536x1024', '9:16': '1024x1536', '4:3': '1536x1024', '3:4': '1024x1536' } + : { '1:1': '2048x2048', '16:9': '2560x1440', '9:16': '1440x2560', '4:3': '2304x1728', '3:4': '1728x2304' } + if (!sizes[ratio]) throw new ImageError('RATIO', 'Choose a supported image aspect ratio.') + return { + model: spec.model, + prompt: [prompt, style && `Art direction: ${style}`, args.purpose && `Intended use: ${args.purpose}`, `Compose for ${ratio}. Keep important subjects inside the safe central area for cropping.`].filter(Boolean).join('\n\n'), + size: sizes[ratio], + ...(provider === 'openai' + ? { n: 1, quality: 'auto', output_format: 'png' } + : { response_format: 'b64_json', sequential_image_generation: 'disabled', watermark: false }), + } +} + +export async function generate(provider, spec, key, args, options = {}) { + const body = generationBody(provider, spec, args) + const signal = AbortSignal.any([AbortSignal.timeout(180_000), ...(options.signal ? [options.signal] : [])]) + const result = await request(`${spec.baseUrl}/images/generations`, key, { ...options, signal, body, maxBytes: Math.ceil(MAX_IMAGE_BYTES * 4 / 3) + 1024 * 1024 }) + const b64 = result?.data?.[0]?.b64_json + if (typeof b64 !== 'string' || b64.length === 0 || b64.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}$/.test(b64)) { + throw new ImageError('IMAGE', 'The provider returned no valid base64 image.', 502) + } + const bytes = Buffer.from(b64, 'base64') + if (bytes.length > MAX_IMAGE_BYTES) throw new ImageError('TOO_LARGE', 'The generated image exceeds 20 MB.', 502) + signal.throwIfAborted() + return bytes +} diff --git a/packages/dsh-image-generation/lib/settings.js b/packages/dsh-image-generation/lib/settings.js new file mode 100644 index 000000000..111c6bf9f --- /dev/null +++ b/packages/dsh-image-generation/lib/settings.js @@ -0,0 +1,67 @@ +import { credentialKey } from '@deepseek-ai/dsh-credentials' +import { DEFAULTS, ImageError, profile, safeError, validateConnection } from './provider.js' + +const KEY = credentialKey('dsh-image-generation', 'configuration') +const EMPTY = () => ({ revision: 0, provider: 'bytedance', profiles: {} }) + +/** Keep each validated endpoint/model/key together in one atomic host credential record. */ +export function createSettings(ctx, validate = validateConnection) { + async function read() { + const record = await ctx.credentials.readRecord(KEY) + if (!record) return EMPTY() + if (record.kind !== 'grant' || !Number.isSafeInteger(record.payload?.revision) || !record.payload?.profiles) { + throw new ImageError('CONFIGURATION', 'The stored image configuration needs to be saved again.', 500) + } + return record.payload + } + async function describe() { + const state = await read() + const info = await ctx.credentials.describeRecord(KEY) + return { + revision: state.revision, provider: state.provider, writable: info.writable, + profiles: Object.fromEntries(Object.entries(DEFAULTS).map(([provider, defaults]) => { + const stored = state.profiles[provider] + return [provider, { ...defaults, ...(stored ? profile(provider, stored) : {}), configured: Boolean(stored?.key), validation: stored?.validation ?? null }] + })), + } + } + async function active() { + const state = await read() + const spec = state.profiles[state.provider] + if (!spec?.key) throw new ImageError('NOT_CONFIGURED', 'Configure the image tool in Settings > Plugins before generating an image.') + return { provider: state.provider, ...profile(state.provider, spec), key: spec.key } + } + async function save(input, signal) { + if (!input || typeof input !== 'object' || Array.isArray(input)) throw new ImageError('CONFIGURATION', 'Enter a valid image configuration.') + const before = await read() + if (!Number.isSafeInteger(input.revision) || input.revision !== before.revision) throw new ImageError('CONFLICT', 'The settings changed in another window. Reopen this card before saving.', 409) + const spec = profile(input.provider, input) + const previous = before.profiles[input.provider] + const key = typeof input.apiKey === 'string' ? input.apiKey.trim() : '' + if (!key && previous && new URL(previous.baseUrl).origin !== new URL(spec.baseUrl).origin) { + throw new ImageError('KEY_REQUIRED', 'Enter the API key again for the new API origin.') + } + const effectiveKey = key || previous?.key + if (!effectiveKey || effectiveKey.length > 4096 || /[\s\x00-\x1f\x7f]/u.test(effectiveKey)) throw new ImageError('KEY_REQUIRED', 'Enter a valid API key.') + if (!(await ctx.credentials.describeRecord(KEY)).writable) throw new ImageError('READ_ONLY', 'The host credential store is read-only.', 403) + try { + const validation = await validate(input.provider, spec, effectiveKey, { signal }) + signal?.throwIfAborted() + await ctx.credentials.modifyRecord(KEY, async record => { + const latest = record?.kind === 'grant' ? record.payload : EMPTY() + if (latest.revision !== before.revision) throw new ImageError('CONFLICT', 'The settings changed in another window. Reopen this card before saving.', 409) + return { kind: 'grant', payload: { + revision: before.revision + 1, provider: input.provider, + profiles: { ...before.profiles, [input.provider]: { ...spec, key: effectiveKey, validation } }, + } } + }) + ctx.logger.info('image-generation: configuration saved; provider=%s validation=%s', input.provider, validation) + return await describe() + } catch (error) { + const safe = safeError(error) + ctx.logger.info('image-generation: configuration save failed; provider=%s code=%s', input.provider, safe.code) + throw safe + } + } + return { describe, active, save } +} diff --git a/packages/dsh-image-generation/lib/storage.js b/packages/dsh-image-generation/lib/storage.js new file mode 100644 index 000000000..330b0db97 --- /dev/null +++ b/packages/dsh-image-generation/lib/storage.js @@ -0,0 +1,30 @@ +import { createHash, randomUUID } from 'node:crypto' +import { constants } from 'node:fs' +import { open, link, unlink } from 'node:fs/promises' +import path from 'node:path' +import { assetDirectory } from './assets.js' +import { ImageError } from './provider.js' + +export async function materialize(root, data, signal) { + signal?.throwIfAborted() + const directory = await assetDirectory(root, true) + const sha256 = createHash('sha256').update(data).digest('hex') + const filename = `${sha256}.png` + const target = path.join(directory, filename) + const temporary = path.join(directory, `.${randomUUID()}.tmp`) + const handle = await open(temporary, 'wx', 0o600) + try { + await handle.writeFile(data, { signal }); await handle.sync(); await handle.close() + await assetDirectory(root) + signal?.throwIfAborted() + try { await link(temporary, target) } catch (error) { + if (error.code !== 'EEXIST') throw error + const existing = await open(target, constants.O_RDONLY | constants.O_NOFOLLOW) + try { + const info = await existing.stat() + if (!info.isFile() || info.size !== data.length || createHash('sha256').update(await existing.readFile()).digest('hex') !== sha256) throw new ImageError('ASSET_PATH', 'The image asset path contains different content.') + } finally { await existing.close() } + } + } finally { await handle.close().catch(() => {}); await unlink(temporary).catch(() => {}) } + return { workspace_path: `.workbuddy/generated-images/${filename}`, sha256 } +} diff --git a/packages/dsh-image-generation/lib/writer.js b/packages/dsh-image-generation/lib/writer.js new file mode 100644 index 000000000..adb6a7a7d --- /dev/null +++ b/packages/dsh-image-generation/lib/writer.js @@ -0,0 +1,13 @@ +// This child receives image bytes only. The Host wraps it with its configured +// OS sandbox; filesystem confinement remains effective during path-swap races. +import { materialize } from './storage.js' +import { MAX_IMAGE_BYTES } from './provider.js' +const chunks = []; let length = 0 +for await (const chunk of process.stdin) { + length += chunk.length + if (length > Math.ceil(MAX_IMAGE_BYTES * 4 / 3) + 4) throw new Error('Image input too large') + chunks.push(chunk) +} +const data = Buffer.from(Buffer.concat(chunks).toString('utf8'), 'base64') +if (!data.length || data.length > MAX_IMAGE_BYTES) throw new Error('Invalid image input') +process.stdout.write(JSON.stringify(await materialize(process.cwd(), data))) diff --git a/packages/dsh-image-generation/package.json b/packages/dsh-image-generation/package.json new file mode 100644 index 000000000..6c542f99a --- /dev/null +++ b/packages/dsh-image-generation/package.json @@ -0,0 +1,30 @@ +{ + "name": "dsh-image-generation", + "version": "0.1.0", + "description": "Shared OpenAI and ByteDance image generation for DSH conversations, presentations and documents.", + "type": "module", + "main": "./index.js", + "exports": { ".": "./index.js", "./client": "./client.js", "./package.json": "./package.json", "./cordis.patch.yml": "./cordis.patch.yml" }, + "files": ["index.js", "client.js", "lib", "skills", "cordis.patch.yml", "README.md", "LICENSE"], + "license": "MIT", + "publishConfig": { "access": "public" }, + "repository": { "type": "git", "url": "git+https://github.com/dataelement/dsh-desktop.git", "directory": "packages/dsh-image-generation" }, + "keywords": ["dsh-plugin", "image-generation", "seedream", "openai"], + "dsh": { + "bundle": { "patch": "./cordis.patch.yml" }, + "client": { "inject": ["@deepseek-ai/dsh-client-locale", "@deepseek-ai/dsh-client-ui-settings-plugins"], "platform": "web" } + }, + "dependencies": { "@deepseek-ai/schemastery": "^3.18.2", "sharp": "^0.35.4" }, + "peerDependencies": { + "@deepseek-ai/cordis": "^4.0.2", + "@deepseek-ai/dsh-client-connection": "^0.1.2-rc.1", + "@deepseek-ai/dsh-credentials": "^0.1.2-rc.1", + "@deepseek-ai/dsh-sandbox-policy": "^0.1.2-rc.1", + "@deepseek-ai/dsh-sandbox": "^0.1.2-rc.1", + "@deepseek-ai/dsh-subprocess": "^0.1.2-rc.1", + "@deepseek-ai/dsh-settings": "^0.1.2-rc.1", + "@deepseek-ai/dsh-skill": "^0.1.2-rc.1", + "@deepseek-ai/dsh-system-prompt": "^0.1.2-rc.1", + "@deepseek-ai/dsh-tools": "^0.1.2-rc.1" + } +} diff --git a/packages/dsh-image-generation/skills/generate-image/SKILL.md b/packages/dsh-image-generation/skills/generate-image/SKILL.md new file mode 100644 index 000000000..8d2d63a2c --- /dev/null +++ b/packages/dsh-image-generation/skills/generate-image/SKILL.md @@ -0,0 +1,14 @@ +--- +name: generate-image +description: Generate reusable photos, illustrations and backgrounds for presentations, documents and other image requests. +--- + +Use `image_generate` whenever a task needs a generated raster visual. The user configures OpenAI or ByteDance once in Settings → Plugins → Image generation. The tool selects that provider. Credentials remain on the host. + +1. Identify the image's purpose and placement. For PPT/Word, inspect the relevant document or template first. Carry its palette, art direction and whitespace needs into `style_context`, and reuse that style across the document. +2. Write a complete prompt: subject, composition, setting, lighting, palette and required details. Specify room for titles or captions where the document needs it. Keep readable document text as native text; use native editable objects for charts, tables, flowcharts and simple diagrams. +3. Choose the target `aspect_ratio` and `purpose`. A 16:9 slide does not require a 16:9 image when the illustration occupies only part of the slide. The tool returns actual dimensions; use proportional scaling or an intentional crop, preserving the main subject. +4. Call `image_generate` once per needed visual. Inspect the resulting PNG with the available image viewer before placing it. Correct specific visible defects if another generation is needed. Reuse an existing suitable image instead of generating duplicates. +5. Insert the returned `workspace_path` into the PPT/Word workflow, then inspect the rendered document page. Keep visual assets linked to their purpose and reuse their paths across later edits. Return the document and any requested standalone image. + +If the tool reports missing configuration, direct the user to Settings → Plugins → Image generation. Never request an API key in conversation or place one in a document, prompt, sandbox command or source file. Respect tool approval, cancellation and workspace policy. Surface provider errors accurately; use a placeholder only when the user accepts one. Configuration validation checks the connection without submitting an image generation job; successful generation is established by an actual generated image. diff --git a/patches/@deepseek-ai+dsh+0.1.2-rc.1.patch b/patches/@deepseek-ai+dsh+0.1.2-rc.1.patch index 6207816b6..622a03183 100644 --- a/patches/@deepseek-ai+dsh+0.1.2-rc.1.patch +++ b/patches/@deepseek-ai+dsh+0.1.2-rc.1.patch @@ -2,10 +2,11 @@ diff --git a/node_modules/@deepseek-ai/dsh/package.json b/node_modules/@deepseek index 28eaff9..2445f81 100644 --- a/node_modules/@deepseek-ai/dsh/package.json +++ b/node_modules/@deepseek-ai/dsh/package.json -@@ -28,6 +28,11 @@ +@@ -28,6 +28,12 @@ }, "license": "MIT", "dependencies": { ++ "dsh-image-generation": "0.1.0", + "dsh-ppt-composer": "0.1.1-rc.2", + "dsh-desktop-client-ui": "0.1.0", + "dsh-desktop-hmr-fallback": "0.1.0", diff --git a/scripts/verify-image-generation.mjs b/scripts/verify-image-generation.mjs new file mode 100644 index 000000000..c2a1068cc --- /dev/null +++ b/scripts/verify-image-generation.mjs @@ -0,0 +1,99 @@ +/** Real Host/auth/client composition smoke with a local provider, no billable generation. + * node scripts/verify-image-generation.mjs [--keep] + * --keep writes a private browser launch URL under the printed temporary directory. + */ +import assert from 'node:assert/strict' +import { spawn } from 'node:child_process' +import { createServer } from 'node:http' +import { mkdtemp, writeFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { DEFAULTS } from '../packages/dsh-image-generation/lib/provider.js' + +const root = fileURLToPath(new URL('..', import.meta.url)) +const home = await mkdtemp(path.join(tmpdir(), 'dsh-image-host-')) +const calls = [] +const mock = createServer(async (request, response) => { + let body = ''; for await (const chunk of request) body += chunk + calls.push({ method: request.method, path: request.url }) + const valid = request.headers.authorization === 'Bearer smoke-image-key' + const probe = valid && request.url.endsWith('/images/generations') && body && !Object.hasOwn(JSON.parse(body), 'prompt') + assert.ok(!body || !JSON.parse(body).prompt, 'The configuration flow must omit the required generation prompt') + response.writeHead(valid ? probe ? 400 : 200 : 401, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify(valid ? probe ? { error: { code: 'MissingParameter', message: 'The request is missing a required parameter: prompt.' } } : { id: DEFAULTS.openai.model } : { error: { message: 'Invalid API key' } })) +}) +await new Promise((resolve, reject) => { mock.once('error', reject); mock.listen(0, '127.0.0.1', resolve) }) +const mockBase = `http://127.0.0.1:${mock.address().port}/v1` +const portServer = createServer() +await new Promise(resolve => portServer.listen(0, '127.0.0.1', resolve)) +const port = portServer.address().port +await new Promise(resolve => portServer.close(resolve)) +const base = `http://127.0.0.1:${port}` +const node = process.env.IMAGE_SMOKE_NODE || path.join(root, 'node_modules/node/bin/node') +const bin = process.env.IMAGE_SMOKE_DSH || path.join(root, 'node_modules/@deepseek-ai/dsh/lib/bin.js') +const patch = process.env.IMAGE_SMOKE_PATCH || path.join(root, 'build/dsh-desktop.patch.yml') +const child = spawn(node, [bin, 'web', '--patch', patch, '--no-open', '--host', '127.0.0.1', '--port', String(port)], { + cwd: root, env: { ...process.env, DSH_HOME: home, NO_COLOR: '1', DSH_TELEMETRY_DISABLED: '1' }, stdio: ['ignore', 'pipe', 'pipe'], +}) +let output = ''; let cleaned = false +async function cleanup() { + if (cleaned) return; cleaned = true + child.kill('SIGTERM'); mock.closeAllConnections(); mock.close() + await rm(home, { recursive: true, force: true }) +} +process.on('SIGINT', () => { void cleanup().then(() => process.exit(0)) }) +process.on('SIGTERM', () => { void cleanup().then(() => process.exit(0)) }) +try { + const url = await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('Host startup timed out')), 60_000) + const scan = buffer => { + output += buffer.toString() + const match = /dsh web:\s*(\S+)/.exec(output) + if (match) { clearTimeout(timer); resolve(match[1]) } + } + child.stdout.on('data', scan); child.stderr.on('data', scan) + child.once('exit', code => { clearTimeout(timer); reject(new Error(`Host exited ${code}`)) }) + }) + assert.equal((await fetch(`${base}/api/image-generation.settings`)).status, 401) + const exchange = await fetch(url, { redirect: 'manual' }) + const cookie = exchange.headers.getSetCookie().map(value => value.split(';')[0]).join('; ') + assert.ok(cookie) + const api = async (suffix, body, extra = {}) => fetch(`${base}/api/image-generation.${suffix}`, { + method: body ? 'POST' : 'GET', headers: { Cookie: cookie, ...(body ? { 'Content-Type': 'application/json' } : {}), ...extra }, + ...(body ? { body: JSON.stringify(body) } : {}), + }) + const initialResponse = await api('settings') + assert.equal(initialResponse.status, 200) + assert.equal(initialResponse.headers.get('cache-control'), 'no-store') + let state = await initialResponse.json() + assert.equal(state.profiles.openai.configured, false) + assert.equal((await api('save', { revision: 0 }, { Origin: 'https://evil.example' })).status, 403) + for (const provider of ['openai', 'bytedance']) { + const response = await api('save', { revision: state.revision, provider, baseUrl: mockBase, model: DEFAULTS[provider].model, apiKey: 'smoke-image-key' }) + const result = await response.json() + assert.equal(response.status, 200, JSON.stringify(result)) + assert.equal(result.profiles[provider].configured, true) + assert.ok(!JSON.stringify(result).includes('smoke-image-key')) + state = result + } + const failure = await api('save', { revision: state.revision, provider: 'bytedance', baseUrl: mockBase, model: DEFAULTS.bytedance.model, apiKey: 'wrong-key' }) + assert.equal((await failure.json()).code, 'AUTH') + assert.deepEqual(await (await api('settings')).json(), state) + assert.equal(calls.length, 3) + assert.deepEqual(calls.map(call => call.method), ['GET', 'POST', 'POST']) + const page = await (await fetch(base, { headers: { Cookie: cookie } })).text() + assert.ok(page.includes('dsh-image-generation'), 'Image client entry must be present in the composed page') + assert.ok(!/image-generation.*(?:failed|Error)/i.test(output), 'Image plugin must load successfully') + console.log('PASS: Host composition, authentication, origin checks, redacted settings, save success/failure, exactly one non-generating request per save.') + if (process.argv.includes('--keep')) { + await writeFile(path.join(home, 'browser-url.txt'), url, { mode: 0o600 }) + await writeFile(path.join(home, 'smoke-context.json'), JSON.stringify({ base, mockBase }), { mode: 0o600 }) + console.log(`Browser fixture ready: ${home}`) + await new Promise(() => {}) + } +} catch (error) { + console.error(error) + console.error(output.replace(/([?&]token=)[^\s"']+/g, '$1[REDACTED]')) + process.exitCode = 1 +} finally { await cleanup() } diff --git a/test/image-generation.test.mjs b/test/image-generation.test.mjs new file mode 100644 index 000000000..7cb28832d --- /dev/null +++ b/test/image-generation.test.mjs @@ -0,0 +1,194 @@ +import { createServer } from 'node:http' +import { mkdtemp, readFile, realpath, rm, symlink, readdir, stat } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { randomUUID, createHash } from 'node:crypto' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { Context } from '@deepseek-ai/cordis' +import { LocalCredentialProvider } from '@deepseek-ai/dsh-credentials-local' +import { FileSettingsProvider } from '@deepseek-ai/dsh-settings-file' +import { ToolRuntime } from '@deepseek-ai/dsh-tools' +import { SystemPrompt } from '@deepseek-ai/dsh-system-prompt' +import { SkillRegistry } from '@deepseek-ai/dsh-skill' +import { LocalSandboxProvider } from '@deepseek-ai/dsh-sandbox-local' +import { LocalSubprocessRuntime } from '@deepseek-ai/dsh-subprocess-local' +import { Session, SessionId } from '@deepseek-ai/dsh-session' +import sharp from 'sharp' +import { apply, imageTool } from '../packages/dsh-image-generation/index.js' +import { createSettings } from '../packages/dsh-image-generation/lib/settings.js' +import { DEFAULTS, generate, generationBody, ImageError, profile, readBounded, validateConnection } from '../packages/dsh-image-generation/lib/provider.js' +import { normalizeImage } from '../packages/dsh-image-generation/lib/assets.js' +import { materialize } from '../packages/dsh-image-generation/lib/storage.js' + +const cleanups = [] +afterEach(async () => { for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); vi.restoreAllMocks() }) +async function temp() { const p = await realpath(await mkdtemp(path.join(tmpdir(), 'dsh-image-test-'))); cleanups.push(() => rm(p, { recursive: true, force: true })); return p } +async function server() { + const calls = [] + const png = await sharp({ create: { width: 32, height: 24, channels: 4, background: '#11223380' } }).png().toBuffer() + let status = 200; let malformed = false; let delay = 0; let redirect + const http = createServer(async (req, res) => { + let body = ''; for await (const chunk of req) body += chunk + calls.push({ url: req.url, method: req.method, auth: req.headers.authorization, body: body && JSON.parse(body) }) + if (redirect) { res.writeHead(307, { Location: redirect }); res.end(); return } + if (delay) await new Promise(resolve => setTimeout(resolve, delay)) + const probe = status === 200 && !malformed && req.url.endsWith('/images/generations') && body && !Object.hasOwn(JSON.parse(body), 'prompt') + res.writeHead(probe ? 400 : status, { 'content-type': 'application/json' }) + res.end(JSON.stringify(probe ? { error: { code: 'MissingParameter', message: 'The request is missing a required parameter: prompt.' } } : status !== 200 ? { error: { message: 'secret-echo-key' } } : malformed ? {} : req.url.endsWith('/images/generations') + ? { data: [{ b64_json: png.toString('base64') }] } + : req.url.endsWith('/models') ? { data: [{ id: DEFAULTS.bytedance.model }] } : { id: DEFAULTS.openai.model })) + }) + await new Promise((resolve, reject) => { http.once('error', reject); http.listen(0, '127.0.0.1', resolve) }) + cleanups.push(() => { http.closeAllConnections(); return new Promise(resolve => http.close(resolve)) }) + return { calls, png, baseUrl: `http://127.0.0.1:${http.address().port}/v1`, setStatus: value => { status = value }, malformed: () => { malformed = true }, delay: value => { delay = value }, redirect: value => { redirect = value } } +} +async function fixture() { + const home = await temp(); const workspace = await temp() + const ctx = new Context() + for (const [plugin, config] of [[SystemPrompt, {}], [ToolRuntime, {}], [SkillRegistry, {}], [LocalSandboxProvider, {}], [LocalSubprocessRuntime, {}], [FileSettingsProvider, { dshHome: home, watch: false }], [LocalCredentialProvider, { dshHome: home, watch: false }]]) { + const fork = ctx.plugin(plugin, config); await fork; cleanups.push(() => fork.dispose()) + } + const log = vi.fn(); let mode = 'workspace-write' + const services = { credentials: ctx.credentials, sandbox: ctx.sandbox, subprocess: ctx.subprocess, logger: { info: log }, sandboxPolicy: { resolve: () => ({ mode, workspaceRoot: workspace }) } } + const settings = createSettings(services) + const id = SessionId(randomUUID()) + const agent = { session: Session.create(id, undefined, { version: 0, id, createdAt: Date.now(), isSeeded: false, cwd: workspace }) } + return { ctx, home, workspace, services, settings, log, agent, setMode: value => { mode = value } } +} +const saveInput = (provider, baseUrl, revision = 0) => ({ provider, baseUrl, model: DEFAULTS[provider].model, apiKey: 'test-image-key', revision }) + +describe('image settings save and provider requests', () => { + it.each(['openai', 'bytedance'])('saves %s with exactly one non-generating request and redacts credentials', async provider => { + const f = await fixture(); const s = await server() + const result = await f.settings.save(saveInput(provider, s.baseUrl)) + expect(s.calls).toHaveLength(1) + expect(s.calls[0]).toMatchObject({ method: provider === 'openai' ? 'GET' : 'POST', auth: 'Bearer test-image-key' }) + if (provider === 'openai') expect(s.calls[0].url).toContain('/models') + else expect(s.calls[0].body).toEqual({ model: DEFAULTS.bytedance.model }) + expect(result.profiles[provider]).toMatchObject({ configured: true, validation: provider === 'openai' ? 'model' : 'connection' }) + expect(JSON.stringify(result)).not.toContain('test-image-key') + expect(JSON.stringify(f.log.mock.calls)).not.toContain('test-image-key') + expect((await f.settings.active()).provider).toBe(provider) + expect((await stat(path.join(f.home, '.credentials.yaml'))).mode & 0o777).toBe(0o600) + expect(await readdir(f.workspace)).toEqual([]) + }) + it('preserves the whole saved profile after validation fails and keeps provider credentials separate', async () => { + const f = await fixture(); const s = await server() + await f.settings.save(saveInput('openai', s.baseUrl)) + s.setStatus(401) + await expect(f.settings.save({ ...saveInput('bytedance', s.baseUrl, 1), apiKey: 'secret-echo-key' })).rejects.toMatchObject({ code: 'AUTH' }) + expect(await f.settings.active()).toMatchObject({ provider: 'openai', key: 'test-image-key' }) + expect(await readFile(path.join(f.home, '.credentials.yaml'), 'utf8')).not.toContain('secret-echo-key') + s.setStatus(200) + await f.settings.save({ ...saveInput('bytedance', s.baseUrl, 1), apiKey: 'byte-key' }) + await f.settings.save({ ...saveInput('openai', s.baseUrl, 2), apiKey: '' }) + expect(s.calls.at(-1).auth).toBe('Bearer test-image-key') + }) + it('prevents lost updates across two validated saves', async () => { + const f = await fixture(); const s = await server() + const result = await Promise.allSettled([f.settings.save(saveInput('openai', s.baseUrl)), f.settings.save(saveInput('bytedance', s.baseUrl))]) + expect(result.filter(result => result.status === 'fulfilled')).toHaveLength(1) + expect(result.find(result => result.status === 'rejected').reason.code).toBe('CONFLICT') + expect((await f.settings.describe()).revision).toBe(1) + }) + it('requires a re-entered key for a different origin and rejects stale revisions before requests', async () => { + const f = await fixture(); const s = await server() + await f.settings.save(saveInput('openai', s.baseUrl)) + await expect(f.settings.save(saveInput('openai', s.baseUrl))).rejects.toMatchObject({ code: 'CONFLICT' }) + await expect(f.settings.save({ ...saveInput('openai', 'https://example.com/v1', 1), apiKey: '' })).rejects.toMatchObject({ code: 'KEY_REQUIRED' }) + expect(s.calls).toHaveLength(1) + }) + it('handles malformed metadata, cancellation, and custom Ark endpoint validation honestly', async () => { + const s = await server() + expect(await validateConnection('bytedance', { baseUrl: s.baseUrl, model: 'ep-custom' }, 'key')).toBe('connection') + s.malformed() + await expect(validateConnection('bytedance', { baseUrl: s.baseUrl, model: 'ep-custom' }, 'key')).rejects.toMatchObject({ code: 'RESPONSE' }) + const cancelled = new AbortController(); cancelled.abort() + await expect(validateConnection('openai', { baseUrl: s.baseUrl, model: DEFAULTS.openai.model }, 'key', { signal: cancelled.signal })).rejects.toMatchObject({ code: 'CANCELLED' }) + }) + it('rejects redirects without forwarding a key to the redirect target', async () => { + const source = await server(); const target = await server() + source.redirect(target.baseUrl) + await expect(validateConnection('openai', { ...profile('openai'), baseUrl: source.baseUrl }, 'key')).rejects.toMatchObject({ code: 'UNAVAILABLE' }) + expect(source.calls).toHaveLength(1); expect(target.calls).toHaveLength(0) + }) + it('accepts only the specific Ark missing-prompt response and rejects unrelated 400 errors', async () => { + const s = await server(); s.setStatus(400) + await expect(validateConnection('bytedance', { ...profile('bytedance'), baseUrl: s.baseUrl }, 'key')).rejects.toMatchObject({ code: 'PARAMETERS' }) + expect(s.calls).toHaveLength(1) + expect(s.calls[0].body).toEqual({ model: DEFAULTS.bytedance.model }) + }) + it('leaves credentials untouched when an in-flight validation is cancelled', async () => { + const f = await fixture(); const s = await server(); s.delay(50) + const controller = new AbortController() + const pending = f.settings.save(saveInput('openai', s.baseUrl), controller.signal) + setTimeout(() => controller.abort(), 10) + await expect(pending).rejects.toMatchObject({ code: 'CANCELLED' }) + expect((await f.settings.describe()).revision).toBe(0) + }) + it.each(['https://user:key@example.com/v1', 'http://example.com/v1', 'https://example.com/v1?key=x', 'https://example.com/v1/images/generations'])('rejects unsafe or mistaken endpoint %s', baseUrl => { + expect(() => profile('openai', { baseUrl })).toThrow(ImageError) + }) + it('maps provider canvas parameters independently and bounds streamed responses', async () => { + const args = { prompt: 'A mountain', aspect_ratio: '16:9' } + expect(generationBody('openai', profile('openai'), args)).toMatchObject({ size: '1536x1024', output_format: 'png' }) + expect(generationBody('openai', profile('openai'), args)).not.toHaveProperty('response_format') + expect(generationBody('bytedance', profile('bytedance'), args)).toMatchObject({ size: '2560x1440', response_format: 'b64_json' }) + expect(generationBody('bytedance', profile('bytedance'), args)).not.toHaveProperty('quality') + await expect(readBounded(new Response('too much'), 2)).rejects.toMatchObject({ code: 'TOO_LARGE' }) + }) +}) + +describe('image tool and durable Office assets', () => { + it.each(['openai', 'bytedance'])('executes %s through ToolRuntime and writes a verifiable PNG', async provider => { + const f = await fixture(); const s = await server() + await f.settings.save(saveInput(provider, s.baseUrl)) + f.ctx.tools.register(imageTool(f.services, f.settings)) + const result = await f.ctx.tools.execute({ callId: 'test-call', name: 'image_generate', arguments: { prompt: 'A calm forest', purpose: 'presentation', aspect_ratio: '16:9' }, agent: f.agent, signal: new AbortController().signal }) + expect(result.isError, JSON.stringify(result)).toBeFalsy() + const value = JSON.parse(result.content[0].text) + const data = await readFile(path.join(f.workspace, value.workspace_path)) + expect(createHash('sha256').update(data).digest('hex')).toBe(value.sha256) + expect(await sharp(data).metadata()).toMatchObject({ format: 'png', width: value.width, height: value.height, hasAlpha: true }) + expect(s.calls.filter(call => call.body?.prompt)).toHaveLength(1) + expect(JSON.stringify(result)).not.toContain('test-image-key') + }) + it('blocks a read-only workspace and symlink escape before the paid request', async () => { + const f = await fixture(); const s = await server() + await f.settings.save(saveInput('openai', s.baseUrl)) + const tool = imageTool(f.services, f.settings) + f.setMode('read-only') + await expect(tool.execute({ prompt: 'Forest' }, { agent: f.agent })).rejects.toMatchObject({ code: 'POLICY' }) + f.setMode('workspace-write') + const outside = await temp(); await symlink(outside, path.join(f.workspace, '.workbuddy')) + await expect(tool.execute({ prompt: 'Forest' }, { agent: f.agent })).rejects.toMatchObject({ code: 'ASSET_PATH' }) + expect(s.calls).toHaveLength(1); expect(await readdir(outside)).toEqual([]) + }) + it('commits identical assets concurrently, rejects target symlinks, and rejects damaged images', async () => { + const workspace = await temp(); const s = await server(); const image = await normalizeImage(s.png) + const values = await Promise.all([materialize(workspace, image.data), materialize(workspace, image.data)]) + expect(values[0]).toEqual(values[1]) + expect(await readdir(path.dirname(path.join(workspace, values[0].workspace_path)))).toHaveLength(1) + const target = path.join(workspace, values[0].workspace_path) + await rm(target); await symlink(path.join(workspace, 'absent'), target) + await expect(materialize(workspace, image.data)).rejects.toThrow() + await expect(normalizeImage(Buffer.from('not a PNG'))).rejects.toMatchObject({ code: 'IMAGE' }) + }) + it('registers standard authenticated API routes, the shared Skill and default approval', async () => { + const f = await fixture(); const routes = [] + const plugin = f.ctx.plugin({ inject: ['settings', 'skills', 'systemPrompt', 'tools'], apply: ctx => apply({ + ...f.services, settings: ctx.settings, skills: ctx.skills, systemPrompt: ctx.systemPrompt, tools: ctx.tools, + on: ctx.on.bind(ctx), connection: { fetch: { register: route => routes.push(route) } }, + }) }) + await plugin; cleanups.push(() => plugin.dispose()) + expect(f.ctx.settings.describe().map(entry => entry.ns)).toContain('image-generation') + expect(routes.map(route => route.path)).toEqual(['/api/image-generation.settings', '/api/image-generation.save']) + const response = await routes[0].fetch(new Request('http://localhost/api/image-generation.settings')) + expect(response.headers.get('cache-control')).toBe('no-store') + expect((await response.json()).profiles.openai.configured).toBe(false) + const result = await f.ctx.tools.execute({ callId: 'ask', name: 'image_generate', arguments: { prompt: 'Forest' }, agent: f.agent, signal: new AbortController().signal }) + expect(result.isError).toBe(true) + expect(JSON.stringify(result)).toContain('Provider usage may be billed') + expect(await readdir(f.workspace)).toEqual([]) + }) +}) From a325b417682f77c10d552f3990ee5c1a4a82851e Mon Sep 17 00:00:00 2001 From: Jianghan <58623049+SuperstructureJH@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:11:58 +0800 Subject: [PATCH 2/8] fix: align image settings and support Electron image writer --- docs/STATUS.md | 8 ++ packages/dsh-image-generation/README.md | 9 ++- packages/dsh-image-generation/client.js | 76 ++++++++++++++----- packages/dsh-image-generation/index.js | 4 +- packages/dsh-image-generation/lib/commit.js | 17 +++-- packages/dsh-image-generation/lib/provider.js | 21 ++++- packages/dsh-image-generation/lib/settings.js | 16 +++- packages/dsh-image-generation/package.json | 2 +- scripts/verify-image-electron.mjs | 68 +++++++++++++++++ scripts/verify-image-generation.mjs | 10 ++- test/image-generation.test.mjs | 36 ++++++++- 11 files changed, 229 insertions(+), 38 deletions(-) create mode 100644 scripts/verify-image-electron.mjs diff --git a/docs/STATUS.md b/docs/STATUS.md index ebc9f9173..5154fb1f9 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1,5 +1,13 @@ # Implementation and verification status +## Image generation acceptance corrections — 2026-09-10 + +- **Fixed and reproduced:** the original package throws `The image writer returned an invalid result` under its actual Electron Helper. Clearing the environment removed Node launch mode, so the child exited successfully without running the writer. The writer now declares `ELECTRON_RUN_AS_NODE=1` for Electron Hosts while continuing to clear credentials and NODE_OPTIONS. The old package reproduces the exact error; corrected source passes with the same Helper and in a real Electron 43.4.0 utility process, for both simulated providers. +- **Implemented:** OpenAI one-click visible image-model discovery, explicit built-in versus provider-list provenance, ByteDance Seedream 5.0 Pro / 4.5 choices and custom endpoint IDs. Provider catalogs are filtered against the current Images API contract; fetching does not save configuration. API base and full image endpoint inputs normalize to the same stored base URL. +- **PASS:** 21 focused tests plus 2 dependency-closure checks; full suite 90 files / 771 tests; TypeScript and production build; authenticated Host model discovery and save smoke. Browser inspection confirms matching 15 px title / 13 px description, font family, 14×16 px header padding and 74.5 px closed-card height across all five cards, in light/dark themes. The OpenAI model-fetch interaction passes against the loopback provider. +- **PASS:** revised macOS arm64 DMG/ZIP build, strict deep signature check, final packaged Host save/model-list smoke and final packaged plugin code inside a real Electron utility process. Revised artifacts are delivered in `outputs/image-generation-plugin-20260910-r2` with provenance and hashes. The original package evidence below predates this correction. +- **Pending:** user real-provider generation and downstream Office acceptance. + ## Image generation plugin — 2026-09-10 - **Implemented:** public `dsh-image-generation` package, default Desktop composition, shared `image_generate` tool and `generate-image` Skill, OpenAI / ByteDance adapters, plugin configuration card, automatic validation on save, host-only atomic credential storage, governed PNG output. diff --git a/packages/dsh-image-generation/README.md b/packages/dsh-image-generation/README.md index b2591d3d0..90acb8077 100644 --- a/packages/dsh-image-generation/README.md +++ b/packages/dsh-image-generation/README.md @@ -6,6 +6,10 @@ DSH/Cordis 公共生图插件，提供 `image_generate` 工具、`generate-image DSH Desktop 默认装载本插件。在生图工具卡片选择字节或 OpenAI，填写该平台的 API Key 并保存。模型和地址已有默认值，可在高级设置中修改。 +生图模型使用下拉选择。OpenAI 的「获取模型」用当前填写或已保存的 Key 发起一次 `GET /models`，筛选 Images API 支持的 GPT Image 模型；获取过程只更新候选列表，配置在点击保存时生效。列表区分「内置模型」与「此 Key 可见模型」，空列表和查询错误明确显示。字节的管理接口使用独立签名凭据，因此 API Key 模式提供 Seedream 5.0 Pro、4.5 的内置选项和自定义模型/接入点 ID。 + +API 地址同时接受基础地址和控制台提供的完整 `/images/generations` 地址。保存时统一为基础地址，实际请求只添加一次接口路径。 + | 服务商 | 默认模型 | 默认 API 地址 | | --- | --- | --- | | 字节 / 火山方舟 | `doubao-seedream-4-5-251128` | `https://ark.cn-beijing.volces.com/api/v3` | @@ -42,15 +46,18 @@ npm pack ./packages/dsh-image-generation --pack-destination /absolute/output/dir 生图通过标准工具执行和审计流程，并默认要求 Host 审批。请求前及落盘前读取当前会话的 sandboxPolicy，要求工作区写权限；输出目录和目标文件拒绝符号链接。实际写入由 `ctx.sandbox` 包装的 `ctx.subprocess` 子进程执行，使用工作区写权限；仅通过 stdin 接收图片字节，清除继承环境，Key 留在 Host。PNG 使用临时文件、原子创建和内容哈希校验，避免并发生图产生半个文件。HTTP 禁止重定向，只接受 HTTPS 地址（本机回环开发服务允许 HTTP），约束超时、响应大小和解码像素；不下载厂商返回的任意远程 URL。系统沙箱的跨平台执行能力以部署环境为准，本期本地验收环境为 macOS arm64。 +macOS Desktop 的 Host 运行在 Electron utility process 中；写图子进程显式设置 `ELECTRON_RUN_AS_NODE=1`，使同一可执行文件执行 Node 脚本。该启动常量与 Windows 的 SystemRoot 按需保留，凭据和 NODE_OPTIONS 继续清除。回归同时覆盖普通 Node 和真实 Electron utility process。 + ## 验证 ```sh npx vitest run test/image-generation.test.mjs test/desktop-plugin-closure.test.ts node scripts/verify-image-generation.mjs +node scripts/verify-image-electron.mjs npm run typecheck npm test ``` 测试使用本机模拟服务和真实 Harness 凭据存储、工具执行管线，覆盖两个厂商、单请求保存、成功/失败、修订冲突、凭据隔离、PNG 落盘、取消、体积限制和目录越界。Host smoke 启动隔离实例，验证默认装载、Client 入口、鉴权、Origin 及保存结果。真实付费模型与 Office 文档视觉验收在 `docs/STATUS.md` 单独记录。 -接口参考：[OpenAI Images API](https://developers.openai.com/api/reference/resources/images/methods/generate)、[OpenAI Models API](https://developers.openai.com/api/reference/resources/models/methods/retrieve)、[火山方舟生图 API](https://www.volcengine.com/docs/82379/1541523)、[火山方舟官方 Python SDK](https://github.com/volcengine/volcengine-python-sdk/tree/master/volcenginesdkarkruntime/resources/images)。 +接口参考：[OpenAI Images API](https://developers.openai.com/api/reference/resources/images/methods/generate)、[OpenAI 模型列表](https://developers.openai.com/api/reference/resources/models/methods/list)、[火山方舟生图 API](https://www.volcengine.com/docs/82379/1541523)、[火山方舟官方运行时 SDK](https://github.com/volcengine/volcengine-python-sdk/tree/master/volcenginesdkarkruntime/resources/images)、[火山管理接口及签名鉴权](https://github.com/volcengine/volcengine-python-sdk/blob/master/volcenginesdkark/api/ark_api.py)。 diff --git a/packages/dsh-image-generation/client.js b/packages/dsh-image-generation/client.js index 9bf566595..bada5fef5 100644 --- a/packages/dsh-image-generation/client.js +++ b/packages/dsh-image-generation/client.js @@ -2,14 +2,20 @@ window.__ModuleLoader__.load({ id: 'dsh-image-generation', factory: require => { const React = require('react') + const { IconChevronDownOutline14 } = require('@deepseek-ai/dsh-client-ui-primitives') const h = React.createElement const NS = 'settings.imageGeneration' const zh = { title: '生图工具', description: '为 PPT、Word 和对话生成图片。', provider: '服务商', bytedance: '字节 · 豆包 Seedream', openai: 'OpenAI', apiKey: 'API Key', keyPlaceholder: '输入服务商的 API Key', savedKey: '已配置，输入新 Key 可替换', keyLink: '获取 API Key', - advanced: '高级设置', model: '模型 ID', baseUrl: 'API 地址', defaults: '已提供默认模型和地址，通常无需修改。', + advanced: '高级设置', model: '模型 ID', baseUrl: 'API 地址', defaults: '支持基础地址或控制台的完整生图接口地址，保存时自动识别。', save: '保存', saving: '正在校验…', saved: '已保存，连接校验通过', hint: '保存时自动校验连接；模型的生图权限和额度在实际调用时确认。', configured: '已配置', loading: '正在读取配置…', reload: '重新读取配置', readOnly: '当前配置由管理员管理。', + modelSelect: '生图模型', fetchModels: '获取模型', fetchingModels: '正在获取…', customModel: '自定义模型 / 接入点', + builtinModels: '内置模型，可直接选择；模型访问权限在保存及实际调用时确认。', + byteModels: '字节的账号模型查询需要独立管理凭据。当前提供内置模型，也可填写自己的接入点 ID。', + fetchedModels: '已获取此 Key 可见且工具支持的生图模型。', emptyModels: '此 Key 的列表未返回工具支持的生图模型，可检查权限或填写自定义模型。', + MODEL_DISCOVERY: '当前服务商使用内置模型或自定义接入点。', AUTH: 'API Key 无效或已过期，请检查后重新保存。', PERMISSION: '当前 Key 无权访问，请确认模型已开通及账号已完成所需认证。', MODEL: '无法访问所选模型，请检查模型 ID、接入点或模型开通状态。', KEY_REQUIRED: '请填写 API Key；更换服务地址后需要重新填写。', QUOTA: '服务商额度不足或请求受限，请检查账户后重试。', ENDPOINT: '请填写有效的 API 基础地址，例如以 /v1 或 /api/v3 结尾的 HTTPS 地址。', @@ -21,9 +27,14 @@ window.__ModuleLoader__.load({ const en = { title: 'Image generation', description: 'Generate images for PPT, Word and conversations.', provider: 'Provider', bytedance: 'ByteDance · Seedream', openai: 'OpenAI', apiKey: 'API Key', keyPlaceholder: 'Enter your provider API key', savedKey: 'Configured; enter a new key to replace it', keyLink: 'Get an API key', - advanced: 'Advanced settings', model: 'Model ID', baseUrl: 'API base URL', defaults: 'The default model and URL are ready to use.', + advanced: 'Advanced settings', model: 'Model ID', baseUrl: 'API URL', defaults: 'Enter a base URL or the full image-generation endpoint. Saving normalizes it automatically.', save: 'Save', saving: 'Validating…', saved: 'Saved. Connection validated.', hint: 'Saving validates the connection automatically. Image permissions and quota are confirmed during generation.', configured: 'Configured', loading: 'Loading settings…', reload: 'Reload settings', readOnly: 'These settings are managed by your administrator.', + modelSelect: 'Image model', fetchModels: 'Fetch models', fetchingModels: 'Fetching…', customModel: 'Custom model / endpoint', + builtinModels: 'Built-in models. Access is checked when saving and generating.', + byteModels: 'ByteDance account discovery requires separate management credentials. Choose the built-in model or enter your endpoint ID.', + fetchedModels: 'Loaded image models visible to this key and supported by this tool.', emptyModels: 'No supported image models were returned. Check access or enter a custom model.', + MODEL_DISCOVERY: 'Use a built-in model or a custom endpoint for this provider.', AUTH: 'The API key is invalid or expired.', PERMISSION: 'Check model access and account verification.', MODEL: 'Check the model ID, inference endpoint and model access.', KEY_REQUIRED: 'Enter an API key. A new API origin requires you to enter the key again.', QUOTA: 'Check provider quota and rate limits.', ENDPOINT: 'Enter an HTTPS API base URL, normally ending in /v1 or /api/v3.', TIMEOUT: 'The connection timed out. Check your network and save again.', @@ -33,19 +44,23 @@ window.__ModuleLoader__.load({ CANCELLED: 'Validation was cancelled. Save again to retry.', PROVIDER_ERROR: 'The provider is temporarily unavailable. Save again later.', } const css = ` - .dshImageCard{list-style:none;border:1px solid var(--dsw-alias-border-l2);border-radius:14px;color:var(--dsw-alias-label-primary);overflow:hidden} - .dshImageHeader{display:flex;align-items:center;gap:12px;width:100%;padding:18px 20px;background:transparent;border:0;text-align:left;color:inherit;font:inherit;cursor:pointer} - .dshImageHeading{display:flex;flex:1;flex-direction:column;gap:6px;min-width:0}.dshImageTitle{font-size:16px;font-weight:600;line-height:24px} - .dshImageDescription{color:var(--dsw-alias-label-tertiary);font-size:14px;line-height:22px}.dshImageBadge{font-size:12px;color:var(--dsw-alias-label-secondary)} - .dshImageChevron{width:8px;height:8px;border-right:1.5px solid var(--dsw-alias-label-tertiary);border-bottom:1.5px solid var(--dsw-alias-label-tertiary);transform:rotate(45deg);margin-right:4px}.dshImageHeader[aria-expanded=true] .dshImageChevron{transform:rotate(225deg)} - .dshImageBody{padding:0 20px 20px;display:flex;flex-direction:column;gap:16px}.dshImageFields{border:0;margin:0;padding:0;display:flex;flex-direction:column;gap:16px;min-width:0} - .dshImageField{display:flex;flex-direction:column;gap:7px;font-size:13px}.dshImageField input,.dshImageField select{box-sizing:border-box;width:100%;min-width:0;height:36px;padding:0 11px;border:1px solid var(--dsw-alias-border-l3);border-radius:8px;background:var(--dsw-alias-bg-layer-1);color:inherit;font:inherit} + /* Match the Host PluginCard design contract; its component is internal. */ + .dshImageCard{list-style:none;border:.5px solid var(--dsw-alias-border-l4);background:var(--dsw-alias-bg-layer-3);border-radius:16px;transition:border-color .16s,background .16s} + .dshImageCard:hover{border-color:var(--dsw-alias-label-dimmed)}.dshImageCardOpen{background:var(--dsw-alias-bg-layer-2);border-color:var(--dsw-alias-label-dimmed)} + .dshImageHeader{appearance:none;display:flex;align-items:center;gap:12px;width:100%;padding:14px 16px;background:transparent;border:0;border-radius:12px;text-align:left;color:inherit;font:inherit;cursor:pointer} + .dshImageHeading{display:flex;flex:1;flex-direction:column;gap:4px;min-width:0}.dshImageTitle{color:var(--dsw-alias-label-primary);font-size:15px;font-weight:600;line-height:1.4} + .dshImageDescription{color:var(--dsw-alias-label-tertiary);font-size:13px;line-height:1.5}.dshImageBadge{corner-shape:round;white-space:nowrap;background:var(--dsw-alias-bg-module-platform);color:var(--dsw-alias-label-secondary);border-radius:999px;flex:none;padding:1px 8px;font-size:11px;font-weight:500;line-height:17px} + .dshImageChevron{color:var(--dsw-alias-label-tertiary);flex:none;transition:transform .16s}.dshImageHeader[aria-expanded=true] .dshImageChevron{transform:rotate(180deg)} + .dshImageBody{border-top:.5px solid var(--dsw-alias-border-l2);margin:0 16px;padding-bottom:8px;display:flex;flex-direction:column;gap:12px}.dshImageFields{border:0;margin:0;padding:0;display:flex;flex-direction:column;min-width:0} + .dshImageField{display:flex;flex-direction:column;gap:6px;padding:12px 0;font-size:13px;font-weight:500;line-height:1.5;color:var(--dsw-alias-label-primary)}.dshImageField+.dshImageField{border-top:.5px solid var(--dsw-alias-border-l2)}.dshImageField input,.dshImageField select{box-sizing:border-box;width:100%;min-width:0;height:34px;padding:0 12px;border:.5px solid var(--dsw-alias-border-l4);border-radius:8px;background:var(--dsw-alias-bg-layer-3);color:var(--dsw-alias-label-primary);font:inherit;font-weight:400} .dshImageField input::placeholder{color:var(--dsw-alias-label-tertiary)}.dshImageLink{align-self:flex-start;font-size:12px;color:var(--dsw-alias-label-secondary);text-decoration:underline} - .dshImageAdvanced summary{cursor:pointer;font-size:13px;color:var(--dsw-alias-label-secondary)}.dshImageAdvanced[open]{display:flex;flex-direction:column}.dshImageAdvanced .dshImageField{margin-top:14px} - .dshImageHint{font-size:12px;line-height:19px;color:var(--dsw-alias-label-tertiary);margin:0}.dshImageActions{display:flex;align-items:center;gap:12px;flex-wrap:wrap} - .dshImageSave{font:inherit;font-size:13px;font-weight:500;border:0;border-radius:18px;padding:8px 20px;background:var(--dsw-alias-button-primary-fill);color:var(--dsw-alias-label-primary-foreground);cursor:pointer}.dshImageSave:disabled{opacity:.5;cursor:default} - .dshImageStatus{margin:0;font-size:13px;line-height:20px}.dshImageError{color:var(--dsw-alias-state-error-primary)} + .dshImageAdvanced{margin-top:12px}.dshImageAdvanced summary{cursor:pointer;font-size:13px;color:var(--dsw-alias-label-secondary)}.dshImageAdvanced[open]{display:flex;flex-direction:column} + .dshImageHint{font-size:12px;line-height:1.5;color:var(--dsw-alias-label-tertiary);margin:0}.dshImageActions{border-top:.5px solid var(--dsw-alias-border-l2);display:flex;justify-content:flex-end;align-items:center;gap:8px;padding:12px 0 4px;flex-wrap:wrap} + .dshImageSave{appearance:none;font:inherit;font-size:13px;line-height:1.5;border:1px solid transparent;border-radius:8px;padding:5px 14px;background:var(--dsw-alias-label-primary);color:var(--dsw-alias-bg-layer-3);cursor:pointer}.dshImageSave:disabled{opacity:.4;cursor:default} + .dshImageStatus{margin:0;font-size:12px;line-height:1.5}.dshImageError{color:var(--dsw-alias-state-error-primary)} .dshImageCard :focus-visible{outline:2px solid var(--dsw-alias-brand-primary);outline-offset:2px} + .dshImageHeader:focus-visible{outline-offset:-2px}.dshImageSave:focus-visible{outline-offset:1px} + .dshImageFetch{appearance:none;align-self:flex-start;font:inherit;font-size:13px;line-height:1.5;border:1px solid var(--dsw-alias-border-l2);border-radius:8px;padding:5px 14px;background:transparent;color:var(--dsw-alias-label-secondary);cursor:pointer}.dshImageFetch:disabled{opacity:.4;cursor:default} ` async function api(path, options = {}) { const response = await fetch(`/api/image-generation.${path}`, { ...options, cache: 'no-store' }) @@ -63,6 +78,9 @@ window.__ModuleLoader__.load({ const [error, setError] = React.useState('') const [loading, setLoading] = React.useState(true) const [busy, setBusy] = React.useState(false) + const [fetching, setFetching] = React.useState(false) + const [catalogs, setCatalogs] = React.useState({}) + const [customModels, setCustomModels] = React.useState({}) const inFlight = React.useRef(false) const lifetime = React.useRef(null) const id = React.useId() @@ -71,6 +89,7 @@ window.__ModuleLoader__.load({ try { const result = await api('settings', { signal }) setSaved(result); setProvider(result.provider) + setCatalogs({}); setCustomModels(Object.fromEntries(Object.entries(result.profiles).map(([key, value]) => [key, !result.catalogs[key].models.includes(value.model)]))) setDrafts(Object.fromEntries(Object.entries(result.profiles).map(([key, value]) => [key, { ...value, apiKey: '' }]))) } catch (error) { if (!signal.aborted) setError(error.code || 'UNAVAILABLE') } finally { if (!signal.aborted) setLoading(false) } @@ -81,10 +100,23 @@ window.__ModuleLoader__.load({ return () => controller.abort() }, [load]) const draft = drafts[provider] + const catalog = catalogs[provider] || saved?.catalogs[provider] const edit = (key, value) => { setDrafts(previous => ({ ...previous, [provider]: { ...previous[provider], [key]: value } })) + if (key === 'apiKey' || key === 'baseUrl') setCatalogs(previous => ({ ...previous, [provider]: undefined })) setStatus(''); setError('') } + const fetchModels = async () => { + if (inFlight.current || !saved) return + inFlight.current = true; setFetching(true); setError(''); setStatus('') + try { + const result = await api('models', { method: 'POST', headers: { 'Content-Type': 'application/json' }, signal: lifetime.current.signal, + body: JSON.stringify({ revision: saved.revision, provider, model: draft.model || undefined, baseUrl: draft.baseUrl, apiKey: draft.apiKey }) }) + setCatalogs(previous => ({ ...previous, [provider]: result })) + setCustomModels(previous => ({ ...previous, [provider]: !result.models.includes(draft.model) })) + } catch (error) { if (!lifetime.current.signal.aborted) setError(error.code || 'UNAVAILABLE') } + finally { inFlight.current = false; if (!lifetime.current.signal.aborted) setFetching(false) } + } const save = async event => { event.preventDefault() if (inFlight.current || !saved) return @@ -102,21 +134,27 @@ window.__ModuleLoader__.load({ } const field = (key, label, type = 'text', placeholder) => h('label', { className: 'dshImageField', key }, t(label), h('input', { name: key, type, value: draft[key], placeholder, autoComplete: 'off', spellCheck: false, onChange: event => edit(key, event.target.value) })) - return h('li', { className: 'dshImageCard', 'data-testid': 'image-generation-card' }, + return h('li', { className: `dshImageCard${expanded ? ' dshImageCardOpen' : ''}`, 'data-testid': 'image-generation-card' }, h('button', { className: 'dshImageHeader', type: 'button', 'aria-expanded': expanded, 'aria-controls': `${id}-body`, onClick: () => setExpanded(value => !value) }, h('span', { className: 'dshImageHeading' }, h('span', { className: 'dshImageTitle' }, t('title')), h('span', { className: 'dshImageDescription' }, t('description'))), saved?.profiles[saved.provider]?.configured && h('span', { className: 'dshImageBadge' }, t('configured')), - h('span', { className: 'dshImageChevron', 'aria-hidden': true })), - expanded && h('form', { id: `${id}-body`, className: 'dshImageBody', onSubmit: save, 'aria-busy': busy || loading }, + h(IconChevronDownOutline14, { className: 'dshImageChevron' })), + expanded && h('form', { id: `${id}-body`, className: 'dshImageBody', onSubmit: save, 'aria-busy': busy || fetching || loading }, loading ? h('p', { className: 'dshImageHint' }, t('loading')) : draft && h(React.Fragment, null, - h('fieldset', { className: 'dshImageFields', disabled: busy || !saved.writable }, + h('fieldset', { className: 'dshImageFields', disabled: busy || fetching || !saved.writable }, h('label', { className: 'dshImageField' }, t('provider'), h('select', { name: 'provider', value: provider, onChange: event => { setProvider(event.target.value); setStatus(''); setError('') } }, h('option', { value: 'bytedance' }, t('bytedance')), h('option', { value: 'openai' }, t('openai')))), field('apiKey', 'apiKey', 'password', t(draft.configured ? 'savedKey' : 'keyPlaceholder')), h('a', { className: 'dshImageLink', href: provider === 'openai' ? 'https://platform.openai.com/api-keys' : 'https://console.volcengine.com/ark/apiKey', target: '_blank', rel: 'noreferrer' }, t('keyLink')), - h('details', { className: 'dshImageAdvanced' }, h('summary', null, t('advanced')), field('model', 'model'), field('baseUrl', 'baseUrl'))), + h('label', { className: 'dshImageField' }, t('modelSelect'), h('select', { name: 'model', value: customModels[provider] || !catalog.models.includes(draft.model) ? '__custom__' : draft.model, + onChange: event => { const custom = event.target.value === '__custom__'; setCustomModels(previous => ({ ...previous, [provider]: custom })); if (!custom) edit('model', event.target.value) } }, + ...catalog.models.map(model => h('option', { key: model, value: model }, model)), h('option', { value: '__custom__' }, t('customModel')))), + (customModels[provider] || !catalog.models.includes(draft.model)) && field('model', 'model'), + catalog.canFetch && h('button', { className: 'dshImageFetch', type: 'button', disabled: !draft.apiKey.trim() && !draft.configured, onClick: fetchModels }, t(fetching ? 'fetchingModels' : 'fetchModels')), + h('p', { className: 'dshImageHint', role: 'status' }, t(catalog.source === 'provider' ? catalog.models.length ? 'fetchedModels' : 'emptyModels' : provider === 'bytedance' ? 'byteModels' : 'builtinModels')), + h('details', { className: 'dshImageAdvanced' }, h('summary', null, t('advanced')), field('baseUrl', 'baseUrl'))), h('p', { className: 'dshImageHint' }, t('defaults')), - h('div', { className: 'dshImageActions' }, h('button', { className: 'dshImageSave', type: 'submit', disabled: busy || !saved.writable || (!draft.apiKey.trim() && !draft.configured) }, t(busy ? 'saving' : 'save')), + h('div', { className: 'dshImageActions' }, h('button', { className: 'dshImageSave', type: 'submit', disabled: busy || fetching || !saved.writable || !draft.model.trim() || (!draft.apiKey.trim() && !draft.configured) }, t(busy ? 'saving' : 'save')), status && h('p', { className: 'dshImageStatus', role: 'status' }, t(status))), h('p', { className: 'dshImageHint' }, t(saved.writable ? 'hint' : 'readOnly'))), error && h('p', { className: 'dshImageStatus dshImageError', role: 'alert' }, t(Object.hasOwn(en, error) ? error : 'UNAVAILABLE')), diff --git a/packages/dsh-image-generation/index.js b/packages/dsh-image-generation/index.js index 494017856..e348b15d3 100644 --- a/packages/dsh-image-generation/index.js +++ b/packages/dsh-image-generation/index.js @@ -63,10 +63,10 @@ export async function apply(ctx) { // failed save cannot pair an old key with a newly persisted endpoint. ctx.settings.register('image-generation', Config, { applies: 'live' }) const settings = createSettings(ctx) - for (const [suffix, method] of [['settings', 'GET'], ['save', 'POST']]) { + for (const [suffix, method] of [['settings', 'GET'], ['save', 'POST'], ['models', 'POST']]) { ctx.connection.fetch.register({ path: `/api/image-generation.${suffix}`, methods: [method], async fetch(request) { try { - const value = method === 'GET' ? await settings.describe() : await settings.save( + const value = method === 'GET' ? await settings.describe() : await settings[suffix]( JSON.parse((await readBounded(request, 16_384, request.signal)).toString('utf8')), request.signal, ) return Response.json(value, { headers: { 'Cache-Control': 'no-store' } }) diff --git a/packages/dsh-image-generation/lib/commit.js b/packages/dsh-image-generation/lib/commit.js index 0c72b4180..fe356eee0 100644 --- a/packages/dsh-image-generation/lib/commit.js +++ b/packages/dsh-image-generation/lib/commit.js @@ -9,15 +9,22 @@ export function writerPlan(ctx, root, exec) { }) } +export function writerEnvironment(runtime = process) { + // Tombstones clear ordinary parent entries too: no launch credentials or + // NODE_OPTIONS enter the asset writer, even if the host inherited them. + const env = Object.fromEntries(Object.keys(runtime.env).map(key => [key, undefined])) + if (runtime.platform === 'win32') env.SystemRoot = runtime.env.SystemRoot + // Desktop's macOS Host is an Electron utility process. Its executable runs + // this child script only in Node mode; this constant is launch configuration. + if (runtime.versions.electron) env.ELECTRON_RUN_AS_NODE = '1' + return env +} + export async function commitImage(ctx, root, data, exec) { const plan = writerPlan(ctx, root, exec) const signal = AbortSignal.any([AbortSignal.timeout(30_000), ...(exec.signal ? [exec.signal] : [])]) - // Tombstones clear ordinary parent entries too: no launch credentials or - // NODE_OPTIONS enter the asset writer, even if the host inherited them. - const env = Object.fromEntries(Object.keys(process.env).map(key => [key, undefined])) - if (process.platform === 'win32') env.SystemRoot = process.env.SystemRoot const child = ctx.subprocess.spawn({ - argv: plan.argv, cwd: root, env, signal, graceMs: 1000, + argv: plan.argv, cwd: root, env: writerEnvironment(), signal, graceMs: 1000, stdio: { stdin: { data: data.toString('base64') }, stdout: { maxBytes: 1024 }, stderr: { maxBytes: 2048 } }, }) const outcome = await child.done diff --git a/packages/dsh-image-generation/lib/provider.js b/packages/dsh-image-generation/lib/provider.js index dddad33c5..ea21555f7 100644 --- a/packages/dsh-image-generation/lib/provider.js +++ b/packages/dsh-image-generation/lib/provider.js @@ -3,6 +3,13 @@ export const DEFAULTS = Object.freeze({ openai: { baseUrl: 'https://api.openai.com/v1', model: 'gpt-image-1.5' }, }) export const MAX_IMAGE_BYTES = 20 * 1024 * 1024 +// Models documented by Images API; text, legacy DALL-E and unknown adapters +// remain outside this tool's parameter contract. +const OPENAI_IMAGES = new Set(['gpt-image-2.5-sunburst', 'gpt-image-2.5-sunburst-2026-09-08', 'gpt-image-2.5-flare', 'gpt-image-2.5-flare-2026-09-08', 'gpt-image-2', 'gpt-image-2-2026-04-21', 'gpt-image-1.5', 'gpt-image-1', 'gpt-image-1-mini']) +export const MODEL_CATALOG = Object.freeze({ + openai: { source: 'builtin', canFetch: true, models: [...OPENAI_IMAGES] }, + bytedance: { source: 'builtin', canFetch: false, models: ['doubao-seedream-5-0-pro-260628', DEFAULTS.bytedance.model] }, +}) export class ImageError extends Error { constructor(code, message, status = 400) { super(message); this.code = code; this.status = status } @@ -23,7 +30,9 @@ export function profile(provider, input = {}) { if ((url.protocol !== 'https:' && !(url.protocol === 'http:' && loopback)) || url.username || url.password || url.search || url.hash) { throw new ImageError('ENDPOINT', 'Use an HTTPS API base URL without credentials, query parameters or fragments.') } - if (/\/images\/generations\/?$/.test(url.pathname)) throw new ImageError('ENDPOINT', 'Enter the API base URL ending in /v1 or /api/v3.') + // Provider consoles present the full REST endpoint; store one canonical base + // for validation, model discovery and generation so the suffix appears once. + url.pathname = url.pathname.replace(/\/images\/generations\/?$/, '') return { baseUrl: url.href.replace(/\/+$/, ''), model } } @@ -83,6 +92,16 @@ async function request(url, key, { signal, body, maxBytes = 2 * 1024 * 1024, fet } catch (error) { throw safeError(error) } } +/** Read the account's visible models without submitting image generation. */ +export async function listModels(provider, spec, key, options = {}) { + if (provider !== 'openai') throw new ImageError('MODEL_DISCOVERY', 'This provider requires separate management credentials for account model discovery. Choose a built-in model or enter your endpoint ID.') + const signal = AbortSignal.any([AbortSignal.timeout(15_000), ...(options.signal ? [options.signal] : [])]) + const result = await request(`${spec.baseUrl}/models`, key, { ...options, signal }) + if (!Array.isArray(result.data)) throw new ImageError('RESPONSE', 'The provider returned an invalid model list.', 502) + const ids = new Set(result.data.filter(model => model && typeof model.id === 'string' && (!model.shutdown_date || model.shutdown_date > new Date().toISOString().slice(0, 10))).map(model => model.id)) + return { source: 'provider', canFetch: true, models: [...OPENAI_IMAGES].filter(id => ids.has(id)) } +} + /** One non-generating request: OpenAI model metadata or Ark required-prompt validation. */ export async function validateConnection(provider, spec, key, options = {}) { const signal = AbortSignal.any([AbortSignal.timeout(15_000), ...(options.signal ? [options.signal] : [])]) diff --git a/packages/dsh-image-generation/lib/settings.js b/packages/dsh-image-generation/lib/settings.js index 111c6bf9f..7411f01f1 100644 --- a/packages/dsh-image-generation/lib/settings.js +++ b/packages/dsh-image-generation/lib/settings.js @@ -1,5 +1,5 @@ import { credentialKey } from '@deepseek-ai/dsh-credentials' -import { DEFAULTS, ImageError, profile, safeError, validateConnection } from './provider.js' +import { DEFAULTS, MODEL_CATALOG, ImageError, listModels, profile, safeError, validateConnection } from './provider.js' const KEY = credentialKey('dsh-image-generation', 'configuration') const EMPTY = () => ({ revision: 0, provider: 'bytedance', profiles: {} }) @@ -18,7 +18,7 @@ export function createSettings(ctx, validate = validateConnection) { const state = await read() const info = await ctx.credentials.describeRecord(KEY) return { - revision: state.revision, provider: state.provider, writable: info.writable, + revision: state.revision, provider: state.provider, writable: info.writable, catalogs: MODEL_CATALOG, profiles: Object.fromEntries(Object.entries(DEFAULTS).map(([provider, defaults]) => { const stored = state.profiles[provider] return [provider, { ...defaults, ...(stored ? profile(provider, stored) : {}), configured: Boolean(stored?.key), validation: stored?.validation ?? null }] @@ -31,7 +31,7 @@ export function createSettings(ctx, validate = validateConnection) { if (!spec?.key) throw new ImageError('NOT_CONFIGURED', 'Configure the image tool in Settings > Plugins before generating an image.') return { provider: state.provider, ...profile(state.provider, spec), key: spec.key } } - async function save(input, signal) { + async function prepared(input) { if (!input || typeof input !== 'object' || Array.isArray(input)) throw new ImageError('CONFIGURATION', 'Enter a valid image configuration.') const before = await read() if (!Number.isSafeInteger(input.revision) || input.revision !== before.revision) throw new ImageError('CONFLICT', 'The settings changed in another window. Reopen this card before saving.', 409) @@ -43,6 +43,14 @@ export function createSettings(ctx, validate = validateConnection) { } const effectiveKey = key || previous?.key if (!effectiveKey || effectiveKey.length > 4096 || /[\s\x00-\x1f\x7f]/u.test(effectiveKey)) throw new ImageError('KEY_REQUIRED', 'Enter a valid API key.') + return { before, spec, effectiveKey } + } + async function models(input, signal) { + const { spec, effectiveKey } = await prepared(input) + return listModels(input.provider, spec, effectiveKey, { signal }) + } + async function save(input, signal) { + const { before, spec, effectiveKey } = await prepared(input) if (!(await ctx.credentials.describeRecord(KEY)).writable) throw new ImageError('READ_ONLY', 'The host credential store is read-only.', 403) try { const validation = await validate(input.provider, spec, effectiveKey, { signal }) @@ -63,5 +71,5 @@ export function createSettings(ctx, validate = validateConnection) { throw safe } } - return { describe, active, save } + return { describe, active, save, models } } diff --git a/packages/dsh-image-generation/package.json b/packages/dsh-image-generation/package.json index 6c542f99a..8028022be 100644 --- a/packages/dsh-image-generation/package.json +++ b/packages/dsh-image-generation/package.json @@ -12,7 +12,7 @@ "keywords": ["dsh-plugin", "image-generation", "seedream", "openai"], "dsh": { "bundle": { "patch": "./cordis.patch.yml" }, - "client": { "inject": ["@deepseek-ai/dsh-client-locale", "@deepseek-ai/dsh-client-ui-settings-plugins"], "platform": "web" } + "client": { "inject": ["@deepseek-ai/dsh-client-locale", "@deepseek-ai/dsh-client-ui-primitives", "@deepseek-ai/dsh-client-ui-settings-plugins"], "platform": "web" } }, "dependencies": { "@deepseek-ai/schemastery": "^3.18.2", "sharp": "^0.35.4" }, "peerDependencies": { diff --git a/scripts/verify-image-electron.mjs b/scripts/verify-image-electron.mjs new file mode 100644 index 000000000..f8ef7d86a --- /dev/null +++ b/scripts/verify-image-electron.mjs @@ -0,0 +1,68 @@ +/** Exercise the image writer inside a real Electron utility process, as macOS Desktop does. */ +import assert from 'node:assert/strict' +import { spawn } from 'node:child_process' +import { createRequire } from 'node:module' +import { mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises' +import { createHash } from 'node:crypto' +import { createServer } from 'node:http' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' + +const root = fileURLToPath(new URL('..', import.meta.url)) +const appRoot = process.env.IMAGE_SMOKE_APP || root +const req = createRequire(path.join(appRoot, 'package.json')) +const use = name => import(pathToFileURL(req.resolve(name)).href) +if (process.argv.includes('--worker')) { + assert.equal(process.type, 'utility') + assert.ok(process.versions.electron) + const { Context } = await use('@deepseek-ai/cordis') + const { LocalSandboxProvider } = await use('@deepseek-ai/dsh-sandbox-local') + const { LocalSubprocessRuntime } = await use('@deepseek-ai/dsh-subprocess-local') + const { default: sharp } = await use('sharp') + const { imageTool } = await use('dsh-image-generation') + const workspace = await realpath(await mkdtemp(path.join(tmpdir(), 'dsh-image-electron-work-'))) + const ctx = new Context(); const forks = []; let calls = 0 + const png = await sharp({ create: { width: 256, height: 192, channels: 4, background: '#12345680' } }).png().toBuffer() + const server = createServer(async (request, response) => { + let body = ''; for await (const part of request) body += part + assert.ok(JSON.parse(body).prompt); calls++ + response.writeHead(200, { 'content-type': 'application/json' }) + response.end(JSON.stringify({ data: [{ b64_json: png.toString('base64') }] })) + }) + try { + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + for (const plugin of [LocalSandboxProvider, LocalSubprocessRuntime]) { const fork = ctx.plugin(plugin, {}); forks.push(fork); await fork } + const services = { sandbox: ctx.sandbox, subprocess: ctx.subprocess, logger: { info() {} }, sandboxPolicy: { resolve: () => ({ mode: 'workspace-write', workspaceRoot: workspace }) } } + for (const [provider, model] of [['openai', 'gpt-image-2.5-flare'], ['bytedance', 'doubao-seedream-5-0-pro-260628']]) { + const tool = imageTool(services, { active: async () => ({ provider, model, baseUrl: `http://127.0.0.1:${server.address().port}/v1`, key: 'local-smoke-key' }) }) + const value = await tool.execute({ prompt: 'A flower', aspect_ratio: '16:9' }, { agent: { session: { header: { cwd: workspace } } }, signal: new AbortController().signal }) + const data = await readFile(path.join(workspace, value.workspace_path)) + assert.equal(createHash('sha256').update(data).digest('hex'), value.sha256) + const meta = await sharp(data).metadata(); assert.equal(meta.format, 'png'); assert.equal(meta.hasAlpha, true) + } + assert.equal(calls, 2) + console.log(`PASS: real Electron ${process.versions.electron} utility Host, both adapters and sandboxed PNG writer (loopback simulation).`) + } finally { + server.closeAllConnections(); await new Promise(resolve => server.close(resolve)) + for (const fork of forks.reverse()) await fork.dispose() + await rm(workspace, { recursive: true, force: true }) + } +} else { + const temp = await mkdtemp(path.join(tmpdir(), 'dsh-image-electron-')) + const main = path.join(temp, 'main.cjs') + await writeFile(main, `const { app, utilityProcess } = require('electron'); +app.setPath('userData', ${JSON.stringify(path.join(temp, 'profile'))}); app.disableHardwareAcceleration(); +app.whenReady().then(() => { + const worker = utilityProcess.fork(${JSON.stringify(fileURLToPath(import.meta.url))}, ['--worker'], { stdio: 'pipe', execArgv: [] }); + worker.stdout.on('data', part => process.stdout.write(part)); worker.stderr.on('data', part => process.stderr.write(part)); + const timer = setTimeout(() => { worker.kill(); app.exit(1); }, 60000); + worker.once('exit', code => { clearTimeout(timer); app.exit(code ?? 1); }); +});`) + try { + const env = { ...process.env, IMAGE_SMOKE_APP: appRoot }; delete env.ELECTRON_RUN_AS_NODE + const child = spawn(createRequire(import.meta.url)('electron'), [main], { env, stdio: 'inherit', signal: AbortSignal.timeout(90_000) }) + const code = await new Promise((resolve, reject) => { child.once('error', reject); child.once('exit', resolve) }) + assert.equal(code, 0, 'Electron utility-process image acceptance failed') + } finally { await rm(temp, { recursive: true, force: true }) } +} diff --git a/scripts/verify-image-generation.mjs b/scripts/verify-image-generation.mjs index c2a1068cc..95432b8d7 100644 --- a/scripts/verify-image-generation.mjs +++ b/scripts/verify-image-generation.mjs @@ -21,7 +21,7 @@ const mock = createServer(async (request, response) => { const probe = valid && request.url.endsWith('/images/generations') && body && !Object.hasOwn(JSON.parse(body), 'prompt') assert.ok(!body || !JSON.parse(body).prompt, 'The configuration flow must omit the required generation prompt') response.writeHead(valid ? probe ? 400 : 200 : 401, { 'Content-Type': 'application/json' }) - response.end(JSON.stringify(valid ? probe ? { error: { code: 'MissingParameter', message: 'The request is missing a required parameter: prompt.' } } : { id: DEFAULTS.openai.model } : { error: { message: 'Invalid API key' } })) + response.end(JSON.stringify(valid ? probe ? { error: { code: 'MissingParameter', message: 'The request is missing a required parameter: prompt.' } } : request.url.endsWith('/models') ? { data: [{ id: 'gpt-image-2.5-flare' }, { id: DEFAULTS.openai.model }, { id: 'gpt-5' }] } : { id: DEFAULTS.openai.model } : { error: { message: 'Invalid API key' } })) }) await new Promise((resolve, reject) => { mock.once('error', reject); mock.listen(0, '127.0.0.1', resolve) }) const mockBase = `http://127.0.0.1:${mock.address().port}/v1` @@ -82,10 +82,16 @@ try { assert.deepEqual(await (await api('settings')).json(), state) assert.equal(calls.length, 3) assert.deepEqual(calls.map(call => call.method), ['GET', 'POST', 'POST']) + assert.equal((await api('models', { revision: state.revision }, { Origin: 'https://evil.example' })).status, 403) + const models = await api('models', { revision: state.revision, provider: 'openai', baseUrl: mockBase, model: DEFAULTS.openai.model, apiKey: '' }) + assert.equal(models.status, 200) + assert.deepEqual((await models.json()).models, ['gpt-image-2.5-flare', DEFAULTS.openai.model]) + assert.equal(calls.length, 4) + assert.deepEqual(await (await api('settings')).json(), state) const page = await (await fetch(base, { headers: { Cookie: cookie } })).text() assert.ok(page.includes('dsh-image-generation'), 'Image client entry must be present in the composed page') assert.ok(!/image-generation.*(?:failed|Error)/i.test(output), 'Image plugin must load successfully') - console.log('PASS: Host composition, authentication, origin checks, redacted settings, save success/failure, exactly one non-generating request per save.') + console.log('PASS: Host composition, authentication, origin checks, redacted settings, save success/failure, single-request validation and read-only model discovery.') if (process.argv.includes('--keep')) { await writeFile(path.join(home, 'browser-url.txt'), url, { mode: 0o600 }) await writeFile(path.join(home, 'smoke-context.json'), JSON.stringify({ base, mockBase }), { mode: 0o600 }) diff --git a/test/image-generation.test.mjs b/test/image-generation.test.mjs index 7cb28832d..31e244743 100644 --- a/test/image-generation.test.mjs +++ b/test/image-generation.test.mjs @@ -19,6 +19,7 @@ import { createSettings } from '../packages/dsh-image-generation/lib/settings.js import { DEFAULTS, generate, generationBody, ImageError, profile, readBounded, validateConnection } from '../packages/dsh-image-generation/lib/provider.js' import { normalizeImage } from '../packages/dsh-image-generation/lib/assets.js' import { materialize } from '../packages/dsh-image-generation/lib/storage.js' +import { writerEnvironment } from '../packages/dsh-image-generation/lib/commit.js' const cleanups = [] afterEach(async () => { for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); vi.restoreAllMocks() }) @@ -36,7 +37,7 @@ async function server() { res.writeHead(probe ? 400 : status, { 'content-type': 'application/json' }) res.end(JSON.stringify(probe ? { error: { code: 'MissingParameter', message: 'The request is missing a required parameter: prompt.' } } : status !== 200 ? { error: { message: 'secret-echo-key' } } : malformed ? {} : req.url.endsWith('/images/generations') ? { data: [{ b64_json: png.toString('base64') }] } - : req.url.endsWith('/models') ? { data: [{ id: DEFAULTS.bytedance.model }] } : { id: DEFAULTS.openai.model })) + : req.url.endsWith('/models') ? { data: [{ id: 'gpt-image-2.5-flare' }, { id: DEFAULTS.openai.model }, { id: 'gpt-image-2.5-flare' }, { id: 'gpt-5' }, { id: 'dall-e-3' }, { id: 'gpt-image-1', shutdown_date: '2020-01-01' }] } : { id: DEFAULTS.openai.model })) }) await new Promise((resolve, reject) => { http.once('error', reject); http.listen(0, '127.0.0.1', resolve) }) cleanups.push(() => { http.closeAllConnections(); return new Promise(resolve => http.close(resolve)) }) @@ -126,9 +127,31 @@ describe('image settings save and provider requests', () => { await expect(pending).rejects.toMatchObject({ code: 'CANCELLED' }) expect((await f.settings.describe()).revision).toBe(0) }) - it.each(['https://user:key@example.com/v1', 'http://example.com/v1', 'https://example.com/v1?key=x', 'https://example.com/v1/images/generations'])('rejects unsafe or mistaken endpoint %s', baseUrl => { + it.each(['https://user:key@example.com/v1', 'http://example.com/v1', 'https://example.com/v1?key=x'])('rejects unsafe endpoint %s', baseUrl => { expect(() => profile('openai', { baseUrl })).toThrow(ImageError) }) + it('accepts the console REST endpoint and requests generation with exactly one path suffix', async () => { + const s = await server() + const spec = profile('bytedance', { baseUrl: `${s.baseUrl}/images/generations/`, model: 'doubao-seedream-5-0-pro-260628' }) + expect(spec.baseUrl).toBe(s.baseUrl) + await generate('bytedance', spec, 'test-image-key', { prompt: 'A flower' }) + expect(s.calls[0].url).toBe('/v1/images/generations') + }) + it('fetches only supported visible image models with one request and keeps stored configuration unchanged', async () => { + const f = await fixture(); const s = await server() + const before = await f.settings.describe() + const result = await f.settings.models(saveInput('openai', s.baseUrl)) + expect(result).toEqual({ source: 'provider', canFetch: true, models: ['gpt-image-2.5-flare', DEFAULTS.openai.model] }) + expect(s.calls).toHaveLength(1); expect(s.calls[0]).toMatchObject({ method: 'GET', url: '/v1/models' }) + expect(await f.settings.describe()).toEqual(before) + expect(JSON.stringify(result)).not.toContain('test-image-key') + await f.settings.save(saveInput('openai', s.baseUrl)) + await f.settings.models({ ...saveInput('openai', s.baseUrl, 1), apiKey: '' }) + expect(s.calls.at(-1).auth).toBe('Bearer test-image-key') + await expect(f.settings.models({ ...saveInput('openai', 'https://other.example/v1', 1), apiKey: '' })).rejects.toMatchObject({ code: 'KEY_REQUIRED' }) + await expect(f.settings.models(saveInput('bytedance', s.baseUrl, 1))).rejects.toMatchObject({ code: 'MODEL_DISCOVERY' }) + expect(s.calls).toHaveLength(3) + }) it('maps provider canvas parameters independently and bounds streamed responses', async () => { const args = { prompt: 'A mountain', aspect_ratio: '16:9' } expect(generationBody('openai', profile('openai'), args)).toMatchObject({ size: '1536x1024', output_format: 'png' }) @@ -140,6 +163,13 @@ describe('image settings save and provider requests', () => { }) describe('image tool and durable Office assets', () => { + it('starts an Electron Host child in Node mode while keeping credentials and Node options outside the writer', () => { + const env = { ELECTRON_RUN_AS_NODE: '0', NODE_OPTIONS: '--require secret.js', API_KEY: 'private-key', PATH: '/host/bin' } + expect(writerEnvironment({ env, platform: 'darwin', versions: { electron: '43.4.0' } })).toEqual({ + ELECTRON_RUN_AS_NODE: '1', NODE_OPTIONS: undefined, API_KEY: undefined, PATH: undefined, + }) + expect(writerEnvironment({ env, platform: 'darwin', versions: { node: '24.9.0' } }).ELECTRON_RUN_AS_NODE).toBeUndefined() + }) it.each(['openai', 'bytedance'])('executes %s through ToolRuntime and writes a verifiable PNG', async provider => { const f = await fixture(); const s = await server() await f.settings.save(saveInput(provider, s.baseUrl)) @@ -182,7 +212,7 @@ describe('image tool and durable Office assets', () => { }) }) await plugin; cleanups.push(() => plugin.dispose()) expect(f.ctx.settings.describe().map(entry => entry.ns)).toContain('image-generation') - expect(routes.map(route => route.path)).toEqual(['/api/image-generation.settings', '/api/image-generation.save']) + expect(routes.map(route => route.path)).toEqual(['/api/image-generation.settings', '/api/image-generation.save', '/api/image-generation.models']) const response = await routes[0].fetch(new Request('http://localhost/api/image-generation.settings')) expect(response.headers.get('cache-control')).toBe('no-store') expect((await response.json()).profiles.openai.configured).toBe(false) From 7ddd04f9f0a43213b792f9e12a39eaa92e031bff Mon Sep 17 00:00:00 2001 From: Jianghan <58623049+SuperstructureJH@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:42:35 +0800 Subject: [PATCH 3/8] fix: use Seedream single-image contract and retain provider diagnostics --- docs/STATUS.md | 10 ++++ packages/dsh-image-generation/README.md | 4 ++ packages/dsh-image-generation/index.js | 4 +- packages/dsh-image-generation/lib/provider.js | 57 ++++++++++++++----- .../skills/generate-image/SKILL.md | 2 + scripts/verify-image-electron.mjs | 10 +++- test/image-generation.test.mjs | 55 ++++++++++++++++++ 7 files changed, 123 insertions(+), 19 deletions(-) diff --git a/docs/STATUS.md b/docs/STATUS.md index 5154fb1f9..66912f710 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1,5 +1,15 @@ # Implementation and verification status +## Seedream 5.0 Pro request compatibility — 2026-09-10 + +- **Observed:** the user's running r2 application matches commit `a325b41`; its selected provider is ByteDance, model `doubao-seedream-5-0-pro-260628`, with the canonical Ark base URL. The screenshot reports a provider rejection. The previous error handler discarded the provider response, so the exact live rejection details are unavailable. +- **Corrected:** single-image requests use the shared Seedream fields and default single-image mode. The prior unconditional `sequential_image_generation: disabled` group parameter is removed, including for opaque endpoint IDs. Model capability reference: [official Seedream table](https://docs.byteplus.com/api/docs/ModelArk/1824121). +- **Corrected:** bounded provider error parsing retains HTTP status, error code, recognized parameter and request ID in tool errors and Host logs. Free-form upstream text and credential echoes remain private. Skill guidance uses these diagnostics to distinguish adapter, input and account failures. +- **PASS:** regression against the former code reproduces 3 contract failures and 8 diagnostic failures. Corrected focused suite passes 32 image tests plus 2 closure tests. Cases cover 5 canvas ratios across 5.0 Pro, 4.5 and an opaque endpoint, ToolRuntime error delivery, credential/prompt echo exclusion, HTTP classification and malformed/oversized errors. These are loopback contract simulations. +- **PASS:** 90 files / 782 tests, TypeScript and production build. The r2 packaged plugin reproduces the same generic provider rejection inside a real Electron utility Host when the loopback provider applies the 5.0 Pro contract. +- **PASS:** r3 macOS arm64 DMG/ZIP build using local Electron 43.4.0, strict deep signature, final packaged Host authentication/save/model-query smoke and final packaged plugin in an actual Electron utility Host. Both OpenAI and the 5.0 Pro single-image contract complete governed PNG writes against loopback services. Artifacts and provenance: `outputs/image-generation-plugin-20260910-r3`. +- **Pending:** real-provider acceptance. Automatic approval review blocked a live image request using the saved Key because that request may incur provider charges; it was not executed. The exact production rejection and successful real generation remain unverified. + ## Image generation acceptance corrections — 2026-09-10 - **Fixed and reproduced:** the original package throws `The image writer returned an invalid result` under its actual Electron Helper. Clearing the environment removed Node launch mode, so the child exited successfully without running the writer. The writer now declares `ELECTRON_RUN_AS_NODE=1` for Electron Hosts while continuing to clear credentials and NODE_OPTIONS. The old package reproduces the exact error; corrected source passes with the same Helper and in a real Electron 43.4.0 utility process, for both simulated providers. diff --git a/packages/dsh-image-generation/README.md b/packages/dsh-image-generation/README.md index 90acb8077..d9897d0ef 100644 --- a/packages/dsh-image-generation/README.md +++ b/packages/dsh-image-generation/README.md @@ -26,6 +26,10 @@ API 地址同时接受基础地址和控制台提供的完整 `/images/generatio 本期支持文生图，每次生成一张图片。OpenAI 的 16:9 / 4:3 请求使用 1536×1024 画布，竖图使用 1024×1536；文档按实际返回尺寸等比放置或裁剪。字节按目标比例选择支持的画布。编辑图、参考图、批量生图和本地模型留待后续版本。 +字节请求采用 Seedream 4.5 / 5.0 Pro 共用的单图字段，由服务端默认单图模式执行。自定义接入点 ID 也使用这一契约。组图控制属于独立能力，5.0 Pro 的单图请求省略 `sequential_image_generation` 及其 options。模型能力和画布范围参考 [BytePlus 官方 Seedream 能力表](https://docs.byteplus.com/api/docs/ModelArk/1824121)。 + +生图失败时，工具和 Host 日志保留 HTTP 状态、服务商错误码、参数名及请求编号。错误响应最多读取 64 KB，仅提取受限格式的诊断字段；服务商原始错误文本可能回显 Key 或提示词，因此留在 Host 处理。模型根据具体原因反馈，参数错误交由插件修正。 + ## 独立分发 该目录是可公开分发的 MIT npm 包，包含 Host、Client、Skill 和 `dsh.bundle.patch`。本 PR 提供源码与 tarball；npm registry 发布为单独操作。 diff --git a/packages/dsh-image-generation/index.js b/packages/dsh-image-generation/index.js index e348b15d3..5d514a907 100644 --- a/packages/dsh-image-generation/index.js +++ b/packages/dsh-image-generation/index.js @@ -40,7 +40,7 @@ export function imageTool(ctx, settings) { await assetDirectory(root) writerPlan(ctx, root, exec) const spec = await settings.active() - ctx.logger.info('image-generation: generation started; provider=%s', spec.provider) + ctx.logger.info('image-generation: generation started; provider=%s model=%s', spec.provider, spec.model) const raw = await generate(spec.provider, spec, spec.key, args, { signal: exec.signal }) const image = await normalizeImage(raw) // Recheck live session permissions after a potentially long provider call. @@ -50,7 +50,7 @@ export function imageTool(ctx, settings) { return { ...asset, asset_id: `sha256:${asset.sha256}`, media_type: 'image/png', width: image.width, height: image.height, bytes: image.data.length, provider: spec.provider, model: spec.model } } catch (error) { const safe = safeError(error) - ctx.logger.info('image-generation: generation failed; code=%s', safe.code) + ctx.logger.info('image-generation: generation failed; code=%s http=%s provider_code=%s parameter=%s request_id=%s', safe.code, safe.providerStatus, safe.providerCode, safe.parameter, safe.requestId) throw safe } }, diff --git a/packages/dsh-image-generation/lib/provider.js b/packages/dsh-image-generation/lib/provider.js index ea21555f7..f56bc91c8 100644 --- a/packages/dsh-image-generation/lib/provider.js +++ b/packages/dsh-image-generation/lib/provider.js @@ -54,6 +54,39 @@ export async function readBounded(response, maxBytes, signal) { } finally { await reader.cancel().catch(() => {}); reader.releaseLock() } } +const IMAGE_PARAMETERS = new Set(['model', 'prompt', 'size', 'response_format', 'output_format', 'quality', 'n', 'watermark', 'stream', 'sequential_image_generation', 'sequential_image_generation_options', 'image', 'background', 'moderation']) +// Keep actionable identifiers, rather than forwarding a provider's free-form +// message: an upstream error can echo credentials, prompts or a proxy HTML page. +function providerFailure(response, payload, key, probe) { + const error = payload?.error + const identifier = value => typeof value === 'string' && /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/.test(value) && !(key && value.includes(key)) ? value : undefined + const details = { + providerStatus: response.status, + providerCode: identifier(error?.code), + requestId: identifier(response.headers.get('x-request-id') || response.headers.get('x-tt-logid') || payload?.request_id || error?.request_id), + } + const upstreamMessage = typeof error?.message === 'string' ? error.message : '' + // Only infer a field from a parameter-specific phrase, never from arbitrary + // mentions of model/prompt elsewhere in the provider response. + const named = upstreamMessage.match(/\b(?:parameter|argument|field)\s*[:=]?\s*[`'"\[]?([a-z_]+)\b/i)?.[1] + const parameter = IMAGE_PARAMETERS.has(error?.param) ? error.param : IMAGE_PARAMETERS.has(named) ? named : undefined + if (parameter) details.parameter = parameter + const errors = { + 401: ['AUTH', 'The API key is invalid or expired.'], + 403: ['PERMISSION', 'This API key lacks access. Enable the model and check account verification.'], + 404: ['MODEL', 'The model or API endpoint is unavailable.'], + 429: ['QUOTA', 'The provider quota or rate limit has been reached.'], + 400: ['PARAMETERS', probe ? 'The provider did not confirm the required-prompt check. Verify the API URL and model.' : 'The provider rejected the image parameters.'], + } + let [code, message] = errors[response.status] ?? ['PROVIDER_ERROR', 'The image provider could not complete the request.'] + if (response.status === 400 && parameter) { + const unsupported = /not support|unsupported|不支持/i.test(upstreamMessage) + message = unsupported ? `The image parameter "${parameter}" is unsupported for this model.` : `The provider rejected the image parameter "${parameter}".` + } + const context = [`HTTP ${details.providerStatus}`, details.providerCode && `code=${details.providerCode}`, details.requestId && `request_id=${details.requestId}`].filter(Boolean).join('; ') + return Object.assign(new ImageError(code, `${message} (${context})`, 502), details) +} + async function request(url, key, { signal, body, maxBytes = 2 * 1024 * 1024, fetchImpl = fetch, parameterProbe = false } = {}) { try { const response = await fetchImpl(url, { @@ -62,10 +95,11 @@ async function request(url, key, { signal, body, maxBytes = 2 * 1024 * 1024, fet ...(body ? { body: JSON.stringify(body) } : {}), }) if (!response.ok) { + let payload + try { payload = JSON.parse((await readBounded(response, 64 * 1024, signal)).toString('utf8')) } + catch { signal?.throwIfAborted() /* Preserve HTTP status for non-JSON/oversized error bodies. */ } if (parameterProbe && response.status === 400) { - const raw = await readBounded(response, 64 * 1024, signal) - let error - try { error = JSON.parse(raw.toString('utf8')).error } catch { /* fail closed below */ } + const error = payload?.error // Ark has no documented runtime Models API. A request with the required // prompt omitted reaches authentication/parameter validation without // submitting an inference job. Accept only the prompt-required error. @@ -74,18 +108,8 @@ async function request(url, key, { signal, body, maxBytes = 2 * 1024 * 1024, fet if (/^(?:MissingParameter|InvalidParameter)(?:\.[A-Za-z]+)?$/.test(code ?? '') && (error?.param === 'prompt' || /\bprompt\b/i.test(message)) && /required|missing|empty|not provided|not set|不能为空|必填|缺少/i.test(message)) return { probe: 'connection' } - throw new ImageError('PARAMETERS', 'The provider did not confirm the expected parameter check. Verify the API URL and model.', 502) - } - await response.body?.cancel() - const errors = { - 401: ['AUTH', 'The API key is invalid or expired.'], - 403: ['PERMISSION', 'This API key lacks access. Enable the model and check account verification.'], - 404: ['MODEL', 'The model or API endpoint is unavailable.'], - 429: ['QUOTA', 'The provider quota or rate limit has been reached.'], - 400: ['PARAMETERS', 'The provider rejected the request. Check the model and supported image parameters.'], } - const [code, message] = errors[response.status] ?? ['PROVIDER_ERROR', 'The image provider could not complete the request.'] - throw new ImageError(code, message, 502) + throw providerFailure(response, payload, key, parameterProbe) } const data = await readBounded(response, maxBytes, signal) try { return JSON.parse(data.toString('utf8')) } catch { throw new ImageError('RESPONSE', 'The provider returned invalid JSON.', 502) } @@ -130,7 +154,10 @@ export function generationBody(provider, spec, args) { size: sizes[ratio], ...(provider === 'openai' ? { n: 1, quality: 'auto', output_format: 'png' } - : { response_format: 'b64_json', sequential_image_generation: 'disabled', watermark: false }), + // Ark defaults to single-image generation. Use its shared single-image + // fields for 4.5, 5.0 Pro and opaque endpoint IDs; group controls belong + // to a separate capability and 5.0 Pro rejects them. + : { response_format: 'b64_json', watermark: false }), } } diff --git a/packages/dsh-image-generation/skills/generate-image/SKILL.md b/packages/dsh-image-generation/skills/generate-image/SKILL.md index 8d2d63a2c..0eafa890b 100644 --- a/packages/dsh-image-generation/skills/generate-image/SKILL.md +++ b/packages/dsh-image-generation/skills/generate-image/SKILL.md @@ -12,3 +12,5 @@ Use `image_generate` whenever a task needs a generated raster visual. The user c 5. Insert the returned `workspace_path` into the PPT/Word workflow, then inspect the rendered document page. Keep visual assets linked to their purpose and reuse their paths across later edits. Return the document and any requested standalone image. If the tool reports missing configuration, direct the user to Settings → Plugins → Image generation. Never request an API key in conversation or place one in a document, prompt, sandbox command or source file. Respect tool approval, cancellation and workspace policy. Surface provider errors accurately; use a placeholder only when the user accepts one. Configuration validation checks the connection without submitting an image generation job; successful generation is established by an actual generated image. + +For a provider rejection, report the returned HTTP status, error code, parameter and request ID when present. A rejected adapter-owned parameter requires a plugin correction; preserve those diagnostics for the developer. Change a prompt or aspect ratio only when the reported cause points to that input. Describe account access or quota issues when the tool explicitly reports that category. The settings card validates automatically on Save. diff --git a/scripts/verify-image-electron.mjs b/scripts/verify-image-electron.mjs index f8ef7d86a..9f8e41bc9 100644 --- a/scripts/verify-image-electron.mjs +++ b/scripts/verify-image-electron.mjs @@ -26,7 +26,13 @@ if (process.argv.includes('--worker')) { const png = await sharp({ create: { width: 256, height: 192, channels: 4, background: '#12345680' } }).png().toBuffer() const server = createServer(async (request, response) => { let body = ''; for await (const part of request) body += part - assert.ok(JSON.parse(body).prompt); calls++ + const input = JSON.parse(body) + assert.ok(input.prompt); calls++ + if (input.model === 'doubao-seedream-5-0-pro-260628' && (['sequential_image_generation', 'sequential_image_generation_options'].some(field => Object.hasOwn(input, field)) || input.stream === true)) { + response.writeHead(400, { 'content-type': 'application/json' }) + response.end(JSON.stringify({ error: { code: 'InvalidParameter', param: 'sequential_image_generation', message: 'The parameter is unsupported for this model.' } })) + return + } response.writeHead(200, { 'content-type': 'application/json' }) response.end(JSON.stringify({ data: [{ b64_json: png.toString('base64') }] })) }) @@ -42,7 +48,7 @@ if (process.argv.includes('--worker')) { const meta = await sharp(data).metadata(); assert.equal(meta.format, 'png'); assert.equal(meta.hasAlpha, true) } assert.equal(calls, 2) - console.log(`PASS: real Electron ${process.versions.electron} utility Host, both adapters and sandboxed PNG writer (loopback simulation).`) + console.log(`PASS: real Electron ${process.versions.electron} utility Host, OpenAI and Seedream 5.0 Pro single-image contract, sandboxed PNG writer (loopback simulation).`) } finally { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)) for (const fork of forks.reverse()) await fork.dispose() diff --git a/test/image-generation.test.mjs b/test/image-generation.test.mjs index 31e244743..6e2c20b55 100644 --- a/test/image-generation.test.mjs +++ b/test/image-generation.test.mjs @@ -34,6 +34,19 @@ async function server() { if (redirect) { res.writeHead(307, { Location: redirect }); res.end(); return } if (delay) await new Promise(resolve => setTimeout(resolve, delay)) const probe = status === 200 && !malformed && req.url.endsWith('/images/generations') && body && !Object.hasOwn(JSON.parse(body), 'prompt') + const input = body && JSON.parse(body) + // The 5.0 Pro contract supports single images and rejects group controls. + // Treat an opaque endpoint as 5.0 Pro too: its ID conveys no capabilities. + if (status === 200 && input?.prompt && ['doubao-seedream-5-0-pro-260628', 'ep-custom'].includes(input.model)) { + const parameter = ['sequential_image_generation', 'sequential_image_generation_options'].find(field => Object.hasOwn(input, field)) || (input.stream === true ? 'stream' : undefined) + const [width, height] = input.size.split('x').map(Number) + const invalidSize = !(width * height >= 921600 && width * height <= 4624220 && width / height >= 1 / 16 && width / height <= 16) + if (parameter || invalidSize) { + res.writeHead(400, { 'content-type': 'application/json', 'x-request-id': 'seedream-contract-request' }) + res.end(JSON.stringify({ error: { code: 'InvalidParameter', param: parameter || 'size', message: parameter ? `The parameter ${parameter} is not supported by this model.` : 'The size is outside the supported range.' } })) + return + } + } res.writeHead(probe ? 400 : status, { 'content-type': 'application/json' }) res.end(JSON.stringify(probe ? { error: { code: 'MissingParameter', message: 'The request is missing a required parameter: prompt.' } } : status !== 200 ? { error: { message: 'secret-echo-key' } } : malformed ? {} : req.url.endsWith('/images/generations') ? { data: [{ b64_json: png.toString('base64') }] } @@ -160,6 +173,48 @@ describe('image settings save and provider requests', () => { expect(generationBody('bytedance', profile('bytedance'), args)).not.toHaveProperty('quality') await expect(readBounded(new Response('too much'), 2)).rejects.toMatchObject({ code: 'TOO_LARGE' }) }) + it.each(['doubao-seedream-5-0-pro-260628', DEFAULTS.bytedance.model, 'ep-custom'])('generates every supported aspect ratio with the single-image contract for %s', async model => { + const s = await server() + for (const aspect_ratio of ['1:1', '16:9', '9:16', '4:3', '3:4']) { + expect(await generate('bytedance', { baseUrl: s.baseUrl, model }, 'test-image-key', { prompt: 'A flower', aspect_ratio })).toEqual(s.png) + } + expect(s.calls).toHaveLength(5) + }) + it('reports the rejected parameter and request ID through ToolRuntime while keeping provider echoes private', async () => { + const f = await fixture(); const s = await server() + await f.settings.save(saveInput('bytedance', s.baseUrl)) + const original = globalThis.fetch + const stub = vi.spyOn(globalThis, 'fetch').mockImplementation((url, options) => String(url).endsWith('/images/generations') + ? Promise.resolve(Response.json({ error: { code: 'InvalidParameter', message: 'The parameter `sequential_image_generation` is not supported. Bearer test-image-key. PRIVATE-PROMPT' } }, { status: 400, headers: { 'x-request-id': 'req-123' } })) + : original(url, options)) + f.ctx.tools.register(imageTool(f.services, f.settings)) + const result = await f.ctx.tools.execute({ callId: 'provider-error', name: 'image_generate', arguments: { prompt: 'PRIVATE-PROMPT' }, agent: f.agent, signal: new AbortController().signal }) + const text = JSON.stringify(result) + expect(result.isError).toBe(true) + for (const value of ['sequential_image_generation', 'unsupported', 'InvalidParameter', 'req-123', 'HTTP 400']) expect(text).toContain(value) + for (const value of ['test-image-key', 'PRIVATE-PROMPT', 'Bearer']) { + expect(text).not.toContain(value) + expect(JSON.stringify(f.log.mock.calls)).not.toContain(value) + } + expect(JSON.stringify(f.log.mock.calls)).toContain('req-123') + expect(stub).toHaveBeenCalledTimes(1) + }) + it.each([401, 403, 404, 429, 500])('preserves HTTP %s classification with private or malformed error bodies', async status => { + const fetchImpl = vi.fn(async () => Response.json({ error: { code: 'private-key', param: 'private-key', message: 'private-key' }, request_id: 'private-key' }, { status })) + try { + await generate('openai', profile('openai'), 'private-key', { prompt: 'Flower' }, { fetchImpl }) + expect.unreachable() + } catch (error) { + expect(error.code).toBe(({ 401: 'AUTH', 403: 'PERMISSION', 404: 'MODEL', 429: 'QUOTA', 500: 'PROVIDER_ERROR' })[status]) + expect(error.message).toContain(`HTTP ${status}`) + expect(error.message).not.toContain('private-key') + } + expect(fetchImpl).toHaveBeenCalledTimes(1) + }) + it.each(['Gateway error', 'x'.repeat(65537)])('preserves provider status when the error response is invalid or oversized (%#)', async body => { + await expect(generate('bytedance', profile('bytedance'), 'key', { prompt: 'Flower' }, { fetchImpl: async () => new Response(body, { status: 400 }) })) + .rejects.toMatchObject({ code: 'PARAMETERS', providerStatus: 400 }) + }) }) describe('image tool and durable Office assets', () => { From 4b29c8db6876e4d0d6ca2ae681523fa07b6e83e4 Mon Sep 17 00:00:00 2001 From: Jianghan <58623049+SuperstructureJH@users.noreply.github.com> Date: Thu, 10 Sep 2026 12:11:22 +0800 Subject: [PATCH 4/8] feat(image-generation): streamline calls and preview generated images --- docs/STATUS.md | 10 ++ package-lock.json | 3 +- packages/dsh-image-generation/README.md | 10 +- packages/dsh-image-generation/client.js | 109 +++++++++++++----- packages/dsh-image-generation/index.js | 7 +- packages/dsh-image-generation/lib/preview.js | 69 +++++++++++ packages/dsh-image-generation/package.json | 57 +++++++-- .../skills/generate-image/SKILL.md | 2 +- scripts/image-preview-fixture.mjs | 39 +++++++ scripts/verify-image-generation.mjs | 11 +- test/image-generation.test.mjs | 84 ++++++++++++-- 11 files changed, 346 insertions(+), 55 deletions(-) create mode 100644 packages/dsh-image-generation/lib/preview.js create mode 100644 scripts/image-preview-fixture.mjs diff --git a/docs/STATUS.md b/docs/STATUS.md index 66912f710..d4cce1fc8 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1,5 +1,15 @@ # Implementation and verification status +## Direct image calls, concise settings and image preview — 2026-09-10 + +- **Implemented:** saved image configuration authorizes direct `image_generate` calls through the standard ToolRuntime; the plugin's additional approval request is removed. Deployment guards and workspace policy continue to apply. The settings card uses the requested description, Seedream / OpenAI labels and essential fields, with automatic validation on Save and concise error feedback. +- **Implemented:** successful image results render a thumbnail beside the answer using the Harness attachment gallery and lightbox. Click opens the original; Escape closes it and restores focus. Completed images sit before the turn footer so process folding, document deliverable contributions and branch actions remain available. Historical successful native and code-dispatch image results use the same canonical asset contract. +- **Implemented:** the authenticated preview route resolves cold/live session history, requires a successful image tool reference, and reads exactly its generated PNG. Canonical directory, symlink, file size/type and SHA-256 checks precede the response; reads are audited and responses use no-store. Browser Blob URLs and pending reads are released on unmount. Model-facing results remain text asset metadata, compatible with text-only chat models. +- **PASS:** 36 image regressions plus 2 dependency-closure checks; full suite 90 files / 786 tests, TypeScript and production build. Regression cases include direct calls with saved configuration, deployment rejection, history authorization, native/code-dispatch results, invalid references, modified files and symlink escapes. Real Host smoke confirms unauthenticated preview rejection and authenticated historical PNG equality. +- **PASS:** actual Harness browser inspection with an isolated simulated image record: thumbnail survives reload and process folding, lightbox/close/focus restoration, enabled branch action, light/dark display and concise configuration with successful automatic Save validation. This is application UI acceptance using fixture pixels. +- **Observed:** the user's latest screenshot reports successful Seedream generation and a saved 2560 x 1440 PNG. This supersedes the earlier user-visible failure; provider generation and the image's visual quality were not independently re-run with a paid request in this revision. +- **PASS:** r4 macOS arm64 development DMG/ZIP, strict deep signature, final packaged Host configuration/model/historical-preview smoke, final plugin in an actual Electron utility Host with both loopback adapters, and source/tarball/app/ZIP parity (14 source/tar files; 13 Desktop files). Artifacts, UI screenshots and hashes are in `outputs/image-generation-plugin-20260910-r4`. Office document, other-platform and notarization acceptance remain separate. + ## Seedream 5.0 Pro request compatibility — 2026-09-10 - **Observed:** the user's running r2 application matches commit `a325b41`; its selected provider is ByteDance, model `doubao-seedream-5-0-pro-260628`, with the canonical Ark base URL. The screenshot reports a provider rejection. The previous error handler discarded the provider response, so the exact live rejection details are unavailable. diff --git a/package-lock.json b/package-lock.json index 571cb1857..c8fec139e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16445,7 +16445,8 @@ "@deepseek-ai/dsh-skill": "^0.1.2-rc.1", "@deepseek-ai/dsh-subprocess": "^0.1.2-rc.1", "@deepseek-ai/dsh-system-prompt": "^0.1.2-rc.1", - "@deepseek-ai/dsh-tools": "^0.1.2-rc.1" + "@deepseek-ai/dsh-tools": "^0.1.2-rc.1", + "@deepseek-ai/dsh-api-session-controller": "^0.1.2-rc.1" } }, "packages/dshmarket": { diff --git a/packages/dsh-image-generation/README.md b/packages/dsh-image-generation/README.md index d9897d0ef..630cfcf89 100644 --- a/packages/dsh-image-generation/README.md +++ b/packages/dsh-image-generation/README.md @@ -6,7 +6,7 @@ DSH/Cordis 公共生图插件，提供 `image_generate` 工具、`generate-image DSH Desktop 默认装载本插件。在生图工具卡片选择字节或 OpenAI，填写该平台的 API Key 并保存。模型和地址已有默认值，可在高级设置中修改。 -生图模型使用下拉选择。OpenAI 的「获取模型」用当前填写或已保存的 Key 发起一次 `GET /models`，筛选 Images API 支持的 GPT Image 模型；获取过程只更新候选列表，配置在点击保存时生效。列表区分「内置模型」与「此 Key 可见模型」，空列表和查询错误明确显示。字节的管理接口使用独立签名凭据，因此 API Key 模式提供 Seedream 5.0 Pro、4.5 的内置选项和自定义模型/接入点 ID。 +生图模型使用下拉选择。OpenAI 的「获取模型」用当前填写或已保存的 Key 发起一次 `GET /models`，筛选 Images API 支持的 GPT Image 模型；获取过程只更新候选列表，配置在点击保存时生效。查询来源保留在接口中，界面显示候选模型、空列表和查询错误。字节的管理接口使用独立签名凭据，因此 API Key 模式提供 Seedream 5.0 Pro、4.5 的内置选项和自定义模型/接入点 ID。 API 地址同时接受基础地址和控制台提供的完整 `/images/generations` 地址。保存时统一为基础地址，实际请求只添加一次接口路径。 @@ -22,7 +22,7 @@ API 地址同时接受基础地址和控制台提供的完整 `/images/generatio 校验结果与实际生图成功分别记录。生图权限、余额、内容审核和输出效果在真正调用时确认。本插件没有生成测试图按钮。 -提示词示例：「为这份 PPT 生成一张留出左侧标题空间的科技插画，风格沿用当前模板。」Agent 加载 Skill 后调用 `image_generate`，经过当前 Host 审批流程，将图片保存到工作区 `.workbuddy/generated-images/.png`。工具返回路径、尺寸、字节数、哈希、服务商和模型。图片可保留透明通道，标题、表格和简单图表继续使用 Office 原生对象。 +提示词示例：「为这份 PPT 生成一张留出左侧标题空间的科技插画，风格沿用当前模板。」Agent 加载 Skill 后调用 `image_generate`，按当前 Host 权限策略直接执行，将图片保存到工作区 `.workbuddy/generated-images/.png`。工具返回路径、尺寸、字节数、哈希、服务商和模型。图片可保留透明通道，标题、表格和简单图表继续使用 Office 原生对象。 本期支持文生图，每次生成一张图片。OpenAI 的 16:9 / 4:3 请求使用 1536×1024 画布，竖图使用 1024×1536；文档按实际返回尺寸等比放置或裁剪。字节按目标比例选择支持的画布。编辑图、参考图、批量生图和本地模型留待后续版本。 @@ -30,6 +30,8 @@ API 地址同时接受基础地址和控制台提供的完整 `/images/generatio 生图失败时，工具和 Host 日志保留 HTTP 状态、服务商错误码、参数名及请求编号。错误响应最多读取 64 KB，仅提取受限格式的诊断字段；服务商原始错误文本可能回显 Key 或提示词，因此留在 Host 处理。模型根据具体原因反馈，参数错误交由插件修正。 +对话直接展示生成图片，点击缩略图可放大，Esc 或点击背景关闭。已有成功生图记录也能预览。预览接口通过 Host 鉴权，按会话中成功的工具结果授权读取对应 PNG，并校验目录、文件类型、体积和 SHA-256；访问留有审计日志。图片以文本资产信息返回给聊天模型，界面预览与模型视觉能力分别处理。 + ## 独立分发 该目录是可公开分发的 MIT npm 包，包含 Host、Client、Skill 和 `dsh.bundle.patch`。本 PR 提供源码与 tarball；npm registry 发布为单独操作。 @@ -40,7 +42,7 @@ API 地址同时接受基础地址和控制台提供的完整 `/images/generatio npm pack ./packages/dsh-image-generation --pack-destination /absolute/output/directory ``` -在具有匹配 Harness 服务的独立 DSH 安装中，可通过 `dsh plugin --profile web add /absolute/path/dsh-image-generation-0.1.0.tgz` 安装。要求 Harness `0.1.2-rc.1` 对应的 settings、credentials、connection、tools、skills、systemPrompt、sandboxPolicy、sandbox 和 subprocess 服务。Desktop 已默认装载时直接配置即可。 +在具有匹配 Harness 服务的独立 DSH 安装中，可通过 `dsh plugin --profile web add /absolute/path/dsh-image-generation-0.1.0.tgz` 安装。要求 Harness `0.1.2-rc.1` 对应的 settings、credentials、connection、tools、skills、systemPrompt、sandboxPolicy、sandbox、subprocess 和 sessionController 服务。Desktop 已默认装载时直接配置即可。 ## 凭据与执行边界 @@ -48,7 +50,7 @@ npm pack ./packages/dsh-image-generation --pack-destination /absolute/output/dir 服务商、模型、地址和 Key 在 `ctx.credentials` 的 `dsh-image-generation/configuration` 私有记录中一次提交。凭据服务的跨进程锁和修订检查防止并发保存覆盖；失败保存保持整个旧记录。使用同一记录使一次工具调用读到一致的配置。通用 settings 仅注册卡片命名空间。Agent 参数、图片、日志和工作区均不携带 Key。 -生图通过标准工具执行和审计流程，并默认要求 Host 审批。请求前及落盘前读取当前会话的 sandboxPolicy，要求工作区写权限；输出目录和目标文件拒绝符号链接。实际写入由 `ctx.sandbox` 包装的 `ctx.subprocess` 子进程执行，使用工作区写权限；仅通过 stdin 接收图片字节，清除继承环境，Key 留在 Host。PNG 使用临时文件、原子创建和内容哈希校验，避免并发生图产生半个文件。HTTP 禁止重定向，只接受 HTTPS 地址（本机回环开发服务允许 HTTP），约束超时、响应大小和解码像素；不下载厂商返回的任意远程 URL。系统沙箱的跨平台执行能力以部署环境为准，本期本地验收环境为 macOS arm64。 +生图通过标准工具执行和审计流程，使用已保存配置直接调用；部署级工具策略继续生效。请求前及落盘前读取当前会话的 sandboxPolicy，要求工作区写权限；输出目录和目标文件拒绝符号链接。实际写入由 `ctx.sandbox` 包装的 `ctx.subprocess` 子进程执行，使用工作区写权限；仅通过 stdin 接收图片字节，清除继承环境，Key 留在 Host。PNG 使用临时文件、原子创建和内容哈希校验，避免并发生图产生半个文件。HTTP 禁止重定向，只接受 HTTPS 地址（本机回环开发服务允许 HTTP），约束超时、响应大小和解码像素；不下载厂商返回的任意远程 URL。系统沙箱的跨平台执行能力以部署环境为准，本期本地验收环境为 macOS arm64。 macOS Desktop 的 Host 运行在 Electron utility process 中；写图子进程显式设置 `ELECTRON_RUN_AS_NODE=1`，使同一可执行文件执行 Node 脚本。该启动常量与 Windows 的 SystemRoot 按需保留，凭据和 NODE_OPTIONS 继续清除。回归同时覆盖普通 Node 和真实 Electron utility process。 diff --git a/packages/dsh-image-generation/client.js b/packages/dsh-image-generation/client.js index bada5fef5..2baf8197f 100644 --- a/packages/dsh-image-generation/client.js +++ b/packages/dsh-image-generation/client.js @@ -6,15 +6,11 @@ window.__ModuleLoader__.load({ const h = React.createElement const NS = 'settings.imageGeneration' const zh = { - title: '生图工具', description: '为 PPT、Word 和对话生成图片。', provider: '服务商', bytedance: '字节 · 豆包 Seedream', openai: 'OpenAI', - apiKey: 'API Key', keyPlaceholder: '输入服务商的 API Key', savedKey: '已配置，输入新 Key 可替换', keyLink: '获取 API Key', - advanced: '高级设置', model: '模型 ID', baseUrl: 'API 地址', defaults: '支持基础地址或控制台的完整生图接口地址，保存时自动识别。', - save: '保存', saving: '正在校验…', saved: '已保存，连接校验通过', hint: '保存时自动校验连接；模型的生图权限和额度在实际调用时确认。', - configured: '已配置', loading: '正在读取配置…', reload: '重新读取配置', readOnly: '当前配置由管理员管理。', - modelSelect: '生图模型', fetchModels: '获取模型', fetchingModels: '正在获取…', customModel: '自定义模型 / 接入点', - builtinModels: '内置模型，可直接选择；模型访问权限在保存及实际调用时确认。', - byteModels: '字节的账号模型查询需要独立管理凭据。当前提供内置模型，也可填写自己的接入点 ID。', - fetchedModels: '已获取此 Key 可见且工具支持的生图模型。', emptyModels: '此 Key 的列表未返回工具支持的生图模型，可检查权限或填写自定义模型。', + previewLoading: '正在加载图片…', previewRetry: '重新加载图片', generatedImage: '生成的图片', title: '生图工具', description: '接入生图，有生图能力的模型', provider: '服务商', bytedance: 'Seedream', openai: 'OpenAI', + apiKey: 'API Key', keyPlaceholder: '输入服务商的 API Key', savedKey: '输入新 Key 可替换', + advanced: '高级设置', model: '模型 ID', baseUrl: 'API 地址', + save: '保存', saving: '正在校验…', saved: '已保存，连接校验通过', loading: '正在读取配置…', reload: '重新读取配置', readOnly: '当前配置由管理员管理。', + modelSelect: '生图模型', fetchModels: '获取模型', fetchingModels: '正在获取…', customModel: '自定义模型 / 接入点', emptyModels: '此 Key 的列表未返回工具支持的生图模型，可检查权限或填写自定义模型。', MODEL_DISCOVERY: '当前服务商使用内置模型或自定义接入点。', AUTH: 'API Key 无效或已过期，请检查后重新保存。', PERMISSION: '当前 Key 无权访问，请确认模型已开通及账号已完成所需认证。', MODEL: '无法访问所选模型，请检查模型 ID、接入点或模型开通状态。', KEY_REQUIRED: '请填写 API Key；更换服务地址后需要重新填写。', @@ -25,15 +21,11 @@ window.__ModuleLoader__.load({ UNAVAILABLE: '连接失败，请检查网络和 API 地址后重新保存。', CANCELLED: '校验已取消，可以重新保存。', PROVIDER_ERROR: '服务商暂时不可用，请稍后重新保存。', } const en = { - title: 'Image generation', description: 'Generate images for PPT, Word and conversations.', provider: 'Provider', bytedance: 'ByteDance · Seedream', openai: 'OpenAI', - apiKey: 'API Key', keyPlaceholder: 'Enter your provider API key', savedKey: 'Configured; enter a new key to replace it', keyLink: 'Get an API key', - advanced: 'Advanced settings', model: 'Model ID', baseUrl: 'API URL', defaults: 'Enter a base URL or the full image-generation endpoint. Saving normalizes it automatically.', - save: 'Save', saving: 'Validating…', saved: 'Saved. Connection validated.', hint: 'Saving validates the connection automatically. Image permissions and quota are confirmed during generation.', - configured: 'Configured', loading: 'Loading settings…', reload: 'Reload settings', readOnly: 'These settings are managed by your administrator.', - modelSelect: 'Image model', fetchModels: 'Fetch models', fetchingModels: 'Fetching…', customModel: 'Custom model / endpoint', - builtinModels: 'Built-in models. Access is checked when saving and generating.', - byteModels: 'ByteDance account discovery requires separate management credentials. Choose the built-in model or enter your endpoint ID.', - fetchedModels: 'Loaded image models visible to this key and supported by this tool.', emptyModels: 'No supported image models were returned. Check access or enter a custom model.', + previewLoading: 'Loading image…', previewRetry: 'Reload image', generatedImage: 'Generated image', title: 'Image generation', description: 'Connect models with image generation capabilities.', provider: 'Provider', bytedance: 'Seedream', openai: 'OpenAI', + apiKey: 'API Key', keyPlaceholder: 'Enter your provider API key', savedKey: 'Enter a new key to replace it', + advanced: 'Advanced settings', model: 'Model ID', baseUrl: 'API URL', + save: 'Save', saving: 'Validating…', saved: 'Saved. Connection validated.', loading: 'Loading settings…', reload: 'Reload settings', readOnly: 'These settings are managed by your administrator.', + modelSelect: 'Image model', fetchModels: 'Fetch models', fetchingModels: 'Fetching…', customModel: 'Custom model / endpoint', emptyModels: 'No supported image models were returned. Check access or enter a custom model.', MODEL_DISCOVERY: 'Use a built-in model or a custom endpoint for this provider.', AUTH: 'The API key is invalid or expired.', PERMISSION: 'Check model access and account verification.', MODEL: 'Check the model ID, inference endpoint and model access.', KEY_REQUIRED: 'Enter an API key. A new API origin requires you to enter the key again.', QUOTA: 'Check provider quota and rate limits.', @@ -49,11 +41,11 @@ window.__ModuleLoader__.load({ .dshImageCard:hover{border-color:var(--dsw-alias-label-dimmed)}.dshImageCardOpen{background:var(--dsw-alias-bg-layer-2);border-color:var(--dsw-alias-label-dimmed)} .dshImageHeader{appearance:none;display:flex;align-items:center;gap:12px;width:100%;padding:14px 16px;background:transparent;border:0;border-radius:12px;text-align:left;color:inherit;font:inherit;cursor:pointer} .dshImageHeading{display:flex;flex:1;flex-direction:column;gap:4px;min-width:0}.dshImageTitle{color:var(--dsw-alias-label-primary);font-size:15px;font-weight:600;line-height:1.4} - .dshImageDescription{color:var(--dsw-alias-label-tertiary);font-size:13px;line-height:1.5}.dshImageBadge{corner-shape:round;white-space:nowrap;background:var(--dsw-alias-bg-module-platform);color:var(--dsw-alias-label-secondary);border-radius:999px;flex:none;padding:1px 8px;font-size:11px;font-weight:500;line-height:17px} + .dshImageDescription{color:var(--dsw-alias-label-tertiary);font-size:13px;line-height:1.5} .dshImageChevron{color:var(--dsw-alias-label-tertiary);flex:none;transition:transform .16s}.dshImageHeader[aria-expanded=true] .dshImageChevron{transform:rotate(180deg)} .dshImageBody{border-top:.5px solid var(--dsw-alias-border-l2);margin:0 16px;padding-bottom:8px;display:flex;flex-direction:column;gap:12px}.dshImageFields{border:0;margin:0;padding:0;display:flex;flex-direction:column;min-width:0} .dshImageField{display:flex;flex-direction:column;gap:6px;padding:12px 0;font-size:13px;font-weight:500;line-height:1.5;color:var(--dsw-alias-label-primary)}.dshImageField+.dshImageField{border-top:.5px solid var(--dsw-alias-border-l2)}.dshImageField input,.dshImageField select{box-sizing:border-box;width:100%;min-width:0;height:34px;padding:0 12px;border:.5px solid var(--dsw-alias-border-l4);border-radius:8px;background:var(--dsw-alias-bg-layer-3);color:var(--dsw-alias-label-primary);font:inherit;font-weight:400} - .dshImageField input::placeholder{color:var(--dsw-alias-label-tertiary)}.dshImageLink{align-self:flex-start;font-size:12px;color:var(--dsw-alias-label-secondary);text-decoration:underline} + .dshImageField input::placeholder{color:var(--dsw-alias-label-tertiary)} .dshImageAdvanced{margin-top:12px}.dshImageAdvanced summary{cursor:pointer;font-size:13px;color:var(--dsw-alias-label-secondary)}.dshImageAdvanced[open]{display:flex;flex-direction:column} .dshImageHint{font-size:12px;line-height:1.5;color:var(--dsw-alias-label-tertiary);margin:0}.dshImageActions{border-top:.5px solid var(--dsw-alias-border-l2);display:flex;justify-content:flex-end;align-items:center;gap:8px;padding:12px 0 4px;flex-wrap:wrap} .dshImageSave{appearance:none;font:inherit;font-size:13px;line-height:1.5;border:1px solid transparent;border-radius:8px;padding:5px 14px;background:var(--dsw-alias-label-primary);color:var(--dsw-alias-bg-layer-3);cursor:pointer}.dshImageSave:disabled{opacity:.4;cursor:default} @@ -137,7 +129,6 @@ window.__ModuleLoader__.load({ return h('li', { className: `dshImageCard${expanded ? ' dshImageCardOpen' : ''}`, 'data-testid': 'image-generation-card' }, h('button', { className: 'dshImageHeader', type: 'button', 'aria-expanded': expanded, 'aria-controls': `${id}-body`, onClick: () => setExpanded(value => !value) }, h('span', { className: 'dshImageHeading' }, h('span', { className: 'dshImageTitle' }, t('title')), h('span', { className: 'dshImageDescription' }, t('description'))), - saved?.profiles[saved.provider]?.configured && h('span', { className: 'dshImageBadge' }, t('configured')), h(IconChevronDownOutline14, { className: 'dshImageChevron' })), expanded && h('form', { id: `${id}-body`, className: 'dshImageBody', onSubmit: save, 'aria-busy': busy || fetching || loading }, loading ? h('p', { className: 'dshImageHint' }, t('loading')) : draft && h(React.Fragment, null, @@ -145,31 +136,95 @@ window.__ModuleLoader__.load({ h('label', { className: 'dshImageField' }, t('provider'), h('select', { name: 'provider', value: provider, onChange: event => { setProvider(event.target.value); setStatus(''); setError('') } }, h('option', { value: 'bytedance' }, t('bytedance')), h('option', { value: 'openai' }, t('openai')))), field('apiKey', 'apiKey', 'password', t(draft.configured ? 'savedKey' : 'keyPlaceholder')), - h('a', { className: 'dshImageLink', href: provider === 'openai' ? 'https://platform.openai.com/api-keys' : 'https://console.volcengine.com/ark/apiKey', target: '_blank', rel: 'noreferrer' }, t('keyLink')), h('label', { className: 'dshImageField' }, t('modelSelect'), h('select', { name: 'model', value: customModels[provider] || !catalog.models.includes(draft.model) ? '__custom__' : draft.model, onChange: event => { const custom = event.target.value === '__custom__'; setCustomModels(previous => ({ ...previous, [provider]: custom })); if (!custom) edit('model', event.target.value) } }, ...catalog.models.map(model => h('option', { key: model, value: model }, model)), h('option', { value: '__custom__' }, t('customModel')))), (customModels[provider] || !catalog.models.includes(draft.model)) && field('model', 'model'), catalog.canFetch && h('button', { className: 'dshImageFetch', type: 'button', disabled: !draft.apiKey.trim() && !draft.configured, onClick: fetchModels }, t(fetching ? 'fetchingModels' : 'fetchModels')), - h('p', { className: 'dshImageHint', role: 'status' }, t(catalog.source === 'provider' ? catalog.models.length ? 'fetchedModels' : 'emptyModels' : provider === 'bytedance' ? 'byteModels' : 'builtinModels')), + catalog.source === 'provider' && catalog.models.length === 0 && h('p', { className: 'dshImageHint', role: 'status' }, t('emptyModels')), h('details', { className: 'dshImageAdvanced' }, h('summary', null, t('advanced')), field('baseUrl', 'baseUrl'))), - h('p', { className: 'dshImageHint' }, t('defaults')), h('div', { className: 'dshImageActions' }, h('button', { className: 'dshImageSave', type: 'submit', disabled: busy || fetching || !saved.writable || !draft.model.trim() || (!draft.apiKey.trim() && !draft.configured) }, t(busy ? 'saving' : 'save')), status && h('p', { className: 'dshImageStatus', role: 'status' }, t(status))), - h('p', { className: 'dshImageHint' }, t(saved.writable ? 'hint' : 'readOnly'))), + !saved.writable && h('p', { className: 'dshImageHint' }, t('readOnly'))), error && h('p', { className: 'dshImageStatus dshImageError', role: 'alert' }, t(Object.hasOwn(en, error) ? error : 'UNAVAILABLE')), !loading && !saved && h('button', { className: 'dshImageSave', type: 'button', onClick: () => load(lifetime.current.signal) }, t('reload')))) } + function generatedResult(event) { + const native = event.type === 'tool/result' && event.surfaceOp === 'append' + const dispatched = event.type === 'tool/code-dispatch' && event.data.name === 'image_generate' + if (!native && !dispatched) return + const result = native ? event.data.message?.content?.[0] : event.data + const id = native ? event.data.message?.source?.callId : event.data.subCallId + if (!id || result?.isError || !Array.isArray(result?.content)) return + for (const block of result.content) { + if (block?.type !== 'text' || typeof block.text !== 'string' || block.text.length > 16384) continue + let image + try { image = JSON.parse(block.text) } catch { continue } + if (image && /^[a-f0-9]{64}$/.test(image.sha256) && image.asset_id === `sha256:${image.sha256}` + && image.workspace_path === `.workbuddy/generated-images/${image.sha256}.png` && image.media_type === 'image/png' + && Number.isInteger(image.width) && image.width > 0 && Number.isInteger(image.height) && image.height > 0 + && Number.isInteger(image.bytes) && image.bytes > 0) return { id: `${native ? 'call' : 'dispatch'}:${id}`, image } + } + } + const imageDefinition = { + kind: 'generated-image', target: 'chat', + match(event) { const value = generatedResult(event); return value ? { id: value.id, role: 'start' } : null }, + start(_context, match) { return generatedResult(match.event).image }, + update(context) { return context.state }, + buildViewNode(context) { + if (!context.start || !context.state) return null + const location = context.start.location + // Completed images belong beside the answer, after the folded process. + const turn = location.kind === 'turn' || location.kind === 'step' ? location.turn : undefined + const closing = turn?.data.get('turn-tail')?.closing + const anchorSeq = closing ? closing.finalNode.seq + 0.025 : turn?.end ? turn.end.seq - 0.25 : context.start.event.seq + return { key: context.key, id: context.id, kind: 'generated-image', target: 'chat', + anchorSeq, location, visibility: 'visible', data: context.state } + }, + } + function GeneratedImage({ node, loadImage, renderMessageImages, t }) { + const image = node.data + const [url, setUrl] = React.useState(null) + const [error, setError] = React.useState(false) + const [attempt, setAttempt] = React.useState(0) + React.useEffect(() => { + const controller = new AbortController() + let objectUrl + setUrl(null); setError(false) + void loadImage(image.sha256, controller.signal).then(blob => { + if (controller.signal.aborted) return + objectUrl = URL.createObjectURL(blob); setUrl(objectUrl) + }, () => { if (!controller.signal.aborted) setError(true) }) + return () => { controller.abort(); if (objectUrl) URL.revokeObjectURL(objectUrl) } + }, [loadImage, image.sha256, attempt]) + if (error) return h('button', { type: 'button', className: 'dshImageFetch', onClick: () => setAttempt(value => value + 1) }, t('previewRetry')) + if (!url) return h('p', { className: 'dshImageHint', role: 'status' }, t('previewLoading')) + return renderMessageImages({ images: [{ preview: { url, width: image.width, height: image.height, name: t('generatedImage') } }], align: 'start' }) + } + function previewLoader(sessionId) { + return async (sha, signal) => { + if (!/^[a-f0-9]{64}$/.test(sha)) throw new Error('Invalid image reference') + const response = await fetch(`/api/image-generation.preview?session=${encodeURIComponent(sessionId)}&asset=${sha}`, + { cache: 'no-store', signal: AbortSignal.any([signal, AbortSignal.timeout(30000)]) }) + if (!response.ok || response.headers.get('content-type') !== 'image/png') throw new Error('Image preview unavailable') + return response.blob() + } + } return { - inject: ['slots', 'locale'], + inject: ['slots', 'locale', 'uiConversation'], apply(ctx) { + ctx.uiConversation.events.register(imageDefinition) + ctx.slots.inject('conversation.chat.node', () => ctx.slots.register({ + name: 'conversation.chat.node', key: 'generated-image', locale: NS, + inject: sessionId => ({ loadImage: previewLoader(sessionId) }), + }, GeneratedImage)) ctx.effect(() => ctx.locale.register(NS, { zh, en }), 'image-generation locale') ctx.effect(() => { const style = document.createElement('style'); style.dataset.pluginCss = 'dsh-image-generation'; style.textContent = css document.head.appendChild(style) return () => style.remove() }, 'image-generation styles') - ctx.slots.inject('settings.plugin.item', () => ctx.slots.register({ name: 'settings.plugin.item', key: 'image-generation', order: 100, locale: NS }, ImageCard)) + ctx.slots.inject('settings.plugin.item', () => ctx.slots.register({ name: 'settings.plugin.item', key: 'image-generation', order: -100, locale: NS }, ImageCard)) }, } }, diff --git a/packages/dsh-image-generation/index.js b/packages/dsh-image-generation/index.js index 5d514a907..e8a1d58f1 100644 --- a/packages/dsh-image-generation/index.js +++ b/packages/dsh-image-generation/index.js @@ -7,9 +7,10 @@ import { createSettings } from './lib/settings.js' import { generate, readBounded, safeError } from './lib/provider.js' import { assetDirectory, normalizeImage, workspaceFor } from './lib/assets.js' import { commitImage, writerPlan } from './lib/commit.js' +import { previewImage } from './lib/preview.js' export const name = 'dsh-image-generation' -export const inject = ['settings', 'credentials', 'connection', 'tools', 'skills', 'systemPrompt', 'sandboxPolicy', 'sandbox', 'subprocess'] +export const inject = ['settings', 'credentials', 'connection', 'tools', 'skills', 'systemPrompt', 'sandboxPolicy', 'sandbox', 'subprocess', 'sessionController'] export const Config = z.object({}) export function imageTool(ctx, settings) { @@ -77,9 +78,7 @@ export async function apply(ctx) { } }) } ctx.tools.register(imageTool(ctx, settings)) - ctx.on('tools/pre-execute', (exec, next) => exec.name === 'image_generate' - ? Promise.resolve({ kind: 'ask', reason: 'Generate an image with the configured provider and save it in this workspace. Provider usage may be billed.' }) - : next()) + ctx.connection.fetch.register({ path: '/api/image-generation.preview', methods: ['GET'], fetch: request => previewImage(ctx, request) }) const locator = new URL('./skills/generate-image/SKILL.md', import.meta.url) const candidate = { name: 'generate-image', description: 'Create reusable photos, illustrations and backgrounds for presentations, documents and other image requests with the configured image_generate tool.', diff --git a/packages/dsh-image-generation/lib/preview.js b/packages/dsh-image-generation/lib/preview.js new file mode 100644 index 000000000..9c8c6f7fc --- /dev/null +++ b/packages/dsh-image-generation/lib/preview.js @@ -0,0 +1,69 @@ +import { open, realpath } from 'node:fs/promises' +import { constants } from 'node:fs' +import { createHash } from 'node:crypto' +import path from 'node:path' +import { assetDirectory } from './assets.js' +import { ImageError, MAX_IMAGE_BYTES } from './provider.js' + +/** Read only canonical image results, including results created before previews existed. */ +export function imageResult(content) { + if (!Array.isArray(content)) return + for (const block of content) { + if (block?.type !== 'text' || typeof block.text !== 'string' || block.text.length > 16384) continue + let value + try { value = JSON.parse(block.text) } catch { continue } + if (value && /^[a-f0-9]{64}$/.test(value.sha256) && value.asset_id === `sha256:${value.sha256}` + && value.workspace_path === `.workbuddy/generated-images/${value.sha256}.png` && value.media_type === 'image/png' + && Number.isInteger(value.width) && value.width > 0 && Number.isInteger(value.height) && value.height > 0 + && Number.isInteger(value.bytes) && value.bytes > 0 && value.bytes <= MAX_IMAGE_BYTES) return value + } +} + +/** The session's successful image tool result grants access to exactly that immutable PNG. */ +function referencedImage(events, sha) { + const calls = new Map() + for (const event of events) { + if (event.type === 'tool/call') calls.set(event.data.callId, event.data.name) + let result + if (event.type === 'tool/result' && event.surfaceOp === 'append' && calls.get(event.data.message?.source?.callId) === 'image_generate') result = event.data.message.content[0] + if (event.type === 'tool/code-dispatch' && event.data.name === 'image_generate') result = event.data + if (result && !result.isError) { + const image = imageResult(result.content) + if (image?.sha256 === sha) return image + } + } +} + +export async function previewImage(ctx, request) { + let file + try { + const url = new URL(request.url) + const sessionId = url.searchParams.get('session') + const sha = url.searchParams.get('asset') + if (!/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(sessionId ?? '') || !/^[a-f0-9]{64}$/.test(sha ?? '')) throw new ImageError('PREVIEW', 'Invalid image reference.', 400) + // Use the same cold/live session authority as the Harness session API. + const inspection = await ctx.sessionController.inspect(sessionId, request.signal) + const image = referencedImage(inspection.events, sha) + if (!image || !inspection.meta.cwd) throw new ImageError('PREVIEW', 'Image is unavailable in this session.', 404) + const root = await realpath(inspection.meta.cwd) + const directory = await assetDirectory(root) + const target = path.join(directory, `${sha}.png`) + file = await open(target, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK) + const info = await file.stat() + if (!info.isFile() || info.size > MAX_IMAGE_BYTES || info.size !== image.bytes || await realpath(target) !== target) throw new ImageError('PREVIEW', 'Image file is unavailable.', 404) + const data = Buffer.alloc(info.size) + let offset = 0 + while (offset < data.length) { + request.signal.throwIfAborted() + const { bytesRead } = await file.read(data, offset, data.length - offset, offset) + if (!bytesRead) break + offset += bytesRead + } + if (offset !== data.length || createHash('sha256').update(data).digest('hex') !== sha || data.subarray(0, 8).toString('hex') !== '89504e470d0a1a0a') throw new ImageError('PREVIEW', 'The generated image has changed.', 404) + ctx.logger.info('image-generation: preview allowed; session=%s asset=%s', sessionId, sha) + return new Response(data, { headers: { 'Content-Type': 'image/png', 'Content-Length': String(data.length), 'Cache-Control': 'no-store', 'X-Content-Type-Options': 'nosniff', 'Content-Disposition': 'inline; filename="generated-image.png"' } }) + } catch (error) { + ctx.logger.info('image-generation: preview unavailable') + return Response.json({ code: 'PREVIEW', error: 'Image preview is unavailable.' }, { status: error instanceof ImageError && error.code === 'PREVIEW' ? error.status : 404, headers: { 'Cache-Control': 'no-store' } }) + } finally { await file?.close() } +} diff --git a/packages/dsh-image-generation/package.json b/packages/dsh-image-generation/package.json index 8028022be..9caffd38d 100644 --- a/packages/dsh-image-generation/package.json +++ b/packages/dsh-image-generation/package.json @@ -4,17 +4,55 @@ "description": "Shared OpenAI and ByteDance image generation for DSH conversations, presentations and documents.", "type": "module", "main": "./index.js", - "exports": { ".": "./index.js", "./client": "./client.js", "./package.json": "./package.json", "./cordis.patch.yml": "./cordis.patch.yml" }, - "files": ["index.js", "client.js", "lib", "skills", "cordis.patch.yml", "README.md", "LICENSE"], + "exports": { + ".": "./index.js", + "./client": "./client.js", + "./package.json": "./package.json", + "./cordis.patch.yml": "./cordis.patch.yml" + }, + "files": [ + "index.js", + "client.js", + "lib", + "skills", + "cordis.patch.yml", + "README.md", + "LICENSE" + ], "license": "MIT", - "publishConfig": { "access": "public" }, - "repository": { "type": "git", "url": "git+https://github.com/dataelement/dsh-desktop.git", "directory": "packages/dsh-image-generation" }, - "keywords": ["dsh-plugin", "image-generation", "seedream", "openai"], + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/dataelement/dsh-desktop.git", + "directory": "packages/dsh-image-generation" + }, + "keywords": [ + "dsh-plugin", + "image-generation", + "seedream", + "openai" + ], "dsh": { - "bundle": { "patch": "./cordis.patch.yml" }, - "client": { "inject": ["@deepseek-ai/dsh-client-locale", "@deepseek-ai/dsh-client-ui-primitives", "@deepseek-ai/dsh-client-ui-settings-plugins"], "platform": "web" } + "bundle": { + "patch": "./cordis.patch.yml" + }, + "client": { + "inject": [ + "@deepseek-ai/dsh-client-locale", + "@deepseek-ai/dsh-client-ui-primitives", + "@deepseek-ai/dsh-client-ui-settings-plugins", + "@deepseek-ai/dsh-client-ui-chat", + "@deepseek-ai/dsh-client-ui-attachment" + ], + "platform": "web" + } + }, + "dependencies": { + "@deepseek-ai/schemastery": "^3.18.2", + "sharp": "^0.35.4" }, - "dependencies": { "@deepseek-ai/schemastery": "^3.18.2", "sharp": "^0.35.4" }, "peerDependencies": { "@deepseek-ai/cordis": "^4.0.2", "@deepseek-ai/dsh-client-connection": "^0.1.2-rc.1", @@ -25,6 +63,7 @@ "@deepseek-ai/dsh-settings": "^0.1.2-rc.1", "@deepseek-ai/dsh-skill": "^0.1.2-rc.1", "@deepseek-ai/dsh-system-prompt": "^0.1.2-rc.1", - "@deepseek-ai/dsh-tools": "^0.1.2-rc.1" + "@deepseek-ai/dsh-tools": "^0.1.2-rc.1", + "@deepseek-ai/dsh-api-session-controller": "^0.1.2-rc.1" } } diff --git a/packages/dsh-image-generation/skills/generate-image/SKILL.md b/packages/dsh-image-generation/skills/generate-image/SKILL.md index 0eafa890b..576c2c7a8 100644 --- a/packages/dsh-image-generation/skills/generate-image/SKILL.md +++ b/packages/dsh-image-generation/skills/generate-image/SKILL.md @@ -8,7 +8,7 @@ Use `image_generate` whenever a task needs a generated raster visual. The user c 1. Identify the image's purpose and placement. For PPT/Word, inspect the relevant document or template first. Carry its palette, art direction and whitespace needs into `style_context`, and reuse that style across the document. 2. Write a complete prompt: subject, composition, setting, lighting, palette and required details. Specify room for titles or captions where the document needs it. Keep readable document text as native text; use native editable objects for charts, tables, flowcharts and simple diagrams. 3. Choose the target `aspect_ratio` and `purpose`. A 16:9 slide does not require a 16:9 image when the illustration occupies only part of the slide. The tool returns actual dimensions; use proportional scaling or an intentional crop, preserving the main subject. -4. Call `image_generate` once per needed visual. Inspect the resulting PNG with the available image viewer before placing it. Correct specific visible defects if another generation is needed. Reuse an existing suitable image instead of generating duplicates. +4. Call `image_generate` directly once per needed visual using the saved provider configuration. The Host applies workspace and deployment policy. The conversation displays the resulting PNG with click-to-enlarge preview. If an image-viewing tool is available, inspect the PNG before placing it; report visual findings only after viewing it. Correct specific visible defects if another generation is needed. Reuse an existing suitable image instead of generating duplicates. 5. Insert the returned `workspace_path` into the PPT/Word workflow, then inspect the rendered document page. Keep visual assets linked to their purpose and reuse their paths across later edits. Return the document and any requested standalone image. If the tool reports missing configuration, direct the user to Settings → Plugins → Image generation. Never request an API key in conversation or place one in a document, prompt, sandbox command or source file. Respect tool approval, cancellation and workspace policy. Surface provider errors accurately; use a placeholder only when the user accepts one. Configuration validation checks the connection without submitting an image generation job; successful generation is established by an actual generated image. diff --git a/scripts/image-preview-fixture.mjs b/scripts/image-preview-fixture.mjs new file mode 100644 index 000000000..66ed27c79 --- /dev/null +++ b/scripts/image-preview-fixture.mjs @@ -0,0 +1,39 @@ +/** Persist a simulated successful tool result through the real Harness log writer. */ +import path from 'node:path' +import { mkdir, realpath } from 'node:fs/promises' +import { randomUUID } from 'node:crypto' +import { Context } from '@deepseek-ai/cordis' +import { Session, SessionStore } from '@deepseek-ai/dsh-session' +import { JsonlSessionPersistence } from '@deepseek-ai/dsh-session-persistence-jsonl' +import { createUserMessage, createAssistantMessage, createToolResultMessage } from '@deepseek-ai/dsh-llm' +import sharp from 'sharp' +import { materialize } from '../packages/dsh-image-generation/lib/storage.js' + +export async function imagePreviewFixture(home) { + const directory = path.join(home, 'preview-workspace'); await mkdir(directory) + const cwd = await realpath(directory) + const png = await sharp(Buffer.from('Image preview fixture')).png().toBuffer() + const asset = await materialize(cwd, png) + const image = { ...asset, asset_id: `sha256:${asset.sha256}`, media_type: 'image/png', width: 960, height: 540, bytes: png.length, provider: 'bytedance', model: 'doubao-seedream-5-0-pro-260628' } + const id = randomUUID() + const session = Session.create(id, undefined, { version: 0, id, createdAt: Date.now(), isSeeded: false, cwd, delegationDepth: 0 }) + session.append('turn/start', { turn: 1 }) + session.append('step/start', { turn: 1, step: 1 }) + session.append('user/message', createUserMessage({ content: [{ type: 'text', text: '图片预览验收：展示模拟生图结果' }], source: { kind: 'user' } }), { surfaceOp: 'append' }) + const args = { prompt: 'Landscape preview fixture', aspect_ratio: '16:9' } + session.append('assistant/message', { turn: 1, step: 1, message: createAssistantMessage({ source: { provider: 'fixture', model: 'fixture' }, content: [{ type: 'tool-call', callId: 'image-fixture', name: 'image_generate', arguments: JSON.stringify(args) }] }) }, { surfaceOp: 'append' }) + session.append('tool/call', { turn: 1, step: 1, callId: 'image-fixture', name: 'image_generate', arguments: JSON.stringify(args) }) + session.append('tool/result', { turn: 1, step: 1, message: createToolResultMessage({ callId: 'image-fixture', content: [{ type: 'text', text: JSON.stringify(image) }], isError: false }) }, { surfaceOp: 'append' }) + session.append('step/end', { turn: 1, step: 1 }) + session.append('step/start', { turn: 1, step: 2 }) + session.append('assistant/message', { turn: 1, step: 2, message: createAssistantMessage({ source: { provider: 'fixture', model: 'fixture' }, content: [{ type: 'text', text: '图片已生成，点击下方缩略图可放大查看。' }] }) }, { surfaceOp: 'append' }) + session.append('step/end', { turn: 1, step: 2 }) + session.append('turn/end', { turn: 1, reason: { kind: 'completed' } }) + const ctx = new Context(); const forks = [] + try { + for (const [plugin, config] of [[SessionStore, {}], [JsonlSessionPersistence, { root: path.join(home, 'sessions') }]]) { const fork = ctx.plugin(plugin, config); forks.push(fork); await fork } + await ctx.sessionPersistence.create(session.header) + await ctx.sessionPersistence.append(session.id, session.snapshotEvents()) + } finally { for (const fork of forks.reverse()) await fork.dispose() } + return { sessionId: session.id, image, png } +} diff --git a/scripts/verify-image-generation.mjs b/scripts/verify-image-generation.mjs index 95432b8d7..0f8dc242e 100644 --- a/scripts/verify-image-generation.mjs +++ b/scripts/verify-image-generation.mjs @@ -9,10 +9,12 @@ import { mkdtemp, writeFile, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import path from 'node:path' import { fileURLToPath } from 'node:url' +import { imagePreviewFixture } from './image-preview-fixture.mjs' import { DEFAULTS } from '../packages/dsh-image-generation/lib/provider.js' const root = fileURLToPath(new URL('..', import.meta.url)) const home = await mkdtemp(path.join(tmpdir(), 'dsh-image-host-')) +const preview = await imagePreviewFixture(home) const calls = [] const mock = createServer(async (request, response) => { let body = ''; for await (const chunk of request) body += chunk @@ -63,6 +65,11 @@ try { method: body ? 'POST' : 'GET', headers: { Cookie: cookie, ...(body ? { 'Content-Type': 'application/json' } : {}), ...extra }, ...(body ? { body: JSON.stringify(body) } : {}), }) + const previewPath = `preview?session=${preview.sessionId}&asset=${preview.image.sha256}` + assert.equal((await fetch(`${base}/api/image-generation.${previewPath}`)).status, 401) + const imageResponse = await api(previewPath) + assert.equal(imageResponse.status, 200) + assert.deepEqual(Buffer.from(await imageResponse.arrayBuffer()), preview.png) const initialResponse = await api('settings') assert.equal(initialResponse.status, 200) assert.equal(initialResponse.headers.get('cache-control'), 'no-store') @@ -91,10 +98,10 @@ try { const page = await (await fetch(base, { headers: { Cookie: cookie } })).text() assert.ok(page.includes('dsh-image-generation'), 'Image client entry must be present in the composed page') assert.ok(!/image-generation.*(?:failed|Error)/i.test(output), 'Image plugin must load successfully') - console.log('PASS: Host composition, authentication, origin checks, redacted settings, save success/failure, single-request validation and read-only model discovery.') + console.log('PASS: Host composition, authentication, origin checks, redacted settings, save success/failure, single-request validation read-only model discovery and authenticated historical PNG preview.') if (process.argv.includes('--keep')) { await writeFile(path.join(home, 'browser-url.txt'), url, { mode: 0o600 }) - await writeFile(path.join(home, 'smoke-context.json'), JSON.stringify({ base, mockBase }), { mode: 0o600 }) + await writeFile(path.join(home, 'smoke-context.json'), JSON.stringify({ base, mockBase, sessionId: preview.sessionId }), { mode: 0o600 }) console.log(`Browser fixture ready: ${home}`) await new Promise(() => {}) } diff --git a/test/image-generation.test.mjs b/test/image-generation.test.mjs index 6e2c20b55..f964a3f51 100644 --- a/test/image-generation.test.mjs +++ b/test/image-generation.test.mjs @@ -1,5 +1,5 @@ import { createServer } from 'node:http' -import { mkdtemp, readFile, realpath, rm, symlink, readdir, stat } from 'node:fs/promises' +import { mkdtemp, readFile, realpath, rm, symlink, readdir, stat, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import path from 'node:path' import { randomUUID, createHash } from 'node:crypto' @@ -19,6 +19,7 @@ import { createSettings } from '../packages/dsh-image-generation/lib/settings.js import { DEFAULTS, generate, generationBody, ImageError, profile, readBounded, validateConnection } from '../packages/dsh-image-generation/lib/provider.js' import { normalizeImage } from '../packages/dsh-image-generation/lib/assets.js' import { materialize } from '../packages/dsh-image-generation/lib/storage.js' +import { previewImage, imageResult } from '../packages/dsh-image-generation/lib/preview.js' import { writerEnvironment } from '../packages/dsh-image-generation/lib/commit.js' const cleanups = [] @@ -259,7 +260,7 @@ describe('image tool and durable Office assets', () => { await expect(materialize(workspace, image.data)).rejects.toThrow() await expect(normalizeImage(Buffer.from('not a PNG'))).rejects.toMatchObject({ code: 'IMAGE' }) }) - it('registers standard authenticated API routes, the shared Skill and default approval', async () => { + it('registers standard routes and executes with saved configuration while honoring deployment guards', async () => { const f = await fixture(); const routes = [] const plugin = f.ctx.plugin({ inject: ['settings', 'skills', 'systemPrompt', 'tools'], apply: ctx => apply({ ...f.services, settings: ctx.settings, skills: ctx.skills, systemPrompt: ctx.systemPrompt, tools: ctx.tools, @@ -267,13 +268,82 @@ describe('image tool and durable Office assets', () => { }) }) await plugin; cleanups.push(() => plugin.dispose()) expect(f.ctx.settings.describe().map(entry => entry.ns)).toContain('image-generation') - expect(routes.map(route => route.path)).toEqual(['/api/image-generation.settings', '/api/image-generation.save', '/api/image-generation.models']) + expect(routes.map(route => route.path)).toEqual(['/api/image-generation.settings', '/api/image-generation.save', '/api/image-generation.models', '/api/image-generation.preview']) const response = await routes[0].fetch(new Request('http://localhost/api/image-generation.settings')) expect(response.headers.get('cache-control')).toBe('no-store') expect((await response.json()).profiles.openai.configured).toBe(false) - const result = await f.ctx.tools.execute({ callId: 'ask', name: 'image_generate', arguments: { prompt: 'Forest' }, agent: f.agent, signal: new AbortController().signal }) - expect(result.isError).toBe(true) - expect(JSON.stringify(result)).toContain('Provider usage may be billed') - expect(await readdir(f.workspace)).toEqual([]) + const provider = await server() + await f.settings.save(saveInput('bytedance', provider.baseUrl)) + const result = await f.ctx.tools.execute({ callId: 'direct', name: 'image_generate', arguments: { prompt: 'Forest' }, agent: f.agent, signal: new AbortController().signal }) + expect(result.isError, JSON.stringify(result)).toBeFalsy() + expect(provider.calls).toHaveLength(2) + const dispose = f.ctx.tools.guard(exec => exec.name === 'image_generate' ? 'Deployment blocks image generation' : undefined) + const denied = await f.ctx.tools.execute({ callId: 'denied', name: 'image_generate', arguments: { prompt: 'Forest' }, agent: f.agent, signal: new AbortController().signal }) + dispose() + expect(denied.isError).toBe(true) + expect(JSON.stringify(denied)).toContain('Deployment blocks image generation') + expect(provider.calls).toHaveLength(2) + }) +}) + + +describe('session-authorized generated image previews', () => { + async function previewFixture() { + const workspace = await temp() + const png = await sharp({ create: { width: 16, height: 9, channels: 4, background: '#112233' } }).png().toBuffer() + const asset = await materialize(workspace, png) + const image = { ...asset, asset_id: `sha256:${asset.sha256}`, media_type: 'image/png', width: 16, height: 9, bytes: png.length, provider: 'bytedance', model: DEFAULTS.bytedance.model } + const id = randomUUID() + const content = [{ type: 'text', text: JSON.stringify(image) }] + const events = [{ type: 'tool/call', data: { callId: 'image-call', name: 'image_generate' } }, + { type: 'tool/result', surfaceOp: 'append', data: { message: { source: { callId: 'image-call' }, content: [{ type: 'tool-result', content }] } } }] + const log = vi.fn() + const ctx = { sessionController: { inspect: vi.fn(async session => ({ meta: { cwd: workspace }, events: session === id ? events : [] })) }, logger: { info: log } } + const request = (session = id, sha = image.sha256) => new Request(`http://localhost/api/image-generation.preview?session=${session}&asset=${sha}`) + return { workspace, png, image, id, content, events, ctx, request, target: path.join(workspace, image.workspace_path) } + } + it('serves historical successful PNGs with integrity, cache, content-type and audit checks', async () => { + const f = await previewFixture() + const result = await previewImage(f.ctx, f.request()) + expect(result.status).toBe(200) + expect(Buffer.from(await result.arrayBuffer())).toEqual(f.png) + expect(result.headers.get('content-type')).toBe('image/png') + expect(result.headers.get('cache-control')).toBe('no-store') + expect(result.headers.get('x-content-type-options')).toBe('nosniff') + expect(f.ctx.logger.info).toHaveBeenCalledWith(expect.stringContaining('preview allowed'), f.id, f.image.sha256) + }) + it('requires a successful image tool result in the requested session', async () => { + const f = await previewFixture() + expect((await previewImage(f.ctx, f.request(randomUUID()))).status).toBe(404) + expect((await previewImage(f.ctx, f.request(f.id, 'a'.repeat(64)))).status).toBe(404) + f.events[0].data.name = 'other_tool' + expect((await previewImage(f.ctx, f.request())).status).toBe(404) + f.events[0].data.name = 'image_generate' + f.events[1].data.message.content[0].isError = true + expect((await previewImage(f.ctx, f.request())).status).toBe(404) + f.events.splice(0, 2, { type: 'tool/code-dispatch', data: { name: 'image_generate', subCallId: 'ptc', content: f.content } }) + expect((await previewImage(f.ctx, f.request())).status).toBe(200) + }) + it('rejects path traversal and invalid asset contracts before reading a session', async () => { + const f = await previewFixture() + expect((await previewImage(f.ctx, f.request('../outside'))).status).toBe(400) + expect((await previewImage(f.ctx, f.request(f.id, '../secret'))).status).toBe(400) + expect(f.ctx.sessionController.inspect).not.toHaveBeenCalled() + expect(imageResult([null, { type: 'text', text: 'broken' }])).toBeUndefined() + for (const change of [{ bytes: -1 }, { bytes: 1000000000 }, { workspace_path: '/etc/passwd' }, { media_type: 'text/html' }, { asset_id: 'wrong' }]) { + expect(imageResult([{ type: 'text', text: JSON.stringify({ ...f.image, ...change }) }])).toBeUndefined() + } + }) + it('refuses changed files, target symlinks and linked parent directories', async () => { + const f = await previewFixture() + await writeFile(f.target, Buffer.alloc(f.png.length)) + expect((await previewImage(f.ctx, f.request())).status).toBe(404) + await rm(f.target) + const outside = path.join(await temp(), 'image.png'); await writeFile(outside, f.png) + await symlink(outside, f.target) + expect((await previewImage(f.ctx, f.request())).status).toBe(404) + await rm(path.dirname(f.target), { recursive: true }) + await symlink(path.dirname(outside), path.dirname(f.target)) + expect((await previewImage(f.ctx, f.request())).status).toBe(404) }) }) From 2913aa4d9424acd1aaaadd5b55626dae5c96f4ce Mon Sep 17 00:00:00 2001 From: Jianghan <58623049+SuperstructureJH@users.noreply.github.com> Date: Thu, 10 Sep 2026 12:27:47 +0800 Subject: [PATCH 5/8] fix(image-generation): use shared dropdown menus --- docs/STATUS.md | 7 ++++ packages/dsh-image-generation/client.js | 48 +++++++++++++++++++++---- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/docs/STATUS.md b/docs/STATUS.md index d4cce1fc8..ff06f774b 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1,5 +1,12 @@ # Implementation and verification status +## Shared image-setting dropdowns — 2026-09-10 + +- **Implemented:** provider and image-model fields use the Harness `Menu` component, with theme-aware surfaces, selected checks, viewport placement and scrolling. The controls retain their existing dimensions and support arrow keys, Home/End, Enter and Escape with focus restoration. Save and model discovery use the existing configuration contract. +- **PASS:** 36 image regressions plus 2 dependency-closure checks, TypeScript, production build and syntax/diff checks. Actual Harness browser inspection confirms the white light-theme model list, dark-theme provider menu, keyboard selection, custom model input, Escape preserving the settings dialog, and automatic Save validation against the isolated loopback provider. +- **Compatibility:** the OpenAI adapter uses the Images API with a configurable base URL. Compatible services must accept the current generation parameters and return `data[0].b64_json`; Save reads model metadata, and discovery filters the provider list to supported GPT Image identifiers. Custom model IDs remain available. Seedream has its own request adapter. +- **PASS:** r5 macOS arm64 development DMG/ZIP, strict deep signature, final packaged Host configuration/model/history-preview smoke and source/tarball/app/ZIP parity (14 source/tarball files; 13 Desktop files). Artifacts and browser screenshots are in `outputs/image-generation-plugin-20260910-r5`. The previous complete-suite and Electron writer gates remain recorded under r4; this revision changes the configuration client. + ## Direct image calls, concise settings and image preview — 2026-09-10 - **Implemented:** saved image configuration authorizes direct `image_generate` calls through the standard ToolRuntime; the plugin's additional approval request is removed. Deployment guards and workspace policy continue to apply. The settings card uses the requested description, Seedream / OpenAI labels and essential fields, with automatic validation on Save and concise error feedback. diff --git a/packages/dsh-image-generation/client.js b/packages/dsh-image-generation/client.js index 2baf8197f..9253e46f2 100644 --- a/packages/dsh-image-generation/client.js +++ b/packages/dsh-image-generation/client.js @@ -2,7 +2,7 @@ window.__ModuleLoader__.load({ id: 'dsh-image-generation', factory: require => { const React = require('react') - const { IconChevronDownOutline14 } = require('@deepseek-ai/dsh-client-ui-primitives') + const { IconChevronDownOutline14, Menu } = require('@deepseek-ai/dsh-client-ui-primitives') const h = React.createElement const NS = 'settings.imageGeneration' const zh = { @@ -44,7 +44,8 @@ window.__ModuleLoader__.load({ .dshImageDescription{color:var(--dsw-alias-label-tertiary);font-size:13px;line-height:1.5} .dshImageChevron{color:var(--dsw-alias-label-tertiary);flex:none;transition:transform .16s}.dshImageHeader[aria-expanded=true] .dshImageChevron{transform:rotate(180deg)} .dshImageBody{border-top:.5px solid var(--dsw-alias-border-l2);margin:0 16px;padding-bottom:8px;display:flex;flex-direction:column;gap:12px}.dshImageFields{border:0;margin:0;padding:0;display:flex;flex-direction:column;min-width:0} - .dshImageField{display:flex;flex-direction:column;gap:6px;padding:12px 0;font-size:13px;font-weight:500;line-height:1.5;color:var(--dsw-alias-label-primary)}.dshImageField+.dshImageField{border-top:.5px solid var(--dsw-alias-border-l2)}.dshImageField input,.dshImageField select{box-sizing:border-box;width:100%;min-width:0;height:34px;padding:0 12px;border:.5px solid var(--dsw-alias-border-l4);border-radius:8px;background:var(--dsw-alias-bg-layer-3);color:var(--dsw-alias-label-primary);font:inherit;font-weight:400} + .dshImageField{display:flex;flex-direction:column;gap:6px;padding:12px 0;font-size:13px;font-weight:500;line-height:1.5;color:var(--dsw-alias-label-primary)}.dshImageField+.dshImageField{border-top:.5px solid var(--dsw-alias-border-l2)}.dshImageField input,.dshImageSelectTrigger{box-sizing:border-box;width:100%;min-width:0;height:34px;padding:0 12px;border:.5px solid var(--dsw-alias-border-l4);border-radius:8px;background:var(--dsw-alias-bg-layer-3);color:var(--dsw-alias-label-primary);font:inherit;font-weight:400} + .dshImageSelect{width:100%;min-width:0}.dshImageSelectTrigger{display:flex;align-items:center;justify-content:space-between;gap:12px;cursor:pointer;text-align:left}.dshImageSelectValue{overflow:hidden;white-space:nowrap;text-overflow:ellipsis}.dshImageSelectTrigger:disabled{opacity:.4;cursor:default} .dshImageField input::placeholder{color:var(--dsw-alias-label-tertiary)} .dshImageAdvanced{margin-top:12px}.dshImageAdvanced summary{cursor:pointer;font-size:13px;color:var(--dsw-alias-label-secondary)}.dshImageAdvanced[open]{display:flex;flex-direction:column} .dshImageHint{font-size:12px;line-height:1.5;color:var(--dsw-alias-label-tertiary);margin:0}.dshImageActions{border-top:.5px solid var(--dsw-alias-border-l2);display:flex;justify-content:flex-end;align-items:center;gap:8px;padding:12px 0 4px;flex-wrap:wrap} @@ -61,6 +62,38 @@ window.__ModuleLoader__.load({ if (!response.ok) throw data return data } + function ImageSelect({ name, label, value, options, onChange, disabled }) { + const [open, setOpen] = React.useState(false) + const trigger = React.useRef(null) + const labels = React.useRef(new Map()) + const id = React.useId() + const close = restoreFocus => { setOpen(false); if (restoreFocus) trigger.current?.focus() } + React.useEffect(() => { if (disabled) setOpen(false) }, [disabled]) + React.useEffect(() => { + if (!open) return + // The shared portaled Menu completes its measurement before focus moves. + const frame = requestAnimationFrame(() => (labels.current.get(value) || labels.current.get(options[0]?.id))?.closest('button')?.focus()) + return () => cancelAnimationFrame(frame) + }, [open]) + const keyDown = event => { + if (disabled) return + if (event.key === 'Escape' && open) { event.preventDefault(); event.stopPropagation(); close(true); return } + if (!['ArrowDown', 'ArrowUp', 'Home', 'End'].includes(event.key)) return + event.preventDefault(); event.stopPropagation() + if (!open) { setOpen(true); return } + const buttons = options.map(option => labels.current.get(option.id)?.closest('button')).filter(Boolean) + const current = buttons.indexOf(document.activeElement) + const index = event.key === 'Home' ? 0 : event.key === 'End' ? buttons.length - 1 : (current + (event.key === 'ArrowDown' ? 1 : -1) + buttons.length) % buttons.length + buttons[index]?.focus() + } + return h('div', { className: 'dshImageField', onKeyDown: keyDown }, h('span', { id: `${id}-label` }, label), + h(Menu, { open: open && !disabled, onClose: () => close(false), selectedId: value, portal: true, dense: true, className: 'dshImageSelect', + items: options.map(option => ({ ...option, disabled, label: h('span', { ref: node => { if (node) labels.current.set(option.id, node); else labels.current.delete(option.id) } }, option.label) })), + onSelect: selected => { if (!disabled) { close(true); onChange(selected) } }, + anchor: h('button', { ref: trigger, name, type: 'button', disabled, className: 'dshImageSelectTrigger', 'aria-labelledby': `${id}-label ${id}-value`, + 'aria-haspopup': 'menu', 'aria-expanded': open && !disabled, onClick: () => setOpen(previous => !previous) }, + h('span', { id: `${id}-value`, className: 'dshImageSelectValue' }, options.find(option => option.id === value)?.label || value), h(IconChevronDownOutline14)) })) + } function ImageCard({ t }) { const [expanded, setExpanded] = React.useState(false) const [saved, setSaved] = React.useState(null) @@ -133,12 +166,13 @@ window.__ModuleLoader__.load({ expanded && h('form', { id: `${id}-body`, className: 'dshImageBody', onSubmit: save, 'aria-busy': busy || fetching || loading }, loading ? h('p', { className: 'dshImageHint' }, t('loading')) : draft && h(React.Fragment, null, h('fieldset', { className: 'dshImageFields', disabled: busy || fetching || !saved.writable }, - h('label', { className: 'dshImageField' }, t('provider'), h('select', { name: 'provider', value: provider, onChange: event => { setProvider(event.target.value); setStatus(''); setError('') } }, - h('option', { value: 'bytedance' }, t('bytedance')), h('option', { value: 'openai' }, t('openai')))), + h(ImageSelect, { name: 'provider', label: t('provider'), value: provider, disabled: busy || fetching || !saved.writable, + options: [{ id: 'bytedance', label: t('bytedance') }, { id: 'openai', label: t('openai') }], + onChange: value => { setProvider(value); setStatus(''); setError('') } }), field('apiKey', 'apiKey', 'password', t(draft.configured ? 'savedKey' : 'keyPlaceholder')), - h('label', { className: 'dshImageField' }, t('modelSelect'), h('select', { name: 'model', value: customModels[provider] || !catalog.models.includes(draft.model) ? '__custom__' : draft.model, - onChange: event => { const custom = event.target.value === '__custom__'; setCustomModels(previous => ({ ...previous, [provider]: custom })); if (!custom) edit('model', event.target.value) } }, - ...catalog.models.map(model => h('option', { key: model, value: model }, model)), h('option', { value: '__custom__' }, t('customModel')))), + h(ImageSelect, { name: 'model', label: t('modelSelect'), value: customModels[provider] || !catalog.models.includes(draft.model) ? '__custom__' : draft.model, disabled: busy || fetching || !saved.writable, + options: [...catalog.models.map(model => ({ id: model, label: model })), { id: '__custom__', label: t('customModel') }], + onChange: value => { const custom = value === '__custom__'; setCustomModels(previous => ({ ...previous, [provider]: custom })); if (!custom) edit('model', value) } }), (customModels[provider] || !catalog.models.includes(draft.model)) && field('model', 'model'), catalog.canFetch && h('button', { className: 'dshImageFetch', type: 'button', disabled: !draft.apiKey.trim() && !draft.configured, onClick: fetchModels }, t(fetching ? 'fetchingModels' : 'fetchModels')), catalog.source === 'provider' && catalog.models.length === 0 && h('p', { className: 'dshImageHint', role: 'status' }, t('emptyModels')), From 5aaed613053ba53ccd8bf373ba9c4afd0bf89902 Mon Sep 17 00:00:00 2001 From: Jianghan <58623049+SuperstructureJH@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:02:55 +0800 Subject: [PATCH 6/8] fix(image-generation): complete Seedream model presets --- docs/STATUS.md | 7 +++++++ packages/dsh-image-generation/README.md | 16 ++++++++++++-- packages/dsh-image-generation/lib/provider.js | 13 ++++++++++-- test/image-generation.test.mjs | 21 +++++++++++++------ 4 files changed, 47 insertions(+), 10 deletions(-) diff --git a/docs/STATUS.md b/docs/STATUS.md index ff06f774b..62c602e99 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1,5 +1,12 @@ # Implementation and verification status +## Expanded Seedream presets — 2026-09-10 + +- **Implemented:** the built-in catalog includes `doubao-seedream-5-0-pro-260628`, `doubao-seedream-5-0-260128`, `doubao-seedream-5-0-lite-260128`, `doubao-seedream-4-5-251128` and `doubao-seedream-4-0-250828`, with custom model/endpoint entry retained. The existing saved model and default remain in place. +- **Source verified:** the [official Ark image model list](https://docs.volcengine.com/docs/82379/1330310), updated September 9, explicitly lists both 5.0 and 5.0 Lite IDs. The [Images API](https://docs.volcengine.com/docs/82379/1541523) supports their shared single-image/Base64 contract; the existing five canvas ratios meet each documented model's pixel limits. Presets represent supported model IDs, while account access is established by the provider. +- **PASS:** 39 image tests plus 2 dependency-closure checks, TypeScript and production build. Each preset and a custom endpoint cover all five aspect ratios against independent provider-documented canvas ranges. These checks use loopback services. +- **PASS:** r6 macOS arm64 development DMG/ZIP, strict deep signature, packaged Host configuration/model/history-preview smoke, and source/tarball/app/ZIP parity (14 source/tarball files; 13 Desktop files). The actual packaged Harness client displays all five presets plus custom entry, preserves the original selection, saves each newly added ID and retains the selection after reopening. Artifacts and source evidence: `outputs/image-generation-plugin-20260910-r6`. Real-provider requests were not run in this revision. + ## Shared image-setting dropdowns — 2026-09-10 - **Implemented:** provider and image-model fields use the Harness `Menu` component, with theme-aware surfaces, selected checks, viewport placement and scrolling. The controls retain their existing dimensions and support arrow keys, Home/End, Enter and Escape with focus restoration. Save and model discovery use the existing configuration contract. diff --git a/packages/dsh-image-generation/README.md b/packages/dsh-image-generation/README.md index 630cfcf89..10dcc3bad 100644 --- a/packages/dsh-image-generation/README.md +++ b/packages/dsh-image-generation/README.md @@ -4,9 +4,21 @@ DSH/Cordis 公共生图插件，提供 `image_generate` 工具、`generate-image ## 使用 -DSH Desktop 默认装载本插件。在生图工具卡片选择字节或 OpenAI，填写该平台的 API Key 并保存。模型和地址已有默认值，可在高级设置中修改。 +DSH Desktop 默认装载本插件。在生图工具卡片选择 Seedream 或 OpenAI，填写该平台的 API Key 并保存。生图模型支持预设选择或自定义输入，API 地址可在高级设置中修改。 -生图模型使用下拉选择。OpenAI 的「获取模型」用当前填写或已保存的 Key 发起一次 `GET /models`，筛选 Images API 支持的 GPT Image 模型；获取过程只更新候选列表，配置在点击保存时生效。查询来源保留在接口中，界面显示候选模型、空列表和查询错误。字节的管理接口使用独立签名凭据，因此 API Key 模式提供 Seedream 5.0 Pro、4.5 的内置选项和自定义模型/接入点 ID。 +生图模型使用下拉选择。OpenAI 的「获取模型」用当前填写或已保存的 Key 发起一次 `GET /models`，筛选 Images API 支持的 GPT Image 模型；获取过程只更新候选列表，配置在点击保存时生效。查询来源保留在接口中，界面显示候选模型、空列表和查询错误。字节的管理接口使用独立签名凭据，因此 API Key 模式提供内置选项和自定义模型/接入点 ID。 + +Seedream 预设按 [火山方舟模型列表](https://docs.volcengine.com/docs/82379/1330310) 的图片生成能力核对（2026-09-10）： + +| 模型 | 模型 ID | +| --- | --- | +| Seedream 5.0 Pro | `doubao-seedream-5-0-pro-260628` | +| Seedream 5.0 | `doubao-seedream-5-0-260128` | +| Seedream 5.0 Lite | `doubao-seedream-5-0-lite-260128` | +| Seedream 4.5 | `doubao-seedream-4-5-251128` | +| Seedream 4.0 | `doubao-seedream-4-0-250828` | + +官方目录明确同时支持 5.0 和 5.0 Lite 两个 ID。预设表示插件支持的型号，实际可用性取决于账户开通情况；自定义入口用于填写其他模型或接入点 ID。 API 地址同时接受基础地址和控制台提供的完整 `/images/generations` 地址。保存时统一为基础地址，实际请求只添加一次接口路径。 diff --git a/packages/dsh-image-generation/lib/provider.js b/packages/dsh-image-generation/lib/provider.js index f56bc91c8..5dd1da9c9 100644 --- a/packages/dsh-image-generation/lib/provider.js +++ b/packages/dsh-image-generation/lib/provider.js @@ -8,7 +8,16 @@ export const MAX_IMAGE_BYTES = 20 * 1024 * 1024 const OPENAI_IMAGES = new Set(['gpt-image-2.5-sunburst', 'gpt-image-2.5-sunburst-2026-09-08', 'gpt-image-2.5-flare', 'gpt-image-2.5-flare-2026-09-08', 'gpt-image-2', 'gpt-image-2-2026-04-21', 'gpt-image-1.5', 'gpt-image-1', 'gpt-image-1-mini']) export const MODEL_CATALOG = Object.freeze({ openai: { source: 'builtin', canFetch: true, models: [...OPENAI_IMAGES] }, - bytedance: { source: 'builtin', canFetch: false, models: ['doubao-seedream-5-0-pro-260628', DEFAULTS.bytedance.model] }, + // Ark image model list, checked 2026-09-10: + // https://docs.volcengine.com/docs/82379/1330310 + // Both 5.0 and 5.0 Lite IDs are explicitly supported by the provider. + bytedance: { source: 'builtin', canFetch: false, models: [ + 'doubao-seedream-5-0-pro-260628', + 'doubao-seedream-5-0-260128', + 'doubao-seedream-5-0-lite-260128', + DEFAULTS.bytedance.model, + 'doubao-seedream-4-0-250828', + ] }, }) export class ImageError extends Error { @@ -155,7 +164,7 @@ export function generationBody(provider, spec, args) { ...(provider === 'openai' ? { n: 1, quality: 'auto', output_format: 'png' } // Ark defaults to single-image generation. Use its shared single-image - // fields for 4.5, 5.0 Pro and opaque endpoint IDs; group controls belong + // fields for Seedream 4.x / 5.x and opaque endpoint IDs; group controls belong // to a separate capability and 5.0 Pro rejects them. : { response_format: 'b64_json', watermark: false }), } diff --git a/test/image-generation.test.mjs b/test/image-generation.test.mjs index f964a3f51..965f7b1d0 100644 --- a/test/image-generation.test.mjs +++ b/test/image-generation.test.mjs @@ -16,7 +16,7 @@ import { Session, SessionId } from '@deepseek-ai/dsh-session' import sharp from 'sharp' import { apply, imageTool } from '../packages/dsh-image-generation/index.js' import { createSettings } from '../packages/dsh-image-generation/lib/settings.js' -import { DEFAULTS, generate, generationBody, ImageError, profile, readBounded, validateConnection } from '../packages/dsh-image-generation/lib/provider.js' +import { DEFAULTS, MODEL_CATALOG, generate, generationBody, ImageError, profile, readBounded, validateConnection } from '../packages/dsh-image-generation/lib/provider.js' import { normalizeImage } from '../packages/dsh-image-generation/lib/assets.js' import { materialize } from '../packages/dsh-image-generation/lib/storage.js' import { previewImage, imageResult } from '../packages/dsh-image-generation/lib/preview.js' @@ -36,12 +36,21 @@ async function server() { if (delay) await new Promise(resolve => setTimeout(resolve, delay)) const probe = status === 200 && !malformed && req.url.endsWith('/images/generations') && body && !Object.hasOwn(JSON.parse(body), 'prompt') const input = body && JSON.parse(body) - // The 5.0 Pro contract supports single images and rejects group controls. + // Provider-documented canvas ranges, independent of the production catalog. // Treat an opaque endpoint as 5.0 Pro too: its ID conveys no capabilities. - if (status === 200 && input?.prompt && ['doubao-seedream-5-0-pro-260628', 'ep-custom'].includes(input.model)) { - const parameter = ['sequential_image_generation', 'sequential_image_generation_options'].find(field => Object.hasOwn(input, field)) || (input.stream === true ? 'stream' : undefined) + const canvas = { + 'doubao-seedream-5-0-pro-260628': [921600, 4624220], + 'doubao-seedream-5-0-260128': [3686400, 16777216], + 'doubao-seedream-5-0-lite-260128': [3686400, 16777216], + 'doubao-seedream-4-5-251128': [3686400, 16777216], + 'doubao-seedream-4-0-250828': [921600, 16777216], + 'ep-custom': [921600, 4624220], + }[input?.model] + if (status === 200 && input?.prompt && canvas) { + const singleOnly = ['doubao-seedream-5-0-pro-260628', 'ep-custom'].includes(input.model) + const parameter = singleOnly && (['sequential_image_generation', 'sequential_image_generation_options'].find(field => Object.hasOwn(input, field)) || (input.stream === true ? 'stream' : undefined)) const [width, height] = input.size.split('x').map(Number) - const invalidSize = !(width * height >= 921600 && width * height <= 4624220 && width / height >= 1 / 16 && width / height <= 16) + const invalidSize = !(width * height >= canvas[0] && width * height <= canvas[1] && width / height >= 1 / 16 && width / height <= 16) if (parameter || invalidSize) { res.writeHead(400, { 'content-type': 'application/json', 'x-request-id': 'seedream-contract-request' }) res.end(JSON.stringify({ error: { code: 'InvalidParameter', param: parameter || 'size', message: parameter ? `The parameter ${parameter} is not supported by this model.` : 'The size is outside the supported range.' } })) @@ -174,7 +183,7 @@ describe('image settings save and provider requests', () => { expect(generationBody('bytedance', profile('bytedance'), args)).not.toHaveProperty('quality') await expect(readBounded(new Response('too much'), 2)).rejects.toMatchObject({ code: 'TOO_LARGE' }) }) - it.each(['doubao-seedream-5-0-pro-260628', DEFAULTS.bytedance.model, 'ep-custom'])('generates every supported aspect ratio with the single-image contract for %s', async model => { + it.each([...MODEL_CATALOG.bytedance.models, 'ep-custom'])('generates every supported aspect ratio with the single-image contract for %s', async model => { const s = await server() for (const aspect_ratio of ['1:1', '16:9', '9:16', '4:3', '3:4']) { expect(await generate('bytedance', { baseUrl: s.baseUrl, model }, 'test-image-key', { prompt: 'A flower', aspect_ratio })).toEqual(s.png) From ee46268df53cb1442399c5ed8643fee76a3454c0 Mon Sep 17 00:00:00 2001 From: Jianghan <58623049+SuperstructureJH@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:50:50 +0800 Subject: [PATCH 7/8] fix(image): resolve Harness session IDs for conversation previews --- docs/STATUS.md | 8 ++++++++ packages/dsh-image-generation/lib/preview.js | 5 +++-- scripts/image-preview-fixture.mjs | 2 +- test/image-generation.test.mjs | 20 +++++++++++++++----- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/docs/STATUS.md b/docs/STATUS.md index 62c602e99..cf98d1032 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -1,5 +1,13 @@ # Implementation and verification status +## Conversation image preview session identity — 2026-09-10 + +- **Root cause:** Desktop creates `session-` IDs; the preview route accepted bare UUIDs. The existing smoke fixture also used a bare UUID, so it missed the production request rejection. The screenshot's stored successful image result and its 4,635,310-byte PNG are intact. +- **Corrected:** bounded, opaque session IDs are resolved by the Harness session controller. File access comes exclusively from the session's successful image tool result and retains path, symlink, size, PNG and SHA-256 checks. The Host smoke fixture now uses Desktop's `session-` form. +- **PASS:** the updated regression cases reproduce six failures against the old route; the fix passes 42 plugin tests plus 2 dependency-closure checks. TypeScript, production build, source and final-packaged Host authentication/configuration/model/PNG smoke, and strict deep development signature pass. +- **PASS:** read-only replay of the user's actual stored session returns its original 2560×1440 PNG with matching hash. An isolated Harness page replaying that same session displays the thumbnail automatically, opens the decoded original on click, and closes with Esc. Simulated historical records also pass thumbnail/expand/close. Screenshots, checks and r7 macOS arm64 DMG/ZIP/tarball are in `outputs/image-generation-plugin-20260910-r7`. +- **Scope:** the running user application remains unchanged; install r7 to apply the fix there. This revision reuses the existing provider-generated image. Paid generation, native Office delivery, other platforms and notarization are separate acceptance steps. + ## Expanded Seedream presets — 2026-09-10 - **Implemented:** the built-in catalog includes `doubao-seedream-5-0-pro-260628`, `doubao-seedream-5-0-260128`, `doubao-seedream-5-0-lite-260128`, `doubao-seedream-4-5-251128` and `doubao-seedream-4-0-250828`, with custom model/endpoint entry retained. The existing saved model and default remain in place. diff --git a/packages/dsh-image-generation/lib/preview.js b/packages/dsh-image-generation/lib/preview.js index 9c8c6f7fc..daf67becd 100644 --- a/packages/dsh-image-generation/lib/preview.js +++ b/packages/dsh-image-generation/lib/preview.js @@ -40,8 +40,9 @@ export async function previewImage(ctx, request) { const url = new URL(request.url) const sessionId = url.searchParams.get('session') const sha = url.searchParams.get('asset') - if (!/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(sessionId ?? '') || !/^[a-f0-9]{64}$/.test(sha ?? '')) throw new ImageError('PREVIEW', 'Invalid image reference.', 400) - // Use the same cold/live session authority as the Harness session API. + if (!sessionId || sessionId.length > 1024 || /[\u0000-\u001f\u007f]/.test(sessionId) || !/^[a-f0-9]{64}$/.test(sha ?? '')) throw new ImageError('PREVIEW', 'Invalid image reference.', 400) + // Harness IDs are opaque (Desktop uses session-). The controller + // resolves their identity; only the authorized result supplies a file path. const inspection = await ctx.sessionController.inspect(sessionId, request.signal) const image = referencedImage(inspection.events, sha) if (!image || !inspection.meta.cwd) throw new ImageError('PREVIEW', 'Image is unavailable in this session.', 404) diff --git a/scripts/image-preview-fixture.mjs b/scripts/image-preview-fixture.mjs index 66ed27c79..7aa68312a 100644 --- a/scripts/image-preview-fixture.mjs +++ b/scripts/image-preview-fixture.mjs @@ -15,7 +15,7 @@ export async function imagePreviewFixture(home) { const png = await sharp(Buffer.from('Image preview fixture')).png().toBuffer() const asset = await materialize(cwd, png) const image = { ...asset, asset_id: `sha256:${asset.sha256}`, media_type: 'image/png', width: 960, height: 540, bytes: png.length, provider: 'bytedance', model: 'doubao-seedream-5-0-pro-260628' } - const id = randomUUID() + const id = `session-${randomUUID()}` const session = Session.create(id, undefined, { version: 0, id, createdAt: Date.now(), isSeeded: false, cwd, delegationDepth: 0 }) session.append('turn/start', { turn: 1 }) session.append('step/start', { turn: 1, step: 1 }) diff --git a/test/image-generation.test.mjs b/test/image-generation.test.mjs index 965f7b1d0..582dc8307 100644 --- a/test/image-generation.test.mjs +++ b/test/image-generation.test.mjs @@ -297,18 +297,17 @@ describe('image tool and durable Office assets', () => { describe('session-authorized generated image previews', () => { - async function previewFixture() { + async function previewFixture(id = `session-${randomUUID()}`) { const workspace = await temp() const png = await sharp({ create: { width: 16, height: 9, channels: 4, background: '#112233' } }).png().toBuffer() const asset = await materialize(workspace, png) const image = { ...asset, asset_id: `sha256:${asset.sha256}`, media_type: 'image/png', width: 16, height: 9, bytes: png.length, provider: 'bytedance', model: DEFAULTS.bytedance.model } - const id = randomUUID() const content = [{ type: 'text', text: JSON.stringify(image) }] const events = [{ type: 'tool/call', data: { callId: 'image-call', name: 'image_generate' } }, { type: 'tool/result', surfaceOp: 'append', data: { message: { source: { callId: 'image-call' }, content: [{ type: 'tool-result', content }] } } }] const log = vi.fn() const ctx = { sessionController: { inspect: vi.fn(async session => ({ meta: { cwd: workspace }, events: session === id ? events : [] })) }, logger: { info: log } } - const request = (session = id, sha = image.sha256) => new Request(`http://localhost/api/image-generation.preview?session=${session}&asset=${sha}`) + const request = (session = id, sha = image.sha256) => new Request(`http://localhost/api/image-generation.preview?${new URLSearchParams({ session, asset: sha })}`) return { workspace, png, image, id, content, events, ctx, request, target: path.join(workspace, image.workspace_path) } } it('serves historical successful PNGs with integrity, cache, content-type and audit checks', async () => { @@ -321,6 +320,13 @@ describe('session-authorized generated image previews', () => { expect(result.headers.get('x-content-type-options')).toBe('nosniff') expect(f.ctx.logger.info).toHaveBeenCalledWith(expect.stringContaining('preview allowed'), f.id, f.image.sha256) }) + it.each([randomUUID(), 'session-1', 'imported:landscape'])('resolves the opaque Harness session ID %s', async id => { + const f = await previewFixture(id) + const result = await previewImage(f.ctx, f.request()) + expect(result.status).toBe(200) + expect(Buffer.from(await result.arrayBuffer())).toEqual(f.png) + expect(f.ctx.sessionController.inspect).toHaveBeenCalledWith(id, expect.any(AbortSignal)) + }) it('requires a successful image tool result in the requested session', async () => { const f = await previewFixture() expect((await previewImage(f.ctx, f.request(randomUUID()))).status).toBe(404) @@ -333,11 +339,15 @@ describe('session-authorized generated image previews', () => { f.events.splice(0, 2, { type: 'tool/code-dispatch', data: { name: 'image_generate', subCallId: 'ptc', content: f.content } }) expect((await previewImage(f.ctx, f.request())).status).toBe(200) }) - it('rejects path traversal and invalid asset contracts before reading a session', async () => { + it('validates query bounds and resolves session identity through the Host', async () => { const f = await previewFixture() - expect((await previewImage(f.ctx, f.request('../outside'))).status).toBe(400) + for (const id of ['', 'a'.repeat(1025), 'session-\u0000']) { + expect((await previewImage(f.ctx, f.request(id))).status).toBe(400) + } expect((await previewImage(f.ctx, f.request(f.id, '../secret'))).status).toBe(400) expect(f.ctx.sessionController.inspect).not.toHaveBeenCalled() + expect((await previewImage(f.ctx, f.request('../outside'))).status).toBe(404) + expect(f.ctx.sessionController.inspect).toHaveBeenCalledWith('../outside', expect.any(AbortSignal)) expect(imageResult([null, { type: 'text', text: 'broken' }])).toBeUndefined() for (const change of [{ bytes: -1 }, { bytes: 1000000000 }, { workspace_path: '/etc/passwd' }, { media_type: 'text/html' }, { asset_id: 'wrong' }]) { expect(imageResult([{ type: 'text', text: JSON.stringify({ ...f.image, ...change }) }])).toBeUndefined() From c4a49b1440fb403c6c71f4f4d7f90c5ce67bb0e3 Mon Sep 17 00:00:00 2001 From: Jianghan <58623049+SuperstructureJH@users.noreply.github.com> Date: Fri, 11 Sep 2026 17:27:20 +0800 Subject: [PATCH 8/8] ci: validate pull requests targeting v0.9.0 --- .github/workflows/release.yml | 1 + scripts/verify-desktop-routes.mjs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f27663e2b..b2b616407 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,7 @@ on: pull_request: branches: - main + - v0.9.0 workflow_dispatch: inputs: target: diff --git a/scripts/verify-desktop-routes.mjs b/scripts/verify-desktop-routes.mjs index 7c6d4724f..96e129735 100644 --- a/scripts/verify-desktop-routes.mjs +++ b/scripts/verify-desktop-routes.mjs @@ -13,7 +13,7 @@ await new Promise(resolve => portServer.listen(0, '127.0.0.1', resolve)) const port = portServer.address().port await new Promise(resolve => portServer.close(resolve)) const base = `http://127.0.0.1:${port}` -const child = spawn(path.join(root, 'node_modules/node/bin/node'), [path.join(root, 'build/harness-node-entry.mjs'), +const child = spawn(path.join(root, 'node_modules/node/bin', process.platform === 'win32' ? 'node.exe' : 'node'), [path.join(root, 'build/harness-node-entry.mjs'), path.join(root, 'node_modules/@deepseek-ai/dsh/lib/bin.js'), 'web', '--patch', path.join(root, 'build/dsh-desktop.patch.yml'), '--no-open', '--host', '127.0.0.1', '--port', String(port)], { cwd: root, env: { ...process.env, DSH_HOME: home, NO_COLOR: '1', DSH_TELEMETRY_DISABLED: '1' }, stdio: ['ignore', 'pipe', 'pipe'] }) let output = ''