From 220d7da6e196f9dcb9952a3fa688d9721d841855 Mon Sep 17 00:00:00 2001 From: VicarDa Date: Mon, 9 Nov 2026 19:56:25 +0800 Subject: [PATCH 1/3] feat(workbuddy): add UI i18n, host locale detection and emulator picker - Add ui-i18n / ui-messages / ui-formatters and host-ui-locale for zh/en UI - Add emulator picker for the workbench - Update installer, viewer and workbench pages and related docs/tests - Add WorkBuddy editions compatibility and open platform research docs Co-Authored-By: Claude Opus 5.5 --- README.ja-JP.md | 39 +- README.md | 39 +- README.zh-CN.md | 39 +- workbuddy-plugin/CHANGELOG.md | 11 + workbuddy-plugin/INSTALL.md | 41 +- workbuddy-plugin/README.md | 18 +- workbuddy-plugin/README.zh-CN.md | 16 +- .../connector/connector-meta.json | 16 +- .../skills/control/installation.html | 2 +- .../docs/install-from-github.zh-CN.md | 118 ++- .../docs/installer-development.md | 4 +- ...open-platform-onboarding-research.zh-CN.md | 78 ++ .../open-platform-submission-draft.zh-CN.md | 76 ++ .../workbuddy-editions-compatibility.zh-CN.md | 151 ++++ workbuddy-plugin/install.sh | 4 +- workbuddy-plugin/release-readiness.json | 2 +- ...\350\243\205\346\214\207\345\215\227.html" | 2 +- workbuddy-plugin/scripts/install-local.mjs | 2 +- .../scripts/install-macos.command | 40 +- .../scripts/installer-handoff.command | 2 +- .../scripts/test-host-discovery.mjs | 19 +- .../scripts/test-release-installer.mjs | 7 +- .../scripts/test-workbuddy-host.mjs | 7 +- workbuddy-plugin/src/broker-client.ts | 5 +- workbuddy-plugin/src/broker.ts | 5 +- workbuddy-plugin/src/connection-status.ts | 26 +- workbuddy-plugin/src/emulator-picker.ts | 78 ++ workbuddy-plugin/src/host-ui-locale.ts | 36 + workbuddy-plugin/src/installation.ts | 6 +- workbuddy-plugin/src/service.ts | 3 + workbuddy-plugin/src/ui-formatters.ts | 52 ++ workbuddy-plugin/src/ui-i18n.ts | 66 ++ workbuddy-plugin/src/ui-messages.ts | 692 ++++++++++++++++++ workbuddy-plugin/src/viewer-page.ts | 35 +- workbuddy-plugin/src/viewer.ts | 7 +- workbuddy-plugin/src/workbench-page.ts | 619 +++++++++------- workbuddy-plugin/tests/host-ui-locale.spec.ts | 70 ++ workbuddy-plugin/tests/installation.spec.ts | 6 + workbuddy-plugin/tests/ui-i18n.spec.ts | 73 ++ 39 files changed, 2120 insertions(+), 392 deletions(-) create mode 100644 workbuddy-plugin/docs/open-platform-onboarding-research.zh-CN.md create mode 100644 workbuddy-plugin/docs/open-platform-submission-draft.zh-CN.md create mode 100644 workbuddy-plugin/docs/workbuddy-editions-compatibility.zh-CN.md create mode 100644 workbuddy-plugin/src/emulator-picker.ts create mode 100644 workbuddy-plugin/src/host-ui-locale.ts create mode 100644 workbuddy-plugin/src/ui-formatters.ts create mode 100644 workbuddy-plugin/src/ui-i18n.ts create mode 100644 workbuddy-plugin/src/ui-messages.ts create mode 100644 workbuddy-plugin/tests/host-ui-locale.spec.ts create mode 100644 workbuddy-plugin/tests/ui-i18n.spec.ts diff --git a/README.ja-JP.md b/README.ja-JP.md index 02e74f4f..b1b8fb70 100644 --- a/README.ja-JP.md +++ b/README.ja-JP.md @@ -56,24 +56,36 @@ OpenGUI は実際の画面を読み取り、自然言語の指示でスマート ## WorkBuddyへのインストール -**macOS(Apple Silicon / Intel)、WorkBuddy 5.5.3 以降**に対応しています。現在の公開版は [0.4.0 公開テスト版](https://github.com/Core-Mate/OpenGUI/releases/tag/opengui-workbuddy-v0.4.0)です。 +**macOS(Apple Silicon / Intel)、WorkBuddy 5.5.3 以降**に対応しています。現在の公開版は [0.4.1 公開テスト版](https://github.com/Core-Mate/OpenGUI/releases/tag/opengui-workbuddy-v0.4.1)です。中国国内版と国際版は、同じ OpenGUI パッケージとバージョンを使用します。カスタム MCP としてインストールでき、コネクターマーケットへの掲載を待つ必要はありません。 + +OpenGUI と WorkBuddy のアカウントは別です。OpenGUI の SMS ログインには現在も**中国本土の電話番号**が必要です。国際版 WorkBuddy に Google/GitHub でログインしても、OpenGUI には自動ログインされません。 > [!IMPORTANT] > **インストール後は WorkBuddy を一度再起動してください。** 他のタスクを終了し、**⌘Q で完全に終了**してから開き直し、OpenGUI を認証してください。ウィンドウを閉じるだけでは終了しません。 **手順1.Terminalコマンドでインストール** -macOS の Terminal で次を実行します: +macOS の Terminal で、使用する WorkBuddy のエディションに対応するコマンドを実行してください。両方を使う場合は、それぞれのコマンドで個別にインストールします。以降の手順は、ここで選んだエディションで行ってください。 + +**中国国内版 — WorkBuddy.app** + +```sh +cd "$HOME" && curl -fsSL https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh | bash -s -- --app "/Applications/WorkBuddy.app" --download-source cn +``` + +**国際版 — WorkBuddy AI.app** ```sh -cd "$HOME" && curl -fsSL https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh | bash +cd "$HOME" && curl -fsSL https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh | bash -s -- --app "/Applications/WorkBuddy AI.app" --download-source official ``` -スクリプトは必要なファイルを検証し、ランタイムと OpenGUI を設定します。 +インストール先のアプリを明示するため、両方のエディションがあっても対象を取り違えません。アプリの保存先や名前が異なる場合は、`--app` の後のパスを実際のものに置き換えてください。スクリプトは必要なファイルをダウンロードして検証し、ランタイムと OpenGUI を設定します。 -Node と npm の依存関係は既定で中国の npmmirror を使い、失敗時は公式ソースへ切り替えます。この設定は今回のインストールにのみ適用されます。公式ソースを使う場合は、末尾の `bash` を `bash -s -- --download-source official` に変更してください。動画コンポーネントは別途 GitHub から取得します。[ダウンロード元の選択](workbuddy-plugin/INSTALL.md#download-sources)も参照してください。 +中国国内版のコマンドは Node と npm の依存関係を npmmirror から取得し、失敗時は公式ソースへ切り替えます。国際版のコマンドは公式ソースを使います。中国国内版でも公式ソースを使う場合は、`--download-source cn` を `--download-source official` に変更してください。この設定は今回のインストールにのみ適用されます。動画コンポーネントは別途 GitHub から取得します。[ダウンロード元の選択](workbuddy-plugin/INSTALL.md#download-sources)も参照してください。 -[install.sh を個別にダウンロード](https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh)して `bash /実際のパス/install.sh` で実行することもできます。 +[install.sh を個別にダウンロード](https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh)し、`bash /実際のパス/install.sh` に上記と同じ `--app` と `--download-source` の引数を付けて実行することもできます。 + +国際版のアカウント制限は[国際版のインストール説明](workbuddy-plugin/INSTALL.md#international-workbuddy)、認証と接続確認の詳細は[GitHub からのインストール手順(中国語)](workbuddy-plugin/docs/install-from-github.zh-CN.md)を参照してください。 **手順2.WorkBuddyを一度再起動** @@ -81,7 +93,10 @@ Node と npm の依存関係は既定で中国の npmmirror を使い、失敗 **手順3.Skillとコネクターを認証** -WorkBuddy の「专家·技能·连接器 → 技能」で OpenGUI を選び、認証します。続いて「连接器」で OpenGUI を認証してください。**Skill とコネクターの両方に認証が必要です。** +- **中国国内版**:「专家·技能·连接器 → 技能」で OpenGUI を認証して有効化します。続いて「连接器 → 自定义连接器」で `opengui` を選び、信頼を確認して有効化してください。 +- **国際版**:**Experts · Skills · Connectors → Skills** で OpenGUI を認証して有効化します。続いて **Connectors → Custom connectors** で `opengui` の **Trust** をクリックし、有効化してください。 + +更新後に Trust が再表示された場合も、内容を確認して本人が承認してください。**Skill とコネクターの両方に認証が必要です。** ![OpenGUI Skill と MCP の認証手順](./workbuddy-plugin/resources/Skill和MCP授权.gif) @@ -89,7 +104,13 @@ OpenGUI が見つからない場合は「自定义连接器」の MCP サービ **手順4.接続を確認して使い始める** -Android スマートフォンの USB デバッグを有効にして PC に接続し、端末上で USB デバッグを承認します。WorkBuddy で `/opengui` に続けて実行したい指示を入力してください。 +まず WorkBuddy に「`opengui_list_devices` を一度だけ呼び出して接続を確認し、端末数を報告してください。スマートフォンは操作しないでください」と依頼できます。正常な呼び出しで端末数が 0 台でも、プラグインの接続は確認できます。 + +スマートフォンのタスクを開始する手順: + +1. Android スマートフォンの USB デバッグを有効にして PC に接続し、端末上で承認します。作業画面からローカルの Android エミュレーターを起動することもできます。 +2. WorkBuddy で `/opengui` を入力し、右側のタスク画面を開いて OpenGUI にログインします。 +3. 目標を入力し、使用する端末を確認して **开始执行**(タスク開始)をクリックします。実行モデルは現在の WorkBuddy の会話モデルに従います。 試してみる:小紅書の投稿を準備 @@ -103,6 +124,8 @@ Android スマートフォンの USB デバッグを有効にして PC に接続 /opengui 【アプリ/画面】の【機能や操作フロー】をテストし、特に【確認する点】を確認してください。問題があれば手順とスクリーンショットを記録し、【終了条件】に達したら停止してください。 ``` +**表示言語(未公開)**:中国語/英語の切り替え、WorkBuddy の表示言語への追従、エミュレーター選択 UI の改善はテスト中です。公開ソースと 0.4.1 パッケージにはまだ含まれていません。言語切り替えは UI が対象で、タスクの説明、TODO の内容、確認用の下書きは原文を保持します。 + ## DeepSeek HarnessでOpenGUIを使う macOSでは、`main` ブランチの安定したインストーラーSkillをCodexに実行させる方法が最短です。実行するたびに最新の安定版OpenGUIプラグインを解決してインストールし、ロールバック用に明示的なバージョン指定も利用できます。Node.js 22.19以降または24以降が必要で、互換性のあるDSHバージョンは自動的にインストールされます。次の内容を1つのプロンプトとして送信します: diff --git a/README.md b/README.md index a1d7ab9e..45feccad 100644 --- a/README.md +++ b/README.md @@ -57,24 +57,36 @@ After installation and authorization, enter these instructions in the correspond ## Use OpenGUI in WorkBuddy -Supported on **macOS (Apple Silicon / Intel) with WorkBuddy 5.5.3 or later**. The current public release is [0.4.0, a public-testing prerelease](https://github.com/Core-Mate/OpenGUI/releases/tag/opengui-workbuddy-v0.4.0). Download and run the installer on your Mac, restart WorkBuddy once, then authorize OpenGUI. +Supported on **macOS (Apple Silicon / Intel) with WorkBuddy 5.5.3 or later**. The current public release is [0.4.1, a public-testing prerelease](https://github.com/Core-Mate/OpenGUI/releases/tag/opengui-workbuddy-v0.4.1). China and international WorkBuddy use the same OpenGUI package and version. Custom MCP installation is available without a connector marketplace listing. + +OpenGUI and WorkBuddy accounts are separate. OpenGUI SMS sign-in currently requires a **mainland-China phone number**; signing into international WorkBuddy with Google/GitHub does not sign you into OpenGUI. > [!IMPORTANT] > **Restart WorkBuddy once after installation.** Finish other tasks, **fully quit with ⌘Q**, then reopen WorkBuddy before authorizing OpenGUI. Closing its window is not enough. **Step 1. Install with a Terminal command** -Open macOS Terminal, paste this command, and run it: +Open macOS Terminal and run the command for your WorkBuddy edition. If you use both editions, install into each one separately. Complete the remaining steps in the edition selected here. + +**China edition — WorkBuddy.app** + +```sh +cd "$HOME" && curl -fsSL https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh | bash -s -- --app "/Applications/WorkBuddy.app" --download-source cn +``` + +**International edition — WorkBuddy AI.app** ```sh -cd "$HOME" && curl -fsSL https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh | bash +cd "$HOME" && curl -fsSL https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh | bash -s -- --app "/Applications/WorkBuddy AI.app" --download-source official ``` -This downloads and runs the installer without saving it manually first. The script verifies the subsequent installation files, prepares the runtime, and configures OpenGUI. +These commands explicitly select the target application, so having both editions installed will not cause OpenGUI to be installed into the wrong one. If your application is elsewhere or has been renamed, replace the path after `--app` with its actual path. The script downloads and verifies the installation files, prepares the runtime, and configures OpenGUI. -Node and npm dependencies default to the China npmmirror service with official fallback, scoped to this installation. To use official sources, replace the final `bash` with `bash -s -- --download-source official`. Video assets use GitHub separately; an npm mirror does not accelerate them. See [download source options](workbuddy-plugin/INSTALL.md#download-sources) for a custom video mirror. +The China command uses npmmirror for Node and npm dependencies with official fallback; the international command uses official sources. To use official sources with China WorkBuddy, change `--download-source cn` to `--download-source official`. This setting applies only to this installation. Video assets use GitHub separately; an npm mirror does not accelerate them. See [download source options](workbuddy-plugin/INSTALL.md#download-sources) for a custom video mirror. -You can also [download install.sh separately](https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh) and run `bash /actual/path/install.sh` in Terminal. +You can also [download install.sh separately](https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh) and run `bash /actual/path/install.sh` with the same `--app` and `--download-source` arguments shown above. + +See [international installation and account limitations](workbuddy-plugin/INSTALL.md#international-workbuddy), or the [GitHub installation flow (Chinese)](workbuddy-plugin/docs/install-from-github.zh-CN.md) for a copyable WorkBuddy prompt, native authorization and connection verification. **Step 2. Restart WorkBuddy once** @@ -82,7 +94,10 @@ After the installer finishes, complete other WorkBuddy tasks, fully quit with ** **Step 3. Authorize the Skill and connector** -In WorkBuddy, open **Experts · Skills · Connectors → Skills**, find OpenGUI, and complete authorization. Then open **Connectors**, find OpenGUI, and authorize it as well. +- **China edition**: authorize and enable OpenGUI under **专家·技能·连接器 → 技能**. Then open **连接器 → 自定义连接器**, find `opengui`, complete the trust prompt, and enable it. +- **International edition**: authorize and enable OpenGUI under **Experts · Skills · Connectors → Skills**. Then open **Connectors → Custom connectors**, click **Trust** for `opengui`, and enable it. + +If an update triggers another Trust prompt, review and confirm it yourself. **Both the Skill and connector require authorization.** @@ -92,7 +107,13 @@ If OpenGUI is not listed under Connectors, check **Custom connectors** for its M **Step 4. Verify the connection and start using OpenGUI** -Enable USB debugging on your Android phone, connect it to your computer, and approve USB debugging on the phone. Enter /opengui, then enter the instructions you want the phone to execute to control it. +You can first ask WorkBuddy: “Call `opengui_list_devices` once to verify the connection, report the device count, and do not operate a phone.” A successful call returning zero devices also confirms the plugin connection. + +To start a phone task: + +1. Enable USB debugging on your Android phone, connect it to your computer, and allow debugging on the phone. You can also start a local Android emulator from the workbench. +2. Enter `/opengui` in WorkBuddy to open the task home on the right, then sign into OpenGUI as prompted. +3. Describe your goal, confirm the device, and click **开始执行** (Start task). Execution follows the current WorkBuddy conversation model. Try it: Post on Tiktok @@ -106,6 +127,8 @@ Try it: Vibe Testing /opengui Help me test [feature or workflow] in [app/page], focusing on [issues to check]. If you find any problems, record the steps and screenshots. Stop when [end condition] is reached. ``` +**UI language (not yet released)**: Chinese/English switching, following WorkBuddy’s display language, and emulator-picker improvements are being tested. They are not yet included in the public source or 0.4.1 package. The language option translates interface labels; task descriptions, TODO content, and review drafts retain their original text. + ## Use OpenGUI in DeepSeek Harness The shortest path on macOS is to let Codex run the stable installer Skill from `main`. Each run resolves the latest stable OpenGUI plugin release, while an explicit version remains available for rollback. It requires Node.js 22.19+ or 24+ and installs the compatible DSH version automatically. Paste this as one prompt: diff --git a/README.zh-CN.md b/README.zh-CN.md index 4bb5b453..017a74b6 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -56,24 +56,36 @@ OpenGUI 让 AI 根据真实屏幕截图理解 App,用自然语言完成手机 ## WorkBuddy 安装 -支持 **macOS(Apple 芯片 / Intel)和 WorkBuddy 5.5.3 及以上版本**。当前公开版本为 [0.4.0 公测预发布版](https://github.com/Core-Mate/OpenGUI/releases/tag/opengui-workbuddy-v0.4.0)。在电脑上下载并运行安装脚本,安装后重启一次 WorkBuddy,再完成授权。 +支持 **macOS(Apple 芯片 / Intel)和 WorkBuddy 5.5.3 及以上版本**。当前公开版本为 [0.4.1 公测预发布版](https://github.com/Core-Mate/OpenGUI/releases/tag/opengui-workbuddy-v0.4.1),国内版和国际版使用同一个 OpenGUI 安装包和版本号。通过自定义 MCP 安装,无需等待连接器市场上架。 + +OpenGUI 账号与 WorkBuddy 账号独立,当前 OpenGUI 短信登录仍需要**中国大陆手机号**;国际版 WorkBuddy 的 Google/GitHub 登录不会自动登录 OpenGUI。 > [!IMPORTANT] > **安装后需要重启一次 WorkBuddy。** 请先结束其他任务,用 **⌘Q 完全退出** WorkBuddy,再重新打开,然后授权 OpenGUI。只关闭窗口不算退出。 **步骤1.Terminal命令下载安装** -打开 macOS“终端”,粘贴并运行: +打开 macOS“终端”,根据使用的 WorkBuddy 版本选择对应命令。两版都需要使用时,分别执行各自的安装命令;后续步骤均在这里选择的版本中完成。 + +**国内版 — WorkBuddy.app** + +```sh +cd "$HOME" && curl -fsSL https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh | bash -s -- --app "/Applications/WorkBuddy.app" --download-source cn +``` + +**国际版 — WorkBuddy AI.app** ```sh -cd "$HOME" && curl -fsSL https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh | bash +cd "$HOME" && curl -fsSL https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh | bash -s -- --app "/Applications/WorkBuddy AI.app" --download-source official ``` -这条命令会下载并运行安装脚本,无需先手动保存文件。脚本会校验后续安装文件、准备运行环境并配置 OpenGUI。 +上述命令明确指定目标应用,两版同时安装时也不会装错。如果应用不在默认位置或已重命名,请将 `--app` 后的路径替换为实际应用路径。脚本会下载安装文件并校验、准备运行环境,然后配置 OpenGUI。 -Node 和 npm 依赖默认使用国内 npmmirror,下载失败时回退官方源,只影响本次安装。需要官方源时,将命令末尾的 `bash` 改为 `bash -s -- --download-source official`。视频组件来自 GitHub,换 npm 源不会加速它;有自己的视频镜像时,可加 `--video-mirror https://镜像地址/归档目录`,详见[下载源选项](workbuddy-plugin/INSTALL.md#download-sources)。 +国内版命令使用 npmmirror 下载 Node 和 npm 依赖,失败时回退官方源;国际版命令使用官方源。国内版需要使用官方源时,将 `--download-source cn` 改为 `--download-source official`。下载源设置只影响本次安装。视频组件来自 GitHub,换 npm 源不会加速它;有自己的视频镜像时,可加 `--video-mirror https://镜像地址/归档目录`,详见[下载源选项](workbuddy-plugin/INSTALL.md#download-sources)。 -也可以[单独下载安装脚本(install.sh)](https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh),保存后在终端运行 `bash /实际路径/install.sh`。 +也可以[单独下载安装脚本(install.sh)](https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh),保存后使用 `bash /实际路径/install.sh`,并附上对应命令中的 `--app` 和 `--download-source` 参数。 + +国际版现有账号限制见[国际版安装说明](workbuddy-plugin/INSTALL.md#international-workbuddy)。可复制的 WorkBuddy 安装提示词、授权和验证步骤见[从 GitHub 安装 OpenGUI](workbuddy-plugin/docs/install-from-github.zh-CN.md)。 **步骤2.重启一次 WorkBuddy** @@ -81,7 +93,10 @@ Node 和 npm 依赖默认使用国内 npmmirror,下载失败时回退官方源 **步骤3.授权技能和连接器** -打开 WorkBuddy 的“专家·技能·连接器 → 技能”,找到 OpenGUI,按提示完成授权;之后在连接器中,找到OpenGUI,再次授权。 +- **国内版**:“专家·技能·连接器 → 技能”中授权并启用 OpenGUI;再到“连接器 → 自定义连接器”,找到 `opengui`,按提示完成信任并开启开关。 +- **国际版**:在 **Experts · Skills · Connectors → Skills** 中授权并启用 OpenGUI;再到 **Connectors → Custom connectors**,为 `opengui` 点击 **Trust** 并开启开关。 + +更新后若再次出现 Trust 提示,请检查后由本人确认。 **注意:技能 和 连接器,都需要授权哦!** @@ -91,7 +106,13 @@ Node 和 npm 依赖默认使用国内 npmmirror,下载失败时回退官方源 **步骤4.验证连接并开始使用** -把Android手机打开USB调试,连上电脑,输入/opengui,输入需要给手机的执行指令,即可控制手机执行。首次连接时,请在手机上允许 USB 调试授权。 +可先在 WorkBuddy 聊天中发送:“仅调用一次 `opengui_list_devices` 验证连接,报告设备数量,不操作手机。”调用成功且返回 0 台设备,也可以确认插件连接成功。 + +开始手机任务: + +1. Android 手机开启 USB 调试并连接电脑,在手机上允许调试授权;没有手机时,可从工作台启动本地 Android 模拟器。 +2. 在 WorkBuddy 输入 `/opengui`,打开右侧任务首页,按提示登录 OpenGUI。 +3. 填写任务目标、确认设备,然后点击 **开始执行**。执行模型跟随当前 WorkBuddy 对话。 试一试:用小红书发帖 @@ -105,6 +126,8 @@ Node 和 npm 依赖默认使用国内 npmmirror,下载失败时回退官方源 /opengui 帮我测试【应用/页面】的【功能或操作流程】,重点检查【关注的问题】。如果发现异常,记录操作步骤和截图,当做到【结束条件】就停。 ``` +**界面语言(尚未发布)**:中英文切换、默认跟随 WorkBuddy 语言和模拟器下拉框改进正在测试,尚未合入公开源码或 0.4.1 安装包。后续语言选项只翻译界面,任务描述、TODO 内容和审核草稿保留原文。 + ## 在 DeepSeek Harness 中使用 OpenGUI 第一次使用 DSH 插件时,可先阅读 [OpenGUI × DeepSeek Harness 简明说明与 FAQ](./deepseek-harness-plugin/docs/quick-start-and-faq.zh.md)。 diff --git a/workbuddy-plugin/CHANGELOG.md b/workbuddy-plugin/CHANGELOG.md index c8b33ddb..a22f0b5b 100644 --- a/workbuddy-plugin/CHANGELOG.md +++ b/workbuddy-plugin/CHANGELOG.md @@ -1,3 +1,14 @@ +# Unreleased + +- Add Chinese and English workbench UI with an accessible language selector, WorkBuddy display-language defaults, local preference persistence and synchronization across tabs. +- Follow each host's display-language changes independently even through a shared broker; retain Chinese/English overrides and fall back to the browser language when host metadata is unavailable. +- Switch UI language in place while preserving task text, review drafts, skip reasons, device streams and control state; clarify mainland-China SMS sign-in in both languages. +- Replace the native emulator dropdown with a keyboard-accessible picker in both device selection surfaces; wrap long names, bound the menu to the available space and adapt the start button to narrow panels in both languages. + +- Select China or international WorkBuddy explicitly with `--edition cn|international`, retaining existing automatic discovery and product-specific configuration paths. +- Default international WorkBuddy AI downloads to official Node/npm sources while preserving explicit source overrides. +- Verify both current WorkBuddy bundle identities with the isolated native-host MCP directory check; document the separate mainland-China account and marketplace requirements. + # OpenGUI for WorkBuddy 0.4.0 public testing - Publish the task home, workbench, account/model selection, takeover, content review, and report runtime as a distinct plugin version. diff --git a/workbuddy-plugin/INSTALL.md b/workbuddy-plugin/INSTALL.md index 72563a9b..1ce8a392 100644 --- a/workbuddy-plugin/INSTALL.md +++ b/workbuddy-plugin/INSTALL.md @@ -1,21 +1,31 @@ # Install OpenGUI into WorkBuddy -This guide installs the published OpenGUI 0.4.0 WorkBuddy connector on macOS. It uses the [official public-testing release](https://github.com/Core-Mate/OpenGUI/releases/tag/opengui-workbuddy-v0.4.0); no source checkout, system Node installation, full backend deployment, or DSH installation is needed. +This guide installs the published OpenGUI 0.4.1 WorkBuddy connector on macOS. It uses the [official public-testing release](https://github.com/Core-Mate/OpenGUI/releases/tag/opengui-workbuddy-v0.4.1); no source checkout, system Node installation, full backend deployment, or DSH installation is needed. > [!IMPORTANT] > **Restart WorkBuddy once after installation.** Finish other tasks, **fully quit with ⌘Q**, then reopen WorkBuddy before authorizing the Skill and connector. Closing its window is not enough. +For copyable WorkBuddy chat prompts and the full Chinese flow, see [install from GitHub](docs/install-from-github.zh-CN.md). China and international WorkBuddy use the same OpenGUI 0.4.1 package and version. + ## Download and run the installer -The complete download, authorization and first-task instructions are in the [root README](../README.md#use-opengui-in-workbuddy) ([中文](../README.zh-CN.md#workbuddy-安装)). Open macOS Terminal and run this command to download and execute the installer: +The complete download, authorization and first-task instructions are in the [root README](../README.md#use-opengui-in-workbuddy) ([中文](../README.zh-CN.md#workbuddy-安装)). Open macOS Terminal and select the target application explicitly. + +**China edition — WorkBuddy.app:** + +```sh +cd "$HOME" && curl -fsSL https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh | bash -s -- --app "/Applications/WorkBuddy.app" --download-source cn +``` + +**International edition — WorkBuddy AI.app:** ```sh -curl -fsSL https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh | bash +cd "$HOME" && curl -fsSL https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh | bash -s -- --app "/Applications/WorkBuddy AI.app" --download-source official ``` -Alternatively, [download install.sh](https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh) and run `bash /actual/path/install.sh`. Installation runs on the Mac; restart WorkBuddy once afterward, then authorize OpenGUI and verify the connection. +Install separately if you use both editions. Replace the application path if it has been moved or renamed. Alternatively, [download install.sh](https://raw.githubusercontent.com/Core-Mate/OpenGUI/main/workbuddy-plugin/install.sh) and run `bash /actual/path/install.sh` with the same `--app` and `--download-source` arguments. Restart the selected WorkBuddy once afterward, then authorize OpenGUI and verify the connection. -The repository entry and its pinned installer/authorization guide are published together. They install the refreshed public-testing 0.4.0 runtime. This maintainer-requested replacement updates the release assets and their adjacent SHA-256 sidecars; previously downloaded files have different checksums. The [manual installation guide](resources/OpenGUI-安装指南.html) remains available as an alternative. +The repository entry and its pinned installer/authorization guide are published together. They install the refreshed public-testing 0.4.1 runtime. This release uses new versioned assets and adjacent SHA-256 sidecars; earlier releases remain unchanged. The [manual installation guide](resources/OpenGUI-安装指南.html) remains available as an alternative. The entry downloads the installer and a self-contained [authorization guide](resources/OpenGUI-授权指南.html), verifies their SHA-256 pins, and runs installation without a Terminal confirmation or Finder step. After successful configuration it opens the guide in the default browser and prints `AUTHORIZATION_GUIDE` and `INSTALLATION_RESULT` paths. WorkBuddy can display that same HTML if browser opening is unavailable. Browser failure does not mean installation failed; do not reinstall just to reopen the instructions. @@ -41,7 +51,7 @@ If you have a video archive mirror, pass `--video-mirror https://your-mirror.exa Installation downloads a private Node runtime and the published OpenGUI package, adds the `opengui` MCP server and Skill, and registers seven lifecycle Hooks: `UserPromptSubmit`, `PreToolUse`, `Stop`, `SubagentStop`, `FinalStop`, `SessionEnd`, and `StopFailure`. These local commands bind tasks and handle stopping and cleanup; review them in `/hooks` when prompted. Affected settings are backed up and unrelated plugins are retained. -No phone is required during installation. Later Android use requires USB debugging authorization. Accounts and configured models use the official CoreMate service; task screenshots go to the selected execution model. Local preview video is not sent frame by frame. +No phone is required during installation. Later Android use requires USB debugging authorization. The official CoreMate account service handles SMS sign-in and automatic registration, currently requiring a mainland-China phone number. Execution follows the current WorkBuddy conversation model. Task screenshots go through WorkBuddy’s model/tool loop; local preview video is not sent frame by frame. ## Requirements @@ -53,6 +63,18 @@ The script entry stores each attempt's log, receipt and guide in `~/.workbuddy/o Automatic discovery recognizes `com.workbuddy.workbuddy`, `com.workbuddy.workbuddy-ai`, and legacy `com.tencent.workbuddy.*` bundles. WorkBuddy (including legacy bundles) takes priority over WorkBuddy AI. If multiple bundles remain at the preferred priority, choose one with `--app /absolute/path/WorkBuddy.app`. Use `--app` to select WorkBuddy AI explicitly even when WorkBuddy is installed. The installer reads the selected application's product metadata to resolve its configuration directory and prints the selected application and directory during preflight. +## International WorkBuddy + +Custom MCP installation does not require a marketplace listing. Select **WorkBuddy AI.app** with `--app` and use `--download-source official`, as shown above. The OpenGUI 0.4.1 installer supports `--app`. + +The installer reads the selected application’s product metadata. Current international builds use `~/.workbuddy-ai` for MCP, Skill and Hook configuration; China builds use their own configuration root, normally `~/.workbuddy`. OpenGUI keeps shared private runtime/account/device state under `~/.workbuddy/opengui`, with separate installation receipts for each configuration root. Do not operate the same phone through both hosts concurrently. + +OpenGUI’s SMS account is separate from WorkBuddy’s Google/GitHub account. OpenGUI currently requires a mainland-China phone number. Chinese/English switching and following WorkBuddy’s display language are being tested and are not yet included in the public source or 0.4.1 package. + +In international WorkBuddy AI, authorize and enable OpenGUI under **Experts · Skills · Connectors → Skills**. Then open **Connectors → Custom connectors**, find `opengui`, click **Trust**, and enable it. The user completes both authorizations. An update may trigger another Trust prompt when the runtime path changes. The [official MCP guide](https://www.workbuddy.ai/docs/workbuddy/From-Beginner-to-Expert-Guide/Function-Description/MCP-Guide) also covers custom servers. + +See [install from GitHub](docs/install-from-github.zh-CN.md) for chat installation, read-only verification and installing an explicitly selected local archive. + ## Manual alternative The existing [OpenGUI installation guide](resources/OpenGUI-安装指南.html) retains the ZIP download and manual launcher. Use it when the user explicitly prefers a manual installation. Its embedded archive is built ahead of time; do not generate another guide during installation. The manual launcher's receipt is in its `installation-result.*` directory. @@ -71,7 +93,8 @@ The connection-check request starts verification; it is not proof that the host If Skill or connector authorization remains incomplete, reopen the authorization guide: -打开 WorkBuddy 的“专家·技能·连接器 → 技能”,找到 OpenGUI,按提示完成授权;之后在连接器中,找到OpenGUI,再次授权。 +- 国内版:在“专家·技能·连接器 → 技能”中授权 OpenGUI;再到“连接器 → 自定义连接器”,为 `opengui` 完成信任并开启开关。 +- International edition: authorize OpenGUI under **Experts · Skills · Connectors → Skills**, then click **Trust** for `opengui` under **Connectors → Custom connectors** and enable it. **注意:技能 和 连接器,都需要授权哦!** @@ -79,7 +102,7 @@ If Skill or connector authorization remains incomplete, reopen the authorization ## MCP missing after installation -The published 0.4.0 installer can write configuration while newer WorkBuddy versions are running, but that does not establish live reload. A local WorkBuddy 5.7.6 comparison found that neither external in-place writing nor atomic replacement updated the custom MCP list. Saving the same configuration through WorkBuddy's built-in MCP editor updated the list without restarting. See the [local verification record](docs/mcp-config-reload-verification.zh-CN.md). +The published 0.4.1 installer can write configuration while newer WorkBuddy versions are running, but that does not establish live reload. A local WorkBuddy 5.7.6 comparison found that neither external in-place writing nor atomic replacement updated the custom MCP list. Saving the same configuration through WorkBuddy's built-in MCP editor updated the list without restarting. See the [local verification record](docs/mcp-config-reload-verification.zh-CN.md). The standard installation flow requires one restart: finish other tasks, fully quit WorkBuddy with Command-Q, reopen it, then continue Skill authorization and MCP trust. If OpenGUI is still missing after that, open **Experts · Skills · Connectors → Connectors → Custom connectors → Add MCP** and check for the `opengui` entry in the expected configuration file. If it is absent, inspect the installer receipt and configuration path. The built-in editor's **Save** action is a verified manual refresh option; it is not the default onboarding step. If configuration is already loaded but a connection fails, inspect the actual error before restarting again. @@ -89,7 +112,7 @@ If the row appears with a first-use trust prompt, the remaining step is the nati After receipt and native tool-call verification, Android setup and task examples are: -把 Android 手机打开 USB 调试,连上电脑并在手机上允许 USB 调试授权。在 WorkBuddy 输入 `/opengui`,再输入需要给手机的执行指令,即可控制手机执行。 +Android 手机开启 USB 调试、连接电脑并允许调试授权,也可使用工作台中的本地 Android 模拟器。在 WorkBuddy 输入 `/opengui`,打开右侧任务首页并登录 OpenGUI,填写目标、确认设备,点击“开始执行”。执行模型跟随当前 WorkBuddy 对话。 [试一试:用小红书发帖](workbuddy://task?action=start&prompt=%E5%B8%AE%E6%88%91%E6%B5%8B%E8%AF%95%E3%80%90%E5%BA%94%E7%94%A8%EF%BC%8F%E9%A1%B5%E9%9D%A2%E3%80%91%E7%9A%84%E3%80%90%E5%8A%9F%E8%83%BD%E6%88%96%E6%93%8D%E4%BD%9C%E6%B5%81%E7%A8%8B%E3%80%91%EF%BC%8C%E9%87%8D%E7%82%B9%E6%A3%80%E6%9F%A5%E3%80%90%E5%85%B3%E6%B3%A8%E7%9A%84%E9%97%AE%E9%A2%98%E3%80%91%E3%80%82%E5%A6%82%E6%9E%9C%E5%8F%91%E7%8E%B0%E5%BC%82%E5%B8%B8%EF%BC%8C%E8%AE%B0%E5%BD%95%E6%93%8D%E4%BD%9C%E6%AD%A5%E9%AA%A4%E5%92%8C%E6%88%AA%E5%9B%BE%EF%BC%8C%E5%BD%93%E5%81%9A%E5%88%B0%E3%80%90%E7%BB%93%E6%9D%9F%E6%9D%A1%E4%BB%B6%E3%80%91%E5%B0%B1%E5%81%9C%E3%80%82) diff --git a/workbuddy-plugin/README.md b/workbuddy-plugin/README.md index ef9d1d3e..91e0f202 100644 --- a/workbuddy-plugin/README.md +++ b/workbuddy-plugin/README.md @@ -1,6 +1,8 @@ -# OpenGUI for WorkBuddy 0.4.0 +# OpenGUI for WorkBuddy 0.4.1 -[中文说明](README.zh-CN.md). macOS public-testing candidate, protocol 8. Marketplace approval and the remaining real-device gates are separate. +[中文说明](README.zh-CN.md). The current public release is [0.4.1, a public-testing prerelease](https://github.com/Core-Mate/OpenGUI/releases/tag/opengui-workbuddy-v0.4.1) for macOS, protocol 8. China and international WorkBuddy use the same OpenGUI package and version. Custom MCP installation does not require marketplace publication; real-device acceptance remains a separate check. + +See the [root installation steps](../README.md#use-opengui-in-workbuddy) for edition-specific commands, or the [GitHub installation flow (Chinese)](docs/install-from-github.zh-CN.md) to install from a WorkBuddy chat. OpenGUI has a separate SMS account that currently requires a mainland-China phone number. OpenGUI opens the device display beside the current chat using WorkBuddy built-in `present_files` with the workbenchUrl and current working directory. A visible decoded H.264 frame (Android) or interval JPEG image (iOS simulator) must reach the backend before the first observation or action. An opening request or screenshot preview is insufficient. Display transport is read-only and local; model observations still use explicit screenshots. @@ -19,15 +21,17 @@ After first display authorization, video failure or page closure does not cancel ## Workbench +**UI language update — not yet released:** Chinese/English switching, following WorkBuddy’s display language, and an improved emulator picker are being tested and are not yet in the public source or 0.4.1 package. The planned default is **Follow WorkBuddy**, with manual **中文 / English** overrides and a browser-language fallback when the host setting is unavailable. Interface labels will change; task descriptions, TODO content, phone content, review drafts and recorded results will retain their original text. + An exhausted blocked run can receive a human-confirmed finite continuation budget in its original workbench: 1–100 additional phone operations and plugin inference calls. The grant starts no work, retains the task ID and earlier result, and rejects stale/duplicate revisions. Copy its continuation prompt into the current chat, recover the same resumeTaskId, and establish fresh original-phone display and observation. This does not extend comment quantity/deadline endpoints, change task scope/model, clear unknown sends, or resume completed/cancelled/stopped tasks. Follow-mode host model calls remain unmeasured. -The light workbench follows the supplied prototype: device controls, a proportionate screen, the existing TODO panel, tool Trace with filtering/details, and a report pane. A bare @opengui mention uses opengui_open_guide without preparing video. A requested task uses open_viewer with its original goal: an available account device preference is reused; otherwise multiple phones or an unavailable preference require graphical single-choice confirmation with manufacturer/model, OS version, serial suffix and connection type. Detection and confirmation never grant phone control; a new decoded frame is required. Waiting host tasks survive a normal final stop and a follow-up prompt, but the page cannot wake an ended host turn. It provides two copyable scenario templates and USB authorization/troubleshooting guidance. Normal MCP tasks bind one Android phone, Android emulator or supported macOS iOS simulator; lower-level legacy display primitives retain multi-device compatibility. Device switching requires a new task. Physical iOS devices and cloud phones are not implemented. Configured models reuse the existing unified admin/backend service; see the account dialog and deployment notes below. +The light workbench follows the supplied prototype: device controls, a proportionate screen, the existing TODO panel, tool Trace with filtering/details, and a report pane. A bare @opengui mention uses opengui_open_guide without preparing video. A requested task uses open_viewer with its original goal: an available account device preference is reused; otherwise multiple phones or an unavailable preference require graphical single-choice confirmation with manufacturer/model, OS version, serial suffix and connection type. Detection and confirmation never grant phone control; a new decoded frame is required. Waiting host tasks survive a normal final stop and a follow-up prompt, but the page cannot wake an ended host turn. It provides two copyable scenario templates and USB authorization/troubleshooting guidance. Normal MCP tasks bind one Android phone, Android emulator or supported macOS iOS simulator; lower-level legacy display primitives retain multi-device compatibility. Device switching requires a new task. Physical iOS devices and cloud phones are not implemented. Execution follows the current WorkBuddy conversation model. The official CoreMate service handles account sign-in; the plugin does not read an online model catalog. Open workbenchUrl to enable takeover, 恢复控制 (resume) and comment decisions. Takeover is the only user pause: no screenshots or model calls until the person resumes control. On Android the person can operate the phone directly on the workbench picture: click/drag for touch, wheel to scroll, right click for Back, and the keyboard including input methods. Input travels over the same scrcpy connection, is accepted only during takeover for the bound device, and stops as soon as control resumes. The separate capability lives in the URL fragment and the current tab session storage; it is never included in HTTP status. The plain url remains read-only. Board controls cannot dispatch arbitrary phone inputs. Pause aborts in-flight work; an already issued mutation may remain unknown. Handback/recheck invalidates previous screenshots and requires a fresh observation before acting. Host Stop hooks yield for human review/takeover instead of forcing autonomous continuation; an explicit stop still cancels. Comment drafts use opengui_review_comment and human approval/edit/skip in the workbench. Input must match the approved draft. opengui_comment_input copies the identified focused comment field without exposing unrelated clipboard text. Different originals require a separate human keep/replace decision; replacement and sending recheck the live input. Identical text is never appended. Native copy requires a fully selectable/copyable Android field (API 24+); unsupported/custom fields or nontext clipboards require handoff. Destination/focus/selection and real platform compatibility still require visual and device acceptance. Sending records intent once; only a latest-image verification via opengui_verify_comment counts as sent. Target/account interpretation and business verification remain the executor model's responsibility. The private local archive blocks previously sent and unresolved submissions for the same signed-in principal and stable platform account/target; unreadable history blocks sending. -Export Markdown, genuine DOCX, PDF, or a ZIP containing the report and archived screenshot evidence. Direct PDF embeds bundled Chinese/emoji fonts and local screenshots; browser Print/Save as PDF remains an additional option. Completion/cancellation automatically archives PDF and Word and returns actual paths, or an explicit export error. Configured-model inference records actual HTTP elapsed time, including network latency. Follow WorkBuddy has no host model timing. Tool inputs are omitted from Trace. Reports and evidence are automatically stored under the private local reports directory; the 32 MB memory cache does not evict archived screenshots. Standalone Markdown references the companion evidence directory. +Export Markdown, genuine DOCX, PDF, or a ZIP containing the report and archived screenshot evidence. Direct PDF embeds bundled Chinese/emoji fonts and local screenshots; browser Print/Save as PDF remains an additional option. Completion/cancellation automatically archives PDF and Word and returns actual paths, or an explicit export error. The current WorkBuddy-following mode has no host model timing; historical reports may retain timing from earlier model configurations. Tool inputs are omitted from Trace. Reports and evidence are automatically stored under the private local reports directory; the 32 MB memory cache does not evict archived screenshots. Standalone Markdown references the companion evidence directory. Execution follows WorkBuddy and waits on saved state events without inspecting the workbench webpage. Keep one owned MCP status wait pending for Start, reviews or human handback; renew timeouts silently. Notifications cannot wake an ended host turn. If the connection recycles, recover the same frozen task with a fresh owned session and screenshot. @@ -43,7 +47,7 @@ The complete steps and download link are in the [root README](../README.md#use-o Use the supplied installer and matching archive with adjacent SHA-256 sidecars: ```sh -bash scripts/install-macos.command --archive /absolute/path/opengui-mcp-0.4.0.tgz +bash scripts/install-macos.command --app "/Applications/WorkBuddy AI.app" --download-source official --archive /absolute/path/opengui-mcp-0.4.1.tgz ``` The installer prepares private Node, verifies the package and scrcpy resources, and only then changes this host's configuration. After installation, restart WorkBuddy once to load the new MCP configuration, review external Hook changes in `/hooks`, and confirm the installed Skill in `/skills`. @@ -54,7 +58,7 @@ WorkBuddy 5.5.3 domestic and overseas configuration discovery, version-aware run Automatic discovery accepts `com.workbuddy.workbuddy`, `com.workbuddy.workbuddy-ai`, and legacy `com.tencent.workbuddy.*` identities. It prefers WorkBuddy (including legacy bundles) over WorkBuddy AI; multiple bundles in the preferred group require `--app`. An explicit `--app` overrides this preference. Configuration paths come from the selected application's product metadata, including `.workbuddy` and `.workbuddy-ai`. -Node and npm default to npmmirror with official fallback for this installation only. Use `--download-source official` to select official sources. Video archives remain separate GitHub downloads; `--video-mirror https://host/archive-directory` optionally selects an identical archive mirror with size/checksum verification and official fallback. See [download sources](INSTALL.md#download-sources); older packages cannot honor the new video option. +The root README’s China command selects npmmirror with official fallback for Node/npm; its international command selects official sources. The public installer defaults to npmmirror when `--download-source` is omitted. This selection applies only to the current installation. Video archives remain separate GitHub downloads; `--video-mirror https://host/archive-directory` optionally selects an identical archive mirror with size/checksum verification and official fallback. See [download sources](INSTALL.md#download-sources); older packages cannot honor the new video option. ## Content review before publishing @@ -100,7 +104,7 @@ The report's 新建任务 button copies `@opengui` for a new WorkBuddy request a The plugin connects to the official CoreMate account service (`https://cm2backend.dmyh.tech`, defined in `src/service-config.ts`) for SMS login and automatic registration through `/api/user-auth/*`. Builds bundle the service URL; `OPENGUI_ACCOUNT_SERVICE_URL` supports another service at build time or runtime. The service receives the phone number, SMS code and account session. Execution only follows the current WorkBuddy conversation model: the plugin does not fetch an online model catalog or send task prompts/screenshots to the account service. Prompts and screenshots are handled by WorkBuddy's normal model/tool loop. Legacy model preferences are cleared while account-scoped device preferences remain. Provider keys are not stored by the plugin. -The companion admin/backend routes must be deployed before login/catalog/inference work. Source integration and fixture tests do not prove production deployment or real-provider/device acceptance. Model/device preferences and task history are scoped to the service and signed-in identity. The device preference stores only an opaque local ID in the private account file, survives restart/logout, and never overrides explicit targets, frozen tasks or recovery. Anonymous device choices are not persisted. Unavailable/unauthorized/busy/incompatible preferences require a human choice; a failed preference save is shown while the successful task binding remains valid. Cross-machine synchronization is not implemented. +SMS sign-in requires the official account service to be reachable. Local fixture tests do not establish production availability or real WorkBuddy/device acceptance. Device preferences and task history are scoped to the service and signed-in identity. The device preference stores only an opaque local ID in the private account file, survives restart/logout, and never overrides explicit targets, frozen tasks or recovery. Anonymous device choices are not persisted. Unavailable/unauthorized/busy/incompatible preferences require a human choice; a failed preference save is shown while the successful task binding remains valid. Cross-machine synchronization is not implemented. Use `opengui_test_case` to define/begin/record/read bounded checks on existing stepIds. A passed/failed verdict requires a known original expectation, executed steps and the latest valid current-task screenshot on that step. Failed checks require the actual observed page and derive a linked defect ID; unknown expectations remain unverified. Dependencies must pass before dependent execution; omitted checks preserve reasons without invented steps. Retest copies a same-account archived case into a new run, preserving original expectations, inputs, steps and stopping conditions, and reports changed conditions without modifying old results/reports or claiming a universal fix. Reports expose each check's actual result and evidence; the runtime records the model's comparison rather than independently interpreting images. diff --git a/workbuddy-plugin/README.zh-CN.md b/workbuddy-plugin/README.zh-CN.md index baad80f8..6005836c 100644 --- a/workbuddy-plugin/README.zh-CN.md +++ b/workbuddy-plugin/README.zh-CN.md @@ -1,10 +1,14 @@ -# OpenGUI for WorkBuddy 0.4.0 候选版 +# OpenGUI for WorkBuddy 0.4.1 公测预发布版 + +国内版和国际版 WorkBuddy 使用同一个 [OpenGUI 0.4.1 安装包](https://github.com/Core-Mate/OpenGUI/releases/tag/opengui-workbuddy-v0.4.1)和版本号,通过自定义 MCP 安装,无需等待市场上架。两版安装命令见[根 README](../README.zh-CN.md#workbuddy-安装),WorkBuddy 聊天安装提示词和授权验证见[从 GitHub 安装](docs/install-from-github.zh-CN.md)。 + +OpenGUI 账号与 WorkBuddy 账号独立,当前 OpenGUI 短信登录仍需中国大陆手机号;国际版 WorkBuddy 的 Google/GitHub 登录不会自动登录 OpenGUI。 新任务默认手机操作/插件内推理预算各 1000 次;可在执行前声明 1–10000 次,旧任务恢复不扩大原上限。2/3 次只用于特定回归测试,不是产品默认值。用户明确约定总操作次数时,在首次观察/动作前向 `opengui_open_session` 传 `executionBudget: {operationLimit?, inferenceLimit?}`(1–10000)。观察和动作共用操作上限;推理上限只统计插件内调用,不能统计宿主模型调用。须核对工具返回的硬上限;目标/结束条件中的部分阿拉伯数字中文/英文总次数句式也会收紧上限,其他表述需用结构化字段声明。初始预算、已用次数随原任务恢复,模型不能调大或重置。工作台和报告展示实际限制;追加仍须原工作台用户授权。 “停在提交前”已增加任务级运行时限制:开会话时传 `stopBeforeSubmit: true`,明确的原生用户提示、目标/结束条件和已开始的检查点也能锁定限制。恢复与归档保留;禁止已标注的提交/发送/发布/付款/删除及 Enter,点击/滑动必须显式声明 `externalSideEffect`,省略则不执行。文本使用剪贴板避免换行模拟 Enter。工作台和报告展示该限制,提交本身应记录为未检查。只有从当前画面确认属于准备或导航的操作才可声明 `none`;该声明仍依赖模型正确识别,不能据此声称任意页面都有独立防提交保证,无法分类时应人工接管。 -本轮支持 macOS,协议版本 8。这是本地候选交付,不代表已公开发布或所有真机验收通过。 +当前公开版支持 macOS,协议版本 8,仍为公测预发布版本;安装成功与真机任务验收分别以实际结果为准。 执行预算耗尽且运行受阻时,原工作台提供用户确认的有限续跑入口:每次追加 1–100 次手机操作及插件内模型调用,保留同一任务 ID、目标、清单和上一轮结论。确认只记录预算;复制继续任务指令到当前聊天后,按原归档任务恢复并重新核对原手机与最后一步。模型不能自行追加,重复或旧版本请求会被拒绝。不会增加评论数量/时限,也不能继续已完成、已取消或按用户停止条件结束的任务;跟随模式的宿主模型调用量仍不可测。 @@ -20,6 +24,8 @@ WorkBuddy 使用内置 present_files,传入 URL 与当前工作目录。国内 ## 手机任务工作台 +**界面语言更新(尚未发布)**:中英文切换、默认跟随 WorkBuddy 语言和模拟器下拉框改进正在测试,尚未合入公开源码或 0.4.1 安装包。计划默认提供“跟随 WorkBuddy”,支持手动选择“中文 / English”,读取不到宿主设置时回退到浏览器语言。语言选项只翻译界面控件,任务描述、TODO 内容、手机内容、审核草稿和已记录结果保留原文。 + 看板按 `docs/preview.html` 原型布局:顶部设备 / 模型选择器(仅提供跟随 WorkBuddy)与接管操作,画面上方环境状态与连接帮助,等比例视频,下方任务步骤、任务报告和「本次运行」双轨时间线(模型 / 设备)及可展开的步骤记录,另有场景示例与连接帮助弹窗。正常 MCP 任务只绑定一台 Android 真机、Android 模拟器或已支持的 macOS iOS 模拟器;旧底层展示接口保留兼容。无设备时保留目标并展示接入指引,不准备视频或摆空手机框;可复用登录账号仍可用的上次选择,否则多台设备或失效偏好通过图形单选确认,无偏好时可选择唯一可用设备。选择器显示厂商型号、系统版本、尾号与连接方式,确认后等待新首帧。等待时保留宿主任务;看板不能自行唤醒结束的宿主回合。裸 mention 使用无目标的 opengui_open_guide。切换设备需先结束原任务。iOS 真机不支持,云手机尚未接入。执行模型固定跟随当前 WorkBuddy 对话,不读取线上模型配置。 通过 workbenchUrl 打开可接管、恢复控制及审核评论;接管即暂停,期间不截图、不调用模型;Android 设备可直接在工作台画面上用鼠标点击、拖动(触摸)、滚轮(滑动)、右键(返回)和键盘(含中文输入法)操作手机,输入经同一 scrcpy 连接发送,只在接管期间、只对当前绑定设备生效,恢复控制后立即失效。普通 url 保持只读。控制能力通过独立 fragment 与当前标签页 sessionStorage 保留,不出现在 HTTP status 中,也不能派发任意手机动作。接管会中止在途工作,已发出的动作可能结果未知;恢复控制或重新检测后旧截图失效,必须重新观察并核对最后一步再继续。宿主 Hook 在接管和等待审核时允许停下来等待用户,明确的停止仍取消任务。 @@ -44,7 +50,7 @@ WorkBuddy 使用内置 present_files,传入 URL 与当前工作目录。国内 核对安装器及归档旁的 SHA-256 文件,结束旧任务、关闭旧展示后运行: ```sh -bash scripts/install-macos.command --archive /绝对路径/opengui-mcp-0.4.0.tgz +bash scripts/install-macos.command --app "/Applications/WorkBuddy.app" --download-source cn --archive /绝对路径/opengui-mcp-0.4.1.tgz ``` 安装器自动准备独立 Node 与 scrcpy 资源,缓存完整时复用;准备失败保留旧配置,并输出恢复步骤。安装完成后重启一次 WorkBuddy,使新 MCP 配置加载生效;外部 Hook 变更在 `/hooks` 中审查应用,并在 `/skills` 中确认 `opengui`。可用 --check 做只读预检,用 --app 指定国内或海外应用。 @@ -53,7 +59,7 @@ bash scripts/install-macos.command --archive /绝对路径/opengui-mcp-0.4.0.tgz 自动发现支持 `com.workbuddy.workbuddy`、`com.workbuddy.workbuddy-ai` 和旧的 `com.tencent.workbuddy.*` 标识。两种应用同时存在时,优先 WorkBuddy(含旧标识),只有 WorkBuddy AI 时选择 AI 版。同一优先级有多个副本时,需要用 `--app` 明确选择;显式指定应用会覆盖默认优先级。配置目录从所选应用的产品信息读取,支持 `.workbuddy` 和 `.workbuddy-ai`。 -Node 与 npm 依赖默认使用 npmmirror,失败时回退官方源;用 `--download-source official` 可切回官方源,选择只影响本次安装。视频组件另从 GitHub 下载,可用 `--video-mirror https://镜像地址/归档目录` 指定相同归档的镜像,保留大小和 SHA-256 校验并支持官方回退。详见[下载源选项](INSTALL.md#download-sources);旧包不支持新的视频镜像选项。 +根 README 的国内版命令使用 npmmirror 下载 Node/npm,失败时回退官方源;国际版命令明确使用官方源。公开安装器省略 `--download-source` 时默认使用 npmmirror,选择只影响本次安装。视频组件另从 GitHub 下载,可用 `--video-mirror https://镜像地址/归档目录` 指定相同归档的镜像,保留大小和 SHA-256 校验并支持官方回退。详见[下载源选项](INSTALL.md#download-sources);旧包不支持新的视频镜像选项。 安装结果分别报告“配置已写入”“宿主已加载”“设备墙可用”,写入配置不是验收通过。安装后在实际宿主选择 OpenGUI Skill,先检查只读设备发现,再验收右侧视频与截图操作。 @@ -77,7 +83,7 @@ macOS 上可运行 `npm run test:host -- /Applications/WorkBuddy.app`,用实 插件默认连接官方 CoreMate 账号服务(`https://cm2backend.dmyh.tech`,定义在 `src/service-config.ts`),通过 `/api/user-auth/*` 完成手机号验证码登录与自动注册。构建时会打包服务地址;可在构建或运行时用 `OPENGUI_ACCOUNT_SERVICE_URL` 覆盖。该服务接收手机号、验证码与登录会话。执行模型只提供「跟随 WorkBuddy」,不读取线上模型目录,也不向账号服务发送任务文字或手机截图;这些内容通过 WorkBuddy 正常的模型与工具链路处理。旧模型偏好会清除,账号关联的设备偏好保留,插件不保存模型供应商密钥。 -插件不再依赖 admin 的插件接入代理;Backend 直接复用现有登录、已发布配置与模型调用接口,没有新增插件专用接口;本地测试通过不能等同于线上可用或真实模型/手机验收。模型/设备偏好与任务历史按服务地址和登录身份隔离。本机私有账号文件保存上次设备的不透明 ID,重启及同账号重新登录后保留;匿名选择不持久化。新任务可复用仍可用的上次设备,上次设备离线/未授权/占用/不兼容时要求人工重新选择,不自动替换成其他手机。明确指定、正在执行及历史恢复的原设备优先;偏好不授予画面或控制许可,保存失败明确提示。目前未提供跨电脑同步。 +短信登录需要能够访问官方账号服务;本地测试通过不能等同于线上服务可用或真实 WorkBuddy/手机验收。设备偏好与任务历史按服务地址和登录身份隔离。本机私有账号文件保存上次设备的不透明 ID,重启及同账号重新登录后保留;匿名选择不持久化。新任务可复用仍可用的上次设备,上次设备离线/未授权/占用/不兼容时要求人工重新选择,不自动替换成其他手机。明确指定、正在执行及历史恢复的原设备优先;偏好不授予画面或控制许可,保存失败明确提示。目前未提供跨电脑同步。 结构化测试使用 `opengui_test_case` 定义、开始、记录和读取检查点;通过/失败要求原预期来源、实际步骤和本步骤的最新有效截图。失败记录实际发生页面、缺陷编号与复现信息;未知预期保持待确认,未执行项保留原因,失败依赖阻止后续依赖执行。`retest` 在新任务中关联同账号归档用例,保留原预期/输入/停止条件及旧报告,并展示条件变化。报告逐项展示结果;这些是模型的视觉判断记录,仍需真实应用验收。 diff --git a/workbuddy-plugin/connector/connector-meta.json b/workbuddy-plugin/connector/connector-meta.json index eb179183..3ef5d22f 100644 --- a/workbuddy-plugin/connector/connector-meta.json +++ b/workbuddy-plugin/connector/connector-meta.json @@ -2,21 +2,19 @@ "name": "OpenGUI", "name_zh": "OpenGUI 手机助手", "name_en": "OpenGUI Android Assistant", - "description": "Let WorkBuddy operate a locally connected Android phone or emulator for GUI testing and reviewed comment operations, with a live workbench for the screen, task plan and run trace.", - "description_zh": "连接手机,在右侧确认任务后开始。支持真实截图操作、App 测试和发布内容审核;执行进度自动更新,结束后生成报告。", - "description_en": "Let WorkBuddy operate a locally connected Android phone or emulator for GUI testing and reviewed comment operations, with a live workbench for the screen, task plan and run trace.", + "description": "Automate your phone for everyday tasks, app testing, content operations, chat and more. Say what you need, and let it take action. After initial setup and authorization, just plug in your phone and go. Leave repetitive tasks to AI, take back your time, and unleash your imagination.", + "description_zh": "自动操作你的手机,处理日常事务、测试 App、运营、聊天……说出你的需求,它来动手。 首次安装并授权后,即插即用。把繁琐操作交给 AI,把时间留给自己,释放你的想象力。", + "description_en": "Automate your phone for everyday tasks, app testing, content operations, chat and more. Say what you need, and let it take action. After initial setup and authorization, just plug in your phone and go. Leave repetitive tasks to AI, take back your time, and unleash your imagination.", "source": "opengui", "type": "mcp", "version": "0.4.0", "minWorkbuddyVersion": "5.5.3", "examples_zh": [ - "@opengui", - "@opengui 打开手机设置,再返回桌面,确认回到桌面后结束。", - "@opengui 帮我测试【应用/页面】的【功能】,输入【测试数据】,预期【结果】,做到【停止位置】就停。" + "/opengui 在小红书App中搜索“美食”,帮我获取前3篇贴子的内容整理给我。", + "/opengui 帮我打开微信App,查看有哪些未读的好友或群消息,告诉我好友/群的名字和未读条数。" ], "examples_en": [ - "@opengui", - "@opengui Open phone Settings, then return to the home screen. Verify the home screen and finish.", - "@opengui Test [feature] in [app/page] using [test data], expect [result], and stop at [endpoint]." + "/opengui Search for “美食” in Xiaohongshu, retrieve the content of the first 3 posts, and organize it for me.", + "/opengui Open WeChat and check which friend or group conversations have unread messages. Tell me each friend/group name and its unread message count." ] } diff --git a/workbuddy-plugin/connector/skills/control/installation.html b/workbuddy-plugin/connector/skills/control/installation.html index 6918577c..ac0f4c29 100644 --- a/workbuddy-plugin/connector/skills/control/installation.html +++ b/workbuddy-plugin/connector/skills/control/installation.html @@ -40,7 +40,7 @@

安装 OpenGUI

- 下载 macOS 安装包 + 下载 macOS 安装包

适用于 Apple 芯片和 Intel Mac · 下载后解压