Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 36 additions & 1 deletion .github/workflows/linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -180,10 +180,45 @@ jobs:
echo "=== AT-SPI2 Python bindings ==="
python3 -c "import gi; gi.require_version('Atspi', '2.0'); from gi.repository import Atspi; print('OK')"
echo "=== AT-SPI2 tree dump (quick test) ==="
python3 linux/atspi-dump.py --surface desktop --max-nodes 5 | python3 -m json.tool | head -20 || echo "::warning::AT-SPI2 tree dump returned no nodes (expected before any app is launched)"
python3 packages/platform-linux/linux/atspi-dump.py --surface desktop --max-nodes 5 | python3 -m json.tool | head -20 || echo "::warning::AT-SPI2 tree dump returned no nodes (expected before any app is launched)"
echo "=== xdotool ==="
xdotool version

# The Linux family is a managed plugin now, so the device lanes serve it from the plugin store
# built by this job. The plugin is installed once here and then answers the visible replies below
# and both Linux gates, which is the evidence that a user's `plugins add` install drives this
# desktop. The store is the one the gate action stages the replay pair into: that step merges its
# selections into the same config, so the daemon a gate starts carries all three packages.
- name: Install the Linux platform plugin
if: steps.impact.outputs.run != 'false'
run: |
set -euo pipefail
# The steps below drive the packed entry point, which refuses to run without the build it
# ships beside it, and `plugins add` is a CLI command.
pnpm build
pnpm --filter @agent-device/linux pack --out "$RUNNER_TEMP/linux-plugin.tgz"
mkdir -p "$RUNNER_TEMP/agent-device-replay-plugins"
export AGENT_DEVICE_HOME="$RUNNER_TEMP/agent-device-replay-plugins"
echo "AGENT_DEVICE_HOME=$AGENT_DEVICE_HOME" >> "$GITHUB_ENV"
node bin/agent-device.mjs plugins add "$RUNNER_TEMP/linux-plugin.tgz" --json
node bin/agent-device.mjs plugins list --json
# The store is read when a daemon starts, so the first Linux command below must not reach a
# daemon that was already running without this entry.
pnpm clean:daemon

- name: Drive the Linux desktop through the installed plugin
if: steps.impact.outputs.run != 'false'
run: |
set -euo pipefail
run() { printf '\n$ agent-device %s\n' "$*"; node bin/agent-device.mjs "$@" --json; }
run devices --platform linux
run doctor --platform linux
run open gnome-calculator --platform linux
run snapshot --platform linux
run press 60 100 --platform linux
run screenshot "$RUNNER_TEMP/plugin-desktop.png" --platform linux
run close gnome-calculator --platform linux

- name: Run Linux replay smoke test
if: steps.impact.outputs.run != 'false'
run: pnpm clean:daemon
Expand Down
2 changes: 1 addition & 1 deletion .jscpd-native.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"path": ["apple", "android", "linux", "packages/host-swift/swift"],
"path": ["apple", "android", "packages/platform-linux/linux", "packages/host-swift/swift"],
"format": ["swift", "objectivec", "kotlin", "java", "python"],
"formatsExts": { "objectivec": ["m", "h"] },
"mode": "weak",
Expand Down
4 changes: 1 addition & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@
"depgraph:test": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/depgraph/model.test.ts scripts/depgraph/entries.test.ts scripts/depgraph/import-graph.test.ts scripts/depgraph/affected.test.ts scripts/depgraph/structure-summary.test.ts tools/depgraph-view/*.test.mjs",
"check:production-exports": "fallow dead-code --config fallow-production-exports.json --production --unused-exports --fail-on-issues",
"check:bundle-owner-files": "node --experimental-strip-types scripts/check-bundle-owner-files.ts",
"check:package": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test packages/maestro/test/package-artifact.test.mjs packages/maestro/test/package-smoke.mjs packages/replay/test/package-smoke.mjs packages/platform-web/test/package-smoke.mjs packages/platform-vega/test/package-smoke.mjs packages/platform-harmonyos/test/package-smoke.mjs && node --experimental-strip-types scripts/check-package.ts",
"check:package": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test packages/maestro/test/package-artifact.test.mjs packages/maestro/test/package-smoke.mjs packages/replay/test/package-smoke.mjs packages/platform-web/test/package-smoke.mjs packages/platform-vega/test/package-smoke.mjs packages/platform-harmonyos/test/package-smoke.mjs packages/platform-linux/test/package-smoke.mjs && node --experimental-strip-types scripts/check-package.ts",
"check:provider-plugin": "node scripts/check-provider-plugin.mjs",
"check:command-docs": "vitest run --project unit-core test/repo-guards/command-doc-coverage.test.ts test/repo-guards/command-doc-generated-blocks.test.ts",
"generate:command-docs": "node --experimental-strip-types scripts/command-docs/generate.ts --write",
Expand Down Expand Up @@ -249,7 +249,6 @@
"android/ime-helper/dist",
"!android/ime-helper/dist/*.idsig",
"!android/ime-helper/README.md",
"linux/atspi-dump.py",
"server.json",
"smithery.yaml",
"README.md",
Expand Down Expand Up @@ -328,7 +327,6 @@
"@agent-device/kernel": "workspace:*",
"@agent-device/platform-android": "workspace:*",
"@agent-device/platform-apple": "workspace:*",
"@agent-device/platform-linux": "workspace:*",
"@agent-device/plugins": "workspace:*",
"@agent-device/provider-limrun": "workspace:*",
"@agent-device/provider-webdriver": "workspace:*",
Expand Down
8 changes: 6 additions & 2 deletions packages/capture-kit/src/snapshot/snapshot-desktop-surface.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,18 @@ export type SnapshotSurfaceLoader = (
signal?: AbortSignal,
) => Promise<SnapshotResult>;

/**
* The surface captures the core still owns. The Linux desktop capture is not one of them: the AT-SPI
* read is that family's own mechanics, so it lives in the Linux plugin and runs over the host that
* plugin was set up with (#4676). What remains here is the macOS surface, whose projector the core
* keeps because the Apple runtime reaches it for a non-app surface.
*/
export type SnapshotSurfaceLoaders = Readonly<{
linux: SnapshotSurfaceLoader;
macos: SnapshotSurfaceLoader;
}>;

export function createSnapshotRuntimeHost(loaders: SnapshotSurfaceLoaders): SnapshotRuntimeHost {
const captureSurface: SnapshotRuntimeHost['captureSurface'] = async (device, options, signal) => {
if (device.platform === 'linux') return await loaders.linux(options, signal);
requireMacOsSurfaceDevice(device);
return await loaders.macos(options, signal);
};
Expand Down
4 changes: 4 additions & 0 deletions packages/contracts/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,10 @@
"types": "./src/logs-runtime-plan.ts",
"default": "./src/logs-runtime-plan.ts"
},
"./linux-tool-provider": {
"types": "./src/linux-tool-provider.ts",
"default": "./src/linux-tool-provider.ts"
},
"./vega-tool-provider": {
"types": "./src/vega-tool-provider.ts",
"default": "./src/vega-tool-provider.ts"
Expand Down
129 changes: 129 additions & 0 deletions packages/contracts/src/linux-tool-provider.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
/**
* The Linux host-tool surface a request scopes (ADR 0019 host execution).
*
* The Linux family drives desktop tools — `xdotool`, `ydotool`, a clipboard tool, a screenshot
* tool, and the AT-SPI helper — through this vocabulary. Root composition passes a resolved
* provider back to the Linux plugin without naming its implementation, exactly as it does for the
* Vega family's CLI adapter, so a request that owns a display decision and a command route hands
* them to the family instead of the family reaching process globals of its own.
*/

import type { RawSnapshotNode } from '@agent-device/kernel/snapshot';
import type { ClickButton } from './click-button.ts';
import type { ScrollDirection } from './scroll-gesture.ts';
import type { SessionSurface } from './session-surface.ts';

/** The Linux family's display-server vocabulary, from one detector. */
export type LinuxDisplayServer = 'wayland' | 'x11';

/** The input tool one provider resolved, tagged with the display decision it resolved for. */
export type LinuxInputTool = 'xdotool' | 'ydotool';

export type LinuxInputSelection = Readonly<{ tool: LinuxInputTool; display: LinuxDisplayServer }>;

/** The AT-SPI surface one Linux capture reads: the desktop or the frontmost app. */
export type LinuxSnapshotSurface = 'desktop' | 'frontmost-app';

/** What the host answered one Linux tool: its streams and its exit code. */
export type LinuxCommandResult = Readonly<{
stdout: string;
stderr: string;
exitCode: number;
}>;

/** How one Linux tool call runs. */
export type LinuxCommandOptions = Readonly<{
allowFailure?: boolean;
timeoutMs?: number;
signal?: AbortSignal;
stdin?: string;
}>;

/** Why a process that outlived its launch ended: a start failure, a non-zero exit, or a signal. */
export type LinuxDetachedExit = Readonly<{
pid: number;
exitCode?: number;
signal?: string;
error?: string;
}>;

/**
* The desktop-provider parts one request may supply. Each part answers one semantic intent of the
* Linux interactor, so a caller that owns a surface — a scripted desktop, a display server it
* drives directly — substitutes it for the family's own tool route without reimplementing the
* interactor. The command primitives stay mandatory: every Linux tool call goes through them.
*/
export type LinuxToolProvider = Readonly<{
/** The display decision every local tool selection in this provider shares (#3794). */
display: Readonly<{ detectDisplayServer(): LinuxDisplayServer }>;
runCommand(
cmd: string,
args: readonly string[],
options?: LinuxCommandOptions,
): Promise<LinuxCommandResult>;
whichCommand(cmd: string): Promise<boolean>;
/**
* Starts a process that outlives the call, with its output discarded, and settles `exited` when
* it ends: with `error` when it could not start, else with its exit code or signal.
*/
launchDetached(
cmd: string,
args: readonly string[],
): Readonly<{ exited: Promise<LinuxDetachedExit> }>;
/** Waits for interaction pacing between synthesized input steps. */
sleep(milliseconds: number): Promise<void>;
desktop: LinuxDesktopProvider;
clipboard?: LinuxClipboardProvider;
screenshot?: LinuxScreenshotProvider;
accessibility?: LinuxAccessibilityProvider;
input?: LinuxInputProvider;
}>;

export type LinuxDesktopProvider = Readonly<{
openTarget(target: string): Promise<void>;
closeApp(app: string): Promise<void>;
}>;

export type LinuxClipboardProvider = Readonly<{
readText(): Promise<string>;
writeText(text: string): Promise<void>;
}>;
Comment thread
thymikee marked this conversation as resolved.
Resolved

export type LinuxScreenshotProvider = Readonly<{
capture(
outPath: string,
options?: Readonly<{ fullscreen?: boolean; stabilize?: boolean; surface?: SessionSurface }>,
): Promise<void>;
}>;

export type LinuxAccessibilityProvider = Readonly<{
captureTree(
surface: LinuxSnapshotSurface,
options?: Readonly<{
maxNodes?: number;
maxDepth?: number;
maxApps?: number;
signal?: AbortSignal;
}>,
): Promise<
Readonly<{
nodes: readonly RawSnapshotNode[];
truncated: boolean;
surface: LinuxSnapshotSurface;
}>
>;
}>;

export type LinuxInputProvider = Readonly<{
click(x: number, y: number, button: ClickButton): Promise<void>;
doubleClick(x: number, y: number): Promise<void>;
longPress(x: number, y: number, durationMs: number): Promise<void>;
drag(x1: number, y1: number, x2: number, y2: number, durationMs: number): Promise<void>;
scroll(
direction: ScrollDirection,
options?: Readonly<{ amount?: number; pixels?: number; durationMs?: number }>,
): Promise<void>;
typeText(text: string, options?: Readonly<{ delayMs?: number }>): Promise<void>;
/** Sends one combination: `combo` is xdotool keysym notation, `scancodes` ydotool pairs. */
key(combo: string, scancodes: readonly string[]): Promise<void>;
}>;
31 changes: 31 additions & 0 deletions packages/contracts/src/platform-plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import type { ManagedWebBackend } from './managed-web-backend.ts';
import type { DoctorCheck } from './doctor.ts';
import type { HostDiagnosticsContext } from './host-diagnostics.ts';
import type { InventoryPlatformModule, PlatformModuleRegistry } from './platform-module.ts';
import type { LinuxToolProvider } from './linux-tool-provider.ts';
import type { PlatformRuntimeModule } from './platform-runtime-operations.ts';
import type { WebProvider } from './web-provider.ts';
import type { VegaToolProvider } from './vega-tool-provider.ts';
Expand Down Expand Up @@ -142,6 +143,35 @@ export type HarmonyPlatformSelection = Readonly<{
get(): HarmonyPlatformRegistration;
}>;

/**
* What the core asks of the Linux plugin: the toolchain probe that answers `doctor --platform
* linux`, and the request scope the family owns. The desktop capture and every interaction the
* family runs go through the registration's runtime and interactor, which use the host the plugin
* was set up with.
*/
export type LinuxPlatformOperations = Readonly<{
toolchainCheck(): Promise<DoctorCheck>;
}>;

export type LinuxPlatformRegistration = Readonly<{
family: 'linux';
inventoryModule: InventoryPlatformModule<'linux'>;
runtimeModule: PlatformRuntimeModule & { readonly family: 'linux' };
createInteractor: PlatformPlugin['createInteractor'];
withRequestProvider<T>(provider: LinuxToolProvider, task: () => Promise<T>): Promise<T>;
operations: LinuxPlatformOperations;
}>;

/**
* How the running process answers for the Linux family, installed or not. `get()` raises the
* selection's own typed outcome when the plugin is absent or failed to load, so `doctor --platform
* linux` names the install command rather than reporting nothing.
*/
export type LinuxPlatformSelection = Readonly<{
active?: LinuxPlatformRegistration;
get(): LinuxPlatformRegistration;
}>;

export type PlatformPluginSet<Plugin extends PlatformPlugin = PlatformPlugin> =
PlatformModuleRegistry<{ family: Platform; plugin: Plugin }>;

Expand All @@ -150,6 +180,7 @@ export type PlatformModuleSelection = Readonly<{
web: WebPlatformSelection;
vega: VegaPlatformSelection;
harmony: HarmonyPlatformSelection;
linux: LinuxPlatformSelection;
inventoryModules: readonly InventoryPlatformModule[];
runtimeModules: readonly PlatformRuntimeModule[];
reportAbsent(family: Platform): AppError;
Expand Down
14 changes: 14 additions & 0 deletions packages/platform-linux/build.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
import { build } from 'tsdown';
import {
assertManagedPlatformPluginFactory,
managedPlatformPluginBuildConfig,
} from '@agent-device/build-config';
import linuxPlatformPlugin from './src/index.ts';

assertManagedPlatformPluginFactory(
new URL('./package.json', import.meta.url),
linuxPlatformPlugin,
'Linux',
);

await build(managedPlatformPluginBuildConfig);
File renamed without changes.
59 changes: 43 additions & 16 deletions packages/platform-linux/package.json
Original file line number Diff line number Diff line change
@@ -1,27 +1,54 @@
{
"name": "@agent-device/platform-linux",
"version": "0.0.0",
"name": "@agent-device/linux",
"version": "0.22.0-dev",
"private": true,
"type": "module",
"description": "Linux-family platform runtime metadata and implementations for agent-device.",
"dependencies": {
"description": "Managed Linux platform plugin for agent-device.",
"exports": {
"./agent-device": {
"types": "./src/index.ts",
"default": "./src/index.ts"
}
},
"devDependencies": {
"@agent-device/build-config": "workspace:*",
"@agent-device/capture-kit": "workspace:*",
"@agent-device/contracts": "workspace:*",
"@agent-device/gestures": "workspace:*",
"@agent-device/host-kit": "workspace:*",
"@agent-device/kernel": "workspace:*"
"@agent-device/kernel": "workspace:*",
"@agent-device/test-kit": "workspace:*",
"agent-device": "workspace:*"
},
"devDependencies": {
"@agent-device/test-kit": "workspace:*"
"engines": {
"node": ">=22.12"
},
"exports": {
".": {
"types": "./src/index.ts",
"default": "./src/index.ts"
"files": [
"dist",
"linux"
],
"license": "MIT",
"agent-device": {
"apiVersion": 3,
"platform": "linux"
},
"scripts": {
"build": "node --experimental-strip-types build.mjs",
"prepack": "pnpm build"
},
"publishConfig": {
"exports": {
"./agent-device": {
"types": "./dist/agent-device.d.mts",
"import": "./dist/agent-device.mjs"
}
},
"./interactor": {
"types": "./src/interactor-facade.ts",
"default": "./src/interactor-facade.ts"
}
}
"access": "public"
},
"repository": {
"type": "git",
"url": "git+https://github.com/callstack/agent-device.git",
"directory": "packages/platform-linux"
},
"homepage": "https://agent-device.dev/docs/commands#linux-desktop-targets"
}
Loading
Loading