From a2ecbaae7f16d23f95b281d522a84245a9237ff2 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 19:45:11 +0530 Subject: [PATCH 01/57] refactor(provider-webdriver): share hub upload and app-reference helpers BrowserStack's app upload, install adapter, --provider-app resolution, session-details URL artifacts, and orientation check were written for one vendor. A second hosted Appium hub needs the same mechanics with a different form field, reference scheme, and response shape, so move them into shared helpers and have BrowserStack use them: - webdriver-utils.ts: postHubAppUpload, createHubUploadApp, resolveHubAppReference, appFileUploadForm, asRecord, readProviderJsonBody, and requireProviderDeviceOrientation. - artifact-results.ts: urlArtifactFromDetails. - browserstack.ts: resolveBrowserStackAppReference, moved out of provider-definitions.ts. Two small BrowserStack behaviour changes come with the shared code: - An upload response that is not JSON (a gateway error page, an empty body) now fails with a typed COMMAND_FAILED that carries the HTTP status, instead of a raw JSON SyntaxError. - The http(s) scheme of a --provider-app URL is matched case-insensitively, so HTTPS://... is passed through to the hub rather than being treated as a local path. Co-Authored-By: Claude Opus 5.5 --- .../src/artifact-results.ts | 14 ++ .../src/browserstack-device-features.ts | 16 +- .../src/browserstack.test.ts | 57 +++++++- .../provider-webdriver/src/browserstack.ts | 101 ++++++------- .../src/provider-definitions.ts | 64 ++------ .../src/webdriver-utils.test.ts | 121 +++++++++++++++- .../provider-webdriver/src/webdriver-utils.ts | 137 +++++++++++++++++- 7 files changed, 386 insertions(+), 124 deletions(-) diff --git a/packages/provider-webdriver/src/artifact-results.ts b/packages/provider-webdriver/src/artifact-results.ts index bf02c7c77e..de29b4ab3a 100644 --- a/packages/provider-webdriver/src/artifact-results.ts +++ b/packages/provider-webdriver/src/artifact-results.ts @@ -28,3 +28,17 @@ export function unavailableCloudArtifactsResult(options: { message: options.error instanceof Error ? options.error.message : String(options.error), }; } + +/** A ready URL artifact read off a provider's session-details record, or nothing when the field is absent. */ +export function urlArtifactFromDetails( + provider: string, + providerSessionId: string, + details: Record, + field: string, + kind: CloudArtifact['kind'], + name: string, +): CloudArtifact | undefined { + const url = details[field]; + if (typeof url !== 'string' || url.length === 0) return undefined; + return { provider, providerSessionId, kind, name, url, availability: 'ready' }; +} diff --git a/packages/provider-webdriver/src/browserstack-device-features.ts b/packages/provider-webdriver/src/browserstack-device-features.ts index 38669957ac..3ea041add7 100644 --- a/packages/provider-webdriver/src/browserstack-device-features.ts +++ b/packages/provider-webdriver/src/browserstack-device-features.ts @@ -3,6 +3,7 @@ import { type CloudProviderProfileFields, } from '@agent-device/contracts/remote'; import { AppError } from '@agent-device/kernel/errors'; +import { requireProviderDeviceOrientation } from './webdriver-utils.ts'; import type { CloudWebDriverPlatform } from './runtime.ts'; /** @@ -199,26 +200,13 @@ function assignStringField( value: string, ): void { if (spec.field === 'providerDeviceOrientation') { - fields.providerDeviceOrientation = requireDeviceOrientation(spec, value); + fields.providerDeviceOrientation = requireProviderDeviceOrientation(spec, value); return; } if (spec.field === 'providerNoResignApp') return; fields[spec.field] = value; } -function requireDeviceOrientation( - spec: BrowserStackDeviceFeatureSpec, - value: string, -): (typeof PROVIDER_DEVICE_ORIENTATIONS)[number] { - const match = PROVIDER_DEVICE_ORIENTATIONS.find((orientation) => orientation === value); - if (match) return match; - throw new AppError('INVALID_ARGS', `Invalid ${spec.flag} value: ${value}.`, { - hint: `Use ${PROVIDER_DEVICE_ORIENTATIONS.join('|')}.`, - flag: spec.flag, - capability: spec.capability, - }); -} - function requireSupportedPlatform( spec: BrowserStackDeviceFeatureSpec, platform: CloudWebDriverPlatform, diff --git a/packages/provider-webdriver/src/browserstack.test.ts b/packages/provider-webdriver/src/browserstack.test.ts index 125bc8ea8c..61cf7bdbb6 100644 --- a/packages/provider-webdriver/src/browserstack.test.ts +++ b/packages/provider-webdriver/src/browserstack.test.ts @@ -3,7 +3,8 @@ import { promises as fs } from 'node:fs'; import path from 'node:path'; import { afterEach, test } from 'vitest'; -import { uploadBrowserStackApp } from './browserstack.ts'; +import { AppError } from '@agent-device/kernel/errors'; +import { resolveBrowserStackAppReference, uploadBrowserStackApp } from './browserstack.ts'; import { mkdtempForTest } from './tmp-dir.fixtures.ts'; const realFetch = globalThis.fetch; @@ -46,3 +47,57 @@ test('BrowserStack upload aborts while the provider request is in flight', async await fs.rm(tempDir, { recursive: true, force: true }); } }); + +const upload = { clientVersion: '0.0.0-test', username: 'user', accessKey: 'key' }; + +test('BrowserStack upload sends the file field and fails typed on a gateway error page', async () => { + const tempDir = await mkdtempForTest('agent-device-browserstack-upload-error-'); + const appPath = path.join(tempDir, 'App.apk'); + try { + await fs.writeFile(appPath, 'placeholder'); + globalThis.fetch = async (_input, init) => { + assert.ok(init?.body instanceof FormData); + assert.ok(init.body.get('file') instanceof Blob); + return new Response('502 Bad Gateway', { status: 502 }); + }; + await assert.rejects(uploadBrowserStackApp(appPath, upload), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.message, 'BrowserStack app upload failed.'); + assert.equal(error.details?.status, 502); + return true; + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + +test('BrowserStack passes bs:// ids and URLs to the hub and uploads only local paths', async () => { + const tempDir = await mkdtempForTest('agent-device-browserstack-resolve-'); + try { + await fs.writeFile(path.join(tempDir, 'App.apk'), 'placeholder'); + const fetched: string[] = []; + globalThis.fetch = async (input) => { + fetched.push(String(input)); + return new Response(JSON.stringify({ app_url: 'bs://uploaded' }), { status: 200 }); + }; + const resolve = async (app: string) => + await resolveBrowserStackAppReference(app, { ...upload, cwd: tempDir }); + + assert.equal(await resolve('bs://preuploaded'), 'bs://preuploaded'); + assert.equal(await resolve('https://builds.example/App.apk'), 'https://builds.example/App.apk'); + assert.equal(fetched.length, 0); + assert.equal(await resolve('App.apk'), 'bs://uploaded'); + assert.deepEqual(fetched, ['https://api-cloud.browserstack.com/app-automate/upload']); + await assert.rejects(resolve('missing.apk'), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal( + error.message, + 'BrowserStack --provider-app must be a bs:// app id, URL, or existing local app path.', + ); + return true; + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); diff --git a/packages/provider-webdriver/src/browserstack.ts b/packages/provider-webdriver/src/browserstack.ts index 6c8d10fd4b..74811f78e6 100644 --- a/packages/provider-webdriver/src/browserstack.ts +++ b/packages/provider-webdriver/src/browserstack.ts @@ -1,12 +1,18 @@ -import fs from 'node:fs/promises'; -import path from 'node:path'; import type { CloudArtifact, CloudArtifactsResult } from '@agent-device/contracts/observability'; import type { CloudWebDriverCapabilityOverrides } from './capabilities.ts'; import type { CloudWebDriverUploadApp } from './runtime.ts'; import { AppError } from '@agent-device/kernel/errors'; import { agentDeviceRequestHeaders } from './request-headers.ts'; -import { cloudArtifactsReadyOrPending } from './artifact-results.ts'; -import { basicAuthHeader, trimTrailingSlash } from './webdriver-utils.ts'; +import { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; +import { + appFileUploadForm, + asRecord, + basicAuthHeader, + createHubUploadApp, + postHubAppUpload, + resolveHubAppReference, + trimTrailingSlash, +} from './webdriver-utils.ts'; export const BROWSERSTACK_APP_AUTOMATE_ENDPOINT = 'https://hub-cloud.browserstack.com/wd/hub/'; export const BROWSERSTACK_APP_UPLOAD_ENDPOINT = @@ -76,41 +82,41 @@ export async function uploadBrowserStackApp( signal?: AbortSignal, ): Promise { signal?.throwIfAborted(); - const file = await fs.readFile(appPath); - const form = new FormData(); - form.set('file', new Blob([file]), path.basename(appPath)); - const response = await fetch(options.endpoint ?? BROWSERSTACK_APP_UPLOAD_ENDPOINT, { - method: 'POST', - headers: { - ...agentDeviceRequestHeaders(options.clientVersion), - Authorization: basicAuthHeader(options), + return await postHubAppUpload( + await appFileUploadForm(appPath, 'file'), + { + service: 'BrowserStack', + endpoint: options.endpoint ?? BROWSERSTACK_APP_UPLOAD_ENDPOINT, + clientVersion: options.clientVersion, + auth: options, + readAppReference: readBrowserStackAppUrl, }, - body: form, signal, - }); - const json = (await response.json()) as unknown; - const appUrl = readBrowserStackAppUrl(json); - if (!response.ok || !appUrl) { - throw new AppError('COMMAND_FAILED', 'BrowserStack app upload failed.', { - status: response.status, - response: json, - }); - } - return appUrl; + ); } export function createBrowserStackUploadApp( options: Required, ): CloudWebDriverUploadApp { - return async ({ appPath, options: installOptions, signal }) => { - const appReference = await uploadBrowserStackApp(appPath, options, signal); - return { - appReference, - bundleId: installOptions?.appIdentifierHint, - packageName: installOptions?.packageNameHint, - launchTarget: installOptions?.appIdentifierHint ?? installOptions?.packageNameHint, - }; - }; + return createHubUploadApp( + async (appPath, signal) => await uploadBrowserStackApp(appPath, options, signal), + ); +} + +/** The hub fetches a public URL itself, so only a local path is uploaded. */ +export async function resolveBrowserStackAppReference( + app: string, + options: BrowserStackUploadOptions & { cwd?: string; signal?: AbortSignal }, +): Promise { + return await resolveHubAppReference({ + service: 'BrowserStack', + app, + cwd: options.cwd, + referenceScheme: 'bs://', + referenceLabel: 'a bs:// app id', + uploadFile: async (appPath, signal) => await uploadBrowserStackApp(appPath, options, signal), + signal: options.signal, + }); } /** @@ -138,17 +144,11 @@ export function buildBrowserStackCapabilities( buildName: options.buildName, sessionName: options.sessionName, ...(options.deviceFeatures ?? {}), - ...asRecord(configuredBstackOptions), + ...(asRecord(configuredBstackOptions) ?? {}), }, }; } -function asRecord(value: unknown): Record { - return value && typeof value === 'object' && !Array.isArray(value) - ? (value as Record) - : {}; -} - async function fetchBrowserStackSessionDetails( sessionId: string, options: BrowserStackSessionDetailsOptions, @@ -179,7 +179,7 @@ function mapBrowserStackArtifacts( details: Record, ): CloudArtifact[] { return [ - browserStackUrlArtifact( + urlArtifactFromDetails( provider, providerSessionId, details, @@ -187,7 +187,7 @@ function mapBrowserStackArtifacts( 'video', 'Session video', ), - browserStackUrlArtifact( + urlArtifactFromDetails( provider, providerSessionId, details, @@ -195,7 +195,7 @@ function mapBrowserStackArtifacts( 'appium-log', 'Appium logs', ), - browserStackUrlArtifact( + urlArtifactFromDetails( provider, providerSessionId, details, @@ -203,7 +203,7 @@ function mapBrowserStackArtifacts( 'device-log', 'Device logs', ), - browserStackUrlArtifact( + urlArtifactFromDetails( provider, providerSessionId, details, @@ -211,7 +211,7 @@ function mapBrowserStackArtifacts( 'provider-session', 'BrowserStack dashboard', ), - browserStackUrlArtifact( + urlArtifactFromDetails( provider, providerSessionId, details, @@ -222,19 +222,6 @@ function mapBrowserStackArtifacts( ].filter((artifact): artifact is CloudArtifact => artifact !== undefined); } -function browserStackUrlArtifact( - provider: string, - providerSessionId: string, - details: Record, - field: string, - kind: CloudArtifact['kind'], - name: string, -): CloudArtifact | undefined { - const url = details[field]; - if (typeof url !== 'string' || url.length === 0) return undefined; - return { provider, providerSessionId, kind, name, url, availability: 'ready' }; -} - function readBrowserStackAppUrl(value: unknown): string | undefined { if (!value || typeof value !== 'object') return undefined; const appUrl = (value as { app_url?: unknown }).app_url; diff --git a/packages/provider-webdriver/src/provider-definitions.ts b/packages/provider-webdriver/src/provider-definitions.ts index b2de85b689..713fd7d8ce 100644 --- a/packages/provider-webdriver/src/provider-definitions.ts +++ b/packages/provider-webdriver/src/provider-definitions.ts @@ -1,5 +1,3 @@ -import fs from 'node:fs'; -import path from 'node:path'; import type { CloudArtifactsResult } from '@agent-device/contracts/observability'; import type { LeaseLifecycleContext } from '@agent-device/contracts/device'; import { AppError } from '@agent-device/kernel/errors'; @@ -17,7 +15,7 @@ import { buildBrowserStackCapabilities, createBrowserStackUploadApp, listBrowserStackCloudArtifacts, - uploadBrowserStackApp, + resolveBrowserStackAppReference, } from './browserstack.ts'; import { buildBrowserStackDeviceFeatureCapabilities, @@ -108,22 +106,24 @@ export function createCloudWebDriverProviderDefinitions( 'providerOsVersion', 'BrowserStack requires --provider-os-version .', ); - const app = await resolveBrowserStackAppReference({ - clientVersion: dependencies.clientVersion, - app: requireFlag( + const app = await resolveBrowserStackAppReference( + requireFlag( request, 'providerApp', 'BrowserStack requires --provider-app .', ), - cwd: request.cwd, - username, - accessKey, - uploadEndpoint: env.BROWSERSTACK_APP_UPLOAD_ENDPOINT, - // A local IPA/APK upload can run long (130 MB is routine); an - // upload is not a billed resource, so the request's cancellation - // may simply abort it — unlike the session creation that follows. - signal: request.signal, - }); + { + clientVersion: dependencies.clientVersion, + username, + accessKey, + endpoint: env.BROWSERSTACK_APP_UPLOAD_ENDPOINT, + cwd: request.cwd, + // A local IPA/APK upload can run long (130 MB is routine); an + // upload is not a billed resource, so the request's cancellation + // may simply abort it — unlike the session creation that follows. + signal: request.signal, + }, + ); return { ...base, platform, @@ -249,40 +249,6 @@ export function createCloudWebDriverProviderDefinitions( ]; } -async function resolveBrowserStackAppReference(options: { - clientVersion: string; - app: string; - cwd?: string; - username: string; - accessKey: string; - uploadEndpoint?: string; - signal?: AbortSignal; -}): Promise { - if (isProviderAppReference(options.app)) return options.app; - const appPath = path.resolve(options.cwd ?? process.cwd(), options.app); - if (!fs.existsSync(appPath)) { - throw new AppError( - 'INVALID_ARGS', - 'BrowserStack --provider-app must be a bs:// app id, URL, or existing local app path.', - { providerApp: options.app }, - ); - } - return await uploadBrowserStackApp( - appPath, - { - clientVersion: options.clientVersion, - username: options.username, - accessKey: options.accessKey, - endpoint: options.uploadEndpoint, - }, - options.signal, - ); -} - -function isProviderAppReference(value: string): boolean { - return value.startsWith('bs://') || /^https?:\/\//.test(value); -} - function requireRequest( req: LeaseLifecycleContext | undefined, providerLabel: string, diff --git a/packages/provider-webdriver/src/webdriver-utils.test.ts b/packages/provider-webdriver/src/webdriver-utils.test.ts index 58d326c241..122aafaf80 100644 --- a/packages/provider-webdriver/src/webdriver-utils.test.ts +++ b/packages/provider-webdriver/src/webdriver-utils.test.ts @@ -1,6 +1,23 @@ import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { trimLeadingSlash, trimTrailingSlash } from './webdriver-utils.ts'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { afterEach, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { + asRecord, + createHubUploadApp, + postHubAppUpload, + resolveHubAppReference, + trimLeadingSlash, + trimTrailingSlash, +} from './webdriver-utils.ts'; +import { mkdtempForTest } from './tmp-dir.fixtures.ts'; + +const realFetch = globalThis.fetch; + +afterEach(() => { + globalThis.fetch = realFetch; +}); test('slash trimming utilities handle slash-heavy strings without regular expressions', () => { const slashRun = '/'.repeat(10_000); @@ -15,3 +32,103 @@ test('slash trimming utilities handle slash-heavy strings without regular expres assert.equal(trimLeadingSlash(slashRun), ''); assert.equal(trimTrailingSlash(slashRun), ''); }); + +test('asRecord admits plain objects only', () => { + assert.deepEqual(asRecord({ a: 1 }), { a: 1 }); + assert.equal(asRecord([]), undefined); + assert.equal(asRecord(null), undefined); + assert.equal(asRecord('x'), undefined); +}); + +const hub = { + service: 'Hub', + endpoint: 'https://upload.example.test/app', + clientVersion: '0.0.0-test', + auth: { username: 'user', accessKey: 'key' }, + readAppReference: (body: unknown) => asRecord(body)?.ref as string | undefined, +}; + +test('the hub upload helper posts with credentials and returns the vendor reference', async () => { + const form = new FormData(); + globalThis.fetch = async (input, init) => { + assert.equal(String(input), hub.endpoint); + assert.equal(init?.method, 'POST'); + assert.equal(init?.body, form); + const headers = init?.headers as Record; + assert.equal(headers.Authorization, `Basic ${Buffer.from('user:key').toString('base64')}`); + assert.equal(headers['x-agent-device-version'], '0.0.0-test'); + return new Response(JSON.stringify({ ref: 'hub://APP1' }), { status: 200 }); + }; + assert.equal(await postHubAppUpload(form, hub), 'hub://APP1'); +}); + +test('the hub upload helper fails typed with the status on an error page or a missing reference', async () => { + for (const response of [ + new Response('502 Bad Gateway', { status: 502 }), + new Response(JSON.stringify({ message: 'ok' }), { status: 200 }), + ]) { + globalThis.fetch = async () => response; + await assert.rejects(postHubAppUpload(new FormData(), hub), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.message, 'Hub app upload failed.'); + assert.equal(error.details?.status, response.status); + return true; + }); + } +}); + +test('the hub install adapter uploads the build and launches the hinted app', async () => { + const upload = vi.fn(async () => 'hub://APP2'); + const signal = new AbortController().signal; + const result = await createHubUploadApp(upload)({ + appPath: '/builds/App.ipa', + options: { appIdentifierHint: 'com.example.app' }, + signal, + }); + assert.deepEqual(upload.mock.calls, [['/builds/App.ipa', signal]]); + assert.deepEqual(result, { + appReference: 'hub://APP2', + bundleId: 'com.example.app', + packageName: undefined, + launchTarget: 'com.example.app', + }); +}); + +test('the hub app resolver passes references through, uploads local files, and routes URLs per hub', async () => { + const tempDir = await mkdtempForTest('agent-device-hub-resolve-'); + try { + await fs.writeFile(path.join(tempDir, 'App.apk'), 'placeholder'); + const uploadFile = vi.fn(async (appPath: string) => `hub://${path.basename(appPath)}`); + const resolve = (app: string, uploadUrl?: (url: string) => Promise) => + resolveHubAppReference({ + service: 'Hub', + app, + cwd: tempDir, + referenceScheme: 'hub://', + referenceLabel: 'a hub:// app id', + uploadFile, + uploadUrl, + }); + + assert.equal(await resolve('hub://APP3'), 'hub://APP3'); + assert.equal(await resolve('https://builds.example/App.apk'), 'https://builds.example/App.apk'); + assert.equal( + await resolve('https://builds.example/App.apk', async (url) => `fetched:${url}`), + 'fetched:https://builds.example/App.apk', + ); + assert.equal(await resolve('App.apk'), 'hub://App.apk'); + assert.deepEqual(uploadFile.mock.calls, [[path.join(tempDir, 'App.apk'), undefined]]); + await assert.rejects(resolve('missing.apk'), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'INVALID_ARGS'); + assert.equal( + error.message, + 'Hub --provider-app must be a hub:// app id, URL, or existing local app path.', + ); + return true; + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); diff --git a/packages/provider-webdriver/src/webdriver-utils.ts b/packages/provider-webdriver/src/webdriver-utils.ts index 5af69a9f4c..a6e833c8c9 100644 --- a/packages/provider-webdriver/src/webdriver-utils.ts +++ b/packages/provider-webdriver/src/webdriver-utils.ts @@ -1,5 +1,17 @@ -import type { DeviceLease } from '@agent-device/contracts/device'; +import fs from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import type { + DeviceLease, + ProviderDeviceInstallOptions, + ProviderDeviceInstallResult, +} from '@agent-device/contracts/device'; +import { + PROVIDER_DEVICE_ORIENTATIONS, + type ProviderDeviceOrientation, +} from '@agent-device/contracts/remote'; import { AppError, errorMessage } from '@agent-device/kernel/errors'; +import { agentDeviceRequestHeaders } from './request-headers.ts'; export type LeaseValue = T | ((lease: DeviceLease) => T); @@ -50,3 +62,126 @@ export function withTrailingSlash(url: URL): URL { copy.pathname = `${copy.pathname}/`; return copy; } + +export function asRecord(value: unknown): Record | undefined { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +type HubCredentials = { username: string; accessKey: string }; + +/** A multipart form carrying the local app file under the hub's field name. */ +export async function appFileUploadForm(appPath: string, fileField: string): Promise { + const form = new FormData(); + form.set(fileField, new Blob([await readFile(appPath)]), path.basename(appPath)); + return form; +} + +/** + * POSTs an app upload to a hosted hub and returns the hub's app reference. A non-2xx answer, a + * body that is not JSON, or one without a reference is `COMMAND_FAILED` with the HTTP status. + */ +export async function postHubAppUpload( + form: FormData, + options: { + service: string; + endpoint: string | URL; + clientVersion: string; + auth: HubCredentials; + readAppReference: (body: unknown) => string | undefined; + }, + signal?: AbortSignal, +): Promise { + const response = await fetch(options.endpoint, { + method: 'POST', + headers: { + ...agentDeviceRequestHeaders(options.clientVersion), + Authorization: basicAuthHeader(options.auth), + }, + body: form, + signal, + }); + const json = await readProviderJsonBody(response); + const appReference = options.readAppReference(json); + if (!response.ok || !appReference) { + throw new AppError('COMMAND_FAILED', `${options.service} app upload failed.`, { + status: response.status, + response: json, + }); + } + return appReference; +} + +/** The `install` adapter of a hosted hub: upload the local build, then launch the hinted app. */ +export function createHubUploadApp( + upload: (appPath: string, signal?: AbortSignal) => Promise, +): (params: { + appPath: string; + options?: ProviderDeviceInstallOptions; + signal?: AbortSignal; +}) => Promise { + return async ({ appPath, options, signal }) => ({ + appReference: await upload(appPath, signal), + bundleId: options?.appIdentifierHint, + packageName: options?.packageNameHint, + launchTarget: options?.appIdentifierHint ?? options?.packageNameHint, + }); +} + +/** + * Turns `--provider-app` into a reference the hub accepts: its own reference scheme passes + * through, a public URL passes through unless the hub only takes its own references (then + * `uploadUrl` has the hub fetch it), and anything else must be a local file to upload. + */ +export async function resolveHubAppReference(options: { + service: string; + app: string; + cwd?: string; + referenceScheme: string; + /** How the scheme reads in the error message, e.g. `a bs:// app id`. */ + referenceLabel: string; + uploadFile: (appPath: string, signal?: AbortSignal) => Promise; + uploadUrl?: (url: string, signal?: AbortSignal) => Promise; + signal?: AbortSignal; +}): Promise { + const { app } = options; + if (app.startsWith(options.referenceScheme)) return app; + if (/^https?:\/\//i.test(app)) { + return options.uploadUrl ? await options.uploadUrl(app, options.signal) : app; + } + const appPath = path.resolve(options.cwd ?? process.cwd(), app); + if (!fs.existsSync(appPath)) { + throw new AppError( + 'INVALID_ARGS', + `${options.service} --provider-app must be ${options.referenceLabel}, URL, or existing local app path.`, + { providerApp: app }, + ); + } + return await options.uploadFile(appPath, options.signal); +} + +/** A provider response body parsed as JSON, or `undefined` when it is empty or not JSON (a gateway error page). */ +async function readProviderJsonBody(response: Response): Promise { + const text = await response.text(); + if (text.length === 0) return undefined; + try { + return JSON.parse(text) as unknown; + } catch { + return undefined; + } +} + +/** Validates a device-orientation flag against the shared enum before it reaches a hub that would ignore it. */ +export function requireProviderDeviceOrientation( + spec: { flag: string; capability: string }, + value: string, +): ProviderDeviceOrientation { + const match = PROVIDER_DEVICE_ORIENTATIONS.find((orientation) => orientation === value); + if (match) return match; + throw new AppError('INVALID_ARGS', `Invalid ${spec.flag} value: ${value}.`, { + hint: `Use ${PROVIDER_DEVICE_ORIENTATIONS.join('|')}.`, + flag: spec.flag, + capability: spec.capability, + }); +} From 364bfe7d596bb77002e4edc34faf266ecc3cb18f Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 19:45:11 +0530 Subject: [PATCH 02/57] fix(provider-webdriver): bound and type session-details lookups The BrowserStack session-details lookup behind `artifacts` had no deadline, so a stalled API call could hang the command indefinitely. A transport failure or a body that was not JSON surfaced as an untyped fetch or SyntaxError, and a JSON array passed the object check and was read as session details. Add fetchProviderSessionDetails to webdriver-utils.ts and use it for BrowserStack. It sends basic auth with a 15 second deadline and reports every failure as COMMAND_FAILED: a timeout or network error with a retry hint and the original error as its cause, and a non-2xx answer or a body that is not a JSON object with the HTTP status and the parsed response. Connection verification gets the same treatment through fetchProviderVerificationJson, which BrowserStack now uses in place of its private fetch. Behaviour is unchanged: 401/403 is UNAUTHORIZED with a credential hint, any other HTTP failure points at the provider's service status, and a transport failure points at network access. A new test pins the two non-credential hints. sameOsVersion moves alongside it. Co-Authored-By: Claude Opus 5.5 --- .../browserstack-connection-verification.ts | 54 ++------- .../src/browserstack.test.ts | 36 +++++- .../provider-webdriver/src/browserstack.ts | 23 +--- .../src/connection-verification.test.ts | 29 +++++ .../provider-webdriver/src/webdriver-utils.ts | 103 ++++++++++++++++++ 5 files changed, 183 insertions(+), 62 deletions(-) diff --git a/packages/provider-webdriver/src/browserstack-connection-verification.ts b/packages/provider-webdriver/src/browserstack-connection-verification.ts index efdfb4e3c8..56d36366cc 100644 --- a/packages/provider-webdriver/src/browserstack-connection-verification.ts +++ b/packages/provider-webdriver/src/browserstack-connection-verification.ts @@ -1,7 +1,6 @@ import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; -import { agentDeviceRequestHeaders } from './request-headers.ts'; -import { basicAuthHeader } from './webdriver-utils.ts'; +import { asRecord, fetchProviderVerificationJson, sameOsVersion } from './webdriver-utils.ts'; import type { CloudWebDriverConnectionVerification, CloudWebDriverConnectionVerificationOptions, @@ -105,42 +104,15 @@ async function fetchBrowserStackJson( auth: { username: string; accessKey: string }, clientVersion: string, ): Promise { - try { - const response = await fetch(endpoint, { - headers: { - ...agentDeviceRequestHeaders(clientVersion), - Authorization: basicAuthHeader(auth), - }, - signal: AbortSignal.timeout(15_000), - }); - if (!response.ok) { - const unauthorized = response.status === 401 || response.status === 403; - throw new AppError( - unauthorized ? 'UNAUTHORIZED' : 'COMMAND_FAILED', - 'BrowserStack rejected connection verification.', - { - status: response.status, - hint: unauthorized - ? 'Check BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY.' - : 'Retry connect or check the BrowserStack service status.', - }, - ); - } - return (await response.json()) as unknown; - } catch (error) { - if (error instanceof AppError) throw error; - throw new AppError( - 'COMMAND_FAILED', - 'BrowserStack connection verification failed.', - { hint: 'Check network access to api-cloud.browserstack.com and retry connect.' }, - error, - ); - } -} - -function sameOsVersion(left: string, right: string): boolean { - const normalize = (value: string) => value.replace(/(?:\.0)+$/, ''); - return normalize(left) === normalize(right); + return await fetchProviderVerificationJson(endpoint, { + clientVersion, + auth, + hints: { + service: 'BrowserStack', + unauthorizedHint: 'Check BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY.', + networkHint: 'Check network access to api-cloud.browserstack.com and retry connect.', + }, + }); } function readBrowserStackDevices( @@ -184,9 +156,3 @@ function readBrowserStackApps( ]; }); } - -function asRecord(value: unknown): Record | undefined { - return value && typeof value === 'object' && !Array.isArray(value) - ? (value as Record) - : undefined; -} diff --git a/packages/provider-webdriver/src/browserstack.test.ts b/packages/provider-webdriver/src/browserstack.test.ts index 61cf7bdbb6..1d8ca76b1a 100644 --- a/packages/provider-webdriver/src/browserstack.test.ts +++ b/packages/provider-webdriver/src/browserstack.test.ts @@ -4,7 +4,11 @@ import { promises as fs } from 'node:fs'; import path from 'node:path'; import { afterEach, test } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; -import { resolveBrowserStackAppReference, uploadBrowserStackApp } from './browserstack.ts'; +import { + listBrowserStackCloudArtifacts, + resolveBrowserStackAppReference, + uploadBrowserStackApp, +} from './browserstack.ts'; import { mkdtempForTest } from './tmp-dir.fixtures.ts'; const realFetch = globalThis.fetch; @@ -101,3 +105,33 @@ test('BrowserStack passes bs:// ids and URLs to the hub and uploads only local p await fs.rm(tempDir, { recursive: true, force: true }); } }); + +test('BrowserStack session details lookup has a deadline and fails typed', async () => { + const lookup = async () => + await listBrowserStackCloudArtifacts('browserstack', 'SESSION1', upload); + const timeout = new DOMException('The operation was aborted due to timeout', 'TimeoutError'); + const transportFailures: unknown[] = [timeout, new TypeError('fetch failed')]; + for (const failure of transportFailures) { + globalThis.fetch = async (_input, init) => { + assert.ok(init?.signal instanceof AbortSignal); + throw failure; + }; + await assert.rejects(lookup(), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.message, 'BrowserStack session details lookup failed.'); + assert.equal(error.cause, failure); + return true; + }); + } + + for (const body of ['gateway', '[]']) { + globalThis.fetch = async () => new Response(body, { status: 200 }); + await assert.rejects(lookup(), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.details?.status, 200); + return true; + }); + } +}); diff --git a/packages/provider-webdriver/src/browserstack.ts b/packages/provider-webdriver/src/browserstack.ts index 74811f78e6..a5954eda00 100644 --- a/packages/provider-webdriver/src/browserstack.ts +++ b/packages/provider-webdriver/src/browserstack.ts @@ -1,14 +1,12 @@ import type { CloudArtifact, CloudArtifactsResult } from '@agent-device/contracts/observability'; import type { CloudWebDriverCapabilityOverrides } from './capabilities.ts'; import type { CloudWebDriverUploadApp } from './runtime.ts'; -import { AppError } from '@agent-device/kernel/errors'; -import { agentDeviceRequestHeaders } from './request-headers.ts'; import { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; import { appFileUploadForm, asRecord, - basicAuthHeader, createHubUploadApp, + fetchProviderSessionDetails, postHubAppUpload, resolveHubAppReference, trimTrailingSlash, @@ -156,21 +154,12 @@ async function fetchBrowserStackSessionDetails( const endpoint = new URL( `${trimTrailingSlash(String(options.endpoint ?? BROWSERSTACK_SESSION_DETAILS_ENDPOINT))}/${sessionId}.json`, ); - const response = await fetch(endpoint, { - headers: { - ...agentDeviceRequestHeaders(options.clientVersion), - Authorization: basicAuthHeader(options), - }, + const json = await fetchProviderSessionDetails(endpoint, { + clientVersion: options.clientVersion, + auth: options, + service: 'BrowserStack', }); - const json = (await response.json()) as unknown; - if (!response.ok || !json || typeof json !== 'object') { - throw new AppError('COMMAND_FAILED', 'BrowserStack session details lookup failed.', { - status: response.status, - response: json, - }); - } - const details = (json as { automation_session?: unknown }).automation_session ?? json; - return details && typeof details === 'object' ? (details as Record) : {}; + return asRecord(json.automation_session) ?? json; } function mapBrowserStackArtifacts( diff --git a/packages/provider-webdriver/src/connection-verification.test.ts b/packages/provider-webdriver/src/connection-verification.test.ts index e9e7f9c726..c93e40109e 100644 --- a/packages/provider-webdriver/src/connection-verification.test.ts +++ b/packages/provider-webdriver/src/connection-verification.test.ts @@ -80,6 +80,35 @@ test('BrowserStack classifies rejected credentials without exposing them', async }); }); +test('BrowserStack points HTTP failures at its service status and transport failures at the network', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => jsonResponse({}, 503)), + ); + await assert.rejects(createProvider().verifyConnection(browserStackOptions), (error: unknown) => { + assert.equal((error as { code?: string }).code, 'COMMAND_FAILED'); + assert.equal( + (error as { details?: { hint?: string } }).details?.hint, + 'Retry connect or check the BrowserStack service status.', + ); + return true; + }); + + vi.stubGlobal( + 'fetch', + vi.fn(async () => { + throw new TypeError('fetch failed'); + }), + ); + await assert.rejects(createProvider().verifyConnection(browserStackOptions), (error: unknown) => { + assert.equal( + (error as { details?: { hint?: string } }).details?.hint, + 'Check network access to api-cloud.browserstack.com and retry connect.', + ); + return true; + }); +}); + test('BrowserStack defers a bs app reference outside the recent upload window', async () => { vi.stubGlobal( 'fetch', diff --git a/packages/provider-webdriver/src/webdriver-utils.ts b/packages/provider-webdriver/src/webdriver-utils.ts index a6e833c8c9..4b750d5439 100644 --- a/packages/provider-webdriver/src/webdriver-utils.ts +++ b/packages/provider-webdriver/src/webdriver-utils.ts @@ -161,6 +161,103 @@ export async function resolveHubAppReference(options: { return await options.uploadFile(appPath, options.signal); } +const PROVIDER_API_TIMEOUT_MS = 15_000; + +/** The provider rejected or could not answer a verification call; typed so callers never sniff text. */ +export type ProviderJsonFailureHints = { + service: string; + unauthorizedHint: string; + networkHint: string; +}; + +/** + * Fetches JSON from a hosted provider's API during connection verification. A 401/403 is + * `UNAUTHORIZED` with a credential hint, any other non-2xx is `COMMAND_FAILED`, and a transport + * failure is wrapped so its cause survives without leaking the credentials. + */ +export async function fetchProviderVerificationJson( + endpoint: string | URL, + options: { + clientVersion: string; + auth?: { username: string; accessKey: string }; + hints: ProviderJsonFailureHints; + }, +): Promise { + const { service, unauthorizedHint, networkHint } = options.hints; + try { + const response = await fetch(endpoint, { + headers: { + ...agentDeviceRequestHeaders(options.clientVersion), + ...(options.auth ? { Authorization: basicAuthHeader(options.auth) } : {}), + }, + signal: AbortSignal.timeout(PROVIDER_API_TIMEOUT_MS), + }); + if (!response.ok) { + const unauthorized = response.status === 401 || response.status === 403; + throw new AppError( + unauthorized ? 'UNAUTHORIZED' : 'COMMAND_FAILED', + `${service} rejected connection verification.`, + { + status: response.status, + hint: unauthorized + ? unauthorizedHint + : `Retry connect or check the ${service} service status.`, + }, + ); + } + return (await response.json()) as unknown; + } catch (error) { + if (error instanceof AppError) throw error; + throw new AppError( + 'COMMAND_FAILED', + `${service} connection verification failed.`, + { hint: networkHint }, + error, + ); + } +} + +/** + * Fetches a provider's session-details JSON with basic auth under a deadline. A transport failure, + * a non-2xx answer, or a body that is not a JSON object is `COMMAND_FAILED`. + */ +export async function fetchProviderSessionDetails( + endpoint: string | URL, + options: { + clientVersion: string; + auth: { username: string; accessKey: string }; + service: string; + }, +): Promise> { + let response: Response; + let json: unknown; + try { + response = await fetch(endpoint, { + headers: { + ...agentDeviceRequestHeaders(options.clientVersion), + Authorization: basicAuthHeader(options.auth), + }, + signal: AbortSignal.timeout(PROVIDER_API_TIMEOUT_MS), + }); + json = await readProviderJsonBody(response); + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + `${options.service} session details lookup failed.`, + { hint: `Check network access to the ${options.service} API, then retry.` }, + error, + ); + } + const details = asRecord(json); + if (!response.ok || !details) { + throw new AppError('COMMAND_FAILED', `${options.service} session details lookup failed.`, { + status: response.status, + response: json, + }); + } + return details; +} + /** A provider response body parsed as JSON, or `undefined` when it is empty or not JSON (a gateway error page). */ async function readProviderJsonBody(response: Response): Promise { const text = await response.text(); @@ -172,6 +269,12 @@ async function readProviderJsonBody(response: Response): Promise { } } +/** `1.0` and `1` name the same OS release on BrowserStack's catalog. */ +export function sameOsVersion(left: string, right: string): boolean { + const normalize = (value: string) => value.replace(/(?:\.0)+$/, ''); + return normalize(left) === normalize(right); +} + /** Validates a device-orientation flag against the shared enum before it reaches a hub that would ignore it. */ export function requireProviderDeviceOrientation( spec: { flag: string; capability: string }, From 465a0a7a9fdcd79102227ddde2de5c5c425bc8f7 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 19:45:11 +0530 Subject: [PATCH 03/57] feat(provider-webdriver): add TestMu AI emulator and simulator provider Add `testmu`, a hosted WebDriver provider for TestMu AI (formerly LambdaTest) virtual devices: Android emulators and iOS simulators behind the TestMu AI Appium hub. It follows the BrowserStack shape: `connect testmu` verifies and saves a local profile, and `open` creates the hosted session. The service hostnames still carry the lambdatest.com domain. connect - Reads LT_USERNAME and LT_ACCESS_KEY, the variables TestMu AI SDKs use. - Checks the device and OS version against the public virtual-device catalog. The match is exact because the hub rejects `18` for a device listed as `18.0`; the error lists the versions the device offers. - Uses the authenticated app listing as the credential check, and looks an lt:// id up in the list for the session's runtime (`emulator` or `simulator`). An id that is not listed is reported as configured, since TestMu AI validates it at session creation. - Verifies against TESTMU_API_ENDPOINT when it is set, as the runtime does, and composes catalog and listing URLs with URL so a base or override that carries a query keeps it. - Never creates a session. Sessions - Standard Appium keys stay `appium:`-prefixed; everything vendor- specific goes in `lt:options`, merged per key with any configured `lt:options`. `isRealMobile: false` and `w3c: true` are applied last, so configuration cannot move the session to another device pool or off the W3C dialect agent-device speaks. - `appiumVersion` is sent only when --provider-appium-version pins one; otherwise TestMu AI starts its default server for the device. - Orientation, geo-location, timezone, Appium version, language, and locale map onto `lt:options` through a table. The BrowserStack-only network-profile, custom-network, and no-resign flags are refused by flag name at connect and at session preparation, instead of being silently dropped. Apps - --provider-app takes an lt:// id, an http(s) URL, or a local path. The hub only accepts lt:// references, so a URL is handed to the upload API to fetch (`storage=url`) and a local file is uploaded. - An unzipped iOS `.app` directory is rejected before any request, with a hint to zip it. - Only a well-formed lt:// reference or app id in the upload response counts as success. Artifacts - Read from the session-details API through the shared bounded lookup. A 404 reads as pending until TestMu AI publishes the details. `console_logs_url` is the device log on virtual devices. - Session video, Appium, network, and command logs, screenshots, and the dashboard link are returned once at least one artifact URL exists. TESTMU_WEBDRIVER_ENDPOINT, TESTMU_APP_UPLOAD_ENDPOINT, and TESTMU_API_ENDPOINT redirect the endpoints. The session, upload, verification, and device-feature modules are loaded with dynamic import, so the package entry's eager module graph does not grow. The CLI reaches the device-feature checks through a new `./testmu-device-features` subpath export. A .fallowrc entry covers the exports that are read only through the dynamic import. `connect testmu` is listed in the connect usage, the remote help topic, and the artifacts provider description. Co-authored-by: gautam-jain-dev Co-Authored-By: Claude Opus 5.5 --- .fallowrc.json | 5 + .../src/flag-definitions-connection.ts | 2 +- packages/provider-webdriver/package.json | 4 + .../src/connection-verification.test.ts | 248 ++++++++++ .../src/connection-verification.ts | 48 +- .../src/provider-definitions.ts | 125 +++++ packages/provider-webdriver/src/providers.ts | 1 + .../src/testmu-connection-verification.ts | 190 ++++++++ .../src/testmu-device-features.test.ts | 103 ++++ .../src/testmu-device-features.ts | 115 +++++ .../provider-webdriver/src/testmu.test.ts | 439 ++++++++++++++++++ packages/provider-webdriver/src/testmu.ts | 264 +++++++++++ .../provider-webdriver/src/webdriver-utils.ts | 2 +- scripts/layering/package-boundaries.test.ts | 1 + src/__tests__/cloud-connect-profile.test.ts | 47 +- src/__tests__/cloud-connect-testmu.test.ts | 192 ++++++++ src/cli/connection/cloud-webdriver-profile.ts | 70 ++- .../connection/connect-provider-adapters.ts | 23 + src/cli/connection/provider-policy.ts | 1 + src/commands/management/artifacts.ts | 4 +- src/commands/schema/cli-help-topics.test.ts | 12 +- src/commands/schema/cli-help.ts | 20 +- src/commands/schema/command-overrides.ts | 2 +- 23 files changed, 1864 insertions(+), 54 deletions(-) create mode 100644 packages/provider-webdriver/src/testmu-connection-verification.ts create mode 100644 packages/provider-webdriver/src/testmu-device-features.test.ts create mode 100644 packages/provider-webdriver/src/testmu-device-features.ts create mode 100644 packages/provider-webdriver/src/testmu.test.ts create mode 100644 packages/provider-webdriver/src/testmu.ts create mode 100644 src/__tests__/cloud-connect-testmu.test.ts diff --git a/.fallowrc.json b/.fallowrc.json index 659ad8259f..494c2f42f2 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -379,6 +379,11 @@ "file": "packages/provider-limrun/src/index.ts", "exports": ["LimrunIosCommandExecution"] }, + { + "comment": "TestMu session preparation reads these off the lazy loadTestMuDeviceFeatures() import in packages/provider-webdriver/src/provider-definitions.ts, which keeps the package entry's eager closure unchanged; Fallow cannot connect the dynamic member reads.", + "file": "packages/provider-webdriver/src/testmu-device-features.ts", + "exports": ["buildTestMuDeviceFeatureCapabilities", "readTestMuDeviceFeatureFields"] + }, { "comment": "Converting the contracts façades from `export *` to explicit named re-exports (the pin-table retirement) made these individually visible to --production analysis for the first time; a bare star previously hid them from this exact check. isRecord/IOS_SAFARI_BUNDLE_ID/REPLAY_DIVERGENCE_* have no production consumer. Kept rather than narrowed here so the façade's re-export surface stays byte-identical to the symbol set the retired pin table asserted — narrowing the surface is a follow-up with its own review, not a side effect of this mechanical conversion.", "file": "packages/contracts/src/facades/{client,command,divergence,recording}.ts", diff --git a/packages/command-registry/src/flag-definitions-connection.ts b/packages/command-registry/src/flag-definitions-connection.ts index 0b01b00d23..0c9a34b0f1 100644 --- a/packages/command-registry/src/flag-definitions-connection.ts +++ b/packages/command-registry/src/flag-definitions-connection.ts @@ -236,7 +236,7 @@ export const CONNECTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'string', usageLabel: '--provider-appium-version ', usageDescription: - 'Hosted cloud provider Appium server version, for example 3.2.0. Without it BrowserStack falls back to its default (Appium 1.x)', + 'Hosted cloud provider Appium server version, for example 3.2.0. Without it each provider starts its own default (Appium 1.x on BrowserStack)', projectConfig: false, recorded: false, }, diff --git a/packages/provider-webdriver/package.json b/packages/provider-webdriver/package.json index d4d56b8d0e..a9f01ebacb 100644 --- a/packages/provider-webdriver/package.json +++ b/packages/provider-webdriver/package.json @@ -19,6 +19,10 @@ "./providers": { "types": "./src/providers.ts", "default": "./src/providers.ts" + }, + "./testmu-device-features": { + "types": "./src/testmu-device-features.ts", + "default": "./src/testmu-device-features.ts" } } } diff --git a/packages/provider-webdriver/src/connection-verification.test.ts b/packages/provider-webdriver/src/connection-verification.test.ts index c93e40109e..5b2d872493 100644 --- a/packages/provider-webdriver/src/connection-verification.test.ts +++ b/packages/provider-webdriver/src/connection-verification.test.ts @@ -213,6 +213,254 @@ test('AWS Device Farm rejects a device from the wrong platform before allocation ); }); +const testMuOptions = { + provider: 'testmu' as const, + username: 'lt-user', + accessKey: 'lt-key', + platform: 'android' as const, + deviceName: 'Pixel 8', + osVersion: '14', + app: 'lt://APP1', + devicesEndpoint: 'https://testmu.test/capability/generator?isVirtualDevice=true', + appsEndpoint: 'https://testmu.test/app/data', +}; + +const testMuCatalog = { + app: { + devices: { + android: { + brands: { + Google: [ + { name: 'Pixel 8', osVersion: ['14', '15'] }, + { name: 'Pixel 4a', osVersion: ['13'] }, + ], + }, + }, + ios: { brands: { Apple: [{ name: 'iPhone 16', osVersion: ['18.0'] }] } }, + }, + }, +}; + +test('TestMu verifies the virtual device and uploaded app without creating a session', async () => { + const fetchMock = vi.fn(async (input, init) => { + const headers = (init?.headers ?? {}) as Record; + if (String(input).includes('capability/generator')) { + assert.equal(headers.Authorization, undefined); + return jsonResponse(testMuCatalog); + } + assert.match(String(headers.Authorization), /^Basic /); + return jsonResponse({ + data: [{ app_id: 'APP1', name: 'sample.apk', version: '1.2.3', type: 'android' }], + metaData: { total: 1 }, + }); + }); + vi.stubGlobal('fetch', fetchMock); + + const result = await createProvider().verifyConnection(testMuOptions); + + assert.equal(result.provider, 'testmu'); + assert.equal(result.service, 'TestMu AI'); + assert.deepEqual(result.device, { + status: 'verified', + name: 'Pixel 8', + platform: 'android', + osVersion: '14', + }); + assert.deepEqual(result.app, { + status: 'verified', + name: 'sample.apk', + reference: 'lt://APP1', + version: '1.2.3', + }); + assert.deepEqual( + fetchMock.mock.calls.map(([input]) => String(input)), + [ + 'https://testmu.test/capability/generator?isVirtualDevice=true', + 'https://testmu.test/app/data?type=emulator&level=user', + ], + ); +}); + +test('TestMu checks the catalog of the configured API endpoint', async () => { + const fetchMock = vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(testMuCatalog) + : jsonResponse({ data: [{ app_id: 'APP1' }] }), + ); + vi.stubGlobal('fetch', fetchMock); + const { devicesEndpoint: _devicesEndpoint, ...options } = testMuOptions; + + await createProvider().verifyConnection({ + ...options, + apiEndpoint: 'https://staging.testmu.test/mobile-automation/api/v1/', + }); + + assert.equal( + String(fetchMock.mock.calls[0]?.[0]), + 'https://staging.testmu.test/mobile-automation/api/v1/capability/generator?isVirtualDevice=true', + ); +}); + +test('TestMu keeps the query of an overridden endpoint and adds its own filters', async () => { + const fetchMock = vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(testMuCatalog) + : jsonResponse({ data: [{ app_id: 'APP1' }] }), + ); + vi.stubGlobal('fetch', fetchMock); + const { devicesEndpoint: _devicesEndpoint, ...options } = testMuOptions; + + await createProvider().verifyConnection({ + ...options, + apiEndpoint: 'https://staging.testmu.test/api/v1/?region=eu', + appsEndpoint: 'https://staging.testmu.test/app/data?org=42', + }); + await createProvider().verifyConnection({ + ...testMuOptions, + devicesEndpoint: 'https://testmu.test/capability/generator?region=eu', + }); + + assert.deepEqual( + fetchMock.mock.calls.map(([input]) => String(input)), + [ + 'https://staging.testmu.test/api/v1/capability/generator?region=eu&isVirtualDevice=true', + 'https://staging.testmu.test/app/data?org=42&type=emulator&level=user', + 'https://testmu.test/capability/generator?region=eu&isVirtualDevice=true', + 'https://testmu.test/app/data?type=emulator&level=user', + ], + ); +}); + +test('TestMu rejects a device or OS version missing from the virtual-device catalog', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => jsonResponse(testMuCatalog)), + ); + await assert.rejects( + createProvider().verifyConnection({ ...testMuOptions, osVersion: '12' }), + (error: unknown) => + error instanceof Error && /"Pixel 8" with android 12 is not available/.test(error.message), + ); + await assert.rejects( + createProvider().verifyConnection({ ...testMuOptions, platform: 'ios', deviceName: 'Pixel 8' }), + /is not available/, + ); +}); + +// The hub rejects `platformVersion: '18'` for a catalog entry spelled `18.0`, so connect must too. +test('TestMu matches the catalog OS version spelling exactly and lists the offered versions', async () => { + const catalog = { + app: { + devices: { + ios: { + brands: { + Apple: [{ name: 'iPhone 16', osVersion: ['18.1', '26.0', '18.0', '18.5', '26.2'] }], + }, + }, + }, + }, + }; + vi.stubGlobal( + 'fetch', + vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(catalog) + : jsonResponse({ data: [], metaData: { total: 0 } }), + ), + ); + const iosOptions = { ...testMuOptions, platform: 'ios' as const, deviceName: 'iPhone 16' }; + + await assert.rejects( + createProvider().verifyConnection({ ...iosOptions, osVersion: '18' }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'INVALID_ARGS'); + assert.match(error.message, /iPhone 16 offers 18\.0, 18\.1, 18\.5, 26\.0, 26\.2/); + return true; + }, + ); + + const result = await createProvider().verifyConnection({ ...iosOptions, osVersion: '18.0' }); + assert.deepEqual(result.device, { + status: 'verified', + name: 'iPhone 16', + platform: 'ios', + osVersion: '18.0', + }); +}); + +test('TestMu classifies rejected credentials without exposing them', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(testMuCatalog) + : jsonResponse({ message: 'Unauthorized' }, 401), + ), + ); + await assert.rejects(createProvider().verifyConnection(testMuOptions), (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'UNAUTHORIZED'); + assert.doesNotMatch(error.message, /lt-key/); + return true; + }); +}); + +test('TestMu defers an lt:// reference it cannot find and a local path it will upload', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(testMuCatalog) + : jsonResponse({ data: [], metaData: { total: 0 } }), + ), + ); + const unknownApp = await createProvider().verifyConnection(testMuOptions); + assert.equal(unknownApp.app.status, 'configured'); + assert.equal(unknownApp.app.reference, 'lt://APP1'); + + const localApp = await createProvider().verifyConnection({ + ...testMuOptions, + app: '/tmp/builds/App.apk', + }); + assert.deepEqual(localApp.app, { + status: 'configured', + name: 'App.apk', + reference: '/tmp/builds/App.apk', + message: 'Local app artifact is ready and will be uploaded when creating the session.', + }); +}); + +// The listing is keyed by runtime: an Android emulator upload is listed under `emulator` and an +// iOS simulator upload under `simulator`, never under the platform name. +test('TestMu checks an lt:// id against the virtual-device app list of its platform', async () => { + const cases = [ + { platform: 'android', deviceName: 'Pixel 8', osVersion: '14', listType: 'emulator' }, + { platform: 'ios', deviceName: 'iPhone 16', osVersion: '18.0', listType: 'simulator' }, + ] as const; + for (const { platform, deviceName, osVersion, listType } of cases) { + const fetchMock = vi.fn(async (input) => { + const url = String(input); + if (url.includes('capability/generator')) return jsonResponse(testMuCatalog); + return jsonResponse({ + data: new URL(url).searchParams.get('type') === listType ? [{ app_id: 'APP1' }] : [], + }); + }); + vi.stubGlobal('fetch', fetchMock); + const result = await createProvider().verifyConnection({ + ...testMuOptions, + platform, + deviceName, + osVersion, + }); + assert.equal(result.app.status, 'verified', platform); + assert.equal( + String(fetchMock.mock.calls[1]?.[0]), + `https://testmu.test/app/data?type=${listType}&level=user`, + ); + } +}); + function createProvider(runHostCommand: RunHostCommand = vi.fn()) { return createProviderWebDriver({ clientVersion: '1.2.3', runHostCommand }); } diff --git a/packages/provider-webdriver/src/connection-verification.ts b/packages/provider-webdriver/src/connection-verification.ts index 0ef8f5ce97..505fb2cf09 100644 --- a/packages/provider-webdriver/src/connection-verification.ts +++ b/packages/provider-webdriver/src/connection-verification.ts @@ -15,20 +15,32 @@ export type CloudWebDriverConnectionVerification = provider: 'aws-device-farm'; service: 'AWS Device Farm'; project: { name?: string; reference: string }; + }) + | (ProviderConnectionVerification & { + provider: 'testmu'; + service: 'TestMu AI'; + project?: never; }); +/** Credentials plus the exact device, OS, and app a hosted Appium hub session is created with. */ +type HubSelectionVerificationOptions = { + username: string; + accessKey: string; + platform: 'android' | 'ios'; + deviceName: string; + osVersion: string; + app: string; + devicesEndpoint?: string | URL; + appsEndpoint?: string | URL; +}; + export type CloudWebDriverConnectionVerificationOptions = - | { - provider: 'browserstack'; - username: string; - accessKey: string; - platform: 'android' | 'ios'; - deviceName: string; - osVersion: string; - app: string; - devicesEndpoint?: string | URL; - appsEndpoint?: string | URL; - } + | (HubSelectionVerificationOptions & { provider: 'browserstack' }) + | (HubSelectionVerificationOptions & { + provider: 'testmu'; + /** Base of the catalog API, as `TESTMU_API_ENDPOINT` sets it for the runtime. */ + apiEndpoint?: string | URL; + }) | { provider: 'aws-device-farm'; platform: 'android' | 'ios'; @@ -42,7 +54,15 @@ export async function verifyCloudWebDriverConnection( options: CloudWebDriverConnectionVerificationOptions, dependencies: ProviderWebDriverDependencies, ): Promise { - return options.provider === 'browserstack' - ? await verifyBrowserStackConnection(options, dependencies.clientVersion) - : await verifyAwsDeviceFarmConnection(options, dependencies.runHostCommand); + switch (options.provider) { + case 'browserstack': + return await verifyBrowserStackConnection(options, dependencies.clientVersion); + case 'testmu': { + // Loaded on demand: the package entry must not grow its eager closure for a new vendor. + const { verifyTestMuConnection } = await import('./testmu-connection-verification.ts'); + return await verifyTestMuConnection(options, dependencies.clientVersion); + } + case 'aws-device-farm': + return await verifyAwsDeviceFarmConnection(options, dependencies.runHostCommand); + } } diff --git a/packages/provider-webdriver/src/provider-definitions.ts b/packages/provider-webdriver/src/provider-definitions.ts index 713fd7d8ce..06e147c5eb 100644 --- a/packages/provider-webdriver/src/provider-definitions.ts +++ b/packages/provider-webdriver/src/provider-definitions.ts @@ -22,6 +22,7 @@ import { readBrowserStackDeviceFeatureFields, rejectBrowserStackOnlyDeviceFeatures, } from './browserstack-device-features.ts'; +import type { CloudWebDriverCapabilityOverrides } from './capabilities.ts'; import { CLOUD_WEBDRIVER_PROVIDERS, type CloudWebDriverKnownProviderName } from './providers.ts'; import { readAwsDeviceFarmRegionFromArn } from './connection-verification.ts'; import { @@ -37,11 +38,16 @@ export type DefaultCloudWebDriverArtifactEnv = { BROWSERSTACK_SESSION_DETAILS_ENDPOINT?: string; AWS_REGION?: string; AWS_DEFAULT_REGION?: string; + LT_USERNAME?: string; + LT_ACCESS_KEY?: string; + TESTMU_API_ENDPOINT?: string; }; export type DefaultCloudWebDriverProviderRuntimeEnv = DefaultCloudWebDriverArtifactEnv & { BROWSERSTACK_WEBDRIVER_ENDPOINT?: string; BROWSERSTACK_APP_UPLOAD_ENDPOINT?: string; + TESTMU_WEBDRIVER_ENDPOINT?: string; + TESTMU_APP_UPLOAD_ENDPOINT?: string; AGENT_DEVICE_AWS_DEVICE_FARM_PROJECT_ARN?: string; AWS_DEVICE_FARM_PROJECT_ARN?: string; AGENT_DEVICE_AWS_DEVICE_FARM_DEVICE_ARN?: string; @@ -50,6 +56,30 @@ export type DefaultCloudWebDriverProviderRuntimeEnv = DefaultCloudWebDriverArtif AWS_DEVICE_FARM_APP_ARN?: string; }; +/** + * TestMu (formerly LambdaTest) virtual devices: emulators and simulators behind one Appium hub. + * Only what `createRuntime` needs synchronously lives here; the session, upload, and artifact + * code loads on first use so the package entry stays as lean as it was. + */ +const TESTMU_WEBDRIVER_ENDPOINT = 'https://mobile-hub.lambdatest.com/wd/hub/'; +const TESTMU_CAPABILITY_OVERRIDES = { + install: { + support: 'partial', + note: 'Local app artifacts are uploaded to TestMu AI as virtual-device apps (lt://), then installed with Appium.', + }, + portReverse: { + support: 'unsupported', + note: 'Use the TestMu AI tunnel for network access to local hosts; agent-device port reverse is not available.', + }, + artifacts: { + support: 'supported', + note: 'TestMu AI session details expose provider-hosted video, Appium logs, device logs, network logs, and dashboard links.', + }, +} as const satisfies CloudWebDriverCapabilityOverrides; + +const loadTestMu = async () => await import('./testmu.ts'); +const loadTestMuDeviceFeatures = async () => await import('./testmu-device-features.ts'); + export type CloudWebDriverProviderDefinition = { provider: CloudWebDriverKnownProviderName; createRuntime: (env: DefaultCloudWebDriverProviderRuntimeEnv) => CloudWebDriverRuntime; @@ -246,7 +276,102 @@ export function createCloudWebDriverProviderDefinitions( ); }, }, + { + provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, + createRuntime: (env) => + createCloudWebDriverRuntime({ + clientVersion: dependencies.clientVersion, + provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, + platform: 'android', + deviceName: 'TestMu AI device', + endpoint: env.TESTMU_WEBDRIVER_ENDPOINT ?? TESTMU_WEBDRIVER_ENDPOINT, + capabilityOverrides: TESTMU_CAPABILITY_OVERRIDES, + listArtifacts: async ({ provider, providerSessionId }) => + await listTestMuArtifactsFromEnv(provider, providerSessionId, env), + prepareSession: async ({ req, lease, base }) => { + const request = requireRequest(req, 'TestMu AI'); + const { buildTestMuCapabilities, createTestMuUploadApp, resolveTestMuAppReference } = + await loadTestMu(); + const { + buildTestMuDeviceFeatureCapabilities, + readTestMuDeviceFeatureFields, + rejectUnsupportedTestMuDeviceFeatures, + } = await loadTestMuDeviceFeatures(); + rejectUnsupportedTestMuDeviceFeatures(request.flags); + const credentials = requireTestMuCredentials(env, 'TestMu AI'); + const platform = requireRequestPlatform(request, 'TestMu AI'); + const deviceName = requireFlag( + request, + 'device', + 'TestMu AI requires --device .', + ); + const osVersion = requireFlag( + request, + 'providerOsVersion', + 'TestMu AI requires --provider-os-version .', + ); + const upload = { + clientVersion: dependencies.clientVersion, + ...credentials, + endpoint: env.TESTMU_APP_UPLOAD_ENDPOINT, + }; + const app = await resolveTestMuAppReference( + requireFlag( + request, + 'providerApp', + 'TestMu AI requires --provider-app .', + ), + { ...upload, cwd: request.cwd, signal: request.signal }, + ); + return { + ...base, + platform, + deviceName, + auth: credentials, + uploadApp: createTestMuUploadApp(upload), + webdriverCapabilities: buildTestMuCapabilities({ + platform, + deviceName, + osVersion, + app, + projectName: readFlag(request, 'providerProject'), + buildName: readFlag(request, 'providerBuild') ?? lease.runId, + sessionName: readFlag(request, 'providerSessionName') ?? lease.leaseId, + deviceFeatures: buildTestMuDeviceFeatureCapabilities( + readTestMuDeviceFeatureFields(request.flags), + ), + configured: buildCloudWebDriverBaseCapabilities(platform, deviceName), + }), + }; + }, + }), + listArtifactsFromEnv: async (providerSessionId, env) => + await listTestMuArtifactsFromEnv(CLOUD_WEBDRIVER_PROVIDERS.testMu, providerSessionId, env), + }, ]; + + async function listTestMuArtifactsFromEnv( + provider: string, + providerSessionId: string | undefined, + env: DefaultCloudWebDriverArtifactEnv, + ): Promise { + const { listTestMuCloudArtifacts } = await loadTestMu(); + return await listTestMuCloudArtifacts(provider, providerSessionId, { + clientVersion: dependencies.clientVersion, + ...requireTestMuCredentials(env, 'TestMu AI artifact lookup'), + endpoint: env.TESTMU_API_ENDPOINT, + }); + } +} + +function requireTestMuCredentials( + env: DefaultCloudWebDriverArtifactEnv, + providerLabel: string, +): { username: string; accessKey: string } { + return { + username: requireEnv(env, 'LT_USERNAME', providerLabel), + accessKey: requireEnv(env, 'LT_ACCESS_KEY', providerLabel), + }; } function requireRequest( diff --git a/packages/provider-webdriver/src/providers.ts b/packages/provider-webdriver/src/providers.ts index 814a0c19ae..4160d465a5 100644 --- a/packages/provider-webdriver/src/providers.ts +++ b/packages/provider-webdriver/src/providers.ts @@ -1,6 +1,7 @@ export const CLOUD_WEBDRIVER_PROVIDERS = { browserStack: 'browserstack', awsDeviceFarm: 'aws-device-farm', + testMu: 'testmu', } as const; export type CloudWebDriverKnownProviderName = diff --git a/packages/provider-webdriver/src/testmu-connection-verification.ts b/packages/provider-webdriver/src/testmu-connection-verification.ts new file mode 100644 index 0000000000..91d913ce23 --- /dev/null +++ b/packages/provider-webdriver/src/testmu-connection-verification.ts @@ -0,0 +1,190 @@ +import path from 'node:path'; +import { AppError } from '@agent-device/kernel/errors'; +import { asRecord, fetchProviderVerificationJson, trimTrailingSlash } from './webdriver-utils.ts'; +import { TESTMU_API_ENDPOINT, TESTMU_APPS_ENDPOINT, isTestMuAppReference } from './testmu.ts'; +import type { + CloudWebDriverConnectionVerification, + CloudWebDriverConnectionVerificationOptions, +} from './connection-verification.ts'; +import type { ProviderConnectionResource } from '@agent-device/contracts/remote'; + +type TestMuOptions = Extract; + +type TestMuAuth = { username: string; accessKey: string }; + +/** `/app/data?type=` keys virtual-device uploads by runtime, not by platform. */ +const TESTMU_APP_LIST_TYPES: Record<'android' | 'ios', string> = { + android: 'emulator', + ios: 'simulator', +}; + +/** + * Verifies a TestMu virtual-device selection without creating a session: the public capability + * catalog confirms the device/OS pair exists in the emulator and simulator pool, and the + * authenticated app listing confirms the credentials and, for an `lt://` reference, the upload. + */ +export async function verifyTestMuConnection( + options: TestMuOptions, + clientVersion: string, +): Promise { + const auth = { username: options.username, accessKey: options.accessKey }; + const catalogUrl = options.devicesEndpoint + ? new URL(options.devicesEndpoint) + : apiUrl(options.apiEndpoint ?? TESTMU_API_ENDPOINT, 'capability/generator'); + catalogUrl.searchParams.set('isVirtualDevice', 'true'); + const catalog = await fetchTestMuJson(catalogUrl, undefined, clientVersion); + const namedDevices = readTestMuVirtualDevices(catalog, options.platform).filter( + (device) => device.name === options.deviceName, + ); + // Exact match on purpose: the hub rejects `18` for a device the catalog lists as `18.0`. + const matchedDevice = namedDevices.find((device) => + device.osVersions.includes(options.osVersion), + ); + if (!matchedDevice) { + const offered = [...new Set(namedDevices.flatMap((device) => device.osVersions))].sort( + (left, right) => left.localeCompare(right, undefined, { numeric: true }), + ); + throw new AppError( + 'INVALID_ARGS', + `TestMu AI virtual device "${options.deviceName}" with ${options.platform} ${options.osVersion} is not available${ + offered.length > 0 ? `; ${options.deviceName} offers ${offered.join(', ')}` : '' + }.`, + { + hint: 'Choose an exact device name and OS version from the TestMu AI virtual-device capability generator.', + ...(offered.length > 0 ? { availableOsVersions: offered } : {}), + }, + ); + } + + const app = await verifyTestMuApp(options, auth, clientVersion); + return { + provider: 'testmu', + service: 'TestMu AI', + verificationMessage: + app.status === 'verified' + ? 'Credentials, virtual device, and uploaded app verified.' + : 'Credentials and virtual device verified; app availability is checked when the session is created.', + device: { + status: 'verified', + name: matchedDevice.name, + platform: options.platform, + osVersion: options.osVersion, + }, + app, + }; +} + +async function verifyTestMuApp( + options: TestMuOptions, + auth: TestMuAuth, + clientVersion: string, +): Promise { + const { app } = options; + // The listing is authenticated, so it doubles as the credential check for every app kind. + const appsUrl = new URL(options.appsEndpoint ?? TESTMU_APPS_ENDPOINT); + appsUrl.searchParams.set('type', TESTMU_APP_LIST_TYPES[options.platform]); + appsUrl.searchParams.set('level', 'user'); + const apps = await fetchTestMuJson(appsUrl, auth, clientVersion); + if (isTestMuAppReference(app)) { + const matched = readTestMuApps(apps).find((entry) => entry.reference === app); + if (!matched) { + return { + status: 'configured', + reference: app, + message: + 'App reference was not found among your virtual-device uploads; TestMu AI validates it when creating the session.', + }; + } + return { status: 'verified', ...matched }; + } + if (/^https?:\/\//i.test(app)) { + return { + status: 'configured', + reference: app, + message: 'Public app URL configured; TestMu AI fetches it when creating the session.', + }; + } + return { + status: 'configured', + name: path.basename(app), + reference: app, + message: 'Local app artifact is ready and will be uploaded when creating the session.', + }; +} + +/** Appends `route` to the base's path; the base may carry a query, which is kept. */ +function apiUrl(base: string | URL, route: string): URL { + const url = new URL(base); + url.pathname = `${trimTrailingSlash(url.pathname)}/${route}`; + return url; +} + +async function fetchTestMuJson( + endpoint: string | URL, + auth: TestMuAuth | undefined, + clientVersion: string, +): Promise { + return await fetchProviderVerificationJson(endpoint, { + clientVersion, + auth, + hints: { + service: 'TestMu AI', + unauthorizedHint: 'Check LT_USERNAME and LT_ACCESS_KEY.', + networkHint: + 'Check network access to mobile-api.lambdatest.com and manual-api.lambdatest.com, then retry connect.', + }, + }); +} + +/** + * The capability generator lists virtual devices per platform under + * `app.devices..brands.[]` as `{ name, osVersion: string[] }`. + */ +function readTestMuVirtualDevices( + value: unknown, + platform: 'android' | 'ios', +): Array<{ name: string; osVersions: string[] }> { + const platformCatalog = asRecord(asRecord(asRecord(asRecord(value)?.app)?.devices)?.[platform]); + const brandRecord = asRecord(platformCatalog?.brands); + if (!brandRecord) { + throw new AppError( + 'COMMAND_FAILED', + 'TestMu AI virtual-device catalog response did not list devices for the platform.', + { platform }, + ); + } + return Object.values(brandRecord).flatMap((devices) => { + if (!Array.isArray(devices)) return []; + return devices.flatMap((entry) => { + const record = asRecord(entry); + if (!record || typeof record.name !== 'string' || !Array.isArray(record.osVersion)) return []; + const osVersions = record.osVersion.flatMap((osVersion) => + typeof osVersion === 'string' || typeof osVersion === 'number' ? [String(osVersion)] : [], + ); + return [{ name: record.name, osVersions }]; + }); + }); +} + +/** `/app/data` answers `{ data: [{ app_id, name, version, ... }], metaData }`. */ +function readTestMuApps( + value: unknown, +): Array<{ name?: string; reference: string; version?: string }> { + const record = asRecord(value); + const data = record?.data; + if (!Array.isArray(data)) { + throw new AppError('COMMAND_FAILED', 'TestMu AI app listing response was not a list.'); + } + return data.flatMap((entry) => { + const app = asRecord(entry); + if (!app || typeof app.app_id !== 'string') return []; + const reference = isTestMuAppReference(app.app_id) ? app.app_id : `lt://${app.app_id}`; + return [ + { + reference, + ...(typeof app.name === 'string' ? { name: app.name } : {}), + ...(typeof app.version === 'string' ? { version: app.version } : {}), + }, + ]; + }); +} diff --git a/packages/provider-webdriver/src/testmu-device-features.test.ts b/packages/provider-webdriver/src/testmu-device-features.test.ts new file mode 100644 index 0000000000..da38e77e3b --- /dev/null +++ b/packages/provider-webdriver/src/testmu-device-features.test.ts @@ -0,0 +1,103 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; + +import { AppError } from '@agent-device/kernel/errors'; +import { + TESTMU_DEVICE_FEATURE_SPECS, + buildTestMuDeviceFeatureCapabilities, + readTestMuDeviceFeatureFields, + rejectUnsupportedTestMuDeviceFeatures, +} from './testmu-device-features.ts'; + +// Every hosted-provider device-feature field is either a TestMu spec row or an explicit rejection: +// a field in neither parses off the CLI, rides the profile, and is silently dropped at the hub. +const SUPPORTED_FIELDS = [ + 'providerDeviceOrientation', + 'providerGeoLocation', + 'providerTimezone', + 'providerAppiumVersion', + 'providerLanguage', + 'providerLocale', +] as const; +const REJECTED_FIELDS = [ + 'providerNetworkProfile', + 'providerCustomNetwork', + 'providerNoResignApp', +] as const; + +test('every supported device-feature field maps to exactly one lt:options key', () => { + const fields = TESTMU_DEVICE_FEATURE_SPECS.map((spec) => spec.field); + assert.deepEqual([...fields].sort(), [...SUPPORTED_FIELDS].sort()); + const capabilities = TESTMU_DEVICE_FEATURE_SPECS.map((spec) => spec.capability); + assert.equal(new Set(capabilities).size, capabilities.length); +}); + +test('configured device features project onto TestMu capability keys', () => { + const capabilities = buildTestMuDeviceFeatureCapabilities({ + providerDeviceOrientation: 'landscape', + providerGeoLocation: 'US', + providerTimezone: 'UTC+05:30', + providerAppiumVersion: '2.16.2', + providerLanguage: 'fr', + providerLocale: 'fr_FR', + }); + assert.deepEqual(capabilities, { + deviceOrientation: 'LANDSCAPE', + geoLocation: 'US', + timezone: 'UTC+05:30', + appiumVersion: '2.16.2', + language: 'fr', + locale: 'fr_FR', + }); +}); + +test('unset and empty device features emit nothing', () => { + assert.deepEqual(buildTestMuDeviceFeatureCapabilities({}), {}); + assert.deepEqual(buildTestMuDeviceFeatureCapabilities({ providerGeoLocation: '' }), {}); +}); + +test('daemon flag bags are read through the same table with orientation validated', () => { + assert.deepEqual( + readTestMuDeviceFeatureFields({ + providerDeviceOrientation: 'portrait', + providerLocale: 'de_DE', + providerNetworkProfile: 'ignored-here', + providerGeoLocation: 7, + }), + { providerDeviceOrientation: 'portrait', providerLocale: 'de_DE' }, + ); + assert.throws( + () => readTestMuDeviceFeatureFields({ providerDeviceOrientation: 'sideways' }), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + error.details?.flag === '--provider-device-orientation', + ); +}); + +test('BrowserStack-only flags are rejected by flag name instead of being dropped', () => { + assert.doesNotThrow(() => rejectUnsupportedTestMuDeviceFeatures(undefined)); + assert.doesNotThrow(() => + rejectUnsupportedTestMuDeviceFeatures({ + providerGeoLocation: 'US', + providerNoResignApp: false, + }), + ); + for (const field of REJECTED_FIELDS) { + assert.throws( + () => + rejectUnsupportedTestMuDeviceFeatures({ + [field]: field === 'providerNoResignApp' ? true : 'x', + }), + (error: unknown) => error instanceof AppError && error.code === 'INVALID_ARGS', + ); + } + assert.throws( + () => + rejectUnsupportedTestMuDeviceFeatures({ + providerNetworkProfile: '4g-lte-good', + providerCustomNetwork: '1000', + }), + /--provider-network-profile, --provider-custom-network are not supported by TestMu AI/, + ); +}); diff --git a/packages/provider-webdriver/src/testmu-device-features.ts b/packages/provider-webdriver/src/testmu-device-features.ts new file mode 100644 index 0000000000..6f1ab049c4 --- /dev/null +++ b/packages/provider-webdriver/src/testmu-device-features.ts @@ -0,0 +1,115 @@ +import type { CloudProviderProfileFields } from '@agent-device/contracts/remote'; +import { AppError } from '@agent-device/kernel/errors'; +import { requireProviderDeviceOrientation } from './webdriver-utils.ts'; + +/** + * TestMu "device feature" session capabilities: the hosted-provider flags TestMu can act on, + * projected onto their `lt:options` keys. The table is the contract; adding a capability means + * adding a row, not a branch. + */ +export type TestMuDeviceFeatureFields = Pick< + CloudProviderProfileFields, + | 'providerDeviceOrientation' + | 'providerGeoLocation' + | 'providerTimezone' + | 'providerAppiumVersion' + | 'providerLanguage' + | 'providerLocale' +>; + +type TestMuDeviceFeatureSpec = { + field: keyof TestMuDeviceFeatureFields; + /** Key emitted inside `lt:options`. */ + capability: string; + /** Canonical CLI flag, so an error can name a recovery action. */ + flag: string; + /** Projects the validated flag value onto what the hub expects. */ + project?: (value: string) => unknown; +}; + +export const TESTMU_DEVICE_FEATURE_SPECS: readonly TestMuDeviceFeatureSpec[] = [ + { + field: 'providerDeviceOrientation', + capability: 'deviceOrientation', + flag: '--provider-device-orientation', + // The hub matches the orientation enum case-sensitively in upper case. + project: (value) => value.toUpperCase(), + }, + { field: 'providerGeoLocation', capability: 'geoLocation', flag: '--provider-geo-location' }, + { field: 'providerTimezone', capability: 'timezone', flag: '--provider-timezone' }, + { + field: 'providerAppiumVersion', + capability: 'appiumVersion', + flag: '--provider-appium-version', + }, + { field: 'providerLanguage', capability: 'language', flag: '--provider-language' }, + { field: 'providerLocale', capability: 'locale', flag: '--provider-locale' }, +]; + +/** Hosted-provider flags other vendors own and TestMu has no capability for. */ +const TESTMU_UNSUPPORTED_DEVICE_FEATURE_FLAGS: ReadonlyArray<{ + field: keyof CloudProviderProfileFields; + flag: string; +}> = [ + { field: 'providerNetworkProfile', flag: '--provider-network-profile' }, + { field: 'providerCustomNetwork', flag: '--provider-custom-network' }, + { field: 'providerNoResignApp', flag: '--provider-no-resign-app' }, +]; + +/** Builds the `lt:options` fragment for the configured device features. */ +export function buildTestMuDeviceFeatureCapabilities( + fields: TestMuDeviceFeatureFields, +): Record { + const capabilities: Record = {}; + for (const spec of TESTMU_DEVICE_FEATURE_SPECS) { + const value = fields[spec.field]; + if (value === undefined || value === '') continue; + capabilities[spec.capability] = spec.project ? spec.project(value) : value; + } + return capabilities; +} + +/** + * Fails when flags TestMu cannot act on were given. Called from both `connect testmu` (through the + * `./testmu-device-features` subpath, so the package entry stays lazy) and session preparation, + * since the typed client and hand-authored profiles skip `connect`. + */ +export function rejectUnsupportedTestMuDeviceFeatures( + flags: Record | undefined, +): void { + const configured = TESTMU_UNSUPPORTED_DEVICE_FEATURE_FLAGS.filter(({ field }) => { + const value = flags?.[field]; + return value !== undefined && value !== false && value !== ''; + }).map(({ flag }) => flag); + if (configured.length === 0) return; + const plural = configured.length !== 1; + throw new AppError( + 'INVALID_ARGS', + `${configured.join(', ')} ${plural ? 'are' : 'is'} not supported by TestMu AI.`, + { + hint: `Drop ${plural ? 'those flags' : 'the flag'}; TestMu AI has no equivalent capability.`, + provider: 'testmu', + flags: configured, + }, + ); +} + +/** + * Reads device-feature fields off an untyped flag bag (a daemon request). Enum values are + * validated here rather than forwarded to the hub, where an unrecognized value is ignored. + */ +export function readTestMuDeviceFeatureFields( + flags: Record | undefined, +): TestMuDeviceFeatureFields { + const fields: TestMuDeviceFeatureFields = {}; + for (const spec of TESTMU_DEVICE_FEATURE_SPECS) { + const value = flags?.[spec.field]; + if (typeof value !== 'string' || value.length === 0) continue; + if (spec.field === 'providerDeviceOrientation') { + fields.providerDeviceOrientation = requireProviderDeviceOrientation(spec, value); + continue; + } + fields[spec.field] = value; + } + return fields; +} diff --git a/packages/provider-webdriver/src/testmu.test.ts b/packages/provider-webdriver/src/testmu.test.ts new file mode 100644 index 0000000000..9b02320c57 --- /dev/null +++ b/packages/provider-webdriver/src/testmu.test.ts @@ -0,0 +1,439 @@ +import assert from 'node:assert/strict'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { afterEach, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { + buildTestMuCapabilities, + createTestMuUploadApp, + listTestMuCloudArtifacts, + resolveTestMuAppReference, + uploadTestMuApp, + uploadTestMuAppFromUrl, +} from './testmu.ts'; +import { buildCloudWebDriverBaseCapabilities } from './runtime.ts'; +import { mkdtempForTest } from './tmp-dir.fixtures.ts'; + +const realFetch = globalThis.fetch; +const auth = { clientVersion: '0.0.0-test', username: 'user', accessKey: 'key' }; + +afterEach(() => { + globalThis.fetch = realFetch; + vi.unstubAllGlobals(); +}); + +// `isRealMobile: false` is the one capability that routes to the emulator/simulator pool; a +// session without it lands on a real device and bills differently. +test('TestMu capabilities select the virtual-device pool and keep vendor keys in lt:options', () => { + const capabilities = buildTestMuCapabilities({ + platform: 'android', + deviceName: 'Pixel 8', + osVersion: '14', + app: 'lt://APP1', + projectName: 'agent-device', + buildName: 'run-1', + sessionName: 'lease-1', + deviceFeatures: { geoLocation: 'US' }, + configured: buildCloudWebDriverBaseCapabilities('android', 'Pixel 8'), + }); + + assert.deepEqual(capabilities, { + platformName: 'Android', + 'appium:deviceName': 'Pixel 8', + 'appium:platformVersion': '14', + 'appium:app': 'lt://APP1', + 'lt:options': { + isRealMobile: false, + w3c: true, + platformName: 'Android', + deviceName: 'Pixel 8', + platformVersion: '14', + app: 'lt://APP1', + project: 'agent-device', + build: 'run-1', + name: 'lease-1', + video: true, + devicelog: true, + geoLocation: 'US', + }, + }); + for (const key of Object.keys(capabilities)) { + assert.ok( + key === 'platformName' || key.startsWith('appium:') || key === 'lt:options', + `legacy top-level key ${key} would make the hub ignore lt:options`, + ); + } +}); + +// Unpinned, TestMu AI starts its own default Appium server for the device, as BrowserStack does. +test('a configured lt:options merges per key and only a pinned Appium version is sent', () => { + const capabilities = buildTestMuCapabilities({ + platform: 'ios', + deviceName: 'iPhone 16', + osVersion: '18.0', + buildName: 'run-1', + sessionName: 'lease-1', + deviceFeatures: { appiumVersion: '2.16.2' }, + configured: { 'lt:options': { tunnel: true } }, + }); + const ltOptions = capabilities['lt:options'] as Record; + assert.equal(ltOptions.appiumVersion, '2.16.2'); + assert.equal(ltOptions.tunnel, true); + assert.equal(ltOptions.build, 'run-1'); + assert.equal(ltOptions.platformName, 'iOS'); + assert.equal('appium:app' in capabilities, false); + + const unpinned = buildTestMuCapabilities({ + platform: 'ios', + deviceName: 'iPhone 16', + osVersion: '18.0', + buildName: 'run-1', + sessionName: 'lease-1', + }); + assert.equal('appiumVersion' in (unpinned['lt:options'] as Record), false); +}); + +test('a configured lt:options cannot turn off the W3C dialect', () => { + const capabilities = buildTestMuCapabilities({ + platform: 'android', + deviceName: 'Pixel 8', + osVersion: '14', + buildName: 'run-1', + sessionName: 'lease-1', + configured: { 'lt:options': { w3c: false, tunnel: true } }, + }); + const ltOptions = capabilities['lt:options'] as Record; + assert.equal(ltOptions.w3c, true); + assert.equal(ltOptions.tunnel, true); +}); + +// A configured `isRealMobile` would silently move the session to the real-device pool, which +// bills differently. +test('a configured lt:options cannot move the session off the virtual-device pool', () => { + const capabilities = buildTestMuCapabilities({ + platform: 'ios', + deviceName: 'iPhone 16', + osVersion: '18.0', + buildName: 'run-1', + sessionName: 'lease-1', + configured: { 'lt:options': { isRealMobile: true, tunnel: true } }, + }); + const ltOptions = capabilities['lt:options'] as Record; + assert.equal(ltOptions.isRealMobile, false); + assert.equal(ltOptions.tunnel, true); +}); + +test('TestMu uploads go to the virtual-device upload API unless an endpoint is configured', async () => { + const tempDir = await mkdtempForTest('agent-device-testmu-upload-endpoint-'); + const appPath = path.join(tempDir, 'MyApp.apk'); + const endpoints: string[] = []; + try { + await fs.writeFile(appPath, 'placeholder'); + globalThis.fetch = async (input) => { + endpoints.push(String(input)); + return jsonResponse({ app_url: 'lt://APP1' }); + }; + await uploadTestMuApp(appPath, auth); + await uploadTestMuAppFromUrl('https://example.test/App.apk', auth); + await uploadTestMuApp(appPath, { ...auth, endpoint: 'https://upload.test/virtual' }); + assert.deepEqual(endpoints, [ + 'https://manual-api.lambdatest.com/app/upload/virtualDevice', + 'https://manual-api.lambdatest.com/app/upload/virtualDevice', + 'https://upload.test/virtual', + ]); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + +test('TestMu upload reads the lt:// reference and aborts while the request is in flight', async () => { + const tempDir = await mkdtempForTest('agent-device-testmu-upload-'); + const appPath = path.join(tempDir, 'App.apk'); + const controller = new AbortController(); + const abortReason = new Error('request cancelled during TestMu AI upload'); + try { + await fs.writeFile(appPath, 'placeholder'); + globalThis.fetch = async (_input, init) => + await new Promise((_resolve, reject) => { + assert.equal(init?.signal, controller.signal); + if (init?.signal?.aborted) { + reject(init.signal.reason); + return; + } + init?.signal?.addEventListener('abort', () => reject(init.signal?.reason), { once: true }); + }); + + const pending = uploadTestMuApp(appPath, auth, controller.signal); + await Promise.resolve(); + controller.abort(abortReason); + await assert.rejects(pending, (error: unknown) => error === abortReason); + + globalThis.fetch = async (_input, init) => { + const body = init?.body as FormData; + assert.ok(body.get('appFile') instanceof Blob); + assert.equal(body.get('name'), 'App'); + return jsonResponse({ app_id: 'APP123', name: 'App' }); + }; + assert.equal(await uploadTestMuApp(appPath, auth), 'lt://APP123'); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + +// iOS simulator builds are `.app` directories; the upload API only takes a file. +test('TestMu upload rejects an unzipped .app bundle before calling the upload API', async () => { + const tempDir = await mkdtempForTest('agent-device-testmu-app-dir-'); + const appPath = path.join(tempDir, 'Demo.app'); + try { + await fs.mkdir(appPath); + const fetchMock = vi.fn(); + globalThis.fetch = fetchMock; + await assert.rejects(uploadTestMuApp(appPath, auth), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'INVALID_ARGS'); + assert.match(String(error.details?.hint), /[Zz]ip the \.app bundle/); + return true; + }); + assert.equal(fetchMock.mock.calls.length, 0); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + +test('the install adapter uploads the local build and launches the hinted app id', async () => { + const tempDir = await mkdtempForTest('agent-device-testmu-install-'); + const appPath = path.join(tempDir, 'Demo.apk'); + try { + await fs.writeFile(appPath, 'placeholder'); + globalThis.fetch = async () => jsonResponse({ app_url: 'lt://APP77' }); + const uploadApp = createTestMuUploadApp(auth); + const result = await uploadApp({ + provider: 'testmu', + lease: {} as never, + device: {} as never, + app: 'com.example.demo', + appPath, + options: { packageNameHint: 'com.example.demo' }, + }); + assert.deepEqual(result, { + appReference: 'lt://APP77', + bundleId: undefined, + packageName: 'com.example.demo', + launchTarget: 'com.example.demo', + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + +test('TestMu passes lt:// ids through and has the upload API fetch a public URL', async () => { + const forms: FormData[] = []; + globalThis.fetch = async (_input, init) => { + forms.push(init?.body as FormData); + return jsonResponse({ app_id: 'APP9' }); + }; + assert.equal(await resolveTestMuAppReference('lt://APP1', auth), 'lt://APP1'); + assert.equal(forms.length, 0); + assert.equal( + await resolveTestMuAppReference('https://builds.example/App.apk', auth), + 'lt://APP9', + ); + assert.equal(forms[0]?.get('url'), 'https://builds.example/App.apk'); + await assert.rejects( + resolveTestMuAppReference('missing.apk', { ...auth, cwd: '/nonexistent' }), + /must be an lt:\/\/ app id, URL, or existing local app path/, + ); +}); + +test('TestMu upload accepts only an lt:// reference or a valid app id from the response', async () => { + const cases: Array<[unknown, string | undefined]> = [ + [{ app_url: 'lt://APP6' }, 'lt://APP6'], + [{ app_url: 'https://cdn.example/app.apk', app_id: 'APP5' }, 'lt://APP5'], + [{ app_id: 'lt://APP7' }, 'lt://APP7'], + [{ app_url: 'https://cdn.example/app.apk' }, undefined], + [{ app_url: 'lt://' }, undefined], + [{ app_id: 'bs://APP8' }, undefined], + ]; + for (const [body, expected] of cases) { + globalThis.fetch = async () => jsonResponse(body); + const pending = uploadTestMuAppFromUrl('https://builds.example/App.apk', auth); + if (expected) { + assert.equal(await pending, expected); + continue; + } + await assert.rejects(pending, (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.deepEqual(error.details?.response, body); + return true; + }); + } +}); + +test('TestMu URL upload hands the URL to the upload API and surfaces a failed upload', async () => { + globalThis.fetch = async (_input, init) => { + const body = init?.body as FormData; + assert.equal(body.get('url'), 'https://example.test/builds/App.apk'); + assert.equal(body.get('storage'), 'url'); + assert.equal(body.get('name'), 'App.apk'); + assert.equal(body.get('appFile'), null); + return jsonResponse({ app_url: 'lt://APP9' }); + }; + assert.equal( + await uploadTestMuAppFromUrl('https://example.test/builds/App.apk', auth), + 'lt://APP9', + ); + + globalThis.fetch = async () => jsonResponse({ message: 'invalid app' }, 400); + await assert.rejects( + uploadTestMuAppFromUrl('https://example.test/builds/App.apk', auth), + (error: unknown) => + error instanceof AppError && error.code === 'COMMAND_FAILED' && error.details?.status === 400, + ); +}); + +test('TestMu artifacts come from the jsend session payload and stay pending until a URL exists', async () => { + const calls: string[] = []; + globalThis.fetch = async (input, init) => { + calls.push(String(input)); + const headers = (init?.headers ?? {}) as Record; + assert.match(String(headers.Authorization), /^Basic /); + return jsonResponse({ + status: 'success', + data: { + test_id: 'SESSION1', + video_url: 'https://cdn.test/video.mp4', + appium_logs_url: 'https://api.test/sessions/SESSION1/log/appium', + device_logs_url: '', + }, + }); + }; + const result = await listTestMuCloudArtifacts('testmu', 'SESSION1', { + ...auth, + endpoint: 'https://api.test/mobile-automation/api/v1/', + }); + assert.deepEqual(calls, ['https://api.test/mobile-automation/api/v1/sessions/SESSION1']); + assert.equal(result?.status, 'ready'); + assert.deepEqual( + result?.cloudArtifacts.map((artifact) => [artifact.kind, artifact.url]), + [ + ['video', 'https://cdn.test/video.mp4'], + ['appium-log', 'https://api.test/sessions/SESSION1/log/appium'], + ['provider-session', 'https://appautomation.lambdatest.com/test?testID=SESSION1'], + ], + ); + + globalThis.fetch = async () => jsonResponse({ status: 'success', data: { test_id: 'SESSION1' } }); + const pending = await listTestMuCloudArtifacts('testmu', 'SESSION1', auth); + assert.equal(pending?.status, 'pending'); + assert.deepEqual(pending?.cloudArtifacts, []); +}); + +// Virtual-device session details carry the device log as `console_logs_url`. +test('TestMu reads the console log as the device log and falls back to device_logs_url', async () => { + globalThis.fetch = async () => + jsonResponse({ + status: 'success', + data: { + console_logs_url: 'https://api.test/sessions/SESSION1/log/console', + device_logs_url: 'https://api.test/sessions/SESSION1/log/device', + }, + }); + const consoleLog = await listTestMuCloudArtifacts('testmu', 'SESSION1', auth); + assert.deepEqual( + consoleLog?.cloudArtifacts + .filter((artifact) => artifact.kind === 'device-log') + .map((artifact) => artifact.url), + ['https://api.test/sessions/SESSION1/log/console'], + ); + + globalThis.fetch = async () => + jsonResponse({ + status: 'success', + data: { device_logs_url: 'https://api.test/sessions/SESSION1/log/device' }, + }); + const deviceLog = await listTestMuCloudArtifacts('testmu', 'SESSION1', auth); + assert.deepEqual( + deviceLog?.cloudArtifacts + .filter((artifact) => artifact.kind === 'device-log') + .map((artifact) => artifact.url), + ['https://api.test/sessions/SESSION1/log/device'], + ); +}); + +test('TestMu session details read as pending on 404 and fail typed on a body that is not JSON', async () => { + let signal: AbortSignal | undefined; + globalThis.fetch = async (_input, init) => { + signal = init?.signal ?? undefined; + return jsonResponse({ status: 'fail', message: 'session not found' }, 404); + }; + const notFound = await listTestMuCloudArtifacts('testmu', 'SESSION1', auth); + assert.equal(notFound?.status, 'pending'); + assert.deepEqual(notFound?.cloudArtifacts, []); + assert.ok(signal instanceof AbortSignal, 'session details lookup should carry a timeout'); + + globalThis.fetch = async () => new Response('Bad Gateway', { status: 502 }); + await assert.rejects( + listTestMuCloudArtifacts('testmu', 'SESSION1', auth), + (error: unknown) => + error instanceof AppError && error.code === 'COMMAND_FAILED' && error.details?.status === 502, + ); + + globalThis.fetch = async () => new Response('', { status: 200 }); + await assert.rejects( + listTestMuCloudArtifacts('testmu', 'SESSION1', auth), + (error: unknown) => + error instanceof AppError && error.code === 'COMMAND_FAILED' && error.details?.status === 200, + ); +}); + +test('TestMu session details require the jsend data envelope', async () => { + globalThis.fetch = async () => jsonResponse({ video_url: 'https://cdn.test/video.mp4' }); + await assert.rejects( + listTestMuCloudArtifacts('testmu', 'SESSION1', auth), + (error: unknown) => error instanceof AppError && error.code === 'COMMAND_FAILED', + ); +}); + +test('TestMu upload reports the HTTP status when the response is not JSON', async () => { + globalThis.fetch = async () => new Response('Bad Gateway', { status: 502 }); + await assert.rejects( + uploadTestMuAppFromUrl('https://example.test/builds/App.apk', auth), + (error: unknown) => + error instanceof AppError && error.code === 'COMMAND_FAILED' && error.details?.status === 502, + ); + + globalThis.fetch = async () => new Response('', { status: 200 }); + await assert.rejects( + uploadTestMuAppFromUrl('https://example.test/builds/App.apk', auth), + (error: unknown) => + error instanceof AppError && error.code === 'COMMAND_FAILED' && error.details?.status === 200, + ); +}); + +test('TestMu session details lookup types a timeout and a network failure', async () => { + const timeout = new DOMException('The operation was aborted due to timeout', 'TimeoutError'); + globalThis.fetch = async () => { + throw timeout; + }; + await assert.rejects(listTestMuCloudArtifacts('testmu', 'SESSION1', auth), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.match(error.message, /TestMu AI session details lookup failed/); + assert.match(String(error.details?.hint), /retry/); + assert.equal(error.cause, timeout); + return true; + }); + + globalThis.fetch = async () => { + throw new TypeError('fetch failed'); + }; + await assert.rejects( + listTestMuCloudArtifacts('testmu', 'SESSION1', auth), + (error: unknown) => error instanceof AppError && error.code === 'COMMAND_FAILED', + ); +}); + +function jsonResponse(value: unknown, status = 200): Response { + return new Response(JSON.stringify(value), { status }); +} diff --git a/packages/provider-webdriver/src/testmu.ts b/packages/provider-webdriver/src/testmu.ts new file mode 100644 index 0000000000..a06b14a0dd --- /dev/null +++ b/packages/provider-webdriver/src/testmu.ts @@ -0,0 +1,264 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { CloudArtifact, CloudArtifactsResult } from '@agent-device/contracts/observability'; +import type { CloudWebDriverPlatform, CloudWebDriverUploadApp } from './runtime.ts'; +import { AppError } from '@agent-device/kernel/errors'; +import { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; +import { + appFileUploadForm, + asRecord, + createHubUploadApp, + fetchProviderSessionDetails, + postHubAppUpload, + resolveHubAppReference, + trimTrailingSlash, +} from './webdriver-utils.ts'; + +/** + * TestMu session, upload, and artifact mechanics. Loaded on demand by the provider definition; + * `isRealMobile: false` in `lt:options` is what routes a session to the virtual-device pool, and + * the hostnames still carry the lambdatest.com brand. + */ +const TESTMU_APP_UPLOAD_ENDPOINT = 'https://manual-api.lambdatest.com/app/upload/virtualDevice'; +export const TESTMU_APPS_ENDPOINT = 'https://manual-api.lambdatest.com/app/data'; +export const TESTMU_API_ENDPOINT = 'https://mobile-api.lambdatest.com/mobile-automation/api/v1'; +const TESTMU_DASHBOARD_TEST_URL = 'https://appautomation.lambdatest.com/test?testID='; + +export type TestMuCapabilitiesOptions = { + platform: CloudWebDriverPlatform; + deviceName: string; + osVersion: string; + app?: string; + projectName?: string; + buildName: string; + sessionName: string; + /** Vendor device-feature capabilities, already projected onto their `lt:options` keys. */ + deviceFeatures?: Record; + configured?: Record; +}; + +export type TestMuAuth = { + username: string; + accessKey: string; +}; + +export type TestMuSessionDetailsOptions = TestMuAuth & { + clientVersion: string; + endpoint?: string | URL; +}; + +export async function listTestMuCloudArtifacts( + provider: string, + providerSessionId: string | undefined, + options: TestMuSessionDetailsOptions, +): Promise { + if (!providerSessionId) return undefined; + const details = await fetchTestMuSessionDetails(providerSessionId, options); + const artifacts = mapTestMuArtifacts(provider, providerSessionId, details); + return cloudArtifactsReadyOrPending({ + provider, + providerSessionId, + artifacts, + pendingMessage: 'TestMu AI artifacts are not ready yet.', + }); +} + +export type TestMuUploadOptions = TestMuAuth & { + clientVersion: string; + endpoint?: string | URL; +}; + +/** Uploads a local `.apk`, `.aab`, `.ipa`, or zipped simulator `.app` and returns its `lt://` reference. */ +export async function uploadTestMuApp( + appPath: string, + options: TestMuUploadOptions, + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted(); + if (!(await fs.stat(appPath)).isFile()) { + throw new AppError('INVALID_ARGS', `TestMu AI can only upload an app file: ${appPath}`, { + appPath, + hint: 'Zip the .app bundle of an iOS simulator build and pass the .zip.', + }); + } + const form = await appFileUploadForm(appPath, 'appFile'); + form.set('name', path.parse(appPath).name); + return await postTestMuUpload(form, options, signal); +} + +/** Has TestMu fetch a public app URL itself, returning its `lt://` reference. */ +export async function uploadTestMuAppFromUrl( + url: string, + options: TestMuUploadOptions, + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted(); + const form = new FormData(); + form.set('url', url); + form.set('storage', 'url'); + form.set('name', path.basename(new URL(url).pathname) || 'app'); + return await postTestMuUpload(form, options, signal); +} + +async function postTestMuUpload( + form: FormData, + options: TestMuUploadOptions, + signal?: AbortSignal, +): Promise { + return await postHubAppUpload( + form, + { + service: 'TestMu AI', + endpoint: options.endpoint ?? TESTMU_APP_UPLOAD_ENDPOINT, + clientVersion: options.clientVersion, + auth: options, + readAppReference: readTestMuAppReference, + }, + signal, + ); +} + +export function createTestMuUploadApp(options: TestMuUploadOptions): CloudWebDriverUploadApp { + return createHubUploadApp( + async (appPath, signal) => await uploadTestMuApp(appPath, options, signal), + ); +} + +/** The hub only accepts `lt://` references, so a public URL is handed to the upload API to fetch. */ +export async function resolveTestMuAppReference( + app: string, + options: TestMuUploadOptions & { cwd?: string; signal?: AbortSignal }, +): Promise { + return await resolveHubAppReference({ + service: 'TestMu AI', + app, + cwd: options.cwd, + referenceScheme: 'lt://', + referenceLabel: 'an lt:// app id', + uploadFile: async (appPath, signal) => await uploadTestMuApp(appPath, options, signal), + uploadUrl: async (url, signal) => await uploadTestMuAppFromUrl(url, options, signal), + signal: options.signal, + }); +} + +/** + * Builds the W3C `alwaysMatch` capabilities for a TestMu virtual-device session. + * + * Standard Appium keys stay `appium:`-prefixed at the top level; everything TestMu-specific lives + * in `lt:options`. `isRealMobile: false` selects an emulator or simulator, and `w3c: true` keeps + * the hub on the W3C dialect agent-device speaks. `appiumVersion` is sent only when the caller + * pins one; otherwise TestMu AI starts its default server for the device. + */ +export function buildTestMuCapabilities( + options: TestMuCapabilitiesOptions, +): Record { + const { 'lt:options': configuredLtOptions, ...configured } = options.configured ?? {}; + const deviceFeatures = options.deviceFeatures ?? {}; + return { + 'appium:deviceName': options.deviceName, + 'appium:platformVersion': options.osVersion, + ...(options.app ? { 'appium:app': options.app } : {}), + ...configured, + // Merged per key, never assigned: a configured `lt:options` must not drop the labels below. + 'lt:options': { + platformName: options.platform === 'ios' ? 'iOS' : 'Android', + deviceName: options.deviceName, + platformVersion: options.osVersion, + ...(options.app ? { app: options.app } : {}), + ...(options.projectName ? { project: options.projectName } : {}), + build: options.buildName, + name: options.sessionName, + video: true, + devicelog: true, + ...deviceFeatures, + ...(asRecord(configuredLtOptions) ?? {}), + // A configured value cannot switch the device pool or drop the W3C dialect agent-device speaks. + isRealMobile: false, + w3c: true, + }, + }; +} + +export function isTestMuAppReference(value: string): boolean { + return value.startsWith('lt://'); +} + +async function fetchTestMuSessionDetails( + sessionId: string, + options: TestMuSessionDetailsOptions, +): Promise> { + const endpoint = new URL( + `${trimTrailingSlash(String(options.endpoint ?? TESTMU_API_ENDPOINT))}/sessions/${encodeURIComponent(sessionId)}`, + ); + let json: Record; + try { + json = await fetchProviderSessionDetails(endpoint, { + clientVersion: options.clientVersion, + auth: options, + service: 'TestMu AI', + }); + } catch (error) { + // Details are published a little after the session ends; until then the API answers 404. + if (error instanceof AppError && error.details?.status === 404) return {}; + throw error; + } + // The API wraps the session in a jsend envelope: `{ status, data: {...}, message }`. + const details = asRecord(json.data); + if (!details) { + throw new AppError('COMMAND_FAILED', 'TestMu AI session details response had no data.', { + response: json, + }); + } + return details; +} + +function mapTestMuArtifacts( + provider: string, + providerSessionId: string, + details: Record, +): CloudArtifact[] { + // Virtual-device sessions report the device log as `console_logs_url`. + const deviceLogField = + typeof details.console_logs_url === 'string' && details.console_logs_url.length > 0 + ? 'console_logs_url' + : 'device_logs_url'; + const fromDetails = ( + [ + ['video_url', 'video', 'Session video'], + ['appium_logs_url', 'appium-log', 'Appium logs'], + [deviceLogField, 'device-log', 'Device logs'], + ['network_logs_url', 'raw', 'Network logs'], + ['command_logs_url', 'automation-log', 'Command logs'], + ['screenshot_url', 'raw', 'Screenshots'], + ] as const + ).map(([field, kind, name]) => + urlArtifactFromDetails(provider, providerSessionId, details, field, kind, name), + ); + const dashboard: CloudArtifact = { + provider, + providerSessionId, + kind: 'provider-session', + name: 'TestMu AI dashboard', + url: `${TESTMU_DASHBOARD_TEST_URL}${encodeURIComponent(providerSessionId)}`, + availability: 'ready', + }; + const ready = fromDetails.filter((artifact): artifact is CloudArtifact => artifact !== undefined); + // The dashboard link alone does not mean the session finished uploading; keep "pending" until + // the API reports at least one artifact URL. + return ready.length > 0 ? [...ready, dashboard] : []; +} + +const TESTMU_APP_ID = /^[\w.-]+$/; + +/** The upload answers with `app_url` (`lt://…`) and/or a bare `app_id`; anything else is a failed upload. */ +function readTestMuAppReference(value: unknown): string | undefined { + const { app_url: appUrl, app_id: appId } = asRecord(value) ?? {}; + if (typeof appUrl === 'string' && isValidTestMuAppReference(appUrl)) return appUrl; + if (typeof appId !== 'string') return undefined; + const reference = isTestMuAppReference(appId) ? appId : `lt://${appId}`; + return isValidTestMuAppReference(reference) ? reference : undefined; +} + +function isValidTestMuAppReference(value: string): boolean { + return isTestMuAppReference(value) && TESTMU_APP_ID.test(value.slice('lt://'.length)); +} diff --git a/packages/provider-webdriver/src/webdriver-utils.ts b/packages/provider-webdriver/src/webdriver-utils.ts index 4b750d5439..88724187dc 100644 --- a/packages/provider-webdriver/src/webdriver-utils.ts +++ b/packages/provider-webdriver/src/webdriver-utils.ts @@ -269,7 +269,7 @@ async function readProviderJsonBody(response: Response): Promise { } } -/** `1.0` and `1` name the same OS release on BrowserStack's catalog. */ +/** `1.0` and `1` name the same OS release on BrowserStack's catalog; TestMu's hub matches spellings exactly. */ export function sameOsVersion(left: string, right: string): boolean { const normalize = (value: string) => value.replace(/(?:\.0)+$/, ''); return normalize(left) === normalize(right); diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index 3f41c6c817..de24acb872 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -756,6 +756,7 @@ test('the real tree parses, declares, and passes R11', () => { assert.deepEqual([...providerWebDriverPackage.exportTargets.keys()].sort(), [ '@agent-device/provider-webdriver', '@agent-device/provider-webdriver/providers', + '@agent-device/provider-webdriver/testmu-device-features', ]); assert.deepEqual([...providerWebDriverPackage.workspaceDependencies].sort(), [ '@agent-device/capture-kit', diff --git a/src/__tests__/cloud-connect-profile.test.ts b/src/__tests__/cloud-connect-profile.test.ts index fb3f486168..3b0118d9e9 100644 --- a/src/__tests__/cloud-connect-profile.test.ts +++ b/src/__tests__/cloud-connect-profile.test.ts @@ -70,24 +70,37 @@ beforeEach(() => { }, app: { status: 'verified', reference: options.app }, } - : { - provider: 'aws-device-farm', - service: 'AWS Device Farm', - verificationMessage: 'Credentials, project, and device verified.', - project: { name: 'Agent Device', reference: options.projectArn }, - device: { - status: 'verified', - name: 'iPhone 15', - reference: options.deviceArn, - platform: options.platform, - osVersion: '17', - }, - app: { - status: 'missing', - message: - 'No app upload is attached; AWS Device Farm does not support install after allocation.', + : options.provider === 'testmu' + ? { + provider: 'testmu', + service: 'TestMu AI', + verificationMessage: 'Credentials, virtual device, and uploaded app verified.', + device: { + status: 'verified', + name: options.deviceName, + platform: options.platform, + osVersion: options.osVersion, + }, + app: { status: 'verified', reference: options.app }, + } + : { + provider: 'aws-device-farm', + service: 'AWS Device Farm', + verificationMessage: 'Credentials, project, and device verified.', + project: { name: 'Agent Device', reference: options.projectArn }, + device: { + status: 'verified', + name: 'iPhone 15', + reference: options.deviceArn, + platform: options.platform, + osVersion: '17', + }, + app: { + status: 'missing', + message: + 'No app upload is attached; AWS Device Farm does not support install after allocation.', + }, }, - }, ); }); diff --git a/src/__tests__/cloud-connect-testmu.test.ts b/src/__tests__/cloud-connect-testmu.test.ts new file mode 100644 index 0000000000..1255a4af6a --- /dev/null +++ b/src/__tests__/cloud-connect-testmu.test.ts @@ -0,0 +1,192 @@ +import { afterEach, beforeEach, test, vi } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { connectCommand } from '../cli/commands/connection.ts'; +import { + readActiveConnectionState, + type RemoteConnectionState, +} from '../remote/remote-connection-state.ts'; +import type { AgentDeviceClient } from '../agent-device-client.ts'; +import { resolveCloudWebDriverConnectProfile } from '../cli/connection/cloud-webdriver-profile.ts'; +import { AppError } from '@agent-device/kernel/errors'; +import { providerWebDriver } from '../provider-webdriver.ts'; +import { mkdtempForTestSync } from './test-utils/tmp-dir.ts'; + +vi.mock('../provider-webdriver.ts', () => ({ + providerWebDriver: { verifyConnection: vi.fn() }, +})); + +afterEach(() => { + vi.clearAllMocks(); + vi.unstubAllEnvs(); +}); + +const mockedVerifyWebDriverConnection = vi.mocked(providerWebDriver.verifyConnection); + +beforeEach(() => { + mockedVerifyWebDriverConnection.mockImplementation(async (options) => { + assert.equal(options.provider, 'testmu'); + return { + provider: 'testmu', + service: 'TestMu AI', + verificationMessage: 'Credentials, device, and uploaded app verified.', + device: { + status: 'verified', + name: options.deviceName, + platform: options.platform, + osVersion: options.osVersion, + }, + app: { status: 'verified', reference: options.app }, + }; + }); +}); + +test('connect testmu generates a local provider profile and verifies the virtual device', async () => { + const tempRoot = mkdtempForTestSync('agent-device-connect-testmu-'); + const stateDir = path.join(tempRoot, '.state'); + vi.stubEnv('LT_USERNAME', 'lt-user'); + vi.stubEnv('LT_ACCESS_KEY', 'lt-key'); + + try { + await connectWithGeneratedProviderProfile({ + stateDir, + positionals: ['testmu'], + flags: { + platform: 'ios', + device: 'iPhone 16', + providerOsVersion: '18.0', + providerApp: 'lt://APP1', + providerBuild: 'build-a', + }, + }); + + assert.deepEqual(mockedVerifyWebDriverConnection.mock.calls[0]?.[0], { + provider: 'testmu', + username: 'lt-user', + accessKey: 'lt-key', + platform: 'ios', + deviceName: 'iPhone 16', + osVersion: '18.0', + app: 'lt://APP1', + }); + const state = readRequiredActiveState(stateDir); + assert.equal(state.tenant, 'testmu'); + assert.equal(state.leaseProvider, 'testmu'); + assert.match(state.remoteConfigPath, /generated\/testmu-[a-f0-9]{16}\.json$/); + const generated = readGeneratedConfig(state.remoteConfigPath); + assert.equal(generated.providerApp, 'lt://APP1'); + assert.equal(generated.providerOsVersion, '18.0'); + assert.equal(generated.providerBuild, 'build-a'); + assert.equal(JSON.stringify(generated).includes('lt-key'), false); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test('connect testmu verifies against TESTMU_API_ENDPOINT', async () => { + const tempRoot = mkdtempForTestSync('agent-device-connect-testmu-endpoint-'); + vi.stubEnv('LT_USERNAME', 'lt-user'); + vi.stubEnv('LT_ACCESS_KEY', 'lt-key'); + vi.stubEnv('TESTMU_API_ENDPOINT', 'https://staging.testmu.test/mobile-automation/api/v1'); + + try { + await connectWithGeneratedProviderProfile({ + stateDir: path.join(tempRoot, '.state'), + positionals: ['testmu'], + flags: { + platform: 'android', + device: 'Pixel 8', + providerOsVersion: '14', + providerApp: 'lt://APP1', + }, + }); + + const options = mockedVerifyWebDriverConnection.mock.calls[0]?.[0]; + assert.equal(options?.provider, 'testmu'); + assert.equal(options.apiEndpoint, 'https://staging.testmu.test/mobile-automation/api/v1'); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test('connect testmu rejects BrowserStack network and re-sign flags before saving a profile', () => { + const tempRoot = mkdtempForTestSync('agent-device-connect-testmu-reject-'); + + try { + assert.throws( + () => + resolveCloudWebDriverConnectProfile({ + provider: 'testmu', + stateDir: path.join(tempRoot, '.state'), + cwd: tempRoot, + env: { LT_USERNAME: 'lt-user', LT_ACCESS_KEY: 'lt-key' }, + flags: { + json: false, + help: false, + version: false, + platform: 'ios', + device: 'iPhone 16', + providerOsVersion: '18.0', + providerApp: 'lt://APP1', + providerNetworkProfile: '3g-lossy', + providerNoResignApp: true, + }, + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'INVALID_ARGS'); + assert.match(error.message, /not supported by TestMu AI/); + assert.deepEqual(error.details?.flags, [ + '--provider-network-profile', + '--provider-no-resign-app', + ]); + return true; + }, + ); + assert.equal(fs.existsSync(path.join(tempRoot, '.state')), false); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +async function connectWithGeneratedProviderProfile(options: { + stateDir: string; + positionals: string[]; + flags: Partial[0]['flags']>; +}): Promise { + const stdoutWrite = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + try { + await connectCommand({ + positionals: options.positionals, + flags: { + json: true, + help: false, + version: false, + stateDir: options.stateDir, + ...options.flags, + }, + client: {} as AgentDeviceClient, + }); + } finally { + stdoutWrite.mockRestore(); + } +} + +function readGeneratedConfig(configPath: string): { + providerApp?: string; + providerOsVersion?: string; + providerBuild?: string; +} { + return JSON.parse(fs.readFileSync(configPath, 'utf8')) as { + providerApp?: string; + providerOsVersion?: string; + providerBuild?: string; + }; +} + +function readRequiredActiveState(stateDir: string): RemoteConnectionState { + const state = readActiveConnectionState({ stateDir }); + assert.ok(state); + return state; +} diff --git a/src/cli/connection/cloud-webdriver-profile.ts b/src/cli/connection/cloud-webdriver-profile.ts index ffa64093af..031d48c8c7 100644 --- a/src/cli/connection/cloud-webdriver-profile.ts +++ b/src/cli/connection/cloud-webdriver-profile.ts @@ -4,6 +4,7 @@ import { rejectBrowserStackOnlyDeviceFeatures, type CloudWebDriverKnownProviderName, } from '@agent-device/provider-webdriver'; +import { rejectUnsupportedTestMuDeviceFeatures } from '@agent-device/provider-webdriver/testmu-device-features'; import type { RemoteConfigProfile } from '../../remote/remote-config-schema.ts'; import { AppError } from '@agent-device/kernel/errors'; import type { PlatformSelector } from '@agent-device/kernel/device'; @@ -70,6 +71,10 @@ const CLOUD_WEBDRIVER_CONNECT_PROFILE_BUILDERS: readonly { provider: CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, buildProfileFields: awsDeviceFarmProfileFields, }, + { + provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, + buildProfileFields: testMuProfileFields, + }, ]; function requireConnectProfileBuilder( @@ -82,29 +87,68 @@ function requireConnectProfileBuilder( throw new AppError('INVALID_ARGS', `Unsupported cloud WebDriver provider "${provider}".`); } +/** A hosted Appium hub picks its device by exact name + OS version and installs one app reference. */ +type HubProviderProfile = { + command: string; + label: string; + credentialEnv: readonly [string, string]; + /** Scheme of the provider's own app references, e.g. `bs://` or `lt://`. */ + appScheme: string; + appHint: string; +}; + +const BROWSERSTACK_HUB_PROFILE: HubProviderProfile = { + command: 'connect browserstack', + label: 'BrowserStack', + credentialEnv: ['BROWSERSTACK_USERNAME', 'BROWSERSTACK_ACCESS_KEY'], + appScheme: 'bs://', + appHint: '', +}; + +const TESTMU_HUB_PROFILE: HubProviderProfile = { + command: 'connect testmu', + label: 'TestMu AI', + credentialEnv: ['LT_USERNAME', 'LT_ACCESS_KEY'], + appScheme: 'lt://', + appHint: '', +}; + function browserStackProfileFields(options: { flags: CliFlags; env?: EnvMap; cwd: string; }): RemoteConfigProfile { - requireEnv(options.env, 'BROWSERSTACK_USERNAME', 'connect browserstack'); - requireEnv(options.env, 'BROWSERSTACK_ACCESS_KEY', 'connect browserstack'); + return hubProviderProfileFields(BROWSERSTACK_HUB_PROFILE, options); +} + +function testMuProfileFields(options: { + flags: CliFlags; + env?: EnvMap; + cwd: string; +}): RemoteConfigProfile { + rejectUnsupportedTestMuDeviceFeatures(options.flags); + return hubProviderProfileFields(TESTMU_HUB_PROFILE, options); +} + +function hubProviderProfileFields( + hub: HubProviderProfile, + options: { flags: CliFlags; env?: EnvMap; cwd: string }, +): RemoteConfigProfile { + for (const name of hub.credentialEnv) requireEnv(options.env, name, hub.command); const platform = requireCloudWebDriverPlatform( options.flags.platform, - 'connect browserstack requires --platform ios|android.', - ); - const device = requireFlag( - options.flags.device, - 'connect browserstack requires --device .', + `${hub.command} requires --platform ios|android.`, ); + const device = requireFlag(options.flags.device, `${hub.command} requires --device .`); const providerOsVersion = requireFlag( options.flags.providerOsVersion, - 'connect browserstack requires --provider-os-version .', + `${hub.command} requires --provider-os-version .`, ); - const providerApp = normalizeBrowserStackAppReference( + const providerApp = normalizeHubAppReference( + hub, requireFlag( options.flags.providerApp, - 'connect browserstack requires --provider-app .', + `${hub.command} requires --provider-app ${hub.appHint}.`, ), options.cwd, ); @@ -120,15 +164,15 @@ function browserStackProfileFields(options: { }; } -function normalizeBrowserStackAppReference(app: string, cwd: string): string { - if (app.startsWith('bs://') || /^https?:\/\//i.test(app)) return app; +function normalizeHubAppReference(hub: HubProviderProfile, app: string, cwd: string): string { + if (app.startsWith(hub.appScheme) || /^https?:\/\//i.test(app)) return app; const resolvedPath = path.resolve(cwd, app); try { if (fs.statSync(resolvedPath).isFile()) return resolvedPath; } catch { // Report one stable profile error below. } - throw new AppError('INVALID_ARGS', `BrowserStack app file not found: ${resolvedPath}`); + throw new AppError('INVALID_ARGS', `${hub.label} app file not found: ${resolvedPath}`); } function awsDeviceFarmProfileFields(options: { diff --git a/src/cli/connection/connect-provider-adapters.ts b/src/cli/connection/connect-provider-adapters.ts index 27d9c5c564..a83c548365 100644 --- a/src/cli/connection/connect-provider-adapters.ts +++ b/src/cli/connection/connect-provider-adapters.ts @@ -69,6 +69,10 @@ const CONNECT_PROVIDER_ADAPTERS = { resolveCloudWebDriverConnectProfile({ provider: 'aws-device-farm', ...context }), verify: verifyAwsDeviceFarm, }, + testmu: { + resolve: (context) => resolveCloudWebDriverConnectProfile({ provider: 'testmu', ...context }), + verify: verifyTestMu, + }, limrun: { resolve: resolveLimrunConnectProfile, verify: verifyLimrun, @@ -153,6 +157,25 @@ async function verifyBrowserStack( }); } +async function verifyTestMu( + context: Pick, +): Promise { + const { flags, env } = context; + return await providerWebDriver.verifyConnection({ + provider: 'testmu', + username: requiredResolvedValue(env.LT_USERNAME, 'TestMu AI profile missed LT_USERNAME.'), + accessKey: requiredResolvedValue(env.LT_ACCESS_KEY, 'TestMu AI profile missed LT_ACCESS_KEY.'), + platform: requiredResolvedPlatform(flags.platform, 'TestMu AI'), + deviceName: requiredResolvedValue(flags.device, 'TestMu AI profile missed device.'), + osVersion: requiredResolvedValue( + flags.providerOsVersion, + 'TestMu AI profile missed OS version.', + ), + app: requiredResolvedValue(flags.providerApp, 'TestMu AI profile missed app.'), + ...(env.TESTMU_API_ENDPOINT ? { apiEndpoint: env.TESTMU_API_ENDPOINT } : {}), + }); +} + async function verifyAwsDeviceFarm( context: Pick, ): Promise { diff --git a/src/cli/connection/provider-policy.ts b/src/cli/connection/provider-policy.ts index ebb86bc6d4..1756751af5 100644 --- a/src/cli/connection/provider-policy.ts +++ b/src/cli/connection/provider-policy.ts @@ -33,6 +33,7 @@ export function connectProviderNamesForError(): string { 'proxy', CLOUD_WEBDRIVER_PROVIDERS.browserStack, CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, + CLOUD_WEBDRIVER_PROVIDERS.testMu, 'limrun', ].join(', '); } diff --git a/src/commands/management/artifacts.ts b/src/commands/management/artifacts.ts index 11cea82c1b..1fbb683529 100644 --- a/src/commands/management/artifacts.ts +++ b/src/commands/management/artifacts.ts @@ -11,7 +11,9 @@ const artifactsCommandMetadata = defineFieldCommandMetadata( 'artifacts', 'List daemon or cloud provider artifacts for an active or completed session.', { - provider: stringField('Cloud provider name, for example browserstack or aws-device-farm.'), + provider: stringField( + 'Cloud provider name, for example browserstack, aws-device-farm, or testmu.', + ), providerSessionId: stringField('Cloud provider session id or ARN.'), }, ); diff --git a/src/commands/schema/cli-help-topics.test.ts b/src/commands/schema/cli-help-topics.test.ts index 2364272b00..cc92b8b6e7 100644 --- a/src/commands/schema/cli-help-topics.test.ts +++ b/src/commands/schema/cli-help-topics.test.ts @@ -436,7 +436,17 @@ test('usageForCommand resolves remote help topic', async () => { assert.match(help, /AGENT_DEVICE_HTTP_AUTH_HOOK configured treats HTTP requests as remote/); assert.match(help, /host-path install sources are rejected/); assert.match(help, /uploaded artifacts remain supported/); - assert.match(help, /Limrun, BrowserStack, and AWS Device Farm through local provider profiles/); + assert.match( + help, + /Limrun, BrowserStack, AWS Device Farm, and TestMu AI through local provider profiles/, + ); + assert.match(help, /TestMu AI uses LT_USERNAME and LT_ACCESS_KEY/); + const testMuFlow = help.slice( + help.indexOf('TestMu AI virtual-device flow'), + help.indexOf('BrowserStack hosted-device flow'), + ); + assert.match(testMuFlow, /--device "iPhone 16" --provider-os-version 18\.0/); + assert.match(testMuFlow, /agent-device disconnect/); assert.match(help, /Limrun uses LIMRUN_API_KEY/); assert.match(help, /BrowserStack uses BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY/); assert.match(help, /Generated connection profiles store app\/device selectors and ARNs/); diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 24957278e4..1211c54f88 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -577,17 +577,18 @@ Providers: Direct proxy: agent-device connect proxy --daemon-base-url stores the shared proxy profile and client identity. BrowserStack: agent-device connect browserstack verifies credentials, the exact device, and a bs:// app reference, then stores a local provider profile. It does not create an App Automate session. AWS Device Farm: agent-device connect aws-device-farm verifies credentials and the exact project, device, and optional app upload, then stores a local provider profile. It does not create a remote access session. + TestMu AI: agent-device connect testmu verifies credentials, the exact virtual device (emulator or simulator) and OS version, and an lt:// app reference, then stores a local provider profile. It does not create a hub session. Limrun: agent-device connect limrun verifies access to the selected iOS or Android instance service, then stores a local provider profile. It does not create an instance. After direct-provider connect: Read the printed Device, App, Next, and workflow-note lines. They are also available as verification/device/app/liveSession/nextSteps/notes in --json output. - BrowserStack and AWS Device Farm create the hosted session on open. open needs the installed package or bundle identifier, not the app artifact name or ARN. + BrowserStack, AWS Device Farm, and TestMu AI create the hosted session on open. open needs the installed package or bundle identifier, not the app artifact name, ARN, or lt:// id. Before provider allocation, apps lists compatible uploaded app assets without creating an instance when the selected provider exposes a catalog. open creates the instance with that asset, resolves its installed app id, and launches it. install remains available when the app comes from a fresh local path or URL. AWS Device Farm cannot install after allocation. If connect reports no attached app, run its printed reconnect command, which includes --session --force, before open. Do not run devices as a pre-open catalog probe for direct providers; it can allocate the deferred provider session. Limrun is the exception for apps: before allocation it lists uploaded assets for the selected platform. Device cloud interfaces: - CLI is the canonical bootstrap path: connect limrun/browserstack/aws-device-farm, then use normal open/snapshot/click/close/artifacts/disconnect commands. + CLI is the canonical bootstrap path: connect limrun/browserstack/aws-device-farm/testmu, then use normal open/snapshot/click/close/artifacts/disconnect commands. JavaScript can skip persisted connect state by passing leaseProvider plus provider fields to createAgentDeviceClient or per-command options. MCP exposes operational tools such as open, snapshot, click, close, and artifacts. It does not expose connect/disconnect; run CLI connect first in the same state dir before relying on MCP tools. @@ -617,6 +618,15 @@ Cloud profile flow: agent-device snapshot agent-device disconnect +TestMu AI virtual-device flow (emulators and simulators): + LT_USERNAME=... LT_ACCESS_KEY=... + agent-device connect testmu --platform ios --device "iPhone 16" --provider-os-version 18.0 --provider-app lt://APP-id + agent-device open com.example.app + agent-device snapshot -i + agent-device close + agent-device artifacts --json + agent-device disconnect + BrowserStack hosted-device flow: BROWSERSTACK_USERNAME=... BROWSERSTACK_ACCESS_KEY=... agent-device connect browserstack --platform android --device "Google Pixel 8" --provider-os-version 14.0 --provider-app bs://app-id @@ -663,12 +673,12 @@ Rules: Use connect without --remote-config when the cloud control plane owns the connection profile. Prefer connect --remote-config over --daemon-base-url, --tenant, --run-id, and --lease-id when using a local profile. Use agent-device proxy for direct tunnel access to a Mac you control. Expose the printed proxy URL through cloudflared/ngrok, then run agent-device connect proxy with the tunnel URL and printed token before normal commands. - Use Limrun, BrowserStack, and AWS Device Farm through local provider profiles; they do not accept a remote agent-device daemon URL. - Device cloud credentials must be available before the command starts. Limrun uses LIMRUN_API_KEY. BrowserStack uses BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY. AWS Device Farm uses the AWS CLI credential chain, including CI-provided AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_SESSION_TOKEN, AWS profiles, or web identity role variables. + Use Limrun, BrowserStack, AWS Device Farm, and TestMu AI through local provider profiles; they do not accept a remote agent-device daemon URL. + Device cloud credentials must be available before the command starts. Limrun uses LIMRUN_API_KEY. BrowserStack uses BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY. TestMu AI uses LT_USERNAME and LT_ACCESS_KEY. AWS Device Farm uses the AWS CLI credential chain, including CI-provided AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_SESSION_TOKEN, AWS profiles, or web identity role variables. Direct-provider connect performs read-only provider calls and saves active connection state only after verification succeeds. It never creates a device, instance, App Automate session, or AWS remote access session. connect without --session always creates a fresh remote session and prints that session in its next-step commands. Concurrent callers must pass the returned --session on every command; the ambient active connection is only a single-workflow convenience. To replace an existing connection, pass its returned session explicitly with --session --force. --force without --session creates another fresh session and does not release or overwrite an unrelated active connection. - Prefer short-lived AWS role credentials in CI. Generated connection profiles store app/device selectors and ARNs, not Limrun API keys, BrowserStack access keys, or AWS credentials. + Prefer short-lived AWS role credentials in CI. Generated connection profiles store app/device selectors and ARNs, not Limrun API keys, BrowserStack or TestMu AI access keys, or AWS credentials. Limrun Android supports direct ADB port reverse for local Metro. Limrun iOS requires a public Metro/React DevTools URL because it cannot reach local host ports directly. After closing a device cloud session, run agent-device artifacts --json to retrieve provider video/log/dashboard URLs when the provider has made them available. connect proxy stores the connection profile and client identity. Proxy device leases are acquired on open and expire after five minutes without commands; devices may inspect proxy inventory without allocating. diff --git a/src/commands/schema/command-overrides.ts b/src/commands/schema/command-overrides.ts index 728c32ffd1..f69ccda125 100644 --- a/src/commands/schema/command-overrides.ts +++ b/src/commands/schema/command-overrides.ts @@ -64,7 +64,7 @@ const SCHEMA_ONLY_CLI_COMMAND_SCHEMAS = { 'Configure remote access without allocating a device. Direct providers validate credentials/resources before saving state and print the exact device/app preparation needed before open. AGENT_DEVICE_CLOUD_BASE_URL is the bridge/control-plane API origin; use AGENT_DEVICE_DAEMON_AUTH_TOKEN=adc_live_... for CI/service-token automation.', }, usageOverride: - 'connect [cloud|proxy|limrun|browserstack|aws-device-farm] [--remote-config ] [--daemon-base-url ] [--tenant ] [--run-id ] [--lease-id ] [--lease-backend ] [--force] [--no-login]', + 'connect [cloud|proxy|limrun|browserstack|aws-device-farm|testmu] [--remote-config ] [--daemon-base-url ] [--tenant ] [--run-id ] [--lease-id ] [--lease-backend ] [--force] [--no-login]', usageFlags: [], listUsageOverride: 'connect', positionalArgs: ['provider?'], From 95c21f7fc910cfaabd8dc17347c5ace74b840993 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 19:45:11 +0530 Subject: [PATCH 04/57] feat(provider-webdriver): support TestMu AI real devices TestMu AI serves real devices and virtual devices from the same Appium hub; `lt:options.isRealMobile` selects the pool. Add `--provider-device-type real|virtual` to choose it. The default is `virtual`, so existing `testmu` profiles and sessions keep their current behaviour. The value is a cloud provider profile field like the others. It is defined in contracts as PROVIDER_DEVICE_TYPES and carried through the lease_allocate projection, request overrides, the remote-config schema, `connect`, the CLI request flags, and the session doctor's provider keys. It reaches session preparation from a `connect testmu` profile and from `client.leases.allocate({ providerDeviceType })`. For `real`, TestMu AI sessions: - Set `isRealMobile: true`. It is applied after any configured `lt:options`, so configuration cannot switch pools. - Upload through the real-device upload API. It has its own override, TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT; TESTMU_APP_UPLOAD_ENDPOINT keeps redirecting virtual-device uploads only. An `.app` directory is refused with a hint to pass a signed .ipa. - Verify against the real-device catalog (`capability/generator?isVirtualDevice=false`). It lists devices directly under the platform key rather than under `app.devices`. The OS version must still match exactly, and real iOS devices are listed by major version, such as `18`. A catalog response without the expected pool shape fails with a typed error. - Look an lt:// id up in the real-device app list (`type=android` or `type=ios`), the same way virtual uploads are looked up under `emulator` or `simulator`. BrowserStack, AWS Device Farm, and Limrun refuse the flag at connect. BrowserStack and AWS Device Farm also refuse it at session preparation, which the typed client and hand-written profiles reach without `connect`. Co-Authored-By: Claude Opus 5.5 --- .fallowrc.json | 6 +- .../src/flag-definitions-connection.ts | 16 +- packages/command-registry/src/flag-groups.ts | 1 + .../src/__tests__/lease-scope.test.ts | 2 + packages/contracts/src/client-connection.ts | 1 + packages/contracts/src/facades/remote.ts | 3 +- packages/contracts/src/lease-scope.ts | 1 + .../contracts/src/remote-config-fields.ts | 5 + .../src/connection-verification.test.ts | 165 +++++++++++++++++- .../src/connection-verification.ts | 7 +- .../src/provider-definitions.ts | 35 +++- .../src/testmu-connection-verification.ts | 59 ++++--- .../src/testmu-device-features.test.ts | 34 ++++ .../src/testmu-device-features.ts | 41 ++++- .../provider-webdriver/src/testmu.test.ts | 76 ++++++-- packages/provider-webdriver/src/testmu.ts | 27 ++- scripts/integration-progress-model.ts | 1 + src/__tests__/client-leases.test.ts | 35 ++++ src/__tests__/cloud-connect-testmu.test.ts | 116 ++++++++++++ src/cli.ts | 1 + src/cli/commands/connection-runtime.ts | 1 + src/cli/connection/cloud-webdriver-profile.ts | 12 +- .../connection/connect-provider-adapters.ts | 1 + src/cli/connection/limrun-profile.ts | 2 + .../args-parse-provider-device-type.test.ts | 17 ++ src/commands/command-flags.ts | 1 + src/commands/schema/cli-help-topics.test.ts | 5 + src/commands/schema/cli-help.ts | 6 +- src/commands/schema/command-overrides.ts | 1 + src/daemon-client/daemon-client-rpc.test.ts | 2 + src/daemon/handlers/session-doctor-options.ts | 1 + src/remote/remote-config-schema.ts | 2 + .../cloud-webdriver-provider-adapters.test.ts | 81 +++++++++ .../daemon-http-lease-allocate.test.ts | 2 + 34 files changed, 696 insertions(+), 70 deletions(-) create mode 100644 src/__tests__/client-leases.test.ts create mode 100644 src/cli/parser/__tests__/args-parse-provider-device-type.test.ts diff --git a/.fallowrc.json b/.fallowrc.json index 494c2f42f2..865583d4d3 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -382,7 +382,11 @@ { "comment": "TestMu session preparation reads these off the lazy loadTestMuDeviceFeatures() import in packages/provider-webdriver/src/provider-definitions.ts, which keeps the package entry's eager closure unchanged; Fallow cannot connect the dynamic member reads.", "file": "packages/provider-webdriver/src/testmu-device-features.ts", - "exports": ["buildTestMuDeviceFeatureCapabilities", "readTestMuDeviceFeatureFields"] + "exports": [ + "buildTestMuDeviceFeatureCapabilities", + "readTestMuDeviceFeatureFields", + "readTestMuDeviceType" + ] }, { "comment": "Converting the contracts façades from `export *` to explicit named re-exports (the pin-table retirement) made these individually visible to --production analysis for the first time; a bare star previously hid them from this exact check. isRecord/IOS_SAFARI_BUNDLE_ID/REPLAY_DIVERGENCE_* have no production consumer. Kept rather than narrowed here so the façade's re-export surface stays byte-identical to the symbol set the retired pin table asserted — narrowing the surface is a follow-up with its own review, not a side effect of this mechanical conversion.", diff --git a/packages/command-registry/src/flag-definitions-connection.ts b/packages/command-registry/src/flag-definitions-connection.ts index 0c9a34b0f1..dd698ce7e3 100644 --- a/packages/command-registry/src/flag-definitions-connection.ts +++ b/packages/command-registry/src/flag-definitions-connection.ts @@ -1,4 +1,7 @@ -import { PROVIDER_DEVICE_ORIENTATIONS } from '@agent-device/contracts/remote'; +import { + PROVIDER_DEVICE_ORIENTATIONS, + PROVIDER_DEVICE_TYPES, +} from '@agent-device/contracts/remote'; import type { FlagDefinition } from './flag-types.ts'; export const CONNECTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ @@ -174,6 +177,17 @@ export const CONNECTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ projectConfig: false, recorded: false, }, + { + key: 'providerDeviceType', + names: ['--provider-device-type'], + type: 'enum', + enumValues: PROVIDER_DEVICE_TYPES, + usageLabel: '--provider-device-type real|virtual', + usageDescription: + 'TestMu AI device pool: real devices or virtual devices (emulators and simulators). Defaults to virtual', + projectConfig: false, + recorded: false, + }, { key: 'providerProject', names: ['--provider-project'], diff --git a/packages/command-registry/src/flag-groups.ts b/packages/command-registry/src/flag-groups.ts index c8dc16507d..f3f4ae1804 100644 --- a/packages/command-registry/src/flag-groups.ts +++ b/packages/command-registry/src/flag-groups.ts @@ -78,6 +78,7 @@ export const COMMON_COMMAND_SUPPORTED_FLAG_KEYS = flagKeys( 'device', 'providerApp', 'providerOsVersion', + 'providerDeviceType', 'providerProject', 'providerBuild', 'providerSessionName', diff --git a/packages/contracts/src/__tests__/lease-scope.test.ts b/packages/contracts/src/__tests__/lease-scope.test.ts index 3da34f2be7..2aa8ed53df 100644 --- a/packages/contracts/src/__tests__/lease-scope.test.ts +++ b/packages/contracts/src/__tests__/lease-scope.test.ts @@ -182,6 +182,7 @@ test('readLeaseAllocateProviderFlags carries the provider-allocation flags and d device: 'iPhone 15', providerApp: 'bs://abc', providerOsVersion: '17', + providerDeviceType: 'real', providerProject: 'MyProject', providerBuild: 'Build-1', providerSessionName: 'smoke', @@ -198,6 +199,7 @@ test('readLeaseAllocateProviderFlags carries the provider-allocation flags and d device: 'iPhone 15', providerApp: 'bs://abc', providerOsVersion: '17', + providerDeviceType: 'real', providerProject: 'MyProject', providerBuild: 'Build-1', providerSessionName: 'smoke', diff --git a/packages/contracts/src/client-connection.ts b/packages/contracts/src/client-connection.ts index 7568768a33..a0eee1368b 100644 --- a/packages/contracts/src/client-connection.ts +++ b/packages/contracts/src/client-connection.ts @@ -63,6 +63,7 @@ export type AgentDeviceRequestOverrides = Pick< | 'clientId' | 'providerApp' | 'providerOsVersion' + | 'providerDeviceType' | 'providerProject' | 'providerBuild' | 'providerSessionName' diff --git a/packages/contracts/src/facades/remote.ts b/packages/contracts/src/facades/remote.ts index 852c8272cb..bfaecc5086 100644 --- a/packages/contracts/src/facades/remote.ts +++ b/packages/contracts/src/facades/remote.ts @@ -14,10 +14,11 @@ export type { ProviderConnectionResource, ProviderConnectionVerification, } from '../provider-connection.ts'; -export { PROVIDER_DEVICE_ORIENTATIONS } from '../remote-config-fields.ts'; +export { PROVIDER_DEVICE_ORIENTATIONS, PROVIDER_DEVICE_TYPES } from '../remote-config-fields.ts'; export type { CloudProviderProfileFields, ProviderDeviceOrientation, + ProviderDeviceType, RemoteConfigMetroOptions, RemoteConnectionProfileFields, } from '../remote-config-fields.ts'; diff --git a/packages/contracts/src/lease-scope.ts b/packages/contracts/src/lease-scope.ts index 51bb05550c..9f27b661cc 100644 --- a/packages/contracts/src/lease-scope.ts +++ b/packages/contracts/src/lease-scope.ts @@ -212,6 +212,7 @@ const LEASE_ALLOCATE_PROVIDER_FLAG_KEYS = [ // The Cloud provider profile fields; pinned exhaustive against that vocabulary below. 'providerApp', 'providerOsVersion', + 'providerDeviceType', 'providerProject', 'providerBuild', 'providerSessionName', diff --git a/packages/contracts/src/remote-config-fields.ts b/packages/contracts/src/remote-config-fields.ts index 5ab0b6a6b2..a2fe91b62d 100644 --- a/packages/contracts/src/remote-config-fields.ts +++ b/packages/contracts/src/remote-config-fields.ts @@ -20,9 +20,14 @@ import type { MetroPrepareKind } from './metro.ts'; export const PROVIDER_DEVICE_ORIENTATIONS = ['portrait', 'landscape'] as const; export type ProviderDeviceOrientation = (typeof PROVIDER_DEVICE_ORIENTATIONS)[number]; +/** Device pool a hosted provider session is created in: physical devices or emulators/simulators. */ +export const PROVIDER_DEVICE_TYPES = ['real', 'virtual'] as const; +export type ProviderDeviceType = (typeof PROVIDER_DEVICE_TYPES)[number]; + export type CloudProviderProfileFields = { providerApp?: string; providerOsVersion?: string; + providerDeviceType?: ProviderDeviceType; providerProject?: string; providerBuild?: string; providerSessionName?: string; diff --git a/packages/provider-webdriver/src/connection-verification.test.ts b/packages/provider-webdriver/src/connection-verification.test.ts index 5b2d872493..fbd3978638 100644 --- a/packages/provider-webdriver/src/connection-verification.test.ts +++ b/packages/provider-webdriver/src/connection-verification.test.ts @@ -431,17 +431,134 @@ test('TestMu defers an lt:// reference it cannot find and a local path it will u }); }); -// The listing is keyed by runtime: an Android emulator upload is listed under `emulator` and an -// iOS simulator upload under `simulator`, never under the platform name. -test('TestMu checks an lt:// id against the virtual-device app list of its platform', async () => { +// The real-device catalog is keyed by platform at the top level, not under `app.devices`. +const testMuRealCatalog = { + android: { + brands: { + Google: [ + { name: 'Pixel 6', osVersion: ['12', '13', '14', '15', '16'] }, + { name: 'Pixel 8', osVersion: ['14'] }, + ], + }, + }, + ios: { + brands: { + Apple: [ + { name: 'iPhone 16', osVersion: ['18'] }, + { name: 'iPhone 15', osVersion: ['17', '18', '26'] }, + ], + }, + }, + roku: { brands: {} }, + tvos: { brands: {} }, +}; + +test('TestMu verifies a real device against the real-device catalog shape', async () => { + const fetchMock = vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(testMuRealCatalog) + : jsonResponse({ data: [{ app_id: 'APP1', name: 'MyApp.ipa' }], metaData: { total: 1 } }), + ); + vi.stubGlobal('fetch', fetchMock); + const { devicesEndpoint: _devicesEndpoint, ...defaultCatalog } = testMuOptions; + + const result = await createProvider().verifyConnection({ + ...defaultCatalog, + deviceType: 'real', + platform: 'ios', + deviceName: 'iPhone 16', + osVersion: '18', + }); + + assert.equal(result.verificationMessage, 'Credentials, real device, and uploaded app verified.'); + assert.deepEqual(result.device, { + status: 'verified', + name: 'iPhone 16', + platform: 'ios', + osVersion: '18', + }); + assert.equal( + String(fetchMock.mock.calls[0]?.[0]), + 'https://mobile-api.lambdatest.com/mobile-automation/api/v1/capability/generator?isVirtualDevice=false', + ); + + const android = await createProvider().verifyConnection({ + ...testMuOptions, + deviceType: 'real', + deviceName: 'Pixel 6', + osVersion: '14', + }); + assert.equal(android.device.name, 'Pixel 6'); +}); + +// Real iOS devices are listed by major version, so `18.0` is the wrong spelling for the real pool. +test('TestMu matches real-device OS versions exactly and lists what the device offers', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => jsonResponse(testMuRealCatalog)), + ); + const realIos = { + ...testMuOptions, + deviceType: 'real' as const, + platform: 'ios' as const, + deviceName: 'iPhone 15', + }; + await assert.rejects( + createProvider().verifyConnection({ ...realIos, deviceName: 'iPhone 16', osVersion: '18.0' }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'INVALID_ARGS'); + assert.match( + error.message, + /TestMu AI real device "iPhone 16" with ios 18\.0 is not available/, + ); + assert.match(error.message, /iPhone 16 offers 18\.$/); + return true; + }, + ); + await assert.rejects( + createProvider().verifyConnection({ ...realIos, osVersion: '16' }), + /iPhone 15 offers 17, 18, 26/, + ); +}); + +test('TestMu fails typed when a catalog does not have the selected pool shape', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => jsonResponse(testMuCatalog)), + ); + await assert.rejects( + createProvider().verifyConnection({ ...testMuOptions, deviceType: 'real' }), + (error: unknown) => + error instanceof Error && + (error as { code?: string }).code === 'COMMAND_FAILED' && + /real-device catalog response did not list devices/.test(error.message), + ); + vi.stubGlobal( + 'fetch', + vi.fn(async () => jsonResponse(testMuRealCatalog)), + ); + await assert.rejects( + createProvider().verifyConnection(testMuOptions), + /virtual-device catalog response did not list devices/, + ); +}); + +// The listing is keyed by pool: `emulator`/`simulator` hold virtual uploads, `android`/`ios` real +// ones, so an id must be looked up in the list of the pool the session will run on. +test('TestMu checks an lt:// id against the app list of the selected pool and platform', async () => { const cases = [ - { platform: 'android', deviceName: 'Pixel 8', osVersion: '14', listType: 'emulator' }, - { platform: 'ios', deviceName: 'iPhone 16', osVersion: '18.0', listType: 'simulator' }, + { deviceType: 'virtual', platform: 'android', deviceName: 'Pixel 8', listType: 'emulator' }, + { deviceType: 'virtual', platform: 'ios', deviceName: 'iPhone 16', listType: 'simulator' }, + { deviceType: 'real', platform: 'android', deviceName: 'Pixel 6', listType: 'android' }, + { deviceType: 'real', platform: 'ios', deviceName: 'iPhone 16', listType: 'ios' }, ] as const; - for (const { platform, deviceName, osVersion, listType } of cases) { + for (const { deviceType, platform, deviceName, listType } of cases) { const fetchMock = vi.fn(async (input) => { const url = String(input); - if (url.includes('capability/generator')) return jsonResponse(testMuCatalog); + if (url.includes('capability/generator')) { + return jsonResponse(deviceType === 'real' ? testMuRealCatalog : testMuCatalog); + } return jsonResponse({ data: new URL(url).searchParams.get('type') === listType ? [{ app_id: 'APP1' }] : [], }); @@ -449,11 +566,19 @@ test('TestMu checks an lt:// id against the virtual-device app list of its platf vi.stubGlobal('fetch', fetchMock); const result = await createProvider().verifyConnection({ ...testMuOptions, + deviceType, platform, deviceName, - osVersion, + osVersion: + deviceType === 'real' + ? platform === 'ios' + ? '18' + : '14' + : platform === 'ios' + ? '18.0' + : '14', }); - assert.equal(result.app.status, 'verified', platform); + assert.equal(result.app.status, 'verified', `${deviceType} ${platform}`); assert.equal( String(fetchMock.mock.calls[1]?.[0]), `https://testmu.test/app/data?type=${listType}&level=user`, @@ -461,6 +586,28 @@ test('TestMu checks an lt:// id against the virtual-device app list of its platf } }); +test('TestMu defers a real-device lt:// id missing from the real-device app list', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(testMuRealCatalog) + : jsonResponse({ data: [], metaData: { total: 0 } }), + ), + ); + const result = await createProvider().verifyConnection({ + ...testMuOptions, + deviceType: 'real', + deviceName: 'Pixel 6', + }); + assert.equal(result.app.status, 'configured'); + assert.match(String(result.app.message), /not found among your real-device uploads/); + assert.equal( + result.verificationMessage, + 'Credentials and real device verified; app availability is checked when the session is created.', + ); +}); + function createProvider(runHostCommand: RunHostCommand = vi.fn()) { return createProviderWebDriver({ clientVersion: '1.2.3', runHostCommand }); } diff --git a/packages/provider-webdriver/src/connection-verification.ts b/packages/provider-webdriver/src/connection-verification.ts index 505fb2cf09..7e8685da1b 100644 --- a/packages/provider-webdriver/src/connection-verification.ts +++ b/packages/provider-webdriver/src/connection-verification.ts @@ -1,5 +1,8 @@ import type { ProviderWebDriverDependencies } from './dependencies.ts'; -import type { ProviderConnectionVerification } from '@agent-device/contracts/remote'; +import type { + ProviderConnectionVerification, + ProviderDeviceType, +} from '@agent-device/contracts/remote'; import { verifyAwsDeviceFarmConnection } from './aws-device-farm-connection-verification.ts'; import { verifyBrowserStackConnection } from './browserstack-connection-verification.ts'; @@ -38,6 +41,8 @@ export type CloudWebDriverConnectionVerificationOptions = | (HubSelectionVerificationOptions & { provider: 'browserstack' }) | (HubSelectionVerificationOptions & { provider: 'testmu'; + /** Defaults to `virtual`. */ + deviceType?: ProviderDeviceType; /** Base of the catalog API, as `TESTMU_API_ENDPOINT` sets it for the runtime. */ apiEndpoint?: string | URL; }) diff --git a/packages/provider-webdriver/src/provider-definitions.ts b/packages/provider-webdriver/src/provider-definitions.ts index 06e147c5eb..dbc2092f7a 100644 --- a/packages/provider-webdriver/src/provider-definitions.ts +++ b/packages/provider-webdriver/src/provider-definitions.ts @@ -1,5 +1,6 @@ import type { CloudArtifactsResult } from '@agent-device/contracts/observability'; import type { LeaseLifecycleContext } from '@agent-device/contracts/device'; +import type { ProviderDeviceType } from '@agent-device/contracts/remote'; import { AppError } from '@agent-device/kernel/errors'; import type { ProviderWebDriverDependencies } from './dependencies.ts'; import { @@ -48,6 +49,7 @@ export type DefaultCloudWebDriverProviderRuntimeEnv = DefaultCloudWebDriverArtif BROWSERSTACK_APP_UPLOAD_ENDPOINT?: string; TESTMU_WEBDRIVER_ENDPOINT?: string; TESTMU_APP_UPLOAD_ENDPOINT?: string; + TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT?: string; AGENT_DEVICE_AWS_DEVICE_FARM_PROJECT_ARN?: string; AWS_DEVICE_FARM_PROJECT_ARN?: string; AGENT_DEVICE_AWS_DEVICE_FARM_DEVICE_ARN?: string; @@ -57,15 +59,15 @@ export type DefaultCloudWebDriverProviderRuntimeEnv = DefaultCloudWebDriverArtif }; /** - * TestMu (formerly LambdaTest) virtual devices: emulators and simulators behind one Appium hub. - * Only what `createRuntime` needs synchronously lives here; the session, upload, and artifact - * code loads on first use so the package entry stays as lean as it was. + * TestMu (formerly LambdaTest) real devices, and virtual devices (emulators and simulators), behind + * one Appium hub. Only what `createRuntime` needs synchronously lives here; the session, + * upload, and artifact code loads on first use so the package entry stays as lean as it was. */ const TESTMU_WEBDRIVER_ENDPOINT = 'https://mobile-hub.lambdatest.com/wd/hub/'; const TESTMU_CAPABILITY_OVERRIDES = { install: { support: 'partial', - note: 'Local app artifacts are uploaded to TestMu AI as virtual-device apps (lt://), then installed with Appium.', + note: 'Local app artifacts are uploaded to TestMu AI as real- or virtual-device apps (lt://), then installed with Appium.', }, portReverse: { support: 'unsupported', @@ -123,6 +125,10 @@ export function createCloudWebDriverProviderDefinitions( }, prepareSession: async ({ req, lease, base }) => { const request = requireRequest(req, 'BrowserStack'); + (await loadTestMuDeviceFeatures()).rejectTestMuOnlyProviderFlags( + request.flags, + CLOUD_WEBDRIVER_PROVIDERS.browserStack, + ); const username = requireEnv(env, 'BROWSERSTACK_USERNAME', 'BrowserStack'); const accessKey = requireEnv(env, 'BROWSERSTACK_ACCESS_KEY', 'BrowserStack'); const platform = requireRequestPlatform(request, 'BrowserStack'); @@ -229,6 +235,10 @@ export function createCloudWebDriverProviderDefinitions( request.flags, CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, ); + (await loadTestMuDeviceFeatures()).rejectTestMuOnlyProviderFlags( + request.flags, + CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, + ); const platform = requireRequestPlatform(request, 'AWS Device Farm'); const sessionOptions = { client: createAwsCliDeviceFarmClient({ @@ -295,9 +305,12 @@ export function createCloudWebDriverProviderDefinitions( const { buildTestMuDeviceFeatureCapabilities, readTestMuDeviceFeatureFields, + readTestMuDeviceType, rejectUnsupportedTestMuDeviceFeatures, } = await loadTestMuDeviceFeatures(); rejectUnsupportedTestMuDeviceFeatures(request.flags); + const deviceType = readTestMuDeviceType(request.flags); + const uploadEndpoint = testMuAppUploadEndpoint(env, deviceType); const credentials = requireTestMuCredentials(env, 'TestMu AI'); const platform = requireRequestPlatform(request, 'TestMu AI'); const deviceName = requireFlag( @@ -313,7 +326,8 @@ export function createCloudWebDriverProviderDefinitions( const upload = { clientVersion: dependencies.clientVersion, ...credentials, - endpoint: env.TESTMU_APP_UPLOAD_ENDPOINT, + deviceType, + endpoint: uploadEndpoint, }; const app = await resolveTestMuAppReference( requireFlag( @@ -331,6 +345,7 @@ export function createCloudWebDriverProviderDefinitions( uploadApp: createTestMuUploadApp(upload), webdriverCapabilities: buildTestMuCapabilities({ platform, + deviceType, deviceName, osVersion, app, @@ -374,6 +389,16 @@ function requireTestMuCredentials( }; } +/** Each pool has its own upload API, so each has its own override. */ +function testMuAppUploadEndpoint( + env: DefaultCloudWebDriverProviderRuntimeEnv, + deviceType: ProviderDeviceType, +): string | undefined { + return deviceType === 'real' + ? env.TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT + : env.TESTMU_APP_UPLOAD_ENDPOINT; +} + function requireRequest( req: LeaseLifecycleContext | undefined, providerLabel: string, diff --git a/packages/provider-webdriver/src/testmu-connection-verification.ts b/packages/provider-webdriver/src/testmu-connection-verification.ts index 91d913ce23..f54c28358f 100644 --- a/packages/provider-webdriver/src/testmu-connection-verification.ts +++ b/packages/provider-webdriver/src/testmu-connection-verification.ts @@ -6,21 +6,24 @@ import type { CloudWebDriverConnectionVerification, CloudWebDriverConnectionVerificationOptions, } from './connection-verification.ts'; -import type { ProviderConnectionResource } from '@agent-device/contracts/remote'; +import type { + ProviderConnectionResource, + ProviderDeviceType, +} from '@agent-device/contracts/remote'; type TestMuOptions = Extract; type TestMuAuth = { username: string; accessKey: string }; -/** `/app/data?type=` keys virtual-device uploads by runtime, not by platform. */ -const TESTMU_APP_LIST_TYPES: Record<'android' | 'ios', string> = { - android: 'emulator', - ios: 'simulator', +/** `/app/data?type=` keys uploads by pool: real-device apps by platform, virtual ones by runtime. */ +const TESTMU_APP_LIST_TYPES: Record> = { + real: { android: 'android', ios: 'ios' }, + virtual: { android: 'emulator', ios: 'simulator' }, }; /** - * Verifies a TestMu virtual-device selection without creating a session: the public capability - * catalog confirms the device/OS pair exists in the emulator and simulator pool, and the + * Verifies a TestMu device selection without creating a session: the public capability catalog + * of the selected pool (real or virtual) confirms the device/OS pair exists, and the * authenticated app listing confirms the credentials and, for an `lt://` reference, the upload. */ export async function verifyTestMuConnection( @@ -28,15 +31,17 @@ export async function verifyTestMuConnection( clientVersion: string, ): Promise { const auth = { username: options.username, accessKey: options.accessKey }; + const deviceType = options.deviceType ?? 'virtual'; const catalogUrl = options.devicesEndpoint ? new URL(options.devicesEndpoint) : apiUrl(options.apiEndpoint ?? TESTMU_API_ENDPOINT, 'capability/generator'); - catalogUrl.searchParams.set('isVirtualDevice', 'true'); + catalogUrl.searchParams.set('isVirtualDevice', String(deviceType === 'virtual')); const catalog = await fetchTestMuJson(catalogUrl, undefined, clientVersion); - const namedDevices = readTestMuVirtualDevices(catalog, options.platform).filter( + const namedDevices = readTestMuCatalogDevices(catalog, options.platform, deviceType).filter( (device) => device.name === options.deviceName, ); - // Exact match on purpose: the hub rejects `18` for a device the catalog lists as `18.0`. + // Exact match on purpose: the hub rejects `18` for a virtual device the catalog lists as `18.0`, + // and real iOS devices are listed by major version only. const matchedDevice = namedDevices.find((device) => device.osVersions.includes(options.osVersion), ); @@ -46,24 +51,25 @@ export async function verifyTestMuConnection( ); throw new AppError( 'INVALID_ARGS', - `TestMu AI virtual device "${options.deviceName}" with ${options.platform} ${options.osVersion} is not available${ + `TestMu AI ${deviceType} device "${options.deviceName}" with ${options.platform} ${options.osVersion} is not available${ offered.length > 0 ? `; ${options.deviceName} offers ${offered.join(', ')}` : '' }.`, { - hint: 'Choose an exact device name and OS version from the TestMu AI virtual-device capability generator.', + hint: `Choose an exact device name and OS version from the TestMu AI ${deviceType}-device capability generator.`, + deviceType, ...(offered.length > 0 ? { availableOsVersions: offered } : {}), }, ); } - const app = await verifyTestMuApp(options, auth, clientVersion); + const app = await verifyTestMuApp(options, deviceType, auth, clientVersion); return { provider: 'testmu', service: 'TestMu AI', verificationMessage: app.status === 'verified' - ? 'Credentials, virtual device, and uploaded app verified.' - : 'Credentials and virtual device verified; app availability is checked when the session is created.', + ? `Credentials, ${deviceType} device, and uploaded app verified.` + : `Credentials and ${deviceType} device verified; app availability is checked when the session is created.`, device: { status: 'verified', name: matchedDevice.name, @@ -76,13 +82,14 @@ export async function verifyTestMuConnection( async function verifyTestMuApp( options: TestMuOptions, + deviceType: ProviderDeviceType, auth: TestMuAuth, clientVersion: string, ): Promise { const { app } = options; // The listing is authenticated, so it doubles as the credential check for every app kind. const appsUrl = new URL(options.appsEndpoint ?? TESTMU_APPS_ENDPOINT); - appsUrl.searchParams.set('type', TESTMU_APP_LIST_TYPES[options.platform]); + appsUrl.searchParams.set('type', TESTMU_APP_LIST_TYPES[deviceType][options.platform]); appsUrl.searchParams.set('level', 'user'); const apps = await fetchTestMuJson(appsUrl, auth, clientVersion); if (isTestMuAppReference(app)) { @@ -91,8 +98,7 @@ async function verifyTestMuApp( return { status: 'configured', reference: app, - message: - 'App reference was not found among your virtual-device uploads; TestMu AI validates it when creating the session.', + message: `App reference was not found among your ${deviceType}-device uploads; TestMu AI validates it when creating the session.`, }; } return { status: 'verified', ...matched }; @@ -137,20 +143,25 @@ async function fetchTestMuJson( } /** - * The capability generator lists virtual devices per platform under - * `app.devices..brands.[]` as `{ name, osVersion: string[] }`. + * The capability generator lists devices per platform as `brands.[]` of + * `{ name, osVersion: string[] }`: under `app.devices.` for the virtual pool + * (`isVirtualDevice=true`), and directly under `` for the real pool. */ -function readTestMuVirtualDevices( +function readTestMuCatalogDevices( value: unknown, platform: 'android' | 'ios', + deviceType: ProviderDeviceType, ): Array<{ name: string; osVersions: string[] }> { - const platformCatalog = asRecord(asRecord(asRecord(asRecord(value)?.app)?.devices)?.[platform]); + const platformCatalog = + deviceType === 'real' + ? asRecord(asRecord(value)?.[platform]) + : asRecord(asRecord(asRecord(asRecord(value)?.app)?.devices)?.[platform]); const brandRecord = asRecord(platformCatalog?.brands); if (!brandRecord) { throw new AppError( 'COMMAND_FAILED', - 'TestMu AI virtual-device catalog response did not list devices for the platform.', - { platform }, + `TestMu AI ${deviceType}-device catalog response did not list devices for the platform.`, + { platform, deviceType }, ); } return Object.values(brandRecord).flatMap((devices) => { diff --git a/packages/provider-webdriver/src/testmu-device-features.test.ts b/packages/provider-webdriver/src/testmu-device-features.test.ts index da38e77e3b..33eb5afd49 100644 --- a/packages/provider-webdriver/src/testmu-device-features.test.ts +++ b/packages/provider-webdriver/src/testmu-device-features.test.ts @@ -6,6 +6,8 @@ import { TESTMU_DEVICE_FEATURE_SPECS, buildTestMuDeviceFeatureCapabilities, readTestMuDeviceFeatureFields, + readTestMuDeviceType, + rejectTestMuOnlyProviderFlags, rejectUnsupportedTestMuDeviceFeatures, } from './testmu-device-features.ts'; @@ -101,3 +103,35 @@ test('BrowserStack-only flags are rejected by flag name instead of being dropped /--provider-network-profile, --provider-custom-network are not supported by TestMu AI/, ); }); + +test('the device type defaults to the virtual pool and rejects unknown values', () => { + assert.equal(readTestMuDeviceType(undefined), 'virtual'); + assert.equal(readTestMuDeviceType({ providerDeviceType: '' }), 'virtual'); + assert.equal(readTestMuDeviceType({ providerDeviceType: 'virtual' }), 'virtual'); + assert.equal(readTestMuDeviceType({ providerDeviceType: 'real' }), 'real'); + assert.throws( + () => readTestMuDeviceType({ providerDeviceType: 'physical' }), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + error.details?.flag === '--provider-device-type', + ); +}); + +test('other providers refuse --provider-device-type by flag name', () => { + assert.doesNotThrow(() => rejectTestMuOnlyProviderFlags(undefined, 'browserstack')); + assert.doesNotThrow(() => + rejectTestMuOnlyProviderFlags({ providerGeoLocation: 'US' }, 'browserstack'), + ); + for (const providerDeviceType of ['real', 'virtual']) { + assert.throws( + () => rejectTestMuOnlyProviderFlags({ providerDeviceType }, 'aws-device-farm'), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /--provider-device-type is only supported by TestMu AI, not aws-device-farm/.test( + error.message, + ), + ); + } +}); diff --git a/packages/provider-webdriver/src/testmu-device-features.ts b/packages/provider-webdriver/src/testmu-device-features.ts index 6f1ab049c4..84ea7d8e95 100644 --- a/packages/provider-webdriver/src/testmu-device-features.ts +++ b/packages/provider-webdriver/src/testmu-device-features.ts @@ -1,4 +1,8 @@ -import type { CloudProviderProfileFields } from '@agent-device/contracts/remote'; +import { + PROVIDER_DEVICE_TYPES, + type CloudProviderProfileFields, + type ProviderDeviceType, +} from '@agent-device/contracts/remote'; import { AppError } from '@agent-device/kernel/errors'; import { requireProviderDeviceOrientation } from './webdriver-utils.ts'; @@ -113,3 +117,38 @@ export function readTestMuDeviceFeatureFields( } return fields; } + +/** Reads the TestMu device pool off an untyped flag bag; an unset value keeps the virtual pool. */ +export function readTestMuDeviceType( + flags: Record | undefined, +): ProviderDeviceType { + const value = flags?.providerDeviceType; + if (value === undefined || value === '') return 'virtual'; + const match = PROVIDER_DEVICE_TYPES.find((deviceType) => deviceType === value); + if (match) return match; + throw new AppError('INVALID_ARGS', `Invalid --provider-device-type value: ${String(value)}.`, { + hint: `Use ${PROVIDER_DEVICE_TYPES.join('|')}.`, + flag: '--provider-device-type', + }); +} + +/** + * Fails when another provider was given a TestMu-only flag. Like the BrowserStack-only check, it + * runs in both the connect profile builder and session preparation. + */ +export function rejectTestMuOnlyProviderFlags( + flags: Record | undefined, + provider: string, +): void { + const value = flags?.providerDeviceType; + if (value === undefined || value === '') return; + throw new AppError( + 'INVALID_ARGS', + `--provider-device-type is only supported by TestMu AI, not ${provider}.`, + { + hint: 'Drop the flag or use the testmu provider.', + provider, + flags: ['--provider-device-type'], + }, + ); +} diff --git a/packages/provider-webdriver/src/testmu.test.ts b/packages/provider-webdriver/src/testmu.test.ts index 9b02320c57..fd00b6782e 100644 --- a/packages/provider-webdriver/src/testmu.test.ts +++ b/packages/provider-webdriver/src/testmu.test.ts @@ -107,25 +107,40 @@ test('a configured lt:options cannot turn off the W3C dialect', () => { assert.equal(ltOptions.tunnel, true); }); -// A configured `isRealMobile` would silently move the session to the real-device pool, which -// bills differently. -test('a configured lt:options cannot move the session off the virtual-device pool', () => { - const capabilities = buildTestMuCapabilities({ - platform: 'ios', +// A configured `isRealMobile` would silently move the session to the other pool, which bills +// differently. +test('the device type selects the TestMu pool and a configured lt:options cannot override it', () => { + const base = { + platform: 'ios' as const, deviceName: 'iPhone 16', - osVersion: '18.0', + osVersion: '18', buildName: 'run-1', sessionName: 'lease-1', - configured: { 'lt:options': { isRealMobile: true, tunnel: true } }, + }; + const real = buildTestMuCapabilities({ + ...base, + deviceType: 'real', + configured: { 'lt:options': { isRealMobile: false, tunnel: true } }, }); - const ltOptions = capabilities['lt:options'] as Record; - assert.equal(ltOptions.isRealMobile, false); - assert.equal(ltOptions.tunnel, true); + const realOptions = real['lt:options'] as Record; + assert.equal(realOptions.isRealMobile, true); + assert.equal(realOptions.tunnel, true); + assert.equal(realOptions.platformVersion, '18'); + + const virtual = buildTestMuCapabilities({ + ...base, + deviceType: 'virtual', + configured: { 'lt:options': { isRealMobile: true } }, + }); + assert.equal((virtual['lt:options'] as Record).isRealMobile, false); + + const unset = buildTestMuCapabilities(base); + assert.equal((unset['lt:options'] as Record).isRealMobile, false); }); -test('TestMu uploads go to the virtual-device upload API unless an endpoint is configured', async () => { - const tempDir = await mkdtempForTest('agent-device-testmu-upload-endpoint-'); - const appPath = path.join(tempDir, 'MyApp.apk'); +test('TestMu uploads go to the upload API of the selected device pool', async () => { + const tempDir = await mkdtempForTest('agent-device-testmu-upload-pool-'); + const appPath = path.join(tempDir, 'MyApp.ipa'); const endpoints: string[] = []; try { await fs.writeFile(appPath, 'placeholder'); @@ -133,19 +148,48 @@ test('TestMu uploads go to the virtual-device upload API unless an endpoint is c endpoints.push(String(input)); return jsonResponse({ app_url: 'lt://APP1' }); }; + await uploadTestMuApp(appPath, { ...auth, deviceType: 'real' }); + await uploadTestMuAppFromUrl('https://example.test/App.apk', { ...auth, deviceType: 'real' }); await uploadTestMuApp(appPath, auth); - await uploadTestMuAppFromUrl('https://example.test/App.apk', auth); - await uploadTestMuApp(appPath, { ...auth, endpoint: 'https://upload.test/virtual' }); + await uploadTestMuApp(appPath, { ...auth, deviceType: 'virtual' }); + await uploadTestMuApp(appPath, { + ...auth, + deviceType: 'real', + endpoint: 'https://upload.test/real', + }); assert.deepEqual(endpoints, [ + 'https://manual-api.lambdatest.com/app/upload/realDevice', + 'https://manual-api.lambdatest.com/app/upload/realDevice', 'https://manual-api.lambdatest.com/app/upload/virtualDevice', 'https://manual-api.lambdatest.com/app/upload/virtualDevice', - 'https://upload.test/virtual', + 'https://upload.test/real', ]); } finally { await fs.rm(tempDir, { recursive: true, force: true }); } }); +test('a real-device upload of an .app directory asks for a signed .ipa', async () => { + const tempDir = await mkdtempForTest('agent-device-testmu-real-app-dir-'); + const appPath = path.join(tempDir, 'Demo.app'); + try { + await fs.mkdir(appPath); + const fetchMock = vi.fn(); + globalThis.fetch = fetchMock; + await assert.rejects( + uploadTestMuApp(appPath, { ...auth, deviceType: 'real' }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.match(String(error.details?.hint), /signed \.ipa/); + return true; + }, + ); + assert.equal(fetchMock.mock.calls.length, 0); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + test('TestMu upload reads the lt:// reference and aborts while the request is in flight', async () => { const tempDir = await mkdtempForTest('agent-device-testmu-upload-'); const appPath = path.join(tempDir, 'App.apk'); diff --git a/packages/provider-webdriver/src/testmu.ts b/packages/provider-webdriver/src/testmu.ts index a06b14a0dd..a632cba50e 100644 --- a/packages/provider-webdriver/src/testmu.ts +++ b/packages/provider-webdriver/src/testmu.ts @@ -1,6 +1,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import type { CloudArtifact, CloudArtifactsResult } from '@agent-device/contracts/observability'; +import type { ProviderDeviceType } from '@agent-device/contracts/remote'; import type { CloudWebDriverPlatform, CloudWebDriverUploadApp } from './runtime.ts'; import { AppError } from '@agent-device/kernel/errors'; import { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; @@ -16,16 +17,21 @@ import { /** * TestMu session, upload, and artifact mechanics. Loaded on demand by the provider definition; - * `isRealMobile: false` in `lt:options` is what routes a session to the virtual-device pool, and + * `isRealMobile` in `lt:options` is what routes a session to the real or virtual device pool, and * the hostnames still carry the lambdatest.com brand. */ -const TESTMU_APP_UPLOAD_ENDPOINT = 'https://manual-api.lambdatest.com/app/upload/virtualDevice'; +const TESTMU_APP_UPLOAD_ENDPOINTS: Record = { + real: 'https://manual-api.lambdatest.com/app/upload/realDevice', + virtual: 'https://manual-api.lambdatest.com/app/upload/virtualDevice', +}; export const TESTMU_APPS_ENDPOINT = 'https://manual-api.lambdatest.com/app/data'; export const TESTMU_API_ENDPOINT = 'https://mobile-api.lambdatest.com/mobile-automation/api/v1'; const TESTMU_DASHBOARD_TEST_URL = 'https://appautomation.lambdatest.com/test?testID='; export type TestMuCapabilitiesOptions = { platform: CloudWebDriverPlatform; + /** Defaults to `virtual`. */ + deviceType?: ProviderDeviceType; deviceName: string; osVersion: string; app?: string; @@ -65,6 +71,8 @@ export async function listTestMuCloudArtifacts( export type TestMuUploadOptions = TestMuAuth & { clientVersion: string; + /** Selects the pool's upload API when no endpoint override is given; defaults to `virtual`. */ + deviceType?: ProviderDeviceType; endpoint?: string | URL; }; @@ -78,7 +86,10 @@ export async function uploadTestMuApp( if (!(await fs.stat(appPath)).isFile()) { throw new AppError('INVALID_ARGS', `TestMu AI can only upload an app file: ${appPath}`, { appPath, - hint: 'Zip the .app bundle of an iOS simulator build and pass the .zip.', + hint: + options.deviceType === 'real' + ? 'Real iOS devices install a signed .ipa; pass the .ipa file.' + : 'Zip the .app bundle of an iOS simulator build and pass the .zip.', }); } const form = await appFileUploadForm(appPath, 'appFile'); @@ -109,7 +120,7 @@ async function postTestMuUpload( form, { service: 'TestMu AI', - endpoint: options.endpoint ?? TESTMU_APP_UPLOAD_ENDPOINT, + endpoint: options.endpoint ?? TESTMU_APP_UPLOAD_ENDPOINTS[options.deviceType ?? 'virtual'], clientVersion: options.clientVersion, auth: options, readAppReference: readTestMuAppReference, @@ -142,11 +153,11 @@ export async function resolveTestMuAppReference( } /** - * Builds the W3C `alwaysMatch` capabilities for a TestMu virtual-device session. + * Builds the W3C `alwaysMatch` capabilities for a TestMu session. * * Standard Appium keys stay `appium:`-prefixed at the top level; everything TestMu-specific lives - * in `lt:options`. `isRealMobile: false` selects an emulator or simulator, and `w3c: true` keeps - * the hub on the W3C dialect agent-device speaks. `appiumVersion` is sent only when the caller + * in `lt:options`. `isRealMobile` selects a real device or an emulator/simulator, and `w3c: true` + * keeps the hub on the W3C dialect agent-device speaks. `appiumVersion` is sent only when the caller * pins one; otherwise TestMu AI starts its default server for the device. */ export function buildTestMuCapabilities( @@ -173,7 +184,7 @@ export function buildTestMuCapabilities( ...deviceFeatures, ...(asRecord(configuredLtOptions) ?? {}), // A configured value cannot switch the device pool or drop the W3C dialect agent-device speaks. - isRealMobile: false, + isRealMobile: options.deviceType === 'real', w3c: true, }, }; diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 3a1be8a0b8..2d9eb9a809 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -254,6 +254,7 @@ function summarizeProviderScenarioFlagExclusions() { 'providerSessionId', 'providerApp', 'providerOsVersion', + 'providerDeviceType', 'providerProject', 'providerBuild', 'providerSessionName', diff --git a/src/__tests__/client-leases.test.ts b/src/__tests__/client-leases.test.ts new file mode 100644 index 0000000000..02ea6d0d58 --- /dev/null +++ b/src/__tests__/client-leases.test.ts @@ -0,0 +1,35 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { createAgentDeviceClient } from '../agent-device-client.ts'; +import { createTransport } from './client-transport-fixture.ts'; + +test('lease allocation carries the TestMu device type to the provider flags', async () => { + const setup = createTransport(async (req) => ({ + ok: true, + data: { + lease: { + leaseId: 'lease-new', + tenantId: req.meta?.tenantId, + runId: req.meta?.runId, + backend: req.meta?.leaseBackend, + }, + }, + })); + const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); + + await client.leases.allocate({ + tenant: 'testmu', + runId: 'remote-run', + leaseBackend: 'ios-instance', + leaseProvider: 'testmu', + platform: 'ios', + device: 'iPhone 16', + providerOsVersion: '18', + providerDeviceType: 'real', + providerApp: 'lt://APP1', + }); + + assert.equal(setup.calls[0]?.command, 'lease_allocate'); + assert.equal(setup.calls[0]?.flags?.providerDeviceType, 'real'); + assert.equal(setup.calls[0]?.flags?.providerOsVersion, '18'); +}); diff --git a/src/__tests__/cloud-connect-testmu.test.ts b/src/__tests__/cloud-connect-testmu.test.ts index 1255a4af6a..1d882c96ff 100644 --- a/src/__tests__/cloud-connect-testmu.test.ts +++ b/src/__tests__/cloud-connect-testmu.test.ts @@ -3,6 +3,7 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { connectCommand } from '../cli/commands/connection.ts'; +import { runCliCapture } from './cli-capture.ts'; import { readActiveConnectionState, type RemoteConnectionState, @@ -150,6 +151,119 @@ test('connect testmu rejects BrowserStack network and re-sign flags before savin } }); +test('connect testmu stores and verifies the real-device pool', async () => { + const tempRoot = mkdtempForTestSync('agent-device-connect-testmu-real-'); + const stateDir = path.join(tempRoot, '.state'); + vi.stubEnv('LT_USERNAME', 'lt-user'); + vi.stubEnv('LT_ACCESS_KEY', 'lt-key'); + + try { + await connectWithGeneratedProviderProfile({ + stateDir, + positionals: ['testmu'], + flags: { + platform: 'ios', + device: 'iPhone 16', + providerOsVersion: '18', + providerDeviceType: 'real', + providerApp: 'lt://APP1', + }, + }); + + assert.deepEqual(mockedVerifyWebDriverConnection.mock.calls[0]?.[0], { + provider: 'testmu', + username: 'lt-user', + accessKey: 'lt-key', + platform: 'ios', + deviceName: 'iPhone 16', + osVersion: '18', + app: 'lt://APP1', + deviceType: 'real', + }); + const state = readRequiredActiveState(stateDir); + const generated = readGeneratedConfig(state.remoteConfigPath); + assert.equal(generated.providerDeviceType, 'real'); + assert.equal(generated.providerOsVersion, '18'); + + // The saved profile reproduces the same verification when it is loaded again. + mockedVerifyWebDriverConnection.mockClear(); + await connectWithGeneratedProviderProfile({ + stateDir, + positionals: [], + flags: { remoteConfig: state.remoteConfigPath, force: true }, + }); + const reloaded = mockedVerifyWebDriverConnection.mock.calls[0]?.[0]; + assert.equal(reloaded?.provider, 'testmu'); + assert.equal(reloaded.deviceType, 'real'); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test('providers other than TestMu refuse --provider-device-type before saving a profile', () => { + const tempRoot = mkdtempForTestSync('agent-device-connect-device-type-reject-'); + const base = { json: false, help: false, version: false, platform: 'android' as const }; + + try { + for (const [provider, flags, env] of [ + [ + 'browserstack', + { + ...base, + device: 'Google Pixel 8', + providerOsVersion: '14.0', + providerApp: 'bs://app-id', + }, + { BROWSERSTACK_USERNAME: 'u', BROWSERSTACK_ACCESS_KEY: 'k' }, + ], + [ + 'aws-device-farm', + { + ...base, + awsProjectArn: 'arn:aws:devicefarm:us-west-2:123:project/p', + awsDeviceArn: 'arn:aws:devicefarm:us-west-2::device/d', + }, + {}, + ], + ] as const) { + assert.throws( + () => + resolveCloudWebDriverConnectProfile({ + provider, + stateDir: path.join(tempRoot, '.state'), + cwd: tempRoot, + env, + flags: { ...flags, providerDeviceType: 'real' }, + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'INVALID_ARGS'); + assert.match( + error.message, + new RegExp(`--provider-device-type is only supported by TestMu AI, not ${provider}`), + ); + return true; + }, + ); + } + assert.equal(fs.existsSync(path.join(tempRoot, '.state')), false); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + +test('connect limrun refuses the TestMu device type', async () => { + const result = await runCliCapture( + ['connect', 'limrun', '--platform', 'ios', '--provider-device-type', 'real', '--json'], + { + env: { LIMRUN_API_KEY: 'lim_test_key' }, + stateDirPrefix: 'agent-device-connect-limrun-device-type-', + }, + ); + assert.equal(result.code, 1); + assert.match(result.stdout, /--provider-device-type is only supported by TestMu AI, not limrun/); +}); + async function connectWithGeneratedProviderProfile(options: { stateDir: string; positionals: string[]; @@ -176,11 +290,13 @@ async function connectWithGeneratedProviderProfile(options: { function readGeneratedConfig(configPath: string): { providerApp?: string; providerOsVersion?: string; + providerDeviceType?: string; providerBuild?: string; } { return JSON.parse(fs.readFileSync(configPath, 'utf8')) as { providerApp?: string; providerOsVersion?: string; + providerDeviceType?: string; providerBuild?: string; }; } diff --git a/src/cli.ts b/src/cli.ts index 67246cd596..5ace00beb7 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -456,6 +456,7 @@ function buildClientConfig(ctx: CliRunContext): AgentDeviceClientConfig { deviceKey: connection?.deviceKey, providerApp: currentFlags.providerApp, providerOsVersion: currentFlags.providerOsVersion, + providerDeviceType: currentFlags.providerDeviceType, providerProject: currentFlags.providerProject, providerBuild: currentFlags.providerBuild, providerSessionName: currentFlags.providerSessionName, diff --git a/src/cli/commands/connection-runtime.ts b/src/cli/commands/connection-runtime.ts index db52353600..b645609807 100644 --- a/src/cli/commands/connection-runtime.ts +++ b/src/cli/commands/connection-runtime.ts @@ -883,6 +883,7 @@ async function allocateOrReuseLease( serial: flags.serial, providerApp: initialApp ?? flags.providerApp, providerOsVersion: flags.providerOsVersion, + providerDeviceType: flags.providerDeviceType, providerProject: flags.providerProject, providerBuild: flags.providerBuild, providerSessionName: flags.providerSessionName, diff --git a/src/cli/connection/cloud-webdriver-profile.ts b/src/cli/connection/cloud-webdriver-profile.ts index 031d48c8c7..c6ed0ebbb9 100644 --- a/src/cli/connection/cloud-webdriver-profile.ts +++ b/src/cli/connection/cloud-webdriver-profile.ts @@ -4,7 +4,10 @@ import { rejectBrowserStackOnlyDeviceFeatures, type CloudWebDriverKnownProviderName, } from '@agent-device/provider-webdriver'; -import { rejectUnsupportedTestMuDeviceFeatures } from '@agent-device/provider-webdriver/testmu-device-features'; +import { + rejectTestMuOnlyProviderFlags, + rejectUnsupportedTestMuDeviceFeatures, +} from '@agent-device/provider-webdriver/testmu-device-features'; import type { RemoteConfigProfile } from '../../remote/remote-config-schema.ts'; import { AppError } from '@agent-device/kernel/errors'; import type { PlatformSelector } from '@agent-device/kernel/device'; @@ -118,6 +121,7 @@ function browserStackProfileFields(options: { env?: EnvMap; cwd: string; }): RemoteConfigProfile { + rejectTestMuOnlyProviderFlags(options.flags, CLOUD_WEBDRIVER_PROVIDERS.browserStack); return hubProviderProfileFields(BROWSERSTACK_HUB_PROFILE, options); } @@ -127,7 +131,10 @@ function testMuProfileFields(options: { cwd: string; }): RemoteConfigProfile { rejectUnsupportedTestMuDeviceFeatures(options.flags); - return hubProviderProfileFields(TESTMU_HUB_PROFILE, options); + return { + ...hubProviderProfileFields(TESTMU_HUB_PROFILE, options), + providerDeviceType: options.flags.providerDeviceType, + }; } function hubProviderProfileFields( @@ -181,6 +188,7 @@ function awsDeviceFarmProfileFields(options: { }): RemoteConfigProfile { const { env, flags } = options; rejectBrowserStackOnlyDeviceFeatures(flags, CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm); + rejectTestMuOnlyProviderFlags(flags, CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm); const platform = requireCloudWebDriverPlatform( flags.platform, 'connect aws-device-farm requires --platform ios|android.', diff --git a/src/cli/connection/connect-provider-adapters.ts b/src/cli/connection/connect-provider-adapters.ts index a83c548365..7fcfabd34b 100644 --- a/src/cli/connection/connect-provider-adapters.ts +++ b/src/cli/connection/connect-provider-adapters.ts @@ -172,6 +172,7 @@ async function verifyTestMu( 'TestMu AI profile missed OS version.', ), app: requiredResolvedValue(flags.providerApp, 'TestMu AI profile missed app.'), + ...(flags.providerDeviceType ? { deviceType: flags.providerDeviceType } : {}), ...(env.TESTMU_API_ENDPOINT ? { apiEndpoint: env.TESTMU_API_ENDPOINT } : {}), }); } diff --git a/src/cli/connection/limrun-profile.ts b/src/cli/connection/limrun-profile.ts index b3025a78da..2033dec412 100644 --- a/src/cli/connection/limrun-profile.ts +++ b/src/cli/connection/limrun-profile.ts @@ -3,6 +3,7 @@ import type { RemoteConfigProfile } from '../../remote/remote-config-schema.ts'; import { AppError } from '@agent-device/kernel/errors'; import type { CliFlags } from '@agent-device/contracts/command'; import { type EnvMap } from '@agent-device/kernel/source-value'; +import { rejectTestMuOnlyProviderFlags } from '@agent-device/provider-webdriver/testmu-device-features'; import { readMetroProfileFields } from './profile-fields.ts'; import { persistAndResolveGeneratedProfile } from './generated-config.ts'; import { resolveRequestedLeaseBackend } from '../commands/connection-runtime.ts'; @@ -53,6 +54,7 @@ function buildLimrunRemoteProfile(options: { flags: CliFlags }): RemoteConfigPro } function validateLimrunConnectFlags(flags: CliFlags): 'android-instance' | 'ios-instance' { + rejectTestMuOnlyProviderFlags(flags, 'limrun'); if (flags.platform !== 'android' && flags.platform !== 'ios') { throw new AppError('INVALID_ARGS', 'connect limrun requires --platform ios or android.'); } diff --git a/src/cli/parser/__tests__/args-parse-provider-device-type.test.ts b/src/cli/parser/__tests__/args-parse-provider-device-type.test.ts new file mode 100644 index 0000000000..534d2d6a3d --- /dev/null +++ b/src/cli/parser/__tests__/args-parse-provider-device-type.test.ts @@ -0,0 +1,17 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { parseArgs } from '../args.ts'; + +test('parseArgs reads the TestMu device type and rejects an unknown pool', () => { + const parsed = parseArgs(['connect', 'testmu', '--provider-device-type', 'real'], { + strictFlags: true, + }); + assert.equal(parsed.flags.providerDeviceType, 'real'); + assert.throws( + () => + parseArgs(['connect', 'testmu', '--provider-device-type', 'physical'], { + strictFlags: true, + }), + /Invalid provider-device-type: physical/, + ); +}); diff --git a/src/commands/command-flags.ts b/src/commands/command-flags.ts index f7b44a4618..fa04c2d2ef 100644 --- a/src/commands/command-flags.ts +++ b/src/commands/command-flags.ts @@ -28,6 +28,7 @@ function buildFlags(options: InternalRequestOptions): CommandFlags { providerSessionId: options.providerSessionId, providerApp: options.providerApp, providerOsVersion: options.providerOsVersion, + providerDeviceType: options.providerDeviceType, providerProject: options.providerProject, providerBuild: options.providerBuild, providerSessionName: options.providerSessionName, diff --git a/src/commands/schema/cli-help-topics.test.ts b/src/commands/schema/cli-help-topics.test.ts index cc92b8b6e7..c7f47c5157 100644 --- a/src/commands/schema/cli-help-topics.test.ts +++ b/src/commands/schema/cli-help-topics.test.ts @@ -446,6 +446,11 @@ test('usageForCommand resolves remote help topic', async () => { help.indexOf('BrowserStack hosted-device flow'), ); assert.match(testMuFlow, /--device "iPhone 16" --provider-os-version 18\.0/); + assert.match( + testMuFlow, + /connect testmu --provider-device-type real .*--provider-os-version 18 --provider-app \.\/MyApp\.ipa/, + ); + assert.match(testMuFlow, /major OS version \(18, not 18\.0\)/); assert.match(testMuFlow, /agent-device disconnect/); assert.match(help, /Limrun uses LIMRUN_API_KEY/); assert.match(help, /BrowserStack uses BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY/); diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 1211c54f88..b4d396fe9f 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -577,7 +577,7 @@ Providers: Direct proxy: agent-device connect proxy --daemon-base-url stores the shared proxy profile and client identity. BrowserStack: agent-device connect browserstack verifies credentials, the exact device, and a bs:// app reference, then stores a local provider profile. It does not create an App Automate session. AWS Device Farm: agent-device connect aws-device-farm verifies credentials and the exact project, device, and optional app upload, then stores a local provider profile. It does not create a remote access session. - TestMu AI: agent-device connect testmu verifies credentials, the exact virtual device (emulator or simulator) and OS version, and an lt:// app reference, then stores a local provider profile. It does not create a hub session. + TestMu AI: agent-device connect testmu verifies credentials, the exact virtual device (emulator or simulator) or, with --provider-device-type real, real device and OS version, and an lt:// app reference, then stores a local provider profile. It does not create a hub session. Limrun: agent-device connect limrun verifies access to the selected iOS or Android instance service, then stores a local provider profile. It does not create an instance. After direct-provider connect: @@ -627,6 +627,10 @@ TestMu AI virtual-device flow (emulators and simulators): agent-device artifacts --json agent-device disconnect +TestMu AI real-device flow: + agent-device connect testmu --provider-device-type real --platform ios --device "iPhone 16" --provider-os-version 18 --provider-app ./MyApp.ipa + Real iOS devices are listed by major OS version (18, not 18.0) and install a signed .ipa. Real and virtual devices have separate upload APIs, so pass an lt:// id uploaded for the pool you connect to. + BrowserStack hosted-device flow: BROWSERSTACK_USERNAME=... BROWSERSTACK_ACCESS_KEY=... agent-device connect browserstack --platform android --device "Google Pixel 8" --provider-os-version 14.0 --provider-app bs://app-id diff --git a/src/commands/schema/command-overrides.ts b/src/commands/schema/command-overrides.ts index f69ccda125..fcd1464609 100644 --- a/src/commands/schema/command-overrides.ts +++ b/src/commands/schema/command-overrides.ts @@ -77,6 +77,7 @@ const SCHEMA_ONLY_CLI_COMMAND_SCHEMAS = { 'leaseBackend', 'providerApp', 'providerOsVersion', + 'providerDeviceType', 'providerProject', 'providerBuild', 'providerSessionName', diff --git a/src/daemon-client/daemon-client-rpc.test.ts b/src/daemon-client/daemon-client-rpc.test.ts index faa8c6e835..a8656d63f2 100644 --- a/src/daemon-client/daemon-client-rpc.test.ts +++ b/src/daemon-client/daemon-client-rpc.test.ts @@ -45,6 +45,7 @@ test('lease allocation transports the provider configuration the session needs ( device: 'iPhone 15', providerApp: 'bs://app-id', providerOsVersion: '17', + providerDeviceType: 'real', providerProject: 'MyProject', providerBuild: 'Build-2026-09-11', providerSessionName: 'smoke — iOS', @@ -72,6 +73,7 @@ test('lease allocation transports the provider configuration the session needs ( device: 'iPhone 15', providerApp: 'bs://app-id', providerOsVersion: '17', + providerDeviceType: 'real', providerProject: 'MyProject', providerBuild: 'Build-2026-09-11', providerSessionName: 'smoke — iOS', diff --git a/src/daemon/handlers/session-doctor-options.ts b/src/daemon/handlers/session-doctor-options.ts index 4c9f6eb1de..7481ac986d 100644 --- a/src/daemon/handlers/session-doctor-options.ts +++ b/src/daemon/handlers/session-doctor-options.ts @@ -20,6 +20,7 @@ const REMOTE_PROVIDER_FLAG_KEYS = [ 'providerSessionId', 'providerApp', 'providerOsVersion', + 'providerDeviceType', 'providerProject', 'providerBuild', 'providerSessionName', diff --git a/src/remote/remote-config-schema.ts b/src/remote/remote-config-schema.ts index d28e68a892..24a9d7e272 100644 --- a/src/remote/remote-config-schema.ts +++ b/src/remote/remote-config-schema.ts @@ -1,5 +1,6 @@ import { PROVIDER_DEVICE_ORIENTATIONS, + PROVIDER_DEVICE_TYPES, type CloudProviderProfileFields, type RemoteConfigMetroOptions, type RemoteConnectionProfileFields, @@ -78,6 +79,7 @@ export const REMOTE_CONFIG_FIELD_SPECS = [ { key: 'session', type: 'string' }, { key: 'providerApp', type: 'string' }, { key: 'providerOsVersion', type: 'string' }, + { key: 'providerDeviceType', type: 'enum', enumValues: PROVIDER_DEVICE_TYPES }, { key: 'providerProject', type: 'string' }, { key: 'providerBuild', type: 'string' }, { key: 'providerSessionName', type: 'string' }, diff --git a/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts b/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts index d98a17f151..0fae68261d 100644 --- a/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts +++ b/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts @@ -185,6 +185,83 @@ test('AWS Device Farm facade rejects BrowserStack-owned device features at sessi }); }, 15_000); +test('TestMu facade routes a real-device session to the real pool and its upload API', async () => { + await withProviderScenarioResource(FakeCloudProviderServer.start, async (server) => { + const provider = createProviderWebDriver({ + clientVersion: CLIENT_VERSION, + runHostCommand: unexpectedHostCommand, + }); + const runtime = runtimeFor( + provider.createDefaultRuntimes({ + LT_USERNAME: 'user', + LT_ACCESS_KEY: 'key', + TESTMU_WEBDRIVER_ENDPOINT: `${server.url}/wd/hub/`, + TESTMU_APP_UPLOAD_ENDPOINT: `${server.url}/lt/upload/virtualDevice`, + TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT: `${server.url}/lt/upload/realDevice`, + }), + CLOUD_WEBDRIVER_PROVIDERS.testMu, + ); + const lease = makeLease(CLOUD_WEBDRIVER_PROVIDERS.testMu); + try { + await runtime.leaseLifecycle.allocate?.(lease, { + flags: { + platform: 'android', + device: 'Pixel 6', + providerOsVersion: '14', + providerDeviceType: 'real', + providerApp: 'https://builds.example/app.apk', + }, + }); + } finally { + await runtime.shutdown(); + } + + assert.deepEqual( + server.calls.filter((call) => call.path.startsWith('/lt/upload/')).map((call) => call.path), + ['/lt/upload/realDevice'], + ); + const session = server.calls.find((call) => call.path === '/wd/hub/session'); + const alwaysMatch = ( + session?.body as { capabilities?: { alwaysMatch?: Record } } | undefined + )?.capabilities?.alwaysMatch; + const ltOptions = alwaysMatch?.['lt:options'] as Record | undefined; + assert.equal(ltOptions?.isRealMobile, true); + assert.equal(ltOptions?.app, 'lt://REAL1'); + assert.equal(ltOptions?.platformVersion, '14'); + }); +}, 15_000); + +test('BrowserStack facade rejects the TestMu device type at session preparation', async () => { + await withProviderScenarioResource(FakeCloudProviderServer.start, async (server) => { + const provider = createProviderWebDriver({ + clientVersion: CLIENT_VERSION, + runHostCommand: unexpectedHostCommand, + }); + const runtime = runtimeFor( + provider.createDefaultRuntimes({ + BROWSERSTACK_USERNAME: 'user', + BROWSERSTACK_ACCESS_KEY: 'key', + BROWSERSTACK_WEBDRIVER_ENDPOINT: `${server.url}/wd/hub/`, + }), + CLOUD_WEBDRIVER_PROVIDERS.browserStack, + ); + const lease = makeLease(CLOUD_WEBDRIVER_PROVIDERS.browserStack); + const context = browserStackContext(lease); + try { + await assert.rejects( + async () => + await runtime.leaseLifecycle.allocate?.(lease, { + flags: { ...context.flags, providerDeviceType: 'real' }, + }), + /--provider-device-type is only supported by TestMu AI, not browserstack/, + ); + assert.deepEqual(server.calls, []); + } finally { + await runtime.shutdown(); + } + }); +}, 15_000); + test('AWS Device Farm facade uses the injected host-command capability for its full lifecycle', async () => { await withProviderScenarioResource(FakeCloudProviderServer.start, async (server) => { const host = new FakeAwsHostCommand(`${server.url}/wd/hub/`); @@ -518,6 +595,10 @@ class FakeCloudProviderServer extends CloudWebDriverTestServer { }); case 'POST /app-automate/upload': return cloudWebDriverTestJson({ app_url: 'bs://uploaded-app' }); + case 'POST /lt/upload/realDevice': + return cloudWebDriverTestJson({ app_url: 'lt://REAL1' }); + case 'POST /lt/upload/virtualDevice': + return cloudWebDriverTestJson({ app_url: 'lt://VIRTUAL1' }); case 'GET /app-automate/sessions/wd-1.json': return cloudWebDriverTestJson({ automation_session: { diff --git a/test/integration/provider-scenarios/daemon-http-lease-allocate.test.ts b/test/integration/provider-scenarios/daemon-http-lease-allocate.test.ts index fd615dd7fc..275c09b659 100644 --- a/test/integration/provider-scenarios/daemon-http-lease-allocate.test.ts +++ b/test/integration/provider-scenarios/daemon-http-lease-allocate.test.ts @@ -47,6 +47,7 @@ test('Provider-backed integration daemon HTTP lease allocate forwards provider a device: 'iPhone 15', providerApp: 'bs://app-id', providerOsVersion: '17', + providerDeviceType: 'real', providerProject: 'MyProject', providerBuild: 'Build-1', providerSessionName: 'smoke', @@ -64,6 +65,7 @@ test('Provider-backed integration daemon HTTP lease allocate forwards provider a device: 'iPhone 15', providerApp: 'bs://app-id', providerOsVersion: '17', + providerDeviceType: 'real', providerProject: 'MyProject', providerBuild: 'Build-1', providerSessionName: 'smoke', From add8062e2693d5b1d8c88168c30629041468f040 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 19:45:11 +0530 Subject: [PATCH 05/57] fix(provider-webdriver): upload the archive of materialized iOS builds Install from a remote source materializes an iOS build by extracting the `.app` bundle from a zipped simulator build or an .ipa. The WebDriver deployment runtime handed that extracted `.app` directory to the provider's uploader. Hosted upload APIs take a file, not a directory, so the upload could not succeed. When the materialized artifact is an iOS `.app` extracted from a `.zip` or `.ipa` archive, upload the archive it came from. Every other case uploads the installable path as before: no archive, an archive of another type, or an Android build. A provider without an uploader still installs the extracted bundle path. The bundle id and launch target hints are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../src/runtime-deployment.test.ts | 113 +++++++++++++++++- .../src/runtime-deployment.ts | 34 +++++- 2 files changed, 140 insertions(+), 7 deletions(-) diff --git a/packages/provider-webdriver/src/runtime-deployment.test.ts b/packages/provider-webdriver/src/runtime-deployment.test.ts index 67b06e14a9..559f3c37c9 100644 --- a/packages/provider-webdriver/src/runtime-deployment.test.ts +++ b/packages/provider-webdriver/src/runtime-deployment.test.ts @@ -1,6 +1,10 @@ -import { expect, test, vi } from 'vitest'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { afterEach, expect, test, vi } from 'vitest'; import { createCloudWebDriverCapabilities } from './capabilities.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; +import { createTestMuUploadApp } from './testmu.ts'; +import { mkdtempForTest } from './tmp-dir.fixtures.ts'; import { createWebDriverDeploymentRuntime } from './runtime-deployment.ts'; import type { WebDriverProviderSession } from './runtime-session.ts'; import type { CloudWebDriverUploadApp } from './runtime.ts'; @@ -14,6 +18,13 @@ const device: DeviceInfo = { booted: true, }; +const iosDevice: DeviceInfo = { ...device, platform: 'apple', id: 'webdriver:ios' }; +const realFetch = globalThis.fetch; + +afterEach(() => { + globalThis.fetch = realFetch; +}); + test('keeps a stale WebDriver owner unavailable before any deployment attempt', () => { const deployment = createWebDriverDeploymentRuntime({ provider: 'webdriver-test', @@ -75,6 +86,106 @@ test('aborts a WebDriver provider deployment while its install is in flight', as expect(installApp).toHaveBeenCalledWith('bs://uploaded-app', controller.signal); }); +// Materialization extracts `App.app.zip` (or an .ipa) to a `.app` directory, which no hosted +// upload API accepts; the archive it came from is the uploadable build. +test('a hosted upload of a materialized iOS bundle sends the archive it was extracted from', async () => { + const uploaded: string[] = []; + const installApp = vi.fn(async () => undefined); + const deployment = createWebDriverDeploymentRuntime({ + provider: 'webdriver-test', + uploadApp: async ({ appPath }) => { + uploaded.push(appPath); + return { appReference: `lt://${uploaded.length}` }; + }, + findSessionForDevice: () => activeSession(installApp), + }); + const deploy = async (selected: DeviceInfo, artifact: Record) => + await deployment.deployMaterializedApp( + selected, + { artifact: { installablePath: '', ...artifact, cleanup: async () => {} } }, + new AbortController().signal, + ); + + const result = await deploy(iosDevice, { + archivePath: '/m/App.app.zip', + installablePath: '/m/extracted/App.app', + bundleId: 'com.example.app', + }); + await deploy(iosDevice, { archivePath: '/m/App.ipa', installablePath: '/m/x/Payload/App.app' }); + await deploy(iosDevice, { installablePath: '/m/App.app' }); + await deploy(iosDevice, { archivePath: '/m/App.tar.gz', installablePath: '/m/x/App.app' }); + await deploy(device, { archivePath: '/m/build.zip', installablePath: '/m/x/app.apk' }); + + expect(uploaded).toEqual([ + '/m/App.app.zip', + '/m/App.ipa', + '/m/App.app', + '/m/x/App.app', + '/m/x/app.apk', + ]); + expect(result).toEqual({ bundleId: 'com.example.app', launchTarget: 'com.example.app' }); + expect(installApp).toHaveBeenNthCalledWith(1, 'lt://1', expect.any(AbortSignal)); +}); + +test('a provider without an uploader still installs the materialized bundle path', async () => { + const installApp = vi.fn(async () => undefined); + const deployment = createWebDriverDeploymentRuntime({ + provider: 'webdriver-test', + findSessionForDevice: () => activeSession(installApp), + }); + await deployment.deployMaterializedApp( + iosDevice, + { + artifact: { + archivePath: '/m/App.app.zip', + installablePath: '/m/extracted/App.app', + bundleId: 'com.example.app', + cleanup: async () => {}, + }, + }, + new AbortController().signal, + ); + expect(installApp).toHaveBeenCalledWith('/m/extracted/App.app', expect.any(AbortSignal)); +}); + +test('TestMu uploads the zipped simulator build that install-from-source extracted', async () => { + const tempDir = await mkdtempForTest('agent-device-materialized-upload-'); + try { + const archivePath = path.join(tempDir, 'App.app.zip'); + const installablePath = path.join(tempDir, 'extracted', 'App.app'); + await fs.writeFile(archivePath, 'zip bytes'); + await fs.mkdir(installablePath, { recursive: true }); + const uploadedNames: unknown[] = []; + globalThis.fetch = async (_input, init) => { + const body = init?.body; + if (!(body instanceof FormData)) throw new Error('expected a multipart upload'); + uploadedNames.push((body.get('appFile') as File).name); + return new Response(JSON.stringify({ app_id: 'APP42' }), { status: 200 }); + }; + const installApp = vi.fn(async () => undefined); + const deployment = createWebDriverDeploymentRuntime({ + provider: 'testmu', + uploadApp: createTestMuUploadApp({ + clientVersion: '0.0.0-test', + username: 'user', + accessKey: 'key', + }), + findSessionForDevice: () => activeSession(installApp), + }); + + await deployment.deployMaterializedApp( + iosDevice, + { artifact: { archivePath, installablePath, cleanup: async () => {} } }, + new AbortController().signal, + ); + + expect(uploadedNames).toEqual(['App.app.zip']); + expect(installApp).toHaveBeenCalledWith('lt://APP42', expect.any(AbortSignal)); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + function activeSession( installApp: (appPath: string, signal?: AbortSignal) => Promise, ): WebDriverProviderSession { diff --git a/packages/provider-webdriver/src/runtime-deployment.ts b/packages/provider-webdriver/src/runtime-deployment.ts index 07e3ecf16d..f88fe4f304 100644 --- a/packages/provider-webdriver/src/runtime-deployment.ts +++ b/packages/provider-webdriver/src/runtime-deployment.ts @@ -6,6 +6,7 @@ import type { AppDeploymentInput, AppDeploymentResult, DeployMaterializedAppInput, + MaterializedAppSource, } from '@agent-device/contracts/app-deployment-runtime'; import type { RuntimeOperationFact } from '@agent-device/contracts/platform-runtime'; import { publicPlatformString, type DeviceInfo } from '@agent-device/kernel/device'; @@ -49,10 +50,10 @@ export function createWebDriverDeploymentRuntime( findSessionForDevice(device: DeviceInfo): WebDriverProviderSession | undefined; }>, ): WebDriverDeploymentRuntime { - const installApp = async ( + const install = async ( device: DeviceInfo, app: string, - appPath: string, + paths: Readonly<{ appPath: string; uploadPath: string }>, installOptions?: ProviderDeviceInstallOptions, signal?: AbortSignal, ): Promise => { @@ -63,13 +64,20 @@ export function createWebDriverDeploymentRuntime( session, device, app, - appPath, + paths.uploadPath, installOptions, signal, ); - await session.client.installApp(upload?.appReference ?? appPath, signal); + await session.client.installApp(upload?.appReference ?? paths.appPath, signal); return providerInstallResult(upload, installOptions); }; + const installApp = async ( + device: DeviceInfo, + app: string, + appPath: string, + installOptions?: ProviderDeviceInstallOptions, + signal?: AbortSignal, + ) => await install(device, app, { appPath, uploadPath: appPath }, installOptions, signal); return Object.freeze({ fact: (device) => deploymentFact(options.findSessionForDevice(device)), installApp, @@ -92,10 +100,13 @@ export function createWebDriverDeploymentRuntime( ), deployMaterializedApp: async (device, input, signal) => deploymentResult( - await installApp( + await install( device, '', - input.artifact.installablePath, + { + appPath: input.artifact.installablePath, + uploadPath: materializedUploadPath(input.artifact), + }, { appIdentifierHint: input.artifact.bundleId, packageNameHint: input.artifact.packageName, @@ -106,6 +117,17 @@ export function createWebDriverDeploymentRuntime( }); } +/** + * Materialization extracts an iOS `.app` bundle out of a zipped simulator build or an .ipa, and no + * hosted upload API takes a directory, so the uploader gets the archive the bundle came from. + */ +function materializedUploadPath(artifact: MaterializedAppSource): string { + const { archivePath, installablePath } = artifact; + return archivePath && /\.(zip|ipa)$/i.test(archivePath) && /\.app\/?$/i.test(installablePath) + ? archivePath + : installablePath; +} + function deploymentFact(session: WebDriverProviderSession | undefined): RuntimeOperationFact { if (!session) { return Object.freeze({ From abf7a0a25a1dad2fe6451dfa2beaedbb84f1ae16 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 19:45:11 +0530 Subject: [PATCH 06/57] docs: add the TestMu AI device cloud guide Add a TestMu AI guide next to the BrowserStack and AWS Device Farm ones and link it from the sidebar. It covers credentials and `connect`, the exact device and OS version match, app references and uploads, device features, real devices with `--provider-device-type real`, the CLI and Node.js client workflows, artifacts, and endpoint overrides. List TestMu AI among the device clouds in the README, the device clouds overview, the client API page, and the command reference. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- website/docs/docs/_meta.json | 5 + website/docs/docs/client-api.md | 6 +- website/docs/docs/commands.md | 4 +- website/docs/docs/device-clouds.md | 5 +- website/docs/docs/testmu.md | 200 +++++++++++++++++++++++++++++ 6 files changed, 215 insertions(+), 9 deletions(-) create mode 100644 website/docs/docs/testmu.md diff --git a/README.md b/README.md index a5f9cb069c..9c056a81cb 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ The same session and evidence model works at every step: the agent explores the | --- | --- | --- | | Local | Trying commands and debugging apps on simulators, emulators, physical devices, macOS, and Linux. | Follow the Quick Start. | | CI/CD | Automated pull request and merge validation with replay scripts and captured artifacts. | Try the [EAS workflow template](https://github.com/callstackincubator/eas-agent-device/blob/main/.eas/workflows/agent-qa-mobile.yml). | -| Cloud / remote | Linux runners, managed devices, and remote jobs. | Set up a [remote proxy](https://oss.callstack.com/agent-device/docs/remote-proxy), connect a [device cloud](https://oss.callstack.com/agent-device/docs/device-clouds) (BrowserStack, AWS Device Farm, Limrun), or [contact Callstack](mailto:hello@callstack.com) for team QA. | +| Cloud / remote | Linux runners, managed devices, and remote jobs. | Set up a [remote proxy](https://oss.callstack.com/agent-device/docs/remote-proxy), connect a [device cloud](https://oss.callstack.com/agent-device/docs/device-clouds) (BrowserStack, AWS Device Farm, TestMu AI, Limrun), or [contact Callstack](mailto:hello@callstack.com) for team QA. | ## How it works @@ -145,7 +145,7 @@ The same session and evidence model works at every step: the agent explores the Support depth varies by target. Newer backends such as HarmonyOS and Vega OS cover a subset of commands; run `agent-device capabilities --platform ` to see what a target supports. -Sessions are scoped to the caller's git worktree, and host-local device claims stop parallel agents from taking over each other's simulators and emulators. Inspect ownership without a daemon via `agent-device device status`, and settle provably dead owners with `agent-device device release --stale`. The same commands drive hosted devices on [BrowserStack, AWS Device Farm, and Limrun](https://oss.callstack.com/agent-device/docs/device-clouds). +Sessions are scoped to the caller's git worktree, and host-local device claims stop parallel agents from taking over each other's simulators and emulators. Inspect ownership without a daemon via `agent-device device status`, and settle provably dead owners with `agent-device device release --stale`. The same commands drive hosted devices on [BrowserStack, AWS Device Farm, TestMu AI, and Limrun](https://oss.callstack.com/agent-device/docs/device-clouds). `agent-device` uses the inspect-act-verify process from Vercel's [agent-browser](https://github.com/vercel-labs/agent-browser) for mobile, TV, and desktop apps. Basic `--platform web` support runs `agent-browser` in the same session and replay system. diff --git a/website/docs/docs/_meta.json b/website/docs/docs/_meta.json index 68655a6001..b7d84f29fa 100644 --- a/website/docs/docs/_meta.json +++ b/website/docs/docs/_meta.json @@ -113,6 +113,11 @@ "label": "AWS Device Farm", "link": "/docs/aws-device-farm" }, + { + "type": "custom-link", + "label": "TestMu AI", + "link": "/docs/testmu" + }, { "type": "custom-link", "label": "Limrun", diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 9265e32296..7c3563fe35 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -117,7 +117,7 @@ stdout/stderr. The option mirrors `open --launch-console` and is not valid for U or contacts the daemon. Pass `{ stateDir }` to resolve an explicit override the same way the CLI resolves `--state-dir`. `client.sessions.artifacts({ provider, providerSessionId })` mirrors `artifacts --provider ... --provider-session ...` and returns provider-hosted `cloudArtifacts`. -Use it for BrowserStack or AWS Device Farm session videos/logs after a cloud session has stopped, or omit `providerSessionId` when an embedding host has registered a provider runtime that can infer the active lease. Limrun does not currently expose provider artifacts through this command. +Use it for BrowserStack, AWS Device Farm, or TestMu AI session videos/logs after a cloud session has stopped, or omit `providerSessionId` when an embedding host has registered a provider runtime that can infer the active lease. Limrun does not currently expose provider artifacts through this command. ```ts const result = await client.sessions.artifacts({ @@ -134,7 +134,7 @@ if ('cloudArtifacts' in result) { ## Device cloud sessions -Limrun, BrowserStack, and AWS Device Farm can be driven through the normal typed client methods. Use the corresponding CLI `connect` flow when you want persisted local connection state. Use direct client config when a Node integration already owns credentials and provider selectors. +Limrun, BrowserStack, AWS Device Farm, and TestMu AI can be driven through the normal typed client methods. Use the corresponding CLI `connect` flow when you want persisted local connection state. Use direct client config when a Node integration already owns credentials and provider selectors. ```ts import { createAgentDeviceClient } from 'agent-device'; @@ -158,7 +158,7 @@ from an explicit selector, an existing session, one local booted/bootable candid simulator with the app installed, or one provider-owned candidate. Ambiguous requests fail with structured retry selectors instead of silently retargeting. -Use `client.sessions.artifacts({ provider, providerSessionId })` with `closed.provider?.providerSessionId` to fetch provider-hosted video and log URLs after close. See the [BrowserStack](/docs/browserstack), [AWS Device Farm](/docs/aws-device-farm), and [Limrun](/docs/limrun) guides for provider-specific setup. +Use `client.sessions.artifacts({ provider, providerSessionId })` with `closed.provider?.providerSessionId` to fetch provider-hosted video and log URLs after close. See the [BrowserStack](/docs/browserstack), [AWS Device Farm](/docs/aws-device-farm), [TestMu AI](/docs/testmu), and [Limrun](/docs/limrun) guides for provider-specific setup. ## Web sessions diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 5af97672f6..44cffce0d2 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -114,7 +114,7 @@ agent-device fold open - Remote daemon clients can pass `--daemon-base-url http(s)://host:port[/base-path]` to skip local daemon discovery/startup and call a remote HTTP daemon directly. - Use `--daemon-auth-token ` (or `AGENT_DEVICE_DAEMON_AUTH_TOKEN`) for explicit service/API-token automation against non-loopback remote daemon URLs; the client sends it in both the JSON-RPC request token and HTTP auth headers. - Use [Remote Proxy](/docs/remote-proxy) when you need to run `agent-device proxy` on a Mac with simulator/device access and drive it from another machine through cloudflared, ngrok, or another HTTP tunnel. -- Use [BrowserStack](/docs/browserstack) or [AWS Device Farm](/docs/aws-device-farm) when a CI agent needs a hosted device session without interactive login. +- Use [BrowserStack](/docs/browserstack), [AWS Device Farm](/docs/aws-device-farm), or [TestMu AI](/docs/testmu) when a CI agent needs a hosted device session without interactive login. - For human cloud access, `connect` can discover a cloud connection profile, while `connect --remote-config ...` uses a local profile. Both refresh a stored CLI session into a short-lived `adc_agent_...` token when needed. If no CLI session exists, interactive shells start login automatically; CI and non-interactive shells fail with API-token setup instructions. Use `--no-login` to disable implicit login. `AGENT_DEVICE_CLOUD_BASE_URL` is the bridge/control-plane API origin; its `/api-keys` route may redirect to the dashboard for token creation. - For remote `connect` and `connect --remote-config` flows, see [Remote Metro workflow](#remote-metro-workflow). - Android React Native relaunch flows require an installed package name for `open --relaunch`; install/reinstall the APK first, then relaunch by package. `open --relaunch` is rejected because runtime hints are written through the installed app sandbox. @@ -1274,7 +1274,7 @@ agent-device artifacts --provider aws-device-farm --provider-session ` and `--provider `. BrowserStack uses `BROWSERSTACK_USERNAME` and `BROWSERSTACK_ACCESS_KEY`. AWS Device Farm uses the AWS CLI credential chain and infers the region from the session ARN when possible. See [BrowserStack](/docs/browserstack) and [AWS Device Farm](/docs/aws-device-farm) for CI credential setup. +- Historical lookup requires `--provider-session ` and `--provider `. BrowserStack uses `BROWSERSTACK_USERNAME` and `BROWSERSTACK_ACCESS_KEY`. TestMu AI uses `LT_USERNAME` and `LT_ACCESS_KEY`. AWS Device Farm uses the AWS CLI credential chain and infers the region from the session ARN when possible. See [BrowserStack](/docs/browserstack), [AWS Device Farm](/docs/aws-device-farm), and [TestMu AI](/docs/testmu) for CI credential setup. - When a cloud runtime is registered in-process by an embedding host, `artifacts` can infer the active provider session from the current lease before disconnect. - `disconnect --json` and `close --json` include provider release data when the runtime returns final cloud artifacts after session teardown. Some providers only finalize video/log URLs after the remote session is stopped, so retry `agent-device artifacts --provider --json` if the first response is `pending`. diff --git a/website/docs/docs/device-clouds.md b/website/docs/docs/device-clouds.md index b7f165dd5f..0e4333c012 100644 --- a/website/docs/docs/device-clouds.md +++ b/website/docs/docs/device-clouds.md @@ -9,9 +9,10 @@ Use a device cloud or farm when an agent needs to automate a hosted mobile devic - [BrowserStack](/docs/browserstack): Android and iOS App Automate sessions over WebDriver. - [AWS Device Farm](/docs/aws-device-farm): Android and iOS remote-access sessions through AWS. +- [TestMu AI](/docs/testmu): Android emulator, iOS simulator, and real-device sessions over WebDriver. - [Limrun](/docs/limrun): direct iOS simulator and Android emulator instances. -All three integrations run through the local `agent-device` daemon. `connect` checks the credentials and configuration, then saves non-secret connection state. It does not allocate a device. BrowserStack and AWS Device Farm allocate a hosted session on `open`. Limrun allocates an instance on the first device command, such as `install` or `open`. +All four integrations run through the local `agent-device` daemon. `connect` checks the credentials and configuration, then saves non-secret connection state. It does not allocate a device. BrowserStack, AWS Device Farm, and TestMu AI allocate a hosted session on `open`. Limrun allocates an instance on the first device command, such as `install` or `open`. For each provider, the standard lifecycle is: @@ -20,4 +21,4 @@ For each provider, the standard lifecycle is: 3. Follow the printed next command to install or open the app. 4. Run normal device commands, then `agent-device close` and `agent-device disconnect`. -Each provider guide covers its connection selectors, client configuration, MCP setup, artifacts, and troubleshooting. Generated remote profiles are safe to store as non-secret configuration. They may include app IDs, ARNs, device names, OS versions, and labels, but never provider API keys or AWS secret keys. +Each provider guide covers its connection selectors, client configuration, MCP setup, artifacts, and troubleshooting. Generated remote profiles are safe to store as non-secret configuration. They may include app IDs, ARNs, device names, OS versions, and labels, but never provider API keys, access keys, or AWS secret keys. diff --git a/website/docs/docs/testmu.md b/website/docs/docs/testmu.md new file mode 100644 index 0000000000..c2adc8266d --- /dev/null +++ b/website/docs/docs/testmu.md @@ -0,0 +1,200 @@ +--- +title: TestMu AI +description: Drive TestMu AI (formerly LambdaTest) virtual devices, Android emulators and iOS simulators, and real devices with agent-device. +--- + +# TestMu AI + +TestMu AI (formerly LambdaTest) hosts virtual devices for Android emulator and iOS simulator +WebDriver sessions, and real devices you select with `--provider-device-type real`. One Appium hub +fronts both pools; agent-device selects the pool with `isRealMobile` and defaults to the +virtual-device pool. + +## Credentials and connection + +Set TestMu AI credentials in a non-interactive environment. These are the same variables every +TestMu AI SDK reads: + +```bash +export LT_USERNAME=... +export LT_ACCESS_KEY=... +``` + +Connect with the platform, exact device name and OS version, and the app to test: + +```bash +agent-device connect testmu \ + --platform android \ + --device "Galaxy S22 Ultra 5G" \ + --provider-os-version 14 \ + --provider-app lt://APP-id +``` + +`--device` and `--provider-os-version` must match the virtual-device catalog spelling exactly. The +hub rejects `--provider-os-version 18` for a device listed with `18.0`, so `connect` does too and +lists the versions the device offers. + +`--provider-app` accepts a TestMu AI app reference such as `lt://APP...`, an HTTP(S) app URL, or +an existing local app path (`.apk`, or a zipped simulator `.app` for iOS). TestMu AI uploads a local +path or fetches a URL when it creates the hosted session, through the virtual-device upload API. + +During `connect`, agent-device checks the device/OS pair against TestMu AI's virtual-device +catalog (`/capability/generator?isVirtualDevice=true`), verifies the credentials against your +uploaded-app listing, matches an `lt://` reference against that listing, and confirms that a local +artifact exists before saving its absolute path. `open` still needs the app's installed package or +bundle identifier, not the upload name or `lt://` id. + +Optional labels: + +```bash +--provider-project agent-device +--provider-build "$GITHUB_RUN_ID" +--provider-session-name "$GITHUB_JOB" +``` + +Optional device features: + +```bash +--provider-device-orientation portrait # or landscape (alias --device-orientation) +--provider-geo-location US # (alias --geo-location) +--provider-timezone UTC+05:30 # (alias --timezone) +--provider-appium-version 2.16.2 # (alias --appium-version) +--provider-language fr # (alias --language) +--provider-locale fr_FR # (alias --locale) +``` + +TestMu AI receives these values in `lt:options` when it creates the hosted session. + +- Without `--provider-appium-version`, agent-device sends no Appium version and TestMu AI starts + its default server for the device. Pin a version when a suite depends on one. +- `--provider-network-profile`, `--provider-custom-network`, and `--provider-no-resign-app` are + BrowserStack capabilities; `connect testmu` and TestMu AI session creation refuse them by flag + name rather than ignoring them. +- Session video and device logs are requested on every session so `artifacts` has something to + return. + +## Real devices + +Pass `--provider-device-type real` to run on a physical device. Everything else works as for +virtual devices: `connect` checks the device/OS pair against the real-device catalog +(`/capability/generator?isVirtualDevice=false`), and a local path or URL is uploaded through the +real-device upload API. + +```bash +agent-device connect testmu \ + --provider-device-type real \ + --platform ios \ + --device "iPhone 16" \ + --provider-os-version 18 \ + --provider-app ./MyApp.ipa + +agent-device connect testmu \ + --provider-device-type real \ + --platform android \ + --device "Pixel 6" \ + --provider-os-version 14 \ + --provider-app https://example.com/builds/app.apk +``` + +- Real iOS devices are listed by major OS version only: use `--provider-os-version 18`, not `18.0`. + The exact-spelling check still applies, so `connect` rejects `18.0` for a real iPhone 16 and lists + the versions it offers. +- Real iOS devices install a signed `.ipa`; a zipped simulator `.app` only runs on simulators. + Android takes an `.apk` or `.aab`. +- Real and virtual devices have separate upload APIs. Pass an `lt://` id that was uploaded for the + pool you connect to; when in doubt, pass the local path or URL and let agent-device upload it. +- `TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT` redirects real-device uploads, as + `TESTMU_APP_UPLOAD_ENDPOINT` does for virtual-device uploads. +- `--provider-device-type` applies only to TestMu AI; other providers refuse it. + +## CLI workflow + +```bash +export LT_USERNAME=... +export LT_ACCESS_KEY=... + +agent-device connect testmu \ + --platform ios \ + --device "iPhone 16" \ + --provider-os-version 18.0 \ + --provider-app ./MyApp.app.zip \ + --provider-build "$GITHUB_RUN_ID" + +agent-device open com.example.app +agent-device snapshot -i +agent-device click 'label="Continue"' +agent-device close +agent-device artifacts --json +agent-device disconnect +``` + +For MCP-only use, run `connect` in the same effective state directory before starting +`agent-device mcp`. MCP exposes `open`, `snapshot`, `click`, `close`, and `artifacts`, but not +provider `connect` commands. + +## Node.js client + +The typed client reaches TestMu AI through a lease. Allocate one with the provider selectors, then +scope a client to it for normal commands. `sessions.close()` ends the hosted session and releases +the lease; `leases.release()` in `finally` is then a no-op, and still releases the lease when a +command fails first. The daemon reads `LT_USERNAME` and `LT_ACCESS_KEY` from its environment. Add +`providerDeviceType: 'real'` to `leases.allocate` to run on a real device. + +```ts +import { createAgentDeviceClient } from 'agent-device'; + +const scope = { + tenant: 'testmu', + runId: process.env.GITHUB_RUN_ID ?? 'local-run', + leaseBackend: 'ios-instance', + leaseProvider: 'testmu', +} as const; + +const lease = await createAgentDeviceClient().leases.allocate({ + ...scope, + platform: 'ios', + device: 'iPhone 16', + providerOsVersion: '18.0', + providerApp: 'lt://APP-id', + providerProject: 'agent-device', + providerBuild: process.env.GITHUB_RUN_ID, +}); +const client = createAgentDeviceClient({ ...scope, leaseId: lease.leaseId }); + +let providerSessionId: string | undefined; +try { + await client.apps.open({ app: 'com.example.app' }); + await client.capture.snapshot({ interactiveOnly: true }); + await client.interactions.click({ selector: 'label="Continue"' }); + const closed = await client.sessions.close(); + providerSessionId = closed.provider?.providerSessionId; +} finally { + await client.leases.release({ ...scope, leaseId: lease.leaseId }); +} + +if (providerSessionId) { + const artifacts = await client.sessions.artifacts({ provider: 'testmu', providerSessionId }); + if ('cloudArtifacts' in artifacts) console.log(artifacts.cloudArtifacts); +} +``` + +## Artifacts and troubleshooting + +After `close`, TestMu AI can return session video, Appium logs, device logs, network and command +logs, a screenshot archive, and the App Automation dashboard link. Run `agent-device artifacts +--json`, or look up a previous session explicitly: + +```bash +agent-device artifacts --provider testmu --json +``` + +The TestMu AI session id is the WebDriver session id. If artifact lookup is pending immediately +after `close`, retry it; TestMu AI finalizes video and log URLs after the session ends. + +Endpoints can be redirected for a staging or private TestMu AI deployment with +`TESTMU_WEBDRIVER_ENDPOINT`, `TESTMU_APP_UPLOAD_ENDPOINT` (virtual devices), +`TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT` (real devices), and `TESTMU_API_ENDPOINT`. + +On hosted WebDriver sessions, `fill` checks that the field received focus before it sends keys. If +it cannot confirm focus, it fails without typing. Use `snapshot -i` to confirm the target, or +`press ` followed by `type `. From 5495cfab3ad4a3b8efbfe94144aad333e4df7d94 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 22:10:57 +0530 Subject: [PATCH 07/57] fix(daemon): start a lease's TTL when its allocation completes The registry records a lease, and stamps its expiry, before the lease lifecycle provider allocates the session behind it. Hosted providers can spend 30-80 seconds creating that session, so a lease on the default one-minute inactivity window was expired, or nearly so, by the time its client received it: the next command failed with "Lease is not active", release reported nothing to release, and the paid provider session was left running. The expiry sweep could also reap the lease while the provider was still allocating. Hold a lease work pass for the duration of the provider allocation, the same protection admitted request work gets. The lease cannot expire underneath the allocation, and ending the pass while the requester is still waiting renews the lease for its own TTL from that moment. The response now carries the renewed lease. A requester that hung up still protects nothing, and its allocation is released as before. Co-Authored-By: Claude Opus 5.5 --- docs/adr/0007-remote-device-leases.md | 10 +++ .../__tests__/lease-artifacts.test.ts | 57 +++++++++++-- src/daemon/handlers/__tests__/lease.test.ts | 79 +++++++++++++++++++ src/daemon/handlers/lease.ts | 13 ++- website/docs/docs/remote-proxy.md | 2 +- 5 files changed, 151 insertions(+), 10 deletions(-) diff --git a/docs/adr/0007-remote-device-leases.md b/docs/adr/0007-remote-device-leases.md index d64f08b6ae..d386648e08 100644 --- a/docs/adr/0007-remote-device-leases.md +++ b/docs/adr/0007-remote-device-leases.md @@ -67,6 +67,16 @@ one minute, while a cloud WebDriver connection profile asks for ten. A single co longer than its own lease is therefore ordinary on the default and only reachable through a profile on the longer one. +## Provider allocation + +Allocation is admitted work too. The registry records a lease before a hosted provider creates the +session behind it, and creating that session can take longer than the lease's inactivity TTL, so a +lease timed from its record was already expired, or nearly so, when its client first received it, and +the paid session it pointed at was orphaned. The allocation therefore holds a work pass while the +provider allocates: the lease cannot expire underneath it, and a successful allocation still wanted by +its requester starts the inactivity TTL from the moment allocation completed. A requester that hung up +preserves nothing, and its allocation is released as before. + ## Client-side work that precedes admission Protecting admitted work covers nothing that happens before a request is admitted. Installing an diff --git a/src/daemon/handlers/__tests__/lease-artifacts.test.ts b/src/daemon/handlers/__tests__/lease-artifacts.test.ts index 8067a4fdae..a6fd63a376 100644 --- a/src/daemon/handlers/__tests__/lease-artifacts.test.ts +++ b/src/daemon/handlers/__tests__/lease-artifacts.test.ts @@ -2,7 +2,8 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import type { CloudArtifactsQuery } from '@agent-device/contracts/observability'; import type { DeviceLease } from '@agent-device/contracts/device'; -import { AppError } from '@agent-device/kernel/errors'; +import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors'; +import { clearRequestCanceled, markRequestCanceled } from '@agent-device/host-kit/request'; import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; import type { DaemonRequest, DaemonResponse } from '../../daemon-request.ts'; import { handleLeaseCommands } from '../lease.ts'; @@ -117,7 +118,7 @@ test('artifacts refuses an expired provider session after retention before lazy assert.deepEqual(world.providerCalls, []); }); -test('artifacts refuses a provider session returned after allocation expiry retention', async () => { +test('artifacts lists a provider session whose allocation outlasted the lease TTL', async () => { let now = 1_000; const world = createWorld({ now: () => now, @@ -128,16 +129,60 @@ test('artifacts refuses a provider session returned after allocation expiry rete }); world.lifecycle.allocate = async (lease) => { now = lease.expiresAt + 51; - return { providerSessionId: 'late-allocation-session' }; + return { providerSessionId: 'slow-allocation-session' }; }; await allocateLease(world, 'tenant-a', 'run-a'); - await assertProviderSessionNotOwned(world, { + const listed = await listArtifacts(world, { tenantId: 'tenant-a', runId: 'run-a', - providerSessionId: 'late-allocation-session', + providerSessionId: 'slow-allocation-session', }); - assert.deepEqual(world.providerCalls, []); + assert.equal(listed.ok, true); +}); + +test('artifacts refuses a provider session returned after a canceled allocation outlived retention', async () => { + let now = 1_000; + const world = createWorld({ + now: () => now, + defaultLeaseTtlMs: 100, + minLeaseTtlMs: 1, + maxLeaseTtlMs: 100, + providerSessionRetentionMs: 50, + }); + const requestId = 'late-canceled-allocation'; + world.lifecycle.allocate = async (lease) => { + markRequestCanceled(requestId); + now = lease.expiresAt + 51; + return { providerSessionId: 'late-allocation-session' }; + }; + + try { + await assert.rejects( + handleLeaseCommands({ + req: leaseRequest('lease_allocate', { + requestId, + tenantId: 'tenant-a', + runId: 'run-a', + leaseBackend: 'android-instance', + leaseProvider: CLOUD_PROVIDER, + }), + sessionName: 'artifact-test', + sessionStore: world.sessionStore, + leaseRegistry: world.leaseRegistry, + leaseLifecycleProvider: world.lifecycle, + }), + isRequestCanceledError, + ); + await assertProviderSessionNotOwned(world, { + tenantId: 'tenant-a', + runId: 'run-a', + providerSessionId: 'late-allocation-session', + }); + assert.deepEqual(world.providerCalls, []); + } finally { + clearRequestCanceled(requestId); + } }); test('artifacts refuses a provider session returned after release expiry retention', async () => { diff --git a/src/daemon/handlers/__tests__/lease.test.ts b/src/daemon/handlers/__tests__/lease.test.ts index 08fdb22c7f..5cbd378ea6 100644 --- a/src/daemon/handlers/__tests__/lease.test.ts +++ b/src/daemon/handlers/__tests__/lease.test.ts @@ -2,7 +2,9 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import { handleLeaseCommands } from '../lease.ts'; import { LeaseRegistry } from '../../lease-registry.ts'; +import type { DaemonRequest } from '../../daemon-request.ts'; import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; +import type { DeviceLease } from '@agent-device/contracts/device'; import { AppError } from '@agent-device/kernel/errors'; import { clearRequestCanceled, markRequestCanceled } from '@agent-device/host-kit/request'; import { @@ -93,3 +95,80 @@ test('activation drains canceled provider allocation and its release cleanup', a clearRequestCanceled(requestId); } }); + +function allocateRequest(): DaemonRequest { + return { + token: 'test-token', + session: 'lease-ttl-test', + command: 'lease_allocate', + positionals: [], + flags: {}, + meta: { + tenantId: 'tenant-a', + runId: 'run-a', + clientId: 'client-a', + leaseBackend: 'android-instance', + leaseProvider: 'cloud', + }, + }; +} + +// A hosted provider can spend longer creating its session than the lease's inactivity +// TTL. Stamping the TTL when the registry record is created handed the client a lease +// that was already expired, and the paid session behind it was orphaned. +test('a lease whose provider allocation outlasts its TTL is active when allocation returns', async () => { + let now = 0; + const registry = new LeaseRegistry({ now: () => now, defaultLeaseTtlMs: 60_000 }); + const response = await handleLeaseCommands({ + req: allocateRequest(), + sessionName: 'lease-ttl-test', + sessionStore: makeSessionStore('agent-device-slow-provider-'), + leaseRegistry: registry, + leaseLifecycleProvider: { + allocate: async (lease) => { + now = 80_000; + assert.deepEqual( + registry.consumeExpiredLeases(), + [], + 'the sweeper must not reap a lease mid-allocation', + ); + now = 90_000; + return { providerSessionId: `session-${lease.leaseId}` }; + }, + }, + }); + + assert.equal(response?.ok, true); + const lease = (response?.ok ? response.data?.lease : undefined) as DeviceLease; + assert.equal(lease.expiresAt, 150_000); + assert.deepEqual( + registry.listActiveLeases().map((entry) => [entry.leaseId, entry.expiresAt]), + [[lease.leaseId, 150_000]], + ); + now = 149_999; + registry.assertLeaseAdmission({ + leaseId: lease.leaseId, + tenantId: lease.tenantId, + runId: lease.runId, + leaseBackend: lease.backend, + leaseProvider: lease.leaseProvider, + }); +}); + +test('a lease allocated without a provider keeps the TTL it was created with', async () => { + const now = 5_000; + const registry = new LeaseRegistry({ now: () => now, defaultLeaseTtlMs: 60_000 }); + const response = await handleLeaseCommands({ + req: allocateRequest(), + sessionName: 'lease-ttl-test', + sessionStore: makeSessionStore('agent-device-no-provider-'), + leaseRegistry: registry, + }); + + assert.equal(response?.ok, true); + const lease = (response?.ok ? response.data?.lease : undefined) as DeviceLease; + assert.equal(lease.createdAt, 5_000); + assert.equal(lease.heartbeatAt, 5_000); + assert.equal(lease.expiresAt, 65_000); + assert.deepEqual(registry.listActiveLeases(), [lease]); +}); diff --git a/src/daemon/handlers/lease.ts b/src/daemon/handlers/lease.ts index fa597823e2..f0826bb87f 100644 --- a/src/daemon/handlers/lease.ts +++ b/src/daemon/handlers/lease.ts @@ -71,20 +71,26 @@ export async function handleLeaseCommands(args: LeaseHandlerArgs): Promise { let providerData: Record | undefined; + // A hosted provider can take longer than the lease TTL to create its session; the work + // pass keeps the lease alive until it does, and ending the pass restarts the TTL then. + const work = leaseRegistry.retainLeaseWork(lease, () => !isRequestCanceled(requestId)); try { providerData = await leaseLifecycleProvider?.allocate?.(lease, { ...leaseLifecycleContext(req), - signal: getRequestSignal(req.meta?.requestId), + signal: getRequestSignal(requestId), deadline: Date.now() + LEASE_ALLOCATION_BUDGET_MS, }); recordProviderSession(leaseRegistry, lease, providerData); } catch (error) { leaseRegistry.releaseLease(leaseReleaseRequestFor(lease)); throw error; + } finally { + work.release(); } - if (isRequestCanceled(req.meta?.requestId)) { + if (isRequestCanceled(requestId)) { // The requester left while the provider was allocating; the lease it // produced is real (and billed) and nobody will ever release it. throw await releaseAllocationForGoneRequester( @@ -93,9 +99,10 @@ export async function handleLeaseCommands(args: LeaseHandlerArgs): Promise` instead of exporting the environment variable also works, but only authenticates the single command it is passed to; subsequent commands need the token again through the env var, a `daemonAuthToken` entry in your remote config profile, or a repeated `--daemon-auth-token` flag. -`connect proxy` stores the proxy profile and client identity. Device leases are automatic on `open` and expire after five minutes without commands. That five minutes is the window `open` asks for; a lease allocated directly over the RPC without `ttlMs` keeps the daemon's one-minute inactivity default instead. `close` releases the active session and device lease; `disconnect` clears local connection state. +`connect proxy` stores the proxy profile and client identity. Device leases are automatic on `open` and expire after five minutes without commands. That five minutes is the window `open` asks for; a lease allocated directly over the RPC without `ttlMs` keeps the daemon's one-minute inactivity default instead. Either window starts when allocation completes, not when it was requested. `close` releases the active session and device lease; `disconnect` clears local connection state. Multiple agents can share one proxy when each uses the normal `connect proxy`, `open`, commands, `close`, and `disconnect` flow. A busy device error means another agent owns the device until it closes or its inactivity lease expires. From 313a7d71aebee3b745278120515ae12229d35d0a Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 22:25:12 +0530 Subject: [PATCH 08/57] refactor(provider-webdriver): declare the profile fields each provider consumes Lease allocation forwards every provider profile field, but only some routes checked which ones a provider could act on. The connect builders and the BrowserStack/AWS session preparation refused flags they did not own, while the Limrun lease runtime and a remote-config profile skipped that check. A typed client asking Limrun for providerDeviceType 'real' was therefore given a simulator without an error. Each lease provider now declares, for every Cloud provider profile field, whether it consumes or refuses it. The declaration is a total record over CloudProviderProfileFields, so adding a field fails to build until every provider decides. One refusal, derived from that declaration, runs at lease preparation (the WebDriver session manager for BrowserStack, AWS Device Farm and TestMu AI, and Limrun allocation) and at connect, so connect, leases.allocate and remote-config profiles all fail the same way with INVALID_ARGS naming the flag and the provider. This replaces the per-provider TestMu-only, TestMu-unsupported and BrowserStack-only checks, the hand-copied complement of the BrowserStack feature table, and the ./testmu-device-features package subpath. AWS Device Farm now also refuses the hub-only app, OS version, project and build flags it never read. Co-Authored-By: Claude Opus 5.5 --- packages/contracts/package.json | 4 + .../src/provider-profile-fields.test.ts | 73 ++++++++++ .../contracts/src/provider-profile-fields.ts | 66 +++++++++ packages/provider-limrun/src/device.ts | 29 ++++ packages/provider-limrun/src/index.ts | 2 +- packages/provider-limrun/src/runtime.ts | 2 + .../provider-limrun/src/session-allocation.ts | 7 +- packages/provider-webdriver/package.json | 4 - .../src/browserstack-device-features.ts | 41 ------ packages/provider-webdriver/src/index.ts | 4 +- .../src/provider-definitions.ts | 117 +++++++++++++--- .../src/provider-profile-fields.test.ts | 56 ++++++++ .../provider-webdriver/src/runtime-session.ts | 6 + packages/provider-webdriver/src/runtime.ts | 3 + .../src/testmu-device-features.test.ts | 67 +--------- .../src/testmu-device-features.ts | 56 -------- .../layering/contracts-exports.snapshot.json | 1 + scripts/layering/package-boundaries.test.ts | 1 - src/__tests__/cloud-connect-profile.test.ts | 4 +- src/__tests__/cloud-connect-testmu.test.ts | 25 +++- .../lease-provider-profile-fields.test.ts | 126 ++++++++++++++++++ src/cli/connection/cloud-webdriver-profile.ts | 18 ++- src/cli/connection/limrun-profile.ts | 5 +- .../cloud-webdriver-provider-adapters.test.ts | 8 +- website/docs/docs/device-clouds.md | 2 + website/docs/docs/testmu.md | 3 +- 26 files changed, 514 insertions(+), 216 deletions(-) create mode 100644 packages/contracts/src/provider-profile-fields.test.ts create mode 100644 packages/contracts/src/provider-profile-fields.ts create mode 100644 packages/provider-webdriver/src/provider-profile-fields.test.ts create mode 100644 src/__tests__/lease-provider-profile-fields.test.ts diff --git a/packages/contracts/package.json b/packages/contracts/package.json index 328cdce13a..bee7b0f6f9 100644 --- a/packages/contracts/package.json +++ b/packages/contracts/package.json @@ -364,6 +364,10 @@ "types": "./src/runtime-operation-names.ts", "default": "./src/runtime-operation-names.ts" }, + "./provider-profile-fields": { + "types": "./src/provider-profile-fields.ts", + "default": "./src/provider-profile-fields.ts" + }, "./progress": { "types": "./src/facades/progress.ts", "default": "./src/facades/progress.ts" diff --git a/packages/contracts/src/provider-profile-fields.test.ts b/packages/contracts/src/provider-profile-fields.test.ts new file mode 100644 index 0000000000..000d72a020 --- /dev/null +++ b/packages/contracts/src/provider-profile-fields.test.ts @@ -0,0 +1,73 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { AppError } from '@agent-device/kernel/errors'; +import { + rejectRefusedProviderProfileFields, + type ProviderProfileFieldDeclaration, +} from './provider-profile-fields.ts'; + +const DECLARATION: ProviderProfileFieldDeclaration = { + provider: 'fake', + label: 'Fake Cloud', + fields: { + providerApp: 'consumed', + providerOsVersion: 'consumed', + providerDeviceType: 'refused', + providerProject: 'consumed', + providerBuild: 'consumed', + providerSessionName: 'consumed', + providerDeviceOrientation: 'consumed', + providerGeoLocation: 'refused', + providerTimezone: 'consumed', + providerAppiumVersion: 'consumed', + providerLanguage: 'consumed', + providerLocale: 'consumed', + providerNetworkProfile: 'consumed', + providerCustomNetwork: 'consumed', + providerNoResignApp: 'refused', + awsProjectArn: 'consumed', + awsDeviceArn: 'consumed', + awsAppArn: 'consumed', + awsRegion: 'consumed', + awsInteractionMode: 'consumed', + }, +}; + +test('consumed, unset, empty, and false fields pass', () => { + assert.doesNotThrow(() => rejectRefusedProviderProfileFields(undefined, DECLARATION)); + assert.doesNotThrow(() => + rejectRefusedProviderProfileFields( + { + providerApp: 'app', + providerDeviceType: '', + providerGeoLocation: undefined, + providerNoResignApp: false, + unrelated: 'x', + }, + DECLARATION, + ), + ); +}); + +test('a refused field fails with its flag and the provider named', () => { + assert.throws( + () => rejectRefusedProviderProfileFields({ providerDeviceType: 'real' }, DECLARATION), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + error.message === '--provider-device-type is not supported by Fake Cloud.' && + error.details?.provider === 'fake' && + JSON.stringify(error.details?.flags) === '["--provider-device-type"]', + ); +}); + +test('every refused field is reported at once', () => { + assert.throws( + () => + rejectRefusedProviderProfileFields( + { providerGeoLocation: 'US', providerNoResignApp: true, providerDeviceType: 'virtual' }, + DECLARATION, + ), + /--provider-device-type, --provider-geo-location, --provider-no-resign-app are not supported by Fake Cloud\./, + ); +}); diff --git a/packages/contracts/src/provider-profile-fields.ts b/packages/contracts/src/provider-profile-fields.ts new file mode 100644 index 0000000000..800dd145a8 --- /dev/null +++ b/packages/contracts/src/provider-profile-fields.ts @@ -0,0 +1,66 @@ +import { AppError } from '@agent-device/kernel/errors'; +import type { CloudProviderProfileFields } from './remote-config-fields.ts'; + +export type ProviderProfileField = keyof CloudProviderProfileFields; + +/** + * What one lease provider does with every Cloud provider profile field. The record is total, so a + * field added to the profile fails to build until each provider says whether it consumes it; a + * field a provider neither reads nor refuses would otherwise ride the profile and be dropped. + */ +export type ProviderProfileFieldDeclaration = Readonly<{ + provider: string; + label: string; + fields: Readonly>; +}>; + +const PROVIDER_PROFILE_FIELD_FLAGS: Readonly> = { + providerApp: '--provider-app', + providerOsVersion: '--provider-os-version', + providerDeviceType: '--provider-device-type', + providerProject: '--provider-project', + providerBuild: '--provider-build', + providerSessionName: '--provider-session-name', + providerDeviceOrientation: '--provider-device-orientation', + providerGeoLocation: '--provider-geo-location', + providerTimezone: '--provider-timezone', + providerAppiumVersion: '--provider-appium-version', + providerLanguage: '--provider-language', + providerLocale: '--provider-locale', + providerNetworkProfile: '--provider-network-profile', + providerCustomNetwork: '--provider-custom-network', + providerNoResignApp: '--provider-no-resign-app', + awsProjectArn: '--aws-project-arn', + awsDeviceArn: '--aws-device-arn', + awsAppArn: '--aws-app-arn', + awsRegion: '--aws-region', + awsInteractionMode: '--aws-interaction-mode', +}; + +/** + * Fails when `flags` set a profile field the provider refuses. Every route to a provider — connect, + * `leases.allocate`, a hand-authored remote-config profile — runs this against the same declaration. + */ +export function rejectRefusedProviderProfileFields( + flags: Readonly> | undefined, + declaration: ProviderProfileFieldDeclaration, +): void { + const refused = (Object.keys(declaration.fields) as ProviderProfileField[]) + .filter((field) => declaration.fields[field] === 'refused' && isSet(flags?.[field])) + .map((field) => PROVIDER_PROFILE_FIELD_FLAGS[field]); + if (refused.length === 0) return; + const plural = refused.length !== 1; + throw new AppError( + 'INVALID_ARGS', + `${refused.join(', ')} ${plural ? 'are' : 'is'} not supported by ${declaration.label}.`, + { + hint: `Drop ${plural ? 'those flags' : 'the flag'} or use a provider that supports ${plural ? 'them' : 'it'}.`, + provider: declaration.provider, + flags: refused, + }, + ); +} + +function isSet(value: unknown): boolean { + return value !== undefined && value !== null && value !== false && value !== ''; +} diff --git a/packages/provider-limrun/src/device.ts b/packages/provider-limrun/src/device.ts index a61e41b3e2..2d19e5e59b 100644 --- a/packages/provider-limrun/src/device.ts +++ b/packages/provider-limrun/src/device.ts @@ -1,10 +1,39 @@ import type { DeviceLease } from '@agent-device/contracts/device'; import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { ProviderProfileFieldDeclaration } from '@agent-device/contracts/provider-profile-fields'; export type LimrunPlatform = 'ios' | 'android'; export const LIMRUN_PROVIDER = 'limrun'; +/** Limrun reads only the app to preinstall; it picks the instance itself. */ +export const LIMRUN_PROFILE_FIELDS: ProviderProfileFieldDeclaration = { + provider: LIMRUN_PROVIDER, + label: 'Limrun', + fields: { + providerApp: 'consumed', + providerOsVersion: 'refused', + providerDeviceType: 'refused', + providerProject: 'refused', + providerBuild: 'refused', + providerSessionName: 'refused', + providerDeviceOrientation: 'refused', + providerGeoLocation: 'refused', + providerTimezone: 'refused', + providerAppiumVersion: 'refused', + providerLanguage: 'refused', + providerLocale: 'refused', + providerNetworkProfile: 'refused', + providerCustomNetwork: 'refused', + providerNoResignApp: 'refused', + awsProjectArn: 'refused', + awsDeviceArn: 'refused', + awsAppArn: 'refused', + awsRegion: 'refused', + awsInteractionMode: 'refused', + }, +}; + const LIMRUN_DEVICE_ID_PREFIX = LIMRUN_PROVIDER; export function platformForLimrunLeaseBackend(backend: string): LimrunPlatform | undefined { diff --git a/packages/provider-limrun/src/index.ts b/packages/provider-limrun/src/index.ts index d29e531c0c..f3ee00a35e 100644 --- a/packages/provider-limrun/src/index.ts +++ b/packages/provider-limrun/src/index.ts @@ -1,4 +1,4 @@ -export { LIMRUN_PROVIDER } from './device.ts'; +export { LIMRUN_PROFILE_FIELDS, LIMRUN_PROVIDER } from './device.ts'; export { createLimrunRuntime, type LimrunRuntime, type LimrunRuntimeOptions } from './runtime.ts'; export { verifyLimrunConnection } from './connection-verification.ts'; diff --git a/packages/provider-limrun/src/runtime.ts b/packages/provider-limrun/src/runtime.ts index d3528ea1b1..f1fb902b1e 100644 --- a/packages/provider-limrun/src/runtime.ts +++ b/packages/provider-limrun/src/runtime.ts @@ -237,9 +237,11 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { const { allocateLimrunAndroidSession, allocateLimrunIosSession, + rejectRefusedLimrunProfileFields, resolvePreinstalledAppId, resolveRequestedLimrunAppAsset, } = await import('./session-allocation.ts'); + rejectRefusedLimrunProfileFields(context); const requestedAsset = await resolveRequestedLimrunAppAsset(this.limrun, platform, context); const session = platform === 'ios' diff --git a/packages/provider-limrun/src/session-allocation.ts b/packages/provider-limrun/src/session-allocation.ts index de83a71366..a78bb4edd2 100644 --- a/packages/provider-limrun/src/session-allocation.ts +++ b/packages/provider-limrun/src/session-allocation.ts @@ -1,8 +1,9 @@ import type Limrun from '@limrun/api'; import type { DeviceLease, LeaseLifecycleContext } from '@agent-device/contracts/device'; import { AppError } from '@agent-device/kernel/errors'; +import { rejectRefusedProviderProfileFields } from '@agent-device/contracts/provider-profile-fields'; import { createLimrunAndroidSession, type LimrunAndroidSession } from './android.ts'; -import { buildLimrunDevice } from './device.ts'; +import { buildLimrunDevice, LIMRUN_PROFILE_FIELDS } from './device.ts'; import { createLimrunIosSession, type LimrunIosSession } from './ios.ts'; import { assertLimrunUploadedAppAccess, @@ -34,6 +35,10 @@ type SessionAllocationParams = Readonly<{ dependencies: LimrunRuntimeDependencies; }>; +export function rejectRefusedLimrunProfileFields(context?: LeaseLifecycleContext): void { + rejectRefusedProviderProfileFields(context?.flags, LIMRUN_PROFILE_FIELDS); +} + export async function resolveRequestedLimrunAppAsset( limrun: Limrun, platform: 'android' | 'ios', diff --git a/packages/provider-webdriver/package.json b/packages/provider-webdriver/package.json index a9f01ebacb..d4d56b8d0e 100644 --- a/packages/provider-webdriver/package.json +++ b/packages/provider-webdriver/package.json @@ -19,10 +19,6 @@ "./providers": { "types": "./src/providers.ts", "default": "./src/providers.ts" - }, - "./testmu-device-features": { - "types": "./src/testmu-device-features.ts", - "default": "./src/testmu-device-features.ts" } } } diff --git a/packages/provider-webdriver/src/browserstack-device-features.ts b/packages/provider-webdriver/src/browserstack-device-features.ts index 3ea041add7..275ae64ff3 100644 --- a/packages/provider-webdriver/src/browserstack-device-features.ts +++ b/packages/provider-webdriver/src/browserstack-device-features.ts @@ -130,47 +130,6 @@ export function buildBrowserStackDeviceFeatureCapabilities( return capabilities; } -/** - * Canonical CLI flags for every device-feature capability set on `flags`. - * - * These capabilities are BrowserStack-owned. Other providers have no equivalent, so a caller who - * passes them to AWS Device Farm would otherwise have them accepted, persisted into the profile, - * and then silently dropped — the session runs with provider defaults and nothing says why. - * Callers use this to reject them at the point the provider is known. - */ -function browserStackOnlyDeviceFeatureFlags(flags: Record | undefined): string[] { - return BROWSERSTACK_DEVICE_FEATURE_SPECS.filter((spec) => { - const value = flags?.[spec.field]; - return value !== undefined && value !== false && value !== ''; - }).map((spec) => spec.flag); -} - -/** - * Fails when a non-BrowserStack provider was given BrowserStack-owned device features. - * - * Called from both the CLI profile builder and the provider's own session preparation. The second - * is the one that actually closes the hole: the typed client and hand-authored remote-config - * profiles reach session preparation without passing through `connect`, so a CLI-only check leaves - * those routes accepting the capabilities and dropping them. - */ -export function rejectBrowserStackOnlyDeviceFeatures( - flags: Record | undefined, - provider: string, -): void { - const configured = browserStackOnlyDeviceFeatureFlags(flags); - if (configured.length === 0) return; - const plural = configured.length !== 1; - throw new AppError( - 'INVALID_ARGS', - `${configured.join(', ')} ${plural ? 'are' : 'is'} only supported by BrowserStack, not ${provider}.`, - { - hint: `Drop ${plural ? 'those flags' : 'the flag'} or use the browserstack provider.`, - provider, - flags: configured, - }, - ); -} - /** * Reads device-feature fields off an untyped flag bag (a daemon request), so the daemon-side * capability build and the CLI-side profile build stay driven by the same table. diff --git a/packages/provider-webdriver/src/index.ts b/packages/provider-webdriver/src/index.ts index cba755ce82..12c30acf60 100644 --- a/packages/provider-webdriver/src/index.ts +++ b/packages/provider-webdriver/src/index.ts @@ -4,6 +4,7 @@ import type { } from '@agent-device/contracts/observability'; import type { ProviderWebDriverDependencies } from './dependencies.ts'; import { + CLOUD_WEBDRIVER_PROFILE_FIELDS, createCloudWebDriverProviderDefinitions, type DefaultCloudWebDriverArtifactEnv, type DefaultCloudWebDriverProviderRuntimeEnv, @@ -17,9 +18,8 @@ import { } from './connection-verification.ts'; import type { CloudWebDriverRuntime } from './runtime.ts'; -export { CLOUD_WEBDRIVER_PROVIDERS }; +export { CLOUD_WEBDRIVER_PROFILE_FIELDS, CLOUD_WEBDRIVER_PROVIDERS }; export { readAwsDeviceFarmRegionFromArn }; -export { rejectBrowserStackOnlyDeviceFeatures } from './browserstack-device-features.ts'; export type { CloudWebDriverKnownProviderName } from './providers.ts'; export type { ProviderWebDriverDependencies, RunHostCommand } from './dependencies.ts'; export type { diff --git a/packages/provider-webdriver/src/provider-definitions.ts b/packages/provider-webdriver/src/provider-definitions.ts index dbc2092f7a..6981043fc5 100644 --- a/packages/provider-webdriver/src/provider-definitions.ts +++ b/packages/provider-webdriver/src/provider-definitions.ts @@ -1,6 +1,7 @@ import type { CloudArtifactsResult } from '@agent-device/contracts/observability'; import type { LeaseLifecycleContext } from '@agent-device/contracts/device'; import type { ProviderDeviceType } from '@agent-device/contracts/remote'; +import type { ProviderProfileFieldDeclaration } from '@agent-device/contracts/provider-profile-fields'; import { AppError } from '@agent-device/kernel/errors'; import type { ProviderWebDriverDependencies } from './dependencies.ts'; import { @@ -21,7 +22,6 @@ import { import { buildBrowserStackDeviceFeatureCapabilities, readBrowserStackDeviceFeatureFields, - rejectBrowserStackOnlyDeviceFeatures, } from './browserstack-device-features.ts'; import type { CloudWebDriverCapabilityOverrides } from './capabilities.ts'; import { CLOUD_WEBDRIVER_PROVIDERS, type CloudWebDriverKnownProviderName } from './providers.ts'; @@ -82,8 +82,100 @@ const TESTMU_CAPABILITY_OVERRIDES = { const loadTestMu = async () => await import('./testmu.ts'); const loadTestMuDeviceFeatures = async () => await import('./testmu-device-features.ts'); +const AWS_DEVICE_FARM_FIELDS_REFUSED = { + awsProjectArn: 'refused', + awsDeviceArn: 'refused', + awsAppArn: 'refused', + awsRegion: 'refused', + awsInteractionMode: 'refused', +} as const; + +const BROWSERSTACK_PROFILE_FIELDS: ProviderProfileFieldDeclaration = { + provider: CLOUD_WEBDRIVER_PROVIDERS.browserStack, + label: 'BrowserStack', + fields: { + providerApp: 'consumed', + providerOsVersion: 'consumed', + providerDeviceType: 'refused', + providerProject: 'consumed', + providerBuild: 'consumed', + providerSessionName: 'consumed', + providerDeviceOrientation: 'consumed', + providerGeoLocation: 'consumed', + providerTimezone: 'consumed', + providerAppiumVersion: 'consumed', + providerLanguage: 'consumed', + providerLocale: 'consumed', + providerNetworkProfile: 'consumed', + providerCustomNetwork: 'consumed', + providerNoResignApp: 'consumed', + ...AWS_DEVICE_FARM_FIELDS_REFUSED, + }, +}; + +const AWS_DEVICE_FARM_PROFILE_FIELDS: ProviderProfileFieldDeclaration = { + provider: CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, + label: 'AWS Device Farm', + fields: { + providerApp: 'refused', + providerOsVersion: 'refused', + providerDeviceType: 'refused', + providerProject: 'refused', + providerBuild: 'refused', + providerSessionName: 'consumed', + providerDeviceOrientation: 'refused', + providerGeoLocation: 'refused', + providerTimezone: 'refused', + providerAppiumVersion: 'refused', + providerLanguage: 'refused', + providerLocale: 'refused', + providerNetworkProfile: 'refused', + providerCustomNetwork: 'refused', + providerNoResignApp: 'refused', + awsProjectArn: 'consumed', + awsDeviceArn: 'consumed', + awsAppArn: 'consumed', + awsRegion: 'consumed', + awsInteractionMode: 'consumed', + }, +}; + +const TESTMU_PROFILE_FIELDS: ProviderProfileFieldDeclaration = { + provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, + label: 'TestMu AI', + fields: { + providerApp: 'consumed', + providerOsVersion: 'consumed', + providerDeviceType: 'consumed', + providerProject: 'consumed', + providerBuild: 'consumed', + providerSessionName: 'consumed', + providerDeviceOrientation: 'consumed', + providerGeoLocation: 'consumed', + providerTimezone: 'consumed', + providerAppiumVersion: 'consumed', + providerLanguage: 'consumed', + providerLocale: 'consumed', + providerNetworkProfile: 'refused', + providerCustomNetwork: 'refused', + providerNoResignApp: 'refused', + ...AWS_DEVICE_FARM_FIELDS_REFUSED, + }, +}; + +/** The profile fields each hub provider reads, for routes that check them before a runtime exists. */ +export const CLOUD_WEBDRIVER_PROFILE_FIELDS: Readonly< + Record +> = { + [CLOUD_WEBDRIVER_PROVIDERS.browserStack]: BROWSERSTACK_PROFILE_FIELDS, + [CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm]: AWS_DEVICE_FARM_PROFILE_FIELDS, + [CLOUD_WEBDRIVER_PROVIDERS.testMu]: TESTMU_PROFILE_FIELDS, +}; + export type CloudWebDriverProviderDefinition = { provider: CloudWebDriverKnownProviderName; + /** Every profile field, consumed or refused; session preparation refuses the refused ones. */ + profileFields: ProviderProfileFieldDeclaration; createRuntime: (env: DefaultCloudWebDriverProviderRuntimeEnv) => CloudWebDriverRuntime; listArtifactsFromEnv: ( providerSessionId: string, @@ -97,10 +189,12 @@ export function createCloudWebDriverProviderDefinitions( return [ { provider: CLOUD_WEBDRIVER_PROVIDERS.browserStack, + profileFields: BROWSERSTACK_PROFILE_FIELDS, createRuntime: (env) => createCloudWebDriverRuntime({ clientVersion: dependencies.clientVersion, provider: CLOUD_WEBDRIVER_PROVIDERS.browserStack, + profileFields: BROWSERSTACK_PROFILE_FIELDS, platform: 'android', deviceName: 'BrowserStack device', endpoint: env.BROWSERSTACK_WEBDRIVER_ENDPOINT ?? BROWSERSTACK_APP_AUTOMATE_ENDPOINT, @@ -125,10 +219,6 @@ export function createCloudWebDriverProviderDefinitions( }, prepareSession: async ({ req, lease, base }) => { const request = requireRequest(req, 'BrowserStack'); - (await loadTestMuDeviceFeatures()).rejectTestMuOnlyProviderFlags( - request.flags, - CLOUD_WEBDRIVER_PROVIDERS.browserStack, - ); const username = requireEnv(env, 'BROWSERSTACK_USERNAME', 'BrowserStack'); const accessKey = requireEnv(env, 'BROWSERSTACK_ACCESS_KEY', 'BrowserStack'); const platform = requireRequestPlatform(request, 'BrowserStack'); @@ -208,10 +298,12 @@ export function createCloudWebDriverProviderDefinitions( }, { provider: CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, + profileFields: AWS_DEVICE_FARM_PROFILE_FIELDS, createRuntime: (env) => createCloudWebDriverRuntime({ clientVersion: dependencies.clientVersion, provider: CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, + profileFields: AWS_DEVICE_FARM_PROFILE_FIELDS, endpoint: 'http://127.0.0.1/', platform: 'android', deviceName: 'AWS Device Farm device', @@ -228,17 +320,6 @@ export function createCloudWebDriverProviderDefinitions( }, prepareSession: async ({ req, lease, base }) => { const request = requireRequest(req, 'AWS Device Farm'); - // Enforced here, not only in the CLI profile builder: the typed client and - // hand-authored remote-config profiles both reach session preparation without passing - // through `connect`, and would otherwise have these capabilities silently dropped. - rejectBrowserStackOnlyDeviceFeatures( - request.flags, - CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, - ); - (await loadTestMuDeviceFeatures()).rejectTestMuOnlyProviderFlags( - request.flags, - CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, - ); const platform = requireRequestPlatform(request, 'AWS Device Farm'); const sessionOptions = { client: createAwsCliDeviceFarmClient({ @@ -288,10 +369,12 @@ export function createCloudWebDriverProviderDefinitions( }, { provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, + profileFields: TESTMU_PROFILE_FIELDS, createRuntime: (env) => createCloudWebDriverRuntime({ clientVersion: dependencies.clientVersion, provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, + profileFields: TESTMU_PROFILE_FIELDS, platform: 'android', deviceName: 'TestMu AI device', endpoint: env.TESTMU_WEBDRIVER_ENDPOINT ?? TESTMU_WEBDRIVER_ENDPOINT, @@ -306,9 +389,7 @@ export function createCloudWebDriverProviderDefinitions( buildTestMuDeviceFeatureCapabilities, readTestMuDeviceFeatureFields, readTestMuDeviceType, - rejectUnsupportedTestMuDeviceFeatures, } = await loadTestMuDeviceFeatures(); - rejectUnsupportedTestMuDeviceFeatures(request.flags); const deviceType = readTestMuDeviceType(request.flags); const uploadEndpoint = testMuAppUploadEndpoint(env, deviceType); const credentials = requireTestMuCredentials(env, 'TestMu AI'); diff --git a/packages/provider-webdriver/src/provider-profile-fields.test.ts b/packages/provider-webdriver/src/provider-profile-fields.test.ts new file mode 100644 index 0000000000..84f3221ab3 --- /dev/null +++ b/packages/provider-webdriver/src/provider-profile-fields.test.ts @@ -0,0 +1,56 @@ +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import type { ProviderProfileField } from '@agent-device/contracts/provider-profile-fields'; +import { CLOUD_WEBDRIVER_PROFILE_FIELDS } from './provider-definitions.ts'; +import { CLOUD_WEBDRIVER_PROVIDERS } from './providers.ts'; +import { BROWSERSTACK_DEVICE_FEATURE_SPECS } from './browserstack-device-features.ts'; +import { TESTMU_DEVICE_FEATURE_SPECS } from './testmu-device-features.ts'; + +// Fields a hub reads directly while building its session, outside the device-feature tables. +const HUB_SESSION_FIELDS: readonly ProviderProfileField[] = [ + 'providerApp', + 'providerOsVersion', + 'providerDeviceType', + 'providerProject', + 'providerBuild', + 'providerSessionName', +]; + +function consumedFields(provider: keyof typeof CLOUD_WEBDRIVER_PROFILE_FIELDS): string[] { + const { fields } = CLOUD_WEBDRIVER_PROFILE_FIELDS[provider]; + return (Object.keys(fields) as ProviderProfileField[]) + .filter((field) => fields[field] === 'consumed') + .sort(); +} + +test('every declaration names the provider it is registered under', () => { + for (const [provider, declaration] of Object.entries(CLOUD_WEBDRIVER_PROFILE_FIELDS)) { + assert.equal(declaration.provider, provider); + } +}); + +// A consumed device feature with no capability row would be accepted and then dropped at the hub. +test('hub declarations consume exactly the fields their capability builders read', () => { + assert.deepEqual( + consumedFields(CLOUD_WEBDRIVER_PROVIDERS.browserStack), + [ + ...HUB_SESSION_FIELDS.filter((field) => field !== 'providerDeviceType'), + ...BROWSERSTACK_DEVICE_FEATURE_SPECS.map((spec) => spec.field), + ].sort(), + ); + assert.deepEqual( + consumedFields(CLOUD_WEBDRIVER_PROVIDERS.testMu), + [...HUB_SESSION_FIELDS, ...TESTMU_DEVICE_FEATURE_SPECS.map((spec) => spec.field)].sort(), + ); +}); + +test('AWS Device Farm consumes only its own fields and the session name', () => { + assert.deepEqual(consumedFields(CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm), [ + 'awsAppArn', + 'awsDeviceArn', + 'awsInteractionMode', + 'awsProjectArn', + 'awsRegion', + 'providerSessionName', + ]); +}); diff --git a/packages/provider-webdriver/src/runtime-session.ts b/packages/provider-webdriver/src/runtime-session.ts index cce3f7ca86..293aa51b0d 100644 --- a/packages/provider-webdriver/src/runtime-session.ts +++ b/packages/provider-webdriver/src/runtime-session.ts @@ -203,6 +203,12 @@ export class WebDriverSessionManager { lease: DeviceLease, req: LeaseLifecycleContext | undefined, ): Promise { + if (this.options.profileFields) { + // Loaded on first allocation so the package entry's eager closure stays unchanged. + const { rejectRefusedProviderProfileFields } = + await import('@agent-device/contracts/provider-profile-fields'); + rejectRefusedProviderProfileFields(req?.flags, this.options.profileFields); + } const base = this.baseSessionForLease(lease); return this.options.prepareSession ? await this.options.prepareSession({ lease, req, base }) diff --git a/packages/provider-webdriver/src/runtime.ts b/packages/provider-webdriver/src/runtime.ts index 3ee5aab125..146cb89787 100644 --- a/packages/provider-webdriver/src/runtime.ts +++ b/packages/provider-webdriver/src/runtime.ts @@ -18,6 +18,7 @@ import type { PlatformRuntimeProviderModule, } from '@agent-device/contracts/platform-runtime-operations'; import { providerRuntimeOwner } from '@agent-device/contracts/platform-runtime'; +import type { ProviderProfileFieldDeclaration } from '@agent-device/contracts/provider-profile-fields'; import type { DeviceInfo } from '@agent-device/kernel/device'; import { createCloudWebDriverCapabilities, @@ -100,6 +101,8 @@ export type CloudWebDriverRuntimeOptions = { deviceId?: (lease: DeviceLease) => string; prepareSession?: CloudWebDriverPrepareSession; capabilityOverrides?: CloudWebDriverCapabilityOverrides; + /** Profile fields this provider reads; any field it refuses fails session preparation. */ + profileFields?: ProviderProfileFieldDeclaration; }; export function createCloudWebDriverRuntime( diff --git a/packages/provider-webdriver/src/testmu-device-features.test.ts b/packages/provider-webdriver/src/testmu-device-features.test.ts index 33eb5afd49..2faba2e427 100644 --- a/packages/provider-webdriver/src/testmu-device-features.test.ts +++ b/packages/provider-webdriver/src/testmu-device-features.test.ts @@ -7,29 +7,9 @@ import { buildTestMuDeviceFeatureCapabilities, readTestMuDeviceFeatureFields, readTestMuDeviceType, - rejectTestMuOnlyProviderFlags, - rejectUnsupportedTestMuDeviceFeatures, } from './testmu-device-features.ts'; -// Every hosted-provider device-feature field is either a TestMu spec row or an explicit rejection: -// a field in neither parses off the CLI, rides the profile, and is silently dropped at the hub. -const SUPPORTED_FIELDS = [ - 'providerDeviceOrientation', - 'providerGeoLocation', - 'providerTimezone', - 'providerAppiumVersion', - 'providerLanguage', - 'providerLocale', -] as const; -const REJECTED_FIELDS = [ - 'providerNetworkProfile', - 'providerCustomNetwork', - 'providerNoResignApp', -] as const; - -test('every supported device-feature field maps to exactly one lt:options key', () => { - const fields = TESTMU_DEVICE_FEATURE_SPECS.map((spec) => spec.field); - assert.deepEqual([...fields].sort(), [...SUPPORTED_FIELDS].sort()); +test('every device-feature spec maps to exactly one lt:options key', () => { const capabilities = TESTMU_DEVICE_FEATURE_SPECS.map((spec) => spec.capability); assert.equal(new Set(capabilities).size, capabilities.length); }); @@ -77,33 +57,6 @@ test('daemon flag bags are read through the same table with orientation validate ); }); -test('BrowserStack-only flags are rejected by flag name instead of being dropped', () => { - assert.doesNotThrow(() => rejectUnsupportedTestMuDeviceFeatures(undefined)); - assert.doesNotThrow(() => - rejectUnsupportedTestMuDeviceFeatures({ - providerGeoLocation: 'US', - providerNoResignApp: false, - }), - ); - for (const field of REJECTED_FIELDS) { - assert.throws( - () => - rejectUnsupportedTestMuDeviceFeatures({ - [field]: field === 'providerNoResignApp' ? true : 'x', - }), - (error: unknown) => error instanceof AppError && error.code === 'INVALID_ARGS', - ); - } - assert.throws( - () => - rejectUnsupportedTestMuDeviceFeatures({ - providerNetworkProfile: '4g-lte-good', - providerCustomNetwork: '1000', - }), - /--provider-network-profile, --provider-custom-network are not supported by TestMu AI/, - ); -}); - test('the device type defaults to the virtual pool and rejects unknown values', () => { assert.equal(readTestMuDeviceType(undefined), 'virtual'); assert.equal(readTestMuDeviceType({ providerDeviceType: '' }), 'virtual'); @@ -117,21 +70,3 @@ test('the device type defaults to the virtual pool and rejects unknown values', error.details?.flag === '--provider-device-type', ); }); - -test('other providers refuse --provider-device-type by flag name', () => { - assert.doesNotThrow(() => rejectTestMuOnlyProviderFlags(undefined, 'browserstack')); - assert.doesNotThrow(() => - rejectTestMuOnlyProviderFlags({ providerGeoLocation: 'US' }, 'browserstack'), - ); - for (const providerDeviceType of ['real', 'virtual']) { - assert.throws( - () => rejectTestMuOnlyProviderFlags({ providerDeviceType }, 'aws-device-farm'), - (error: unknown) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /--provider-device-type is only supported by TestMu AI, not aws-device-farm/.test( - error.message, - ), - ); - } -}); diff --git a/packages/provider-webdriver/src/testmu-device-features.ts b/packages/provider-webdriver/src/testmu-device-features.ts index 84ea7d8e95..6d886b8438 100644 --- a/packages/provider-webdriver/src/testmu-device-features.ts +++ b/packages/provider-webdriver/src/testmu-device-features.ts @@ -50,16 +50,6 @@ export const TESTMU_DEVICE_FEATURE_SPECS: readonly TestMuDeviceFeatureSpec[] = [ { field: 'providerLocale', capability: 'locale', flag: '--provider-locale' }, ]; -/** Hosted-provider flags other vendors own and TestMu has no capability for. */ -const TESTMU_UNSUPPORTED_DEVICE_FEATURE_FLAGS: ReadonlyArray<{ - field: keyof CloudProviderProfileFields; - flag: string; -}> = [ - { field: 'providerNetworkProfile', flag: '--provider-network-profile' }, - { field: 'providerCustomNetwork', flag: '--provider-custom-network' }, - { field: 'providerNoResignApp', flag: '--provider-no-resign-app' }, -]; - /** Builds the `lt:options` fragment for the configured device features. */ export function buildTestMuDeviceFeatureCapabilities( fields: TestMuDeviceFeatureFields, @@ -73,31 +63,6 @@ export function buildTestMuDeviceFeatureCapabilities( return capabilities; } -/** - * Fails when flags TestMu cannot act on were given. Called from both `connect testmu` (through the - * `./testmu-device-features` subpath, so the package entry stays lazy) and session preparation, - * since the typed client and hand-authored profiles skip `connect`. - */ -export function rejectUnsupportedTestMuDeviceFeatures( - flags: Record | undefined, -): void { - const configured = TESTMU_UNSUPPORTED_DEVICE_FEATURE_FLAGS.filter(({ field }) => { - const value = flags?.[field]; - return value !== undefined && value !== false && value !== ''; - }).map(({ flag }) => flag); - if (configured.length === 0) return; - const plural = configured.length !== 1; - throw new AppError( - 'INVALID_ARGS', - `${configured.join(', ')} ${plural ? 'are' : 'is'} not supported by TestMu AI.`, - { - hint: `Drop ${plural ? 'those flags' : 'the flag'}; TestMu AI has no equivalent capability.`, - provider: 'testmu', - flags: configured, - }, - ); -} - /** * Reads device-feature fields off an untyped flag bag (a daemon request). Enum values are * validated here rather than forwarded to the hub, where an unrecognized value is ignored. @@ -131,24 +96,3 @@ export function readTestMuDeviceType( flag: '--provider-device-type', }); } - -/** - * Fails when another provider was given a TestMu-only flag. Like the BrowserStack-only check, it - * runs in both the connect profile builder and session preparation. - */ -export function rejectTestMuOnlyProviderFlags( - flags: Record | undefined, - provider: string, -): void { - const value = flags?.providerDeviceType; - if (value === undefined || value === '') return; - throw new AppError( - 'INVALID_ARGS', - `--provider-device-type is only supported by TestMu AI, not ${provider}.`, - { - hint: 'Drop the flag or use the testmu provider.', - provider, - flags: ['--provider-device-type'], - }, - ); -} diff --git a/scripts/layering/contracts-exports.snapshot.json b/scripts/layering/contracts-exports.snapshot.json index 840c9d49cb..d5f7f69d10 100644 --- a/scripts/layering/contracts-exports.snapshot.json +++ b/scripts/layering/contracts-exports.snapshot.json @@ -87,6 +87,7 @@ "@agent-device/contracts/platform-runtime-operations", "@agent-device/contracts/platform-runtime-unavailable", "@agent-device/contracts/progress", + "@agent-device/contracts/provider-profile-fields", "@agent-device/contracts/react-native-overlay", "@agent-device/contracts/record-runtime-execution", "@agent-device/contracts/recording", diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index de24acb872..3f41c6c817 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -756,7 +756,6 @@ test('the real tree parses, declares, and passes R11', () => { assert.deepEqual([...providerWebDriverPackage.exportTargets.keys()].sort(), [ '@agent-device/provider-webdriver', '@agent-device/provider-webdriver/providers', - '@agent-device/provider-webdriver/testmu-device-features', ]); assert.deepEqual([...providerWebDriverPackage.workspaceDependencies].sort(), [ '@agent-device/capture-kit', diff --git a/src/__tests__/cloud-connect-profile.test.ts b/src/__tests__/cloud-connect-profile.test.ts index 3b0118d9e9..e91650c8b2 100644 --- a/src/__tests__/cloud-connect-profile.test.ts +++ b/src/__tests__/cloud-connect-profile.test.ts @@ -747,7 +747,7 @@ test('connect does not activate provider state when verification fails', async ( } }); -test('connect aws-device-farm rejects BrowserStack-only device-feature flags', () => { +test('connect aws-device-farm rejects device-feature flags it does not read', () => { const tempRoot = mkdtempForTestSync('agent-device-connect-aws-reject-'); try { @@ -774,7 +774,7 @@ test('connect aws-device-farm rejects BrowserStack-only device-feature flags', ( // Names every offending flag, and fires before the provider's own required-arg checks so // the caller is told what is unsupported rather than what else is missing. assert.match(error.message, /--provider-device-orientation, --provider-timezone/); - assert.match(error.message, /only supported by BrowserStack, not aws-device-farm/); + assert.match(error.message, /are not supported by AWS Device Farm/); assert.deepEqual(error.details?.flags, [ '--provider-device-orientation', '--provider-timezone', diff --git a/src/__tests__/cloud-connect-testmu.test.ts b/src/__tests__/cloud-connect-testmu.test.ts index 1d882c96ff..df9358b53d 100644 --- a/src/__tests__/cloud-connect-testmu.test.ts +++ b/src/__tests__/cloud-connect-testmu.test.ts @@ -238,10 +238,8 @@ test('providers other than TestMu refuse --provider-device-type before saving a (error: unknown) => { assert.ok(error instanceof AppError); assert.equal(error.code, 'INVALID_ARGS'); - assert.match( - error.message, - new RegExp(`--provider-device-type is only supported by TestMu AI, not ${provider}`), - ); + assert.match(error.message, /^--provider-device-type is not supported by /); + assert.equal(error.details?.provider, provider); return true; }, ); @@ -252,16 +250,29 @@ test('providers other than TestMu refuse --provider-device-type before saving a } }); -test('connect limrun refuses the TestMu device type', async () => { +test('connect limrun refuses profile fields Limrun does not read', async () => { const result = await runCliCapture( - ['connect', 'limrun', '--platform', 'ios', '--provider-device-type', 'real', '--json'], + [ + 'connect', + 'limrun', + '--platform', + 'ios', + '--provider-device-type', + 'real', + '--provider-os-version', + '18', + '--json', + ], { env: { LIMRUN_API_KEY: 'lim_test_key' }, stateDirPrefix: 'agent-device-connect-limrun-device-type-', }, ); assert.equal(result.code, 1); - assert.match(result.stdout, /--provider-device-type is only supported by TestMu AI, not limrun/); + assert.match( + result.stdout, + /--provider-os-version, --provider-device-type are not supported by Limrun/, + ); }); async function connectWithGeneratedProviderProfile(options: { diff --git a/src/__tests__/lease-provider-profile-fields.test.ts b/src/__tests__/lease-provider-profile-fields.test.ts new file mode 100644 index 0000000000..141e8d3234 --- /dev/null +++ b/src/__tests__/lease-provider-profile-fields.test.ts @@ -0,0 +1,126 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { afterEach, test, vi } from 'vitest'; +import { AppError, normalizeError } from '@agent-device/kernel/errors'; +import { + errorResponse, + type DaemonRequest, + type DaemonResponse, +} from '@agent-device/kernel/contracts'; +import { LimrunRuntime } from '../sdk/limrun.ts'; +import { createAgentDeviceClient } from '../agent-device-client.ts'; +import { handleLeaseCommands } from '../daemon/handlers/lease.ts'; +import { LeaseRegistry } from '../daemon/lease-registry.ts'; +import { createProviderDeviceRuntimeRequestProviders } from '../provider-device-runtime.ts'; +import { + hashRemoteConfigFile, + writeRemoteConnectionState, +} from '../remote/remote-connection-state.ts'; +import { createTransport } from './client-transport-fixture.ts'; +import { runCliCapture } from './cli-capture.ts'; +import { makeTempWorkspace } from './cli-config-fixtures.ts'; +import { makeSessionStore } from './test-utils/store-factory.ts'; + +const limrunInstances = vi.hoisted(() => ({ + iosCreate: vi.fn(async () => { + throw new Error('a refused allocation must not create a Limrun instance'); + }), +})); + +vi.mock('@limrun/api', () => ({ + default: class MockLimrun { + readonly iosInstances = { create: limrunInstances.iosCreate }; + }, +})); + +afterEach(() => { + vi.clearAllMocks(); +}); + +// The daemon's lease handler over the same provider composition a daemon builds, so a request +// reaches the Limrun runtime exactly as it does in production. +function limrunDaemon(): (req: Omit) => Promise { + const runtime = new LimrunRuntime({ apiKey: 'lim_test_key' }); + const providers = createProviderDeviceRuntimeRequestProviders([runtime]); + const leaseRegistry = new LeaseRegistry(); + const sessionStore = makeSessionStore('agent-device-limrun-profile-fields-'); + return async (req) => { + try { + return (await handleLeaseCommands({ + req: { ...req, token: 'test-token' } as Parameters[0]['req'], + sessionName: req.session ?? 'default', + sessionStore, + leaseRegistry, + providerRuntimeIds: providers.providerRuntimeIds, + providerRuntimeRequiredIds: providers.providerRuntimeRequiredIds, + leaseLifecycleProvider: providers.leaseLifecycleProvider, + })) as DaemonResponse; + } catch (error) { + const normalized = normalizeError(error); + return errorResponse(normalized.code, normalized.message, normalized.details); + } + }; +} + +test('leases.allocate refuses a real device from Limrun instead of handing out a simulator', async () => { + const setup = createTransport(limrunDaemon()); + const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); + + await assert.rejects( + client.leases.allocate({ + tenant: 'limrun', + runId: 'run-real', + leaseBackend: 'ios-instance', + leaseProvider: 'limrun', + platform: 'ios', + providerDeviceType: 'real', + }), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /--provider-device-type is not supported by Limrun/.test(error.message), + ); + assert.equal(limrunInstances.iosCreate.mock.calls.length, 0); +}); + +test('a remote-config profile cannot carry a field Limrun refuses past lease allocation', async () => { + const { root, home, project } = makeTempWorkspace(); + const stateDir = path.join(root, 'state'); + const remoteConfig = path.join(project, 'limrun.remote.json'); + fs.writeFileSync(remoteConfig, JSON.stringify({ providerDeviceType: 'real' }), 'utf8'); + const now = new Date().toISOString(); + writeRemoteConnectionState({ + stateDir, + state: { + version: 1, + session: 'limrun-profile', + remoteConfigPath: remoteConfig, + remoteConfigHash: hashRemoteConfigFile(remoteConfig), + tenant: 'limrun', + runId: 'run-profile', + leaseBackend: 'ios-instance', + leaseProvider: 'limrun', + platform: 'ios', + connectedAt: now, + updatedAt: now, + }, + }); + const daemon = limrunDaemon(); + + try { + const result = await runCliCapture(['open', '--state-dir', stateDir, '--json'], { + cwd: project, + env: { HOME: home }, + sendToDaemon: async (req) => await daemon(req), + }); + + assert.equal(result.code, 1); + assert.equal(result.calls[0]?.command, 'lease_allocate'); + assert.equal(result.calls[0]?.flags?.providerDeviceType, 'real'); + assert.match(result.stdout, /--provider-device-type is not supported by Limrun/); + assert.equal(limrunInstances.iosCreate.mock.calls.length, 0); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/src/cli/connection/cloud-webdriver-profile.ts b/src/cli/connection/cloud-webdriver-profile.ts index c6ed0ebbb9..fe4c8b31ab 100644 --- a/src/cli/connection/cloud-webdriver-profile.ts +++ b/src/cli/connection/cloud-webdriver-profile.ts @@ -1,13 +1,10 @@ import { + CLOUD_WEBDRIVER_PROFILE_FIELDS, CLOUD_WEBDRIVER_PROVIDERS, readAwsDeviceFarmRegionFromArn, - rejectBrowserStackOnlyDeviceFeatures, type CloudWebDriverKnownProviderName, } from '@agent-device/provider-webdriver'; -import { - rejectTestMuOnlyProviderFlags, - rejectUnsupportedTestMuDeviceFeatures, -} from '@agent-device/provider-webdriver/testmu-device-features'; +import { rejectRefusedProviderProfileFields } from '@agent-device/contracts/provider-profile-fields'; import type { RemoteConfigProfile } from '../../remote/remote-config-schema.ts'; import { AppError } from '@agent-device/kernel/errors'; import type { PlatformSelector } from '@agent-device/kernel/device'; @@ -27,7 +24,12 @@ export function resolveCloudWebDriverConnectProfile(options: { cwd: string; env?: EnvMap; }): { flags: CliFlags; remoteConfigPath: string } { - const providerConfig = requireConnectProfileBuilder(options.provider)(options); + const buildProfileFields = requireConnectProfileBuilder(options.provider); + rejectRefusedProviderProfileFields( + options.flags, + CLOUD_WEBDRIVER_PROFILE_FIELDS[options.provider], + ); + const providerConfig = buildProfileFields(options); const clientId = buildConnectClientId( options.provider, options.stateDir, @@ -121,7 +123,6 @@ function browserStackProfileFields(options: { env?: EnvMap; cwd: string; }): RemoteConfigProfile { - rejectTestMuOnlyProviderFlags(options.flags, CLOUD_WEBDRIVER_PROVIDERS.browserStack); return hubProviderProfileFields(BROWSERSTACK_HUB_PROFILE, options); } @@ -130,7 +131,6 @@ function testMuProfileFields(options: { env?: EnvMap; cwd: string; }): RemoteConfigProfile { - rejectUnsupportedTestMuDeviceFeatures(options.flags); return { ...hubProviderProfileFields(TESTMU_HUB_PROFILE, options), providerDeviceType: options.flags.providerDeviceType, @@ -187,8 +187,6 @@ function awsDeviceFarmProfileFields(options: { env?: EnvMap; }): RemoteConfigProfile { const { env, flags } = options; - rejectBrowserStackOnlyDeviceFeatures(flags, CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm); - rejectTestMuOnlyProviderFlags(flags, CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm); const platform = requireCloudWebDriverPlatform( flags.platform, 'connect aws-device-farm requires --platform ios|android.', diff --git a/src/cli/connection/limrun-profile.ts b/src/cli/connection/limrun-profile.ts index 2033dec412..a50c5dcbeb 100644 --- a/src/cli/connection/limrun-profile.ts +++ b/src/cli/connection/limrun-profile.ts @@ -3,7 +3,8 @@ import type { RemoteConfigProfile } from '../../remote/remote-config-schema.ts'; import { AppError } from '@agent-device/kernel/errors'; import type { CliFlags } from '@agent-device/contracts/command'; import { type EnvMap } from '@agent-device/kernel/source-value'; -import { rejectTestMuOnlyProviderFlags } from '@agent-device/provider-webdriver/testmu-device-features'; +import { rejectRefusedProviderProfileFields } from '@agent-device/contracts/provider-profile-fields'; +import { LIMRUN_PROFILE_FIELDS } from '@agent-device/provider-limrun'; import { readMetroProfileFields } from './profile-fields.ts'; import { persistAndResolveGeneratedProfile } from './generated-config.ts'; import { resolveRequestedLeaseBackend } from '../commands/connection-runtime.ts'; @@ -54,7 +55,7 @@ function buildLimrunRemoteProfile(options: { flags: CliFlags }): RemoteConfigPro } function validateLimrunConnectFlags(flags: CliFlags): 'android-instance' | 'ios-instance' { - rejectTestMuOnlyProviderFlags(flags, 'limrun'); + rejectRefusedProviderProfileFields(flags, LIMRUN_PROFILE_FIELDS); if (flags.platform !== 'android' && flags.platform !== 'ios') { throw new AppError('INVALID_ARGS', 'connect limrun requires --platform ios or android.'); } diff --git a/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts b/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts index 0fae68261d..f66534ba6b 100644 --- a/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts +++ b/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts @@ -142,7 +142,7 @@ test('BrowserStack facade nests device-feature capabilities inside bstack:option }); }, 15_000); -test('AWS Device Farm facade rejects BrowserStack-owned device features at session preparation', async () => { +test('AWS Device Farm facade rejects device features it does not read at session preparation', async () => { await withProviderScenarioResource(FakeCloudProviderServer.start, async (server) => { const host = new FakeAwsHostCommand(`${server.url}/wd/hub/`); const provider = createProviderWebDriver({ @@ -172,7 +172,7 @@ test('AWS Device Farm facade rejects BrowserStack-owned device features at sessi (error: unknown) => { assert.match( (error as Error).message, - /--provider-device-orientation, --provider-network-profile are only supported by BrowserStack, not aws-device-farm/, + /--provider-device-orientation, --provider-network-profile are not supported by AWS Device Farm/, ); return true; }, @@ -231,7 +231,7 @@ test('TestMu facade routes a real-device session to the real pool and its upload }); }, 15_000); -test('BrowserStack facade rejects the TestMu device type at session preparation', async () => { +test('BrowserStack facade rejects the device type at session preparation', async () => { await withProviderScenarioResource(FakeCloudProviderServer.start, async (server) => { const provider = createProviderWebDriver({ clientVersion: CLIENT_VERSION, @@ -253,7 +253,7 @@ test('BrowserStack facade rejects the TestMu device type at session preparation' await runtime.leaseLifecycle.allocate?.(lease, { flags: { ...context.flags, providerDeviceType: 'real' }, }), - /--provider-device-type is only supported by TestMu AI, not browserstack/, + /--provider-device-type is not supported by BrowserStack/, ); assert.deepEqual(server.calls, []); } finally { diff --git a/website/docs/docs/device-clouds.md b/website/docs/docs/device-clouds.md index 0e4333c012..fc1923adba 100644 --- a/website/docs/docs/device-clouds.md +++ b/website/docs/docs/device-clouds.md @@ -21,4 +21,6 @@ For each provider, the standard lifecycle is: 3. Follow the printed next command to install or open the app. 4. Run normal device commands, then `agent-device close` and `agent-device disconnect`. +Each provider reads only its own provider flags (`--provider-*` and `--aws-*`). A flag the provider does not use fails with `INVALID_ARGS` naming the flag, whether it arrives through `connect`, `client.leases.allocate()`, or a remote-config profile, so a setting is never silently dropped. + Each provider guide covers its connection selectors, client configuration, MCP setup, artifacts, and troubleshooting. Generated remote profiles are safe to store as non-secret configuration. They may include app IDs, ARNs, device names, OS versions, and labels, but never provider API keys, access keys, or AWS secret keys. diff --git a/website/docs/docs/testmu.md b/website/docs/docs/testmu.md index c2adc8266d..40d80a9717 100644 --- a/website/docs/docs/testmu.md +++ b/website/docs/docs/testmu.md @@ -105,7 +105,8 @@ agent-device connect testmu \ pool you connect to; when in doubt, pass the local path or URL and let agent-device upload it. - `TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT` redirects real-device uploads, as `TESTMU_APP_UPLOAD_ENDPOINT` does for virtual-device uploads. -- `--provider-device-type` applies only to TestMu AI; other providers refuse it. +- `--provider-device-type` applies only to TestMu AI; BrowserStack, AWS Device Farm, and Limrun + refuse it on every route, including `client.leases.allocate()`. ## CLI workflow From 9b5e8182c2998b2bc32072efcef8727af2b50f9b Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 22:28:39 +0530 Subject: [PATCH 09/57] fix(provider-webdriver): let the Apple materializer name the uploadable file The WebDriver deployment runtime guessed what to upload for a materialized iOS build from file extensions: an extracted .app with a .zip or .ipa archive uploaded that archive. The archive it saw is the outermost one the source arrived as, so a URL .zip wrapping an .ipa uploaded the wrapper zip instead of the .ipa, and a zip holding a .app.tar.gz uploaded a zip with no app bundle at its top. Materialization now records the archive an installable was extracted from directly, and the Apple materializer declares the file a hosted provider uploads: the .ipa itself, or the zip a simulator .app was extracted from. It names nothing when no such file exists. The deployment runtime uploads the declared file and otherwise the installable path, with no inference of its own. Co-Authored-By: Claude Opus 5.5 --- .../contracts/src/app-deployment-runtime.ts | 5 +++ .../src/core/install-artifact.ts | 27 ++++++++++++- .../src/runtime-deployment.test.ts | 32 +++++++++------- .../src/runtime-deployment.ts | 14 +------ .../src/install-source-archive.ts | 29 ++++++++++++-- packages/provision-kit/src/install-source.ts | 5 +++ src/__tests__/install-source.test.ts | 38 +++++++++++++++++++ 7 files changed, 120 insertions(+), 30 deletions(-) diff --git a/packages/contracts/src/app-deployment-runtime.ts b/packages/contracts/src/app-deployment-runtime.ts index 52e581c3c9..711a6858c6 100644 --- a/packages/contracts/src/app-deployment-runtime.ts +++ b/packages/contracts/src/app-deployment-runtime.ts @@ -13,6 +13,11 @@ export type AppDeploymentSource = export type MaterializedAppSource = Readonly<{ archivePath?: string; installablePath: string; + /** + * The single file a hosted provider uploads for this build, named by the materializer that knows + * its format. Absent when the installable is already that file or no such file exists. + */ + uploadPath?: string; bundleId?: string; packageName?: string; appName?: string; diff --git a/packages/platform-apple/src/core/install-artifact.ts b/packages/platform-apple/src/core/install-artifact.ts index 4217ed13b3..959c43ca7d 100644 --- a/packages/platform-apple/src/core/install-artifact.ts +++ b/packages/platform-apple/src/core/install-artifact.ts @@ -2,7 +2,11 @@ import path from 'node:path'; import type { LocalInstallSource } from '@agent-device/kernel/contracts'; import { readIosBundleInfo } from './bundle-info.ts'; import { AppError } from '@agent-device/kernel/errors'; -import { extractArchiveSafely, ArchiveBudget } from '@agent-device/host-kit/archive'; +import { + archiveTypeFromPath, + extractArchiveSafely, + ArchiveBudget, +} from '@agent-device/host-kit/archive'; import { installArtifactArchiveBudget, @@ -35,6 +39,7 @@ type IosPayloadAppBundle = { export type PreparedIosInstallArtifact = { archivePath?: string; installablePath: string; + uploadPath?: string; bundleId?: string; appName?: string; cleanup: () => Promise; @@ -80,9 +85,11 @@ async function prepareIosInstallArtifactInScope( ? materialized.installablePath : undefined); + const uploadPath = iosUploadPath(materialized); return { archivePath, installablePath: resolvedInstallable.installPath, + ...(uploadPath ? { uploadPath } : {}), bundleId: bundleInfo.bundleId, appName: bundleInfo.appName, cleanup: async () => { @@ -102,6 +109,24 @@ async function prepareIosInstallArtifactInScope( export { readIosBundleInfo } from './bundle-info.ts'; +/** + * The installable is an extracted `.app` directory, which no hosted upload API accepts. The file + * that carries it is the `.ipa` it was unpacked from, or the zip the `.app` was extracted from + * directly; an outer archive that merely wrapped either is never it. + */ +function iosUploadPath(materialized: { + containingArchivePath?: string; + installablePath: string; +}): string | undefined { + if (materialized.installablePath.toLowerCase().endsWith('.ipa')) { + return materialized.installablePath; + } + const { containingArchivePath } = materialized; + return containingArchivePath && archiveTypeFromPath(containingArchivePath) === 'zip' + ? containingArchivePath + : undefined; +} + async function resolveIosInstallablePath( appPath: string, options?: InstallIosArtifactOptions, diff --git a/packages/provider-webdriver/src/runtime-deployment.test.ts b/packages/provider-webdriver/src/runtime-deployment.test.ts index 559f3c37c9..41aaa508d3 100644 --- a/packages/provider-webdriver/src/runtime-deployment.test.ts +++ b/packages/provider-webdriver/src/runtime-deployment.test.ts @@ -86,9 +86,9 @@ test('aborts a WebDriver provider deployment while its install is in flight', as expect(installApp).toHaveBeenCalledWith('bs://uploaded-app', controller.signal); }); -// Materialization extracts `App.app.zip` (or an .ipa) to a `.app` directory, which no hosted -// upload API accepts; the archive it came from is the uploadable build. -test('a hosted upload of a materialized iOS bundle sends the archive it was extracted from', async () => { +// The materializer knows which file carries the build; the deployment runtime must not second-guess +// it from extensions, or a URL zip wrapping an .ipa would upload the wrapper. +test('a hosted upload sends the file the materializer names, else the installable', async () => { const uploaded: string[] = []; const installApp = vi.fn(async () => undefined); const deployment = createWebDriverDeploymentRuntime({ @@ -109,20 +109,18 @@ test('a hosted upload of a materialized iOS bundle sends the archive it was extr const result = await deploy(iosDevice, { archivePath: '/m/App.app.zip', installablePath: '/m/extracted/App.app', + uploadPath: '/m/App.app.zip', bundleId: 'com.example.app', }); - await deploy(iosDevice, { archivePath: '/m/App.ipa', installablePath: '/m/x/Payload/App.app' }); - await deploy(iosDevice, { installablePath: '/m/App.app' }); + await deploy(iosDevice, { + archivePath: '/m/wrapper.zip', + installablePath: '/m/x/Payload/App.app', + uploadPath: '/m/x/App.ipa', + }); await deploy(iosDevice, { archivePath: '/m/App.tar.gz', installablePath: '/m/x/App.app' }); await deploy(device, { archivePath: '/m/build.zip', installablePath: '/m/x/app.apk' }); - expect(uploaded).toEqual([ - '/m/App.app.zip', - '/m/App.ipa', - '/m/App.app', - '/m/x/App.app', - '/m/x/app.apk', - ]); + expect(uploaded).toEqual(['/m/App.app.zip', '/m/x/App.ipa', '/m/x/App.app', '/m/x/app.apk']); expect(result).toEqual({ bundleId: 'com.example.app', launchTarget: 'com.example.app' }); expect(installApp).toHaveBeenNthCalledWith(1, 'lt://1', expect.any(AbortSignal)); }); @@ -139,6 +137,7 @@ test('a provider without an uploader still installs the materialized bundle path artifact: { archivePath: '/m/App.app.zip', installablePath: '/m/extracted/App.app', + uploadPath: '/m/App.app.zip', bundleId: 'com.example.app', cleanup: async () => {}, }, @@ -175,7 +174,14 @@ test('TestMu uploads the zipped simulator build that install-from-source extract await deployment.deployMaterializedApp( iosDevice, - { artifact: { archivePath, installablePath, cleanup: async () => {} } }, + { + artifact: { + archivePath, + installablePath, + uploadPath: archivePath, + cleanup: async () => {}, + }, + }, new AbortController().signal, ); diff --git a/packages/provider-webdriver/src/runtime-deployment.ts b/packages/provider-webdriver/src/runtime-deployment.ts index f88fe4f304..fb2a058b86 100644 --- a/packages/provider-webdriver/src/runtime-deployment.ts +++ b/packages/provider-webdriver/src/runtime-deployment.ts @@ -6,7 +6,6 @@ import type { AppDeploymentInput, AppDeploymentResult, DeployMaterializedAppInput, - MaterializedAppSource, } from '@agent-device/contracts/app-deployment-runtime'; import type { RuntimeOperationFact } from '@agent-device/contracts/platform-runtime'; import { publicPlatformString, type DeviceInfo } from '@agent-device/kernel/device'; @@ -105,7 +104,7 @@ export function createWebDriverDeploymentRuntime( '', { appPath: input.artifact.installablePath, - uploadPath: materializedUploadPath(input.artifact), + uploadPath: input.artifact.uploadPath ?? input.artifact.installablePath, }, { appIdentifierHint: input.artifact.bundleId, @@ -117,17 +116,6 @@ export function createWebDriverDeploymentRuntime( }); } -/** - * Materialization extracts an iOS `.app` bundle out of a zipped simulator build or an .ipa, and no - * hosted upload API takes a directory, so the uploader gets the archive the bundle came from. - */ -function materializedUploadPath(artifact: MaterializedAppSource): string { - const { archivePath, installablePath } = artifact; - return archivePath && /\.(zip|ipa)$/i.test(archivePath) && /\.app\/?$/i.test(installablePath) - ? archivePath - : installablePath; -} - function deploymentFact(session: WebDriverProviderSession | undefined): RuntimeOperationFact { if (!session) { return Object.freeze({ diff --git a/packages/provision-kit/src/install-source-archive.ts b/packages/provision-kit/src/install-source-archive.ts index 7449c3d480..1f5fca61b8 100644 --- a/packages/provision-kit/src/install-source-archive.ts +++ b/packages/provision-kit/src/install-source-archive.ts @@ -15,10 +15,32 @@ type InstallableMatcher = ( stat: { isFile(): boolean; isDirectory(): boolean }, ) => boolean; +type ResolvedInstallableCandidate = { + /** The outermost archive the source arrived as. */ + archivePath?: string; + /** The innermost archive, the one the installable was extracted from directly. */ + containingArchivePath?: string; + installablePath: string; +}; + +function resolvedCandidate( + installablePath: string, + params: { archivePath: string | undefined; containingArchivePath?: string }, +): ResolvedInstallableCandidate { + return { + archivePath: params.archivePath, + ...(params.containingArchivePath + ? { containingArchivePath: params.containingArchivePath } + : {}), + installablePath, + }; +} + export async function resolveInstallableCandidate( candidatePath: string, params: { archivePath: string | undefined; + containingArchivePath?: string; isInstallablePath: InstallableMatcher; installableLabel: string; allowArchiveExtraction: boolean; @@ -27,11 +49,11 @@ export async function resolveInstallableCandidate( archiveDepth: number; onArchiveAccepted?: (depth: number) => void; }, -): Promise<{ archivePath?: string; installablePath: string }> { +): Promise { const stat = await fs.stat(candidatePath).catch(() => null); if (!stat) throw new AppError('INVALID_ARGS', `App source not found: ${candidatePath}`); if (params.isInstallablePath(candidatePath, stat)) { - return { archivePath: params.archivePath, installablePath: candidatePath }; + return resolvedCandidate(candidatePath, params); } if (stat.isFile() && isArchivePath(candidatePath)) { assertArchiveExtractionAllowed(candidatePath, params, false); @@ -40,7 +62,7 @@ export async function resolveInstallableCandidate( if (stat.isDirectory()) { const installables = await collectMatchingPaths(candidatePath, params.isInstallablePath); if (installables.length === 1) { - return { archivePath: params.archivePath, installablePath: installables[0]! }; + return resolvedCandidate(installables[0]!, params); } if (installables.length > 1) { throw new AppError( @@ -80,6 +102,7 @@ async function resolveExtractedArchive( return await resolveInstallableCandidate(extracted.outputPath, { ...params, archivePath: params.archivePath ?? archivePath, + containingArchivePath: archivePath, archiveDepth: params.archiveDepth + 1, }); } diff --git a/packages/provision-kit/src/install-source.ts b/packages/provision-kit/src/install-source.ts index e73c4aafdb..26119979e9 100644 --- a/packages/provision-kit/src/install-source.ts +++ b/packages/provision-kit/src/install-source.ts @@ -33,6 +33,8 @@ export type MaterializeInstallableOptions = { export type MaterializedInstallable = { archivePath?: string; + /** The archive the installable was extracted from directly, when it came out of one. */ + containingArchivePath?: string; installablePath: string; cleanup: () => Promise; }; @@ -69,6 +71,9 @@ export async function materializeInstallablePath( }); return { archivePath: resolved.archivePath, + ...(resolved.containingArchivePath + ? { containingArchivePath: resolved.containingArchivePath } + : {}), installablePath: resolved.installablePath, cleanup: async () => { await runCleanupTasks(cleanupTasks); diff --git a/src/__tests__/install-source.test.ts b/src/__tests__/install-source.test.ts index 58206934e5..974a6ba509 100644 --- a/src/__tests__/install-source.test.ts +++ b/src/__tests__/install-source.test.ts @@ -381,6 +381,8 @@ test('prepareIosInstallArtifact extracts trusted GitHub artifact ZIP containing try { assert.equal(path.basename(result.installablePath), 'Demo.app'); + // A tar has no hosted upload API, so nothing is named uploadable. + assert.equal(result.uploadPath, undefined); assert.equal(result.bundleId, 'com.example.githubtar'); assert.equal(result.appName, 'GitHub Tar'); } finally { @@ -420,6 +422,9 @@ test('prepareIosInstallArtifact extracts trusted GitHub artifact ZIP containing try { assert.equal(path.basename(result.installablePath), 'Demo.app'); + // The .ipa, not the artifact zip that wrapped it, is what a hosted provider uploads. + assert.equal(path.basename(result.uploadPath ?? ''), 'Demo.ipa'); + assert.equal((await fs.stat(result.uploadPath ?? '')).isFile(), true); assert.equal(result.bundleId, 'com.example.githubipa'); assert.equal(result.appName, 'GitHub IPA'); } finally { @@ -431,6 +436,39 @@ test('prepareIosInstallArtifact extracts trusted GitHub artifact ZIP containing ); }); +test('prepareIosInstallArtifact names the zip a simulator app arrived in as its upload', async () => { + await withArchiveFixture( + { + extractions: [ + { + command: 'unzip', + populate: async (outputPath) => { + await fs.mkdir(path.join(outputPath, 'Demo.app')); + }, + }, + ], + }, + async () => { + await withIosBundleInfo('com.example.githubapp', 'GitHub App', async () => { + await withMockedInstallSourceFetch(Buffer.from('artifact fixture'), async () => { + const result = await prepareIosInstallArtifact({ + kind: 'url', + url: 'https://api.github.com/repos/acme/app/actions/artifacts/989/zip', + }); + + try { + assert.equal(path.basename(result.installablePath), 'Demo.app'); + assert.equal(result.uploadPath, result.archivePath); + assert.equal((await fs.stat(result.uploadPath ?? '')).isFile(), true); + } finally { + await result.cleanup(); + } + }); + }); + }, + ); +}); + test('prepareIosInstallArtifact cleans URL materialization when IPA payload resolution fails', async () => { await withArchiveFixture( { From 402370e0f6dd13bcb5aacb02d143051140138ede Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 22:35:09 +0530 Subject: [PATCH 10/57] fix(provider-webdriver): tighten app-reference, endpoint and upload input handling Several inputs reached a hosted provider in a shape it could not use, or failed with an untyped error: - An empty `lt://` passed as a TestMu AI app reference and reached session creation. A reference is now validated against the app-id grammar, and an invalid one is INVALID_ARGS on connect and at session preparation. - `LT://APP1` or `BS://id` was treated as a file path. App schemes now match case-insensitively and are canonicalized to the spelling the hub accepts. - A directory given as `--provider-app` reached the upload and failed with a raw EISDIR; it is now INVALID_ARGS with a hint. - A missing path given straight to the TestMu AI upload failed with a bare ENOENT; it now gets the same typed refusal as a directory. - Session-detail URLs were built by string concatenation, so a query on TESTMU_API_ENDPOINT or the BrowserStack details endpoint swallowed the route. Routes are now appended to the URL path, keeping the query. - A whitespace-only artifact URL was reported as a ready artifact; it is now treated as absent, and URLs are trimmed. - A 2xx connection-verification answer that is not JSON was reported as a network failure. It is now COMMAND_FAILED with the HTTP status, and each hub supplies its own service-status hint. The connect help and the TestMu AI guide now say what connect checks for `--provider-app` and what is only validated when the session is created. Co-Authored-By: Claude Opus 5.5 --- .../src/artifact-results.ts | 5 +- .../browserstack-connection-verification.ts | 1 + .../src/browserstack.test.ts | 16 ++++ .../provider-webdriver/src/browserstack.ts | 7 +- .../src/testmu-connection-verification.ts | 21 ++--- .../provider-webdriver/src/testmu.test.ts | 34 ++++++- packages/provider-webdriver/src/testmu.ts | 33 ++++--- .../src/webdriver-utils.test.ts | 92 +++++++++++++++++++ .../provider-webdriver/src/webdriver-utils.ts | 63 +++++++++++-- src/__tests__/cloud-connect-testmu.test.ts | 38 ++++++++ src/cli/connection/cloud-webdriver-profile.ts | 10 +- src/commands/schema/cli-help.ts | 2 +- website/docs/docs/testmu.md | 13 ++- 13 files changed, 287 insertions(+), 48 deletions(-) diff --git a/packages/provider-webdriver/src/artifact-results.ts b/packages/provider-webdriver/src/artifact-results.ts index de29b4ab3a..3aaabc69b5 100644 --- a/packages/provider-webdriver/src/artifact-results.ts +++ b/packages/provider-webdriver/src/artifact-results.ts @@ -38,7 +38,8 @@ export function urlArtifactFromDetails( kind: CloudArtifact['kind'], name: string, ): CloudArtifact | undefined { - const url = details[field]; - if (typeof url !== 'string' || url.length === 0) return undefined; + const value = details[field]; + const url = typeof value === 'string' ? value.trim() : ''; + if (url.length === 0) return undefined; return { provider, providerSessionId, kind, name, url, availability: 'ready' }; } diff --git a/packages/provider-webdriver/src/browserstack-connection-verification.ts b/packages/provider-webdriver/src/browserstack-connection-verification.ts index 56d36366cc..dfb18bd989 100644 --- a/packages/provider-webdriver/src/browserstack-connection-verification.ts +++ b/packages/provider-webdriver/src/browserstack-connection-verification.ts @@ -110,6 +110,7 @@ async function fetchBrowserStackJson( hints: { service: 'BrowserStack', unauthorizedHint: 'Check BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY.', + serviceHint: 'Retry connect or check the BrowserStack service status.', networkHint: 'Check network access to api-cloud.browserstack.com and retry connect.', }, }); diff --git a/packages/provider-webdriver/src/browserstack.test.ts b/packages/provider-webdriver/src/browserstack.test.ts index 1d8ca76b1a..7d7af5610a 100644 --- a/packages/provider-webdriver/src/browserstack.test.ts +++ b/packages/provider-webdriver/src/browserstack.test.ts @@ -135,3 +135,19 @@ test('BrowserStack session details lookup has a deadline and fails typed', async }); } }); + +test('BrowserStack session details keep a query on the endpoint override', async () => { + const calls: string[] = []; + globalThis.fetch = async (input) => { + calls.push(String(input)); + return new Response(JSON.stringify({ automation_session: { video_url: ' ' } }), { + status: 200, + }); + }; + const result = await listBrowserStackCloudArtifacts('browserstack', 'SESSION1', { + ...upload, + endpoint: 'https://api.example.test/sessions?region=eu', + }); + assert.deepEqual(calls, ['https://api.example.test/sessions/SESSION1.json?region=eu']); + assert.equal(result?.status, 'pending'); +}); diff --git a/packages/provider-webdriver/src/browserstack.ts b/packages/provider-webdriver/src/browserstack.ts index a5954eda00..ffd854052f 100644 --- a/packages/provider-webdriver/src/browserstack.ts +++ b/packages/provider-webdriver/src/browserstack.ts @@ -3,13 +3,13 @@ import type { CloudWebDriverCapabilityOverrides } from './capabilities.ts'; import type { CloudWebDriverUploadApp } from './runtime.ts'; import { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; import { + appendUrlPath, appFileUploadForm, asRecord, createHubUploadApp, fetchProviderSessionDetails, postHubAppUpload, resolveHubAppReference, - trimTrailingSlash, } from './webdriver-utils.ts'; export const BROWSERSTACK_APP_AUTOMATE_ENDPOINT = 'https://hub-cloud.browserstack.com/wd/hub/'; @@ -151,8 +151,9 @@ async function fetchBrowserStackSessionDetails( sessionId: string, options: BrowserStackSessionDetailsOptions, ): Promise> { - const endpoint = new URL( - `${trimTrailingSlash(String(options.endpoint ?? BROWSERSTACK_SESSION_DETAILS_ENDPOINT))}/${sessionId}.json`, + const endpoint = appendUrlPath( + options.endpoint ?? BROWSERSTACK_SESSION_DETAILS_ENDPOINT, + `${sessionId}.json`, ); const json = await fetchProviderSessionDetails(endpoint, { clientVersion: options.clientVersion, diff --git a/packages/provider-webdriver/src/testmu-connection-verification.ts b/packages/provider-webdriver/src/testmu-connection-verification.ts index f54c28358f..7e37a4f2f5 100644 --- a/packages/provider-webdriver/src/testmu-connection-verification.ts +++ b/packages/provider-webdriver/src/testmu-connection-verification.ts @@ -1,7 +1,12 @@ import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; -import { asRecord, fetchProviderVerificationJson, trimTrailingSlash } from './webdriver-utils.ts'; -import { TESTMU_API_ENDPOINT, TESTMU_APPS_ENDPOINT, isTestMuAppReference } from './testmu.ts'; +import { appendUrlPath, asRecord, fetchProviderVerificationJson } from './webdriver-utils.ts'; +import { + TESTMU_API_ENDPOINT, + TESTMU_APPS_ENDPOINT, + isTestMuAppReference, + testMuAppReferenceFromId, +} from './testmu.ts'; import type { CloudWebDriverConnectionVerification, CloudWebDriverConnectionVerificationOptions, @@ -34,7 +39,7 @@ export async function verifyTestMuConnection( const deviceType = options.deviceType ?? 'virtual'; const catalogUrl = options.devicesEndpoint ? new URL(options.devicesEndpoint) - : apiUrl(options.apiEndpoint ?? TESTMU_API_ENDPOINT, 'capability/generator'); + : appendUrlPath(options.apiEndpoint ?? TESTMU_API_ENDPOINT, 'capability/generator'); catalogUrl.searchParams.set('isVirtualDevice', String(deviceType === 'virtual')); const catalog = await fetchTestMuJson(catalogUrl, undefined, clientVersion); const namedDevices = readTestMuCatalogDevices(catalog, options.platform, deviceType).filter( @@ -118,13 +123,6 @@ async function verifyTestMuApp( }; } -/** Appends `route` to the base's path; the base may carry a query, which is kept. */ -function apiUrl(base: string | URL, route: string): URL { - const url = new URL(base); - url.pathname = `${trimTrailingSlash(url.pathname)}/${route}`; - return url; -} - async function fetchTestMuJson( endpoint: string | URL, auth: TestMuAuth | undefined, @@ -136,6 +134,7 @@ async function fetchTestMuJson( hints: { service: 'TestMu AI', unauthorizedHint: 'Check LT_USERNAME and LT_ACCESS_KEY.', + serviceHint: 'Retry connect or check the TestMu AI service status.', networkHint: 'Check network access to mobile-api.lambdatest.com and manual-api.lambdatest.com, then retry connect.', }, @@ -189,7 +188,7 @@ function readTestMuApps( return data.flatMap((entry) => { const app = asRecord(entry); if (!app || typeof app.app_id !== 'string') return []; - const reference = isTestMuAppReference(app.app_id) ? app.app_id : `lt://${app.app_id}`; + const reference = testMuAppReferenceFromId(app.app_id); return [ { reference, diff --git a/packages/provider-webdriver/src/testmu.test.ts b/packages/provider-webdriver/src/testmu.test.ts index fd00b6782e..d14039a9fc 100644 --- a/packages/provider-webdriver/src/testmu.test.ts +++ b/packages/provider-webdriver/src/testmu.test.ts @@ -244,6 +244,20 @@ test('TestMu upload rejects an unzipped .app bundle before calling the upload AP } }); +// The install adapter reaches the upload without the resolver's existence check in front of it. +test('TestMu upload of a missing path fails typed, not with a bare ENOENT', async () => { + const fetchMock = vi.fn(); + globalThis.fetch = fetchMock; + await assert.rejects(uploadTestMuApp('/nonexistent/App.apk', auth), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'INVALID_ARGS'); + assert.equal(error.message, 'TestMu AI can only upload an app file: /nonexistent/App.apk'); + assert.ok(error.details?.hint); + return true; + }); + assert.equal(fetchMock.mock.calls.length, 0); +}); + test('the install adapter uploads the local build and launches the hinted app id', async () => { const tempDir = await mkdtempForTest('agent-device-testmu-install-'); const appPath = path.join(tempDir, 'Demo.apk'); @@ -277,6 +291,14 @@ test('TestMu passes lt:// ids through and has the upload API fetch a public URL' return jsonResponse({ app_id: 'APP9' }); }; assert.equal(await resolveTestMuAppReference('lt://APP1', auth), 'lt://APP1'); + assert.equal(await resolveTestMuAppReference('LT://APP1', auth), 'lt://APP1'); + await assert.rejects( + resolveTestMuAppReference('lt://', auth), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /--provider-app lt:\/\/ is not an lt:\/\/ app id/.test(error.message), + ); assert.equal(forms.length, 0); assert.equal( await resolveTestMuAppReference('https://builds.example/App.apk', auth), @@ -346,9 +368,10 @@ test('TestMu artifacts come from the jsend session payload and stay pending unti status: 'success', data: { test_id: 'SESSION1', - video_url: 'https://cdn.test/video.mp4', + video_url: ' https://cdn.test/video.mp4\n', appium_logs_url: 'https://api.test/sessions/SESSION1/log/appium', device_logs_url: '', + network_logs_url: ' ', }, }); }; @@ -356,7 +379,14 @@ test('TestMu artifacts come from the jsend session payload and stay pending unti ...auth, endpoint: 'https://api.test/mobile-automation/api/v1/', }); - assert.deepEqual(calls, ['https://api.test/mobile-automation/api/v1/sessions/SESSION1']); + await listTestMuCloudArtifacts('testmu', 'SESSION 2', { + ...auth, + endpoint: 'https://api.test/mobile-automation/api/v1?region=eu', + }); + assert.deepEqual(calls, [ + 'https://api.test/mobile-automation/api/v1/sessions/SESSION1', + 'https://api.test/mobile-automation/api/v1/sessions/SESSION%202?region=eu', + ]); assert.equal(result?.status, 'ready'); assert.deepEqual( result?.cloudArtifacts.map((artifact) => [artifact.kind, artifact.url]), diff --git a/packages/provider-webdriver/src/testmu.ts b/packages/provider-webdriver/src/testmu.ts index a632cba50e..1217fd6c05 100644 --- a/packages/provider-webdriver/src/testmu.ts +++ b/packages/provider-webdriver/src/testmu.ts @@ -6,13 +6,13 @@ import type { CloudWebDriverPlatform, CloudWebDriverUploadApp } from './runtime. import { AppError } from '@agent-device/kernel/errors'; import { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; import { + appendUrlPath, appFileUploadForm, asRecord, createHubUploadApp, fetchProviderSessionDetails, postHubAppUpload, resolveHubAppReference, - trimTrailingSlash, } from './webdriver-utils.ts'; /** @@ -83,7 +83,9 @@ export async function uploadTestMuApp( signal?: AbortSignal, ): Promise { signal?.throwIfAborted(); - if (!(await fs.stat(appPath)).isFile()) { + // A missing path is the same caller mistake as a directory, so both get the typed refusal. + const stat = await fs.stat(appPath).catch(() => undefined); + if (!stat?.isFile()) { throw new AppError('INVALID_ARGS', `TestMu AI can only upload an app file: ${appPath}`, { appPath, hint: @@ -146,6 +148,7 @@ export async function resolveTestMuAppReference( cwd: options.cwd, referenceScheme: 'lt://', referenceLabel: 'an lt:// app id', + isReference: isTestMuAppReference, uploadFile: async (appPath, signal) => await uploadTestMuApp(appPath, options, signal), uploadUrl: async (url, signal) => await uploadTestMuAppFromUrl(url, options, signal), signal: options.signal, @@ -190,16 +193,24 @@ export function buildTestMuCapabilities( }; } +const TESTMU_APP_REFERENCE = /^lt:\/\/[\w.-]+$/; + export function isTestMuAppReference(value: string): boolean { - return value.startsWith('lt://'); + return TESTMU_APP_REFERENCE.test(value); +} + +/** The upload and app-list APIs answer with a bare app id or an `lt://` reference. */ +export function testMuAppReferenceFromId(id: string): string { + return id.startsWith('lt://') ? id : `lt://${id}`; } async function fetchTestMuSessionDetails( sessionId: string, options: TestMuSessionDetailsOptions, ): Promise> { - const endpoint = new URL( - `${trimTrailingSlash(String(options.endpoint ?? TESTMU_API_ENDPOINT))}/sessions/${encodeURIComponent(sessionId)}`, + const endpoint = appendUrlPath( + options.endpoint ?? TESTMU_API_ENDPOINT, + `sessions/${encodeURIComponent(sessionId)}`, ); let json: Record; try { @@ -259,17 +270,11 @@ function mapTestMuArtifacts( return ready.length > 0 ? [...ready, dashboard] : []; } -const TESTMU_APP_ID = /^[\w.-]+$/; - /** The upload answers with `app_url` (`lt://…`) and/or a bare `app_id`; anything else is a failed upload. */ function readTestMuAppReference(value: unknown): string | undefined { const { app_url: appUrl, app_id: appId } = asRecord(value) ?? {}; - if (typeof appUrl === 'string' && isValidTestMuAppReference(appUrl)) return appUrl; + if (typeof appUrl === 'string' && isTestMuAppReference(appUrl)) return appUrl; if (typeof appId !== 'string') return undefined; - const reference = isTestMuAppReference(appId) ? appId : `lt://${appId}`; - return isValidTestMuAppReference(reference) ? reference : undefined; -} - -function isValidTestMuAppReference(value: string): boolean { - return isTestMuAppReference(value) && TESTMU_APP_ID.test(value.slice('lt://'.length)); + const reference = testMuAppReferenceFromId(appId); + return isTestMuAppReference(reference) ? reference : undefined; } diff --git a/packages/provider-webdriver/src/webdriver-utils.test.ts b/packages/provider-webdriver/src/webdriver-utils.test.ts index 122aafaf80..58a182c789 100644 --- a/packages/provider-webdriver/src/webdriver-utils.test.ts +++ b/packages/provider-webdriver/src/webdriver-utils.test.ts @@ -4,8 +4,10 @@ import path from 'node:path'; import { afterEach, test, vi } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; import { + appendUrlPath, asRecord, createHubUploadApp, + fetchProviderVerificationJson, postHubAppUpload, resolveHubAppReference, trimLeadingSlash, @@ -112,6 +114,7 @@ test('the hub app resolver passes references through, uploads local files, and r }); assert.equal(await resolve('hub://APP3'), 'hub://APP3'); + assert.equal(await resolve('HUB://APP3'), 'hub://APP3'); assert.equal(await resolve('https://builds.example/App.apk'), 'https://builds.example/App.apk'); assert.equal( await resolve('https://builds.example/App.apk', async (url) => `fetched:${url}`), @@ -132,3 +135,92 @@ test('the hub app resolver passes references through, uploads local files, and r await fs.rm(tempDir, { recursive: true, force: true }); } }); + +test('the hub app resolver refuses an empty reference and a directory, typed', async () => { + const tempDir = await mkdtempForTest('agent-device-hub-resolve-invalid-'); + try { + await fs.mkdir(path.join(tempDir, 'App.app')); + const uploadFile = vi.fn(async () => 'hub://never'); + const resolve = (app: string) => + resolveHubAppReference({ + service: 'Hub', + app, + cwd: tempDir, + referenceScheme: 'hub://', + referenceLabel: 'a hub:// app id', + isReference: (reference) => /^hub:\/\/\w+$/.test(reference), + uploadFile, + }); + + for (const [app, message] of [ + ['hub://', /^Hub --provider-app hub:\/\/ is not a hub:\/\/ app id\.$/], + ['hub://a b', /is not a hub:\/\/ app id/], + ['App.app', /must be an app file, not a directory: .*App\.app$/], + ] as const) { + await assert.rejects( + resolve(app), + (error: unknown) => + error instanceof AppError && error.code === 'INVALID_ARGS' && message.test(error.message), + ); + } + assert.equal(uploadFile.mock.calls.length, 0); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + +test('appending a route keeps a query on the base endpoint', () => { + assert.equal( + appendUrlPath('https://api.example.test/v1/?region=eu', 'sessions/S%201').toString(), + 'https://api.example.test/v1/sessions/S%201?region=eu', + ); + assert.equal( + appendUrlPath('https://api.example.test/v1', 'sessions/S1').toString(), + 'https://api.example.test/v1/sessions/S1', + ); +}); + +const verificationHints = { + service: 'Hub', + unauthorizedHint: 'Check HUB_KEY.', + serviceHint: 'Retry connect or check the Hub service status.', + networkHint: 'Check network access to the hub.', +}; + +test('connection verification reports a non-JSON success typed, with its status', async () => { + globalThis.fetch = async () => new Response('maintenance', { status: 200 }); + await assert.rejects( + fetchProviderVerificationJson('https://api.example.test/apps', { + clientVersion: '0.0.0-test', + hints: verificationHints, + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.message, 'Hub connection verification answer was not JSON.'); + assert.equal(error.details?.status, 200); + assert.equal(error.details?.hint, verificationHints.serviceHint); + return true; + }, + ); +}); + +test('connection verification gives each failure its provider hint', async () => { + for (const [status, code, hint] of [ + [401, 'UNAUTHORIZED', verificationHints.unauthorizedHint], + [503, 'COMMAND_FAILED', verificationHints.serviceHint], + ] as const) { + globalThis.fetch = async () => new Response('nope', { status }); + await assert.rejects( + fetchProviderVerificationJson('https://api.example.test/apps', { + clientVersion: '0.0.0-test', + hints: verificationHints, + }), + (error: unknown) => + error instanceof AppError && + error.code === code && + error.details?.status === status && + error.details?.hint === hint, + ); + } +}); diff --git a/packages/provider-webdriver/src/webdriver-utils.ts b/packages/provider-webdriver/src/webdriver-utils.ts index 88724187dc..d75fd560ba 100644 --- a/packages/provider-webdriver/src/webdriver-utils.ts +++ b/packages/provider-webdriver/src/webdriver-utils.ts @@ -56,6 +56,13 @@ export function trimTrailingSlash(value: string): string { return lastNonSlash === value.length - 1 ? value : value.slice(0, lastNonSlash + 1); } +/** Appends `route` to the base's path; a query on the base is kept rather than swallowing the route. */ +export function appendUrlPath(base: string | URL, route: string): URL { + const url = new URL(base); + url.pathname = `${trimTrailingSlash(url.pathname)}/${route}`; + return url; +} + export function withTrailingSlash(url: URL): URL { if (url.pathname.endsWith('/')) return url; const copy = new URL(url); @@ -141,39 +148,71 @@ export async function resolveHubAppReference(options: { referenceScheme: string; /** How the scheme reads in the error message, e.g. `a bs:// app id`. */ referenceLabel: string; + /** Validates a canonical reference; by default any non-empty id after the scheme is accepted. */ + isReference?: (reference: string) => boolean; uploadFile: (appPath: string, signal?: AbortSignal) => Promise; uploadUrl?: (url: string, signal?: AbortSignal) => Promise; signal?: AbortSignal; }): Promise { const { app } = options; - if (app.startsWith(options.referenceScheme)) return app; + const reference = canonicalHubAppReference(app, options.referenceScheme); + if (reference !== undefined) { + const isReference = + options.isReference ?? ((value: string) => value.length > options.referenceScheme.length); + if (isReference(reference)) return reference; + throw new AppError( + 'INVALID_ARGS', + `${options.service} --provider-app ${app} is not ${options.referenceLabel}.`, + { providerApp: app }, + ); + } if (/^https?:\/\//i.test(app)) { return options.uploadUrl ? await options.uploadUrl(app, options.signal) : app; } const appPath = path.resolve(options.cwd ?? process.cwd(), app); - if (!fs.existsSync(appPath)) { + const stat = fs.statSync(appPath, { throwIfNoEntry: false }); + if (!stat) { throw new AppError( 'INVALID_ARGS', `${options.service} --provider-app must be ${options.referenceLabel}, URL, or existing local app path.`, { providerApp: app }, ); } + if (!stat.isFile()) { + throw new AppError( + 'INVALID_ARGS', + `${options.service} --provider-app must be an app file, not a directory: ${appPath}`, + { + providerApp: app, + hint: 'Zip an iOS simulator .app bundle and pass the .zip, or pass the .ipa, .apk, or .aab.', + }, + ); + } return await options.uploadFile(appPath, options.signal); } +/** URI schemes are case-insensitive, so `LT://id` is the hub reference `lt://id`. */ +function canonicalHubAppReference(app: string, scheme: string): string | undefined { + if (app.slice(0, scheme.length).toLowerCase() !== scheme) return undefined; + return `${scheme}${app.slice(scheme.length)}`; +} + const PROVIDER_API_TIMEOUT_MS = 15_000; /** The provider rejected or could not answer a verification call; typed so callers never sniff text. */ export type ProviderJsonFailureHints = { service: string; unauthorizedHint: string; + /** For any other non-2xx answer, or a 2xx answer that is not JSON. */ + serviceHint: string; networkHint: string; }; /** * Fetches JSON from a hosted provider's API during connection verification. A 401/403 is - * `UNAUTHORIZED` with a credential hint, any other non-2xx is `COMMAND_FAILED`, and a transport - * failure is wrapped so its cause survives without leaking the credentials. + * `UNAUTHORIZED` with a credential hint, any other non-2xx or a body that is not JSON is + * `COMMAND_FAILED` with the status, and a transport failure is wrapped so its cause survives + * without leaking the credentials. */ export async function fetchProviderVerificationJson( endpoint: string | URL, @@ -183,7 +222,7 @@ export async function fetchProviderVerificationJson( hints: ProviderJsonFailureHints; }, ): Promise { - const { service, unauthorizedHint, networkHint } = options.hints; + const { service, unauthorizedHint, serviceHint, networkHint } = options.hints; try { const response = await fetch(endpoint, { headers: { @@ -199,13 +238,19 @@ export async function fetchProviderVerificationJson( `${service} rejected connection verification.`, { status: response.status, - hint: unauthorized - ? unauthorizedHint - : `Retry connect or check the ${service} service status.`, + hint: unauthorized ? unauthorizedHint : serviceHint, }, ); } - return (await response.json()) as unknown; + const json = await readProviderJsonBody(response); + if (json === undefined) { + throw new AppError( + 'COMMAND_FAILED', + `${service} connection verification answer was not JSON.`, + { status: response.status, hint: serviceHint }, + ); + } + return json; } catch (error) { if (error instanceof AppError) throw error; throw new AppError( diff --git a/src/__tests__/cloud-connect-testmu.test.ts b/src/__tests__/cloud-connect-testmu.test.ts index df9358b53d..e9b7a3413c 100644 --- a/src/__tests__/cloud-connect-testmu.test.ts +++ b/src/__tests__/cloud-connect-testmu.test.ts @@ -85,6 +85,44 @@ test('connect testmu generates a local provider profile and verifies the virtual } }); +test('connect canonicalizes an upper-case app scheme and refuses an empty app id', () => { + const tempRoot = mkdtempForTestSync('agent-device-connect-app-scheme-'); + const base = { json: false, help: false, version: false, platform: 'ios' as const }; + const connect = (provider: 'testmu' | 'browserstack', providerApp: string) => + resolveCloudWebDriverConnectProfile({ + provider, + stateDir: path.join(tempRoot, `.state-${provider}`), + cwd: tempRoot, + env: { + LT_USERNAME: 'u', + LT_ACCESS_KEY: 'k', + BROWSERSTACK_USERNAME: 'u', + BROWSERSTACK_ACCESS_KEY: 'k', + }, + flags: { ...base, device: 'iPhone 16', providerOsVersion: '18.0', providerApp }, + }); + + try { + assert.equal( + readGeneratedConfig(connect('testmu', 'LT://APP1').remoteConfigPath).providerApp, + 'lt://APP1', + ); + assert.equal( + readGeneratedConfig(connect('browserstack', 'Bs://abc').remoteConfigPath).providerApp, + 'bs://abc', + ); + assert.throws( + () => connect('testmu', 'lt://'), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /--provider-app lt:\/\/ has no app id/.test(error.message), + ); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } +}); + test('connect testmu verifies against TESTMU_API_ENDPOINT', async () => { const tempRoot = mkdtempForTestSync('agent-device-connect-testmu-endpoint-'); vi.stubEnv('LT_USERNAME', 'lt-user'); diff --git a/src/cli/connection/cloud-webdriver-profile.ts b/src/cli/connection/cloud-webdriver-profile.ts index fe4c8b31ab..5611c86eaa 100644 --- a/src/cli/connection/cloud-webdriver-profile.ts +++ b/src/cli/connection/cloud-webdriver-profile.ts @@ -172,7 +172,15 @@ function hubProviderProfileFields( } function normalizeHubAppReference(hub: HubProviderProfile, app: string, cwd: string): string { - if (app.startsWith(hub.appScheme) || /^https?:\/\//i.test(app)) return app; + if (/^https?:\/\//i.test(app)) return app; + // URI schemes are case-insensitive; the hub only matches the lower-case spelling. + if (app.slice(0, hub.appScheme.length).toLowerCase() === hub.appScheme) { + const id = app.slice(hub.appScheme.length); + if (id.length > 0) return `${hub.appScheme}${id}`; + throw new AppError('INVALID_ARGS', `${hub.command} --provider-app ${app} has no app id.`, { + hint: `Pass ${hub.appHint}.`, + }); + } const resolvedPath = path.resolve(cwd, app); try { if (fs.statSync(resolvedPath).isFile()) return resolvedPath; diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index b4d396fe9f..37c359bd63 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -577,7 +577,7 @@ Providers: Direct proxy: agent-device connect proxy --daemon-base-url stores the shared proxy profile and client identity. BrowserStack: agent-device connect browserstack verifies credentials, the exact device, and a bs:// app reference, then stores a local provider profile. It does not create an App Automate session. AWS Device Farm: agent-device connect aws-device-farm verifies credentials and the exact project, device, and optional app upload, then stores a local provider profile. It does not create a remote access session. - TestMu AI: agent-device connect testmu verifies credentials, the exact virtual device (emulator or simulator) or, with --provider-device-type real, real device and OS version, and an lt:// app reference, then stores a local provider profile. It does not create a hub session. + TestMu AI: agent-device connect testmu verifies credentials and the exact virtual device (emulator or simulator) or, with --provider-device-type real, real device and OS version, then stores a local provider profile. --provider-app takes an lt:// app reference, an https URL, or a local path: connect looks an lt:// id up among your uploads for that device pool, while URL and local sources are uploaded and validated when open creates the session. It does not create a hub session. Limrun: agent-device connect limrun verifies access to the selected iOS or Android instance service, then stores a local provider profile. It does not create an instance. After direct-provider connect: diff --git a/website/docs/docs/testmu.md b/website/docs/docs/testmu.md index 40d80a9717..87912239ae 100644 --- a/website/docs/docs/testmu.md +++ b/website/docs/docs/testmu.md @@ -35,14 +35,17 @@ hub rejects `--provider-os-version 18` for a device listed with `18.0`, so `conn lists the versions the device offers. `--provider-app` accepts a TestMu AI app reference such as `lt://APP...`, an HTTP(S) app URL, or -an existing local app path (`.apk`, or a zipped simulator `.app` for iOS). TestMu AI uploads a local -path or fetches a URL when it creates the hosted session, through the virtual-device upload API. +an existing local app path (`.apk`, or a zipped simulator `.app` for iOS). When `open` creates the +hosted session, agent-device uploads a local path, and TestMu AI fetches a URL, through the +virtual-device upload API. During `connect`, agent-device checks the device/OS pair against TestMu AI's virtual-device catalog (`/capability/generator?isVirtualDevice=true`), verifies the credentials against your -uploaded-app listing, matches an `lt://` reference against that listing, and confirms that a local -artifact exists before saving its absolute path. `open` still needs the app's installed package or -bundle identifier, not the upload name or `lt://` id. +uploaded-app listing, looks an `lt://` reference up in that listing, and confirms that a local +artifact exists before saving its absolute path. `connect` does not prove the app usable: an `lt://` +id missing from the listing is still accepted, and TestMu AI validates it, like a URL or local +upload, only when the session is created. `open` still needs the app's installed package or bundle +identifier, not the upload name or `lt://` id. Optional labels: From 12bab7abe70715553eef8310bd18f35c7b05749c Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 22:36:44 +0530 Subject: [PATCH 11/57] test(provider-webdriver): exercise in-flight deadlines and aborts for real The BrowserStack session-details timeout test threw a prebuilt TimeoutError from the fetch stub, so it passed even with the deadline removed. The fetch now stays pending until the signal the lookup armed aborts, and the test checks that signal is the 15 second deadline. The TestMu AI upload cancellation test aborted before the upload reached fetch, so it never covered a request in flight. It now waits for the stub to start before aborting. The connect suites shared a copied connectWithGeneratedProviderProfile helper; it now lives in the shared test utilities. Co-Authored-By: Claude Opus 5.5 --- .../src/browserstack.test.ts | 43 ++++++++++++++----- .../provider-webdriver/src/testmu.test.ts | 11 ++--- src/__tests__/cloud-connect-profile.test.ts | 24 +---------- src/__tests__/cloud-connect-testmu.test.ts | 26 +---------- src/__tests__/test-utils/connect-command.ts | 27 ++++++++++++ 5 files changed, 68 insertions(+), 63 deletions(-) create mode 100644 src/__tests__/test-utils/connect-command.ts diff --git a/packages/provider-webdriver/src/browserstack.test.ts b/packages/provider-webdriver/src/browserstack.test.ts index 7d7af5610a..ff64a05c9e 100644 --- a/packages/provider-webdriver/src/browserstack.test.ts +++ b/packages/provider-webdriver/src/browserstack.test.ts @@ -2,7 +2,7 @@ import assert from 'node:assert/strict'; import { promises as fs } from 'node:fs'; import path from 'node:path'; -import { afterEach, test } from 'vitest'; +import { afterEach, test, vi } from 'vitest'; import { AppError } from '@agent-device/kernel/errors'; import { listBrowserStackCloudArtifacts, @@ -109,22 +109,45 @@ test('BrowserStack passes bs:// ids and URLs to the hub and uploads only local p test('BrowserStack session details lookup has a deadline and fails typed', async () => { const lookup = async () => await listBrowserStackCloudArtifacts('browserstack', 'SESSION1', upload); - const timeout = new DOMException('The operation was aborted due to timeout', 'TimeoutError'); - const transportFailures: unknown[] = [timeout, new TypeError('fetch failed')]; - for (const failure of transportFailures) { - globalThis.fetch = async (_input, init) => { - assert.ok(init?.signal instanceof AbortSignal); - throw failure; - }; - await assert.rejects(lookup(), (error: unknown) => { + // The deadline is the only thing that ends a hung lookup: the fetch stays pending until the + // signal the lookup armed for 15 s aborts, so removing the deadline would hang this test. + const deadline = new AbortController(); + const timeoutSpy = vi.spyOn(AbortSignal, 'timeout').mockImplementation(() => deadline.signal); + let started: () => void = () => {}; + const fetchStarted = new Promise((resolve) => { + started = resolve; + }); + globalThis.fetch = async (_input, init) => + await new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => reject(init.signal?.reason), { once: true }); + started(); + }); + try { + const pending = lookup(); + await fetchStarted; + assert.deepEqual(timeoutSpy.mock.calls, [[15_000]]); + const timeout = new DOMException('The operation was aborted due to timeout', 'TimeoutError'); + deadline.abort(timeout); + await assert.rejects(pending, (error: unknown) => { assert.ok(error instanceof AppError); assert.equal(error.code, 'COMMAND_FAILED'); assert.equal(error.message, 'BrowserStack session details lookup failed.'); - assert.equal(error.cause, failure); + assert.equal(error.cause, timeout); return true; }); + } finally { + timeoutSpy.mockRestore(); } + const networkFailure = new TypeError('fetch failed'); + globalThis.fetch = async () => { + throw networkFailure; + }; + await assert.rejects( + lookup(), + (error: unknown) => error instanceof AppError && error.cause === networkFailure, + ); + for (const body of ['gateway', '[]']) { globalThis.fetch = async () => new Response(body, { status: 200 }); await assert.rejects(lookup(), (error: unknown) => { diff --git a/packages/provider-webdriver/src/testmu.test.ts b/packages/provider-webdriver/src/testmu.test.ts index d14039a9fc..084e7fb084 100644 --- a/packages/provider-webdriver/src/testmu.test.ts +++ b/packages/provider-webdriver/src/testmu.test.ts @@ -197,18 +197,19 @@ test('TestMu upload reads the lt:// reference and aborts while the request is in const abortReason = new Error('request cancelled during TestMu AI upload'); try { await fs.writeFile(appPath, 'placeholder'); + let started: () => void = () => {}; + const fetchStarted = new Promise((resolve) => { + started = resolve; + }); globalThis.fetch = async (_input, init) => await new Promise((_resolve, reject) => { assert.equal(init?.signal, controller.signal); - if (init?.signal?.aborted) { - reject(init.signal.reason); - return; - } init?.signal?.addEventListener('abort', () => reject(init.signal?.reason), { once: true }); + started(); }); const pending = uploadTestMuApp(appPath, auth, controller.signal); - await Promise.resolve(); + await fetchStarted; controller.abort(abortReason); await assert.rejects(pending, (error: unknown) => error === abortReason); diff --git a/src/__tests__/cloud-connect-profile.test.ts b/src/__tests__/cloud-connect-profile.test.ts index e91650c8b2..5a0a3cd070 100644 --- a/src/__tests__/cloud-connect-profile.test.ts +++ b/src/__tests__/cloud-connect-profile.test.ts @@ -17,6 +17,7 @@ import { AppError } from '@agent-device/kernel/errors'; import { verifyLimrunConnection } from '@agent-device/provider-limrun'; import { providerWebDriver } from '../provider-webdriver.ts'; import { mkdtempForTestSync } from './test-utils/tmp-dir.ts'; +import { connectWithGeneratedProviderProfile } from './test-utils/connect-command.ts'; vi.mock('../cli/auth-session.ts', async (importOriginal) => ({ ...(await importOriginal()), @@ -857,29 +858,6 @@ async function captureConnectStdout(task: () => Promise): Promise { } } -async function connectWithGeneratedProviderProfile(options: { - stateDir: string; - positionals: string[]; - flags: Partial[0]['flags']>; -}): Promise { - const stdoutWrite = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); - try { - await connectCommand({ - positionals: options.positionals, - flags: { - json: true, - help: false, - version: false, - stateDir: options.stateDir, - ...options.flags, - }, - client: {} as AgentDeviceClient, - }); - } finally { - stdoutWrite.mockRestore(); - } -} - function readGeneratedConfig(configPath: string): { tenant?: string; leaseProvider?: string; diff --git a/src/__tests__/cloud-connect-testmu.test.ts b/src/__tests__/cloud-connect-testmu.test.ts index e9b7a3413c..ee37923166 100644 --- a/src/__tests__/cloud-connect-testmu.test.ts +++ b/src/__tests__/cloud-connect-testmu.test.ts @@ -2,17 +2,16 @@ import { afterEach, beforeEach, test, vi } from 'vitest'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; -import { connectCommand } from '../cli/commands/connection.ts'; import { runCliCapture } from './cli-capture.ts'; import { readActiveConnectionState, type RemoteConnectionState, } from '../remote/remote-connection-state.ts'; -import type { AgentDeviceClient } from '../agent-device-client.ts'; import { resolveCloudWebDriverConnectProfile } from '../cli/connection/cloud-webdriver-profile.ts'; import { AppError } from '@agent-device/kernel/errors'; import { providerWebDriver } from '../provider-webdriver.ts'; import { mkdtempForTestSync } from './test-utils/tmp-dir.ts'; +import { connectWithGeneratedProviderProfile } from './test-utils/connect-command.ts'; vi.mock('../provider-webdriver.ts', () => ({ providerWebDriver: { verifyConnection: vi.fn() }, @@ -313,29 +312,6 @@ test('connect limrun refuses profile fields Limrun does not read', async () => { ); }); -async function connectWithGeneratedProviderProfile(options: { - stateDir: string; - positionals: string[]; - flags: Partial[0]['flags']>; -}): Promise { - const stdoutWrite = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); - try { - await connectCommand({ - positionals: options.positionals, - flags: { - json: true, - help: false, - version: false, - stateDir: options.stateDir, - ...options.flags, - }, - client: {} as AgentDeviceClient, - }); - } finally { - stdoutWrite.mockRestore(); - } -} - function readGeneratedConfig(configPath: string): { providerApp?: string; providerOsVersion?: string; diff --git a/src/__tests__/test-utils/connect-command.ts b/src/__tests__/test-utils/connect-command.ts new file mode 100644 index 0000000000..c16a40ca43 --- /dev/null +++ b/src/__tests__/test-utils/connect-command.ts @@ -0,0 +1,27 @@ +import { vi } from 'vitest'; +import { connectCommand } from '../../cli/commands/connection.ts'; +import type { AgentDeviceClient } from '../../agent-device-client.ts'; + +/** Runs `connect` the way the CLI does, writing its generated profile under `stateDir`. */ +export async function connectWithGeneratedProviderProfile(options: { + stateDir: string; + positionals: string[]; + flags: Partial[0]['flags']>; +}): Promise { + const stdoutWrite = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + try { + await connectCommand({ + positionals: options.positionals, + flags: { + json: true, + help: false, + version: false, + stateDir: options.stateDir, + ...options.flags, + }, + client: {} as AgentDeviceClient, + }); + } finally { + stdoutWrite.mockRestore(); + } +} From 0d70437b3cd9f810bdd610d856ef100c6f8df9aa Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 22:53:32 +0530 Subject: [PATCH 12/57] fix(provider-webdriver): refuse profile fields on a repeat allocation too The daemon hands a run's repeat lease_allocate the lease it already holds, and both the WebDriver runtime and the Limrun runtime returned the live session for a known lease before checking the request's profile fields. A second allocation on the same run that set a refused field, such as providerDeviceType on BrowserStack, therefore succeeded. Both runtimes now refuse before reusing a live session. Because a refusal of that repeat request is a provider allocate failure, the lease handler no longer releases a lease it reused when allocate throws; doing so would end the run's lease and leave the session the first allocation created without an owner. The runtime's profileFields option is now required, and each provider definition passes its own declaration into createRuntime, so a provider that forgets to wire its declaration fails to compile instead of silently refusing nothing. Co-Authored-By: Claude Opus 5.5 --- packages/provider-limrun/src/runtime.ts | 6 ++-- packages/provider-webdriver/src/index.ts | 2 +- .../src/profile-fields.fixtures.ts | 31 ++++++++++++++++ .../src/provider-definitions.ts | 20 ++++++----- .../src/runtime-session.test.ts | 2 ++ .../provider-webdriver/src/runtime-session.ts | 11 +++--- .../provider-webdriver/src/runtime.test.ts | 2 ++ packages/provider-webdriver/src/runtime.ts | 4 +-- src/__tests__/limrun-runtime.test.ts | 26 ++++++++++++++ src/daemon/handlers/__tests__/lease.test.ts | 36 +++++++++++++++++++ src/daemon/handlers/lease.ts | 8 ++++- .../cloud-webdriver-provider-adapters.test.ts | 36 +++++++++++++++++++ 12 files changed, 163 insertions(+), 21 deletions(-) create mode 100644 packages/provider-webdriver/src/profile-fields.fixtures.ts diff --git a/packages/provider-limrun/src/runtime.ts b/packages/provider-limrun/src/runtime.ts index f1fb902b1e..df8b581e47 100644 --- a/packages/provider-limrun/src/runtime.ts +++ b/packages/provider-limrun/src/runtime.ts @@ -231,9 +231,6 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { if (lease.leaseProvider !== this.provider) return undefined; const platform = platformForLimrunLeaseBackend(lease.backend); if (!platform) return undefined; - const existing = this.sessions.get(lease.leaseId); - if (existing) return { limrunInstanceId: existing.instanceId, device: existing.device }; - const { allocateLimrunAndroidSession, allocateLimrunIosSession, @@ -241,7 +238,10 @@ class LimrunRuntimeImplementation implements ProviderDeviceRuntime { resolvePreinstalledAppId, resolveRequestedLimrunAppAsset, } = await import('./session-allocation.ts'); + // Before the reuse below: a repeat allocation of a live lease carries flags of its own. rejectRefusedLimrunProfileFields(context); + const existing = this.sessions.get(lease.leaseId); + if (existing) return { limrunInstanceId: existing.instanceId, device: existing.device }; const requestedAsset = await resolveRequestedLimrunAppAsset(this.limrun, platform, context); const session = platform === 'ios' diff --git a/packages/provider-webdriver/src/index.ts b/packages/provider-webdriver/src/index.ts index 12c30acf60..b8c785eb3c 100644 --- a/packages/provider-webdriver/src/index.ts +++ b/packages/provider-webdriver/src/index.ts @@ -50,7 +50,7 @@ export function createProviderWebDriver( return { providerIds: definitions.map((definition) => definition.provider), createDefaultRuntimes: (env = process.env) => - definitions.map((definition) => definition.createRuntime(env)), + definitions.map((definition) => definition.createRuntime(env, definition.profileFields)), listArtifactsFromEnv: async (query, env) => { if (!query.providerSessionId) return undefined; return await definitions diff --git a/packages/provider-webdriver/src/profile-fields.fixtures.ts b/packages/provider-webdriver/src/profile-fields.fixtures.ts new file mode 100644 index 0000000000..f52e710f25 --- /dev/null +++ b/packages/provider-webdriver/src/profile-fields.fixtures.ts @@ -0,0 +1,31 @@ +import type { ProviderProfileFieldDeclaration } from '@agent-device/contracts/provider-profile-fields'; + +/** A test hub that reads every profile field, so no flag is refused. */ +export function consumeAllProfileFields(provider: string): ProviderProfileFieldDeclaration { + return { + provider, + label: provider, + fields: { + providerApp: 'consumed', + providerOsVersion: 'consumed', + providerDeviceType: 'consumed', + providerProject: 'consumed', + providerBuild: 'consumed', + providerSessionName: 'consumed', + providerDeviceOrientation: 'consumed', + providerGeoLocation: 'consumed', + providerTimezone: 'consumed', + providerAppiumVersion: 'consumed', + providerLanguage: 'consumed', + providerLocale: 'consumed', + providerNetworkProfile: 'consumed', + providerCustomNetwork: 'consumed', + providerNoResignApp: 'consumed', + awsProjectArn: 'consumed', + awsDeviceArn: 'consumed', + awsAppArn: 'consumed', + awsRegion: 'consumed', + awsInteractionMode: 'consumed', + }, + }; +} diff --git a/packages/provider-webdriver/src/provider-definitions.ts b/packages/provider-webdriver/src/provider-definitions.ts index 6981043fc5..d701f29595 100644 --- a/packages/provider-webdriver/src/provider-definitions.ts +++ b/packages/provider-webdriver/src/provider-definitions.ts @@ -174,9 +174,13 @@ export const CLOUD_WEBDRIVER_PROFILE_FIELDS: Readonly< export type CloudWebDriverProviderDefinition = { provider: CloudWebDriverKnownProviderName; - /** Every profile field, consumed or refused; session preparation refuses the refused ones. */ + /** Every profile field, consumed or refused; lease allocation refuses the refused ones. */ profileFields: ProviderProfileFieldDeclaration; - createRuntime: (env: DefaultCloudWebDriverProviderRuntimeEnv) => CloudWebDriverRuntime; + /** Receives `profileFields`, which the runtime requires, so the two cannot drift apart. */ + createRuntime: ( + env: DefaultCloudWebDriverProviderRuntimeEnv, + profileFields: ProviderProfileFieldDeclaration, + ) => CloudWebDriverRuntime; listArtifactsFromEnv: ( providerSessionId: string, env: DefaultCloudWebDriverArtifactEnv, @@ -190,11 +194,11 @@ export function createCloudWebDriverProviderDefinitions( { provider: CLOUD_WEBDRIVER_PROVIDERS.browserStack, profileFields: BROWSERSTACK_PROFILE_FIELDS, - createRuntime: (env) => + createRuntime: (env, profileFields) => createCloudWebDriverRuntime({ clientVersion: dependencies.clientVersion, provider: CLOUD_WEBDRIVER_PROVIDERS.browserStack, - profileFields: BROWSERSTACK_PROFILE_FIELDS, + profileFields, platform: 'android', deviceName: 'BrowserStack device', endpoint: env.BROWSERSTACK_WEBDRIVER_ENDPOINT ?? BROWSERSTACK_APP_AUTOMATE_ENDPOINT, @@ -299,11 +303,11 @@ export function createCloudWebDriverProviderDefinitions( { provider: CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, profileFields: AWS_DEVICE_FARM_PROFILE_FIELDS, - createRuntime: (env) => + createRuntime: (env, profileFields) => createCloudWebDriverRuntime({ clientVersion: dependencies.clientVersion, provider: CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, - profileFields: AWS_DEVICE_FARM_PROFILE_FIELDS, + profileFields, endpoint: 'http://127.0.0.1/', platform: 'android', deviceName: 'AWS Device Farm device', @@ -370,11 +374,11 @@ export function createCloudWebDriverProviderDefinitions( { provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, profileFields: TESTMU_PROFILE_FIELDS, - createRuntime: (env) => + createRuntime: (env, profileFields) => createCloudWebDriverRuntime({ clientVersion: dependencies.clientVersion, provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, - profileFields: TESTMU_PROFILE_FIELDS, + profileFields, platform: 'android', deviceName: 'TestMu AI device', endpoint: env.TESTMU_WEBDRIVER_ENDPOINT ?? TESTMU_WEBDRIVER_ENDPOINT, diff --git a/packages/provider-webdriver/src/runtime-session.test.ts b/packages/provider-webdriver/src/runtime-session.test.ts index e435e5571b..4429903b7e 100644 --- a/packages/provider-webdriver/src/runtime-session.test.ts +++ b/packages/provider-webdriver/src/runtime-session.test.ts @@ -3,6 +3,7 @@ import { afterEach, test } from 'vitest'; import type { DeviceLease } from '@agent-device/contracts/device'; import { AppError } from '@agent-device/kernel/errors'; import { createCloudWebDriverRuntime, type CloudWebDriverRuntimeOptions } from './runtime.ts'; +import { consumeAllProfileFields } from './profile-fields.fixtures.ts'; const realFetch = globalThis.fetch; @@ -106,6 +107,7 @@ function makeRuntime(overrides: Partial = {}) { endpoint: 'https://webdriver.test/wd/hub/', platform: 'android', deviceName: 'Test device', + profileFields: consumeAllProfileFields('webdriver-test'), requestPolicy: { retryAttempts: 0 }, ...overrides, }); diff --git a/packages/provider-webdriver/src/runtime-session.ts b/packages/provider-webdriver/src/runtime-session.ts index 293aa51b0d..7274789f6c 100644 --- a/packages/provider-webdriver/src/runtime-session.ts +++ b/packages/provider-webdriver/src/runtime-session.ts @@ -82,6 +82,11 @@ export class WebDriverSessionManager { async allocate(lease: DeviceLease, req?: LeaseLifecycleContext): Promise { if (lease.leaseProvider !== this.options.provider) return undefined; + // Before the reuse below: a repeat allocation of a live lease carries flags of its own. + // Loaded here rather than eagerly so the package entry's closure stays unchanged. + const { rejectRefusedProviderProfileFields } = + await import('@agent-device/contracts/provider-profile-fields'); + rejectRefusedProviderProfileFields(req?.flags, this.options.profileFields); if (this.sessionsByLeaseId.has(lease.leaseId)) return this.heartbeat(lease); const prepared = await this.prepareSession(lease, req); const client = new WebDriverClient({ @@ -203,12 +208,6 @@ export class WebDriverSessionManager { lease: DeviceLease, req: LeaseLifecycleContext | undefined, ): Promise { - if (this.options.profileFields) { - // Loaded on first allocation so the package entry's eager closure stays unchanged. - const { rejectRefusedProviderProfileFields } = - await import('@agent-device/contracts/provider-profile-fields'); - rejectRefusedProviderProfileFields(req?.flags, this.options.profileFields); - } const base = this.baseSessionForLease(lease); return this.options.prepareSession ? await this.options.prepareSession({ lease, req, base }) diff --git a/packages/provider-webdriver/src/runtime.test.ts b/packages/provider-webdriver/src/runtime.test.ts index 134e8a4a78..e94d1faece 100644 --- a/packages/provider-webdriver/src/runtime.test.ts +++ b/packages/provider-webdriver/src/runtime.test.ts @@ -1,5 +1,6 @@ import { expect, test } from 'vitest'; import { createCloudWebDriverRuntime } from './runtime.ts'; +import { consumeAllProfileFields } from './profile-fields.fixtures.ts'; test('publishes eager provider metadata without loading a WebDriver session', async () => { const runtime = createCloudWebDriverRuntime({ @@ -8,6 +9,7 @@ test('publishes eager provider metadata without loading a WebDriver session', as endpoint: 'https://webdriver.test/wd/hub/', platform: 'android', deviceName: 'Test device', + profileFields: consumeAllProfileFields('webdriver-test'), }); try { diff --git a/packages/provider-webdriver/src/runtime.ts b/packages/provider-webdriver/src/runtime.ts index 146cb89787..4b94ac2d72 100644 --- a/packages/provider-webdriver/src/runtime.ts +++ b/packages/provider-webdriver/src/runtime.ts @@ -101,8 +101,8 @@ export type CloudWebDriverRuntimeOptions = { deviceId?: (lease: DeviceLease) => string; prepareSession?: CloudWebDriverPrepareSession; capabilityOverrides?: CloudWebDriverCapabilityOverrides; - /** Profile fields this provider reads; any field it refuses fails session preparation. */ - profileFields?: ProviderProfileFieldDeclaration; + /** Profile fields this provider reads; any field it refuses fails lease allocation. */ + profileFields: ProviderProfileFieldDeclaration; }; export function createCloudWebDriverRuntime( diff --git a/src/__tests__/limrun-runtime.test.ts b/src/__tests__/limrun-runtime.test.ts index dd3446fbeb..5aa066d8b5 100644 --- a/src/__tests__/limrun-runtime.test.ts +++ b/src/__tests__/limrun-runtime.test.ts @@ -172,6 +172,32 @@ test('Limrun runtime identifies direct CLI usage to the Limrun API', async () => } }); +test('Limrun refuses a refused field on a repeat allocation of its live lease', async () => { + const runtime = new LimrunRuntime({ apiKey: 'lim_test_key' }); + const lease: SimulatorLease = { + leaseId: 'lease-repeat', + tenantId: 'team-a', + runId: 'run-a', + backend: 'ios-instance', + leaseProvider: 'limrun', + createdAt: 1, + heartbeatAt: 1, + expiresAt: 60_001, + }; + try { + const allocateLease = runtime.leaseLifecycle.allocate; + if (!allocateLease) throw new Error('Limrun runtime must provide lease allocation'); + await allocateLease(lease); + await assert.rejects( + allocateLease(lease, { flags: { providerDeviceType: 'real' } }), + /--provider-device-type is not supported by Limrun/, + ); + assert.equal(limrunMockState.iosCreate.mock.calls.length, 1); + } finally { + await runtime.shutdown(); + } +}); + test('Limrun iOS uses shared deep-link classification', async () => { const runtime = new LimrunRuntime({ apiKey: 'lim_test_key' }); diff --git a/src/daemon/handlers/__tests__/lease.test.ts b/src/daemon/handlers/__tests__/lease.test.ts index 5cbd378ea6..076cc0c07b 100644 --- a/src/daemon/handlers/__tests__/lease.test.ts +++ b/src/daemon/handlers/__tests__/lease.test.ts @@ -172,3 +172,39 @@ test('a lease allocated without a provider keeps the TTL it was created with', a assert.equal(lease.expiresAt, 65_000); assert.deepEqual(registry.listActiveLeases(), [lease]); }); + +// The registry hands a run's repeat allocation the lease it already holds. A provider refusing the +// repeat request (a profile field it does not read) must leave that lease and its session alone. +test("a refused repeat allocation keeps the run's live lease", async () => { + const registry = new LeaseRegistry(); + const sessionStore = makeSessionStore('agent-device-refused-repeat-'); + let calls = 0; + const allocate = async (req: DaemonRequest) => + await handleLeaseCommands({ + req, + sessionName: 'lease-ttl-test', + sessionStore, + leaseRegistry: registry, + leaseLifecycleProvider: { + allocate: async () => { + calls += 1; + if (calls === 1) return { providerSessionId: 'session-1' }; + throw new AppError('INVALID_ARGS', '--provider-device-type is not supported by Cloud.'); + }, + }, + }); + + const first = await allocate(allocateRequest()); + const lease = (first?.ok ? first.data?.lease : undefined) as DeviceLease; + const repeat = allocateRequest(); + repeat.flags = { providerDeviceType: 'real' }; + await assert.rejects( + allocate(repeat), + (error: unknown) => error instanceof AppError && error.code === 'INVALID_ARGS', + ); + assert.equal(calls, 2); + assert.deepEqual( + registry.listActiveLeases().map((entry) => entry.leaseId), + [lease.leaseId], + ); +}); diff --git a/src/daemon/handlers/lease.ts b/src/daemon/handlers/lease.ts index f0826bb87f..06d46126cd 100644 --- a/src/daemon/handlers/lease.ts +++ b/src/daemon/handlers/lease.ts @@ -70,7 +70,13 @@ export async function handleLeaseCommands(args: LeaseHandlerArgs): Promise entry.leaseId), + ); const lease = leaseRegistry.allocateLease(leaseScopeToAllocateRequest(leaseScope)); + // A run's repeat allocation reuses its live lease; refusing that request must not end the + // lease, or the provider session the first allocation created is left without an owner. + const reused = activeLeaseIds.has(lease.leaseId); const requestId = req.meta?.requestId; return await leaseRegistry.runDeviceMutation(lease, async () => { let providerData: Record | undefined; @@ -85,7 +91,7 @@ export async function handleLeaseCommands(args: LeaseHandlerArgs): Promise { + await withProviderScenarioResource(FakeCloudProviderServer.start, async (server) => { + const provider = createProviderWebDriver({ + clientVersion: CLIENT_VERSION, + runHostCommand: unexpectedHostCommand, + }); + const runtime = runtimeFor( + provider.createDefaultRuntimes({ + BROWSERSTACK_USERNAME: 'user', + BROWSERSTACK_ACCESS_KEY: 'key', + BROWSERSTACK_WEBDRIVER_ENDPOINT: `${server.url}/wd/hub/`, + }), + CLOUD_WEBDRIVER_PROVIDERS.browserStack, + ); + const lease = makeLease(CLOUD_WEBDRIVER_PROVIDERS.browserStack); + const context = browserStackContext(lease); + try { + await runtime.leaseLifecycle.allocate?.(lease, context); + const sessionCalls = server.calls.length; + await assert.rejects( + async () => + await runtime.leaseLifecycle.allocate?.(lease, { + flags: { ...context.flags, providerDeviceType: 'real' }, + }), + /--provider-device-type is not supported by BrowserStack/, + ); + assert.equal(server.calls.length, sessionCalls); + assert.deepEqual(await runtime.leaseLifecycle.heartbeat?.(lease), { + provider: CLOUD_WEBDRIVER_PROVIDERS.browserStack, + }); + } finally { + await runtime.shutdown(); + } + }); +}, 15_000); + test('AWS Device Farm facade uses the injected host-command capability for its full lifecycle', async () => { await withProviderScenarioResource(FakeCloudProviderServer.start, async (server) => { const host = new FakeAwsHostCommand(`${server.url}/wd/hub/`); From 80aa05f862528ba4d0b300898ef381e6aa91cd16 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 22:54:48 +0530 Subject: [PATCH 13/57] fix(provider-webdriver): validate and canonicalize app references at connect Connect only checked that an lt:// or bs:// reference had something after the scheme, so `lt://a b` and `lt://a/b` were saved and failed only when the hub created the session. Both hubs' reference grammars now live in the light providers module, and connect, session preparation and the TestMu AI upload reader all validate against them. Connect also verified the raw spelling typed on the command line: the generated profile stored `lt://APP1` for `LT://APP1`, but the flags handed to verification were the raw CLI flags laid over the profile, so the uploaded-app lookup missed and reported a local artifact. The canonical reference now wins in the flags verification reads. Co-Authored-By: Claude Opus 5.5 --- .../provider-webdriver/src/browserstack.ts | 2 ++ packages/provider-webdriver/src/providers.ts | 12 ++++++++ packages/provider-webdriver/src/testmu.ts | 9 ++---- src/__tests__/cloud-connect-testmu.test.ts | 29 ++++++++++++------- src/cli/connection/cloud-webdriver-profile.ts | 25 ++++++++++++---- 5 files changed, 55 insertions(+), 22 deletions(-) diff --git a/packages/provider-webdriver/src/browserstack.ts b/packages/provider-webdriver/src/browserstack.ts index ffd854052f..dc55978c8f 100644 --- a/packages/provider-webdriver/src/browserstack.ts +++ b/packages/provider-webdriver/src/browserstack.ts @@ -2,6 +2,7 @@ import type { CloudArtifact, CloudArtifactsResult } from '@agent-device/contract import type { CloudWebDriverCapabilityOverrides } from './capabilities.ts'; import type { CloudWebDriverUploadApp } from './runtime.ts'; import { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; +import { isBrowserStackAppReference } from './providers.ts'; import { appendUrlPath, appFileUploadForm, @@ -112,6 +113,7 @@ export async function resolveBrowserStackAppReference( cwd: options.cwd, referenceScheme: 'bs://', referenceLabel: 'a bs:// app id', + isReference: isBrowserStackAppReference, uploadFile: async (appPath, signal) => await uploadBrowserStackApp(appPath, options, signal), signal: options.signal, }); diff --git a/packages/provider-webdriver/src/providers.ts b/packages/provider-webdriver/src/providers.ts index 4160d465a5..c64c3c7c00 100644 --- a/packages/provider-webdriver/src/providers.ts +++ b/packages/provider-webdriver/src/providers.ts @@ -14,3 +14,15 @@ export function isCloudWebDriverProviderName( ): provider is CloudWebDriverKnownProviderName { return provider !== undefined && CLOUD_WEBDRIVER_KNOWN_PROVIDERS.has(provider); } + +/** + * The app references each hub accepts. An id outside the grammar would otherwise pass every local + * check and fail only when the hub creates the session. + */ +export function isBrowserStackAppReference(value: string): boolean { + return /^bs:\/\/[\w.-]+$/.test(value); +} + +export function isTestMuAppReference(value: string): boolean { + return /^lt:\/\/[\w.-]+$/.test(value); +} diff --git a/packages/provider-webdriver/src/testmu.ts b/packages/provider-webdriver/src/testmu.ts index 1217fd6c05..68b970d971 100644 --- a/packages/provider-webdriver/src/testmu.ts +++ b/packages/provider-webdriver/src/testmu.ts @@ -4,6 +4,7 @@ import type { CloudArtifact, CloudArtifactsResult } from '@agent-device/contract import type { ProviderDeviceType } from '@agent-device/contracts/remote'; import type { CloudWebDriverPlatform, CloudWebDriverUploadApp } from './runtime.ts'; import { AppError } from '@agent-device/kernel/errors'; +import { isTestMuAppReference } from './providers.ts'; import { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; import { appendUrlPath, @@ -26,6 +27,8 @@ const TESTMU_APP_UPLOAD_ENDPOINTS: Record = { }; export const TESTMU_APPS_ENDPOINT = 'https://manual-api.lambdatest.com/app/data'; export const TESTMU_API_ENDPOINT = 'https://mobile-api.lambdatest.com/mobile-automation/api/v1'; +export { isTestMuAppReference }; + const TESTMU_DASHBOARD_TEST_URL = 'https://appautomation.lambdatest.com/test?testID='; export type TestMuCapabilitiesOptions = { @@ -193,12 +196,6 @@ export function buildTestMuCapabilities( }; } -const TESTMU_APP_REFERENCE = /^lt:\/\/[\w.-]+$/; - -export function isTestMuAppReference(value: string): boolean { - return TESTMU_APP_REFERENCE.test(value); -} - /** The upload and app-list APIs answer with a bare app id or an `lt://` reference. */ export function testMuAppReferenceFromId(id: string): string { return id.startsWith('lt://') ? id : `lt://${id}`; diff --git a/src/__tests__/cloud-connect-testmu.test.ts b/src/__tests__/cloud-connect-testmu.test.ts index ee37923166..ea4b2f5dfa 100644 --- a/src/__tests__/cloud-connect-testmu.test.ts +++ b/src/__tests__/cloud-connect-testmu.test.ts @@ -102,21 +102,28 @@ test('connect canonicalizes an upper-case app scheme and refuses an empty app id }); try { - assert.equal( - readGeneratedConfig(connect('testmu', 'LT://APP1').remoteConfigPath).providerApp, - 'lt://APP1', - ); + const upperCase = connect('testmu', 'LT://APP1'); + assert.equal(readGeneratedConfig(upperCase.remoteConfigPath).providerApp, 'lt://APP1'); + // Connect verification reads these flags, so they must carry the canonical reference too. + assert.equal(upperCase.flags.providerApp, 'lt://APP1'); assert.equal( readGeneratedConfig(connect('browserstack', 'Bs://abc').remoteConfigPath).providerApp, 'bs://abc', ); - assert.throws( - () => connect('testmu', 'lt://'), - (error: unknown) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /--provider-app lt:\/\/ has no app id/.test(error.message), - ); + for (const [provider, app] of [ + ['testmu', 'lt://'], + ['testmu', 'lt://a b'], + ['testmu', 'LT://a/b'], + ['browserstack', 'bs://'], + ] as const) { + assert.throws( + () => connect(provider, app), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + error.message.includes(`--provider-app ${app} is not a valid`), + ); + } } finally { fs.rmSync(tempRoot, { recursive: true, force: true }); } diff --git a/src/cli/connection/cloud-webdriver-profile.ts b/src/cli/connection/cloud-webdriver-profile.ts index 5611c86eaa..a9e0f61db2 100644 --- a/src/cli/connection/cloud-webdriver-profile.ts +++ b/src/cli/connection/cloud-webdriver-profile.ts @@ -4,6 +4,10 @@ import { readAwsDeviceFarmRegionFromArn, type CloudWebDriverKnownProviderName, } from '@agent-device/provider-webdriver'; +import { + isBrowserStackAppReference, + isTestMuAppReference, +} from '@agent-device/provider-webdriver/providers'; import { rejectRefusedProviderProfileFields } from '@agent-device/contracts/provider-profile-fields'; import type { RemoteConfigProfile } from '../../remote/remote-config-schema.ts'; import { AppError } from '@agent-device/kernel/errors'; @@ -55,6 +59,11 @@ export function resolveCloudWebDriverConnectProfile(options: { cwd: options.cwd, env: options.env, flags: options.flags, + // Verification reads these flags; it must see the canonical reference the profile saved, + // not the spelling typed on the command line. + ...(providerConfig.providerApp + ? { extraFlags: { providerApp: providerConfig.providerApp } } + : {}), }); } @@ -99,6 +108,8 @@ type HubProviderProfile = { credentialEnv: readonly [string, string]; /** Scheme of the provider's own app references, e.g. `bs://` or `lt://`. */ appScheme: string; + /** The hub's own reference grammar, checked here so a malformed id fails at connect. */ + isAppReference: (reference: string) => boolean; appHint: string; }; @@ -107,6 +118,7 @@ const BROWSERSTACK_HUB_PROFILE: HubProviderProfile = { label: 'BrowserStack', credentialEnv: ['BROWSERSTACK_USERNAME', 'BROWSERSTACK_ACCESS_KEY'], appScheme: 'bs://', + isAppReference: isBrowserStackAppReference, appHint: '', }; @@ -115,6 +127,7 @@ const TESTMU_HUB_PROFILE: HubProviderProfile = { label: 'TestMu AI', credentialEnv: ['LT_USERNAME', 'LT_ACCESS_KEY'], appScheme: 'lt://', + isAppReference: isTestMuAppReference, appHint: '', }; @@ -175,11 +188,13 @@ function normalizeHubAppReference(hub: HubProviderProfile, app: string, cwd: str if (/^https?:\/\//i.test(app)) return app; // URI schemes are case-insensitive; the hub only matches the lower-case spelling. if (app.slice(0, hub.appScheme.length).toLowerCase() === hub.appScheme) { - const id = app.slice(hub.appScheme.length); - if (id.length > 0) return `${hub.appScheme}${id}`; - throw new AppError('INVALID_ARGS', `${hub.command} --provider-app ${app} has no app id.`, { - hint: `Pass ${hub.appHint}.`, - }); + const reference = `${hub.appScheme}${app.slice(hub.appScheme.length)}`; + if (hub.isAppReference(reference)) return reference; + throw new AppError( + 'INVALID_ARGS', + `${hub.command} --provider-app ${app} is not a valid ${hub.appScheme} app reference.`, + { hint: `Pass ${hub.appHint}.` }, + ); } const resolvedPath = path.resolve(cwd, app); try { From 720e8927cc5e9cc8420c02e5c8315b8bfbb2431c Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 22:55:35 +0530 Subject: [PATCH 14/57] fix(provider-webdriver): refuse a directory before any hosted upload A materialized build that names no uploadable file falls back to its installable path, which for iOS is the extracted .app directory. The deployment runtime handed that to the hub's uploader, and BrowserStack's read it as a file and failed with a raw EISDIR. The deployment runtime now refuses a directory before calling any hub's uploader, with INVALID_ARGS naming the provider and path and a hint to zip the .app or pass the .ipa, .apk, or .aab. Co-Authored-By: Claude Opus 5.5 --- .../src/runtime-deployment.test.ts | 30 +++++++++++++++++++ .../src/runtime-deployment.ts | 17 +++++++++++ 2 files changed, 47 insertions(+) diff --git a/packages/provider-webdriver/src/runtime-deployment.test.ts b/packages/provider-webdriver/src/runtime-deployment.test.ts index 41aaa508d3..efe9835962 100644 --- a/packages/provider-webdriver/src/runtime-deployment.test.ts +++ b/packages/provider-webdriver/src/runtime-deployment.test.ts @@ -125,6 +125,36 @@ test('a hosted upload sends the file the materializer names, else the installabl expect(installApp).toHaveBeenNthCalledWith(1, 'lt://1', expect.any(AbortSignal)); }); +test('a hosted upload refuses a directory typed instead of reading it', async () => { + const tempDir = await mkdtempForTest('agent-device-materialized-directory-'); + try { + const installablePath = path.join(tempDir, 'extracted', 'App.app'); + await fs.mkdir(installablePath, { recursive: true }); + const uploadApp = vi.fn(); + const installApp = vi.fn(async () => undefined); + const deployment = createWebDriverDeploymentRuntime({ + provider: 'browserstack', + uploadApp, + findSessionForDevice: () => activeSession(installApp), + }); + + await expect( + deployment.deployMaterializedApp( + iosDevice, + { artifact: { installablePath, cleanup: async () => {} } }, + new AbortController().signal, + ), + ).rejects.toMatchObject({ + code: 'INVALID_ARGS', + message: `browserstack can only upload an app file, not a directory: ${installablePath}`, + }); + expect(uploadApp).not.toHaveBeenCalled(); + expect(installApp).not.toHaveBeenCalled(); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + test('a provider without an uploader still installs the materialized bundle path', async () => { const installApp = vi.fn(async () => undefined); const deployment = createWebDriverDeploymentRuntime({ diff --git a/packages/provider-webdriver/src/runtime-deployment.ts b/packages/provider-webdriver/src/runtime-deployment.ts index fb2a058b86..2f96f1ace4 100644 --- a/packages/provider-webdriver/src/runtime-deployment.ts +++ b/packages/provider-webdriver/src/runtime-deployment.ts @@ -1,3 +1,4 @@ +import fs from 'node:fs/promises'; import type { ProviderDeviceInstallOptions, ProviderDeviceInstallResult, @@ -155,6 +156,7 @@ async function uploadAppIfNeeded( } const uploadApp = session.prepared.uploadApp ?? options.uploadApp; if (!uploadApp) return undefined; + await assertUploadableFile(options.provider, appPath); return await uploadApp({ provider: options.provider, lease: session.lease, @@ -166,6 +168,21 @@ async function uploadAppIfNeeded( }); } +/** Hosted upload APIs take one file; an extracted `.app` with no declared archive is a directory. */ +async function assertUploadableFile(provider: string, appPath: string): Promise { + const stat = await fs.stat(appPath).catch(() => undefined); + if (!stat || stat.isFile()) return; + throw new AppError( + 'INVALID_ARGS', + `${provider} can only upload an app file, not a directory: ${appPath}`, + { + provider, + appPath, + hint: 'Zip the iOS simulator .app bundle and install the .zip, or install the .ipa, .apk, or .aab.', + }, + ); +} + function deploymentResult( result: ProviderDeviceInstallResult | undefined, fallbackTarget?: string, From 417ff5ba7a40d0f168d7caf9b78057944397a494 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Fri, 2 Oct 2026 23:01:22 +0530 Subject: [PATCH 15/57] fix(cli): refuse a non-URL install-from-source source with a typed error install-from-source sends its positional as a URL source. A local path reached the iOS trusted-host check, where `new URL()` threw a TypeError that surfaced as an untyped UNKNOWN "Invalid URL". The CLI now refuses a positional that is not an http(s) URL with INVALID_ARGS and points at `install ` for local builds, and the trusted-host check reports an unparsable source as INVALID_ARGS instead of throwing. Co-Authored-By: Claude Opus 5.5 --- packages/provision-kit/src/install-source.ts | 3 +- src/__tests__/cli-install-from-source.test.ts | 30 +++++++++++++++++++ src/__tests__/install-source.test.ts | 8 +++++ src/commands/management/install.ts | 5 ++++ 4 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 src/__tests__/cli-install-from-source.test.ts diff --git a/packages/provision-kit/src/install-source.ts b/packages/provision-kit/src/install-source.ts index 26119979e9..3daada22f3 100644 --- a/packages/provision-kit/src/install-source.ts +++ b/packages/provision-kit/src/install-source.ts @@ -160,7 +160,8 @@ export async function validateDownloadSourceUrl(parsedUrl: URL): Promise { } export function isTrustedInstallSourceUrl(sourceUrl: string | URL): boolean { - const parsed = sourceUrl instanceof URL ? sourceUrl : new URL(sourceUrl); + const parsed = sourceUrl instanceof URL ? sourceUrl : URL.parse(sourceUrl); + if (!parsed) throw new AppError('INVALID_ARGS', 'Invalid source URL'); const hostname = parsed.hostname.toLowerCase(); if (!hostname) return false; const pathname = parsed.pathname; diff --git a/src/__tests__/cli-install-from-source.test.ts b/src/__tests__/cli-install-from-source.test.ts new file mode 100644 index 0000000000..f99e8b3705 --- /dev/null +++ b/src/__tests__/cli-install-from-source.test.ts @@ -0,0 +1,30 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import type { AgentDeviceClient } from '../agent-device-client.ts'; +import { tryRunClientBackedCommand } from '../cli/commands/router.ts'; + +test('install-from-source refuses a local path with a typed error', async () => { + const client = { + apps: { + installFromSource: async () => { + throw new Error('unexpected call'); + }, + }, + } as unknown as AgentDeviceClient; + + await assert.rejects( + () => + tryRunClientBackedCommand({ + command: 'install-from-source', + positionals: ['/abs/path/app.zip'], + flags: { json: false, help: false, version: false }, + client, + }), + (error) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + error.message === 'install-from-source must be an http(s) URL: /abs/path/app.zip' && + String(error.details?.hint).includes('install '), + ); +}); diff --git a/src/__tests__/install-source.test.ts b/src/__tests__/install-source.test.ts index 974a6ba509..7059e83d0f 100644 --- a/src/__tests__/install-source.test.ts +++ b/src/__tests__/install-source.test.ts @@ -24,6 +24,7 @@ import { withAppleToolProvider, } from '@agent-device/platform-apple/tool-provider'; import { prepareIosInstallArtifact } from '@agent-device/platform-apple/install-artifact'; +import { AppError } from '@agent-device/kernel/errors'; import { ANDROID_INSTALL_SOURCE_CONTRACT_EVIDENCE } from './install-source.coverage.ts'; import { mkdtempForTest } from './test-utils/tmp-dir.ts'; import * as networkTransport from '@agent-device/provision-kit/install-source-network-transport'; @@ -104,6 +105,13 @@ test('isTrustedInstallSourceUrl recognizes supported artifact services', () => { false, ); assert.equal(isTrustedInstallSourceUrl('https://expo.dev/pricing'), false); + assert.throws( + () => isTrustedInstallSourceUrl('/abs/path/app.zip'), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + error.message === 'Invalid source URL', + ); }); test('materializeInstallablePath rejects archive extraction when disabled', async () => { diff --git a/src/commands/management/install.ts b/src/commands/management/install.ts index 7155d10e09..015858f4fd 100644 --- a/src/commands/management/install.ts +++ b/src/commands/management/install.ts @@ -204,6 +204,11 @@ function resolveInstallSource(positionals: string[], flags: CliFlags) { } if (githubArtifactSource) return githubArtifactSource; if (configuredSource) return configuredSource; + if (!/^https?:\/\//i.test(url!)) { + throw new AppError('INVALID_ARGS', `install-from-source must be an http(s) URL: ${url}`, { + hint: 'Install a local build with install .', + }); + } return { kind: 'url' as const, url: url!, From 4b81820698ea0e99c4046e10db88918884b9c9e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 16:54:29 +0200 Subject: [PATCH 16/57] feat: support provider-owned plugin connections and WebDriver adapters --- packages/kernel/src/app-error.ts | 14 + packages/kernel/src/errors.ts | 15 +- .../src/artifact-results.ts | 15 + .../browserstack-connection-verification.ts | 55 +--- .../provider-webdriver/src/browserstack.ts | 129 ++++---- packages/provider-webdriver/src/plugin.ts | 17 ++ packages/provider-webdriver/src/providers.ts | 4 + .../src/webdriver-utils.test.ts | 213 ++++++++++++- .../provider-webdriver/src/webdriver-utils.ts | 285 +++++++++++++++++- .../connection/connect-provider-adapters.ts | 65 +++- src/cli/connection/provider-policy.ts | 30 +- src/plugins/connection.ts | 32 ++ src/plugins/host.ts | 63 ++++ src/plugins/load.test.ts | 43 ++- src/plugins/load.ts | 68 ++++- src/plugins/manifest.ts | 30 +- src/plugins/store.test.ts | 1 + src/plugins/store.ts | 1 + src/sdk/plugin-webdriver.ts | 3 + src/sdk/plugins.ts | 10 + website/docs/docs/client-api.md | 2 + website/docs/docs/plugins.md | 23 +- 22 files changed, 955 insertions(+), 163 deletions(-) create mode 100644 packages/kernel/src/app-error.ts create mode 100644 packages/provider-webdriver/src/plugin.ts create mode 100644 src/plugins/connection.ts create mode 100644 src/plugins/host.ts create mode 100644 src/sdk/plugin-webdriver.ts diff --git a/packages/kernel/src/app-error.ts b/packages/kernel/src/app-error.ts new file mode 100644 index 0000000000..d748efb9b2 --- /dev/null +++ b/packages/kernel/src/app-error.ts @@ -0,0 +1,14 @@ +import type { AppErrorCode, AppErrorDetails } from './errors.ts'; + +export class AppError extends Error { + code: AppErrorCode; + details?: AppErrorDetails; + cause?: unknown; + + constructor(code: AppErrorCode, message: string, details?: AppErrorDetails, cause?: unknown) { + super(message); + this.code = code; + this.details = details; + this.cause = cause; + } +} diff --git a/packages/kernel/src/errors.ts b/packages/kernel/src/errors.ts index 747541f4e5..42dd4f1a85 100644 --- a/packages/kernel/src/errors.ts +++ b/packages/kernel/src/errors.ts @@ -1,3 +1,5 @@ +import { AppError } from './app-error.ts'; +export { AppError } from './app-error.ts'; import { redactDiagnosticData, sanitizeErrorCause as normalizeErrorCause } from './redaction.ts'; /** @@ -219,19 +221,6 @@ export type DaemonError = { supportedOn?: string; }; -export class AppError extends Error { - code: AppErrorCode; - details?: AppErrorDetails; - cause?: unknown; - - constructor(code: AppErrorCode, message: string, details?: AppErrorDetails, cause?: unknown) { - super(message); - this.code = code; - this.details = details; - this.cause = cause; - } -} - /** Rehydrate a daemon transport error into the error type used by local callers. */ export function throwDaemonError(error: DaemonError): never { throw new AppError( diff --git a/packages/provider-webdriver/src/artifact-results.ts b/packages/provider-webdriver/src/artifact-results.ts index bf02c7c77e..3aaabc69b5 100644 --- a/packages/provider-webdriver/src/artifact-results.ts +++ b/packages/provider-webdriver/src/artifact-results.ts @@ -28,3 +28,18 @@ export function unavailableCloudArtifactsResult(options: { message: options.error instanceof Error ? options.error.message : String(options.error), }; } + +/** A ready URL artifact read off a provider's session-details record, or nothing when the field is absent. */ +export function urlArtifactFromDetails( + provider: string, + providerSessionId: string, + details: Record, + field: string, + kind: CloudArtifact['kind'], + name: string, +): CloudArtifact | undefined { + const value = details[field]; + const url = typeof value === 'string' ? value.trim() : ''; + if (url.length === 0) return undefined; + return { provider, providerSessionId, kind, name, url, availability: 'ready' }; +} diff --git a/packages/provider-webdriver/src/browserstack-connection-verification.ts b/packages/provider-webdriver/src/browserstack-connection-verification.ts index efdfb4e3c8..dfb18bd989 100644 --- a/packages/provider-webdriver/src/browserstack-connection-verification.ts +++ b/packages/provider-webdriver/src/browserstack-connection-verification.ts @@ -1,7 +1,6 @@ import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; -import { agentDeviceRequestHeaders } from './request-headers.ts'; -import { basicAuthHeader } from './webdriver-utils.ts'; +import { asRecord, fetchProviderVerificationJson, sameOsVersion } from './webdriver-utils.ts'; import type { CloudWebDriverConnectionVerification, CloudWebDriverConnectionVerificationOptions, @@ -105,42 +104,16 @@ async function fetchBrowserStackJson( auth: { username: string; accessKey: string }, clientVersion: string, ): Promise { - try { - const response = await fetch(endpoint, { - headers: { - ...agentDeviceRequestHeaders(clientVersion), - Authorization: basicAuthHeader(auth), - }, - signal: AbortSignal.timeout(15_000), - }); - if (!response.ok) { - const unauthorized = response.status === 401 || response.status === 403; - throw new AppError( - unauthorized ? 'UNAUTHORIZED' : 'COMMAND_FAILED', - 'BrowserStack rejected connection verification.', - { - status: response.status, - hint: unauthorized - ? 'Check BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY.' - : 'Retry connect or check the BrowserStack service status.', - }, - ); - } - return (await response.json()) as unknown; - } catch (error) { - if (error instanceof AppError) throw error; - throw new AppError( - 'COMMAND_FAILED', - 'BrowserStack connection verification failed.', - { hint: 'Check network access to api-cloud.browserstack.com and retry connect.' }, - error, - ); - } -} - -function sameOsVersion(left: string, right: string): boolean { - const normalize = (value: string) => value.replace(/(?:\.0)+$/, ''); - return normalize(left) === normalize(right); + return await fetchProviderVerificationJson(endpoint, { + clientVersion, + auth, + hints: { + service: 'BrowserStack', + unauthorizedHint: 'Check BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY.', + serviceHint: 'Retry connect or check the BrowserStack service status.', + networkHint: 'Check network access to api-cloud.browserstack.com and retry connect.', + }, + }); } function readBrowserStackDevices( @@ -184,9 +157,3 @@ function readBrowserStackApps( ]; }); } - -function asRecord(value: unknown): Record | undefined { - return value && typeof value === 'object' && !Array.isArray(value) - ? (value as Record) - : undefined; -} diff --git a/packages/provider-webdriver/src/browserstack.ts b/packages/provider-webdriver/src/browserstack.ts index 6c8d10fd4b..dc55978c8f 100644 --- a/packages/provider-webdriver/src/browserstack.ts +++ b/packages/provider-webdriver/src/browserstack.ts @@ -1,12 +1,17 @@ -import fs from 'node:fs/promises'; -import path from 'node:path'; import type { CloudArtifact, CloudArtifactsResult } from '@agent-device/contracts/observability'; import type { CloudWebDriverCapabilityOverrides } from './capabilities.ts'; import type { CloudWebDriverUploadApp } from './runtime.ts'; -import { AppError } from '@agent-device/kernel/errors'; -import { agentDeviceRequestHeaders } from './request-headers.ts'; -import { cloudArtifactsReadyOrPending } from './artifact-results.ts'; -import { basicAuthHeader, trimTrailingSlash } from './webdriver-utils.ts'; +import { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; +import { isBrowserStackAppReference } from './providers.ts'; +import { + appendUrlPath, + appFileUploadForm, + asRecord, + createHubUploadApp, + fetchProviderSessionDetails, + postHubAppUpload, + resolveHubAppReference, +} from './webdriver-utils.ts'; export const BROWSERSTACK_APP_AUTOMATE_ENDPOINT = 'https://hub-cloud.browserstack.com/wd/hub/'; export const BROWSERSTACK_APP_UPLOAD_ENDPOINT = @@ -76,41 +81,42 @@ export async function uploadBrowserStackApp( signal?: AbortSignal, ): Promise { signal?.throwIfAborted(); - const file = await fs.readFile(appPath); - const form = new FormData(); - form.set('file', new Blob([file]), path.basename(appPath)); - const response = await fetch(options.endpoint ?? BROWSERSTACK_APP_UPLOAD_ENDPOINT, { - method: 'POST', - headers: { - ...agentDeviceRequestHeaders(options.clientVersion), - Authorization: basicAuthHeader(options), + return await postHubAppUpload( + await appFileUploadForm(appPath, 'file'), + { + service: 'BrowserStack', + endpoint: options.endpoint ?? BROWSERSTACK_APP_UPLOAD_ENDPOINT, + clientVersion: options.clientVersion, + auth: options, + readAppReference: readBrowserStackAppUrl, }, - body: form, signal, - }); - const json = (await response.json()) as unknown; - const appUrl = readBrowserStackAppUrl(json); - if (!response.ok || !appUrl) { - throw new AppError('COMMAND_FAILED', 'BrowserStack app upload failed.', { - status: response.status, - response: json, - }); - } - return appUrl; + ); } export function createBrowserStackUploadApp( options: Required, ): CloudWebDriverUploadApp { - return async ({ appPath, options: installOptions, signal }) => { - const appReference = await uploadBrowserStackApp(appPath, options, signal); - return { - appReference, - bundleId: installOptions?.appIdentifierHint, - packageName: installOptions?.packageNameHint, - launchTarget: installOptions?.appIdentifierHint ?? installOptions?.packageNameHint, - }; - }; + return createHubUploadApp( + async (appPath, signal) => await uploadBrowserStackApp(appPath, options, signal), + ); +} + +/** The hub fetches a public URL itself, so only a local path is uploaded. */ +export async function resolveBrowserStackAppReference( + app: string, + options: BrowserStackUploadOptions & { cwd?: string; signal?: AbortSignal }, +): Promise { + return await resolveHubAppReference({ + service: 'BrowserStack', + app, + cwd: options.cwd, + referenceScheme: 'bs://', + referenceLabel: 'a bs:// app id', + isReference: isBrowserStackAppReference, + uploadFile: async (appPath, signal) => await uploadBrowserStackApp(appPath, options, signal), + signal: options.signal, + }); } /** @@ -138,39 +144,25 @@ export function buildBrowserStackCapabilities( buildName: options.buildName, sessionName: options.sessionName, ...(options.deviceFeatures ?? {}), - ...asRecord(configuredBstackOptions), + ...(asRecord(configuredBstackOptions) ?? {}), }, }; } -function asRecord(value: unknown): Record { - return value && typeof value === 'object' && !Array.isArray(value) - ? (value as Record) - : {}; -} - async function fetchBrowserStackSessionDetails( sessionId: string, options: BrowserStackSessionDetailsOptions, ): Promise> { - const endpoint = new URL( - `${trimTrailingSlash(String(options.endpoint ?? BROWSERSTACK_SESSION_DETAILS_ENDPOINT))}/${sessionId}.json`, + const endpoint = appendUrlPath( + options.endpoint ?? BROWSERSTACK_SESSION_DETAILS_ENDPOINT, + `${sessionId}.json`, ); - const response = await fetch(endpoint, { - headers: { - ...agentDeviceRequestHeaders(options.clientVersion), - Authorization: basicAuthHeader(options), - }, + const json = await fetchProviderSessionDetails(endpoint, { + clientVersion: options.clientVersion, + auth: options, + service: 'BrowserStack', }); - const json = (await response.json()) as unknown; - if (!response.ok || !json || typeof json !== 'object') { - throw new AppError('COMMAND_FAILED', 'BrowserStack session details lookup failed.', { - status: response.status, - response: json, - }); - } - const details = (json as { automation_session?: unknown }).automation_session ?? json; - return details && typeof details === 'object' ? (details as Record) : {}; + return asRecord(json.automation_session) ?? json; } function mapBrowserStackArtifacts( @@ -179,7 +171,7 @@ function mapBrowserStackArtifacts( details: Record, ): CloudArtifact[] { return [ - browserStackUrlArtifact( + urlArtifactFromDetails( provider, providerSessionId, details, @@ -187,7 +179,7 @@ function mapBrowserStackArtifacts( 'video', 'Session video', ), - browserStackUrlArtifact( + urlArtifactFromDetails( provider, providerSessionId, details, @@ -195,7 +187,7 @@ function mapBrowserStackArtifacts( 'appium-log', 'Appium logs', ), - browserStackUrlArtifact( + urlArtifactFromDetails( provider, providerSessionId, details, @@ -203,7 +195,7 @@ function mapBrowserStackArtifacts( 'device-log', 'Device logs', ), - browserStackUrlArtifact( + urlArtifactFromDetails( provider, providerSessionId, details, @@ -211,7 +203,7 @@ function mapBrowserStackArtifacts( 'provider-session', 'BrowserStack dashboard', ), - browserStackUrlArtifact( + urlArtifactFromDetails( provider, providerSessionId, details, @@ -222,19 +214,6 @@ function mapBrowserStackArtifacts( ].filter((artifact): artifact is CloudArtifact => artifact !== undefined); } -function browserStackUrlArtifact( - provider: string, - providerSessionId: string, - details: Record, - field: string, - kind: CloudArtifact['kind'], - name: string, -): CloudArtifact | undefined { - const url = details[field]; - if (typeof url !== 'string' || url.length === 0) return undefined; - return { provider, providerSessionId, kind, name, url, availability: 'ready' }; -} - function readBrowserStackAppUrl(value: unknown): string | undefined { if (!value || typeof value !== 'object') return undefined; const appUrl = (value as { app_url?: unknown }).app_url; diff --git a/packages/provider-webdriver/src/plugin.ts b/packages/provider-webdriver/src/plugin.ts new file mode 100644 index 0000000000..e1b46966b4 --- /dev/null +++ b/packages/provider-webdriver/src/plugin.ts @@ -0,0 +1,17 @@ +export { createCloudWebDriverRuntime } from './runtime.ts'; +export type { CloudWebDriverRuntimeOptions } from './runtime.ts'; +export { + appFileUploadForm, + appendUrlPath, + asRecord, + basicAuthHeader, + createHubUploadApp, + fetchProviderSessionDetails, + fetchProviderVerificationJson, + postHubAppUpload, + requireProviderDeviceOrientation, + resolveHubAppReference, + sameOsVersion, +} from './webdriver-utils.ts'; +export { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; +export { buildCloudWebDriverBaseCapabilities } from './runtime-session.ts'; diff --git a/packages/provider-webdriver/src/providers.ts b/packages/provider-webdriver/src/providers.ts index 814a0c19ae..a4799ba975 100644 --- a/packages/provider-webdriver/src/providers.ts +++ b/packages/provider-webdriver/src/providers.ts @@ -13,3 +13,7 @@ export function isCloudWebDriverProviderName( ): provider is CloudWebDriverKnownProviderName { return provider !== undefined && CLOUD_WEBDRIVER_KNOWN_PROVIDERS.has(provider); } + +export function isBrowserStackAppReference(value: string): boolean { + return /^bs:\/\/[\w.-]+$/.test(value); +} diff --git a/packages/provider-webdriver/src/webdriver-utils.test.ts b/packages/provider-webdriver/src/webdriver-utils.test.ts index 58d326c241..58a182c789 100644 --- a/packages/provider-webdriver/src/webdriver-utils.test.ts +++ b/packages/provider-webdriver/src/webdriver-utils.test.ts @@ -1,6 +1,25 @@ import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { trimLeadingSlash, trimTrailingSlash } from './webdriver-utils.ts'; +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { afterEach, test, vi } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { + appendUrlPath, + asRecord, + createHubUploadApp, + fetchProviderVerificationJson, + postHubAppUpload, + resolveHubAppReference, + trimLeadingSlash, + trimTrailingSlash, +} from './webdriver-utils.ts'; +import { mkdtempForTest } from './tmp-dir.fixtures.ts'; + +const realFetch = globalThis.fetch; + +afterEach(() => { + globalThis.fetch = realFetch; +}); test('slash trimming utilities handle slash-heavy strings without regular expressions', () => { const slashRun = '/'.repeat(10_000); @@ -15,3 +34,193 @@ test('slash trimming utilities handle slash-heavy strings without regular expres assert.equal(trimLeadingSlash(slashRun), ''); assert.equal(trimTrailingSlash(slashRun), ''); }); + +test('asRecord admits plain objects only', () => { + assert.deepEqual(asRecord({ a: 1 }), { a: 1 }); + assert.equal(asRecord([]), undefined); + assert.equal(asRecord(null), undefined); + assert.equal(asRecord('x'), undefined); +}); + +const hub = { + service: 'Hub', + endpoint: 'https://upload.example.test/app', + clientVersion: '0.0.0-test', + auth: { username: 'user', accessKey: 'key' }, + readAppReference: (body: unknown) => asRecord(body)?.ref as string | undefined, +}; + +test('the hub upload helper posts with credentials and returns the vendor reference', async () => { + const form = new FormData(); + globalThis.fetch = async (input, init) => { + assert.equal(String(input), hub.endpoint); + assert.equal(init?.method, 'POST'); + assert.equal(init?.body, form); + const headers = init?.headers as Record; + assert.equal(headers.Authorization, `Basic ${Buffer.from('user:key').toString('base64')}`); + assert.equal(headers['x-agent-device-version'], '0.0.0-test'); + return new Response(JSON.stringify({ ref: 'hub://APP1' }), { status: 200 }); + }; + assert.equal(await postHubAppUpload(form, hub), 'hub://APP1'); +}); + +test('the hub upload helper fails typed with the status on an error page or a missing reference', async () => { + for (const response of [ + new Response('502 Bad Gateway', { status: 502 }), + new Response(JSON.stringify({ message: 'ok' }), { status: 200 }), + ]) { + globalThis.fetch = async () => response; + await assert.rejects(postHubAppUpload(new FormData(), hub), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.message, 'Hub app upload failed.'); + assert.equal(error.details?.status, response.status); + return true; + }); + } +}); + +test('the hub install adapter uploads the build and launches the hinted app', async () => { + const upload = vi.fn(async () => 'hub://APP2'); + const signal = new AbortController().signal; + const result = await createHubUploadApp(upload)({ + appPath: '/builds/App.ipa', + options: { appIdentifierHint: 'com.example.app' }, + signal, + }); + assert.deepEqual(upload.mock.calls, [['/builds/App.ipa', signal]]); + assert.deepEqual(result, { + appReference: 'hub://APP2', + bundleId: 'com.example.app', + packageName: undefined, + launchTarget: 'com.example.app', + }); +}); + +test('the hub app resolver passes references through, uploads local files, and routes URLs per hub', async () => { + const tempDir = await mkdtempForTest('agent-device-hub-resolve-'); + try { + await fs.writeFile(path.join(tempDir, 'App.apk'), 'placeholder'); + const uploadFile = vi.fn(async (appPath: string) => `hub://${path.basename(appPath)}`); + const resolve = (app: string, uploadUrl?: (url: string) => Promise) => + resolveHubAppReference({ + service: 'Hub', + app, + cwd: tempDir, + referenceScheme: 'hub://', + referenceLabel: 'a hub:// app id', + uploadFile, + uploadUrl, + }); + + assert.equal(await resolve('hub://APP3'), 'hub://APP3'); + assert.equal(await resolve('HUB://APP3'), 'hub://APP3'); + assert.equal(await resolve('https://builds.example/App.apk'), 'https://builds.example/App.apk'); + assert.equal( + await resolve('https://builds.example/App.apk', async (url) => `fetched:${url}`), + 'fetched:https://builds.example/App.apk', + ); + assert.equal(await resolve('App.apk'), 'hub://App.apk'); + assert.deepEqual(uploadFile.mock.calls, [[path.join(tempDir, 'App.apk'), undefined]]); + await assert.rejects(resolve('missing.apk'), (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'INVALID_ARGS'); + assert.equal( + error.message, + 'Hub --provider-app must be a hub:// app id, URL, or existing local app path.', + ); + return true; + }); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + +test('the hub app resolver refuses an empty reference and a directory, typed', async () => { + const tempDir = await mkdtempForTest('agent-device-hub-resolve-invalid-'); + try { + await fs.mkdir(path.join(tempDir, 'App.app')); + const uploadFile = vi.fn(async () => 'hub://never'); + const resolve = (app: string) => + resolveHubAppReference({ + service: 'Hub', + app, + cwd: tempDir, + referenceScheme: 'hub://', + referenceLabel: 'a hub:// app id', + isReference: (reference) => /^hub:\/\/\w+$/.test(reference), + uploadFile, + }); + + for (const [app, message] of [ + ['hub://', /^Hub --provider-app hub:\/\/ is not a hub:\/\/ app id\.$/], + ['hub://a b', /is not a hub:\/\/ app id/], + ['App.app', /must be an app file, not a directory: .*App\.app$/], + ] as const) { + await assert.rejects( + resolve(app), + (error: unknown) => + error instanceof AppError && error.code === 'INVALID_ARGS' && message.test(error.message), + ); + } + assert.equal(uploadFile.mock.calls.length, 0); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + +test('appending a route keeps a query on the base endpoint', () => { + assert.equal( + appendUrlPath('https://api.example.test/v1/?region=eu', 'sessions/S%201').toString(), + 'https://api.example.test/v1/sessions/S%201?region=eu', + ); + assert.equal( + appendUrlPath('https://api.example.test/v1', 'sessions/S1').toString(), + 'https://api.example.test/v1/sessions/S1', + ); +}); + +const verificationHints = { + service: 'Hub', + unauthorizedHint: 'Check HUB_KEY.', + serviceHint: 'Retry connect or check the Hub service status.', + networkHint: 'Check network access to the hub.', +}; + +test('connection verification reports a non-JSON success typed, with its status', async () => { + globalThis.fetch = async () => new Response('maintenance', { status: 200 }); + await assert.rejects( + fetchProviderVerificationJson('https://api.example.test/apps', { + clientVersion: '0.0.0-test', + hints: verificationHints, + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'COMMAND_FAILED'); + assert.equal(error.message, 'Hub connection verification answer was not JSON.'); + assert.equal(error.details?.status, 200); + assert.equal(error.details?.hint, verificationHints.serviceHint); + return true; + }, + ); +}); + +test('connection verification gives each failure its provider hint', async () => { + for (const [status, code, hint] of [ + [401, 'UNAUTHORIZED', verificationHints.unauthorizedHint], + [503, 'COMMAND_FAILED', verificationHints.serviceHint], + ] as const) { + globalThis.fetch = async () => new Response('nope', { status }); + await assert.rejects( + fetchProviderVerificationJson('https://api.example.test/apps', { + clientVersion: '0.0.0-test', + hints: verificationHints, + }), + (error: unknown) => + error instanceof AppError && + error.code === code && + error.details?.status === status && + error.details?.hint === hint, + ); + } +}); diff --git a/packages/provider-webdriver/src/webdriver-utils.ts b/packages/provider-webdriver/src/webdriver-utils.ts index 5af69a9f4c..d75fd560ba 100644 --- a/packages/provider-webdriver/src/webdriver-utils.ts +++ b/packages/provider-webdriver/src/webdriver-utils.ts @@ -1,5 +1,17 @@ -import type { DeviceLease } from '@agent-device/contracts/device'; +import fs from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; +import type { + DeviceLease, + ProviderDeviceInstallOptions, + ProviderDeviceInstallResult, +} from '@agent-device/contracts/device'; +import { + PROVIDER_DEVICE_ORIENTATIONS, + type ProviderDeviceOrientation, +} from '@agent-device/contracts/remote'; import { AppError, errorMessage } from '@agent-device/kernel/errors'; +import { agentDeviceRequestHeaders } from './request-headers.ts'; export type LeaseValue = T | ((lease: DeviceLease) => T); @@ -44,9 +56,280 @@ export function trimTrailingSlash(value: string): string { return lastNonSlash === value.length - 1 ? value : value.slice(0, lastNonSlash + 1); } +/** Appends `route` to the base's path; a query on the base is kept rather than swallowing the route. */ +export function appendUrlPath(base: string | URL, route: string): URL { + const url = new URL(base); + url.pathname = `${trimTrailingSlash(url.pathname)}/${route}`; + return url; +} + export function withTrailingSlash(url: URL): URL { if (url.pathname.endsWith('/')) return url; const copy = new URL(url); copy.pathname = `${copy.pathname}/`; return copy; } + +export function asRecord(value: unknown): Record | undefined { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +type HubCredentials = { username: string; accessKey: string }; + +/** A multipart form carrying the local app file under the hub's field name. */ +export async function appFileUploadForm(appPath: string, fileField: string): Promise { + const form = new FormData(); + form.set(fileField, new Blob([await readFile(appPath)]), path.basename(appPath)); + return form; +} + +/** + * POSTs an app upload to a hosted hub and returns the hub's app reference. A non-2xx answer, a + * body that is not JSON, or one without a reference is `COMMAND_FAILED` with the HTTP status. + */ +export async function postHubAppUpload( + form: FormData, + options: { + service: string; + endpoint: string | URL; + clientVersion: string; + auth: HubCredentials; + readAppReference: (body: unknown) => string | undefined; + }, + signal?: AbortSignal, +): Promise { + const response = await fetch(options.endpoint, { + method: 'POST', + headers: { + ...agentDeviceRequestHeaders(options.clientVersion), + Authorization: basicAuthHeader(options.auth), + }, + body: form, + signal, + }); + const json = await readProviderJsonBody(response); + const appReference = options.readAppReference(json); + if (!response.ok || !appReference) { + throw new AppError('COMMAND_FAILED', `${options.service} app upload failed.`, { + status: response.status, + response: json, + }); + } + return appReference; +} + +/** The `install` adapter of a hosted hub: upload the local build, then launch the hinted app. */ +export function createHubUploadApp( + upload: (appPath: string, signal?: AbortSignal) => Promise, +): (params: { + appPath: string; + options?: ProviderDeviceInstallOptions; + signal?: AbortSignal; +}) => Promise { + return async ({ appPath, options, signal }) => ({ + appReference: await upload(appPath, signal), + bundleId: options?.appIdentifierHint, + packageName: options?.packageNameHint, + launchTarget: options?.appIdentifierHint ?? options?.packageNameHint, + }); +} + +/** + * Turns `--provider-app` into a reference the hub accepts: its own reference scheme passes + * through, a public URL passes through unless the hub only takes its own references (then + * `uploadUrl` has the hub fetch it), and anything else must be a local file to upload. + */ +export async function resolveHubAppReference(options: { + service: string; + app: string; + cwd?: string; + referenceScheme: string; + /** How the scheme reads in the error message, e.g. `a bs:// app id`. */ + referenceLabel: string; + /** Validates a canonical reference; by default any non-empty id after the scheme is accepted. */ + isReference?: (reference: string) => boolean; + uploadFile: (appPath: string, signal?: AbortSignal) => Promise; + uploadUrl?: (url: string, signal?: AbortSignal) => Promise; + signal?: AbortSignal; +}): Promise { + const { app } = options; + const reference = canonicalHubAppReference(app, options.referenceScheme); + if (reference !== undefined) { + const isReference = + options.isReference ?? ((value: string) => value.length > options.referenceScheme.length); + if (isReference(reference)) return reference; + throw new AppError( + 'INVALID_ARGS', + `${options.service} --provider-app ${app} is not ${options.referenceLabel}.`, + { providerApp: app }, + ); + } + if (/^https?:\/\//i.test(app)) { + return options.uploadUrl ? await options.uploadUrl(app, options.signal) : app; + } + const appPath = path.resolve(options.cwd ?? process.cwd(), app); + const stat = fs.statSync(appPath, { throwIfNoEntry: false }); + if (!stat) { + throw new AppError( + 'INVALID_ARGS', + `${options.service} --provider-app must be ${options.referenceLabel}, URL, or existing local app path.`, + { providerApp: app }, + ); + } + if (!stat.isFile()) { + throw new AppError( + 'INVALID_ARGS', + `${options.service} --provider-app must be an app file, not a directory: ${appPath}`, + { + providerApp: app, + hint: 'Zip an iOS simulator .app bundle and pass the .zip, or pass the .ipa, .apk, or .aab.', + }, + ); + } + return await options.uploadFile(appPath, options.signal); +} + +/** URI schemes are case-insensitive, so `LT://id` is the hub reference `lt://id`. */ +function canonicalHubAppReference(app: string, scheme: string): string | undefined { + if (app.slice(0, scheme.length).toLowerCase() !== scheme) return undefined; + return `${scheme}${app.slice(scheme.length)}`; +} + +const PROVIDER_API_TIMEOUT_MS = 15_000; + +/** The provider rejected or could not answer a verification call; typed so callers never sniff text. */ +export type ProviderJsonFailureHints = { + service: string; + unauthorizedHint: string; + /** For any other non-2xx answer, or a 2xx answer that is not JSON. */ + serviceHint: string; + networkHint: string; +}; + +/** + * Fetches JSON from a hosted provider's API during connection verification. A 401/403 is + * `UNAUTHORIZED` with a credential hint, any other non-2xx or a body that is not JSON is + * `COMMAND_FAILED` with the status, and a transport failure is wrapped so its cause survives + * without leaking the credentials. + */ +export async function fetchProviderVerificationJson( + endpoint: string | URL, + options: { + clientVersion: string; + auth?: { username: string; accessKey: string }; + hints: ProviderJsonFailureHints; + }, +): Promise { + const { service, unauthorizedHint, serviceHint, networkHint } = options.hints; + try { + const response = await fetch(endpoint, { + headers: { + ...agentDeviceRequestHeaders(options.clientVersion), + ...(options.auth ? { Authorization: basicAuthHeader(options.auth) } : {}), + }, + signal: AbortSignal.timeout(PROVIDER_API_TIMEOUT_MS), + }); + if (!response.ok) { + const unauthorized = response.status === 401 || response.status === 403; + throw new AppError( + unauthorized ? 'UNAUTHORIZED' : 'COMMAND_FAILED', + `${service} rejected connection verification.`, + { + status: response.status, + hint: unauthorized ? unauthorizedHint : serviceHint, + }, + ); + } + const json = await readProviderJsonBody(response); + if (json === undefined) { + throw new AppError( + 'COMMAND_FAILED', + `${service} connection verification answer was not JSON.`, + { status: response.status, hint: serviceHint }, + ); + } + return json; + } catch (error) { + if (error instanceof AppError) throw error; + throw new AppError( + 'COMMAND_FAILED', + `${service} connection verification failed.`, + { hint: networkHint }, + error, + ); + } +} + +/** + * Fetches a provider's session-details JSON with basic auth under a deadline. A transport failure, + * a non-2xx answer, or a body that is not a JSON object is `COMMAND_FAILED`. + */ +export async function fetchProviderSessionDetails( + endpoint: string | URL, + options: { + clientVersion: string; + auth: { username: string; accessKey: string }; + service: string; + }, +): Promise> { + let response: Response; + let json: unknown; + try { + response = await fetch(endpoint, { + headers: { + ...agentDeviceRequestHeaders(options.clientVersion), + Authorization: basicAuthHeader(options.auth), + }, + signal: AbortSignal.timeout(PROVIDER_API_TIMEOUT_MS), + }); + json = await readProviderJsonBody(response); + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + `${options.service} session details lookup failed.`, + { hint: `Check network access to the ${options.service} API, then retry.` }, + error, + ); + } + const details = asRecord(json); + if (!response.ok || !details) { + throw new AppError('COMMAND_FAILED', `${options.service} session details lookup failed.`, { + status: response.status, + response: json, + }); + } + return details; +} + +/** A provider response body parsed as JSON, or `undefined` when it is empty or not JSON (a gateway error page). */ +async function readProviderJsonBody(response: Response): Promise { + const text = await response.text(); + if (text.length === 0) return undefined; + try { + return JSON.parse(text) as unknown; + } catch { + return undefined; + } +} + +/** `1.0` and `1` name the same OS release on BrowserStack's catalog; TestMu's hub matches spellings exactly. */ +export function sameOsVersion(left: string, right: string): boolean { + const normalize = (value: string) => value.replace(/(?:\.0)+$/, ''); + return normalize(left) === normalize(right); +} + +/** Validates a device-orientation flag against the shared enum before it reaches a hub that would ignore it. */ +export function requireProviderDeviceOrientation( + spec: { flag: string; capability: string }, + value: string, +): ProviderDeviceOrientation { + const match = PROVIDER_DEVICE_ORIENTATIONS.find((orientation) => orientation === value); + if (match) return match; + throw new AppError('INVALID_ARGS', `Invalid ${spec.flag} value: ${value}.`, { + hint: `Use ${PROVIDER_DEVICE_ORIENTATIONS.join('|')}.`, + flag: spec.flag, + capability: spec.capability, + }); +} diff --git a/src/cli/connection/connect-provider-adapters.ts b/src/cli/connection/connect-provider-adapters.ts index 27d9c5c564..b7312fca31 100644 --- a/src/cli/connection/connect-provider-adapters.ts +++ b/src/cli/connection/connect-provider-adapters.ts @@ -12,7 +12,15 @@ import { resolveCloudWebDriverConnectProfile } from './cloud-webdriver-profile.t import { resolveLimrunConnectProfile } from './limrun-profile.ts'; import { resolveProxyConnectProfile } from './proxy-profile.ts'; import { profileToCliFlags } from '../remote-config-flags.ts'; -import { isConnectProviderName, type ConnectProvider } from './provider-policy.ts'; +import { + isConnectProviderName, + type ConnectProvider, + type BuiltinConnectProvider, +} from './provider-policy.ts'; +import { withPluginConnection } from '../../plugins/load.ts'; +import { readMetroProfileFields } from './profile-fields.ts'; +import { buildConnectClientId } from './client-id.ts'; +import { persistAndResolveGeneratedProfile } from './generated-config.ts'; type ConnectProfile = { flags: CliFlags; remoteConfigPath: string }; type ResolvedConnectProfile = ConnectProfile & { provider?: ConnectProvider }; @@ -73,7 +81,7 @@ const CONNECT_PROVIDER_ADAPTERS = { resolve: resolveLimrunConnectProfile, verify: verifyLimrun, }, -} satisfies Record; +} satisfies Record; export async function resolveConnectProviderProfile(options: { provider?: ConnectProvider; @@ -103,12 +111,47 @@ export async function resolveConnectProviderProfile(options: { } const provider = options.provider ?? (shouldUseProxyConnectShortcut(options.flags) ? 'proxy' : 'cloud'); - const profile = await CONNECT_PROVIDER_ADAPTERS[provider].resolve({ + const context = { flags: options.flags, stateDir: options.stateDir, cwd, env, - }); + }; + const adapter = (CONNECT_PROVIDER_ADAPTERS as Partial>)[ + provider + ]; + const profile = adapter + ? await adapter.resolve(context) + : await withPluginConnection(provider, env, async (connection) => { + const resolved = await connection.resolve(context); + if (resolved.profile.leaseProvider !== provider) + throw new AppError( + 'INVALID_ARGS', + 'Plugin connection profile must select its declared provider', + ); + const clientId = buildConnectClientId( + provider, + context.stateDir, + context.flags.session, + resolved.profile.device, + ); + return persistAndResolveGeneratedProfile({ + ...context, + ...resolved, + provider, + profile: { + tenant: context.flags.tenant ?? provider, + sessionIsolation: context.flags.sessionIsolation ?? 'tenant', + runId: context.flags.runId ?? `${provider}-${clientId}`, + clientId, + target: context.flags.target ?? 'mobile', + session: context.flags.session, + stateDir: context.stateDir, + ...readMetroProfileFields(context.flags), + ...resolved.profile, + }, + }); + }); return { ...profile, provider }; } @@ -123,10 +166,20 @@ export async function verifyResolvedConnectProvider( 'Remote connection profile loaded. Access is checked by the first remote command.', }; } - return await CONNECT_PROVIDER_ADAPTERS[resolved.provider].verify({ + const context = { flags: resolved.flags, env: process.env, - }); + }; + const adapter = (CONNECT_PROVIDER_ADAPTERS as Partial>)[ + resolved.provider + ]; + return adapter + ? await adapter.verify(context) + : await withPluginConnection( + resolved.provider, + context.env, + async (connection) => await connection.verify(context), + ); } async function verifyBrowserStack( diff --git a/src/cli/connection/provider-policy.ts b/src/cli/connection/provider-policy.ts index ebb86bc6d4..d7fd72b2ee 100644 --- a/src/cli/connection/provider-policy.ts +++ b/src/cli/connection/provider-policy.ts @@ -3,9 +3,18 @@ import { isCloudWebDriverProviderName, type CloudWebDriverKnownProviderName, } from '@agent-device/provider-webdriver/providers'; +import { pluginConnectionCapabilities, pluginConnectionNames } from '../../plugins/connection.ts'; export type DirectDeviceConnectProvider = CloudWebDriverKnownProviderName | 'limrun'; -export type ConnectProvider = 'cloud' | 'proxy' | DirectDeviceConnectProvider; +export const BUILTIN_CONNECT_PROVIDERS = [ + 'cloud', + 'proxy', + CLOUD_WEBDRIVER_PROVIDERS.browserStack, + CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, + 'limrun', +] as const; +export type BuiltinConnectProvider = (typeof BUILTIN_CONNECT_PROVIDERS)[number]; +export type ConnectProvider = BuiltinConnectProvider | (string & {}); export type ConnectionProviderCapabilities = { leaseKind: 'proxy' | 'direct-device-provider' | 'remote-provider'; @@ -18,7 +27,12 @@ export type ConnectionProviderCapabilities = { }; export function isConnectProviderName(value: string | undefined): value is ConnectProvider { - return value === 'cloud' || value === 'proxy' || isDirectDeviceConnectProvider(value); + return ( + value === 'cloud' || + value === 'proxy' || + isDirectDeviceConnectProvider(value) || + pluginConnectionCapabilities(value) !== undefined + ); } function isDirectDeviceConnectProvider( @@ -28,13 +42,7 @@ function isDirectDeviceConnectProvider( } export function connectProviderNamesForError(): string { - return [ - 'cloud', - 'proxy', - CLOUD_WEBDRIVER_PROVIDERS.browserStack, - CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, - 'limrun', - ].join(', '); + return [...BUILTIN_CONNECT_PROVIDERS, ...pluginConnectionNames()].join(', '); } export function connectionProviderCapabilities( @@ -42,6 +50,10 @@ export function connectionProviderCapabilities( ): ConnectionProviderCapabilities { const directDeviceProvider = isDirectDeviceConnectProvider(provider); const cloudWebDriver = isCloudWebDriverProviderName(provider); + if (!directDeviceProvider && provider !== 'cloud' && provider !== 'proxy') { + const plugin = pluginConnectionCapabilities(provider); + if (plugin) return plugin; + } return { leaseKind: provider === 'proxy' diff --git a/src/plugins/connection.ts b/src/plugins/connection.ts new file mode 100644 index 0000000000..734b14dc2d --- /dev/null +++ b/src/plugins/connection.ts @@ -0,0 +1,32 @@ +import type { CliFlags } from '@agent-device/contracts/command'; +import type { ProviderConnectionVerification } from '@agent-device/contracts/remote'; +import type { EnvMap } from '@agent-device/kernel/source-value'; +import type { RemoteConfigProfile } from '../remote/remote-config-schema.ts'; +import type { ConnectionProviderCapabilities } from '../cli/connection/provider-policy.ts'; +import { installedPlugins } from './store.ts'; + +export type PluginConnection = Readonly<{ + resolve(context: { flags: CliFlags; stateDir: string; cwd: string; env: EnvMap }): + | Promise<{ + profile: RemoteConfigProfile; + extraFlags?: Partial; + }> + | { profile: RemoteConfigProfile; extraFlags?: Partial }; + verify(context: { flags: CliFlags; env: EnvMap }): Promise; +}>; + +export function pluginConnectionCapabilities( + provider: string | undefined, + env: NodeJS.ProcessEnv = process.env, +): ConnectionProviderCapabilities | undefined { + return provider === undefined + ? undefined + : installedPlugins(env).find((plugin) => plugin.agentDevicePlugin.provider === provider) + ?.agentDevicePlugin.connection; +} + +export function pluginConnectionNames(env: NodeJS.ProcessEnv = process.env): string[] { + return installedPlugins(env) + .filter((plugin) => plugin.agentDevicePlugin.connection) + .map((plugin) => plugin.agentDevicePlugin.provider); +} diff --git a/src/plugins/host.ts b/src/plugins/host.ts new file mode 100644 index 0000000000..66ce4e70db --- /dev/null +++ b/src/plugins/host.ts @@ -0,0 +1,63 @@ +import { register } from 'node:module'; +import { pathToFileURL } from 'node:url'; +import { AppError } from '@agent-device/kernel/errors'; +import { execFailureDetails, runCmd } from '@agent-device/host-kit/command'; +import { findProjectRoot, readVersion } from '@agent-device/host-kit/version'; +import type { ProviderPluginHost } from '../sdk/plugins.ts'; + +let hostBound = false; + +export function bindPluginHost(): void { + if (hostBound) return; + const root = pathToFileURL(`${findProjectRoot()}/`).href; + const source = import.meta.url.startsWith(`${root}src/`); + register( + `data:text/javascript,${encodeURIComponent(` + let host; + export function initialize(data) { host = data; } + export async function resolve(specifier, context, nextResolve) { + if (specifier === 'agent-device' && host.source) { + return { url: new URL('src/sdk/index.ts', host.root).href, shortCircuit: true }; + } + if (specifier === 'agent-device' || specifier.startsWith('agent-device/')) { + return nextResolve(specifier, { ...context, parentURL: new URL('plugin-host.mjs', host.root).href }); + } + return nextResolve(specifier, context); + } + `)}`, + { parentURL: import.meta.url, data: { root, source } }, + ); + hostBound = true; +} + +export function createPluginHost( + env: NodeJS.ProcessEnv, + options: Record | undefined, +): ProviderPluginHost { + return Object.freeze({ + env: Object.freeze({ ...env }), + options: Object.freeze({ ...options }), + clientVersion: readVersion(), + createError: (code, message, details) => new AppError(code, message, details), + apple: Object.freeze({ + archiveDirectory: async ({ sourceDirectory, entryName, archivePath }) => { + const args = ['-qr', archivePath, entryName]; + const result = await runCmd('zip', args, { cwd: sourceDirectory, timeoutMs: 120_000 }); + if (result.exitCode !== 0) { + throw new AppError('COMMAND_FAILED', 'Failed to package iOS app for provider install', { + command: ['zip', ...args].join(' '), + ...execFailureDetails(result), + }); + } + }, + resolveAppAlias: async (app) => { + const { resolveIosAppAlias } = await import('@agent-device/platform-apple/app-resolution'); + return await resolveIosAppAlias(app); + }, + readBundleAppName: async (appPath) => { + const { readIosBundleInfo } = await import('@agent-device/platform-apple/install-artifact'); + return (await readIosBundleInfo(appPath)).appName; + }, + }), + } satisfies ProviderPluginHost); +} diff --git a/src/plugins/load.test.ts b/src/plugins/load.test.ts index 64afbc9f44..5fd08fde5a 100644 --- a/src/plugins/load.test.ts +++ b/src/plugins/load.test.ts @@ -2,7 +2,7 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { test } from 'vitest'; -import { loadProviderPlugins } from './load.ts'; +import { loadProviderPlugins, withPluginConnection } from './load.ts'; import { pluginHome, selectPlugin, registrationSource } from './plugin.fixtures.ts'; import { AppError } from '@agent-device/kernel/errors'; import type { ProviderPluginHost } from '../sdk/plugins.ts'; @@ -62,3 +62,44 @@ test('cleanup throwing synchronously preserves the factory failure', async () => await assert.rejects(loadProviderPlugins(env, []), /factory failed/); assert.ok(fs.existsSync(marker)); }); + +test('connection callbacks run from the installed plugin and always release runtimes', async () => { + const { home, env } = pluginHome(); + const marker = path.join(home, 'shutdown'); + const source = registrationSource('example', marker).replace( + 'platformModule:', + "connection: { resolve: () => ({ profile: { leaseProvider: 'example', platform: 'android' } }), verify: async () => { throw host.createError('COMMAND_FAILED', 'verification failed'); } }, platformModule:", + ); + selectPlugin(home, 'example', 'example', source); + const profile = await withPluginConnection( + 'example', + env, + async (connection) => await connection.resolve({ flags: {}, stateDir: home, cwd: home, env }), + ); + assert.equal(profile.profile.platform, 'android'); + assert.ok(fs.existsSync(marker)); + fs.unlinkSync(marker); + await assert.rejects( + withPluginConnection( + 'example', + env, + async (connection) => await connection.verify({ flags: {}, env }), + ), + { code: 'COMMAND_FAILED' }, + ); + assert.ok(fs.existsSync(marker)); +}); + +test('WebDriver plugins use the shared engine and refuse mismatched providers', async () => { + const { home, env } = pluginHome(); + const source = + "export default () => ({ webDriver: { provider: 'example', endpoint: 'http://127.0.0.1/', platform: 'android', deviceName: 'example' } });"; + selectPlugin(home, 'example', 'example', source); + const [registration] = await loadProviderPlugins(env, []); + assert.equal(registration!.runtime.provider, 'example'); + assert.equal(registration!.platformModule.owner.provider, 'example'); + await registration!.runtime.shutdown(); + const other = pluginHome(); + selectPlugin(other.home, 'example', 'wrong', source); + await assert.rejects(loadProviderPlugins(other.env, []), { code: 'INVALID_ARGS' }); +}); diff --git a/src/plugins/load.ts b/src/plugins/load.ts index 024b7a9573..0b91f944e5 100644 --- a/src/plugins/load.ts +++ b/src/plugins/load.ts @@ -5,10 +5,15 @@ import type { PlatformRuntimeProviderModule } from '@agent-device/contracts/plat import type { ProviderPluginHost } from '../sdk/plugins.ts'; import { installedPlugins } from './store.ts'; import { resolvePluginEntry, assertUniquePluginProviders } from './manifest.ts'; +import { bindPluginHost, createPluginHost } from './host.ts'; +import type { PluginConnection } from './connection.ts'; +import { BUILTIN_CONNECT_PROVIDERS } from '../cli/connection/provider-policy.ts'; +import type { WebDriverPluginOptions } from '../sdk/plugin-webdriver.ts'; type ProviderPluginRegistration = Readonly<{ runtime: ProviderDeviceRuntime; platformModule: PlatformRuntimeProviderModule; + connection?: PluginConnection; }>; export async function loadProviderPlugins( @@ -20,22 +25,44 @@ export async function loadProviderPlugins( const registrations: ProviderPluginRegistration[] = []; try { for (const plugin of plugins) { + bindPluginHost(); const module = await import( pathToFileURL(resolvePluginEntry(plugin.directory, plugin.agentDevicePlugin.entry)).href ); if (typeof module.default !== 'function') throw new AppError('INVALID_ARGS', `Plugin must export a default factory: ${plugin.name}`); - const registration = await ( + const host = createPluginHost(env, plugin.selection.options); + const result = await ( module.default as ( host: ProviderPluginHost, - ) => ProviderPluginRegistration | Promise - )( - Object.freeze({ - env: Object.freeze({ ...env }), - options: Object.freeze({ ...plugin.selection.options }), - createError: (code, message, details) => new AppError(code, message, details), - }), - ); + ) => + | ProviderPluginRegistration + | { webDriver: WebDriverPluginOptions; connection?: PluginConnection } + | Promise< + | ProviderPluginRegistration + | { webDriver: WebDriverPluginOptions; connection?: PluginConnection } + > + )(host); + let registration: ProviderPluginRegistration; + if (result && 'webDriver' in result) { + if (result.webDriver?.provider !== plugin.agentDevicePlugin.provider) { + throw new AppError( + 'INVALID_ARGS', + `WebDriver plugin provider does not match its declaration: ${plugin.name}`, + ); + } + const { createCloudWebDriverRuntime } = + await import('@agent-device/provider-webdriver/plugin'); + const runtime = createCloudWebDriverRuntime({ + ...result.webDriver, + clientVersion: host.clientVersion, + }); + registration = { + runtime, + platformModule: runtime.platformRuntimeModule, + connection: result.connection, + }; + } else registration = result; if (!registration?.runtime || typeof registration.runtime.shutdown !== 'function') { throw new AppError('INVALID_ARGS', `Plugin must return a provider runtime: ${plugin.name}`); } @@ -51,7 +78,10 @@ export async function loadProviderPlugins( registration.platformModule.owner.provider !== registration.runtime.provider || typeof registration.platformModule.owner.instance !== 'string' || registration.platformModule.owner.instance.trim().length === 0 || - typeof registration.platformModule.loadRuntime !== 'function' + typeof registration.platformModule.loadRuntime !== 'function' || + (plugin.agentDevicePlugin.connection && + (typeof registration.connection?.resolve !== 'function' || + typeof registration.connection?.verify !== 'function')) ) { throw new AppError( 'INVALID_ARGS', @@ -65,3 +95,21 @@ export async function loadProviderPlugins( throw error; } } + +export async function withPluginConnection( + provider: string, + env: NodeJS.ProcessEnv, + use: (connection: PluginConnection) => Promise, +): Promise { + const registrations = await loadProviderPlugins(env, BUILTIN_CONNECT_PROVIDERS); + try { + const connection = registrations.find( + (entry) => entry.runtime.provider === provider, + )?.connection; + if (!connection) + throw new AppError('INVALID_ARGS', `Plugin does not register connect: ${provider}`); + return await use(connection); + } finally { + await Promise.allSettled(registrations.map(async ({ runtime }) => await runtime.shutdown())); + } +} diff --git a/src/plugins/manifest.ts b/src/plugins/manifest.ts index 44b6d2a1c9..c30f6b113b 100644 --- a/src/plugins/manifest.ts +++ b/src/plugins/manifest.ts @@ -1,12 +1,18 @@ import fs from 'node:fs'; import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; +import type { ConnectionProviderCapabilities } from '../cli/connection/provider-policy.ts'; const PROVIDER_PLUGIN_API_VERSION = 1; type PluginManifest = { name: string; version: string; - agentDevicePlugin: { apiVersion: number; provider: string; entry: string }; + agentDevicePlugin: { + apiVersion: number; + provider: string; + entry: string; + connection?: ConnectionProviderCapabilities; + }; }; export function assertUniquePluginProviders( @@ -53,6 +59,28 @@ export function readPluginManifest(directory: string): PluginManifest { }); } resolvePluginEntry(directory, declaration.entry); + if (declaration.connection !== undefined) { + const policy = declaration.connection; + if ( + !policy || + typeof policy !== 'object' || + policy.leaseKind !== 'direct-device-provider' || + [ + 'requiresAppAttachment', + 'requiresRemoteDaemon', + 'supportsArtifacts', + 'supportsDeferredAppSelection', + 'supportsDirectPortReverse', + 'usesCloudWebDriverLease', + ].some((key) => typeof policy[key as keyof ConnectionProviderCapabilities] !== 'boolean') || + policy.requiresRemoteDaemon + ) { + throw new AppError( + 'INVALID_ARGS', + 'Plugin connection must declare local provider capabilities', + ); + } + } return manifest as PluginManifest; } diff --git a/src/plugins/store.test.ts b/src/plugins/store.test.ts index d10648a187..32fcef3d2c 100644 --- a/src/plugins/store.test.ts +++ b/src/plugins/store.test.ts @@ -20,6 +20,7 @@ function installFixture(name = packageName, apiVersion = 1, provider = 'example' assert.equal(argv[argv.indexOf('--prefix') + 1], options!.cwd); assert.ok(argv.includes('--global=false')); assert.ok(argv.includes('--ignore-scripts')); + assert.ok(argv.includes('--omit=peer')); assert.ok(argv.includes('--workspaces=false')); assert.ok(argv.includes('--package-lock=true')); writePlugin(options!.cwd!, name, apiVersion, provider); diff --git a/src/plugins/store.ts b/src/plugins/store.ts index 4435a624cb..000653787c 100644 --- a/src/plugins/store.ts +++ b/src/plugins/store.ts @@ -112,6 +112,7 @@ async function stagePlugin( [ 'install', '--ignore-scripts', + '--omit=peer', '--no-audit', '--no-fund', '--global=false', diff --git a/src/sdk/plugin-webdriver.ts b/src/sdk/plugin-webdriver.ts new file mode 100644 index 0000000000..1f43c9f016 --- /dev/null +++ b/src/sdk/plugin-webdriver.ts @@ -0,0 +1,3 @@ +import type { CloudWebDriverRuntimeOptions } from '@agent-device/provider-webdriver/plugin'; + +export type WebDriverPluginOptions = Omit; diff --git a/src/sdk/plugins.ts b/src/sdk/plugins.ts index 188481315e..753ce229b0 100644 --- a/src/sdk/plugins.ts +++ b/src/sdk/plugins.ts @@ -3,5 +3,15 @@ import type { AppError, AppErrorCode, AppErrorDetails } from '@agent-device/kern export type ProviderPluginHost = Readonly<{ env: Readonly>; options: Readonly>; + clientVersion: string; + apple: Readonly<{ + archiveDirectory(options: { + sourceDirectory: string; + entryName: string; + archivePath: string; + }): Promise; + resolveAppAlias(app: string): Promise; + readBundleAppName(appPath: string): Promise; + }>; createError(code: AppErrorCode, message: string, details?: AppErrorDetails): AppError; }>; diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 19aa0b8749..3ff7ec5e27 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -97,6 +97,8 @@ Supported public entry points for Node consumers: - `runtime.getDeviceSession(device)` - types: `LimrunRuntimeOptions`, `LimrunDeviceSession`, `LimrunAndroidDeviceSession`, `LimrunIosDeviceSession`, `LimrunIosCommandExecution` +- `agent-device/plugins/webdriver` + - experimental `WebDriverPluginOptions` for providers using the shared engine. - `agent-device/plugins` - experimental factory context: `ProviderPluginHost`; see [provider plugins](./plugins.md). - `agent-device/ai-sdk` diff --git a/website/docs/docs/plugins.md b/website/docs/docs/plugins.md index a668163fa2..24618c8451 100644 --- a/website/docs/docs/plugins.md +++ b/website/docs/docs/plugins.md @@ -27,8 +27,29 @@ The interface is experimental. Publish this declaration in your package manifest {"agentDevicePlugin": {"apiVersion": 1, "provider": "example", "entry": "./dist/plugin.mjs"}} ``` -Use `agent-device` as a development dependency. The default factory accepts `ProviderPluginHost` from `agent-device/plugins`: `env`, package-specific `options`, and `createError` for host-recognized errors. Return `{ runtime, platformModule }` implementing the [provider runtime](https://github.com/callstack/agent-device/blob/main/packages/contracts/src/provider-device-runtime.ts) and [platform module](https://github.com/callstack/agent-device/blob/main/packages/contracts/src/platform-runtime-operations.ts) contracts. Both provider IDs must match the manifest; the module owner must declare `kind: 'provider-runtime'` and a nonempty `instance`. Core checks required runtime methods and owner metadata at startup; operation contracts are checked when used. Provider packages own implementation types; the SDK exposes only factory context. Set options through `plugins[""].options` in user config; leave `installation` unchanged. +Use `agent-device` as a development dependency. The default factory accepts `ProviderPluginHost` from `agent-device/plugins`: `env`, package-specific `options`, `clientVersion`, Apple app packaging and resolution helpers under `apple`, and `createError` for host-recognized errors. Return `{ runtime, platformModule }` implementing the [provider runtime](https://github.com/callstack/agent-device/blob/main/packages/contracts/src/provider-device-runtime.ts) and [platform module](https://github.com/callstack/agent-device/blob/main/packages/contracts/src/platform-runtime-operations.ts) contracts. Both provider IDs must match the manifest; the module owner must declare `kind: 'provider-runtime'` and a nonempty `instance`. Core checks required runtime methods and owner metadata at startup; operation contracts are checked when used. Provider packages own implementation types; the SDK exposes only factory context. Set options through `plugins[""].options` in user config; leave `installation` unchanged. Keep initialization prompt and free of network I/O or device allocation; a stalled factory blocks startup. Load platform mechanics through `platformModule.loadRuntime` and perform remote work in request-bound operations. A failing factory cleans up its own resources; core shuts down previously returned runtimes if another plugin fails. Incompatible contract changes require a new API version. Plugins cannot replace bundled providers or register arbitrary commands. Installing a package does not add `connect `; provider-specific connect adapters need separate support. Limrun, BrowserStack, and AWS Device Farm remain bundled. + + +For an Appium or WebDriver service, return `{ webDriver: options }` instead of building an engine. `WebDriverPluginOptions` is available through the type-only `agent-device/plugins/webdriver` import. Core supplies the client version and creates the shared runtime. Provider callbacks prepare sessions, upload apps, and retrieve artifacts. + +To support `agent-device connect example`, declare `agentDevicePlugin.connection` in the package manifest: + +```json +{ + "leaseKind": "direct-device-provider", + "requiresAppAttachment": false, + "requiresRemoteDaemon": false, + "supportsArtifacts": false, + "supportsDeferredAppSelection": true, + "supportsDirectPortReverse": false, + "usesCloudWebDriverLease": false +} +``` + +Return `connection` alongside the runtime or WebDriver options. Its `resolve({ flags, env, cwd, stateDir })` callback validates provider flags and returns `{ profile, extraFlags? }`; `profile.leaseProvider` must match the manifest. Core supplies connection identity, session defaults, Metro settings, and persists the profile. Its async `verify({ flags, env })` callback returns the provider verification result. These callbacks run without allocating a device and their temporary runtimes are shut down afterwards. + +Bundle the plugin implementation and ship ready-to-run ESM: installation disables lifecycle scripts and omits peer dependencies. Runtime imports of public `agent-device` exports resolve against the running host, so plugins can use its `AppError` without installing a second core package. Import types with `import type` to keep them out of the runtime dependency graph. From f3e6d0474b4f97982c4f6041f451259e234fdbfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 16:54:29 +0200 Subject: [PATCH 17/57] chore(gates): expose the optional WebDriver plugin configuration --- .fallowrc.json | 1 + package.json | 4 ++++ packages/provider-webdriver/package.json | 4 ++++ tsdown.config.ts | 1 + 4 files changed, 10 insertions(+) diff --git a/.fallowrc.json b/.fallowrc.json index 635559ff05..bd9b3ad99a 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -10,6 +10,7 @@ "src/sdk/remote-config.ts", "src/sdk/install-source.ts", "src/sdk/plugins.ts", + "src/sdk/plugin-webdriver.ts", "src/sdk/android-adb.ts", "src/sdk/contracts.ts", "src/sdk/selectors.ts", diff --git a/package.json b/package.json index 396323377e..78e463ea72 100644 --- a/package.json +++ b/package.json @@ -73,6 +73,10 @@ "./plugins": { "types": "./dist/src/plugins.d.ts", "import": "./dist/src/plugins.js" + }, + "./plugins/webdriver": { + "types": "./dist/src/plugins/webdriver.d.ts", + "import": "./dist/src/plugins/webdriver.js" } }, "engines": { diff --git a/packages/provider-webdriver/package.json b/packages/provider-webdriver/package.json index d4d56b8d0e..1d445ccdf6 100644 --- a/packages/provider-webdriver/package.json +++ b/packages/provider-webdriver/package.json @@ -19,6 +19,10 @@ "./providers": { "types": "./src/providers.ts", "default": "./src/providers.ts" + }, + "./plugin": { + "types": "./src/plugin.ts", + "default": "./src/plugin.ts" } } } diff --git a/tsdown.config.ts b/tsdown.config.ts index 7131e46e79..163bb4c75d 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -102,6 +102,7 @@ export default defineConfig({ 'android-adb': 'src/sdk/android-adb.ts', limrun: 'src/sdk/limrun.ts', plugins: 'src/sdk/plugins.ts', + 'plugins/webdriver': 'src/sdk/plugin-webdriver.ts', contracts: 'src/sdk/contracts.ts', selectors: 'src/sdk/selectors.ts', finders: 'src/sdk/finders.ts', From 5f362bc6822ddb9a8f0ad4e1b21a2733b823b2a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 17:41:16 +0200 Subject: [PATCH 18/57] feat: load TestMu from its separately installed provider plugin --- packages/contracts/src/facades/remote.ts | 1 + packages/contracts/src/provider-connection.ts | 10 + packages/kernel/src/app-error.ts | 14 - packages/kernel/src/errors.test.ts | 17 + packages/kernel/src/errors.ts | 27 +- packages/provider-testmu/src/connection.ts | 78 ++++ packages/provider-testmu/src/plugin.ts | 196 +++++++++ packages/provider-testmu/src/providers.ts | 3 + .../testmu-connection-verification.test.ts | 408 ++++++++++++++++++ .../src/testmu-connection-verification.ts | 27 +- .../src/testmu-device-features.test.ts | 0 .../src/testmu-device-features.ts | 2 +- .../src/testmu.test.ts | 2 +- .../src/testmu.ts | 12 +- .../provider-testmu/src/tmp-dir.fixtures.ts | 21 + .../provider-testmu/test/package-smoke.mjs | 33 ++ packages/provider-testmu/tsconfig.json | 12 + packages/provider-testmu/tsdown.config.ts | 19 + .../src/connection-verification.test.ts | 395 ----------------- .../src/connection-verification.ts | 22 +- packages/provider-webdriver/src/plugin.ts | 8 +- .../src/provider-definitions.ts | 166 ------- .../src/provider-profile-fields.test.ts | 5 - .../src/runtime-deployment.test.ts | 46 -- scripts/check-provider-plugin.mjs | 130 ++++++ src/__tests__/cloud-connect-profile.test.ts | 47 +- src/__tests__/cloud-connect-testmu.test.ts | 55 ++- src/__tests__/testmu-upload.test.ts | 83 ++++ src/cli/connection/cloud-webdriver-profile.ts | 29 +- .../connection/connect-provider-adapters.ts | 2 +- src/cli/connection/provider-policy.ts | 27 +- src/commands/schema/cli-help-topics.test.ts | 1 + src/commands/schema/cli-help.ts | 1 + src/commands/schema/command-overrides.ts | 2 +- src/plugins/connection.ts | 2 +- src/plugins/host.ts | 29 +- src/plugins/load.test.ts | 12 +- src/plugins/load.ts | 21 +- src/plugins/manifest.ts | 14 +- src/plugins/store.test.ts | 1 - src/plugins/store.ts | 1 - .../cloud-webdriver-provider-adapters.test.ts | 32 +- website/docs/docs/plugins.md | 2 +- website/docs/docs/testmu.md | 10 + 44 files changed, 1219 insertions(+), 806 deletions(-) delete mode 100644 packages/kernel/src/app-error.ts create mode 100644 packages/provider-testmu/src/connection.ts create mode 100644 packages/provider-testmu/src/plugin.ts create mode 100644 packages/provider-testmu/src/providers.ts create mode 100644 packages/provider-testmu/src/testmu-connection-verification.test.ts rename packages/{provider-webdriver => provider-testmu}/src/testmu-connection-verification.ts (93%) rename packages/{provider-webdriver => provider-testmu}/src/testmu-device-features.test.ts (100%) rename packages/{provider-webdriver => provider-testmu}/src/testmu-device-features.ts (97%) rename packages/{provider-webdriver => provider-testmu}/src/testmu.test.ts (99%) rename packages/{provider-webdriver => provider-testmu}/src/testmu.ts (97%) create mode 100644 packages/provider-testmu/src/tmp-dir.fixtures.ts create mode 100644 packages/provider-testmu/test/package-smoke.mjs create mode 100644 packages/provider-testmu/tsconfig.json create mode 100644 packages/provider-testmu/tsdown.config.ts create mode 100644 scripts/check-provider-plugin.mjs create mode 100644 src/__tests__/testmu-upload.test.ts diff --git a/packages/contracts/src/facades/remote.ts b/packages/contracts/src/facades/remote.ts index bfaecc5086..c4692bdc7e 100644 --- a/packages/contracts/src/facades/remote.ts +++ b/packages/contracts/src/facades/remote.ts @@ -11,6 +11,7 @@ export type { ResolvedMetroKind, } from '../metro.ts'; export type { + ConnectionProviderCapabilities, ProviderConnectionResource, ProviderConnectionVerification, } from '../provider-connection.ts'; diff --git a/packages/contracts/src/provider-connection.ts b/packages/contracts/src/provider-connection.ts index 3a3fa1125d..a0ff737e9e 100644 --- a/packages/contracts/src/provider-connection.ts +++ b/packages/contracts/src/provider-connection.ts @@ -17,3 +17,13 @@ export type ProviderConnectionVerification = { device: ProviderConnectionResource; app: ProviderConnectionResource; }; + +export type ConnectionProviderCapabilities = { + leaseKind: 'proxy' | 'direct-device-provider' | 'remote-provider'; + requiresAppAttachment: boolean; + requiresRemoteDaemon: boolean; + supportsArtifacts: boolean; + supportsDeferredAppSelection: boolean; + supportsDirectPortReverse: boolean; + usesCloudWebDriverLease: boolean; +}; diff --git a/packages/kernel/src/app-error.ts b/packages/kernel/src/app-error.ts deleted file mode 100644 index d748efb9b2..0000000000 --- a/packages/kernel/src/app-error.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { AppErrorCode, AppErrorDetails } from './errors.ts'; - -export class AppError extends Error { - code: AppErrorCode; - details?: AppErrorDetails; - cause?: unknown; - - constructor(code: AppErrorCode, message: string, details?: AppErrorDetails, cause?: unknown) { - super(message); - this.code = code; - this.details = details; - this.cause = cause; - } -} diff --git a/packages/kernel/src/errors.test.ts b/packages/kernel/src/errors.test.ts index 3bf8104af2..451a7566e1 100644 --- a/packages/kernel/src/errors.test.ts +++ b/packages/kernel/src/errors.test.ts @@ -117,3 +117,20 @@ test('discloseDispatchAfterSteps keeps no only while no step of the series was d const plain = new Error('socket closed'); assert.equal(discloseDispatchAfterSteps(plain, 4), plain); }); + +test('bundled plugin errors preserve codes and details without changing subclass checks', async () => { + const copyPath = './errors.ts?plugin-copy'; + const { AppError: PluginError } = await import(copyPath); + assert.notEqual(PluginError, AppError); + const foreign = new PluginError('INVALID_ARGS', 'bad plugin profile', { provider: 'example' }); + assert.ok(foreign instanceof AppError); + const normalized = normalizeError(foreign); + assert.equal(normalized.code, 'INVALID_ARGS'); + assert.equal(normalized.message, 'bad plugin profile'); + assert.deepEqual(normalized.details, { provider: 'example' }); + const ordinary = Object.assign(new Error('bad plugin profile'), { code: 'INVALID_ARGS' }); + assert.equal(ordinary instanceof AppError, false); + class SpecificError extends AppError {} + assert.ok(new SpecificError('COMMAND_FAILED', 'specific') instanceof SpecificError); + assert.equal(foreign instanceof SpecificError, false); +}); diff --git a/packages/kernel/src/errors.ts b/packages/kernel/src/errors.ts index 42dd4f1a85..467b630d94 100644 --- a/packages/kernel/src/errors.ts +++ b/packages/kernel/src/errors.ts @@ -1,5 +1,3 @@ -import { AppError } from './app-error.ts'; -export { AppError } from './app-error.ts'; import { redactDiagnosticData, sanitizeErrorCause as normalizeErrorCause } from './redaction.ts'; /** @@ -38,6 +36,31 @@ export type KnownAppErrorCode = (typeof KNOWN_APP_ERROR_CODES)[number]; // include a default branch. export type AppErrorCode = KnownAppErrorCode | (string & {}); +const APP_ERROR_BRAND = Symbol.for('agent-device.AppError'); + +export class AppError extends Error { + static [Symbol.hasInstance](value: unknown): boolean { + if (this !== AppError) return Function.prototype[Symbol.hasInstance].call(this, value); + return ( + typeof value === 'object' && + value !== null && + (value as Record)[APP_ERROR_BRAND] === true + ); + } + + code: AppErrorCode; + details?: AppErrorDetails; + cause?: unknown; + + constructor(code: AppErrorCode, message: string, details?: AppErrorDetails, cause?: unknown) { + super(message); + Object.defineProperty(this, APP_ERROR_BRAND, { value: true }); + this.code = code; + this.details = details; + this.cause = cause; + } +} + export function toAppErrorCode( code: string | undefined, fallback: AppErrorCode = 'COMMAND_FAILED', diff --git a/packages/provider-testmu/src/connection.ts b/packages/provider-testmu/src/connection.ts new file mode 100644 index 0000000000..10c09648d7 --- /dev/null +++ b/packages/provider-testmu/src/connection.ts @@ -0,0 +1,78 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import type { ProviderPluginHost } from 'agent-device/plugins'; +import type { CliFlags } from '@agent-device/contracts/command'; +import { + rejectRefusedProviderProfileFields, + type ProviderProfileFieldDeclaration, +} from '@agent-device/contracts/provider-profile-fields'; +import { isTestMuAppReference } from './providers.ts'; +import { verifyTestMuConnection } from './testmu-connection-verification.ts'; +import { readTestMuDeviceFeatureFields, readTestMuDeviceType } from './testmu-device-features.ts'; + +export function createTestMuConnection( + host: ProviderPluginHost, + fields: ProviderProfileFieldDeclaration, +) { + const required = (value: string | undefined, name: string) => { + if (value?.trim()) return value; + throw host.createError('INVALID_ARGS', `connect testmu requires ${name}.`); + }; + return { + resolve: ({ flags, cwd }: { flags: CliFlags; cwd: string }) => { + rejectRefusedProviderProfileFields(flags, fields); + required(host.env.LT_USERNAME, 'LT_USERNAME'); + required(host.env.LT_ACCESS_KEY, 'LT_ACCESS_KEY'); + if (flags.platform !== 'android' && flags.platform !== 'ios') + throw host.createError('INVALID_ARGS', 'connect testmu requires --platform ios|android.'); + let app = required(flags.providerApp, '--provider-app '); + if (app.slice(0, 5).toLowerCase() === 'lt://') { + app = `lt://${app.slice(5)}`; + if (!isTestMuAppReference(app)) + throw host.createError( + 'INVALID_ARGS', + 'connect testmu requires a valid lt:// app reference.', + ); + } else if (!/^https?:\/\//i.test(app)) { + app = path.resolve(cwd, app); + if (!fs.statSync(app, { throwIfNoEntry: false })?.isFile()) + throw host.createError('INVALID_ARGS', `TestMu AI app file not found: ${app}`); + } + return { + profile: { + leaseProvider: 'testmu', + leaseBackend: + flags.leaseBackend ?? (flags.platform === 'ios' ? 'ios-instance' : 'android-instance'), + platform: flags.platform, + device: required(flags.device, '--device '), + providerOsVersion: required(flags.providerOsVersion, '--provider-os-version '), + providerApp: app, + providerDeviceType: readTestMuDeviceType(flags), + providerProject: flags.providerProject, + providerBuild: flags.providerBuild, + providerSessionName: flags.providerSessionName, + ...readTestMuDeviceFeatureFields(flags), + } as const, + extraFlags: { providerApp: app }, + }; + }, + verify: async ({ flags }: { flags: CliFlags }) => { + if (flags.platform !== 'android' && flags.platform !== 'ios') + throw host.createError('INVALID_ARGS', 'TestMu profile missed platform.'); + return await verifyTestMuConnection( + { + provider: 'testmu', + username: required(host.env.LT_USERNAME, 'LT_USERNAME'), + accessKey: required(host.env.LT_ACCESS_KEY, 'LT_ACCESS_KEY'), + platform: flags.platform, + deviceName: required(flags.device, '--device'), + osVersion: required(flags.providerOsVersion, '--provider-os-version'), + app: required(flags.providerApp, '--provider-app'), + deviceType: readTestMuDeviceType(flags), + apiEndpoint: host.env.TESTMU_API_ENDPOINT, + }, + host.clientVersion, + ); + }, + }; +} diff --git a/packages/provider-testmu/src/plugin.ts b/packages/provider-testmu/src/plugin.ts new file mode 100644 index 0000000000..9dc868046d --- /dev/null +++ b/packages/provider-testmu/src/plugin.ts @@ -0,0 +1,196 @@ +import type { ProviderPluginHost } from 'agent-device/plugins'; +import type { WebDriverPluginOptions } from 'agent-device/plugins/webdriver'; +import type { ProviderProfileFieldDeclaration } from '@agent-device/contracts/provider-profile-fields'; +import type { ProviderDeviceType } from '@agent-device/contracts/remote'; +import type { LeaseLifecycleContext } from '@agent-device/contracts/device'; +import { AppError } from '@agent-device/kernel/errors'; +import { buildCloudWebDriverBaseCapabilities } from '@agent-device/provider-webdriver/plugin'; +import { createTestMuConnection } from './connection.ts'; +const TESTMU_WEBDRIVER_ENDPOINT = 'https://mobile-hub.lambdatest.com/wd/hub/'; +const TESTMU_CAPABILITY_OVERRIDES = { + install: { + support: 'partial', + note: 'Local app artifacts are uploaded to TestMu AI as real- or virtual-device apps (lt://), then installed with Appium.', + }, + portReverse: { + support: 'unsupported', + note: 'Use the TestMu AI tunnel for network access to local hosts; agent-device port reverse is not available.', + }, + artifacts: { + support: 'supported', + note: 'TestMu AI session details expose provider-hosted video, Appium logs, device logs, network logs, and dashboard links.', + }, +} as const; + +const loadTestMu = async () => await import('./testmu.ts'); +const loadTestMuDeviceFeatures = async () => await import('./testmu-device-features.ts'); + +const TESTMU_PROFILE_FIELDS: ProviderProfileFieldDeclaration = { + provider: 'testmu', + label: 'TestMu AI', + fields: { + providerApp: 'consumed', + providerOsVersion: 'consumed', + providerDeviceType: 'consumed', + providerProject: 'consumed', + providerBuild: 'consumed', + providerSessionName: 'consumed', + providerDeviceOrientation: 'consumed', + providerGeoLocation: 'consumed', + providerTimezone: 'consumed', + providerAppiumVersion: 'consumed', + providerLanguage: 'consumed', + providerLocale: 'consumed', + providerNetworkProfile: 'refused', + providerCustomNetwork: 'refused', + providerNoResignApp: 'refused', + awsProjectArn: 'refused', + awsDeviceArn: 'refused', + awsAppArn: 'refused', + awsRegion: 'refused', + awsInteractionMode: 'refused', + }, +}; + +export default function testMuPlugin(host: ProviderPluginHost) { + const env = host.env; + async function listTestMuArtifactsFromEnv( + provider: string, + providerSessionId: string | undefined, + env: ProviderPluginHost['env'], + ) { + const { listTestMuCloudArtifacts } = await loadTestMu(); + return await listTestMuCloudArtifacts(provider, providerSessionId, { + clientVersion: host.clientVersion, + ...requireTestMuCredentials(env, 'TestMu AI artifact lookup'), + endpoint: env.TESTMU_API_ENDPOINT, + }); + } + const webDriver: WebDriverPluginOptions = { + provider: 'testmu', + profileFields: TESTMU_PROFILE_FIELDS, + platform: 'android', + deviceName: 'TestMu AI device', + endpoint: env.TESTMU_WEBDRIVER_ENDPOINT ?? TESTMU_WEBDRIVER_ENDPOINT, + capabilityOverrides: TESTMU_CAPABILITY_OVERRIDES, + listArtifacts: async ({ provider, providerSessionId }) => + await listTestMuArtifactsFromEnv(provider, providerSessionId, env), + prepareSession: async ({ req, lease, base }) => { + const request = requireRequest(req, 'TestMu AI'); + const { buildTestMuCapabilities, createTestMuUploadApp, resolveTestMuAppReference } = + await loadTestMu(); + const { + buildTestMuDeviceFeatureCapabilities, + readTestMuDeviceFeatureFields, + readTestMuDeviceType, + } = await loadTestMuDeviceFeatures(); + const deviceType = readTestMuDeviceType(request.flags); + const uploadEndpoint = testMuAppUploadEndpoint(env, deviceType); + const credentials = requireTestMuCredentials(env, 'TestMu AI'); + const platform = requireRequestPlatform(request, 'TestMu AI'); + const deviceName = requireFlag(request, 'device', 'TestMu AI requires --device .'); + const osVersion = requireFlag( + request, + 'providerOsVersion', + 'TestMu AI requires --provider-os-version .', + ); + const upload = { + clientVersion: host.clientVersion, + + ...credentials, + deviceType, + endpoint: uploadEndpoint, + }; + const app = await resolveTestMuAppReference( + requireFlag( + request, + 'providerApp', + 'TestMu AI requires --provider-app .', + ), + { ...upload, cwd: request.cwd, signal: request.signal }, + ); + return { + ...base, + platform, + deviceName, + auth: credentials, + uploadApp: createTestMuUploadApp(upload), + webdriverCapabilities: buildTestMuCapabilities({ + platform, + deviceType, + deviceName, + osVersion, + app, + projectName: readFlag(request, 'providerProject'), + buildName: readFlag(request, 'providerBuild') ?? lease.runId, + sessionName: readFlag(request, 'providerSessionName') ?? lease.leaseId, + deviceFeatures: buildTestMuDeviceFeatureCapabilities( + readTestMuDeviceFeatureFields(request.flags), + ), + configured: buildCloudWebDriverBaseCapabilities(platform, deviceName), + }), + }; + }, + }; + return { webDriver, connection: createTestMuConnection(host, TESTMU_PROFILE_FIELDS) }; +} +function requireTestMuCredentials( + env: ProviderPluginHost['env'], + providerLabel: string, +): { username: string; accessKey: string } { + return { + username: requireEnv(env, 'LT_USERNAME', providerLabel), + accessKey: requireEnv(env, 'LT_ACCESS_KEY', providerLabel), + }; +} + +/** Each pool has its own upload API, so each has its own override. */ +function testMuAppUploadEndpoint( + env: ProviderPluginHost['env'], + deviceType: ProviderDeviceType, +): string | undefined { + return deviceType === 'real' + ? env.TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT + : env.TESTMU_APP_UPLOAD_ENDPOINT; +} + +function requireRequest( + req: LeaseLifecycleContext | undefined, + providerLabel: string, +): LeaseLifecycleContext { + if (req) return req; + throw new AppError( + 'INVALID_ARGS', + `${providerLabel} lease allocation requires provider profile flags on the request.`, + ); +} + +function requireRequestPlatform( + req: LeaseLifecycleContext, + providerLabel: string, +): 'android' | 'ios' { + const platform = req.flags?.platform; + if (platform === 'android' || platform === 'ios') return platform; + throw new AppError('INVALID_ARGS', `${providerLabel} requires --platform ios|android.`); +} + +function requireFlag(req: LeaseLifecycleContext, key: string, message: string): string { + const value = readFlag(req, key); + if (value) return value; + throw new AppError('INVALID_ARGS', message); +} + +function readFlag(req: LeaseLifecycleContext, key: string): string | undefined { + const value = req.flags?.[key]; + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +function requireEnv( + env: ProviderPluginHost['env'], + key: keyof ProviderPluginHost['env'], + providerLabel: string, +): string { + const value = env[key]; + if (value) return value; + throw new AppError('INVALID_ARGS', `${providerLabel} requires ${key} in the environment.`); +} diff --git a/packages/provider-testmu/src/providers.ts b/packages/provider-testmu/src/providers.ts new file mode 100644 index 0000000000..72b2314564 --- /dev/null +++ b/packages/provider-testmu/src/providers.ts @@ -0,0 +1,3 @@ +export function isTestMuAppReference(value: string): boolean { + return /^lt:\/\/[\w.-]+$/.test(value); +} diff --git a/packages/provider-testmu/src/testmu-connection-verification.test.ts b/packages/provider-testmu/src/testmu-connection-verification.test.ts new file mode 100644 index 0000000000..e82ac69cbd --- /dev/null +++ b/packages/provider-testmu/src/testmu-connection-verification.test.ts @@ -0,0 +1,408 @@ +import assert from 'node:assert/strict'; +import { afterEach, test, vi } from 'vitest'; +import { verifyTestMuConnection } from './testmu-connection-verification.ts'; +import type { TestMuOptions } from './testmu-connection-verification.ts'; +afterEach(() => vi.unstubAllGlobals()); +function createProvider() { + return { + verifyConnection: async (options: TestMuOptions) => + await verifyTestMuConnection(options, '1.2.3'), + }; +} +function jsonResponse(value: unknown, status = 200) { + return new Response(JSON.stringify(value), { status }); +} +const testMuOptions = { + provider: 'testmu' as const, + username: 'lt-user', + accessKey: 'lt-key', + platform: 'android' as const, + deviceName: 'Pixel 8', + osVersion: '14', + app: 'lt://APP1', + devicesEndpoint: 'https://testmu.test/capability/generator?isVirtualDevice=true', + appsEndpoint: 'https://testmu.test/app/data', +}; + +const testMuCatalog = { + app: { + devices: { + android: { + brands: { + Google: [ + { name: 'Pixel 8', osVersion: ['14', '15'] }, + { name: 'Pixel 4a', osVersion: ['13'] }, + ], + }, + }, + ios: { brands: { Apple: [{ name: 'iPhone 16', osVersion: ['18.0'] }] } }, + }, + }, +}; + +test('TestMu verifies the virtual device and uploaded app without creating a session', async () => { + const fetchMock = vi.fn(async (input, init) => { + const headers = (init?.headers ?? {}) as Record; + if (String(input).includes('capability/generator')) { + assert.equal(headers.Authorization, undefined); + return jsonResponse(testMuCatalog); + } + assert.match(String(headers.Authorization), /^Basic /); + return jsonResponse({ + data: [{ app_id: 'APP1', name: 'sample.apk', version: '1.2.3', type: 'android' }], + metaData: { total: 1 }, + }); + }); + vi.stubGlobal('fetch', fetchMock); + + const result = await createProvider().verifyConnection(testMuOptions); + + assert.equal(result.provider, 'testmu'); + assert.equal(result.service, 'TestMu AI'); + assert.deepEqual(result.device, { + status: 'verified', + name: 'Pixel 8', + platform: 'android', + osVersion: '14', + }); + assert.deepEqual(result.app, { + status: 'verified', + name: 'sample.apk', + reference: 'lt://APP1', + version: '1.2.3', + }); + assert.deepEqual( + fetchMock.mock.calls.map(([input]) => String(input)), + [ + 'https://testmu.test/capability/generator?isVirtualDevice=true', + 'https://testmu.test/app/data?type=emulator&level=user', + ], + ); +}); + +test('TestMu checks the catalog of the configured API endpoint', async () => { + const fetchMock = vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(testMuCatalog) + : jsonResponse({ data: [{ app_id: 'APP1' }] }), + ); + vi.stubGlobal('fetch', fetchMock); + const { devicesEndpoint: _devicesEndpoint, ...options } = testMuOptions; + + await createProvider().verifyConnection({ + ...options, + apiEndpoint: 'https://staging.testmu.test/mobile-automation/api/v1/', + }); + + assert.equal( + String(fetchMock.mock.calls[0]?.[0]), + 'https://staging.testmu.test/mobile-automation/api/v1/capability/generator?isVirtualDevice=true', + ); +}); + +test('TestMu keeps the query of an overridden endpoint and adds its own filters', async () => { + const fetchMock = vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(testMuCatalog) + : jsonResponse({ data: [{ app_id: 'APP1' }] }), + ); + vi.stubGlobal('fetch', fetchMock); + const { devicesEndpoint: _devicesEndpoint, ...options } = testMuOptions; + + await createProvider().verifyConnection({ + ...options, + apiEndpoint: 'https://staging.testmu.test/api/v1/?region=eu', + appsEndpoint: 'https://staging.testmu.test/app/data?org=42', + }); + await createProvider().verifyConnection({ + ...testMuOptions, + devicesEndpoint: 'https://testmu.test/capability/generator?region=eu', + }); + + assert.deepEqual( + fetchMock.mock.calls.map(([input]) => String(input)), + [ + 'https://staging.testmu.test/api/v1/capability/generator?region=eu&isVirtualDevice=true', + 'https://staging.testmu.test/app/data?org=42&type=emulator&level=user', + 'https://testmu.test/capability/generator?region=eu&isVirtualDevice=true', + 'https://testmu.test/app/data?type=emulator&level=user', + ], + ); +}); + +test('TestMu rejects a device or OS version missing from the virtual-device catalog', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => jsonResponse(testMuCatalog)), + ); + await assert.rejects( + createProvider().verifyConnection({ ...testMuOptions, osVersion: '12' }), + (error: unknown) => + error instanceof Error && /"Pixel 8" with android 12 is not available/.test(error.message), + ); + await assert.rejects( + createProvider().verifyConnection({ ...testMuOptions, platform: 'ios', deviceName: 'Pixel 8' }), + /is not available/, + ); +}); + +// The hub rejects `platformVersion: '18'` for a catalog entry spelled `18.0`, so connect must too. +test('TestMu matches the catalog OS version spelling exactly and lists the offered versions', async () => { + const catalog = { + app: { + devices: { + ios: { + brands: { + Apple: [{ name: 'iPhone 16', osVersion: ['18.1', '26.0', '18.0', '18.5', '26.2'] }], + }, + }, + }, + }, + }; + vi.stubGlobal( + 'fetch', + vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(catalog) + : jsonResponse({ data: [], metaData: { total: 0 } }), + ), + ); + const iosOptions = { ...testMuOptions, platform: 'ios' as const, deviceName: 'iPhone 16' }; + + await assert.rejects( + createProvider().verifyConnection({ ...iosOptions, osVersion: '18' }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'INVALID_ARGS'); + assert.match(error.message, /iPhone 16 offers 18\.0, 18\.1, 18\.5, 26\.0, 26\.2/); + return true; + }, + ); + + const result = await createProvider().verifyConnection({ ...iosOptions, osVersion: '18.0' }); + assert.deepEqual(result.device, { + status: 'verified', + name: 'iPhone 16', + platform: 'ios', + osVersion: '18.0', + }); +}); + +test('TestMu classifies rejected credentials without exposing them', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(testMuCatalog) + : jsonResponse({ message: 'Unauthorized' }, 401), + ), + ); + await assert.rejects(createProvider().verifyConnection(testMuOptions), (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'UNAUTHORIZED'); + assert.doesNotMatch(error.message, /lt-key/); + return true; + }); +}); + +test('TestMu defers an lt:// reference it cannot find and a local path it will upload', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(testMuCatalog) + : jsonResponse({ data: [], metaData: { total: 0 } }), + ), + ); + const unknownApp = await createProvider().verifyConnection(testMuOptions); + assert.equal(unknownApp.app.status, 'configured'); + assert.equal(unknownApp.app.reference, 'lt://APP1'); + + const localApp = await createProvider().verifyConnection({ + ...testMuOptions, + app: '/tmp/builds/App.apk', + }); + assert.deepEqual(localApp.app, { + status: 'configured', + name: 'App.apk', + reference: '/tmp/builds/App.apk', + message: 'Local app artifact is ready and will be uploaded when creating the session.', + }); +}); + +// The real-device catalog is keyed by platform at the top level, not under `app.devices`. +const testMuRealCatalog = { + android: { + brands: { + Google: [ + { name: 'Pixel 6', osVersion: ['12', '13', '14', '15', '16'] }, + { name: 'Pixel 8', osVersion: ['14'] }, + ], + }, + }, + ios: { + brands: { + Apple: [ + { name: 'iPhone 16', osVersion: ['18'] }, + { name: 'iPhone 15', osVersion: ['17', '18', '26'] }, + ], + }, + }, + roku: { brands: {} }, + tvos: { brands: {} }, +}; + +test('TestMu verifies a real device against the real-device catalog shape', async () => { + const fetchMock = vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(testMuRealCatalog) + : jsonResponse({ data: [{ app_id: 'APP1', name: 'MyApp.ipa' }], metaData: { total: 1 } }), + ); + vi.stubGlobal('fetch', fetchMock); + const { devicesEndpoint: _devicesEndpoint, ...defaultCatalog } = testMuOptions; + + const result = await createProvider().verifyConnection({ + ...defaultCatalog, + deviceType: 'real', + platform: 'ios', + deviceName: 'iPhone 16', + osVersion: '18', + }); + + assert.equal(result.verificationMessage, 'Credentials, real device, and uploaded app verified.'); + assert.deepEqual(result.device, { + status: 'verified', + name: 'iPhone 16', + platform: 'ios', + osVersion: '18', + }); + assert.equal( + String(fetchMock.mock.calls[0]?.[0]), + 'https://mobile-api.lambdatest.com/mobile-automation/api/v1/capability/generator?isVirtualDevice=false', + ); + + const android = await createProvider().verifyConnection({ + ...testMuOptions, + deviceType: 'real', + deviceName: 'Pixel 6', + osVersion: '14', + }); + assert.equal(android.device.name, 'Pixel 6'); +}); + +// Real iOS devices are listed by major version, so `18.0` is the wrong spelling for the real pool. +test('TestMu matches real-device OS versions exactly and lists what the device offers', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => jsonResponse(testMuRealCatalog)), + ); + const realIos = { + ...testMuOptions, + deviceType: 'real' as const, + platform: 'ios' as const, + deviceName: 'iPhone 15', + }; + await assert.rejects( + createProvider().verifyConnection({ ...realIos, deviceName: 'iPhone 16', osVersion: '18.0' }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.equal((error as { code?: string }).code, 'INVALID_ARGS'); + assert.match( + error.message, + /TestMu AI real device "iPhone 16" with ios 18\.0 is not available/, + ); + assert.match(error.message, /iPhone 16 offers 18\.$/); + return true; + }, + ); + await assert.rejects( + createProvider().verifyConnection({ ...realIos, osVersion: '16' }), + /iPhone 15 offers 17, 18, 26/, + ); +}); + +test('TestMu fails typed when a catalog does not have the selected pool shape', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => jsonResponse(testMuCatalog)), + ); + await assert.rejects( + createProvider().verifyConnection({ ...testMuOptions, deviceType: 'real' }), + (error: unknown) => + error instanceof Error && + (error as { code?: string }).code === 'COMMAND_FAILED' && + /real-device catalog response did not list devices/.test(error.message), + ); + vi.stubGlobal( + 'fetch', + vi.fn(async () => jsonResponse(testMuRealCatalog)), + ); + await assert.rejects( + createProvider().verifyConnection(testMuOptions), + /virtual-device catalog response did not list devices/, + ); +}); + +// The listing is keyed by pool: `emulator`/`simulator` hold virtual uploads, `android`/`ios` real +// ones, so an id must be looked up in the list of the pool the session will run on. +test('TestMu checks an lt:// id against the app list of the selected pool and platform', async () => { + const cases = [ + { deviceType: 'virtual', platform: 'android', deviceName: 'Pixel 8', listType: 'emulator' }, + { deviceType: 'virtual', platform: 'ios', deviceName: 'iPhone 16', listType: 'simulator' }, + { deviceType: 'real', platform: 'android', deviceName: 'Pixel 6', listType: 'android' }, + { deviceType: 'real', platform: 'ios', deviceName: 'iPhone 16', listType: 'ios' }, + ] as const; + for (const { deviceType, platform, deviceName, listType } of cases) { + const fetchMock = vi.fn(async (input) => { + const url = String(input); + if (url.includes('capability/generator')) { + return jsonResponse(deviceType === 'real' ? testMuRealCatalog : testMuCatalog); + } + return jsonResponse({ + data: new URL(url).searchParams.get('type') === listType ? [{ app_id: 'APP1' }] : [], + }); + }); + vi.stubGlobal('fetch', fetchMock); + const result = await createProvider().verifyConnection({ + ...testMuOptions, + deviceType, + platform, + deviceName, + osVersion: + deviceType === 'real' + ? platform === 'ios' + ? '18' + : '14' + : platform === 'ios' + ? '18.0' + : '14', + }); + assert.equal(result.app.status, 'verified', `${deviceType} ${platform}`); + assert.equal( + String(fetchMock.mock.calls[1]?.[0]), + `https://testmu.test/app/data?type=${listType}&level=user`, + ); + } +}); + +test('TestMu defers a real-device lt:// id missing from the real-device app list', async () => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input) => + String(input).includes('capability/generator') + ? jsonResponse(testMuRealCatalog) + : jsonResponse({ data: [], metaData: { total: 0 } }), + ), + ); + const result = await createProvider().verifyConnection({ + ...testMuOptions, + deviceType: 'real', + deviceName: 'Pixel 6', + }); + assert.equal(result.app.status, 'configured'); + assert.match(String(result.app.message), /not found among your real-device uploads/); + assert.equal( + result.verificationMessage, + 'Credentials and real device verified; app availability is checked when the session is created.', + ); +}); diff --git a/packages/provider-webdriver/src/testmu-connection-verification.ts b/packages/provider-testmu/src/testmu-connection-verification.ts similarity index 93% rename from packages/provider-webdriver/src/testmu-connection-verification.ts rename to packages/provider-testmu/src/testmu-connection-verification.ts index 7e37a4f2f5..ca092f18de 100644 --- a/packages/provider-webdriver/src/testmu-connection-verification.ts +++ b/packages/provider-testmu/src/testmu-connection-verification.ts @@ -1,6 +1,10 @@ import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; -import { appendUrlPath, asRecord, fetchProviderVerificationJson } from './webdriver-utils.ts'; +import { + appendUrlPath, + asRecord, + fetchProviderVerificationJson, +} from '@agent-device/provider-webdriver/plugin'; import { TESTMU_API_ENDPOINT, TESTMU_APPS_ENDPOINT, @@ -8,15 +12,24 @@ import { testMuAppReferenceFromId, } from './testmu.ts'; import type { - CloudWebDriverConnectionVerification, - CloudWebDriverConnectionVerificationOptions, -} from './connection-verification.ts'; -import type { + ProviderConnectionVerification, ProviderConnectionResource, ProviderDeviceType, } from '@agent-device/contracts/remote'; -type TestMuOptions = Extract; +export type TestMuOptions = { + provider: 'testmu'; + username: string; + accessKey: string; + platform: 'android' | 'ios'; + deviceName: string; + osVersion: string; + app: string; + deviceType?: ProviderDeviceType; + apiEndpoint?: string | URL; + devicesEndpoint?: string | URL; + appsEndpoint?: string | URL; +}; type TestMuAuth = { username: string; accessKey: string }; @@ -34,7 +47,7 @@ const TESTMU_APP_LIST_TYPES: Record { +): Promise { const auth = { username: options.username, accessKey: options.accessKey }; const deviceType = options.deviceType ?? 'virtual'; const catalogUrl = options.devicesEndpoint diff --git a/packages/provider-webdriver/src/testmu-device-features.test.ts b/packages/provider-testmu/src/testmu-device-features.test.ts similarity index 100% rename from packages/provider-webdriver/src/testmu-device-features.test.ts rename to packages/provider-testmu/src/testmu-device-features.test.ts diff --git a/packages/provider-webdriver/src/testmu-device-features.ts b/packages/provider-testmu/src/testmu-device-features.ts similarity index 97% rename from packages/provider-webdriver/src/testmu-device-features.ts rename to packages/provider-testmu/src/testmu-device-features.ts index 6d886b8438..da67971388 100644 --- a/packages/provider-webdriver/src/testmu-device-features.ts +++ b/packages/provider-testmu/src/testmu-device-features.ts @@ -4,7 +4,7 @@ import { type ProviderDeviceType, } from '@agent-device/contracts/remote'; import { AppError } from '@agent-device/kernel/errors'; -import { requireProviderDeviceOrientation } from './webdriver-utils.ts'; +import { requireProviderDeviceOrientation } from '@agent-device/provider-webdriver/plugin'; /** * TestMu "device feature" session capabilities: the hosted-provider flags TestMu can act on, diff --git a/packages/provider-webdriver/src/testmu.test.ts b/packages/provider-testmu/src/testmu.test.ts similarity index 99% rename from packages/provider-webdriver/src/testmu.test.ts rename to packages/provider-testmu/src/testmu.test.ts index 084e7fb084..551ddeda66 100644 --- a/packages/provider-webdriver/src/testmu.test.ts +++ b/packages/provider-testmu/src/testmu.test.ts @@ -11,7 +11,7 @@ import { uploadTestMuApp, uploadTestMuAppFromUrl, } from './testmu.ts'; -import { buildCloudWebDriverBaseCapabilities } from './runtime.ts'; +import { buildCloudWebDriverBaseCapabilities } from '@agent-device/provider-webdriver/plugin'; import { mkdtempForTest } from './tmp-dir.fixtures.ts'; const realFetch = globalThis.fetch; diff --git a/packages/provider-webdriver/src/testmu.ts b/packages/provider-testmu/src/testmu.ts similarity index 97% rename from packages/provider-webdriver/src/testmu.ts rename to packages/provider-testmu/src/testmu.ts index 68b970d971..8055ee2dd7 100644 --- a/packages/provider-webdriver/src/testmu.ts +++ b/packages/provider-testmu/src/testmu.ts @@ -2,10 +2,16 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import type { CloudArtifact, CloudArtifactsResult } from '@agent-device/contracts/observability'; import type { ProviderDeviceType } from '@agent-device/contracts/remote'; -import type { CloudWebDriverPlatform, CloudWebDriverUploadApp } from './runtime.ts'; +import type { + CloudWebDriverPlatform, + CloudWebDriverUploadApp, +} from '@agent-device/provider-webdriver/plugin'; import { AppError } from '@agent-device/kernel/errors'; import { isTestMuAppReference } from './providers.ts'; -import { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; +import { + cloudArtifactsReadyOrPending, + urlArtifactFromDetails, +} from '@agent-device/provider-webdriver/plugin'; import { appendUrlPath, appFileUploadForm, @@ -14,7 +20,7 @@ import { fetchProviderSessionDetails, postHubAppUpload, resolveHubAppReference, -} from './webdriver-utils.ts'; +} from '@agent-device/provider-webdriver/plugin'; /** * TestMu session, upload, and artifact mechanics. Loaded on demand by the provider definition; diff --git a/packages/provider-testmu/src/tmp-dir.fixtures.ts b/packages/provider-testmu/src/tmp-dir.fixtures.ts new file mode 100644 index 0000000000..dbc5c2724c --- /dev/null +++ b/packages/provider-testmu/src/tmp-dir.fixtures.ts @@ -0,0 +1,21 @@ +import fs from 'node:fs'; +import fsPromises from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +/** + * Creates a fresh scratch directory for one test. Cleanup is automatic: the + * unit suite redirects TMPDIR to a per-run directory (scripts/vitest-tmpdir-global-setup.ts) + * that gets removed in one recursive rm after every worker finishes, so + * individual tests never need their own afterEach/afterAll for this. + */ +// fallow-ignore-next-line code-duplication +export async function mkdtempForTest(prefix: string): Promise { + return fsPromises.mkdtemp(path.join(os.tmpdir(), prefix)); +} + +/** Sync counterpart of {@link mkdtempForTest}, for setup code that can't await. */ +// fallow-ignore-next-line code-duplication +export function mkdtempForTestSync(prefix: string): string { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} diff --git a/packages/provider-testmu/test/package-smoke.mjs b/packages/provider-testmu/test/package-smoke.mjs new file mode 100644 index 0000000000..fb29daa306 --- /dev/null +++ b/packages/provider-testmu/test/package-smoke.mjs @@ -0,0 +1,33 @@ +export const args = [ + '--platform', + 'android', + '--device', + 'Pixel 8', + '--provider-os-version', + '14', + '--provider-app', + 'lt://APP1', +]; +export function environment(endpoint) { + return { LT_USERNAME: 'user', LT_ACCESS_KEY: 'key', TESTMU_API_ENDPOINT: endpoint }; +} +export function respond(url, rejectCredentials) { + if (url.includes('capability/generator')) + return { + body: { + app: { + devices: { android: { brands: { Google: [{ name: 'Pixel 8', osVersion: ['14'] }] } } }, + }, + }, + }; + return rejectCredentials + ? { status: 401, body: { error: 'unauthorized' } } + : { + body: { + data: [{ app_id: 'APP1', name: 'app.apk', type: 'android' }], + metaData: { total: 1 }, + }, + }; +} + +export const fetchRedirects = ['https://manual-api.lambdatest.com']; diff --git a/packages/provider-testmu/tsconfig.json b/packages/provider-testmu/tsconfig.json new file mode 100644 index 0000000000..935c871a4d --- /dev/null +++ b/packages/provider-testmu/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "composite": true, + "noEmit": false, + "emitDeclarationOnly": true, + "declaration": true, + "declarationDir": "./dist-types", + "rootDir": "./src" + }, + "include": ["src"] +} diff --git a/packages/provider-testmu/tsdown.config.ts b/packages/provider-testmu/tsdown.config.ts new file mode 100644 index 0000000000..5aeb12099a --- /dev/null +++ b/packages/provider-testmu/tsdown.config.ts @@ -0,0 +1,19 @@ +import { defineConfig } from 'tsdown'; + +export default defineConfig({ + entry: { plugin: 'src/plugin.ts' }, + outDir: 'dist', + format: 'esm', + platform: 'node', + target: 'es2022', + minify: true, + dts: false, + hash: false, + deps: { alwaysBundle: [/^@agent-device\//] }, + inputOptions: { + onLog(level, log, handler) { + if (log.code === 'UNRESOLVED_IMPORT') throw new Error(log.message); + handler(level, log); + }, + }, +}); diff --git a/packages/provider-webdriver/src/connection-verification.test.ts b/packages/provider-webdriver/src/connection-verification.test.ts index fbd3978638..c93e40109e 100644 --- a/packages/provider-webdriver/src/connection-verification.test.ts +++ b/packages/provider-webdriver/src/connection-verification.test.ts @@ -213,401 +213,6 @@ test('AWS Device Farm rejects a device from the wrong platform before allocation ); }); -const testMuOptions = { - provider: 'testmu' as const, - username: 'lt-user', - accessKey: 'lt-key', - platform: 'android' as const, - deviceName: 'Pixel 8', - osVersion: '14', - app: 'lt://APP1', - devicesEndpoint: 'https://testmu.test/capability/generator?isVirtualDevice=true', - appsEndpoint: 'https://testmu.test/app/data', -}; - -const testMuCatalog = { - app: { - devices: { - android: { - brands: { - Google: [ - { name: 'Pixel 8', osVersion: ['14', '15'] }, - { name: 'Pixel 4a', osVersion: ['13'] }, - ], - }, - }, - ios: { brands: { Apple: [{ name: 'iPhone 16', osVersion: ['18.0'] }] } }, - }, - }, -}; - -test('TestMu verifies the virtual device and uploaded app without creating a session', async () => { - const fetchMock = vi.fn(async (input, init) => { - const headers = (init?.headers ?? {}) as Record; - if (String(input).includes('capability/generator')) { - assert.equal(headers.Authorization, undefined); - return jsonResponse(testMuCatalog); - } - assert.match(String(headers.Authorization), /^Basic /); - return jsonResponse({ - data: [{ app_id: 'APP1', name: 'sample.apk', version: '1.2.3', type: 'android' }], - metaData: { total: 1 }, - }); - }); - vi.stubGlobal('fetch', fetchMock); - - const result = await createProvider().verifyConnection(testMuOptions); - - assert.equal(result.provider, 'testmu'); - assert.equal(result.service, 'TestMu AI'); - assert.deepEqual(result.device, { - status: 'verified', - name: 'Pixel 8', - platform: 'android', - osVersion: '14', - }); - assert.deepEqual(result.app, { - status: 'verified', - name: 'sample.apk', - reference: 'lt://APP1', - version: '1.2.3', - }); - assert.deepEqual( - fetchMock.mock.calls.map(([input]) => String(input)), - [ - 'https://testmu.test/capability/generator?isVirtualDevice=true', - 'https://testmu.test/app/data?type=emulator&level=user', - ], - ); -}); - -test('TestMu checks the catalog of the configured API endpoint', async () => { - const fetchMock = vi.fn(async (input) => - String(input).includes('capability/generator') - ? jsonResponse(testMuCatalog) - : jsonResponse({ data: [{ app_id: 'APP1' }] }), - ); - vi.stubGlobal('fetch', fetchMock); - const { devicesEndpoint: _devicesEndpoint, ...options } = testMuOptions; - - await createProvider().verifyConnection({ - ...options, - apiEndpoint: 'https://staging.testmu.test/mobile-automation/api/v1/', - }); - - assert.equal( - String(fetchMock.mock.calls[0]?.[0]), - 'https://staging.testmu.test/mobile-automation/api/v1/capability/generator?isVirtualDevice=true', - ); -}); - -test('TestMu keeps the query of an overridden endpoint and adds its own filters', async () => { - const fetchMock = vi.fn(async (input) => - String(input).includes('capability/generator') - ? jsonResponse(testMuCatalog) - : jsonResponse({ data: [{ app_id: 'APP1' }] }), - ); - vi.stubGlobal('fetch', fetchMock); - const { devicesEndpoint: _devicesEndpoint, ...options } = testMuOptions; - - await createProvider().verifyConnection({ - ...options, - apiEndpoint: 'https://staging.testmu.test/api/v1/?region=eu', - appsEndpoint: 'https://staging.testmu.test/app/data?org=42', - }); - await createProvider().verifyConnection({ - ...testMuOptions, - devicesEndpoint: 'https://testmu.test/capability/generator?region=eu', - }); - - assert.deepEqual( - fetchMock.mock.calls.map(([input]) => String(input)), - [ - 'https://staging.testmu.test/api/v1/capability/generator?region=eu&isVirtualDevice=true', - 'https://staging.testmu.test/app/data?org=42&type=emulator&level=user', - 'https://testmu.test/capability/generator?region=eu&isVirtualDevice=true', - 'https://testmu.test/app/data?type=emulator&level=user', - ], - ); -}); - -test('TestMu rejects a device or OS version missing from the virtual-device catalog', async () => { - vi.stubGlobal( - 'fetch', - vi.fn(async () => jsonResponse(testMuCatalog)), - ); - await assert.rejects( - createProvider().verifyConnection({ ...testMuOptions, osVersion: '12' }), - (error: unknown) => - error instanceof Error && /"Pixel 8" with android 12 is not available/.test(error.message), - ); - await assert.rejects( - createProvider().verifyConnection({ ...testMuOptions, platform: 'ios', deviceName: 'Pixel 8' }), - /is not available/, - ); -}); - -// The hub rejects `platformVersion: '18'` for a catalog entry spelled `18.0`, so connect must too. -test('TestMu matches the catalog OS version spelling exactly and lists the offered versions', async () => { - const catalog = { - app: { - devices: { - ios: { - brands: { - Apple: [{ name: 'iPhone 16', osVersion: ['18.1', '26.0', '18.0', '18.5', '26.2'] }], - }, - }, - }, - }, - }; - vi.stubGlobal( - 'fetch', - vi.fn(async (input) => - String(input).includes('capability/generator') - ? jsonResponse(catalog) - : jsonResponse({ data: [], metaData: { total: 0 } }), - ), - ); - const iosOptions = { ...testMuOptions, platform: 'ios' as const, deviceName: 'iPhone 16' }; - - await assert.rejects( - createProvider().verifyConnection({ ...iosOptions, osVersion: '18' }), - (error: unknown) => { - assert.ok(error instanceof Error); - assert.equal((error as { code?: string }).code, 'INVALID_ARGS'); - assert.match(error.message, /iPhone 16 offers 18\.0, 18\.1, 18\.5, 26\.0, 26\.2/); - return true; - }, - ); - - const result = await createProvider().verifyConnection({ ...iosOptions, osVersion: '18.0' }); - assert.deepEqual(result.device, { - status: 'verified', - name: 'iPhone 16', - platform: 'ios', - osVersion: '18.0', - }); -}); - -test('TestMu classifies rejected credentials without exposing them', async () => { - vi.stubGlobal( - 'fetch', - vi.fn(async (input) => - String(input).includes('capability/generator') - ? jsonResponse(testMuCatalog) - : jsonResponse({ message: 'Unauthorized' }, 401), - ), - ); - await assert.rejects(createProvider().verifyConnection(testMuOptions), (error: unknown) => { - assert.ok(error instanceof Error); - assert.equal((error as { code?: string }).code, 'UNAUTHORIZED'); - assert.doesNotMatch(error.message, /lt-key/); - return true; - }); -}); - -test('TestMu defers an lt:// reference it cannot find and a local path it will upload', async () => { - vi.stubGlobal( - 'fetch', - vi.fn(async (input) => - String(input).includes('capability/generator') - ? jsonResponse(testMuCatalog) - : jsonResponse({ data: [], metaData: { total: 0 } }), - ), - ); - const unknownApp = await createProvider().verifyConnection(testMuOptions); - assert.equal(unknownApp.app.status, 'configured'); - assert.equal(unknownApp.app.reference, 'lt://APP1'); - - const localApp = await createProvider().verifyConnection({ - ...testMuOptions, - app: '/tmp/builds/App.apk', - }); - assert.deepEqual(localApp.app, { - status: 'configured', - name: 'App.apk', - reference: '/tmp/builds/App.apk', - message: 'Local app artifact is ready and will be uploaded when creating the session.', - }); -}); - -// The real-device catalog is keyed by platform at the top level, not under `app.devices`. -const testMuRealCatalog = { - android: { - brands: { - Google: [ - { name: 'Pixel 6', osVersion: ['12', '13', '14', '15', '16'] }, - { name: 'Pixel 8', osVersion: ['14'] }, - ], - }, - }, - ios: { - brands: { - Apple: [ - { name: 'iPhone 16', osVersion: ['18'] }, - { name: 'iPhone 15', osVersion: ['17', '18', '26'] }, - ], - }, - }, - roku: { brands: {} }, - tvos: { brands: {} }, -}; - -test('TestMu verifies a real device against the real-device catalog shape', async () => { - const fetchMock = vi.fn(async (input) => - String(input).includes('capability/generator') - ? jsonResponse(testMuRealCatalog) - : jsonResponse({ data: [{ app_id: 'APP1', name: 'MyApp.ipa' }], metaData: { total: 1 } }), - ); - vi.stubGlobal('fetch', fetchMock); - const { devicesEndpoint: _devicesEndpoint, ...defaultCatalog } = testMuOptions; - - const result = await createProvider().verifyConnection({ - ...defaultCatalog, - deviceType: 'real', - platform: 'ios', - deviceName: 'iPhone 16', - osVersion: '18', - }); - - assert.equal(result.verificationMessage, 'Credentials, real device, and uploaded app verified.'); - assert.deepEqual(result.device, { - status: 'verified', - name: 'iPhone 16', - platform: 'ios', - osVersion: '18', - }); - assert.equal( - String(fetchMock.mock.calls[0]?.[0]), - 'https://mobile-api.lambdatest.com/mobile-automation/api/v1/capability/generator?isVirtualDevice=false', - ); - - const android = await createProvider().verifyConnection({ - ...testMuOptions, - deviceType: 'real', - deviceName: 'Pixel 6', - osVersion: '14', - }); - assert.equal(android.device.name, 'Pixel 6'); -}); - -// Real iOS devices are listed by major version, so `18.0` is the wrong spelling for the real pool. -test('TestMu matches real-device OS versions exactly and lists what the device offers', async () => { - vi.stubGlobal( - 'fetch', - vi.fn(async () => jsonResponse(testMuRealCatalog)), - ); - const realIos = { - ...testMuOptions, - deviceType: 'real' as const, - platform: 'ios' as const, - deviceName: 'iPhone 15', - }; - await assert.rejects( - createProvider().verifyConnection({ ...realIos, deviceName: 'iPhone 16', osVersion: '18.0' }), - (error: unknown) => { - assert.ok(error instanceof Error); - assert.equal((error as { code?: string }).code, 'INVALID_ARGS'); - assert.match( - error.message, - /TestMu AI real device "iPhone 16" with ios 18\.0 is not available/, - ); - assert.match(error.message, /iPhone 16 offers 18\.$/); - return true; - }, - ); - await assert.rejects( - createProvider().verifyConnection({ ...realIos, osVersion: '16' }), - /iPhone 15 offers 17, 18, 26/, - ); -}); - -test('TestMu fails typed when a catalog does not have the selected pool shape', async () => { - vi.stubGlobal( - 'fetch', - vi.fn(async () => jsonResponse(testMuCatalog)), - ); - await assert.rejects( - createProvider().verifyConnection({ ...testMuOptions, deviceType: 'real' }), - (error: unknown) => - error instanceof Error && - (error as { code?: string }).code === 'COMMAND_FAILED' && - /real-device catalog response did not list devices/.test(error.message), - ); - vi.stubGlobal( - 'fetch', - vi.fn(async () => jsonResponse(testMuRealCatalog)), - ); - await assert.rejects( - createProvider().verifyConnection(testMuOptions), - /virtual-device catalog response did not list devices/, - ); -}); - -// The listing is keyed by pool: `emulator`/`simulator` hold virtual uploads, `android`/`ios` real -// ones, so an id must be looked up in the list of the pool the session will run on. -test('TestMu checks an lt:// id against the app list of the selected pool and platform', async () => { - const cases = [ - { deviceType: 'virtual', platform: 'android', deviceName: 'Pixel 8', listType: 'emulator' }, - { deviceType: 'virtual', platform: 'ios', deviceName: 'iPhone 16', listType: 'simulator' }, - { deviceType: 'real', platform: 'android', deviceName: 'Pixel 6', listType: 'android' }, - { deviceType: 'real', platform: 'ios', deviceName: 'iPhone 16', listType: 'ios' }, - ] as const; - for (const { deviceType, platform, deviceName, listType } of cases) { - const fetchMock = vi.fn(async (input) => { - const url = String(input); - if (url.includes('capability/generator')) { - return jsonResponse(deviceType === 'real' ? testMuRealCatalog : testMuCatalog); - } - return jsonResponse({ - data: new URL(url).searchParams.get('type') === listType ? [{ app_id: 'APP1' }] : [], - }); - }); - vi.stubGlobal('fetch', fetchMock); - const result = await createProvider().verifyConnection({ - ...testMuOptions, - deviceType, - platform, - deviceName, - osVersion: - deviceType === 'real' - ? platform === 'ios' - ? '18' - : '14' - : platform === 'ios' - ? '18.0' - : '14', - }); - assert.equal(result.app.status, 'verified', `${deviceType} ${platform}`); - assert.equal( - String(fetchMock.mock.calls[1]?.[0]), - `https://testmu.test/app/data?type=${listType}&level=user`, - ); - } -}); - -test('TestMu defers a real-device lt:// id missing from the real-device app list', async () => { - vi.stubGlobal( - 'fetch', - vi.fn(async (input) => - String(input).includes('capability/generator') - ? jsonResponse(testMuRealCatalog) - : jsonResponse({ data: [], metaData: { total: 0 } }), - ), - ); - const result = await createProvider().verifyConnection({ - ...testMuOptions, - deviceType: 'real', - deviceName: 'Pixel 6', - }); - assert.equal(result.app.status, 'configured'); - assert.match(String(result.app.message), /not found among your real-device uploads/); - assert.equal( - result.verificationMessage, - 'Credentials and real device verified; app availability is checked when the session is created.', - ); -}); - function createProvider(runHostCommand: RunHostCommand = vi.fn()) { return createProviderWebDriver({ clientVersion: '1.2.3', runHostCommand }); } diff --git a/packages/provider-webdriver/src/connection-verification.ts b/packages/provider-webdriver/src/connection-verification.ts index 7e8685da1b..9cd048b951 100644 --- a/packages/provider-webdriver/src/connection-verification.ts +++ b/packages/provider-webdriver/src/connection-verification.ts @@ -1,8 +1,5 @@ import type { ProviderWebDriverDependencies } from './dependencies.ts'; -import type { - ProviderConnectionVerification, - ProviderDeviceType, -} from '@agent-device/contracts/remote'; +import type { ProviderConnectionVerification } from '@agent-device/contracts/remote'; import { verifyAwsDeviceFarmConnection } from './aws-device-farm-connection-verification.ts'; import { verifyBrowserStackConnection } from './browserstack-connection-verification.ts'; @@ -18,11 +15,6 @@ export type CloudWebDriverConnectionVerification = provider: 'aws-device-farm'; service: 'AWS Device Farm'; project: { name?: string; reference: string }; - }) - | (ProviderConnectionVerification & { - provider: 'testmu'; - service: 'TestMu AI'; - project?: never; }); /** Credentials plus the exact device, OS, and app a hosted Appium hub session is created with. */ @@ -39,13 +31,6 @@ type HubSelectionVerificationOptions = { export type CloudWebDriverConnectionVerificationOptions = | (HubSelectionVerificationOptions & { provider: 'browserstack' }) - | (HubSelectionVerificationOptions & { - provider: 'testmu'; - /** Defaults to `virtual`. */ - deviceType?: ProviderDeviceType; - /** Base of the catalog API, as `TESTMU_API_ENDPOINT` sets it for the runtime. */ - apiEndpoint?: string | URL; - }) | { provider: 'aws-device-farm'; platform: 'android' | 'ios'; @@ -62,11 +47,6 @@ export async function verifyCloudWebDriverConnection( switch (options.provider) { case 'browserstack': return await verifyBrowserStackConnection(options, dependencies.clientVersion); - case 'testmu': { - // Loaded on demand: the package entry must not grow its eager closure for a new vendor. - const { verifyTestMuConnection } = await import('./testmu-connection-verification.ts'); - return await verifyTestMuConnection(options, dependencies.clientVersion); - } case 'aws-device-farm': return await verifyAwsDeviceFarmConnection(options, dependencies.runHostCommand); } diff --git a/packages/provider-webdriver/src/plugin.ts b/packages/provider-webdriver/src/plugin.ts index e1b46966b4..827c72969a 100644 --- a/packages/provider-webdriver/src/plugin.ts +++ b/packages/provider-webdriver/src/plugin.ts @@ -1,17 +1,19 @@ export { createCloudWebDriverRuntime } from './runtime.ts'; -export type { CloudWebDriverRuntimeOptions } from './runtime.ts'; +export type { + CloudWebDriverRuntimeOptions, + CloudWebDriverPlatform, + CloudWebDriverUploadApp, +} from './runtime.ts'; export { appFileUploadForm, appendUrlPath, asRecord, - basicAuthHeader, createHubUploadApp, fetchProviderSessionDetails, fetchProviderVerificationJson, postHubAppUpload, requireProviderDeviceOrientation, resolveHubAppReference, - sameOsVersion, } from './webdriver-utils.ts'; export { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; export { buildCloudWebDriverBaseCapabilities } from './runtime-session.ts'; diff --git a/packages/provider-webdriver/src/provider-definitions.ts b/packages/provider-webdriver/src/provider-definitions.ts index d701f29595..42acf03ee1 100644 --- a/packages/provider-webdriver/src/provider-definitions.ts +++ b/packages/provider-webdriver/src/provider-definitions.ts @@ -1,6 +1,5 @@ import type { CloudArtifactsResult } from '@agent-device/contracts/observability'; import type { LeaseLifecycleContext } from '@agent-device/contracts/device'; -import type { ProviderDeviceType } from '@agent-device/contracts/remote'; import type { ProviderProfileFieldDeclaration } from '@agent-device/contracts/provider-profile-fields'; import { AppError } from '@agent-device/kernel/errors'; import type { ProviderWebDriverDependencies } from './dependencies.ts'; @@ -23,7 +22,6 @@ import { buildBrowserStackDeviceFeatureCapabilities, readBrowserStackDeviceFeatureFields, } from './browserstack-device-features.ts'; -import type { CloudWebDriverCapabilityOverrides } from './capabilities.ts'; import { CLOUD_WEBDRIVER_PROVIDERS, type CloudWebDriverKnownProviderName } from './providers.ts'; import { readAwsDeviceFarmRegionFromArn } from './connection-verification.ts'; import { @@ -39,17 +37,11 @@ export type DefaultCloudWebDriverArtifactEnv = { BROWSERSTACK_SESSION_DETAILS_ENDPOINT?: string; AWS_REGION?: string; AWS_DEFAULT_REGION?: string; - LT_USERNAME?: string; - LT_ACCESS_KEY?: string; - TESTMU_API_ENDPOINT?: string; }; export type DefaultCloudWebDriverProviderRuntimeEnv = DefaultCloudWebDriverArtifactEnv & { BROWSERSTACK_WEBDRIVER_ENDPOINT?: string; BROWSERSTACK_APP_UPLOAD_ENDPOINT?: string; - TESTMU_WEBDRIVER_ENDPOINT?: string; - TESTMU_APP_UPLOAD_ENDPOINT?: string; - TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT?: string; AGENT_DEVICE_AWS_DEVICE_FARM_PROJECT_ARN?: string; AWS_DEVICE_FARM_PROJECT_ARN?: string; AGENT_DEVICE_AWS_DEVICE_FARM_DEVICE_ARN?: string; @@ -58,30 +50,6 @@ export type DefaultCloudWebDriverProviderRuntimeEnv = DefaultCloudWebDriverArtif AWS_DEVICE_FARM_APP_ARN?: string; }; -/** - * TestMu (formerly LambdaTest) real devices, and virtual devices (emulators and simulators), behind - * one Appium hub. Only what `createRuntime` needs synchronously lives here; the session, - * upload, and artifact code loads on first use so the package entry stays as lean as it was. - */ -const TESTMU_WEBDRIVER_ENDPOINT = 'https://mobile-hub.lambdatest.com/wd/hub/'; -const TESTMU_CAPABILITY_OVERRIDES = { - install: { - support: 'partial', - note: 'Local app artifacts are uploaded to TestMu AI as real- or virtual-device apps (lt://), then installed with Appium.', - }, - portReverse: { - support: 'unsupported', - note: 'Use the TestMu AI tunnel for network access to local hosts; agent-device port reverse is not available.', - }, - artifacts: { - support: 'supported', - note: 'TestMu AI session details expose provider-hosted video, Appium logs, device logs, network logs, and dashboard links.', - }, -} as const satisfies CloudWebDriverCapabilityOverrides; - -const loadTestMu = async () => await import('./testmu.ts'); -const loadTestMuDeviceFeatures = async () => await import('./testmu-device-features.ts'); - const AWS_DEVICE_FARM_FIELDS_REFUSED = { awsProjectArn: 'refused', awsDeviceArn: 'refused', @@ -140,36 +108,12 @@ const AWS_DEVICE_FARM_PROFILE_FIELDS: ProviderProfileFieldDeclaration = { }, }; -const TESTMU_PROFILE_FIELDS: ProviderProfileFieldDeclaration = { - provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, - label: 'TestMu AI', - fields: { - providerApp: 'consumed', - providerOsVersion: 'consumed', - providerDeviceType: 'consumed', - providerProject: 'consumed', - providerBuild: 'consumed', - providerSessionName: 'consumed', - providerDeviceOrientation: 'consumed', - providerGeoLocation: 'consumed', - providerTimezone: 'consumed', - providerAppiumVersion: 'consumed', - providerLanguage: 'consumed', - providerLocale: 'consumed', - providerNetworkProfile: 'refused', - providerCustomNetwork: 'refused', - providerNoResignApp: 'refused', - ...AWS_DEVICE_FARM_FIELDS_REFUSED, - }, -}; - /** The profile fields each hub provider reads, for routes that check them before a runtime exists. */ export const CLOUD_WEBDRIVER_PROFILE_FIELDS: Readonly< Record > = { [CLOUD_WEBDRIVER_PROVIDERS.browserStack]: BROWSERSTACK_PROFILE_FIELDS, [CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm]: AWS_DEVICE_FARM_PROFILE_FIELDS, - [CLOUD_WEBDRIVER_PROVIDERS.testMu]: TESTMU_PROFILE_FIELDS, }; export type CloudWebDriverProviderDefinition = { @@ -371,117 +315,7 @@ export function createCloudWebDriverProviderDefinitions( ); }, }, - { - provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, - profileFields: TESTMU_PROFILE_FIELDS, - createRuntime: (env, profileFields) => - createCloudWebDriverRuntime({ - clientVersion: dependencies.clientVersion, - provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, - profileFields, - platform: 'android', - deviceName: 'TestMu AI device', - endpoint: env.TESTMU_WEBDRIVER_ENDPOINT ?? TESTMU_WEBDRIVER_ENDPOINT, - capabilityOverrides: TESTMU_CAPABILITY_OVERRIDES, - listArtifacts: async ({ provider, providerSessionId }) => - await listTestMuArtifactsFromEnv(provider, providerSessionId, env), - prepareSession: async ({ req, lease, base }) => { - const request = requireRequest(req, 'TestMu AI'); - const { buildTestMuCapabilities, createTestMuUploadApp, resolveTestMuAppReference } = - await loadTestMu(); - const { - buildTestMuDeviceFeatureCapabilities, - readTestMuDeviceFeatureFields, - readTestMuDeviceType, - } = await loadTestMuDeviceFeatures(); - const deviceType = readTestMuDeviceType(request.flags); - const uploadEndpoint = testMuAppUploadEndpoint(env, deviceType); - const credentials = requireTestMuCredentials(env, 'TestMu AI'); - const platform = requireRequestPlatform(request, 'TestMu AI'); - const deviceName = requireFlag( - request, - 'device', - 'TestMu AI requires --device .', - ); - const osVersion = requireFlag( - request, - 'providerOsVersion', - 'TestMu AI requires --provider-os-version .', - ); - const upload = { - clientVersion: dependencies.clientVersion, - ...credentials, - deviceType, - endpoint: uploadEndpoint, - }; - const app = await resolveTestMuAppReference( - requireFlag( - request, - 'providerApp', - 'TestMu AI requires --provider-app .', - ), - { ...upload, cwd: request.cwd, signal: request.signal }, - ); - return { - ...base, - platform, - deviceName, - auth: credentials, - uploadApp: createTestMuUploadApp(upload), - webdriverCapabilities: buildTestMuCapabilities({ - platform, - deviceType, - deviceName, - osVersion, - app, - projectName: readFlag(request, 'providerProject'), - buildName: readFlag(request, 'providerBuild') ?? lease.runId, - sessionName: readFlag(request, 'providerSessionName') ?? lease.leaseId, - deviceFeatures: buildTestMuDeviceFeatureCapabilities( - readTestMuDeviceFeatureFields(request.flags), - ), - configured: buildCloudWebDriverBaseCapabilities(platform, deviceName), - }), - }; - }, - }), - listArtifactsFromEnv: async (providerSessionId, env) => - await listTestMuArtifactsFromEnv(CLOUD_WEBDRIVER_PROVIDERS.testMu, providerSessionId, env), - }, ]; - - async function listTestMuArtifactsFromEnv( - provider: string, - providerSessionId: string | undefined, - env: DefaultCloudWebDriverArtifactEnv, - ): Promise { - const { listTestMuCloudArtifacts } = await loadTestMu(); - return await listTestMuCloudArtifacts(provider, providerSessionId, { - clientVersion: dependencies.clientVersion, - ...requireTestMuCredentials(env, 'TestMu AI artifact lookup'), - endpoint: env.TESTMU_API_ENDPOINT, - }); - } -} - -function requireTestMuCredentials( - env: DefaultCloudWebDriverArtifactEnv, - providerLabel: string, -): { username: string; accessKey: string } { - return { - username: requireEnv(env, 'LT_USERNAME', providerLabel), - accessKey: requireEnv(env, 'LT_ACCESS_KEY', providerLabel), - }; -} - -/** Each pool has its own upload API, so each has its own override. */ -function testMuAppUploadEndpoint( - env: DefaultCloudWebDriverProviderRuntimeEnv, - deviceType: ProviderDeviceType, -): string | undefined { - return deviceType === 'real' - ? env.TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT - : env.TESTMU_APP_UPLOAD_ENDPOINT; } function requireRequest( diff --git a/packages/provider-webdriver/src/provider-profile-fields.test.ts b/packages/provider-webdriver/src/provider-profile-fields.test.ts index 84f3221ab3..aa414aaecd 100644 --- a/packages/provider-webdriver/src/provider-profile-fields.test.ts +++ b/packages/provider-webdriver/src/provider-profile-fields.test.ts @@ -4,7 +4,6 @@ import type { ProviderProfileField } from '@agent-device/contracts/provider-prof import { CLOUD_WEBDRIVER_PROFILE_FIELDS } from './provider-definitions.ts'; import { CLOUD_WEBDRIVER_PROVIDERS } from './providers.ts'; import { BROWSERSTACK_DEVICE_FEATURE_SPECS } from './browserstack-device-features.ts'; -import { TESTMU_DEVICE_FEATURE_SPECS } from './testmu-device-features.ts'; // Fields a hub reads directly while building its session, outside the device-feature tables. const HUB_SESSION_FIELDS: readonly ProviderProfileField[] = [ @@ -38,10 +37,6 @@ test('hub declarations consume exactly the fields their capability builders read ...BROWSERSTACK_DEVICE_FEATURE_SPECS.map((spec) => spec.field), ].sort(), ); - assert.deepEqual( - consumedFields(CLOUD_WEBDRIVER_PROVIDERS.testMu), - [...HUB_SESSION_FIELDS, ...TESTMU_DEVICE_FEATURE_SPECS.map((spec) => spec.field)].sort(), - ); }); test('AWS Device Farm consumes only its own fields and the session name', () => { diff --git a/packages/provider-webdriver/src/runtime-deployment.test.ts b/packages/provider-webdriver/src/runtime-deployment.test.ts index efe9835962..e5c79598c7 100644 --- a/packages/provider-webdriver/src/runtime-deployment.test.ts +++ b/packages/provider-webdriver/src/runtime-deployment.test.ts @@ -3,7 +3,6 @@ import path from 'node:path'; import { afterEach, expect, test, vi } from 'vitest'; import { createCloudWebDriverCapabilities } from './capabilities.ts'; import type { DeviceInfo } from '@agent-device/kernel/device'; -import { createTestMuUploadApp } from './testmu.ts'; import { mkdtempForTest } from './tmp-dir.fixtures.ts'; import { createWebDriverDeploymentRuntime } from './runtime-deployment.ts'; import type { WebDriverProviderSession } from './runtime-session.ts'; @@ -177,51 +176,6 @@ test('a provider without an uploader still installs the materialized bundle path expect(installApp).toHaveBeenCalledWith('/m/extracted/App.app', expect.any(AbortSignal)); }); -test('TestMu uploads the zipped simulator build that install-from-source extracted', async () => { - const tempDir = await mkdtempForTest('agent-device-materialized-upload-'); - try { - const archivePath = path.join(tempDir, 'App.app.zip'); - const installablePath = path.join(tempDir, 'extracted', 'App.app'); - await fs.writeFile(archivePath, 'zip bytes'); - await fs.mkdir(installablePath, { recursive: true }); - const uploadedNames: unknown[] = []; - globalThis.fetch = async (_input, init) => { - const body = init?.body; - if (!(body instanceof FormData)) throw new Error('expected a multipart upload'); - uploadedNames.push((body.get('appFile') as File).name); - return new Response(JSON.stringify({ app_id: 'APP42' }), { status: 200 }); - }; - const installApp = vi.fn(async () => undefined); - const deployment = createWebDriverDeploymentRuntime({ - provider: 'testmu', - uploadApp: createTestMuUploadApp({ - clientVersion: '0.0.0-test', - username: 'user', - accessKey: 'key', - }), - findSessionForDevice: () => activeSession(installApp), - }); - - await deployment.deployMaterializedApp( - iosDevice, - { - artifact: { - archivePath, - installablePath, - uploadPath: archivePath, - cleanup: async () => {}, - }, - }, - new AbortController().signal, - ); - - expect(uploadedNames).toEqual(['App.app.zip']); - expect(installApp).toHaveBeenCalledWith('lt://APP42', expect.any(AbortSignal)); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } -}); - function activeSession( installApp: (appPath: string, signal?: AbortSignal) => Promise, ): WebDriverProviderSession { diff --git a/scripts/check-provider-plugin.mjs b/scripts/check-provider-plugin.mjs new file mode 100644 index 0000000000..4ab7e5d21d --- /dev/null +++ b/scripts/check-provider-plugin.mjs @@ -0,0 +1,130 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import crypto from 'node:crypto'; +import fs from 'node:fs/promises'; +import http from 'node:http'; +import os from 'node:os'; +import path from 'node:path'; +import { promisify } from 'node:util'; +import { pathToFileURL } from 'node:url'; + +const exec = promisify(execFile); +const root = path.resolve(import.meta.dirname, '..'); +const plugin = path.resolve(root, process.argv[2]); +const fixture = await import(pathToFileURL(path.join(plugin, 'test/package-smoke.mjs')).href); +const scratch = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-plugin-package-')); +const consumer = path.join(scratch, 'consumer'); +const home = path.join(scratch, 'home'); +const installation = crypto.randomUUID(); +const project = path.join(home, 'plugins', installation); +let rejectCredentials = true; +const requests = []; +const server = http.createServer((request, response) => { + requests.push(request.url); + const result = fixture.respond(request.url, rejectCredentials); + response.writeHead(result.status ?? 200, { 'content-type': 'application/json' }); + response.end(JSON.stringify(result.body)); +}); +await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); +const endpoint = `http://127.0.0.1:${server.address().port}`; +async function run(command, args, cwd) { + return await exec(command, args, { cwd, maxBuffer: 16 * 1024 * 1024, timeout: 120_000 }); +} +async function pack(directory) { + const { stdout } = await run( + 'npm', + ['pack', '--ignore-scripts', '--json', '--pack-destination', scratch], + directory, + ); + return path.join(scratch, JSON.parse(stdout)[0].filename); +} +try { + await run('pnpm', ['build'], plugin); + const coreTarball = await pack(root); + const pluginTarball = await pack(plugin); + for (const directory of [consumer, project]) { + await fs.mkdir(directory, { recursive: true }); + await fs.writeFile(path.join(directory, 'package.json'), '{"private":true,"type":"module"}'); + } + await run( + 'npm', + ['install', '--ignore-scripts', '--no-audit', '--no-fund', coreTarball], + consumer, + ); + await run( + 'npm', + ['install', '--ignore-scripts', '--no-audit', '--no-fund', pluginTarball], + project, + ); + const manifest = JSON.parse(await fs.readFile(path.join(plugin, 'package.json'), 'utf8')); + await assert.rejects(fs.stat(path.join(project, 'node_modules', 'agent-device')), { + code: 'ENOENT', + }); + await fs.writeFile( + path.join(home, 'config.json'), + JSON.stringify({ plugins: { [manifest.name]: { installation } } }), + ); + const core = path.join(consumer, 'node_modules', 'agent-device'); + const coreManifest = JSON.parse(await fs.readFile(path.join(core, 'package.json'), 'utf8')); + const bin = path.resolve(core, coreManifest.bin['agent-device']); + const env = { + ...process.env, + ...fixture.environment(endpoint), + AGENT_DEVICE_HOME: home, + AGENT_DEVICE_NO_UPDATE_NOTIFIER: '1', + }; + const preload = path.join(scratch, 'fetch-fixture.mjs'); + await fs.writeFile( + preload, + `const originalFetch = globalThis.fetch; +const redirects = ${JSON.stringify(fixture.fetchRedirects ?? [])}; +globalThis.fetch = (input, init) => { + let url = String(input); + for (const prefix of redirects) if (url.startsWith(prefix)) url = ${JSON.stringify(endpoint)} + url.slice(prefix.length); + if (new URL(url).hostname !== '127.0.0.1') throw new Error('Unexpected external request: ' + url); + return originalFetch(url, init); +};`, + ); + const command = async (args) => { + try { + return await exec(process.execPath, ['--import', preload, bin, ...args, '--json'], { + cwd: consumer, + env, + timeout: 30_000, + }); + } catch (error) { + if (typeof error.stdout !== 'string') throw error; + return error; + } + }; + const listed = await command(['plugins', 'list']); + assert.equal(listed.code, undefined, listed.stderr); + const rejected = await command([ + 'connect', + manifest.agentDevicePlugin.provider, + ...fixture.args, + '--state-dir', + path.join(scratch, 'state'), + ]); + assert.equal( + JSON.parse(rejected.stdout).error.code, + 'UNAUTHORIZED', + rejected.stdout + rejected.stderr, + ); + rejectCredentials = false; + const connected = await command([ + 'connect', + manifest.agentDevicePlugin.provider, + ...fixture.args, + '--state-dir', + path.join(scratch, 'state'), + ]); + assert.equal(connected.code, undefined, connected.stdout + connected.stderr); + assert.ok(requests.length > 0); + console.log( + `Packed ${manifest.name}: isolated install, host-recognized errors, and CLI connect passed.`, + ); +} finally { + await new Promise((resolve) => server.close(resolve)); + await fs.rm(scratch, { recursive: true, force: true }); +} diff --git a/src/__tests__/cloud-connect-profile.test.ts b/src/__tests__/cloud-connect-profile.test.ts index 5a0a3cd070..5d91afcec1 100644 --- a/src/__tests__/cloud-connect-profile.test.ts +++ b/src/__tests__/cloud-connect-profile.test.ts @@ -71,37 +71,24 @@ beforeEach(() => { }, app: { status: 'verified', reference: options.app }, } - : options.provider === 'testmu' - ? { - provider: 'testmu', - service: 'TestMu AI', - verificationMessage: 'Credentials, virtual device, and uploaded app verified.', - device: { - status: 'verified', - name: options.deviceName, - platform: options.platform, - osVersion: options.osVersion, - }, - app: { status: 'verified', reference: options.app }, - } - : { - provider: 'aws-device-farm', - service: 'AWS Device Farm', - verificationMessage: 'Credentials, project, and device verified.', - project: { name: 'Agent Device', reference: options.projectArn }, - device: { - status: 'verified', - name: 'iPhone 15', - reference: options.deviceArn, - platform: options.platform, - osVersion: '17', - }, - app: { - status: 'missing', - message: - 'No app upload is attached; AWS Device Farm does not support install after allocation.', - }, + : { + provider: 'aws-device-farm', + service: 'AWS Device Farm', + verificationMessage: 'Credentials, project, and device verified.', + project: { name: 'Agent Device', reference: options.projectArn }, + device: { + status: 'verified', + name: 'iPhone 15', + reference: options.deviceArn, + platform: options.platform, + osVersion: '17', + }, + app: { + status: 'missing', + message: + 'No app upload is attached; AWS Device Farm does not support install after allocation.', }, + }, ); }); diff --git a/src/__tests__/cloud-connect-testmu.test.ts b/src/__tests__/cloud-connect-testmu.test.ts index ea4b2f5dfa..c3a34412e1 100644 --- a/src/__tests__/cloud-connect-testmu.test.ts +++ b/src/__tests__/cloud-connect-testmu.test.ts @@ -7,24 +7,64 @@ import { readActiveConnectionState, type RemoteConnectionState, } from '../remote/remote-connection-state.ts'; -import { resolveCloudWebDriverConnectProfile } from '../cli/connection/cloud-webdriver-profile.ts'; +import { resolveCloudWebDriverConnectProfile as resolveBuiltinProfile } from '../cli/connection/cloud-webdriver-profile.ts'; import { AppError } from '@agent-device/kernel/errors'; -import { providerWebDriver } from '../provider-webdriver.ts'; +import { verifyTestMuConnection } from '../../packages/provider-testmu/src/testmu-connection-verification.ts'; +import testMuPlugin from '../../packages/provider-testmu/src/plugin.ts'; +import { createPluginHost } from '../plugins/host.ts'; +import { persistAndResolveGeneratedProfile } from '../cli/connection/generated-config.ts'; +import { selectPlugin, pluginHome } from '../plugins/plugin.fixtures.ts'; +import { installedPlugins } from '../plugins/store.ts'; +import manifest from '../../packages/provider-testmu/package.json' with { type: 'json' }; +import type { CliFlags } from '@agent-device/contracts/command'; +import type { PluginConnection } from '../plugins/connection.ts'; import { mkdtempForTestSync } from './test-utils/tmp-dir.ts'; import { connectWithGeneratedProviderProfile } from './test-utils/connect-command.ts'; -vi.mock('../provider-webdriver.ts', () => ({ - providerWebDriver: { verifyConnection: vi.fn() }, +vi.mock('../../packages/provider-testmu/src/testmu-connection-verification.ts', () => ({ + verifyTestMuConnection: vi.fn(), })); +vi.mock('../plugins/load.ts', () => ({ + withPluginConnection: async ( + _provider: string, + env: NodeJS.ProcessEnv, + runConnection: (connection: PluginConnection) => Promise, + ) => { + const registration = testMuPlugin(createPluginHost(env, undefined)); + return await runConnection(registration.connection); + }, +})); +function resolveCloudWebDriverConnectProfile(options: { + provider: 'testmu' | 'browserstack' | 'aws-device-farm'; + flags: CliFlags; + stateDir: string; + cwd: string; + env?: NodeJS.ProcessEnv; +}) { + if (options.provider !== 'testmu') + return resolveBuiltinProfile({ ...options, provider: options.provider }); + const resolved = testMuPlugin(createPluginHost(options.env ?? {}, undefined)).connection.resolve( + options, + ); + return persistAndResolveGeneratedProfile({ ...options, ...resolved }); +} afterEach(() => { vi.clearAllMocks(); vi.unstubAllEnvs(); }); -const mockedVerifyWebDriverConnection = vi.mocked(providerWebDriver.verifyConnection); +const mockedVerifyWebDriverConnection = vi.mocked(verifyTestMuConnection); beforeEach(() => { + const { home, env } = pluginHome(); + selectPlugin(home, manifest.name, 'testmu', 'export default () => {};'); + const [plugin] = installedPlugins(env); + const manifestPath = path.join(plugin!.directory, 'package.json'); + const declared = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + declared.agentDevicePlugin.connection = manifest.agentDevicePlugin.connection; + fs.writeFileSync(manifestPath, JSON.stringify(declared)); + vi.stubEnv('AGENT_DEVICE_HOME', home); mockedVerifyWebDriverConnection.mockImplementation(async (options) => { assert.equal(options.provider, 'testmu'); return { @@ -69,6 +109,8 @@ test('connect testmu generates a local provider profile and verifies the virtual deviceName: 'iPhone 16', osVersion: '18.0', app: 'lt://APP1', + deviceType: 'virtual', + apiEndpoint: undefined, }); const state = readRequiredActiveState(stateDir); assert.equal(state.tenant, 'testmu'); @@ -121,7 +163,7 @@ test('connect canonicalizes an upper-case app scheme and refuses an empty app id (error: unknown) => error instanceof AppError && error.code === 'INVALID_ARGS' && - error.message.includes(`--provider-app ${app} is not a valid`), + error.message.includes('valid'), ); } } finally { @@ -223,6 +265,7 @@ test('connect testmu stores and verifies the real-device pool', async () => { osVersion: '18', app: 'lt://APP1', deviceType: 'real', + apiEndpoint: undefined, }); const state = readRequiredActiveState(stateDir); const generated = readGeneratedConfig(state.remoteConfigPath); diff --git a/src/__tests__/testmu-upload.test.ts b/src/__tests__/testmu-upload.test.ts new file mode 100644 index 0000000000..aadc1a172a --- /dev/null +++ b/src/__tests__/testmu-upload.test.ts @@ -0,0 +1,83 @@ +import { promises as fs } from 'node:fs'; +import path from 'node:path'; +import { afterEach, expect, test, vi } from 'vitest'; +import { createCloudWebDriverCapabilities } from '../../packages/provider-webdriver/src/capabilities.ts'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { createTestMuUploadApp } from '../../packages/provider-testmu/src/testmu.ts'; +import { mkdtempForTest } from '../../packages/provider-webdriver/src/tmp-dir.fixtures.ts'; +import { createWebDriverDeploymentRuntime } from '../../packages/provider-webdriver/src/runtime-deployment.ts'; +import type { WebDriverProviderSession } from '../../packages/provider-webdriver/src/runtime-session.ts'; + +const device: DeviceInfo = { + platform: 'android', + id: 'webdriver:stale', + name: 'Stale WebDriver device', + kind: 'device', + target: 'mobile', + booted: true, +}; + +const iosDevice: DeviceInfo = { ...device, platform: 'apple', id: 'webdriver:ios' }; +const realFetch = globalThis.fetch; + +afterEach(() => { + globalThis.fetch = realFetch; +}); + +test('TestMu uploads the zipped simulator build that install-from-source extracted', async () => { + const tempDir = await mkdtempForTest('agent-device-materialized-upload-'); + try { + const archivePath = path.join(tempDir, 'App.app.zip'); + const installablePath = path.join(tempDir, 'extracted', 'App.app'); + await fs.writeFile(archivePath, 'zip bytes'); + await fs.mkdir(installablePath, { recursive: true }); + const uploadedNames: unknown[] = []; + globalThis.fetch = async (_input, init) => { + const body = init?.body; + if (!(body instanceof FormData)) throw new Error('expected a multipart upload'); + uploadedNames.push((body.get('appFile') as File).name); + return new Response(JSON.stringify({ app_id: 'APP42' }), { status: 200 }); + }; + const installApp = vi.fn(async () => undefined); + const deployment = createWebDriverDeploymentRuntime({ + provider: 'testmu', + uploadApp: createTestMuUploadApp({ + clientVersion: '0.0.0-test', + username: 'user', + accessKey: 'key', + }), + findSessionForDevice: () => activeSession(installApp), + }); + + await deployment.deployMaterializedApp( + iosDevice, + { + artifact: { + archivePath, + installablePath, + uploadPath: archivePath, + cleanup: async () => {}, + }, + }, + new AbortController().signal, + ); + + expect(uploadedNames).toEqual(['App.app.zip']); + expect(installApp).toHaveBeenCalledWith('lt://APP42', expect.any(AbortSignal)); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +}); + +function activeSession( + installApp: (appPath: string, signal?: AbortSignal) => Promise, +): WebDriverProviderSession { + return { + capabilities: createCloudWebDriverCapabilities({ + provider: 'webdriver-test', + platform: 'android', + }), + client: { installApp }, + prepared: {}, + } as unknown as WebDriverProviderSession; +} diff --git a/src/cli/connection/cloud-webdriver-profile.ts b/src/cli/connection/cloud-webdriver-profile.ts index a9e0f61db2..4ac3153d49 100644 --- a/src/cli/connection/cloud-webdriver-profile.ts +++ b/src/cli/connection/cloud-webdriver-profile.ts @@ -4,10 +4,7 @@ import { readAwsDeviceFarmRegionFromArn, type CloudWebDriverKnownProviderName, } from '@agent-device/provider-webdriver'; -import { - isBrowserStackAppReference, - isTestMuAppReference, -} from '@agent-device/provider-webdriver/providers'; +import { isBrowserStackAppReference } from '@agent-device/provider-webdriver/providers'; import { rejectRefusedProviderProfileFields } from '@agent-device/contracts/provider-profile-fields'; import type { RemoteConfigProfile } from '../../remote/remote-config-schema.ts'; import { AppError } from '@agent-device/kernel/errors'; @@ -85,10 +82,6 @@ const CLOUD_WEBDRIVER_CONNECT_PROFILE_BUILDERS: readonly { provider: CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, buildProfileFields: awsDeviceFarmProfileFields, }, - { - provider: CLOUD_WEBDRIVER_PROVIDERS.testMu, - buildProfileFields: testMuProfileFields, - }, ]; function requireConnectProfileBuilder( @@ -122,15 +115,6 @@ const BROWSERSTACK_HUB_PROFILE: HubProviderProfile = { appHint: '', }; -const TESTMU_HUB_PROFILE: HubProviderProfile = { - command: 'connect testmu', - label: 'TestMu AI', - credentialEnv: ['LT_USERNAME', 'LT_ACCESS_KEY'], - appScheme: 'lt://', - isAppReference: isTestMuAppReference, - appHint: '', -}; - function browserStackProfileFields(options: { flags: CliFlags; env?: EnvMap; @@ -139,17 +123,6 @@ function browserStackProfileFields(options: { return hubProviderProfileFields(BROWSERSTACK_HUB_PROFILE, options); } -function testMuProfileFields(options: { - flags: CliFlags; - env?: EnvMap; - cwd: string; -}): RemoteConfigProfile { - return { - ...hubProviderProfileFields(TESTMU_HUB_PROFILE, options), - providerDeviceType: options.flags.providerDeviceType, - }; -} - function hubProviderProfileFields( hub: HubProviderProfile, options: { flags: CliFlags; env?: EnvMap; cwd: string }, diff --git a/src/cli/connection/connect-provider-adapters.ts b/src/cli/connection/connect-provider-adapters.ts index b7312fca31..2b09748915 100644 --- a/src/cli/connection/connect-provider-adapters.ts +++ b/src/cli/connection/connect-provider-adapters.ts @@ -106,7 +106,7 @@ export async function resolveConnectProviderProfile(options: { remoteConfig: resolved.resolvedPath, }, remoteConfigPath: resolved.resolvedPath, - ...(isConnectProviderName(leaseProvider) ? { provider: leaseProvider } : {}), + ...(isConnectProviderName(leaseProvider, env) ? { provider: leaseProvider } : {}), }; } const provider = diff --git a/src/cli/connection/provider-policy.ts b/src/cli/connection/provider-policy.ts index d7fd72b2ee..c2b89edf55 100644 --- a/src/cli/connection/provider-policy.ts +++ b/src/cli/connection/provider-policy.ts @@ -1,3 +1,4 @@ +import type { ConnectionProviderCapabilities } from '@agent-device/contracts/remote'; import { CLOUD_WEBDRIVER_PROVIDERS, isCloudWebDriverProviderName, @@ -6,32 +7,20 @@ import { import { pluginConnectionCapabilities, pluginConnectionNames } from '../../plugins/connection.ts'; export type DirectDeviceConnectProvider = CloudWebDriverKnownProviderName | 'limrun'; -export const BUILTIN_CONNECT_PROVIDERS = [ - 'cloud', - 'proxy', - CLOUD_WEBDRIVER_PROVIDERS.browserStack, - CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, - 'limrun', -] as const; +export { RESERVED_PLUGIN_PROVIDERS as BUILTIN_CONNECT_PROVIDERS } from '../../plugins/manifest.ts'; +import { RESERVED_PLUGIN_PROVIDERS as BUILTIN_CONNECT_PROVIDERS } from '../../plugins/manifest.ts'; export type BuiltinConnectProvider = (typeof BUILTIN_CONNECT_PROVIDERS)[number]; export type ConnectProvider = BuiltinConnectProvider | (string & {}); -export type ConnectionProviderCapabilities = { - leaseKind: 'proxy' | 'direct-device-provider' | 'remote-provider'; - requiresAppAttachment: boolean; - requiresRemoteDaemon: boolean; - supportsArtifacts: boolean; - supportsDeferredAppSelection: boolean; - supportsDirectPortReverse: boolean; - usesCloudWebDriverLease: boolean; -}; - -export function isConnectProviderName(value: string | undefined): value is ConnectProvider { +export function isConnectProviderName( + value: string | undefined, + env: NodeJS.ProcessEnv = process.env, +): value is ConnectProvider { return ( value === 'cloud' || value === 'proxy' || isDirectDeviceConnectProvider(value) || - pluginConnectionCapabilities(value) !== undefined + pluginConnectionCapabilities(value, env) !== undefined ); } diff --git a/src/commands/schema/cli-help-topics.test.ts b/src/commands/schema/cli-help-topics.test.ts index c7f47c5157..7d983f2fcc 100644 --- a/src/commands/schema/cli-help-topics.test.ts +++ b/src/commands/schema/cli-help-topics.test.ts @@ -440,6 +440,7 @@ test('usageForCommand resolves remote help topic', async () => { help, /Limrun, BrowserStack, AWS Device Farm, and TestMu AI through local provider profiles/, ); + assert.match(help, /plugins add @agent-device\/testmu/); assert.match(help, /TestMu AI uses LT_USERNAME and LT_ACCESS_KEY/); const testMuFlow = help.slice( help.indexOf('TestMu AI virtual-device flow'), diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index b78177c4b3..7fbc38d6eb 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -619,6 +619,7 @@ Cloud profile flow: agent-device disconnect TestMu AI virtual-device flow (emulators and simulators): + agent-device plugins add @agent-device/testmu LT_USERNAME=... LT_ACCESS_KEY=... agent-device connect testmu --platform ios --device "iPhone 16" --provider-os-version 18.0 --provider-app lt://APP-id agent-device open com.example.app diff --git a/src/commands/schema/command-overrides.ts b/src/commands/schema/command-overrides.ts index 368b50a72d..5c50607014 100644 --- a/src/commands/schema/command-overrides.ts +++ b/src/commands/schema/command-overrides.ts @@ -75,7 +75,7 @@ const SCHEMA_ONLY_CLI_COMMAND_SCHEMAS = { 'Configure remote access without allocating a device. Direct providers validate credentials/resources before saving state and print the exact device/app preparation needed before open. AGENT_DEVICE_CLOUD_BASE_URL is the bridge/control-plane API origin; use AGENT_DEVICE_DAEMON_AUTH_TOKEN=adc_live_... for CI/service-token automation.', }, usageOverride: - 'connect [cloud|proxy|limrun|browserstack|aws-device-farm|testmu] [--remote-config ] [--daemon-base-url ] [--tenant ] [--run-id ] [--lease-id ] [--lease-backend ] [--force] [--no-login]', + 'connect [provider] [--remote-config ] [--daemon-base-url ] [--tenant ] [--run-id ] [--lease-id ] [--lease-backend ] [--force] [--no-login]', usageFlags: [], listUsageOverride: 'connect', positionalArgs: ['provider?'], diff --git a/src/plugins/connection.ts b/src/plugins/connection.ts index 734b14dc2d..329effd3b1 100644 --- a/src/plugins/connection.ts +++ b/src/plugins/connection.ts @@ -2,7 +2,7 @@ import type { CliFlags } from '@agent-device/contracts/command'; import type { ProviderConnectionVerification } from '@agent-device/contracts/remote'; import type { EnvMap } from '@agent-device/kernel/source-value'; import type { RemoteConfigProfile } from '../remote/remote-config-schema.ts'; -import type { ConnectionProviderCapabilities } from '../cli/connection/provider-policy.ts'; +import type { ConnectionProviderCapabilities } from '@agent-device/contracts/remote'; import { installedPlugins } from './store.ts'; export type PluginConnection = Readonly<{ diff --git a/src/plugins/host.ts b/src/plugins/host.ts index 66ce4e70db..98961a65c2 100644 --- a/src/plugins/host.ts +++ b/src/plugins/host.ts @@ -1,35 +1,8 @@ -import { register } from 'node:module'; -import { pathToFileURL } from 'node:url'; import { AppError } from '@agent-device/kernel/errors'; import { execFailureDetails, runCmd } from '@agent-device/host-kit/command'; -import { findProjectRoot, readVersion } from '@agent-device/host-kit/version'; +import { readVersion } from '@agent-device/host-kit/version'; import type { ProviderPluginHost } from '../sdk/plugins.ts'; -let hostBound = false; - -export function bindPluginHost(): void { - if (hostBound) return; - const root = pathToFileURL(`${findProjectRoot()}/`).href; - const source = import.meta.url.startsWith(`${root}src/`); - register( - `data:text/javascript,${encodeURIComponent(` - let host; - export function initialize(data) { host = data; } - export async function resolve(specifier, context, nextResolve) { - if (specifier === 'agent-device' && host.source) { - return { url: new URL('src/sdk/index.ts', host.root).href, shortCircuit: true }; - } - if (specifier === 'agent-device' || specifier.startsWith('agent-device/')) { - return nextResolve(specifier, { ...context, parentURL: new URL('plugin-host.mjs', host.root).href }); - } - return nextResolve(specifier, context); - } - `)}`, - { parentURL: import.meta.url, data: { root, source } }, - ); - hostBound = true; -} - export function createPluginHost( env: NodeJS.ProcessEnv, options: Record | undefined, diff --git a/src/plugins/load.test.ts b/src/plugins/load.test.ts index 5fd08fde5a..4026c62558 100644 --- a/src/plugins/load.test.ts +++ b/src/plugins/load.test.ts @@ -71,10 +71,17 @@ test('connection callbacks run from the installed plugin and always release runt "connection: { resolve: () => ({ profile: { leaseProvider: 'example', platform: 'android' } }), verify: async () => { throw host.createError('COMMAND_FAILED', 'verification failed'); } }, platformModule:", ); selectPlugin(home, 'example', 'example', source); + selectPlugin(home, 'unrelated', 'unrelated', 'throw new Error("unrelated plugin evaluated");'); const profile = await withPluginConnection( 'example', env, - async (connection) => await connection.resolve({ flags: {}, stateDir: home, cwd: home, env }), + async (connection) => + await connection.resolve({ + flags: { json: false, help: false, version: false }, + stateDir: home, + cwd: home, + env, + }), ); assert.equal(profile.profile.platform, 'android'); assert.ok(fs.existsSync(marker)); @@ -83,7 +90,8 @@ test('connection callbacks run from the installed plugin and always release runt withPluginConnection( 'example', env, - async (connection) => await connection.verify({ flags: {}, env }), + async (connection) => + await connection.verify({ flags: { json: false, help: false, version: false }, env }), ), { code: 'COMMAND_FAILED' }, ); diff --git a/src/plugins/load.ts b/src/plugins/load.ts index 0b91f944e5..a5f48e6690 100644 --- a/src/plugins/load.ts +++ b/src/plugins/load.ts @@ -4,10 +4,13 @@ import type { ProviderDeviceRuntime } from '@agent-device/contracts/device'; import type { PlatformRuntimeProviderModule } from '@agent-device/contracts/platform-runtime-operations'; import type { ProviderPluginHost } from '../sdk/plugins.ts'; import { installedPlugins } from './store.ts'; -import { resolvePluginEntry, assertUniquePluginProviders } from './manifest.ts'; -import { bindPluginHost, createPluginHost } from './host.ts'; +import { + resolvePluginEntry, + assertUniquePluginProviders, + RESERVED_PLUGIN_PROVIDERS, +} from './manifest.ts'; +import { createPluginHost } from './host.ts'; import type { PluginConnection } from './connection.ts'; -import { BUILTIN_CONNECT_PROVIDERS } from '../cli/connection/provider-policy.ts'; import type { WebDriverPluginOptions } from '../sdk/plugin-webdriver.ts'; type ProviderPluginRegistration = Readonly<{ @@ -19,13 +22,15 @@ type ProviderPluginRegistration = Readonly<{ export async function loadProviderPlugins( env: NodeJS.ProcessEnv, reservedProviders: readonly string[], + onlyProvider?: string, ): Promise { const plugins = installedPlugins(env); assertUniquePluginProviders(plugins, reservedProviders); const registrations: ProviderPluginRegistration[] = []; try { - for (const plugin of plugins) { - bindPluginHost(); + for (const plugin of plugins.filter( + (plugin) => onlyProvider === undefined || plugin.agentDevicePlugin.provider === onlyProvider, + )) { const module = await import( pathToFileURL(resolvePluginEntry(plugin.directory, plugin.agentDevicePlugin.entry)).href ); @@ -99,16 +104,16 @@ export async function loadProviderPlugins( export async function withPluginConnection( provider: string, env: NodeJS.ProcessEnv, - use: (connection: PluginConnection) => Promise, + runConnection: (connection: PluginConnection) => Promise, ): Promise { - const registrations = await loadProviderPlugins(env, BUILTIN_CONNECT_PROVIDERS); + const registrations = await loadProviderPlugins(env, RESERVED_PLUGIN_PROVIDERS, provider); try { const connection = registrations.find( (entry) => entry.runtime.provider === provider, )?.connection; if (!connection) throw new AppError('INVALID_ARGS', `Plugin does not register connect: ${provider}`); - return await use(connection); + return await runConnection(connection); } finally { await Promise.allSettled(registrations.map(async ({ runtime }) => await runtime.shutdown())); } diff --git a/src/plugins/manifest.ts b/src/plugins/manifest.ts index c30f6b113b..93fd645c12 100644 --- a/src/plugins/manifest.ts +++ b/src/plugins/manifest.ts @@ -1,7 +1,17 @@ import fs from 'node:fs'; import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; -import type { ConnectionProviderCapabilities } from '../cli/connection/provider-policy.ts'; +import type { ConnectionProviderCapabilities } from '@agent-device/contracts/remote'; + +import { CLOUD_WEBDRIVER_PROVIDERS } from '@agent-device/provider-webdriver/providers'; + +export const RESERVED_PLUGIN_PROVIDERS = [ + 'cloud', + 'proxy', + CLOUD_WEBDRIVER_PROVIDERS.browserStack, + CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, + 'limrun', +] as const; const PROVIDER_PLUGIN_API_VERSION = 1; type PluginManifest = { @@ -19,7 +29,7 @@ export function assertUniquePluginProviders( plugins: readonly PluginManifest[], reserved: readonly string[], ): void { - const providers = new Set(reserved); + const providers = new Set([...RESERVED_PLUGIN_PROVIDERS, ...reserved]); for (const plugin of plugins) { const provider = plugin.agentDevicePlugin.provider; if (providers.has(provider)) diff --git a/src/plugins/store.test.ts b/src/plugins/store.test.ts index 32fcef3d2c..d10648a187 100644 --- a/src/plugins/store.test.ts +++ b/src/plugins/store.test.ts @@ -20,7 +20,6 @@ function installFixture(name = packageName, apiVersion = 1, provider = 'example' assert.equal(argv[argv.indexOf('--prefix') + 1], options!.cwd); assert.ok(argv.includes('--global=false')); assert.ok(argv.includes('--ignore-scripts')); - assert.ok(argv.includes('--omit=peer')); assert.ok(argv.includes('--workspaces=false')); assert.ok(argv.includes('--package-lock=true')); writePlugin(options!.cwd!, name, apiVersion, provider); diff --git a/src/plugins/store.ts b/src/plugins/store.ts index 000653787c..4435a624cb 100644 --- a/src/plugins/store.ts +++ b/src/plugins/store.ts @@ -112,7 +112,6 @@ async function stagePlugin( [ 'install', '--ignore-scripts', - '--omit=peer', '--no-audit', '--no-fund', '--global=false', diff --git a/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts b/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts index efc4445842..95711ad821 100644 --- a/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts +++ b/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts @@ -23,6 +23,10 @@ import { type StartedCloudWebDriverTestServer, } from './cloud-webdriver-test-server.ts'; +import testMuPlugin from '../../../packages/provider-testmu/src/plugin.ts'; +import { createPluginHost } from '../../../src/plugins/host.ts'; +import { createCloudWebDriverRuntime } from '@agent-device/provider-webdriver/plugin'; + const CLIENT_VERSION = '0.20.3-test'; test('BrowserStack facade prepares capabilities, uploads apps, and returns artifacts', async () => { @@ -187,21 +191,23 @@ test('AWS Device Farm facade rejects device features it does not read at session test('TestMu facade routes a real-device session to the real pool and its upload API', async () => { await withProviderScenarioResource(FakeCloudProviderServer.start, async (server) => { - const provider = createProviderWebDriver({ + const registration = testMuPlugin( + createPluginHost( + { + LT_USERNAME: 'user', + LT_ACCESS_KEY: 'key', + TESTMU_WEBDRIVER_ENDPOINT: `${server.url}/wd/hub/`, + TESTMU_APP_UPLOAD_ENDPOINT: `${server.url}/lt/upload/virtualDevice`, + TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT: `${server.url}/lt/upload/realDevice`, + }, + undefined, + ), + ); + const runtime = createCloudWebDriverRuntime({ + ...registration.webDriver, clientVersion: CLIENT_VERSION, - runHostCommand: unexpectedHostCommand, }); - const runtime = runtimeFor( - provider.createDefaultRuntimes({ - LT_USERNAME: 'user', - LT_ACCESS_KEY: 'key', - TESTMU_WEBDRIVER_ENDPOINT: `${server.url}/wd/hub/`, - TESTMU_APP_UPLOAD_ENDPOINT: `${server.url}/lt/upload/virtualDevice`, - TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT: `${server.url}/lt/upload/realDevice`, - }), - CLOUD_WEBDRIVER_PROVIDERS.testMu, - ); - const lease = makeLease(CLOUD_WEBDRIVER_PROVIDERS.testMu); + const lease = makeLease('testmu'); try { await runtime.leaseLifecycle.allocate?.(lease, { flags: { diff --git a/website/docs/docs/plugins.md b/website/docs/docs/plugins.md index 24618c8451..2cb55a16d5 100644 --- a/website/docs/docs/plugins.md +++ b/website/docs/docs/plugins.md @@ -52,4 +52,4 @@ To support `agent-device connect example`, declare `agentDevicePlugin.connection Return `connection` alongside the runtime or WebDriver options. Its `resolve({ flags, env, cwd, stateDir })` callback validates provider flags and returns `{ profile, extraFlags? }`; `profile.leaseProvider` must match the manifest. Core supplies connection identity, session defaults, Metro settings, and persists the profile. Its async `verify({ flags, env })` callback returns the provider verification result. These callbacks run without allocating a device and their temporary runtimes are shut down afterwards. -Bundle the plugin implementation and ship ready-to-run ESM: installation disables lifecycle scripts and omits peer dependencies. Runtime imports of public `agent-device` exports resolve against the running host, so plugins can use its `AppError` without installing a second core package. Import types with `import type` to keep them out of the runtime dependency graph. +Bundle the plugin implementation and ship ready-to-run ESM: installation disables lifecycle scripts. Bundle shared implementation helpers with the plugin; `AppError` carries a shared brand so core preserves its code and details across package copies. Import types with `import type` to keep them out of the runtime dependency graph. diff --git a/website/docs/docs/testmu.md b/website/docs/docs/testmu.md index 87912239ae..26124f191f 100644 --- a/website/docs/docs/testmu.md +++ b/website/docs/docs/testmu.md @@ -10,6 +10,16 @@ WebDriver sessions, and real devices you select with `--provider-device-type rea fronts both pools; agent-device selects the pool with `isRealMobile` and defaults to the virtual-device pool. +## Install the plugin + +```bash +agent-device plugins add @agent-device/testmu +``` + +The provider is an optional npm package installed under `AGENT_DEVICE_HOME`. +After adding or updating it, close sessions and run `agent-device daemon stop` +with the state directory you use; the next device command loads the selected plugin. + ## Credentials and connection Set TestMu AI credentials in a non-interactive environment. These are the same variables every From 11759063c9fff8f4d3adb25516302a6de66b6099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 17:41:16 +0200 Subject: [PATCH 19/57] chore(gates): declare TestMu plugin build dependencies and analysis entrypoints --- .fallowrc.json | 4 ++- package.json | 2 +- packages/provider-testmu/package.json | 38 +++++++++++++++++++++ pnpm-lock.yaml | 15 ++++++++ scripts/layering/model.ts | 1 + scripts/layering/package-boundaries.test.ts | 1 + scripts/layering/package-boundaries.ts | 7 ++-- 7 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 packages/provider-testmu/package.json diff --git a/.fallowrc.json b/.fallowrc.json index b03734c356..3acc520b02 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -2,6 +2,8 @@ "$schema": "https://raw.githubusercontent.com/fallow-rs/fallow/main/schema.json", "entry": [ "tsdown.config.ts", + "packages/provider-testmu/tsdown.config.ts", + "packages/provider-testmu/src/plugin.ts", "vitest.mutation.config.ts", "src/sdk/index.ts", "src/sdk/io.ts", @@ -384,7 +386,7 @@ }, { "comment": "TestMu session preparation reads these off the lazy loadTestMuDeviceFeatures() import in packages/provider-webdriver/src/provider-definitions.ts, which keeps the package entry's eager closure unchanged; Fallow cannot connect the dynamic member reads.", - "file": "packages/provider-webdriver/src/testmu-device-features.ts", + "file": "packages/provider-testmu/src/testmu-device-features.ts", "exports": [ "buildTestMuDeviceFeatureCapabilities", "readTestMuDeviceFeatureFields", diff --git a/package.json b/package.json index 78e463ea72..9a76d4088e 100644 --- a/package.json +++ b/package.json @@ -179,7 +179,7 @@ "check:unit": "pnpm test:unit && pnpm check:tmpdir-leaks && pnpm test:smoke", "check": "pnpm check:tooling && pnpm check:fallow && pnpm check:unit", "prepack": "pnpm check:mcp-metadata && pnpm package:npm", - "typecheck": "tsc -b packages/xml packages/kernel packages/contracts packages/device-selection packages/host-kit packages/capture-kit packages/managed-allocation packages/provision-kit packages/platform-apple packages/platform-android packages/platform-harmonyos packages/platform-vega packages/platform-linux packages/platform-web packages/ad-script packages/selectors packages/command-registry packages/session-journal packages/ad-replay packages/maestro packages/replay-port packages/replay-test packages/provider-webdriver packages/provider-limrun && tsc -p tsconfig.json && tsc -p examples/sdk/tsconfig.json", + "typecheck": "tsc -b packages/xml packages/kernel packages/contracts packages/device-selection packages/host-kit packages/capture-kit packages/managed-allocation packages/provision-kit packages/platform-apple packages/platform-android packages/platform-harmonyos packages/platform-vega packages/platform-linux packages/platform-web packages/ad-script packages/selectors packages/command-registry packages/session-journal packages/ad-replay packages/maestro packages/replay-port packages/replay-test packages/provider-webdriver packages/provider-limrun packages/provider-testmu && tsc -p tsconfig.json && tsc -p examples/sdk/tsconfig.json", "test-app:install": "pnpm install --dir examples/test-app", "test-app:start": "pnpm --dir examples/test-app start", "test-app:ios": "pnpm --dir examples/test-app ios", diff --git a/packages/provider-testmu/package.json b/packages/provider-testmu/package.json new file mode 100644 index 0000000000..0d192501eb --- /dev/null +++ b/packages/provider-testmu/package.json @@ -0,0 +1,38 @@ +{ + "name": "@agent-device/testmu", + "version": "0.1.0", + "type": "module", + "description": "TestMu AI real and virtual mobile device plugin for agent-device.", + "files": [ + "dist" + ], + "scripts": { + "build": "tsdown --config tsdown.config.ts", + "prepack": "pnpm build" + }, + "exports": { + ".": { + "import": "./dist/plugin.mjs" + } + }, + "devDependencies": { + "@agent-device/contracts": "workspace:*", + "@agent-device/kernel": "workspace:*", + "@agent-device/provider-webdriver": "workspace:*", + "agent-device": "workspace:*" + }, + "agentDevicePlugin": { + "apiVersion": 1, + "provider": "testmu", + "entry": "./dist/plugin.mjs", + "connection": { + "leaseKind": "direct-device-provider", + "requiresAppAttachment": false, + "requiresRemoteDaemon": false, + "supportsArtifacts": true, + "supportsDeferredAppSelection": false, + "supportsDirectPortReverse": false, + "usesCloudWebDriverLease": true + } + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bf1452f083..acc2325db1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -470,6 +470,21 @@ importers: specifier: ^0.49.3 version: 0.49.3(supports-color@7.2.0) + packages/provider-testmu: + devDependencies: + '@agent-device/contracts': + specifier: workspace:* + version: link:../contracts + '@agent-device/kernel': + specifier: workspace:* + version: link:../kernel + '@agent-device/provider-webdriver': + specifier: workspace:* + version: link:../provider-webdriver + agent-device: + specifier: workspace:* + version: link:../.. + packages/provider-webdriver: dependencies: '@agent-device/capture-kit': diff --git a/scripts/layering/model.ts b/scripts/layering/model.ts index 17f5590ad1..cc6f0cdaa8 100644 --- a/scripts/layering/model.ts +++ b/scripts/layering/model.ts @@ -108,6 +108,7 @@ export const UNRANKED_ZONES: ReadonlySet = new Set([ ...PLATFORMS.map((family) => `platform-${family}`), 'provider-webdriver', 'provider-limrun', + 'provider-testmu', 'xml', ]); diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index 3f41c6c817..29f42d92f3 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -755,6 +755,7 @@ test('the real tree parses, declares, and passes R11', () => { assert.ok(providerWebDriverPackage, 'provider-webdriver package must exist'); assert.deepEqual([...providerWebDriverPackage.exportTargets.keys()].sort(), [ '@agent-device/provider-webdriver', + '@agent-device/provider-webdriver/plugin', '@agent-device/provider-webdriver/providers', ]); assert.deepEqual([...providerWebDriverPackage.workspaceDependencies].sort(), [ diff --git a/scripts/layering/package-boundaries.ts b/scripts/layering/package-boundaries.ts index cb8c462c4d..5b9d288401 100644 --- a/scripts/layering/package-boundaries.ts +++ b/scripts/layering/package-boundaries.ts @@ -100,13 +100,14 @@ export function workspacePackagesFromManifests( const manifest = JSON.parse(manifests.get(manifestFile)!) as { name?: string; private?: boolean; - exports?: Record; + exports?: Record; + devDependencies?: Record; dependencies?: Record; }; if (!manifest.name) continue; const exportTargets = new Map(); for (const [subpath, target] of Object.entries(manifest.exports ?? {})) { - const targetFile = typeof target === 'string' ? target : target.default; + const targetFile = typeof target === 'string' ? target : (target.default ?? target.import); if (!targetFile) continue; exportTargets.set( path.posix.join(manifest.name, subpath), @@ -114,7 +115,7 @@ export function workspacePackagesFromManifests( ); } const workspaceDependencies = new Set( - Object.entries(manifest.dependencies ?? {}) + Object.entries({ ...manifest.dependencies, ...manifest.devDependencies }) .filter(([, range]) => range.startsWith('workspace:')) .map(([name]) => name), ); From d6bee90e1dfe3d2a8e862b6dcded2d5ec67018c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 18:06:31 +0200 Subject: [PATCH 20/57] fix: retain plugin tests through development-only package exports --- packages/provider-testmu/src/index.ts | 1 + scripts/check-provider-plugin.mjs | 11 ++ src/__tests__/cloud-connect-testmu.test.ts | 8 +- src/__tests__/testmu-upload.test.ts | 83 ---------- src/cli/connection/provider-policy.ts | 1 - src/plugins/connection.ts | 6 +- src/plugins/load.ts | 149 +++++++++++------- src/plugins/manifest.test.ts | 32 ++++ .../cloud-webdriver-provider-adapters.test.ts | 84 +++++++++- .../cloud-webdriver-test-server.ts | 12 +- 10 files changed, 234 insertions(+), 153 deletions(-) create mode 100644 packages/provider-testmu/src/index.ts delete mode 100644 src/__tests__/testmu-upload.test.ts diff --git a/packages/provider-testmu/src/index.ts b/packages/provider-testmu/src/index.ts new file mode 100644 index 0000000000..f00db85a53 --- /dev/null +++ b/packages/provider-testmu/src/index.ts @@ -0,0 +1 @@ +export { default } from './plugin.ts'; diff --git a/scripts/check-provider-plugin.mjs b/scripts/check-provider-plugin.mjs index 4ab7e5d21d..bc713e7332 100644 --- a/scripts/check-provider-plugin.mjs +++ b/scripts/check-provider-plugin.mjs @@ -31,6 +31,11 @@ async function run(command, args, cwd) { return await exec(command, args, { cwd, maxBuffer: 16 * 1024 * 1024, timeout: 120_000 }); } async function pack(directory) { + if (directory === plugin) { + const tarball = path.join(scratch, 'plugin.tgz'); + await run('pnpm', ['pack', '--out', tarball], directory); + return tarball; + } const { stdout } = await run( 'npm', ['pack', '--ignore-scripts', '--json', '--pack-destination', scratch], @@ -57,6 +62,12 @@ try { project, ); const manifest = JSON.parse(await fs.readFile(path.join(plugin, 'package.json'), 'utf8')); + const installed = JSON.parse( + await fs.readFile(path.join(project, 'node_modules', manifest.name, 'package.json'), 'utf8'), + ); + assert.deepEqual(installed.exports, { '.': { import: './dist/plugin.mjs' } }); + assert.equal(installed.dependencies, undefined); + assert.equal(installed.peerDependencies, undefined); await assert.rejects(fs.stat(path.join(project, 'node_modules', 'agent-device')), { code: 'ENOENT', }); diff --git a/src/__tests__/cloud-connect-testmu.test.ts b/src/__tests__/cloud-connect-testmu.test.ts index c3a34412e1..8730eedbbc 100644 --- a/src/__tests__/cloud-connect-testmu.test.ts +++ b/src/__tests__/cloud-connect-testmu.test.ts @@ -9,19 +9,19 @@ import { } from '../remote/remote-connection-state.ts'; import { resolveCloudWebDriverConnectProfile as resolveBuiltinProfile } from '../cli/connection/cloud-webdriver-profile.ts'; import { AppError } from '@agent-device/kernel/errors'; -import { verifyTestMuConnection } from '../../packages/provider-testmu/src/testmu-connection-verification.ts'; -import testMuPlugin from '../../packages/provider-testmu/src/plugin.ts'; +import { verifyTestMuConnection } from '@agent-device/testmu/connection-verification'; +import testMuPlugin from '@agent-device/testmu'; import { createPluginHost } from '../plugins/host.ts'; import { persistAndResolveGeneratedProfile } from '../cli/connection/generated-config.ts'; import { selectPlugin, pluginHome } from '../plugins/plugin.fixtures.ts'; import { installedPlugins } from '../plugins/store.ts'; -import manifest from '../../packages/provider-testmu/package.json' with { type: 'json' }; +import manifest from '@agent-device/testmu/package.json' with { type: 'json' }; import type { CliFlags } from '@agent-device/contracts/command'; import type { PluginConnection } from '../plugins/connection.ts'; import { mkdtempForTestSync } from './test-utils/tmp-dir.ts'; import { connectWithGeneratedProviderProfile } from './test-utils/connect-command.ts'; -vi.mock('../../packages/provider-testmu/src/testmu-connection-verification.ts', () => ({ +vi.mock('@agent-device/testmu/connection-verification', () => ({ verifyTestMuConnection: vi.fn(), })); vi.mock('../plugins/load.ts', () => ({ diff --git a/src/__tests__/testmu-upload.test.ts b/src/__tests__/testmu-upload.test.ts deleted file mode 100644 index aadc1a172a..0000000000 --- a/src/__tests__/testmu-upload.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { promises as fs } from 'node:fs'; -import path from 'node:path'; -import { afterEach, expect, test, vi } from 'vitest'; -import { createCloudWebDriverCapabilities } from '../../packages/provider-webdriver/src/capabilities.ts'; -import type { DeviceInfo } from '@agent-device/kernel/device'; -import { createTestMuUploadApp } from '../../packages/provider-testmu/src/testmu.ts'; -import { mkdtempForTest } from '../../packages/provider-webdriver/src/tmp-dir.fixtures.ts'; -import { createWebDriverDeploymentRuntime } from '../../packages/provider-webdriver/src/runtime-deployment.ts'; -import type { WebDriverProviderSession } from '../../packages/provider-webdriver/src/runtime-session.ts'; - -const device: DeviceInfo = { - platform: 'android', - id: 'webdriver:stale', - name: 'Stale WebDriver device', - kind: 'device', - target: 'mobile', - booted: true, -}; - -const iosDevice: DeviceInfo = { ...device, platform: 'apple', id: 'webdriver:ios' }; -const realFetch = globalThis.fetch; - -afterEach(() => { - globalThis.fetch = realFetch; -}); - -test('TestMu uploads the zipped simulator build that install-from-source extracted', async () => { - const tempDir = await mkdtempForTest('agent-device-materialized-upload-'); - try { - const archivePath = path.join(tempDir, 'App.app.zip'); - const installablePath = path.join(tempDir, 'extracted', 'App.app'); - await fs.writeFile(archivePath, 'zip bytes'); - await fs.mkdir(installablePath, { recursive: true }); - const uploadedNames: unknown[] = []; - globalThis.fetch = async (_input, init) => { - const body = init?.body; - if (!(body instanceof FormData)) throw new Error('expected a multipart upload'); - uploadedNames.push((body.get('appFile') as File).name); - return new Response(JSON.stringify({ app_id: 'APP42' }), { status: 200 }); - }; - const installApp = vi.fn(async () => undefined); - const deployment = createWebDriverDeploymentRuntime({ - provider: 'testmu', - uploadApp: createTestMuUploadApp({ - clientVersion: '0.0.0-test', - username: 'user', - accessKey: 'key', - }), - findSessionForDevice: () => activeSession(installApp), - }); - - await deployment.deployMaterializedApp( - iosDevice, - { - artifact: { - archivePath, - installablePath, - uploadPath: archivePath, - cleanup: async () => {}, - }, - }, - new AbortController().signal, - ); - - expect(uploadedNames).toEqual(['App.app.zip']); - expect(installApp).toHaveBeenCalledWith('lt://APP42', expect.any(AbortSignal)); - } finally { - await fs.rm(tempDir, { recursive: true, force: true }); - } -}); - -function activeSession( - installApp: (appPath: string, signal?: AbortSignal) => Promise, -): WebDriverProviderSession { - return { - capabilities: createCloudWebDriverCapabilities({ - provider: 'webdriver-test', - platform: 'android', - }), - client: { installApp }, - prepared: {}, - } as unknown as WebDriverProviderSession; -} diff --git a/src/cli/connection/provider-policy.ts b/src/cli/connection/provider-policy.ts index c2b89edf55..db4f872186 100644 --- a/src/cli/connection/provider-policy.ts +++ b/src/cli/connection/provider-policy.ts @@ -7,7 +7,6 @@ import { import { pluginConnectionCapabilities, pluginConnectionNames } from '../../plugins/connection.ts'; export type DirectDeviceConnectProvider = CloudWebDriverKnownProviderName | 'limrun'; -export { RESERVED_PLUGIN_PROVIDERS as BUILTIN_CONNECT_PROVIDERS } from '../../plugins/manifest.ts'; import { RESERVED_PLUGIN_PROVIDERS as BUILTIN_CONNECT_PROVIDERS } from '../../plugins/manifest.ts'; export type BuiltinConnectProvider = (typeof BUILTIN_CONNECT_PROVIDERS)[number]; export type ConnectProvider = BuiltinConnectProvider | (string & {}); diff --git a/src/plugins/connection.ts b/src/plugins/connection.ts index 329effd3b1..295f8f8cbe 100644 --- a/src/plugins/connection.ts +++ b/src/plugins/connection.ts @@ -1,8 +1,10 @@ import type { CliFlags } from '@agent-device/contracts/command'; -import type { ProviderConnectionVerification } from '@agent-device/contracts/remote'; +import type { + ConnectionProviderCapabilities, + ProviderConnectionVerification, +} from '@agent-device/contracts/remote'; import type { EnvMap } from '@agent-device/kernel/source-value'; import type { RemoteConfigProfile } from '../remote/remote-config-schema.ts'; -import type { ConnectionProviderCapabilities } from '@agent-device/contracts/remote'; import { installedPlugins } from './store.ts'; export type PluginConnection = Readonly<{ diff --git a/src/plugins/load.ts b/src/plugins/load.ts index a5f48e6690..cf5dbfa845 100644 --- a/src/plugins/load.ts +++ b/src/plugins/load.ts @@ -31,68 +31,9 @@ export async function loadProviderPlugins( for (const plugin of plugins.filter( (plugin) => onlyProvider === undefined || plugin.agentDevicePlugin.provider === onlyProvider, )) { - const module = await import( - pathToFileURL(resolvePluginEntry(plugin.directory, plugin.agentDevicePlugin.entry)).href - ); - if (typeof module.default !== 'function') - throw new AppError('INVALID_ARGS', `Plugin must export a default factory: ${plugin.name}`); - const host = createPluginHost(env, plugin.selection.options); - const result = await ( - module.default as ( - host: ProviderPluginHost, - ) => - | ProviderPluginRegistration - | { webDriver: WebDriverPluginOptions; connection?: PluginConnection } - | Promise< - | ProviderPluginRegistration - | { webDriver: WebDriverPluginOptions; connection?: PluginConnection } - > - )(host); - let registration: ProviderPluginRegistration; - if (result && 'webDriver' in result) { - if (result.webDriver?.provider !== plugin.agentDevicePlugin.provider) { - throw new AppError( - 'INVALID_ARGS', - `WebDriver plugin provider does not match its declaration: ${plugin.name}`, - ); - } - const { createCloudWebDriverRuntime } = - await import('@agent-device/provider-webdriver/plugin'); - const runtime = createCloudWebDriverRuntime({ - ...result.webDriver, - clientVersion: host.clientVersion, - }); - registration = { - runtime, - platformModule: runtime.platformRuntimeModule, - connection: result.connection, - }; - } else registration = result; - if (!registration?.runtime || typeof registration.runtime.shutdown !== 'function') { - throw new AppError('INVALID_ARGS', `Plugin must return a provider runtime: ${plugin.name}`); - } + const registration = await instantiateProviderPlugin(plugin, env); registrations.push(registration); - if ( - registration.runtime.provider !== plugin.agentDevicePlugin.provider || - typeof registration.runtime.ownsDevice !== 'function' || - typeof registration.runtime.getInteractor !== 'function' || - typeof registration.runtime.deviceInventoryProvider !== 'function' || - !registration.runtime.leaseLifecycle || - typeof registration.runtime.leaseLifecycle !== 'object' || - registration.platformModule?.owner?.kind !== 'provider-runtime' || - registration.platformModule.owner.provider !== registration.runtime.provider || - typeof registration.platformModule.owner.instance !== 'string' || - registration.platformModule.owner.instance.trim().length === 0 || - typeof registration.platformModule.loadRuntime !== 'function' || - (plugin.agentDevicePlugin.connection && - (typeof registration.connection?.resolve !== 'function' || - typeof registration.connection?.verify !== 'function')) - ) { - throw new AppError( - 'INVALID_ARGS', - `Plugin runtime owner does not match its declaration: ${plugin.name}`, - ); - } + validateProviderPlugin(registration, plugin); } return registrations; } catch (error) { @@ -118,3 +59,89 @@ export async function withPluginConnection( await Promise.allSettled(registrations.map(async ({ runtime }) => await runtime.shutdown())); } } + +async function instantiateProviderPlugin( + plugin: ReturnType[number], + env: NodeJS.ProcessEnv, +): Promise { + const module = await import( + pathToFileURL(resolvePluginEntry(plugin.directory, plugin.agentDevicePlugin.entry)).href + ); + if (typeof module.default !== 'function') + throw new AppError('INVALID_ARGS', `Plugin must export a default factory: ${plugin.name}`); + const host = createPluginHost(env, plugin.selection.options); + const result = await ( + module.default as ( + host: ProviderPluginHost, + ) => + | ProviderPluginRegistration + | { webDriver: WebDriverPluginOptions; connection?: PluginConnection } + | Promise< + | ProviderPluginRegistration + | { webDriver: WebDriverPluginOptions; connection?: PluginConnection } + > + )(host); + let registration: ProviderPluginRegistration; + if (result && 'webDriver' in result) { + if (result.webDriver?.provider !== plugin.agentDevicePlugin.provider) { + throw new AppError( + 'INVALID_ARGS', + `WebDriver plugin provider does not match its declaration: ${plugin.name}`, + ); + } + const { createCloudWebDriverRuntime } = await import('@agent-device/provider-webdriver/plugin'); + const runtime = createCloudWebDriverRuntime({ + ...result.webDriver, + clientVersion: host.clientVersion, + }); + registration = { + runtime, + platformModule: runtime.platformRuntimeModule, + connection: result.connection, + }; + } else registration = result; + if (!registration?.runtime || typeof registration.runtime.shutdown !== 'function') { + throw new AppError('INVALID_ARGS', `Plugin must return a provider runtime: ${plugin.name}`); + } + return registration; +} + +function validateProviderPlugin( + registration: ProviderPluginRegistration, + plugin: ReturnType[number], +): void { + if ( + registration.runtime.provider !== plugin.agentDevicePlugin.provider || + !hasProviderFacets(registration.runtime) || + !matchesRuntimeOwner(registration) || + (plugin.agentDevicePlugin.connection && + (typeof registration.connection?.resolve !== 'function' || + typeof registration.connection?.verify !== 'function')) + ) { + throw new AppError( + 'INVALID_ARGS', + `Plugin runtime owner does not match its declaration: ${plugin.name}`, + ); + } +} + +function matchesRuntimeOwner(registration: ProviderPluginRegistration): boolean { + const owner = registration.platformModule?.owner; + return ( + owner?.kind === 'provider-runtime' && + owner.provider === registration.runtime.provider && + typeof owner.instance === 'string' && + owner.instance.trim().length > 0 && + typeof registration.platformModule.loadRuntime === 'function' + ); +} + +function hasProviderFacets(runtime: ProviderDeviceRuntime): boolean { + return ( + runtime.leaseLifecycle !== null && + typeof runtime.leaseLifecycle === 'object' && + (['ownsDevice', 'getInteractor', 'deviceInventoryProvider'] as const).every( + (method) => typeof runtime[method] === 'function', + ) + ); +} diff --git a/src/plugins/manifest.test.ts b/src/plugins/manifest.test.ts index a7763c828a..a414e7bd94 100644 --- a/src/plugins/manifest.test.ts +++ b/src/plugins/manifest.test.ts @@ -33,3 +33,35 @@ test('manifest refuses traversal and symlink entries outside the installed packa fs.writeFileSync(path.join(directory, 'package.json'), JSON.stringify(manifest)); assert.throws(() => readPluginManifest(directory), { code: 'INVALID_ARGS' }); }); + +test('connection metadata admits local providers and rejects malformed or remote policies', () => { + const { home } = pluginHome(); + const directory = writePlugin(home); + const file = path.join(directory, 'package.json'); + const manifest = JSON.parse(fs.readFileSync(file, 'utf8')); + const connection = { + leaseKind: 'direct-device-provider', + requiresAppAttachment: true, + requiresRemoteDaemon: false, + supportsArtifacts: true, + supportsDeferredAppSelection: false, + supportsDirectPortReverse: false, + usesCloudWebDriverLease: true, + }; + const read = (policy: unknown) => { + manifest.agentDevicePlugin.connection = policy; + fs.writeFileSync(file, JSON.stringify(manifest)); + return readPluginManifest(directory); + }; + assert.deepEqual(read(connection).agentDevicePlugin.connection, connection); + for (const invalid of [ + null, + {}, + { ...connection, leaseKind: 'remote-daemon' }, + { ...connection, requiresRemoteDaemon: true }, + { ...connection, supportsArtifacts: 'true' }, + { ...connection, supportsDeferredAppSelection: undefined }, + ]) { + assert.throws(() => read(invalid), { code: 'INVALID_ARGS' }); + } +}); diff --git a/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts b/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts index 95711ad821..1e268577d8 100644 --- a/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts +++ b/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts @@ -13,6 +13,7 @@ import type { LeaseLifecycleContext, ProviderDeviceRuntime, } from '@agent-device/contracts/device'; +import type { PlatformRuntimeHost } from '@agent-device/contracts/platform-runtime-operations'; import type { CloudArtifactsResult } from '@agent-device/contracts/observability'; import { withProviderScenarioResource, withProviderScenarioTempDir } from './harness.ts'; import { @@ -23,7 +24,7 @@ import { type StartedCloudWebDriverTestServer, } from './cloud-webdriver-test-server.ts'; -import testMuPlugin from '../../../packages/provider-testmu/src/plugin.ts'; +import testMuPlugin from '@agent-device/testmu'; import { createPluginHost } from '../../../src/plugins/host.ts'; import { createCloudWebDriverRuntime } from '@agent-device/provider-webdriver/plugin'; @@ -237,6 +238,87 @@ test('TestMu facade routes a real-device session to the real pool and its upload }); }, 15_000); +test('TestMu uploads the materializer-selected simulator archive through the plugin runtime', async () => { + await withProviderScenarioResource(FakeCloudProviderServer.start, async (server) => { + await withProviderScenarioTempDir('agent-device-testmu-materialized-', async (tempDir) => { + const archivePath = path.join(tempDir, 'App.app.zip'); + const installablePath = path.join(tempDir, 'extracted', 'App.app'); + fs.writeFileSync(archivePath, 'zip bytes'); + fs.mkdirSync(installablePath, { recursive: true }); + const registration = testMuPlugin( + createPluginHost( + { + LT_USERNAME: 'user', + LT_ACCESS_KEY: 'key', + TESTMU_WEBDRIVER_ENDPOINT: `${server.url}/wd/hub/`, + TESTMU_APP_UPLOAD_ENDPOINT: `${server.url}/lt/upload/virtualDevice`, + }, + undefined, + ), + ); + const runtime = createCloudWebDriverRuntime({ + ...registration.webDriver, + clientVersion: CLIENT_VERSION, + }); + const lease = makeLease('testmu'); + try { + await runtime.leaseLifecycle.allocate?.(lease, { + flags: { + platform: 'ios', + device: 'iPhone 16', + providerOsVersion: '18.0', + providerApp: 'lt://APP1', + }, + }); + const [device] = + (await runtime.deviceInventoryProvider({ + leaseProvider: 'testmu', + leaseId: lease.leaseId, + platform: 'ios', + })) ?? []; + assert.ok(device); + const owner = await runtime.platformRuntimeModule.loadRuntime({ + snapshot: { + presentIosAcquisition: async () => { + throw new Error('Unexpected snapshot'); + }, + }, + } as unknown as PlatformRuntimeHost); + const binding = await owner.bind({ + device, + intent: { kind: 'ordinary' }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, + }); + try { + assert.ok(binding.operations.deployMaterializedApp); + await binding.operations.deployMaterializedApp({ + artifact: { + archivePath, + installablePath, + uploadPath: archivePath, + cleanup: async () => {}, + }, + }); + } finally { + await binding[Symbol.asyncDispose](); + } + const upload = server.calls.find((call) => call.path === '/lt/upload/virtualDevice'); + assert.deepEqual((upload?.body as { filenames?: string[] })?.filenames, ['App.app.zip']); + const install = server.calls.find((call) => + call.path.endsWith('/appium/device/install_app'), + ); + assert.deepEqual(install?.body, { appPath: 'lt://VIRTUAL1' }); + } finally { + await runtime.shutdown(); + } + }); + }); +}, 15_000); + test('BrowserStack facade rejects the device type at session preparation', async () => { await withProviderScenarioResource(FakeCloudProviderServer.start, async (server) => { const provider = createProviderWebDriver({ diff --git a/test/integration/provider-scenarios/cloud-webdriver-test-server.ts b/test/integration/provider-scenarios/cloud-webdriver-test-server.ts index e2b166100f..a1265a6c75 100644 --- a/test/integration/provider-scenarios/cloud-webdriver-test-server.ts +++ b/test/integration/provider-scenarios/cloud-webdriver-test-server.ts @@ -97,9 +97,19 @@ async function neverAnsweredResponse(signal: AbortSignal | undefined): Promise { if (!request.body) return {}; + const multipart = request.headers.get('content-type')?.startsWith('multipart/form-data') + ? await request.clone().formData() + : undefined; const buffer = Buffer.from(await request.arrayBuffer()); if (request.headers.get('content-type')?.startsWith('multipart/form-data')) { - return { body: { multipartBytes: buffer.length } }; + return { + body: { + multipartBytes: buffer.length, + filenames: [...multipart!.values()] + .filter((value): value is File => value instanceof File) + .map((value) => value.name), + }, + }; } const text = buffer.toString('utf8'); return text ? { body: JSON.parse(text) as unknown } : {}; From 48fef3bfc25888355259d27da4e9e8eb19a6ad0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 18:06:31 +0200 Subject: [PATCH 21/57] chore(gates): track TestMu packaging fixtures and workspace test dependencies --- .fallowrc.json | 9 +++++- package.json | 3 +- packages/provider-testmu/package.json | 17 +++++++++-- pnpm-lock.yaml | 3 ++ scripts/layering/package-boundaries.test.ts | 33 +++++++++++++++++++++ 5 files changed, 61 insertions(+), 4 deletions(-) diff --git a/.fallowrc.json b/.fallowrc.json index 3acc520b02..65a1465080 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -2,6 +2,8 @@ "$schema": "https://raw.githubusercontent.com/fallow-rs/fallow/main/schema.json", "entry": [ "tsdown.config.ts", + "scripts/check-provider-plugin.mjs", + "packages/provider-testmu/test/package-smoke.mjs", "packages/provider-testmu/tsdown.config.ts", "packages/provider-testmu/src/plugin.ts", "vitest.mutation.config.ts", @@ -69,6 +71,11 @@ "@arethetypeswrong/cli" ], "ignoreExports": [ + { + "comment": "The standalone packed-install harness loads this fixture from the selected package path and reads its request scenarios dynamically.", + "file": "packages/provider-*/test/package-smoke.mjs", + "exports": ["*"] + }, { "comment": "Android perf mechanics are selected through the lazy platform host so importing agent-device does not eagerly load adb mechanics. Fallow cannot follow the dynamic property read in src/platform-runtime-perf-host.ts.", "file": "packages/platform-android/src/perf.ts", @@ -290,7 +297,7 @@ }, { "comment": "Tool config default exports, loaded by the tool rather than imported.", - "file": "{oxlint.config.ts,tsdown.config.ts,vitest.mutation.config.ts,website/rspress.config.ts}", + "file": "{oxlint.config.ts,tsdown.config.ts,packages/provider-*/tsdown.config.ts,vitest.mutation.config.ts,website/rspress.config.ts}", "exports": ["default"] }, { diff --git a/package.json b/package.json index 9a76d4088e..77ba729ba7 100644 --- a/package.json +++ b/package.json @@ -343,6 +343,7 @@ "vite": "^8.2.1", "vitest": "^4.1.11", "yaml": "^2.9.0", - "yauzl": "^3.4.0" + "yauzl": "^3.4.0", + "@agent-device/testmu": "workspace:*" } } diff --git a/packages/provider-testmu/package.json b/packages/provider-testmu/package.json index 0d192501eb..2430d0f468 100644 --- a/packages/provider-testmu/package.json +++ b/packages/provider-testmu/package.json @@ -12,8 +12,14 @@ }, "exports": { ".": { - "import": "./dist/plugin.mjs" - } + "types": "./src/index.ts", + "default": "./src/index.ts" + }, + "./connection-verification": { + "types": "./src/testmu-connection-verification.ts", + "default": "./src/testmu-connection-verification.ts" + }, + "./package.json": "./package.json" }, "devDependencies": { "@agent-device/contracts": "workspace:*", @@ -34,5 +40,12 @@ "supportsDirectPortReverse": false, "usesCloudWebDriverLease": true } + }, + "publishConfig": { + "exports": { + ".": { + "import": "./dist/plugin.mjs" + } + } } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index acc2325db1..0ae0a8b811 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -86,6 +86,9 @@ importers: '@agent-device/session-journal': specifier: workspace:* version: link:packages/session-journal + '@agent-device/testmu': + specifier: workspace:* + version: link:packages/provider-testmu '@agent-device/xml': specifier: workspace:* version: link:packages/xml diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index 29f42d92f3..1f44a1537a 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -16,6 +16,7 @@ import { checkPackageInternalSites, checkRootSites, readWorkspacePackages, + workspacePackagesFromManifests, rootExternalDependencyRanges, rootWorkspaceDependencyNames, specifierSites, @@ -148,6 +149,38 @@ test('readWorkspacePackages reads tracked manifests only', () => { ); }); +test('published ESM plugins declare bundled workspace build dependencies without runtime dependencies', () => { + const [plugin] = workspacePackagesFromManifests( + new Map([ + [ + 'packages/provider-example/package.json', + JSON.stringify({ + name: '@agent-device/example', + exports: { '.': { import: './dist/plugin.mjs' } }, + devDependencies: { '@agent-device/kernel': 'workspace:*', tsdown: '^0.21.0' }, + }), + ], + ]), + ); + assert.ok(plugin); + assert.equal( + plugin.exportTargets.get('@agent-device/example'), + 'packages/provider-example/dist/plugin.mjs', + ); + assert.deepEqual([...plugin.workspaceDependencies], ['@agent-device/kernel']); + assert.equal(plugin.externalDependencies.size, 0); + const sites = specifierSites( + 'packages/provider-example/src/plugin.ts', + "import { AppError } from '@agent-device/kernel/errors';", + ); + assert.deepEqual(checkPackageInternalSites(plugin, sites, [plugin, kernel]), []); + const undeclared = { ...plugin, workspaceDependencies: new Set() }; + const violations = checkPackageInternalSites(undeclared, sites, [undeclared, kernel]); + assert.equal(violations.length, 1); + assert.equal(violations[0]?.rule, 'R11 package-boundaries'); + assert.match(violations[0]?.message ?? '', /without declaring/); +}); + test('every workspace package façade names its exports explicitly (no bare `export *`)', () => { // #1574 built a hand-maintained pin table (`facade-symbols.ts`, 816 symbols across every // workspace-package façade) plus a ~200-line star-chain resolver (`readFacadeExports`) whose From c33719ac03e21fa7338c2169e5c432859b185cca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 18:10:57 +0200 Subject: [PATCH 22/57] chore(gates): smoke the optional packed WebDriver plugin SDK --- test/integration/installed-package-metro.test.ts | 2 ++ website/docs/docs/plugins.md | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/integration/installed-package-metro.test.ts b/test/integration/installed-package-metro.test.ts index afe2aa0bb1..0fa4f350c6 100644 --- a/test/integration/installed-package-metro.test.ts +++ b/test/integration/installed-package-metro.test.ts @@ -327,6 +327,7 @@ test('installed package exposes Node APIs and packaged companion tunnel entrypoi }, './metro': (mod) => mod.buildBundleUrl('https://public.example.test', 'ios'), './plugins': (mod) => Object.keys(mod).length === 0, + './plugins/webdriver': (mod) => Object.keys(mod).length === 0, './remote-config': (mod) => typeof mod, './selectors': (mod) => mod.isSelectorToken('||') && typeof mod.parseSelectorChain === 'function', @@ -402,6 +403,7 @@ test('installed package exposes Node APIs and packaged companion tunnel entrypoi './io': 'function', './limrun': 'limrun', './plugins': true, + './plugins/webdriver': true, // Type-only subpath: resolving the module from the packed exports map is // the entire runtime check. './remote-config': 'object', diff --git a/website/docs/docs/plugins.md b/website/docs/docs/plugins.md index 2cb55a16d5..62024ec3bc 100644 --- a/website/docs/docs/plugins.md +++ b/website/docs/docs/plugins.md @@ -31,8 +31,7 @@ Use `agent-device` as a development dependency. The default factory accepts `Pro Keep initialization prompt and free of network I/O or device allocation; a stalled factory blocks startup. Load platform mechanics through `platformModule.loadRuntime` and perform remote work in request-bound operations. A failing factory cleans up its own resources; core shuts down previously returned runtimes if another plugin fails. -Incompatible contract changes require a new API version. Plugins cannot replace bundled providers or register arbitrary commands. Installing a package does not add `connect `; provider-specific connect adapters need separate support. Limrun, BrowserStack, and AWS Device Farm remain bundled. - +Incompatible contract changes require a new API version. Plugins cannot replace bundled providers or register arbitrary commands. Plugins can add `connect ` through the connection callbacks described below. Limrun, BrowserStack, and AWS Device Farm remain bundled. For an Appium or WebDriver service, return `{ webDriver: options }` instead of building an engine. `WebDriverPluginOptions` is available through the type-only `agent-device/plugins/webdriver` import. Core supplies the client version and creates the shared runtime. Provider callbacks prepare sessions, upload apps, and retrieve artifacts. From ec53bcb1d5179fce58ff4ef53a447a503a54e139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 18:25:32 +0200 Subject: [PATCH 23/57] perf: load the shared WebDriver engine on demand --- packages/provider-webdriver/src/capabilities.ts | 12 ++++++++++++ packages/provider-webdriver/src/plugin.ts | 11 +++++++++-- packages/provider-webdriver/src/runtime-session.ts | 14 +------------- packages/provider-webdriver/src/runtime.ts | 2 +- src/cli.ts | 5 +++-- src/plugins/load.ts | 2 +- .../cloud-webdriver-provider-adapters.test.ts | 4 ++-- 7 files changed, 29 insertions(+), 21 deletions(-) diff --git a/packages/provider-webdriver/src/capabilities.ts b/packages/provider-webdriver/src/capabilities.ts index c390be3355..f2b1ab345d 100644 --- a/packages/provider-webdriver/src/capabilities.ts +++ b/packages/provider-webdriver/src/capabilities.ts @@ -166,3 +166,15 @@ function applyCapabilityOverrides( } return next; } + +export function buildCloudWebDriverBaseCapabilities( + platform: CloudWebDriverPlatform, + deviceName: string, + configured: Record = {}, +): Record { + return { + platformName: platform === 'ios' ? 'iOS' : 'Android', + 'appium:deviceName': deviceName, + ...configured, + }; +} diff --git a/packages/provider-webdriver/src/plugin.ts b/packages/provider-webdriver/src/plugin.ts index 827c72969a..a8436cc2c4 100644 --- a/packages/provider-webdriver/src/plugin.ts +++ b/packages/provider-webdriver/src/plugin.ts @@ -1,4 +1,11 @@ -export { createCloudWebDriverRuntime } from './runtime.ts'; +import type { CloudWebDriverRuntimeOptions, CloudWebDriverRuntime } from './runtime.ts'; + +export async function createCloudWebDriverRuntime( + options: CloudWebDriverRuntimeOptions, +): Promise { + const runtime = await import('./runtime.ts'); + return runtime.createCloudWebDriverRuntime(options); +} export type { CloudWebDriverRuntimeOptions, CloudWebDriverPlatform, @@ -16,4 +23,4 @@ export { resolveHubAppReference, } from './webdriver-utils.ts'; export { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; -export { buildCloudWebDriverBaseCapabilities } from './runtime-session.ts'; +export { buildCloudWebDriverBaseCapabilities } from './capabilities.ts'; diff --git a/packages/provider-webdriver/src/runtime-session.ts b/packages/provider-webdriver/src/runtime-session.ts index 7274789f6c..4425f761b6 100644 --- a/packages/provider-webdriver/src/runtime-session.ts +++ b/packages/provider-webdriver/src/runtime-session.ts @@ -8,6 +8,7 @@ import { AppError, errorMessage } from '@agent-device/kernel/errors'; import { unavailableCloudArtifactsResult } from './artifact-results.ts'; import { createCloudWebDriverCapabilities, + buildCloudWebDriverBaseCapabilities, type CloudWebDriverProviderCapabilities, } from './capabilities.ts'; import { WebDriverClient, type WebDriverSession } from './webdriver-client.ts'; @@ -17,7 +18,6 @@ import { snapshotBackendForPlatform } from './runtime-helpers.ts'; import { releaseOnFailure } from './webdriver-utils.ts'; import type { CloudWebDriverBaseSession, - CloudWebDriverPlatform, CloudWebDriverPreparedSession, CloudWebDriverRuntimeOptions, } from './runtime.ts'; @@ -293,18 +293,6 @@ export class WebDriverSessionManager { } } -export function buildCloudWebDriverBaseCapabilities( - platform: CloudWebDriverPlatform, - deviceName: string, - configured: Record = {}, -): Record { - return { - platformName: platform === 'ios' ? 'iOS' : 'Android', - 'appium:deviceName': deviceName, - ...configured, - }; -} - /** * The transport gave up on `POST /session`; the provider may still finish it, * and nothing here can learn that session's id — so the error names the lease diff --git a/packages/provider-webdriver/src/runtime.ts b/packages/provider-webdriver/src/runtime.ts index 4b94ac2d72..4f400bf28c 100644 --- a/packages/provider-webdriver/src/runtime.ts +++ b/packages/provider-webdriver/src/runtime.ts @@ -111,7 +111,7 @@ export function createCloudWebDriverRuntime( return new CloudWebDriverRuntimeImplementation(options); } -export { buildCloudWebDriverBaseCapabilities } from './runtime-session.ts'; +export { buildCloudWebDriverBaseCapabilities } from './capabilities.ts'; /** Public façade: provider wiring stays small while session/deployment mechanics stay focused. */ class CloudWebDriverRuntimeImplementation implements CloudWebDriverRuntime { diff --git a/src/cli.ts b/src/cli.ts index d1dbe15ff5..e5c7ce568a 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -25,10 +25,8 @@ import { type AgentDeviceClientConfig, type AgentDeviceDaemonTransport, } from './agent-device-client.ts'; -import { materializeRemoteConnectionForCommand } from './cli/commands/connection-runtime.ts'; import { tryRunClientBackedCommand } from './cli/commands/router.ts'; import { runAgentCdpCommand } from './cli/commands/agent-cdp.ts'; -import { runReactDevtoolsCommand } from './cli/commands/react-devtools.ts'; import { readCliBatchStepsJson } from './commands/batch/batch-steps.ts'; import { createRequestId, @@ -353,6 +351,7 @@ async function resolveRunContextOrExit( } async function runReactDevtoolsCli(ctx: CliRunContext, deps: CliDeps): Promise { + const { runReactDevtoolsCommand } = await import('./cli/commands/react-devtools.ts'); const { daemonAuthToken, ...directRequestFlags } = ctx.effectiveFlags; return await runReactDevtoolsCommand(ctx.positionals, { flags: { @@ -408,6 +407,8 @@ async function resolveRemoteContext(ctx: CliRunContext, deps: CliDeps): Promise< const materializationClient = createAgentDeviceClient(buildClientConfig(ctx), { transport: createClientDaemonTransport(deps.sendToDaemon), }); + const { materializeRemoteConnectionForCommand } = + await import('./cli/commands/connection-runtime.ts'); const materialized = await materializeRemoteConnectionForCommand({ command: ctx.command, flags: ctx.effectiveFlags, diff --git a/src/plugins/load.ts b/src/plugins/load.ts index cf5dbfa845..b9e93d471b 100644 --- a/src/plugins/load.ts +++ b/src/plugins/load.ts @@ -90,7 +90,7 @@ async function instantiateProviderPlugin( ); } const { createCloudWebDriverRuntime } = await import('@agent-device/provider-webdriver/plugin'); - const runtime = createCloudWebDriverRuntime({ + const runtime = await createCloudWebDriverRuntime({ ...result.webDriver, clientVersion: host.clientVersion, }); diff --git a/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts b/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts index 1e268577d8..3076f37803 100644 --- a/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts +++ b/test/integration/provider-scenarios/cloud-webdriver-provider-adapters.test.ts @@ -204,7 +204,7 @@ test('TestMu facade routes a real-device session to the real pool and its upload undefined, ), ); - const runtime = createCloudWebDriverRuntime({ + const runtime = await createCloudWebDriverRuntime({ ...registration.webDriver, clientVersion: CLIENT_VERSION, }); @@ -256,7 +256,7 @@ test('TestMu uploads the materializer-selected simulator archive through the plu undefined, ), ); - const runtime = createCloudWebDriverRuntime({ + const runtime = await createCloudWebDriverRuntime({ ...registration.webDriver, clientVersion: CLIENT_VERSION, }); From 3632ba8c28368d5bc23b384e1557eb410478f0b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 18:26:56 +0200 Subject: [PATCH 24/57] chore(gates): bound the optional plugin SDK declarations --- test/integration/installed-package-metro.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/integration/installed-package-metro.test.ts b/test/integration/installed-package-metro.test.ts index 0fa4f350c6..d024f9d2aa 100644 --- a/test/integration/installed-package-metro.test.ts +++ b/test/integration/installed-package-metro.test.ts @@ -228,6 +228,10 @@ test('installed package exposes Node APIs and packaged companion tunnel entrypoi const pluginTypes = fs.readFileSync(path.join(installedPackageRoot, 'dist/src/plugins.d.ts')); assert.ok(pluginTypes.length < 1024); + const webDriverPluginTypes = fs.readFileSync( + path.join(installedPackageRoot, 'dist/src/plugins/webdriver.d.ts'), + ); + assert.ok(webDriverPluginTypes.length < 5120); assert.match(pluginTypes.toString(), /export \{ ProviderPluginHost \}/); metroPort = await listenOnLoopback(metroServer); t.after(async () => { From 915e57200d29829ff23b76b487d83b7ddb23831d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 3 Oct 2026 18:37:03 +0200 Subject: [PATCH 25/57] fix: consolidate TestMu plugin helper imports --- packages/provider-testmu/src/testmu.ts | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/packages/provider-testmu/src/testmu.ts b/packages/provider-testmu/src/testmu.ts index 8055ee2dd7..9bc3924229 100644 --- a/packages/provider-testmu/src/testmu.ts +++ b/packages/provider-testmu/src/testmu.ts @@ -1,18 +1,8 @@ -import fs from 'node:fs/promises'; -import path from 'node:path'; -import type { CloudArtifact, CloudArtifactsResult } from '@agent-device/contracts/observability'; -import type { ProviderDeviceType } from '@agent-device/contracts/remote'; -import type { - CloudWebDriverPlatform, - CloudWebDriverUploadApp, -} from '@agent-device/provider-webdriver/plugin'; -import { AppError } from '@agent-device/kernel/errors'; -import { isTestMuAppReference } from './providers.ts'; import { + type CloudWebDriverPlatform, + type CloudWebDriverUploadApp, cloudArtifactsReadyOrPending, urlArtifactFromDetails, -} from '@agent-device/provider-webdriver/plugin'; -import { appendUrlPath, appFileUploadForm, asRecord, @@ -21,6 +11,12 @@ import { postHubAppUpload, resolveHubAppReference, } from '@agent-device/provider-webdriver/plugin'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { CloudArtifact, CloudArtifactsResult } from '@agent-device/contracts/observability'; +import type { ProviderDeviceType } from '@agent-device/contracts/remote'; +import { AppError } from '@agent-device/kernel/errors'; +import { isTestMuAppReference } from './providers.ts'; /** * TestMu session, upload, and artifact mechanics. Loaded on demand by the provider definition; From 0cc1af432d453b78a19bb0a87081f91f07c4a69c Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 14:51:08 +0530 Subject: [PATCH 26/57] fix(testmu): trim credentials and log URLs, canonicalize lt:// everywhere Whitespace-only LT_USERNAME/LT_ACCESS_KEY now fail as missing in the runtime, matching connect. Verification of a hand-authored profile, upload responses, and app-list ids share one case-insensitive lt:// canonicalization, and a blank console_logs_url falls back to device_logs_url. Co-Authored-By: Claude Opus 5.5 --- packages/provider-testmu/src/connection.ts | 11 ++-- packages/provider-testmu/src/plugin.test.ts | 69 +++++++++++++++++++++ packages/provider-testmu/src/plugin.ts | 2 +- packages/provider-testmu/src/providers.ts | 5 ++ packages/provider-testmu/src/testmu.test.ts | 31 +++++---- packages/provider-testmu/src/testmu.ts | 19 +++--- 6 files changed, 109 insertions(+), 28 deletions(-) create mode 100644 packages/provider-testmu/src/plugin.test.ts diff --git a/packages/provider-testmu/src/connection.ts b/packages/provider-testmu/src/connection.ts index 10c09648d7..d5e0caf904 100644 --- a/packages/provider-testmu/src/connection.ts +++ b/packages/provider-testmu/src/connection.ts @@ -6,7 +6,7 @@ import { rejectRefusedProviderProfileFields, type ProviderProfileFieldDeclaration, } from '@agent-device/contracts/provider-profile-fields'; -import { isTestMuAppReference } from './providers.ts'; +import { canonicalTestMuAppReference, isTestMuAppReference } from './providers.ts'; import { verifyTestMuConnection } from './testmu-connection-verification.ts'; import { readTestMuDeviceFeatureFields, readTestMuDeviceType } from './testmu-device-features.ts'; @@ -25,9 +25,10 @@ export function createTestMuConnection( required(host.env.LT_ACCESS_KEY, 'LT_ACCESS_KEY'); if (flags.platform !== 'android' && flags.platform !== 'ios') throw host.createError('INVALID_ARGS', 'connect testmu requires --platform ios|android.'); - let app = required(flags.providerApp, '--provider-app '); - if (app.slice(0, 5).toLowerCase() === 'lt://') { - app = `lt://${app.slice(5)}`; + let app = canonicalTestMuAppReference( + required(flags.providerApp, '--provider-app '), + ); + if (app.startsWith('lt://')) { if (!isTestMuAppReference(app)) throw host.createError( 'INVALID_ARGS', @@ -67,7 +68,7 @@ export function createTestMuConnection( platform: flags.platform, deviceName: required(flags.device, '--device'), osVersion: required(flags.providerOsVersion, '--provider-os-version'), - app: required(flags.providerApp, '--provider-app'), + app: canonicalTestMuAppReference(required(flags.providerApp, '--provider-app')), deviceType: readTestMuDeviceType(flags), apiEndpoint: host.env.TESTMU_API_ENDPOINT, }, diff --git a/packages/provider-testmu/src/plugin.test.ts b/packages/provider-testmu/src/plugin.test.ts new file mode 100644 index 0000000000..2811237a32 --- /dev/null +++ b/packages/provider-testmu/src/plugin.test.ts @@ -0,0 +1,69 @@ +import assert from 'node:assert/strict'; +import { afterEach, test, vi } from 'vitest'; +import type { ProviderPluginHost } from 'agent-device/plugins'; +import { AppError } from '@agent-device/kernel/errors'; +import testMuPlugin from './plugin.ts'; +import { verifyTestMuConnection } from './testmu-connection-verification.ts'; + +vi.mock('./testmu-connection-verification.ts', () => ({ verifyTestMuConnection: vi.fn() })); + +const realFetch = globalThis.fetch; + +afterEach(() => { + globalThis.fetch = realFetch; + vi.clearAllMocks(); +}); + +function host(env: Record): ProviderPluginHost { + return { + env, + options: {}, + clientVersion: '0.0.0-test', + apple: {} as ProviderPluginHost['apple'], + createError: (code, message, details) => new AppError(code, message, details), + }; +} + +test('whitespace-only credentials are missing, not a provider authentication failure', async () => { + globalThis.fetch = async () => { + throw new Error('a missing credential must not reach TestMu AI'); + }; + for (const env of [ + { LT_USERNAME: ' ', LT_ACCESS_KEY: 'key' }, + { LT_USERNAME: 'user', LT_ACCESS_KEY: '\t' }, + ]) { + await assert.rejects( + testMuPlugin(host(env)).webDriver.listArtifacts!({ + provider: 'testmu', + providerSessionId: 'SESSION1', + }), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /requires LT_(USERNAME|ACCESS_KEY) in the environment/.test(error.message), + ); + } +}); + +test('verification of a hand-authored profile canonicalizes an upper-case app scheme', async () => { + vi.mocked(verifyTestMuConnection).mockResolvedValue({ + provider: 'testmu', + service: 'TestMu AI', + verificationMessage: 'verified', + device: { status: 'verified', name: 'iPhone 16', platform: 'ios', osVersion: '18.0' }, + app: { status: 'verified', reference: 'lt://APP1' }, + }); + const env = { LT_USERNAME: 'user', LT_ACCESS_KEY: 'key' }; + await testMuPlugin(host(env)).connection.verify({ + flags: { + json: false, + help: false, + version: false, + platform: 'ios', + device: 'iPhone 16', + providerOsVersion: '18.0', + providerApp: 'LT://APP1', + }, + }); + assert.equal(vi.mocked(verifyTestMuConnection).mock.calls[0]?.[0].app, 'lt://APP1'); +}); diff --git a/packages/provider-testmu/src/plugin.ts b/packages/provider-testmu/src/plugin.ts index 9dc868046d..545d93eb55 100644 --- a/packages/provider-testmu/src/plugin.ts +++ b/packages/provider-testmu/src/plugin.ts @@ -191,6 +191,6 @@ function requireEnv( providerLabel: string, ): string { const value = env[key]; - if (value) return value; + if (value?.trim()) return value; throw new AppError('INVALID_ARGS', `${providerLabel} requires ${key} in the environment.`); } diff --git a/packages/provider-testmu/src/providers.ts b/packages/provider-testmu/src/providers.ts index 72b2314564..a2e9ccb5ef 100644 --- a/packages/provider-testmu/src/providers.ts +++ b/packages/provider-testmu/src/providers.ts @@ -1,3 +1,8 @@ export function isTestMuAppReference(value: string): boolean { return /^lt:\/\/[\w.-]+$/.test(value); } + +/** URI schemes are case-insensitive; the hub matches the lower-case `lt://` spelling. */ +export function canonicalTestMuAppReference(value: string): string { + return value.slice(0, 5).toLowerCase() === 'lt://' ? `lt://${value.slice(5)}` : value; +} diff --git a/packages/provider-testmu/src/testmu.test.ts b/packages/provider-testmu/src/testmu.test.ts index 551ddeda66..5825c4ffe8 100644 --- a/packages/provider-testmu/src/testmu.test.ts +++ b/packages/provider-testmu/src/testmu.test.ts @@ -317,6 +317,8 @@ test('TestMu upload accepts only an lt:// reference or a valid app id from the r [{ app_url: 'lt://APP6' }, 'lt://APP6'], [{ app_url: 'https://cdn.example/app.apk', app_id: 'APP5' }, 'lt://APP5'], [{ app_id: 'lt://APP7' }, 'lt://APP7'], + [{ app_id: 'LT://APP7' }, 'lt://APP7'], + [{ app_url: 'Lt://APP6' }, 'lt://APP6'], [{ app_url: 'https://cdn.example/app.apk' }, undefined], [{ app_url: 'lt://' }, undefined], [{ app_id: 'bs://APP8' }, undefined], @@ -422,18 +424,23 @@ test('TestMu reads the console log as the device log and falls back to device_lo ['https://api.test/sessions/SESSION1/log/console'], ); - globalThis.fetch = async () => - jsonResponse({ - status: 'success', - data: { device_logs_url: 'https://api.test/sessions/SESSION1/log/device' }, - }); - const deviceLog = await listTestMuCloudArtifacts('testmu', 'SESSION1', auth); - assert.deepEqual( - deviceLog?.cloudArtifacts - .filter((artifact) => artifact.kind === 'device-log') - .map((artifact) => artifact.url), - ['https://api.test/sessions/SESSION1/log/device'], - ); + for (const consoleLogsUrl of [undefined, ' ']) { + globalThis.fetch = async () => + jsonResponse({ + status: 'success', + data: { + console_logs_url: consoleLogsUrl, + device_logs_url: 'https://api.test/sessions/SESSION1/log/device', + }, + }); + const deviceLog = await listTestMuCloudArtifacts('testmu', 'SESSION1', auth); + assert.deepEqual( + deviceLog?.cloudArtifacts + .filter((artifact) => artifact.kind === 'device-log') + .map((artifact) => artifact.url), + ['https://api.test/sessions/SESSION1/log/device'], + ); + } }); test('TestMu session details read as pending on 404 and fail typed on a body that is not JSON', async () => { diff --git a/packages/provider-testmu/src/testmu.ts b/packages/provider-testmu/src/testmu.ts index e326ae2495..c6ac25fe5c 100644 --- a/packages/provider-testmu/src/testmu.ts +++ b/packages/provider-testmu/src/testmu.ts @@ -16,7 +16,7 @@ import type { CloudArtifact, CloudArtifactsResult } from '@agent-device/contract import type { ProviderDeviceType } from '@agent-device/contracts/remote'; import { AppError } from '@agent-device/kernel/errors'; import { asOptionalRecord } from '@agent-device/kernel/record'; -import { isTestMuAppReference } from './providers.ts'; +import { canonicalTestMuAppReference, isTestMuAppReference } from './providers.ts'; /** * TestMu session, upload, and artifact mechanics. Loaded on demand by the provider definition; @@ -168,13 +168,10 @@ export async function resolveTestMuAppReference( const TESTMU_APP_SCHEME = 'lt://'; -/** - * The canonical `lt://` reference for `app` (URI schemes are case-insensitive; the hub matches the - * lower-case spelling), or undefined when `app` does not use the scheme. - */ +/** The canonical `lt://` reference for `app`, or undefined when `app` does not use the scheme. */ function parseTestMuAppReference(app: string): string | undefined { - if (app.slice(0, TESTMU_APP_SCHEME.length).toLowerCase() !== TESTMU_APP_SCHEME) return undefined; - const reference = `${TESTMU_APP_SCHEME}${app.slice(TESTMU_APP_SCHEME.length)}`; + const reference = canonicalTestMuAppReference(app); + if (!reference.startsWith(TESTMU_APP_SCHEME)) return undefined; if (isTestMuAppReference(reference)) return reference; throw new AppError('INVALID_ARGS', `TestMu AI --provider-app ${app} is not an lt:// app id.`, { providerApp: app, @@ -221,7 +218,8 @@ export function buildTestMuCapabilities( /** The upload and app-list APIs answer with a bare app id or an `lt://` reference. */ export function testMuAppReferenceFromId(id: string): string { - return id.startsWith('lt://') ? id : `lt://${id}`; + const reference = canonicalTestMuAppReference(id); + return reference.startsWith(TESTMU_APP_SCHEME) ? reference : `${TESTMU_APP_SCHEME}${id}`; } async function fetchTestMuSessionDetails( @@ -261,7 +259,7 @@ function mapTestMuArtifacts( ): CloudArtifact[] { // Virtual-device sessions report the device log as `console_logs_url`. const deviceLogField = - typeof details.console_logs_url === 'string' && details.console_logs_url.length > 0 + typeof details.console_logs_url === 'string' && details.console_logs_url.trim().length > 0 ? 'console_logs_url' : 'device_logs_url'; const fromDetails = ( @@ -293,7 +291,8 @@ function mapTestMuArtifacts( /** The upload answers with `app_url` (`lt://…`) and/or a bare `app_id`; anything else is a failed upload. */ function readTestMuAppReference(value: unknown): string | undefined { const { app_url: appUrl, app_id: appId } = asOptionalRecord(value) ?? {}; - if (typeof appUrl === 'string' && isTestMuAppReference(appUrl)) return appUrl; + const url = typeof appUrl === 'string' ? canonicalTestMuAppReference(appUrl) : undefined; + if (url && isTestMuAppReference(url)) return url; if (typeof appId !== 'string') return undefined; const reference = testMuAppReferenceFromId(appId); return isTestMuAppReference(reference) ? reference : undefined; From 229bf7e930f6caea35bb59bcd958e5de30183cdd Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 14:51:32 +0530 Subject: [PATCH 27/57] docs(testmu): export credentials in help and note the fixed app listing The help flow's bare assignments did not reach the next agent-device process. The endpoint overrides do not move the app listing connect uses for the credential check, so say so instead of implying a private deployment is fully redirectable. Co-Authored-By: Claude Opus 5.5 --- src/commands/schema/cli-help.ts | 2 +- website/docs/docs/testmu.md | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 7165cb1951..b4f7084164 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -620,7 +620,7 @@ Cloud profile flow: TestMu AI virtual-device flow (emulators and simulators): agent-device plugins add @agent-device/testmu - LT_USERNAME=... LT_ACCESS_KEY=... + export LT_USERNAME=... LT_ACCESS_KEY=... agent-device connect testmu --platform ios --device "iPhone 16" --provider-os-version 18.0 --provider-app lt://APP-id agent-device open com.example.app agent-device snapshot -i diff --git a/website/docs/docs/testmu.md b/website/docs/docs/testmu.md index 26124f191f..a288d083d6 100644 --- a/website/docs/docs/testmu.md +++ b/website/docs/docs/testmu.md @@ -205,9 +205,11 @@ agent-device artifacts --provider testmu --json The TestMu AI session id is the WebDriver session id. If artifact lookup is pending immediately after `close`, retry it; TestMu AI finalizes video and log URLs after the session ends. -Endpoints can be redirected for a staging or private TestMu AI deployment with -`TESTMU_WEBDRIVER_ENDPOINT`, `TESTMU_APP_UPLOAD_ENDPOINT` (virtual devices), -`TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT` (real devices), and `TESTMU_API_ENDPOINT`. +WebDriver, upload, and catalog/session-detail endpoints can be redirected for a staging or private +TestMu AI deployment with `TESTMU_WEBDRIVER_ENDPOINT`, `TESTMU_APP_UPLOAD_ENDPOINT` (virtual +devices), `TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT` (real devices), and `TESTMU_API_ENDPOINT`. The +app listing `connect` uses to check credentials stays fixed at +`https://manual-api.lambdatest.com/app/data`. On hosted WebDriver sessions, `fill` checks that the field received focus before it sends keys. If it cannot confirm focus, it fails without typing. Use `snapshot -i` to confirm the target, or From c553ed71e9552154a6218ce7e857812a10f0fb57 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 14:51:46 +0530 Subject: [PATCH 28/57] fix(kernel): require a real Error behind the AppError brand A plain object carrying the global brand symbol passed instanceof AppError and left normalizeError with no code or message. Package copies share the realm's Error.prototype, so the plugin-copy case still holds. Co-Authored-By: Claude Opus 5.5 --- packages/kernel/src/errors.test.ts | 3 +++ packages/kernel/src/errors.ts | 6 +----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/kernel/src/errors.test.ts b/packages/kernel/src/errors.test.ts index 0d47220b90..0a0208cc0d 100644 --- a/packages/kernel/src/errors.test.ts +++ b/packages/kernel/src/errors.test.ts @@ -191,6 +191,9 @@ test('bundled plugin errors preserve codes and details without changing subclass assert.deepEqual(normalized.details, { provider: 'example' }); const ordinary = Object.assign(new Error('bad plugin profile'), { code: 'INVALID_ARGS' }); assert.equal(ordinary instanceof AppError, false); + const forged = { [Symbol.for('agent-device.AppError')]: true }; + assert.equal(forged instanceof AppError, false); + assert.equal(normalizeError(forged).code, 'UNKNOWN'); class SpecificError extends AppError {} assert.ok(new SpecificError('COMMAND_FAILED', 'specific') instanceof SpecificError); assert.equal(foreign instanceof SpecificError, false); diff --git a/packages/kernel/src/errors.ts b/packages/kernel/src/errors.ts index 56ed18b6bd..a5d8e744be 100644 --- a/packages/kernel/src/errors.ts +++ b/packages/kernel/src/errors.ts @@ -41,11 +41,7 @@ const APP_ERROR_BRAND = Symbol.for('agent-device.AppError'); export class AppError extends Error { static [Symbol.hasInstance](value: unknown): boolean { if (this !== AppError) return Function.prototype[Symbol.hasInstance].call(this, value); - return ( - typeof value === 'object' && - value !== null && - (value as Record)[APP_ERROR_BRAND] === true - ); + return value instanceof Error && (value as Record)[APP_ERROR_BRAND] === true; } code: AppErrorCode; From 2b2370f26e45aee786655ad025b8798af36da812 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 14:52:29 +0530 Subject: [PATCH 29/57] fix(plugins): refuse a primitive factory result as an invalid plugin 'webDriver' in threw a raw TypeError past the loader's INVALID_ARGS validation. Co-Authored-By: Claude Opus 5.5 --- src/plugins/load.test.ts | 8 ++++++++ src/plugins/load.ts | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/plugins/load.test.ts b/src/plugins/load.test.ts index 4026c62558..231b0ab0b7 100644 --- a/src/plugins/load.test.ts +++ b/src/plugins/load.test.ts @@ -111,3 +111,11 @@ test('WebDriver plugins use the shared engine and refuse mismatched providers', selectPlugin(other.home, 'example', 'wrong', source); await assert.rejects(loadProviderPlugins(other.env, []), { code: 'INVALID_ARGS' }); }); + +test('a factory returning a primitive is refused as an invalid plugin', async () => { + for (const value of ['42', '"runtime"', 'true']) { + const { home, env } = pluginHome(); + selectPlugin(home, 'example', 'example', `export default () => ${value};`); + await assert.rejects(loadProviderPlugins(env, []), { code: 'INVALID_ARGS' }); + } +}); diff --git a/src/plugins/load.ts b/src/plugins/load.ts index b9e93d471b..d68eb51a81 100644 --- a/src/plugins/load.ts +++ b/src/plugins/load.ts @@ -82,7 +82,7 @@ async function instantiateProviderPlugin( > )(host); let registration: ProviderPluginRegistration; - if (result && 'webDriver' in result) { + if (result && typeof result === 'object' && 'webDriver' in result) { if (result.webDriver?.provider !== plugin.agentDevicePlugin.provider) { throw new AppError( 'INVALID_ARGS', From 1d528624a35de5ff9d5082293938a007526ad5ef Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 14:52:29 +0530 Subject: [PATCH 30/57] fix(plugins): archive into a fresh zip, not over a stale one zip -r updates an existing archive in place, so leftover entries from an earlier attempt would ride along in the upload. Co-Authored-By: Claude Opus 5.5 --- src/plugins/host.test.ts | 30 ++++++++++++++++++++++++++++++ src/plugins/host.ts | 3 +++ 2 files changed, 33 insertions(+) create mode 100644 src/plugins/host.test.ts diff --git a/src/plugins/host.test.ts b/src/plugins/host.test.ts new file mode 100644 index 0000000000..af6870f36a --- /dev/null +++ b/src/plugins/host.test.ts @@ -0,0 +1,30 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { test } from 'vitest'; +import { runCmdSync } from '@agent-device/host-kit/command'; +import { createPluginHost } from './host.ts'; +import { mkdtempForTestSync } from '../__tests__/test-utils/tmp-dir.ts'; + +test('archiving replaces a stale archive instead of merging into it', async () => { + const root = mkdtempForTestSync('plugin-host-archive-'); + try { + const archivePath = path.join(root, 'App.zip'); + fs.mkdirSync(path.join(root, 'Stale.app')); + fs.writeFileSync(path.join(root, 'Stale.app', 'Info.plist'), 'stale'); + runCmdSync('zip', ['-qr', archivePath, 'Stale.app'], { cwd: root }); + fs.mkdirSync(path.join(root, 'App.app')); + fs.writeFileSync(path.join(root, 'App.app', 'Info.plist'), 'fresh'); + + await createPluginHost({}, undefined).apple.archiveDirectory({ + sourceDirectory: root, + entryName: 'App.app', + archivePath, + }); + + const entries = runCmdSync('unzip', ['-Z1', archivePath]).stdout.trim().split('\n'); + assert.deepEqual(entries.sort(), ['App.app/', 'App.app/Info.plist']); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/src/plugins/host.ts b/src/plugins/host.ts index 98961a65c2..61e7e22948 100644 --- a/src/plugins/host.ts +++ b/src/plugins/host.ts @@ -1,3 +1,4 @@ +import fs from 'node:fs/promises'; import { AppError } from '@agent-device/kernel/errors'; import { execFailureDetails, runCmd } from '@agent-device/host-kit/command'; import { readVersion } from '@agent-device/host-kit/version'; @@ -14,6 +15,8 @@ export function createPluginHost( createError: (code, message, details) => new AppError(code, message, details), apple: Object.freeze({ archiveDirectory: async ({ sourceDirectory, entryName, archivePath }) => { + // zip updates an existing archive in place, so a stale one would keep its old entries. + await fs.rm(archivePath, { force: true }); const args = ['-qr', archivePath, entryName]; const result = await runCmd('zip', args, { cwd: sourceDirectory, timeoutMs: 120_000 }); if (result.exitCode !== 0) { From 694e4d5459797ae5145ff73fff6fdc7dec5cc848 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 14:52:53 +0530 Subject: [PATCH 31/57] fix(connect): look plugin capabilities up in the caller's environment isConnectProviderName already took an env; connectionProviderCapabilities read process.env, so under another plugin home the two disagreed and a plugin provider fell back to the builtin capability shape. Co-Authored-By: Claude Opus 5.5 --- src/cli/commands/react-devtools.ts | 2 +- src/cli/connection/provider-policy.test.ts | 27 +++++++++++++++++++++- src/cli/connection/provider-policy.ts | 3 ++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/cli/commands/react-devtools.ts b/src/cli/commands/react-devtools.ts index 4a69f206f3..d1adb6e8bd 100644 --- a/src/cli/commands/react-devtools.ts +++ b/src/cli/commands/react-devtools.ts @@ -169,7 +169,7 @@ function shouldConfigureDirectReverse( const { flags } = options; if (!flags) return false; return ( - connectionProviderCapabilities(flags.leaseProvider).supportsDirectPortReverse && + connectionProviderCapabilities(flags.leaseProvider, options.env).supportsDirectPortReverse && flags.leaseBackend === 'android-instance' && flags.metroProxyBaseUrl === undefined && options.configureDirectPortReverse !== undefined diff --git a/src/cli/connection/provider-policy.test.ts b/src/cli/connection/provider-policy.test.ts index d537e91c6a..b6ced9d546 100644 --- a/src/cli/connection/provider-policy.test.ts +++ b/src/cli/connection/provider-policy.test.ts @@ -1,6 +1,10 @@ import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; import { test } from 'vitest'; -import { connectionProviderCapabilities } from './provider-policy.ts'; +import { connectionProviderCapabilities, isConnectProviderName } from './provider-policy.ts'; +import { pluginHome, selectPlugin } from '../../plugins/plugin.fixtures.ts'; +import { installedPlugins } from '../../plugins/store.ts'; test('provider policy projects provider identity into semantic capabilities', () => { assert.deepEqual(connectionProviderCapabilities('limrun'), { @@ -18,3 +22,24 @@ test('provider policy projects provider identity into semantic capabilities', () assert.equal(connectionProviderCapabilities('aws-device-farm').requiresAppAttachment, true); assert.equal(connectionProviderCapabilities('proxy').leaseKind, 'proxy'); }); + +test('plugin capabilities come from the same environment as the provider name check', () => { + const { home, env } = pluginHome(); + selectPlugin(home, 'example', 'example', 'export default () => {};'); + const manifestPath = path.join(installedPlugins(env)[0]!.directory, 'package.json'); + const declared = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + const connection = { + leaseKind: 'direct-device-provider', + requiresAppAttachment: true, + requiresRemoteDaemon: false, + supportsArtifacts: true, + supportsDeferredAppSelection: false, + supportsDirectPortReverse: false, + usesCloudWebDriverLease: true, + } as const; + declared.agentDevicePlugin.connection = connection; + fs.writeFileSync(manifestPath, JSON.stringify(declared)); + + assert.equal(isConnectProviderName('example', env), true); + assert.deepEqual(connectionProviderCapabilities('example', env), connection); +}); diff --git a/src/cli/connection/provider-policy.ts b/src/cli/connection/provider-policy.ts index db4f872186..d0f4d8ba3d 100644 --- a/src/cli/connection/provider-policy.ts +++ b/src/cli/connection/provider-policy.ts @@ -35,11 +35,12 @@ export function connectProviderNamesForError(): string { export function connectionProviderCapabilities( provider: string | undefined, + env: NodeJS.ProcessEnv = process.env, ): ConnectionProviderCapabilities { const directDeviceProvider = isDirectDeviceConnectProvider(provider); const cloudWebDriver = isCloudWebDriverProviderName(provider); if (!directDeviceProvider && provider !== 'cloud' && provider !== 'proxy') { - const plugin = pluginConnectionCapabilities(provider); + const plugin = pluginConnectionCapabilities(provider, env); if (plugin) return plugin; } return { From aab5684cccc4a6f359704c670869c93096c0b02a Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 14:55:14 +0530 Subject: [PATCH 32/57] fix(kernel): type the AppError brand lookup on a narrowed Error Co-Authored-By: Claude Opus 5.5 --- packages/kernel/src/errors.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/kernel/src/errors.ts b/packages/kernel/src/errors.ts index a5d8e744be..a0e383dc9e 100644 --- a/packages/kernel/src/errors.ts +++ b/packages/kernel/src/errors.ts @@ -41,7 +41,9 @@ const APP_ERROR_BRAND = Symbol.for('agent-device.AppError'); export class AppError extends Error { static [Symbol.hasInstance](value: unknown): boolean { if (this !== AppError) return Function.prototype[Symbol.hasInstance].call(this, value); - return value instanceof Error && (value as Record)[APP_ERROR_BRAND] === true; + return ( + value instanceof Error && (value as Error & Record)[APP_ERROR_BRAND] === true + ); } code: AppErrorCode; From 7279a63e52917fd3599a488322893932f82d45de Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 14:55:14 +0530 Subject: [PATCH 33/57] refactor(plugins): reserve provider ids from one list assertUniquePluginProviders merged RESERVED_PLUGIN_PROVIDERS into whatever its callers passed, and the daemon passed the overlapping DEFAULT_PROVIDER_RUNTIME_REQUIRED_IDS. Every caller now passes RESERVED_PLUGIN_PROVIDERS, which takes the WebDriver ids from CLOUD_WEBDRIVER_PROVIDERS, and a test pins every bundled runtime id into it. plugins add/update no longer loads the provider runtimes. Co-Authored-By: Claude Opus 5.5 --- src/cli/commands/plugins.ts | 6 +----- src/plugins/manifest.test.ts | 8 +++++++- src/plugins/manifest.ts | 21 ++++++++++++--------- src/plugins/store.ts | 8 ++++++-- src/provider-device-runtimes.ts | 7 +++++-- 5 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/cli/commands/plugins.ts b/src/cli/commands/plugins.ts index 5987978e8c..8fbf202bb9 100644 --- a/src/cli/commands/plugins.ts +++ b/src/cli/commands/plugins.ts @@ -16,11 +16,7 @@ export const pluginsCommand: ClientCommandHandler = async ({ positionals, flags ); } if (action !== 'list' && name) { - const reserved = - action === 'remove' - ? [] - : (await import('../../provider-device-runtimes.ts')).DEFAULT_PROVIDER_RUNTIME_REQUIRED_IDS; - await changePlugin(action as 'add' | 'update' | 'remove', name, process.env, reserved); + await changePlugin(action as 'add' | 'update' | 'remove', name, process.env); } const plugins = listPlugins(process.env); await writeCommandOutput(flags, { plugins, restartRequired: action !== 'list' }, () => diff --git a/src/plugins/manifest.test.ts b/src/plugins/manifest.test.ts index a414e7bd94..4fbd7dfc5d 100644 --- a/src/plugins/manifest.test.ts +++ b/src/plugins/manifest.test.ts @@ -2,7 +2,8 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import { test } from 'vitest'; -import { readPluginManifest } from './manifest.ts'; +import { readPluginManifest, RESERVED_PLUGIN_PROVIDERS } from './manifest.ts'; +import { DEFAULT_PROVIDER_RUNTIME_REQUIRED_IDS } from '../provider-device-runtimes.ts'; import { pluginHome, writePlugin } from './plugin.fixtures.ts'; test('manifest compatibility is checked without evaluating plugin code', () => { @@ -65,3 +66,8 @@ test('connection metadata admits local providers and rejects malformed or remote assert.throws(() => read(invalid), { code: 'INVALID_ARGS' }); } }); + +test('every bundled provider runtime is reserved from plugins', () => { + for (const provider of DEFAULT_PROVIDER_RUNTIME_REQUIRED_IDS) + assert.ok((RESERVED_PLUGIN_PROVIDERS as readonly string[]).includes(provider), provider); +}); diff --git a/src/plugins/manifest.ts b/src/plugins/manifest.ts index 93fd645c12..675817af96 100644 --- a/src/plugins/manifest.ts +++ b/src/plugins/manifest.ts @@ -3,15 +3,18 @@ import path from 'node:path'; import { AppError } from '@agent-device/kernel/errors'; import type { ConnectionProviderCapabilities } from '@agent-device/contracts/remote'; -import { CLOUD_WEBDRIVER_PROVIDERS } from '@agent-device/provider-webdriver/providers'; +import { + CLOUD_WEBDRIVER_PROVIDERS, + type CloudWebDriverKnownProviderName, +} from '@agent-device/provider-webdriver/providers'; -export const RESERVED_PLUGIN_PROVIDERS = [ - 'cloud', - 'proxy', - CLOUD_WEBDRIVER_PROVIDERS.browserStack, - CLOUD_WEBDRIVER_PROVIDERS.awsDeviceFarm, - 'limrun', -] as const; +/** The one list of provider ids plugins cannot claim: connect routes and bundled runtimes. */ +export const RESERVED_PLUGIN_PROVIDERS: readonly ( + | 'cloud' + | 'proxy' + | 'limrun' + | CloudWebDriverKnownProviderName +)[] = ['cloud', 'proxy', ...Object.values(CLOUD_WEBDRIVER_PROVIDERS), 'limrun']; const PROVIDER_PLUGIN_API_VERSION = 1; type PluginManifest = { @@ -29,7 +32,7 @@ export function assertUniquePluginProviders( plugins: readonly PluginManifest[], reserved: readonly string[], ): void { - const providers = new Set([...RESERVED_PLUGIN_PROVIDERS, ...reserved]); + const providers = new Set(reserved); for (const plugin of plugins) { const provider = plugin.agentDevicePlugin.provider; if (providers.has(provider)) diff --git a/src/plugins/store.ts b/src/plugins/store.ts index 4435a624cb..ec50f038a1 100644 --- a/src/plugins/store.ts +++ b/src/plugins/store.ts @@ -7,7 +7,11 @@ import { runCmd } from '@agent-device/host-kit/command'; import { acquireProcessLock, publishFileSync } from '@agent-device/host-kit/file'; import { readCurrentOwnerIdentity } from '@agent-device/host-kit/process'; import { resolveUserConfigPath } from '../commands/schema/cli-config.ts'; -import { readPluginManifest, assertUniquePluginProviders } from './manifest.ts'; +import { + readPluginManifest, + assertUniquePluginProviders, + RESERVED_PLUGIN_PROVIDERS, +} from './manifest.ts'; type PluginSelection = { installation: string; @@ -136,7 +140,7 @@ export async function changePlugin( action: PluginAction, input: string, env: NodeJS.ProcessEnv = process.env, - reservedProviders: readonly string[] = [], + reservedProviders: readonly string[] = RESERVED_PLUGIN_PROVIDERS, ): Promise { const { name, version } = parsePluginRequest(action, input); const configPath = resolveUserConfigPath(env); diff --git a/src/provider-device-runtimes.ts b/src/provider-device-runtimes.ts index 3d2e0f8a3b..df925fb3d3 100644 --- a/src/provider-device-runtimes.ts +++ b/src/provider-device-runtimes.ts @@ -49,8 +49,11 @@ export async function createDaemonProviderRuntimeComposition( ): Promise { const bundled = await createDefaultProviderRuntimeComposition(env); try { - const { loadProviderPlugins } = await import('./plugins/load.ts'); - const plugins = await loadProviderPlugins(env, DEFAULT_PROVIDER_RUNTIME_REQUIRED_IDS); + const [{ loadProviderPlugins }, { RESERVED_PLUGIN_PROVIDERS }] = await Promise.all([ + import('./plugins/load.ts'), + import('./plugins/manifest.ts'), + ]); + const plugins = await loadProviderPlugins(env, RESERVED_PLUGIN_PROVIDERS); return Object.freeze({ ...bundled, runtimes: Object.freeze([...bundled.runtimes, ...plugins.map(({ runtime }) => runtime)]), From acf35f4c5afc7f85099968a4ac794a441457f759 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 14:55:59 +0530 Subject: [PATCH 34/57] refactor(plugins): name the WebDriver factory result check Keeps instantiateProviderPlugin under the Fallow complexity threshold after the object guard. Co-Authored-By: Claude Opus 5.5 --- src/plugins/load.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/plugins/load.ts b/src/plugins/load.ts index d68eb51a81..f9b2d6472b 100644 --- a/src/plugins/load.ts +++ b/src/plugins/load.ts @@ -82,7 +82,7 @@ async function instantiateProviderPlugin( > )(host); let registration: ProviderPluginRegistration; - if (result && typeof result === 'object' && 'webDriver' in result) { + if (isWebDriverPluginResult(result)) { if (result.webDriver?.provider !== plugin.agentDevicePlugin.provider) { throw new AppError( 'INVALID_ARGS', @@ -106,6 +106,12 @@ async function instantiateProviderPlugin( return registration; } +function isWebDriverPluginResult( + result: unknown, +): result is { webDriver: WebDriverPluginOptions; connection?: PluginConnection } { + return typeof result === 'object' && result !== null && 'webDriver' in result; +} + function validateProviderPlugin( registration: ProviderPluginRegistration, plugin: ReturnType[number], From 027b84851373451f11827b7005d8780992210b13 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 14:55:59 +0530 Subject: [PATCH 35/57] chore(fallow): stop suppressing statically imported TestMu helpers connection.ts imports readTestMuDeviceFeatureFields and readTestMuDeviceType statically, so Fallow sees them; only the lazily read buildTestMuDeviceFeatureCapabilities still needs the exemption. Co-Authored-By: Claude Opus 5.5 --- .fallowrc.json | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.fallowrc.json b/.fallowrc.json index 65a1465080..c910e9013a 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -392,13 +392,9 @@ "exports": ["LimrunIosCommandExecution"] }, { - "comment": "TestMu session preparation reads these off the lazy loadTestMuDeviceFeatures() import in packages/provider-webdriver/src/provider-definitions.ts, which keeps the package entry's eager closure unchanged; Fallow cannot connect the dynamic member reads.", + "comment": "TestMu session preparation reads this off the lazy loadTestMuDeviceFeatures() import in packages/provider-testmu/src/plugin.ts, which keeps the plugin entry's eager closure unchanged; Fallow cannot connect the dynamic member read.", "file": "packages/provider-testmu/src/testmu-device-features.ts", - "exports": [ - "buildTestMuDeviceFeatureCapabilities", - "readTestMuDeviceFeatureFields", - "readTestMuDeviceType" - ] + "exports": ["buildTestMuDeviceFeatureCapabilities"] }, { "comment": "Converting the contracts façades from `export *` to explicit named re-exports (the pin-table retirement) made these individually visible to --production analysis for the first time; a bare star previously hid them from this exact check. isRecord/IOS_SAFARI_BUNDLE_ID/REPLAY_DIVERGENCE_* have no production consumer. Kept rather than narrowed here so the façade's re-export surface stays byte-identical to the symbol set the retired pin table asserted — narrowing the surface is a follow-up with its own review, not a side effect of this mechanical conversion.", From 244c131ccb6a6e5084d29f07ecef064d840eb6eb Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 14:57:11 +0530 Subject: [PATCH 36/57] test: allocate through a WebDriver plugin and assert typed ownership The WebDriver plugin fixture now declares its profile fields and the test allocates a session through the shared engine, refusing a field it marks refused before any request. The refused repeat allocation also asserts session-1 still resolves to the live lease, and the Limrun refusal checks details.provider and details.flags instead of copy. Co-Authored-By: Claude Opus 5.5 --- .../lease-provider-profile-fields.test.ts | 11 ++-- src/daemon/handlers/__tests__/lease.test.ts | 13 +++++ src/plugins/load.test.ts | 56 ++++++++++++++++--- src/plugins/plugin.fixtures.ts | 47 ++++++++++++++++ 4 files changed, 116 insertions(+), 11 deletions(-) diff --git a/src/__tests__/lease-provider-profile-fields.test.ts b/src/__tests__/lease-provider-profile-fields.test.ts index 141e8d3234..56dd8bff2f 100644 --- a/src/__tests__/lease-provider-profile-fields.test.ts +++ b/src/__tests__/lease-provider-profile-fields.test.ts @@ -76,10 +76,13 @@ test('leases.allocate refuses a real device from Limrun instead of handing out a platform: 'ios', providerDeviceType: 'real', }), - (error: unknown) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /--provider-device-type is not supported by Limrun/.test(error.message), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'INVALID_ARGS'); + assert.equal(error.details?.provider, 'limrun'); + assert.deepEqual(error.details?.flags, ['--provider-device-type']); + return true; + }, ); assert.equal(limrunInstances.iosCreate.mock.calls.length, 0); }); diff --git a/src/daemon/handlers/__tests__/lease.test.ts b/src/daemon/handlers/__tests__/lease.test.ts index 8035a90b97..fdb671b54a 100644 --- a/src/daemon/handlers/__tests__/lease.test.ts +++ b/src/daemon/handlers/__tests__/lease.test.ts @@ -210,4 +210,17 @@ test("a refused repeat allocation keeps the run's live lease", async () => { registry.listActiveLeases().map((entry) => entry.leaseId), [lease.leaseId], ); + assert.deepEqual( + registry.resolveProviderSession({ + provider: 'cloud', + providerSessionId: 'session-1', + tenantId: 'tenant-a', + }), + { + provider: 'cloud', + providerSessionId: 'session-1', + leaseId: lease.leaseId, + tenantId: 'tenant-a', + }, + ); }); diff --git a/src/plugins/load.test.ts b/src/plugins/load.test.ts index 231b0ab0b7..3bf512dee2 100644 --- a/src/plugins/load.test.ts +++ b/src/plugins/load.test.ts @@ -3,10 +3,29 @@ import fs from 'node:fs'; import path from 'node:path'; import { test } from 'vitest'; import { loadProviderPlugins, withPluginConnection } from './load.ts'; -import { pluginHome, selectPlugin, registrationSource } from './plugin.fixtures.ts'; +import { + pluginHome, + selectPlugin, + registrationSource, + webDriverPluginSource, +} from './plugin.fixtures.ts'; import { AppError } from '@agent-device/kernel/errors'; import type { ProviderPluginHost } from '../sdk/plugins.ts'; -import type { ProviderDeviceRuntime } from '@agent-device/contracts/device'; +import type { DeviceLease, ProviderDeviceRuntime } from '@agent-device/contracts/device'; + +const realFetch = globalThis.fetch; +const lease: DeviceLease = { + leaseId: 'lease-1', + tenantId: 'team-a', + runId: 'run-a', + clientId: 'client-a', + leaseProvider: 'example', + backend: 'android-instance', + deviceKey: 'example:device-a', + createdAt: 1, + expiresAt: 2, + heartbeatAt: 1, +}; test('startup loads the factory with options and the host error constructor', async () => { const { home, env } = pluginHome(); @@ -98,15 +117,38 @@ test('connection callbacks run from the installed plugin and always release runt assert.ok(fs.existsSync(marker)); }); -test('WebDriver plugins use the shared engine and refuse mismatched providers', async () => { +test('WebDriver plugins allocate through the shared engine and refuse mismatched providers', async () => { const { home, env } = pluginHome(); - const source = - "export default () => ({ webDriver: { provider: 'example', endpoint: 'http://127.0.0.1/', platform: 'android', deviceName: 'example' } });"; + const source = webDriverPluginSource('example', ['awsProjectArn']); selectPlugin(home, 'example', 'example', source); const [registration] = await loadProviderPlugins(env, []); - assert.equal(registration!.runtime.provider, 'example'); + const runtime = registration!.runtime; + assert.equal(runtime.provider, 'example'); assert.equal(registration!.platformModule.owner.provider, 'example'); - await registration!.runtime.shutdown(); + const requests: string[] = []; + globalThis.fetch = async (input, init) => { + requests.push(`${init?.method ?? 'GET'} ${String(input)}`); + return new Response(JSON.stringify({ value: { sessionId: 'SESSION1', capabilities: {} } })); + }; + try { + await assert.rejects( + runtime.leaseLifecycle.allocate!(lease, { flags: { awsProjectArn: 'arn:project' } }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'INVALID_ARGS'); + assert.equal(error.details?.provider, 'example'); + assert.deepEqual(error.details?.flags, ['--aws-project-arn']); + return true; + }, + ); + assert.deepEqual(requests, []); + const allocated = await runtime.leaseLifecycle.allocate!(lease, { flags: {} }); + assert.equal(allocated?.sessionId, 'SESSION1'); + assert.deepEqual(requests, ['POST https://webdriver.test/wd/hub/session']); + } finally { + await runtime.shutdown(); + globalThis.fetch = realFetch; + } const other = pluginHome(); selectPlugin(other.home, 'example', 'wrong', source); await assert.rejects(loadProviderPlugins(other.env, []), { code: 'INVALID_ARGS' }); diff --git a/src/plugins/plugin.fixtures.ts b/src/plugins/plugin.fixtures.ts index 38b80443f9..36d83ed134 100644 --- a/src/plugins/plugin.fixtures.ts +++ b/src/plugins/plugin.fixtures.ts @@ -1,6 +1,7 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; +import type { ProviderProfileField } from '@agent-device/contracts/provider-profile-fields'; import { mkdtempForTestSync } from '../__tests__/test-utils/tmp-dir.ts'; export function pluginHome() { @@ -58,3 +59,49 @@ export function registrationSource(provider: string, shutdownFile?: string) { loadRuntime: async () => { throw new Error('lazy'); } } });`; } + +const CONSUMED_PROFILE_FIELDS: Record = { + providerApp: 'consumed', + providerOsVersion: 'consumed', + providerDeviceType: 'consumed', + providerProject: 'consumed', + providerBuild: 'consumed', + providerSessionName: 'consumed', + providerDeviceOrientation: 'consumed', + providerGeoLocation: 'consumed', + providerTimezone: 'consumed', + providerAppiumVersion: 'consumed', + providerLanguage: 'consumed', + providerLocale: 'consumed', + providerNetworkProfile: 'consumed', + providerCustomNetwork: 'consumed', + providerNoResignApp: 'consumed', + awsProjectArn: 'consumed', + awsDeviceArn: 'consumed', + awsAppArn: 'consumed', + awsRegion: 'consumed', + awsInteractionMode: 'consumed', +}; + +/** A `{ webDriver }` factory with a total field declaration; `connection` is a JS expression. */ +export function webDriverPluginSource( + provider: string, + refused: readonly ProviderProfileField[] = [], + connection?: string, +) { + const fields = { + ...CONSUMED_PROFILE_FIELDS, + ...Object.fromEntries(refused.map((field) => [field, 'refused'])), + }; + const webDriver = { + provider, + endpoint: 'https://webdriver.test/wd/hub/', + platform: 'android', + deviceName: provider, + profileFields: { provider, label: provider, fields }, + requestPolicy: { retryAttempts: 0 }, + }; + return `export default (host) => ({ webDriver: ${JSON.stringify(webDriver)}${ + connection ? `, connection: ${connection}` : '' + } });`; +} From 62e814c6d13fdfd3da8e6c391a92c5a25ff80605 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 14:57:53 +0530 Subject: [PATCH 37/57] fix(connect): refuse a WebDriver plugin's refused fields before resolve Builtin connect routes run rejectRefusedProviderProfileFields against their declaration; the plugin route left it to the plugin. A { webDriver } plugin already hands core a total profileFields declaration, so the connect route now applies it before the plugin's resolve. Raw runtime plugins carry no declaration and still validate their own flags. Co-Authored-By: Claude Opus 5.5 --- .../connect-provider-adapters.test.ts | 49 +++++++++++++++++++ .../connection/connect-provider-adapters.ts | 4 +- src/plugins/load.ts | 17 ++++--- website/docs/docs/plugins.md | 2 +- 4 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 src/cli/connection/connect-provider-adapters.test.ts diff --git a/src/cli/connection/connect-provider-adapters.test.ts b/src/cli/connection/connect-provider-adapters.test.ts new file mode 100644 index 0000000000..d96ff8fb1b --- /dev/null +++ b/src/cli/connection/connect-provider-adapters.test.ts @@ -0,0 +1,49 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { test } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { resolveConnectProviderProfile } from './connect-provider-adapters.ts'; +import { pluginHome, selectPlugin, webDriverPluginSource } from '../../plugins/plugin.fixtures.ts'; + +test('connect refuses a field a WebDriver plugin declares refused before its resolve runs', async () => { + const { home, env } = pluginHome(); + const marker = path.join(home, 'resolved'); + const connection = `{ + resolve: () => { + fs.writeFileSync(${JSON.stringify(marker)}, 'yes'); + return { profile: { leaseProvider: 'example', platform: 'android' } }; + }, + verify: async () => ({}), + }`; + selectPlugin( + home, + 'example', + 'example', + `import fs from 'node:fs';\n${webDriverPluginSource('example', ['awsProjectArn'], connection)}`, + ); + + await assert.rejects( + resolveConnectProviderProfile({ + provider: 'example', + flags: { + json: false, + help: false, + version: false, + platform: 'android', + awsProjectArn: 'arn:project', + }, + stateDir: path.join(home, 'state'), + cwd: home, + env, + }), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'INVALID_ARGS'); + assert.equal(error.details?.provider, 'example'); + assert.deepEqual(error.details?.flags, ['--aws-project-arn']); + return true; + }, + ); + assert.ok(!fs.existsSync(marker)); +}); diff --git a/src/cli/connection/connect-provider-adapters.ts b/src/cli/connection/connect-provider-adapters.ts index 94818de10d..d59b13d6e9 100644 --- a/src/cli/connection/connect-provider-adapters.ts +++ b/src/cli/connection/connect-provider-adapters.ts @@ -2,6 +2,7 @@ import type { CliFlags } from '@agent-device/contracts/command'; import type { ProviderConnectionVerification } from '@agent-device/contracts/remote'; import { verifyLimrunConnection } from '@agent-device/provider-limrun'; import { AppError } from '@agent-device/kernel/errors'; +import { rejectRefusedProviderProfileFields } from '@agent-device/contracts/provider-profile-fields'; import { providerWebDriver } from '../../provider-webdriver.ts'; import { resolveRemoteConfigProfile } from '../../remote/remote-config.ts'; import { readVersion } from '@agent-device/host-kit/version'; @@ -123,7 +124,8 @@ export async function resolveConnectProviderProfile(options: { ]; const profile = adapter ? await adapter.resolve(context) - : await withPluginConnection(provider, env, async (connection) => { + : await withPluginConnection(provider, env, async (connection, profileFields) => { + if (profileFields) rejectRefusedProviderProfileFields(context.flags, profileFields); const resolved = await connection.resolve(context); if (resolved.profile.leaseProvider !== provider) throw new AppError( diff --git a/src/plugins/load.ts b/src/plugins/load.ts index f9b2d6472b..723b00b0cd 100644 --- a/src/plugins/load.ts +++ b/src/plugins/load.ts @@ -12,11 +12,14 @@ import { import { createPluginHost } from './host.ts'; import type { PluginConnection } from './connection.ts'; import type { WebDriverPluginOptions } from '../sdk/plugin-webdriver.ts'; +import type { ProviderProfileFieldDeclaration } from '@agent-device/contracts/provider-profile-fields'; type ProviderPluginRegistration = Readonly<{ runtime: ProviderDeviceRuntime; platformModule: PlatformRuntimeProviderModule; connection?: PluginConnection; + /** Known only for `{ webDriver }` plugins; a raw runtime plugin validates its own fields. */ + profileFields?: ProviderProfileFieldDeclaration; }>; export async function loadProviderPlugins( @@ -45,16 +48,17 @@ export async function loadProviderPlugins( export async function withPluginConnection( provider: string, env: NodeJS.ProcessEnv, - runConnection: (connection: PluginConnection) => Promise, + runConnection: ( + connection: PluginConnection, + profileFields: ProviderProfileFieldDeclaration | undefined, + ) => Promise, ): Promise { const registrations = await loadProviderPlugins(env, RESERVED_PLUGIN_PROVIDERS, provider); try { - const connection = registrations.find( - (entry) => entry.runtime.provider === provider, - )?.connection; - if (!connection) + const registration = registrations.find((entry) => entry.runtime.provider === provider); + if (!registration?.connection) throw new AppError('INVALID_ARGS', `Plugin does not register connect: ${provider}`); - return await runConnection(connection); + return await runConnection(registration.connection, registration.profileFields); } finally { await Promise.allSettled(registrations.map(async ({ runtime }) => await runtime.shutdown())); } @@ -98,6 +102,7 @@ async function instantiateProviderPlugin( runtime, platformModule: runtime.platformRuntimeModule, connection: result.connection, + profileFields: result.webDriver.profileFields, }; } else registration = result; if (!registration?.runtime || typeof registration.runtime.shutdown !== 'function') { diff --git a/website/docs/docs/plugins.md b/website/docs/docs/plugins.md index 62024ec3bc..1626f2a137 100644 --- a/website/docs/docs/plugins.md +++ b/website/docs/docs/plugins.md @@ -49,6 +49,6 @@ To support `agent-device connect example`, declare `agentDevicePlugin.connection } ``` -Return `connection` alongside the runtime or WebDriver options. Its `resolve({ flags, env, cwd, stateDir })` callback validates provider flags and returns `{ profile, extraFlags? }`; `profile.leaseProvider` must match the manifest. Core supplies connection identity, session defaults, Metro settings, and persists the profile. Its async `verify({ flags, env })` callback returns the provider verification result. These callbacks run without allocating a device and their temporary runtimes are shut down afterwards. +Return `connection` alongside the runtime or WebDriver options. Its `resolve({ flags, env, cwd, stateDir })` callback validates provider flags and returns `{ profile, extraFlags? }`; for a WebDriver plugin, core first refuses the flags its `profileFields` marks `refused`; `profile.leaseProvider` must match the manifest. Core supplies connection identity, session defaults, Metro settings, and persists the profile. Its async `verify({ flags, env })` callback returns the provider verification result. These callbacks run without allocating a device and their temporary runtimes are shut down afterwards. Bundle the plugin implementation and ship ready-to-run ESM: installation disables lifecycle scripts. Bundle shared implementation helpers with the plugin; `AppError` carries a shared brand so core preserves its code and details across package copies. Import types with `import type` to keep them out of the runtime dependency graph. From cee110cfd62e071a5f147f8fa217da10d2fb2031 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 14:58:29 +0530 Subject: [PATCH 38/57] docs(testing): say when to run the packed provider plugin check check-provider-plugin.mjs needs pnpm and a built core, which the Node 22.12 package lane cannot run and whose script body CI credits verbatim, so it is a manual pre-push check for plugin package changes. Co-Authored-By: Claude Opus 5.5 --- docs/agents/testing.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/agents/testing.md b/docs/agents/testing.md index f0dbb77db8..9b0992dc5b 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -32,6 +32,9 @@ Two selection traps recur: - A workspace package manifest or TypeScript config can rewire all consumers, so the affected selector fails open to the full gate set on purpose. +Provider plugin packaging has no CI lane: after changing a `packages/provider-*` package or the +plugin SDK, run `pnpm build && node scripts/check-provider-plugin.mjs packages/provider-testmu`. + Docs-only changes with no runtime behavior impact need no runtime tests or new tests asserting prose. Keep required gates; after those and focused checks pass, repeat or broaden only for changes, failures, or unresolved risks. From f3f05e01591eb93b4e2adbdd90ff43b65c41171c Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 15:04:41 +0530 Subject: [PATCH 39/57] docs: document the plugin package check in its script, not testing.md testing.md is over the agent-guidance byte budget with the note, so the how and when now sit in the script header. Co-Authored-By: Claude Opus 5.5 --- docs/agents/testing.md | 3 --- scripts/check-provider-plugin.mjs | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 9b0992dc5b..f0dbb77db8 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -32,9 +32,6 @@ Two selection traps recur: - A workspace package manifest or TypeScript config can rewire all consumers, so the affected selector fails open to the full gate set on purpose. -Provider plugin packaging has no CI lane: after changing a `packages/provider-*` package or the -plugin SDK, run `pnpm build && node scripts/check-provider-plugin.mjs packages/provider-testmu`. - Docs-only changes with no runtime behavior impact need no runtime tests or new tests asserting prose. Keep required gates; after those and focused checks pass, repeat or broaden only for changes, failures, or unresolved risks. diff --git a/scripts/check-provider-plugin.mjs b/scripts/check-provider-plugin.mjs index bc713e7332..13d30d255a 100644 --- a/scripts/check-provider-plugin.mjs +++ b/scripts/check-provider-plugin.mjs @@ -1,3 +1,5 @@ +// Manual pre-push check for provider plugin packages; no CI lane runs it (it needs pnpm and a +// built core). Usage: `pnpm build && node scripts/check-provider-plugin.mjs packages/provider-testmu`. import assert from 'node:assert/strict'; import { execFile } from 'node:child_process'; import crypto from 'node:crypto'; From 3986976795532be33ccf92f4a92eebaf5a5b832d Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 15:17:29 +0530 Subject: [PATCH 40/57] revert: let resolve and allocate own plugin profile-field refusal Reverts 62e814c6d. The maintainer judged the connect-route check redundant: plugin resolve and lease allocation both refuse those fields, and #3168 owns profile-field admission. Co-Authored-By: Claude Opus 5.5 --- .../connect-provider-adapters.test.ts | 49 ------------------- .../connection/connect-provider-adapters.ts | 4 +- src/plugins/load.ts | 17 +++---- website/docs/docs/plugins.md | 2 +- 4 files changed, 8 insertions(+), 64 deletions(-) delete mode 100644 src/cli/connection/connect-provider-adapters.test.ts diff --git a/src/cli/connection/connect-provider-adapters.test.ts b/src/cli/connection/connect-provider-adapters.test.ts deleted file mode 100644 index d96ff8fb1b..0000000000 --- a/src/cli/connection/connect-provider-adapters.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import assert from 'node:assert/strict'; -import fs from 'node:fs'; -import path from 'node:path'; -import { test } from 'vitest'; -import { AppError } from '@agent-device/kernel/errors'; -import { resolveConnectProviderProfile } from './connect-provider-adapters.ts'; -import { pluginHome, selectPlugin, webDriverPluginSource } from '../../plugins/plugin.fixtures.ts'; - -test('connect refuses a field a WebDriver plugin declares refused before its resolve runs', async () => { - const { home, env } = pluginHome(); - const marker = path.join(home, 'resolved'); - const connection = `{ - resolve: () => { - fs.writeFileSync(${JSON.stringify(marker)}, 'yes'); - return { profile: { leaseProvider: 'example', platform: 'android' } }; - }, - verify: async () => ({}), - }`; - selectPlugin( - home, - 'example', - 'example', - `import fs from 'node:fs';\n${webDriverPluginSource('example', ['awsProjectArn'], connection)}`, - ); - - await assert.rejects( - resolveConnectProviderProfile({ - provider: 'example', - flags: { - json: false, - help: false, - version: false, - platform: 'android', - awsProjectArn: 'arn:project', - }, - stateDir: path.join(home, 'state'), - cwd: home, - env, - }), - (error: unknown) => { - assert.ok(error instanceof AppError); - assert.equal(error.code, 'INVALID_ARGS'); - assert.equal(error.details?.provider, 'example'); - assert.deepEqual(error.details?.flags, ['--aws-project-arn']); - return true; - }, - ); - assert.ok(!fs.existsSync(marker)); -}); diff --git a/src/cli/connection/connect-provider-adapters.ts b/src/cli/connection/connect-provider-adapters.ts index d59b13d6e9..94818de10d 100644 --- a/src/cli/connection/connect-provider-adapters.ts +++ b/src/cli/connection/connect-provider-adapters.ts @@ -2,7 +2,6 @@ import type { CliFlags } from '@agent-device/contracts/command'; import type { ProviderConnectionVerification } from '@agent-device/contracts/remote'; import { verifyLimrunConnection } from '@agent-device/provider-limrun'; import { AppError } from '@agent-device/kernel/errors'; -import { rejectRefusedProviderProfileFields } from '@agent-device/contracts/provider-profile-fields'; import { providerWebDriver } from '../../provider-webdriver.ts'; import { resolveRemoteConfigProfile } from '../../remote/remote-config.ts'; import { readVersion } from '@agent-device/host-kit/version'; @@ -124,8 +123,7 @@ export async function resolveConnectProviderProfile(options: { ]; const profile = adapter ? await adapter.resolve(context) - : await withPluginConnection(provider, env, async (connection, profileFields) => { - if (profileFields) rejectRefusedProviderProfileFields(context.flags, profileFields); + : await withPluginConnection(provider, env, async (connection) => { const resolved = await connection.resolve(context); if (resolved.profile.leaseProvider !== provider) throw new AppError( diff --git a/src/plugins/load.ts b/src/plugins/load.ts index 723b00b0cd..f9b2d6472b 100644 --- a/src/plugins/load.ts +++ b/src/plugins/load.ts @@ -12,14 +12,11 @@ import { import { createPluginHost } from './host.ts'; import type { PluginConnection } from './connection.ts'; import type { WebDriverPluginOptions } from '../sdk/plugin-webdriver.ts'; -import type { ProviderProfileFieldDeclaration } from '@agent-device/contracts/provider-profile-fields'; type ProviderPluginRegistration = Readonly<{ runtime: ProviderDeviceRuntime; platformModule: PlatformRuntimeProviderModule; connection?: PluginConnection; - /** Known only for `{ webDriver }` plugins; a raw runtime plugin validates its own fields. */ - profileFields?: ProviderProfileFieldDeclaration; }>; export async function loadProviderPlugins( @@ -48,17 +45,16 @@ export async function loadProviderPlugins( export async function withPluginConnection( provider: string, env: NodeJS.ProcessEnv, - runConnection: ( - connection: PluginConnection, - profileFields: ProviderProfileFieldDeclaration | undefined, - ) => Promise, + runConnection: (connection: PluginConnection) => Promise, ): Promise { const registrations = await loadProviderPlugins(env, RESERVED_PLUGIN_PROVIDERS, provider); try { - const registration = registrations.find((entry) => entry.runtime.provider === provider); - if (!registration?.connection) + const connection = registrations.find( + (entry) => entry.runtime.provider === provider, + )?.connection; + if (!connection) throw new AppError('INVALID_ARGS', `Plugin does not register connect: ${provider}`); - return await runConnection(registration.connection, registration.profileFields); + return await runConnection(connection); } finally { await Promise.allSettled(registrations.map(async ({ runtime }) => await runtime.shutdown())); } @@ -102,7 +98,6 @@ async function instantiateProviderPlugin( runtime, platformModule: runtime.platformRuntimeModule, connection: result.connection, - profileFields: result.webDriver.profileFields, }; } else registration = result; if (!registration?.runtime || typeof registration.runtime.shutdown !== 'function') { diff --git a/website/docs/docs/plugins.md b/website/docs/docs/plugins.md index 1626f2a137..62024ec3bc 100644 --- a/website/docs/docs/plugins.md +++ b/website/docs/docs/plugins.md @@ -49,6 +49,6 @@ To support `agent-device connect example`, declare `agentDevicePlugin.connection } ``` -Return `connection` alongside the runtime or WebDriver options. Its `resolve({ flags, env, cwd, stateDir })` callback validates provider flags and returns `{ profile, extraFlags? }`; for a WebDriver plugin, core first refuses the flags its `profileFields` marks `refused`; `profile.leaseProvider` must match the manifest. Core supplies connection identity, session defaults, Metro settings, and persists the profile. Its async `verify({ flags, env })` callback returns the provider verification result. These callbacks run without allocating a device and their temporary runtimes are shut down afterwards. +Return `connection` alongside the runtime or WebDriver options. Its `resolve({ flags, env, cwd, stateDir })` callback validates provider flags and returns `{ profile, extraFlags? }`; `profile.leaseProvider` must match the manifest. Core supplies connection identity, session defaults, Metro settings, and persists the profile. Its async `verify({ flags, env })` callback returns the provider verification result. These callbacks run without allocating a device and their temporary runtimes are shut down afterwards. Bundle the plugin implementation and ship ready-to-run ESM: installation disables lifecycle scripts. Bundle shared implementation helpers with the plugin; `AppError` carries a shared brand so core preserves its code and details across package copies. Import types with `import type` to keep them out of the runtime dependency graph. From 5e1319d7c7d7c26462c27817c5ebae8b9ed3f264 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 15:17:29 +0530 Subject: [PATCH 41/57] revert: keep the AppError brand check as it was Reverts c553ed71e and aab5684cc. Plugins are trusted in-process code, so a forged brand is not a threat the maintainer wants handled here. Co-Authored-By: Claude Opus 5.5 --- packages/kernel/src/errors.test.ts | 3 --- packages/kernel/src/errors.ts | 4 +++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/kernel/src/errors.test.ts b/packages/kernel/src/errors.test.ts index 0a0208cc0d..0d47220b90 100644 --- a/packages/kernel/src/errors.test.ts +++ b/packages/kernel/src/errors.test.ts @@ -191,9 +191,6 @@ test('bundled plugin errors preserve codes and details without changing subclass assert.deepEqual(normalized.details, { provider: 'example' }); const ordinary = Object.assign(new Error('bad plugin profile'), { code: 'INVALID_ARGS' }); assert.equal(ordinary instanceof AppError, false); - const forged = { [Symbol.for('agent-device.AppError')]: true }; - assert.equal(forged instanceof AppError, false); - assert.equal(normalizeError(forged).code, 'UNKNOWN'); class SpecificError extends AppError {} assert.ok(new SpecificError('COMMAND_FAILED', 'specific') instanceof SpecificError); assert.equal(foreign instanceof SpecificError, false); diff --git a/packages/kernel/src/errors.ts b/packages/kernel/src/errors.ts index a0e383dc9e..56ed18b6bd 100644 --- a/packages/kernel/src/errors.ts +++ b/packages/kernel/src/errors.ts @@ -42,7 +42,9 @@ export class AppError extends Error { static [Symbol.hasInstance](value: unknown): boolean { if (this !== AppError) return Function.prototype[Symbol.hasInstance].call(this, value); return ( - value instanceof Error && (value as Error & Record)[APP_ERROR_BRAND] === true + typeof value === 'object' && + value !== null && + (value as Record)[APP_ERROR_BRAND] === true ); } From 6ad12ebf6d6d5bb0a9b1bc62ff5b8ba58604b7be Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 15:17:29 +0530 Subject: [PATCH 42/57] revert: drop the plugin host archive fix with the apple host block Reverts 1d528624a; ProviderPluginHost.apple is removed next. Co-Authored-By: Claude Opus 5.5 --- src/plugins/host.test.ts | 30 ------------------------------ src/plugins/host.ts | 3 --- 2 files changed, 33 deletions(-) delete mode 100644 src/plugins/host.test.ts diff --git a/src/plugins/host.test.ts b/src/plugins/host.test.ts deleted file mode 100644 index af6870f36a..0000000000 --- a/src/plugins/host.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -import assert from 'node:assert/strict'; -import fs from 'node:fs'; -import path from 'node:path'; -import { test } from 'vitest'; -import { runCmdSync } from '@agent-device/host-kit/command'; -import { createPluginHost } from './host.ts'; -import { mkdtempForTestSync } from '../__tests__/test-utils/tmp-dir.ts'; - -test('archiving replaces a stale archive instead of merging into it', async () => { - const root = mkdtempForTestSync('plugin-host-archive-'); - try { - const archivePath = path.join(root, 'App.zip'); - fs.mkdirSync(path.join(root, 'Stale.app')); - fs.writeFileSync(path.join(root, 'Stale.app', 'Info.plist'), 'stale'); - runCmdSync('zip', ['-qr', archivePath, 'Stale.app'], { cwd: root }); - fs.mkdirSync(path.join(root, 'App.app')); - fs.writeFileSync(path.join(root, 'App.app', 'Info.plist'), 'fresh'); - - await createPluginHost({}, undefined).apple.archiveDirectory({ - sourceDirectory: root, - entryName: 'App.app', - archivePath, - }); - - const entries = runCmdSync('unzip', ['-Z1', archivePath]).stdout.trim().split('\n'); - assert.deepEqual(entries.sort(), ['App.app/', 'App.app/Info.plist']); - } finally { - fs.rmSync(root, { recursive: true, force: true }); - } -}); diff --git a/src/plugins/host.ts b/src/plugins/host.ts index 61e7e22948..98961a65c2 100644 --- a/src/plugins/host.ts +++ b/src/plugins/host.ts @@ -1,4 +1,3 @@ -import fs from 'node:fs/promises'; import { AppError } from '@agent-device/kernel/errors'; import { execFailureDetails, runCmd } from '@agent-device/host-kit/command'; import { readVersion } from '@agent-device/host-kit/version'; @@ -15,8 +14,6 @@ export function createPluginHost( createError: (code, message, details) => new AppError(code, message, details), apple: Object.freeze({ archiveDirectory: async ({ sourceDirectory, entryName, archivePath }) => { - // zip updates an existing archive in place, so a stale one would keep its old entries. - await fs.rm(archivePath, { force: true }); const args = ['-qr', archivePath, entryName]; const result = await runCmd('zip', args, { cwd: sourceDirectory, timeoutMs: 120_000 }); if (result.exitCode !== 0) { From e881abd5d4b4a4db024f6b77883e5b78dc160483 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 15:17:59 +0530 Subject: [PATCH 43/57] refactor(plugins): remove the unused apple block from the plugin host No plugin calls host.apple, and archiveDirectory copied the Limrun dependency's zip helper. A later plugin that needs Apple packaging can share one extracted helper with Limrun. Co-Authored-By: Claude Opus 5.5 --- packages/provider-testmu/src/plugin.test.ts | 1 - src/plugins/host.ts | 21 --------------------- src/sdk/plugins.ts | 9 --------- website/docs/docs/plugins.md | 2 +- 4 files changed, 1 insertion(+), 32 deletions(-) diff --git a/packages/provider-testmu/src/plugin.test.ts b/packages/provider-testmu/src/plugin.test.ts index 2811237a32..29621ae951 100644 --- a/packages/provider-testmu/src/plugin.test.ts +++ b/packages/provider-testmu/src/plugin.test.ts @@ -19,7 +19,6 @@ function host(env: Record): ProviderPluginHost { env, options: {}, clientVersion: '0.0.0-test', - apple: {} as ProviderPluginHost['apple'], createError: (code, message, details) => new AppError(code, message, details), }; } diff --git a/src/plugins/host.ts b/src/plugins/host.ts index 98961a65c2..3021c710ae 100644 --- a/src/plugins/host.ts +++ b/src/plugins/host.ts @@ -1,5 +1,4 @@ import { AppError } from '@agent-device/kernel/errors'; -import { execFailureDetails, runCmd } from '@agent-device/host-kit/command'; import { readVersion } from '@agent-device/host-kit/version'; import type { ProviderPluginHost } from '../sdk/plugins.ts'; @@ -12,25 +11,5 @@ export function createPluginHost( options: Object.freeze({ ...options }), clientVersion: readVersion(), createError: (code, message, details) => new AppError(code, message, details), - apple: Object.freeze({ - archiveDirectory: async ({ sourceDirectory, entryName, archivePath }) => { - const args = ['-qr', archivePath, entryName]; - const result = await runCmd('zip', args, { cwd: sourceDirectory, timeoutMs: 120_000 }); - if (result.exitCode !== 0) { - throw new AppError('COMMAND_FAILED', 'Failed to package iOS app for provider install', { - command: ['zip', ...args].join(' '), - ...execFailureDetails(result), - }); - } - }, - resolveAppAlias: async (app) => { - const { resolveIosAppAlias } = await import('@agent-device/platform-apple/app-resolution'); - return await resolveIosAppAlias(app); - }, - readBundleAppName: async (appPath) => { - const { readIosBundleInfo } = await import('@agent-device/platform-apple/install-artifact'); - return (await readIosBundleInfo(appPath)).appName; - }, - }), } satisfies ProviderPluginHost); } diff --git a/src/sdk/plugins.ts b/src/sdk/plugins.ts index 753ce229b0..8cbc13d56c 100644 --- a/src/sdk/plugins.ts +++ b/src/sdk/plugins.ts @@ -4,14 +4,5 @@ export type ProviderPluginHost = Readonly<{ env: Readonly>; options: Readonly>; clientVersion: string; - apple: Readonly<{ - archiveDirectory(options: { - sourceDirectory: string; - entryName: string; - archivePath: string; - }): Promise; - resolveAppAlias(app: string): Promise; - readBundleAppName(appPath: string): Promise; - }>; createError(code: AppErrorCode, message: string, details?: AppErrorDetails): AppError; }>; diff --git a/website/docs/docs/plugins.md b/website/docs/docs/plugins.md index 62024ec3bc..d127cccbaf 100644 --- a/website/docs/docs/plugins.md +++ b/website/docs/docs/plugins.md @@ -27,7 +27,7 @@ The interface is experimental. Publish this declaration in your package manifest {"agentDevicePlugin": {"apiVersion": 1, "provider": "example", "entry": "./dist/plugin.mjs"}} ``` -Use `agent-device` as a development dependency. The default factory accepts `ProviderPluginHost` from `agent-device/plugins`: `env`, package-specific `options`, `clientVersion`, Apple app packaging and resolution helpers under `apple`, and `createError` for host-recognized errors. Return `{ runtime, platformModule }` implementing the [provider runtime](https://github.com/callstack/agent-device/blob/main/packages/contracts/src/provider-device-runtime.ts) and [platform module](https://github.com/callstack/agent-device/blob/main/packages/contracts/src/platform-runtime-operations.ts) contracts. Both provider IDs must match the manifest; the module owner must declare `kind: 'provider-runtime'` and a nonempty `instance`. Core checks required runtime methods and owner metadata at startup; operation contracts are checked when used. Provider packages own implementation types; the SDK exposes only factory context. Set options through `plugins[""].options` in user config; leave `installation` unchanged. +Use `agent-device` as a development dependency. The default factory accepts `ProviderPluginHost` from `agent-device/plugins`: `env`, package-specific `options`, `clientVersion`, and `createError` for host-recognized errors. Return `{ runtime, platformModule }` implementing the [provider runtime](https://github.com/callstack/agent-device/blob/main/packages/contracts/src/provider-device-runtime.ts) and [platform module](https://github.com/callstack/agent-device/blob/main/packages/contracts/src/platform-runtime-operations.ts) contracts. Both provider IDs must match the manifest; the module owner must declare `kind: 'provider-runtime'` and a nonempty `instance`. Core checks required runtime methods and owner metadata at startup; operation contracts are checked when used. Provider packages own implementation types; the SDK exposes only factory context. Set options through `plugins[""].options` in user config; leave `installation` unchanged. Keep initialization prompt and free of network I/O or device allocation; a stalled factory blocks startup. Load platform mechanics through `platformModule.loadRuntime` and perform remote work in request-bound operations. A failing factory cleans up its own resources; core shuts down previously returned runtimes if another plugin fails. From b7456b47926cf1ea859df77b0210810d138f0bb1 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 15:19:29 +0530 Subject: [PATCH 44/57] refactor(webdriver): one owner for lease-flag and credential readers requireRequest, requireRequestPlatform, requireFlag, readFlag and requireEnv move from provider-definitions.ts into webdriver-utils.ts and are exported from the provider-webdriver/plugin subpath; the TestMu plugin drops its copies. requireEnv keeps the trim-empty rule, so a whitespace-only BrowserStack credential now also fails as missing. Co-Authored-By: Claude Opus 5.5 --- packages/provider-testmu/src/plugin.ts | 52 +++---------------- packages/provider-webdriver/src/plugin.ts | 7 +++ .../src/provider-definitions.ts | 49 +++-------------- .../src/webdriver-utils.test.ts | 22 ++++++++ .../provider-webdriver/src/webdriver-utils.ts | 44 ++++++++++++++++ 5 files changed, 88 insertions(+), 86 deletions(-) diff --git a/packages/provider-testmu/src/plugin.ts b/packages/provider-testmu/src/plugin.ts index 545d93eb55..3ba57bd36e 100644 --- a/packages/provider-testmu/src/plugin.ts +++ b/packages/provider-testmu/src/plugin.ts @@ -2,9 +2,14 @@ import type { ProviderPluginHost } from 'agent-device/plugins'; import type { WebDriverPluginOptions } from 'agent-device/plugins/webdriver'; import type { ProviderProfileFieldDeclaration } from '@agent-device/contracts/provider-profile-fields'; import type { ProviderDeviceType } from '@agent-device/contracts/remote'; -import type { LeaseLifecycleContext } from '@agent-device/contracts/device'; -import { AppError } from '@agent-device/kernel/errors'; -import { buildCloudWebDriverBaseCapabilities } from '@agent-device/provider-webdriver/plugin'; +import { + buildCloudWebDriverBaseCapabilities, + readFlag, + requireEnv, + requireFlag, + requireRequest, + requireRequestPlatform, +} from '@agent-device/provider-webdriver/plugin'; import { createTestMuConnection } from './connection.ts'; const TESTMU_WEBDRIVER_ENDPOINT = 'https://mobile-hub.lambdatest.com/wd/hub/'; const TESTMU_CAPABILITY_OVERRIDES = { @@ -153,44 +158,3 @@ function testMuAppUploadEndpoint( ? env.TESTMU_REAL_DEVICE_APP_UPLOAD_ENDPOINT : env.TESTMU_APP_UPLOAD_ENDPOINT; } - -function requireRequest( - req: LeaseLifecycleContext | undefined, - providerLabel: string, -): LeaseLifecycleContext { - if (req) return req; - throw new AppError( - 'INVALID_ARGS', - `${providerLabel} lease allocation requires provider profile flags on the request.`, - ); -} - -function requireRequestPlatform( - req: LeaseLifecycleContext, - providerLabel: string, -): 'android' | 'ios' { - const platform = req.flags?.platform; - if (platform === 'android' || platform === 'ios') return platform; - throw new AppError('INVALID_ARGS', `${providerLabel} requires --platform ios|android.`); -} - -function requireFlag(req: LeaseLifecycleContext, key: string, message: string): string { - const value = readFlag(req, key); - if (value) return value; - throw new AppError('INVALID_ARGS', message); -} - -function readFlag(req: LeaseLifecycleContext, key: string): string | undefined { - const value = req.flags?.[key]; - return typeof value === 'string' && value.length > 0 ? value : undefined; -} - -function requireEnv( - env: ProviderPluginHost['env'], - key: keyof ProviderPluginHost['env'], - providerLabel: string, -): string { - const value = env[key]; - if (value?.trim()) return value; - throw new AppError('INVALID_ARGS', `${providerLabel} requires ${key} in the environment.`); -} diff --git a/packages/provider-webdriver/src/plugin.ts b/packages/provider-webdriver/src/plugin.ts index 9662a9965e..c7e4b26917 100644 --- a/packages/provider-webdriver/src/plugin.ts +++ b/packages/provider-webdriver/src/plugin.ts @@ -23,3 +23,10 @@ export { } from './webdriver-utils.ts'; export { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; export { buildCloudWebDriverBaseCapabilities } from './capabilities.ts'; +export { + readFlag, + requireEnv, + requireFlag, + requireRequest, + requireRequestPlatform, +} from './webdriver-utils.ts'; diff --git a/packages/provider-webdriver/src/provider-definitions.ts b/packages/provider-webdriver/src/provider-definitions.ts index 42acf03ee1..50b4400f9b 100644 --- a/packages/provider-webdriver/src/provider-definitions.ts +++ b/packages/provider-webdriver/src/provider-definitions.ts @@ -24,10 +24,16 @@ import { } from './browserstack-device-features.ts'; import { CLOUD_WEBDRIVER_PROVIDERS, type CloudWebDriverKnownProviderName } from './providers.ts'; import { readAwsDeviceFarmRegionFromArn } from './connection-verification.ts'; +import { + readFlag, + requireEnv, + requireFlag, + requireRequest, + requireRequestPlatform, +} from './webdriver-utils.ts'; import { buildCloudWebDriverBaseCapabilities, createCloudWebDriverRuntime, - type CloudWebDriverPlatform, type CloudWebDriverRuntime, } from './runtime.ts'; @@ -318,47 +324,6 @@ export function createCloudWebDriverProviderDefinitions( ]; } -function requireRequest( - req: LeaseLifecycleContext | undefined, - providerLabel: string, -): LeaseLifecycleContext { - if (req) return req; - throw new AppError( - 'INVALID_ARGS', - `${providerLabel} lease allocation requires provider profile flags on the request.`, - ); -} - -function requireRequestPlatform( - req: LeaseLifecycleContext, - providerLabel: string, -): CloudWebDriverPlatform { - const platform = req.flags?.platform; - if (platform === 'android' || platform === 'ios') return platform; - throw new AppError('INVALID_ARGS', `${providerLabel} requires --platform ios|android.`); -} - -function requireFlag(req: LeaseLifecycleContext, key: string, message: string): string { - const value = readFlag(req, key); - if (value) return value; - throw new AppError('INVALID_ARGS', message); -} - -function readFlag(req: LeaseLifecycleContext, key: string): string | undefined { - const value = req.flags?.[key]; - return typeof value === 'string' && value.length > 0 ? value : undefined; -} - -function requireEnv( - env: DefaultCloudWebDriverProviderRuntimeEnv, - key: keyof DefaultCloudWebDriverProviderRuntimeEnv, - providerLabel: string, -): string { - const value = env[key]; - if (value) return value; - throw new AppError('INVALID_ARGS', `${providerLabel} requires ${key} in the environment.`); -} - function requireAwsValue( req: LeaseLifecycleContext, env: DefaultCloudWebDriverProviderRuntimeEnv, diff --git a/packages/provider-webdriver/src/webdriver-utils.test.ts b/packages/provider-webdriver/src/webdriver-utils.test.ts index ca026630ec..e0ee9e2abe 100644 --- a/packages/provider-webdriver/src/webdriver-utils.test.ts +++ b/packages/provider-webdriver/src/webdriver-utils.test.ts @@ -9,6 +9,8 @@ import { appFileUploadForm, createHubUploadApp, postHubAppUpload, + readFlag, + requireEnv, resolveHubAppReference, trimLeadingSlash, trimTrailingSlash, @@ -203,3 +205,23 @@ test('the hub app resolver surfaces the grammar rejection of a malformed referen await fs.rm(tempDir, { recursive: true, force: true }); } }); + +test('a whitespace-only credential is missing', () => { + assert.equal(requireEnv({ USER: 'u' }, 'USER', 'Hub'), 'u'); + for (const env of [{}, { USER: '' }, { USER: ' \t' }]) { + assert.throws( + () => requireEnv(env, 'USER', 'Hub'), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + error.message === 'Hub requires USER in the environment.', + ); + } +}); + +test('only non-empty string flags are read', () => { + const req = { flags: { device: 'Pixel 8', empty: '', count: 3 } }; + assert.equal(readFlag(req, 'device'), 'Pixel 8'); + assert.equal(readFlag(req, 'empty'), undefined); + assert.equal(readFlag(req, 'count'), undefined); +}); diff --git a/packages/provider-webdriver/src/webdriver-utils.ts b/packages/provider-webdriver/src/webdriver-utils.ts index 3b12be10db..c4dae20c1f 100644 --- a/packages/provider-webdriver/src/webdriver-utils.ts +++ b/packages/provider-webdriver/src/webdriver-utils.ts @@ -3,6 +3,7 @@ import { readFile, stat } from 'node:fs/promises'; import path from 'node:path'; import type { DeviceLease, + LeaseLifecycleContext, ProviderDeviceInstallOptions, ProviderDeviceInstallResult, } from '@agent-device/contracts/device'; @@ -323,3 +324,46 @@ export function requireProviderDeviceOrientation( capability: spec.capability, }); } + +/** Lease-flag and credential readers every hosted-WebDriver provider shares. */ +export function requireRequest( + req: LeaseLifecycleContext | undefined, + providerLabel: string, +): LeaseLifecycleContext { + if (req) return req; + throw new AppError( + 'INVALID_ARGS', + `${providerLabel} lease allocation requires provider profile flags on the request.`, + ); +} + +export function requireRequestPlatform( + req: LeaseLifecycleContext, + providerLabel: string, +): 'android' | 'ios' { + const platform = req.flags?.platform; + if (platform === 'android' || platform === 'ios') return platform; + throw new AppError('INVALID_ARGS', `${providerLabel} requires --platform ios|android.`); +} + +export function requireFlag(req: LeaseLifecycleContext, key: string, message: string): string { + const value = readFlag(req, key); + if (value) return value; + throw new AppError('INVALID_ARGS', message); +} + +export function readFlag(req: LeaseLifecycleContext, key: string): string | undefined { + const value = req.flags?.[key]; + return typeof value === 'string' && value.length > 0 ? value : undefined; +} + +/** A whitespace-only credential is missing, not one the provider should reject later. */ +export function requireEnv( + env: Readonly>>, + key: Key, + providerLabel: string, +): string { + const value = env[key]; + if (value?.trim()) return value; + throw new AppError('INVALID_ARGS', `${providerLabel} requires ${key} in the environment.`); +} From add09949f634367e523f0170eadc0cb2f343ddb7 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 15:19:55 +0530 Subject: [PATCH 45/57] refactor(testmu): import session and device-feature modules statically connection.ts already imports them statically, so the dynamic imports in plugin.ts saved nothing; the Fallow exemption for the lazy read goes with them. Co-Authored-By: Claude Opus 5.5 --- .fallowrc.json | 5 ----- packages/provider-testmu/src/plugin.ts | 22 +++++++++++----------- packages/provider-testmu/src/testmu.ts | 6 +++--- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/.fallowrc.json b/.fallowrc.json index c910e9013a..96df109096 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -391,11 +391,6 @@ "file": "packages/provider-limrun/src/index.ts", "exports": ["LimrunIosCommandExecution"] }, - { - "comment": "TestMu session preparation reads this off the lazy loadTestMuDeviceFeatures() import in packages/provider-testmu/src/plugin.ts, which keeps the plugin entry's eager closure unchanged; Fallow cannot connect the dynamic member read.", - "file": "packages/provider-testmu/src/testmu-device-features.ts", - "exports": ["buildTestMuDeviceFeatureCapabilities"] - }, { "comment": "Converting the contracts façades from `export *` to explicit named re-exports (the pin-table retirement) made these individually visible to --production analysis for the first time; a bare star previously hid them from this exact check. isRecord/IOS_SAFARI_BUNDLE_ID/REPLAY_DIVERGENCE_* have no production consumer. Kept rather than narrowed here so the façade's re-export surface stays byte-identical to the symbol set the retired pin table asserted — narrowing the surface is a follow-up with its own review, not a side effect of this mechanical conversion.", "file": "packages/contracts/src/facades/{client,command,divergence,recording}.ts", diff --git a/packages/provider-testmu/src/plugin.ts b/packages/provider-testmu/src/plugin.ts index 3ba57bd36e..f844178ad6 100644 --- a/packages/provider-testmu/src/plugin.ts +++ b/packages/provider-testmu/src/plugin.ts @@ -11,6 +11,17 @@ import { requireRequestPlatform, } from '@agent-device/provider-webdriver/plugin'; import { createTestMuConnection } from './connection.ts'; +import { + buildTestMuCapabilities, + createTestMuUploadApp, + listTestMuCloudArtifacts, + resolveTestMuAppReference, +} from './testmu.ts'; +import { + buildTestMuDeviceFeatureCapabilities, + readTestMuDeviceFeatureFields, + readTestMuDeviceType, +} from './testmu-device-features.ts'; const TESTMU_WEBDRIVER_ENDPOINT = 'https://mobile-hub.lambdatest.com/wd/hub/'; const TESTMU_CAPABILITY_OVERRIDES = { install: { @@ -27,9 +38,6 @@ const TESTMU_CAPABILITY_OVERRIDES = { }, } as const; -const loadTestMu = async () => await import('./testmu.ts'); -const loadTestMuDeviceFeatures = async () => await import('./testmu-device-features.ts'); - const TESTMU_PROFILE_FIELDS: ProviderProfileFieldDeclaration = { provider: 'testmu', label: 'TestMu AI', @@ -64,7 +72,6 @@ export default function testMuPlugin(host: ProviderPluginHost) { providerSessionId: string | undefined, env: ProviderPluginHost['env'], ) { - const { listTestMuCloudArtifacts } = await loadTestMu(); return await listTestMuCloudArtifacts(provider, providerSessionId, { clientVersion: host.clientVersion, ...requireTestMuCredentials(env, 'TestMu AI artifact lookup'), @@ -82,13 +89,6 @@ export default function testMuPlugin(host: ProviderPluginHost) { await listTestMuArtifactsFromEnv(provider, providerSessionId, env), prepareSession: async ({ req, lease, base }) => { const request = requireRequest(req, 'TestMu AI'); - const { buildTestMuCapabilities, createTestMuUploadApp, resolveTestMuAppReference } = - await loadTestMu(); - const { - buildTestMuDeviceFeatureCapabilities, - readTestMuDeviceFeatureFields, - readTestMuDeviceType, - } = await loadTestMuDeviceFeatures(); const deviceType = readTestMuDeviceType(request.flags); const uploadEndpoint = testMuAppUploadEndpoint(env, deviceType); const credentials = requireTestMuCredentials(env, 'TestMu AI'); diff --git a/packages/provider-testmu/src/testmu.ts b/packages/provider-testmu/src/testmu.ts index c6ac25fe5c..8ea6b12e87 100644 --- a/packages/provider-testmu/src/testmu.ts +++ b/packages/provider-testmu/src/testmu.ts @@ -19,9 +19,9 @@ import { asOptionalRecord } from '@agent-device/kernel/record'; import { canonicalTestMuAppReference, isTestMuAppReference } from './providers.ts'; /** - * TestMu session, upload, and artifact mechanics. Loaded on demand by the provider definition; - * `isRealMobile` in `lt:options` is what routes a session to the real or virtual device pool, and - * the hostnames still carry the lambdatest.com brand. + * TestMu session, upload, and artifact mechanics. `isRealMobile` in `lt:options` is what routes a + * session to the real or virtual device pool, and the hostnames still carry the lambdatest.com + * brand. */ const TESTMU_APP_UPLOAD_ENDPOINTS: Record = { real: 'https://manual-api.lambdatest.com/app/upload/realDevice', From ffb76e0fd07704434107cebd8afd396de5caca24 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 15:22:07 +0530 Subject: [PATCH 46/57] build(testmu): typecheck the plugin against core source, not dist CI typechecks before it builds, so agent-device/plugins and agent-device/plugins/webdriver resolved to dist types that did not exist yet and ProviderPluginHost became any. Both tsconfigs map the two specifiers to src/sdk, as examples/sdk does; provider-testmu maps through its workspace link so the files stay outside its rootDir. Co-Authored-By: Claude Opus 5.5 --- packages/provider-testmu/tsconfig.json | 6 +++++- tsconfig.json | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/provider-testmu/tsconfig.json b/packages/provider-testmu/tsconfig.json index 935c871a4d..aa0881ab1c 100644 --- a/packages/provider-testmu/tsconfig.json +++ b/packages/provider-testmu/tsconfig.json @@ -6,7 +6,11 @@ "emitDeclarationOnly": true, "declaration": true, "declarationDir": "./dist-types", - "rootDir": "./src" + "rootDir": "./src", + "paths": { + "agent-device/plugins": ["./node_modules/agent-device/src/sdk/plugins.ts"], + "agent-device/plugins/webdriver": ["./node_modules/agent-device/src/sdk/plugin-webdriver.ts"] + } }, "include": ["src"] } diff --git a/tsconfig.json b/tsconfig.json index b77afe95cb..d99f51c1a3 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,7 +16,11 @@ "erasableSyntaxOnly": true, "rewriteRelativeImportExtensions": true, "verbatimModuleSyntax": true, - "types": ["node"] + "types": ["node"], + "paths": { + "agent-device/plugins": ["./src/sdk/plugins.ts"], + "agent-device/plugins/webdriver": ["./src/sdk/plugin-webdriver.ts"] + } }, "include": [ "src", From 789f997c2ea18ae21e5b4010d34e4a998994693c Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 15:27:27 +0530 Subject: [PATCH 47/57] chore(fallow): credit the testmu plugin's type-only agent-device link With the tsconfig mapping to core source, Fallow resolves the plugin's agent-device/plugins imports to src/sdk files and reported the devDependency unused, which a clean checkout surfaced. Co-Authored-By: Claude Opus 5.5 --- .fallowrc.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.fallowrc.json b/.fallowrc.json index 96df109096..a3b17a524c 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -61,6 +61,10 @@ // @agent-device/provider-limrun owns the source import, while the published // root build externalizes @limrun/api and retains runtime imports in packed chunks. "@limrun/api", + // @agent-device/testmu imports agent-device type-only, and its tsconfig maps those specifiers + // through this workspace link to core source so typecheck needs no build; Fallow then credits + // the source files rather than the package. + "agent-device", // Oxlint resolves the shared config's JS plugins dynamically from their package names. "@nkzw/eslint-plugin", "eslint-plugin-no-only-tests", From 891df76261163ed79e8f2d16e08da91d7f1ad52c Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 16:42:28 +0530 Subject: [PATCH 48/57] test(testmu): assert only Limrun's device-type refusal at connect Main's refusal names a field's aliases too, so the combined message this test matched no longer exists, and main already covers Limrun refusing the OS version. Keep the part this PR adds: --provider-device-type is refused by Limrun. Co-Authored-By: Claude Opus 5.5 --- src/__tests__/cloud-connect-testmu.test.ts | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/__tests__/cloud-connect-testmu.test.ts b/src/__tests__/cloud-connect-testmu.test.ts index 3a148e4e74..2a6d157460 100644 --- a/src/__tests__/cloud-connect-testmu.test.ts +++ b/src/__tests__/cloud-connect-testmu.test.ts @@ -337,29 +337,16 @@ test('providers other than TestMu refuse --provider-device-type before saving a } }); -test('connect limrun refuses profile fields Limrun does not read', async () => { +test('connect limrun refuses the device type', async () => { const result = await runCliCapture( - [ - 'connect', - 'limrun', - '--platform', - 'ios', - '--provider-device-type', - 'real', - '--provider-os-version', - '18', - '--json', - ], + ['connect', 'limrun', '--platform', 'ios', '--provider-device-type', 'real', '--json'], { env: { LIMRUN_API_KEY: 'lim_test_key' }, stateDirPrefix: 'agent-device-connect-limrun-device-type-', }, ); assert.equal(result.code, 1); - assert.match( - result.stdout, - /--provider-os-version, --provider-device-type are not supported by Limrun/, - ); + assert.match(result.stdout, /--provider-device-type is not supported by Limrun/); }); function readGeneratedConfig(configPath: string): { From 11b40ece07d60b8364a3dc1b9a4c8ee725cba7ac Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 17:04:57 +0530 Subject: [PATCH 49/57] refactor(providers): share the lower-case scheme rule for hub app references canonicalTestMuAppReference repeated the rule canonicalBrowserStackAppReference applies: a hub matches only the lower-case scheme, so BS:// and LT:// are rewritten to bs:// and lt://. Both now call canonicalSchemeReference from the dependency-free provider-webdriver/providers subpath, so neither eager closure grows. Behaviour is unchanged: BrowserStack still returns undefined without the scheme, and TestMu still returns the value as given. Co-Authored-By: Claude Opus 5.5 --- packages/provider-testmu/src/providers.ts | 5 +++-- packages/provider-webdriver/src/providers.ts | 14 ++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/packages/provider-testmu/src/providers.ts b/packages/provider-testmu/src/providers.ts index a2e9ccb5ef..11260f933f 100644 --- a/packages/provider-testmu/src/providers.ts +++ b/packages/provider-testmu/src/providers.ts @@ -1,8 +1,9 @@ +import { canonicalSchemeReference } from '@agent-device/provider-webdriver/providers'; + export function isTestMuAppReference(value: string): boolean { return /^lt:\/\/[\w.-]+$/.test(value); } -/** URI schemes are case-insensitive; the hub matches the lower-case `lt://` spelling. */ export function canonicalTestMuAppReference(value: string): string { - return value.slice(0, 5).toLowerCase() === 'lt://' ? `lt://${value.slice(5)}` : value; + return canonicalSchemeReference(value, 'lt://') ?? value; } diff --git a/packages/provider-webdriver/src/providers.ts b/packages/provider-webdriver/src/providers.ts index d0d9ab6e4f..742e10b653 100644 --- a/packages/provider-webdriver/src/providers.ts +++ b/packages/provider-webdriver/src/providers.ts @@ -17,14 +17,16 @@ export function isCloudWebDriverProviderName( const BROWSERSTACK_APP_SCHEME = 'bs://'; /** - * URI schemes are case-insensitive, but BrowserStack only matches the lower-case spelling, so - * `BS://id` is returned as `bs://id`. Anything without the scheme returns undefined. + * URI schemes are case-insensitive, but hosted hubs only match the lower-case spelling, so + * `BS://id` is returned as `bs://id`. Anything without the lower-case `scheme` returns undefined. */ +export function canonicalSchemeReference(app: string, scheme: string): string | undefined { + if (app.slice(0, scheme.length).toLowerCase() !== scheme) return undefined; + return `${scheme}${app.slice(scheme.length)}`; +} + export function canonicalBrowserStackAppReference(app: string): string | undefined { - if (app.slice(0, BROWSERSTACK_APP_SCHEME.length).toLowerCase() !== BROWSERSTACK_APP_SCHEME) { - return undefined; - } - return `${BROWSERSTACK_APP_SCHEME}${app.slice(BROWSERSTACK_APP_SCHEME.length)}`; + return canonicalSchemeReference(app, BROWSERSTACK_APP_SCHEME); } /** An id outside this grammar would pass every local check and fail only at session creation. */ From 6b19fdeeebbda43e1f876dbc086530ad5ea37101 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Mon, 5 Oct 2026 17:04:57 +0530 Subject: [PATCH 50/57] refactor(provider-webdriver): export the plugin's webdriver-utils readers in one block Co-Authored-By: Claude Opus 5.5 --- packages/provider-webdriver/src/plugin.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/provider-webdriver/src/plugin.ts b/packages/provider-webdriver/src/plugin.ts index c7e4b26917..8c6dac887b 100644 --- a/packages/provider-webdriver/src/plugin.ts +++ b/packages/provider-webdriver/src/plugin.ts @@ -18,15 +18,13 @@ export { fetchProviderSessionDetails, fetchProviderVerificationJson, postHubAppUpload, - requireProviderDeviceOrientation, - resolveHubAppReference, -} from './webdriver-utils.ts'; -export { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; -export { buildCloudWebDriverBaseCapabilities } from './capabilities.ts'; -export { readFlag, requireEnv, requireFlag, + requireProviderDeviceOrientation, requireRequest, requireRequestPlatform, + resolveHubAppReference, } from './webdriver-utils.ts'; +export { cloudArtifactsReadyOrPending, urlArtifactFromDetails } from './artifact-results.ts'; +export { buildCloudWebDriverBaseCapabilities } from './capabilities.ts'; From ada0ae962d5c6a6b17b293611872ebef53f26742 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Tue, 6 Oct 2026 13:09:39 +0530 Subject: [PATCH 51/57] fix(providers): refuse a lease when the daemon holds other plugin credentials A provider plugin can declare agentDevicePlugin.credentialVariables in its manifest. providerCredentialFingerprint falls back to that declaration for providers without a built-in reader, so the client (shell env) and the daemon (startup env) both fingerprint plugin credentials without loading plugin code. TestMu declares LT_USERNAME and LT_ACCESS_KEY, which brings it under the provider-credentials-changed refusal from #3207. Co-Authored-By: Claude Opus 5.5 --- packages/provider-testmu/package.json | 6 ++- src/commands/schema/cli-help.ts | 2 +- src/daemon/handlers/__tests__/lease.test.ts | 60 +++++++++++++++++++++ src/plugins/manifest.test.ts | 19 +++++++ src/plugins/manifest.ts | 14 +++++ src/plugins/plugin.fixtures.ts | 18 +++++++ src/provider-credential-fingerprint.test.ts | 45 +++++++++++++++- src/provider-credential-fingerprint.ts | 20 ++++++- website/docs/docs/plugins.md | 2 + website/docs/docs/testmu.md | 7 ++- 10 files changed, 187 insertions(+), 6 deletions(-) diff --git a/packages/provider-testmu/package.json b/packages/provider-testmu/package.json index 2430d0f468..4263f02485 100644 --- a/packages/provider-testmu/package.json +++ b/packages/provider-testmu/package.json @@ -39,7 +39,11 @@ "supportsDeferredAppSelection": false, "supportsDirectPortReverse": false, "usesCloudWebDriverLease": true - } + }, + "credentialVariables": [ + "LT_USERNAME", + "LT_ACCESS_KEY" + ] }, "publishConfig": { "exports": { diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index d19d9ae5ce..33ee9411df 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -682,7 +682,7 @@ Rules: Use agent-device proxy for direct tunnel access to a Mac you control. Expose the printed proxy URL through cloudflared/ngrok, then run agent-device connect proxy with the tunnel URL and printed token before normal commands. Use Limrun, BrowserStack, AWS Device Farm, and TestMu AI through local provider profiles; they do not accept a remote agent-device daemon URL. Device cloud credentials must be available before the command starts. Limrun uses LIMRUN_API_KEY, or the LIM_*_INSTANCE_* variables for an existing instance. BrowserStack uses BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY. TestMu AI uses LT_USERNAME and LT_ACCESS_KEY. AWS Device Farm uses the AWS CLI credential chain, including CI-provided AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_SESSION_TOKEN, AWS profiles, or web identity role variables. - A local daemon keeps the Limrun and BrowserStack credentials it started with. When the shell holds different ones, the first command that allocates a lease refuses with reason provider-credentials-changed; run agent-device daemon stop with the same --state-dir, then rerun the command. A shell that sets none of them uses the daemon's. A daemon with an HTTP auth hook serves remote callers and does not compare. + A local daemon keeps the Limrun, BrowserStack, and TestMu AI credentials it started with. When the shell holds different ones, the first command that allocates a lease refuses with reason provider-credentials-changed; run agent-device daemon stop with the same --state-dir, then rerun the command. A shell that sets none of them uses the daemon's. A daemon with an HTTP auth hook serves remote callers and does not compare. Direct-provider connect performs read-only provider calls and saves active connection state only after verification succeeds. It never creates a device, instance, App Automate session, or AWS remote access session. connect without --session always creates a fresh remote session and prints that session in its next-step commands. Concurrent callers must pass the returned --session on every command; the ambient active connection is only a single-workflow convenience. To replace an existing connection, pass its returned session explicitly with --session --force. --force without --session creates another fresh session and does not release or overwrite an unrelated active connection. diff --git a/src/daemon/handlers/__tests__/lease.test.ts b/src/daemon/handlers/__tests__/lease.test.ts index 8de59f7099..1e30815cad 100644 --- a/src/daemon/handlers/__tests__/lease.test.ts +++ b/src/daemon/handlers/__tests__/lease.test.ts @@ -1,4 +1,6 @@ import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; import { test } from 'vitest'; import { handleLeaseCommands } from '../lease.ts'; import { LeaseRegistry } from '../../lease-registry.ts'; @@ -15,6 +17,7 @@ import { providerCredentialFingerprint, readDaemonProviderCredentials, } from '../../../provider-credential-fingerprint.ts'; +import { pluginHome, selectPluginManifest } from '../../../plugins/plugin.fixtures.ts'; import { HUMAN_CONTROL_LEASE_REQUEST, HUMAN_CONTROL_SCOPE, @@ -482,3 +485,60 @@ test('a daemon started without BrowserStack credentials refuses a shell that has assert.equal(outcome.error?.details?.reason, 'provider-credentials-changed'); assert.match(String(outcome.error?.message), /started without the browserstack credentials/); }); + +function testMuPluginEnv(): Record { + const manifest = path.resolve( + import.meta.dirname, + '../../../../packages/provider-testmu/package.json', + ); + const { home, env } = pluginHome(); + selectPluginManifest(home, JSON.parse(fs.readFileSync(manifest, 'utf8'))); + return env; +} + +test('a daemon holding other TestMu AI credentials refuses before allocation', async () => { + const home = testMuPluginEnv(); + const daemonEnv = { ...home, LT_USERNAME: 'user', LT_ACCESS_KEY: 'key-1' }; + const outcome = await allocateWithDaemonEnv( + providerAllocateRequest( + 'testmu', + providerCredentialFingerprint('testmu', { ...daemonEnv, LT_ACCESS_KEY: 'key-2' }), + ), + daemonEnv, + ); + + assert.equal(outcome.allocations, 0); + assert.equal(outcome.error?.code, 'INVALID_ARGS'); + assert.equal(outcome.error?.details?.reason, 'provider-credentials-changed'); + assert.equal(outcome.error?.details?.provider, 'testmu'); +}); + +test('a daemon holding the TestMu AI credentials of the shell allocates', async () => { + const env = { ...testMuPluginEnv(), LT_USERNAME: 'user', LT_ACCESS_KEY: 'key-1' }; + const outcome = await allocateWithDaemonEnv( + providerAllocateRequest('testmu', providerCredentialFingerprint('testmu', env)), + env, + ); + + assert.equal(outcome.error, undefined); + assert.equal(outcome.allocations, 1); +}); + +test('a daemon started without TestMu AI credentials refuses a shell that has them', async () => { + const home = testMuPluginEnv(); + const outcome = await allocateWithDaemonEnv( + providerAllocateRequest( + 'testmu', + providerCredentialFingerprint('testmu', { + ...home, + LT_USERNAME: 'user', + LT_ACCESS_KEY: 'key-1', + }), + ), + home, + ); + + assert.equal(outcome.allocations, 0); + assert.equal(outcome.error?.details?.reason, 'provider-credentials-changed'); + assert.match(String(outcome.error?.message), /started without the testmu credentials/); +}); diff --git a/src/plugins/manifest.test.ts b/src/plugins/manifest.test.ts index 4fbd7dfc5d..58767eb244 100644 --- a/src/plugins/manifest.test.ts +++ b/src/plugins/manifest.test.ts @@ -67,6 +67,25 @@ test('connection metadata admits local providers and rejects malformed or remote } }); +test('credential variables are optional environment variable names', () => { + const { home } = pluginHome(); + const directory = writePlugin(home); + const file = path.join(directory, 'package.json'); + const manifest = JSON.parse(fs.readFileSync(file, 'utf8')); + const read = (credentialVariables: unknown) => { + manifest.agentDevicePlugin.credentialVariables = credentialVariables; + fs.writeFileSync(file, JSON.stringify(manifest)); + return readPluginManifest(directory); + }; + assert.equal(read(undefined).agentDevicePlugin.credentialVariables, undefined); + assert.deepEqual(read(['EXAMPLE_USER', '_KEY_2']).agentDevicePlugin.credentialVariables, [ + 'EXAMPLE_USER', + '_KEY_2', + ]); + for (const invalid of [null, 'EXAMPLE_USER', {}, [''], ['example_user'], ['2KEY'], ['A-B'], [1]]) + assert.throws(() => read(invalid), { code: 'INVALID_ARGS' }, JSON.stringify(invalid)); +}); + test('every bundled provider runtime is reserved from plugins', () => { for (const provider of DEFAULT_PROVIDER_RUNTIME_REQUIRED_IDS) assert.ok((RESERVED_PLUGIN_PROVIDERS as readonly string[]).includes(provider), provider); diff --git a/src/plugins/manifest.ts b/src/plugins/manifest.ts index 675817af96..0be9362502 100644 --- a/src/plugins/manifest.ts +++ b/src/plugins/manifest.ts @@ -25,6 +25,7 @@ type PluginManifest = { provider: string; entry: string; connection?: ConnectionProviderCapabilities; + credentialVariables?: string[]; }; }; @@ -94,6 +95,19 @@ export function readPluginManifest(directory: string): PluginManifest { ); } } + const credentialVariables: unknown = declaration.credentialVariables; + if ( + credentialVariables !== undefined && + (!Array.isArray(credentialVariables) || + !credentialVariables.every( + (name) => typeof name === 'string' && /^[A-Z_][A-Z0-9_]*$/.test(name), + )) + ) { + throw new AppError( + 'INVALID_ARGS', + 'Plugin credentialVariables must list environment variable names', + ); + } return manifest as PluginManifest; } diff --git a/src/plugins/plugin.fixtures.ts b/src/plugins/plugin.fixtures.ts index 36d83ed134..d8d4a8c2f9 100644 --- a/src/plugins/plugin.fixtures.ts +++ b/src/plugins/plugin.fixtures.ts @@ -40,6 +40,24 @@ export function selectPlugin( ) { const installation = crypto.randomUUID(); writePlugin(path.join(home, 'plugins', installation), name, apiVersion, provider, source); + recordSelection(home, name, installation); +} + +/** Installs `manifest` as-is, with an entry file that throws if evaluated. */ +export function selectPluginManifest( + home: string, + manifest: { name: string; agentDevicePlugin: Record & { entry: string } }, +) { + const installation = crypto.randomUUID(); + const directory = path.join(home, 'plugins', installation, 'node_modules', manifest.name); + const entry = path.join(directory, manifest.agentDevicePlugin.entry); + fs.mkdirSync(path.dirname(entry), { recursive: true }); + fs.writeFileSync(path.join(directory, 'package.json'), JSON.stringify(manifest)); + fs.writeFileSync(entry, 'throw new Error("evaluated");'); + recordSelection(home, manifest.name, installation); +} + +function recordSelection(home: string, name: string, installation: string) { const configPath = path.join(home, 'config.json'); const config = fs.existsSync(configPath) ? JSON.parse(fs.readFileSync(configPath, 'utf8')) : {}; config.plugins ??= {}; diff --git a/src/provider-credential-fingerprint.test.ts b/src/provider-credential-fingerprint.test.ts index a73beacb14..6ce107c060 100644 --- a/src/provider-credential-fingerprint.test.ts +++ b/src/provider-credential-fingerprint.test.ts @@ -3,6 +3,7 @@ import { providerCredentialFingerprint, readDaemonProviderCredentials, } from './provider-credential-fingerprint.ts'; +import { pluginHome, selectPluginManifest } from './plugins/plugin.fixtures.ts'; const BROWSERSTACK_ENV = { BROWSERSTACK_USERNAME: 'user', BROWSERSTACK_ACCESS_KEY: 'key-1' }; @@ -58,7 +59,8 @@ test('a fingerprint hashes the exact values each provider reads', () => { }); test('a provider name that only matches an inherited object key has no fingerprint', () => { - expect(providerCredentialFingerprint('constructor', BROWSERSTACK_ENV)).toBe(undefined); + const env = { ...pluginHome().env, ...BROWSERSTACK_ENV }; + expect(providerCredentialFingerprint('constructor', env)).toBe(undefined); }); test('AWS Device Farm has no environment fingerprint', () => { @@ -109,3 +111,44 @@ test('a Limrun lease fingerprint covers only the leased platform and the account readDaemonProviderCredentials(before, '/state').fingerprint('limrun', 'ios-instance'), ).toBe(ios(rotatedAndroid)); }); + +function pluginWithCredentialVariables(credentialVariables?: string[], provider = 'example') { + const { home, env } = pluginHome(); + selectPluginManifest(home, { + name: '@example/provider', + version: '1.2.3', + agentDevicePlugin: { apiVersion: 1, provider, entry: './plugin.js', credentialVariables }, + }); + return env; +} + +test('a plugin provider fingerprint hashes the variables its manifest declares', () => { + const home = pluginWithCredentialVariables(['EXAMPLE_USER', 'EXAMPLE_KEY']); + const env = { ...home, EXAMPLE_USER: 'user', EXAMPLE_KEY: 'key-1' }; + const fingerprint = providerCredentialFingerprint('example', env); + + expect(fingerprint).toMatch(/^v1:[0-9a-f]{16}$/); + expect(providerCredentialFingerprint('example', { ...env, UNRELATED: 'x' })).toBe(fingerprint); + expect(providerCredentialFingerprint('example', { ...env, EXAMPLE_KEY: 'key-2' })).not.toBe( + fingerprint, + ); + expect(providerCredentialFingerprint('example', { ...env, EXAMPLE_KEY: 'key-1 ' })).not.toBe( + fingerprint, + ); + expect(providerCredentialFingerprint('example', { ...home, EXAMPLE_USER: ' ' })).toBe(undefined); + expect(readDaemonProviderCredentials(env, '/state').fingerprint('example')).toBe(fingerprint); +}); + +test('a plugin provider without declared credential variables has no fingerprint', () => { + const env = { ...pluginWithCredentialVariables(), EXAMPLE_USER: 'user' }; + expect(providerCredentialFingerprint('example', env)).toBe(undefined); + expect(providerCredentialFingerprint('other', env)).toBe(undefined); +}); + +test('a plugin declaration never adds a fingerprint to a bundled provider', () => { + const env = { + ...pluginWithCredentialVariables(['AWS_ACCESS_KEY_ID'], 'aws-device-farm'), + AWS_ACCESS_KEY_ID: 'id', + }; + expect(providerCredentialFingerprint('aws-device-farm', env)).toBe(undefined); +}); diff --git a/src/provider-credential-fingerprint.ts b/src/provider-credential-fingerprint.ts index d6d12ee404..df2cf4ce69 100644 --- a/src/provider-credential-fingerprint.ts +++ b/src/provider-credential-fingerprint.ts @@ -7,6 +7,8 @@ import { import type { LIMRUN_PROVIDER } from '@agent-device/provider-limrun'; import type { EnvMap } from '@agent-device/kernel/source-value'; import { readLimrunCredentialValues } from './provider-limrun-credentials.ts'; +import { RESERVED_PLUGIN_PROVIDERS } from './plugins/manifest.ts'; +import { installedPlugins } from './plugins/store.ts'; type CredentialValues = Readonly>; @@ -40,7 +42,23 @@ export function providerCredentialFingerprint( leaseBackend?: string, ): string | undefined { const read = PROVIDER_CREDENTIAL_READERS.get(provider); - return read ? digest(read(env, leaseBackend)) : undefined; + if (read) return digest(read(env, leaseBackend)); + // Whitespace-only counts as unset and other values are kept as is, as the plugin's requireEnv does. + const variables = pluginCredentialVariables(provider, env); + return variables + ? digest( + Object.fromEntries( + variables.map((name) => [name, env[name]?.trim() ? env[name] : undefined]), + ), + ) + : undefined; +} + +// Read from the manifest, so neither the client nor the daemon evaluates plugin code. +function pluginCredentialVariables(provider: string, env: EnvMap): readonly string[] | undefined { + if ((RESERVED_PLUGIN_PROVIDERS as readonly string[]).includes(provider)) return undefined; + return installedPlugins(env).find((plugin) => plugin.agentDevicePlugin.provider === provider) + ?.agentDevicePlugin.credentialVariables; } /** The provider credentials a daemon started with, and the state dir that names that daemon. */ diff --git a/website/docs/docs/plugins.md b/website/docs/docs/plugins.md index d127cccbaf..1a1ff789af 100644 --- a/website/docs/docs/plugins.md +++ b/website/docs/docs/plugins.md @@ -49,6 +49,8 @@ To support `agent-device connect example`, declare `agentDevicePlugin.connection } ``` +If the provider reads credentials from the environment, list the variables in `agentDevicePlugin.credentialVariables`, for example `["EXAMPLE_USERNAME", "EXAMPLE_ACCESS_KEY"]`. A local daemon keeps the values it started with; when the shell holds different ones, the first command that allocates a lease refuses with reason `provider-credentials-changed` until the daemon is stopped. Core reads this list from the manifest without loading the plugin, and treats a whitespace-only value as unset. + Return `connection` alongside the runtime or WebDriver options. Its `resolve({ flags, env, cwd, stateDir })` callback validates provider flags and returns `{ profile, extraFlags? }`; `profile.leaseProvider` must match the manifest. Core supplies connection identity, session defaults, Metro settings, and persists the profile. Its async `verify({ flags, env })` callback returns the provider verification result. These callbacks run without allocating a device and their temporary runtimes are shut down afterwards. Bundle the plugin implementation and ship ready-to-run ESM: installation disables lifecycle scripts. Bundle shared implementation helpers with the plugin; `AppError` carries a shared brand so core preserves its code and details across package copies. Import types with `import type` to keep them out of the runtime dependency graph. diff --git a/website/docs/docs/testmu.md b/website/docs/docs/testmu.md index a288d083d6..883530483e 100644 --- a/website/docs/docs/testmu.md +++ b/website/docs/docs/testmu.md @@ -151,8 +151,11 @@ provider `connect` commands. The typed client reaches TestMu AI through a lease. Allocate one with the provider selectors, then scope a client to it for normal commands. `sessions.close()` ends the hosted session and releases the lease; `leases.release()` in `finally` is then a no-op, and still releases the lease when a -command fails first. The daemon reads `LT_USERNAME` and `LT_ACCESS_KEY` from its environment. Add -`providerDeviceType: 'real'` to `leases.allocate` to run on a real device. +command fails first. The daemon reads `LT_USERNAME` and `LT_ACCESS_KEY` from its environment and +keeps the values it started with. If your shell holds different ones, the first command that +allocates a lease, such as `open`, refuses before it creates a session; run +`agent-device daemon stop` (with the same `--state-dir`) and rerun the command. A shell that sets +neither variable uses the daemon's. Add `providerDeviceType: 'real'` to `leases.allocate` to run on a real device. ```ts import { createAgentDeviceClient } from 'agent-device'; From 2041269b217936123ac7dcfa4798d16051873963 Mon Sep 17 00:00:00 2001 From: amankansal-lt Date: Tue, 6 Oct 2026 13:12:03 +0530 Subject: [PATCH 52/57] refactor(plugins): split manifest connection and credential checks Keeps readPluginManifest under the fallow complexity threshold, and types the manifest fixture's version so the typecheck passes. Co-Authored-By: Claude Opus 5.5 --- src/plugins/manifest.ts | 59 ++++++++++++++++++---------------- src/plugins/plugin.fixtures.ts | 6 +++- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/src/plugins/manifest.ts b/src/plugins/manifest.ts index 0be9362502..87fd0962ec 100644 --- a/src/plugins/manifest.ts +++ b/src/plugins/manifest.ts @@ -73,42 +73,45 @@ export function readPluginManifest(directory: string): PluginManifest { }); } resolvePluginEntry(directory, declaration.entry); - if (declaration.connection !== undefined) { - const policy = declaration.connection; - if ( - !policy || - typeof policy !== 'object' || - policy.leaseKind !== 'direct-device-provider' || - [ - 'requiresAppAttachment', - 'requiresRemoteDaemon', - 'supportsArtifacts', - 'supportsDeferredAppSelection', - 'supportsDirectPortReverse', - 'usesCloudWebDriverLease', - ].some((key) => typeof policy[key as keyof ConnectionProviderCapabilities] !== 'boolean') || - policy.requiresRemoteDaemon - ) { - throw new AppError( - 'INVALID_ARGS', - 'Plugin connection must declare local provider capabilities', - ); - } + assertLocalConnectionPolicy(declaration.connection); + assertCredentialVariables(declaration.credentialVariables); + return manifest as PluginManifest; +} + +function assertLocalConnectionPolicy(policy: ConnectionProviderCapabilities | undefined): void { + if (policy === undefined) return; + if ( + !policy || + typeof policy !== 'object' || + policy.leaseKind !== 'direct-device-provider' || + [ + 'requiresAppAttachment', + 'requiresRemoteDaemon', + 'supportsArtifacts', + 'supportsDeferredAppSelection', + 'supportsDirectPortReverse', + 'usesCloudWebDriverLease', + ].some((key) => typeof policy[key as keyof ConnectionProviderCapabilities] !== 'boolean') || + policy.requiresRemoteDaemon + ) { + throw new AppError( + 'INVALID_ARGS', + 'Plugin connection must declare local provider capabilities', + ); } - const credentialVariables: unknown = declaration.credentialVariables; +} + +function assertCredentialVariables(variables: unknown): void { if ( - credentialVariables !== undefined && - (!Array.isArray(credentialVariables) || - !credentialVariables.every( - (name) => typeof name === 'string' && /^[A-Z_][A-Z0-9_]*$/.test(name), - )) + variables !== undefined && + (!Array.isArray(variables) || + !variables.every((name) => typeof name === 'string' && /^[A-Z_][A-Z0-9_]*$/.test(name))) ) { throw new AppError( 'INVALID_ARGS', 'Plugin credentialVariables must list environment variable names', ); } - return manifest as PluginManifest; } export function resolvePluginEntry(directory: string, entry: string): string { diff --git a/src/plugins/plugin.fixtures.ts b/src/plugins/plugin.fixtures.ts index d8d4a8c2f9..00e6ef75e1 100644 --- a/src/plugins/plugin.fixtures.ts +++ b/src/plugins/plugin.fixtures.ts @@ -46,7 +46,11 @@ export function selectPlugin( /** Installs `manifest` as-is, with an entry file that throws if evaluated. */ export function selectPluginManifest( home: string, - manifest: { name: string; agentDevicePlugin: Record & { entry: string } }, + manifest: { + name: string; + version: string; + agentDevicePlugin: Record & { entry: string }; + }, ) { const installation = crypto.randomUUID(); const directory = path.join(home, 'plugins', installation, 'node_modules', manifest.name); From 704531063fd4af77f5dd4048d5f78de51b75abe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 6 Oct 2026 16:51:47 +0200 Subject: [PATCH 53/57] docs: explain installation of the TestMu package --- packages/provider-testmu/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 packages/provider-testmu/README.md diff --git a/packages/provider-testmu/README.md b/packages/provider-testmu/README.md new file mode 100644 index 0000000000..58ebdf8cf9 --- /dev/null +++ b/packages/provider-testmu/README.md @@ -0,0 +1,15 @@ +# @agent-device/testmu + +Use TestMu AI Android and iOS virtual and real devices with [agent-device](https://agent-device.dev). +You need a TestMu AI account and its API credentials. + +```sh +npm install -g agent-device +agent-device plugins add @agent-device/testmu +export LT_USERNAME=your-username +export LT_ACCESS_KEY=your-access-key +agent-device connect testmu --platform ios --device "iPhone 16" --provider-os-version 18.0 --provider-app ./MyApp.zip +``` + +See the [TestMu AI guide](https://agent-device.dev/docs/testmu) for setup and supported operations. +To update the plugin, run `agent-device plugins update @agent-device/testmu`. From 77579cf8a502edef705edefe77b889ca2b09ff8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 6 Oct 2026 16:51:47 +0200 Subject: [PATCH 54/57] chore(gates): align TestMu package with workspace releases --- packages/provider-testmu/package.json | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/provider-testmu/package.json b/packages/provider-testmu/package.json index 4263f02485..2dd95897f9 100644 --- a/packages/provider-testmu/package.json +++ b/packages/provider-testmu/package.json @@ -1,6 +1,6 @@ { "name": "@agent-device/testmu", - "version": "0.1.0", + "version": "0.21.22", "type": "module", "description": "TestMu AI real and virtual mobile device plugin for agent-device.", "files": [ @@ -50,6 +50,14 @@ ".": { "import": "./dist/plugin.mjs" } - } - } + }, + "access": "public" + }, + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/callstack/agent-device.git", + "directory": "packages/provider-testmu" + }, + "homepage": "https://agent-device.dev/docs/testmu" } From 5db1dd82696b1842fa077643f288c0bcbaf73c2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 6 Oct 2026 16:55:58 +0200 Subject: [PATCH 55/57] docs: explain daemon restart after plugin installation --- packages/provider-testmu/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/provider-testmu/README.md b/packages/provider-testmu/README.md index 58ebdf8cf9..2cf34a621c 100644 --- a/packages/provider-testmu/README.md +++ b/packages/provider-testmu/README.md @@ -13,3 +13,5 @@ agent-device connect testmu --platform ios --device "iPhone 16" --provider-os-ve See the [TestMu AI guide](https://agent-device.dev/docs/testmu) for setup and supported operations. To update the plugin, run `agent-device plugins update @agent-device/testmu`. + +After adding or updating a plugin, close your sessions and run `agent-device daemon stop` before reconnecting. From 7d8cfab2e545e4ca51f4658e4817135c9ddf129f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 6 Oct 2026 21:10:11 +0200 Subject: [PATCH 56/57] docs: show the complete plugin connection manifest --- website/docs/docs/plugins.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/website/docs/docs/plugins.md b/website/docs/docs/plugins.md index 1a1ff789af..0e5fb431f8 100644 --- a/website/docs/docs/plugins.md +++ b/website/docs/docs/plugins.md @@ -39,13 +39,20 @@ To support `agent-device connect example`, declare `agentDevicePlugin.connection ```json { - "leaseKind": "direct-device-provider", - "requiresAppAttachment": false, - "requiresRemoteDaemon": false, - "supportsArtifacts": false, - "supportsDeferredAppSelection": true, - "supportsDirectPortReverse": false, - "usesCloudWebDriverLease": false + "agentDevicePlugin": { + "apiVersion": 1, + "provider": "example", + "entry": "./dist/plugin.mjs", + "connection": { + "leaseKind": "direct-device-provider", + "requiresAppAttachment": false, + "requiresRemoteDaemon": false, + "supportsArtifacts": false, + "supportsDeferredAppSelection": true, + "supportsDirectPortReverse": false, + "usesCloudWebDriverLease": false + } + } } ``` From 66eaad852fa32e4b6d26fb467c60ae2751ec97e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 6 Oct 2026 21:26:10 +0200 Subject: [PATCH 57/57] chore(gates): deduplicate the merged package boundary fixture import --- scripts/layering/package-boundaries.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/layering/package-boundaries.test.ts b/scripts/layering/package-boundaries.test.ts index 195fdf5af9..91e60414e2 100644 --- a/scripts/layering/package-boundaries.test.ts +++ b/scripts/layering/package-boundaries.test.ts @@ -16,7 +16,6 @@ import { checkPackageInternalSites, checkRootSites, readWorkspacePackages, - workspacePackagesFromManifests, rootExternalDependencyRanges, rootWorkspaceDependencyNames, specifierSites,