From 7e67a117c21f6a584e91de7fef558359885a5be8 Mon Sep 17 00:00:00 2001 From: James Long Date: Mon, 9 Nov 2026 20:31:16 +0000 Subject: [PATCH 01/12] feat(core): run local shell commands in the process daemon Local shell commands (no workspace, not Windows) now run in the opencode-pty daemon instead of being spawned by the server. Remote workspaces, Windows and an unreachable daemon keep today's spawner; a request the daemon rejected fails the command instead of running it again. The daemon writes the output file, so the server reads output and the model's tail straight from it. Daemon commands belong to a server-wide registry (Shell.Daemon), so a Location that closes or reloads while the server runs keeps their timeouts, stops and job notifications. Stop, remove, the 25-command limit and the 7-day sweep forget a process in the daemon before deleting its file; a non-zero exit sends SIGTERM to what the command left in its group; a relative cwd resolves against the Location; a command lost with the daemon is reported as killed. Commands still end with the server; the next commit lets them survive a planned restart. Both ship together. User decisions: - Run local shell commands in the daemon and survive a planned restart in one PR - A non-zero exit sends SIGTERM to the command's group - Leftover `cmd &` processes after exit 0 are killed when the command is forgotten (noted, not fixed) --- packages/core/src/file-retention.ts | 3 + packages/core/src/plugin/internal.ts | 2 + packages/core/src/shell.ts | 497 +++++++++++++++++-- packages/core/src/shell/result.ts | 10 +- packages/core/src/tool/plugin/shell.ts | 9 +- packages/core/test/fixture/persistent-pty.ts | 8 + packages/core/test/fixture/shell-daemon.ts | 122 +++++ packages/core/test/shell-cleanup.test.ts | 5 +- packages/core/test/shell-daemon.test.ts | 413 +++++++++++++++ packages/core/test/shell-result.test.ts | 17 + packages/core/test/shell-retention.test.ts | 3 + packages/core/test/tool-shell.test.ts | 150 +++++- packages/server/src/routes.ts | 3 + 13 files changed, 1193 insertions(+), 49 deletions(-) create mode 100644 packages/core/test/fixture/persistent-pty.ts create mode 100644 packages/core/test/fixture/shell-daemon.ts create mode 100644 packages/core/test/shell-daemon.test.ts diff --git a/packages/core/src/file-retention.ts b/packages/core/src/file-retention.ts index a325f2165fc9..ee9f486e219c 100644 --- a/packages/core/src/file-retention.ts +++ b/packages/core/src/file-retention.ts @@ -7,6 +7,8 @@ export const cleanup = Effect.fn("FileRetention.cleanup")(function* ( fs: FSUtil.Interface, files: ReadonlyArray, retention: Duration.Input, + // Runs before an expired file is removed. + beforeRemove?: (file: string) => Effect.Effect, ) { const cutoff = Date.now() - Duration.toMillis(retention) yield* Effect.forEach( @@ -16,6 +18,7 @@ export const cleanup = Effect.fn("FileRetention.cleanup")(function* ( const info = yield* fs.stat(file).pipe(Effect.orElseSucceed(() => undefined)) const mtime = info && Option.getOrUndefined(info.mtime) if (!mtime || mtime.getTime() >= cutoff) return + if (beforeRemove) yield* beforeRemove(file) yield* fs.remove(file).pipe(Effect.ignore) }), { concurrency: 8, discard: true }, diff --git a/packages/core/src/plugin/internal.ts b/packages/core/src/plugin/internal.ts index 180aa4661a81..2d714ab172f2 100644 --- a/packages/core/src/plugin/internal.ts +++ b/packages/core/src/plugin/internal.ts @@ -166,6 +166,7 @@ const services = [ SubagentJob.Service, SharedCache.Service, Shell.Service, + Shell.Daemon, ShellSelect.Service, Snapshot.Service, Skill.Service, @@ -208,6 +209,7 @@ export const requirements = LayerNode.group([ InstructionDiscovery.node, Integration.node, Job.node, + Shell.daemonNode, KV.node, llmClient, Location.node, diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index 054237074457..c8980db9cacc 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -1,7 +1,20 @@ export * as Shell from "./shell.js" import path from "path" -import { Context, Deferred, Duration, Effect, Fiber, Latch, Layer, Option, Schema, Schedule, Stream } from "effect" +import { + Context, + Deferred, + Duration, + Effect, + Exit, + Fiber, + Latch, + Layer, + Option, + Schema, + Schedule, + Stream, +} from "effect" import { ChildProcess } from "effect/process" import { produce } from "immer" import { Shell } from "@opencode/schema/shell" @@ -14,6 +27,7 @@ import { Environment } from "./environment/index.js" import { FileRetention } from "./file-retention.js" import { Location } from "./location.js" import { Global } from "@opencode/util/global" +import { PersistentPty } from "./persistent-pty.js" import { ShellSelect } from "./shell/select.js" import type { ShellCreateBefore } from "@opencode/plugin/effect/shell" import { PluginHooks } from "./plugin/hooks.js" @@ -33,6 +47,8 @@ const EXITED_LIMIT = 25 export const RETENTION = Duration.days(7) export const DIRECTORY = "shell" const SPAN_COMMAND_LIMIT = 200 +const STOP_GRACE = Duration.seconds(3) +const LOST = "Command was killed: the process daemon stopped" type Info = Shell.Info export type StopReason = "timeout" | "killed" @@ -40,10 +56,23 @@ type CreateInput = Shell.CreateInput & { shell?: string } -type Active = { +// What the server records about a daemon-run command in the daemon itself. +const Stored = Schema.Struct({ + id: Shell.ID, + command: Schema.String, + shell: Schema.String, + location: Schema.Struct({ directory: Schema.String, workspaceID: Schema.optionalKey(Schema.String) }), + metadata: Shell.Metadata, + timeoutDeadline: Schema.optional(Schema.Number), + stopReason: Schema.optional(Schema.Literals(["timeout", "killed"])), +}) +type Stored = typeof Stored.Type + +// A command this server spawned itself. It dies with its Location. +type Spawned = { + kind: "spawned" // Immutable snapshot; lifecycle updates replace it via immer `produce`. info: Info - file: string size: number newlines: number // Byte offsets where the last two lines start, so a truncation notice can size a cut-off line. @@ -56,6 +85,27 @@ type Active = { stop?: (reason: StopReason) => Effect.Effect } +// A command the process daemon runs. The daemon writes and counts its output file. It belongs to the server, not +// to the Location that started it, so a Location closing or reloading never stops or forgets it. +type Run = { + kind: "daemon" + info: Info + process: number + timeoutDeadline?: number + stopReason?: StopReason + // From the exit: newlines in the file, and why the daemon stopped writing it early. + outputLines?: number + outputError?: string + // Why it was killed without a signal of its own. + killed?: string + done: Deferred.Deferred + timeoutFiber?: Fiber.Fiber + stopping?: Fiber.Fiber + wait?: Fiber.Fiber +} + +type Command = Spawned | Run + type Ended = { status: "exited" | StopReason exit?: number @@ -66,10 +116,13 @@ type Ended = { /** * Location-owned non-interactive shell command process service. * - * Each `create` spawns one shell command, captures combined stdout/stderr to a + * Each `create` starts one shell command, captures combined stdout/stderr to a * file, and returns an ID. Clients poll `get` for status and `output` for * file-backed output by cursor. No session, message, or permission state lives * here; callers (e.g. `ShellTool`) own that association and store the shell ID. + * + * Local commands run in the process daemon when it is available (`Daemon`); remote workspaces, Windows and an + * unavailable daemon keep spawning them from the server. */ export interface Interface { readonly create: ( @@ -104,6 +157,7 @@ export class Service extends Context.Service()("@opencode/Sh export const cleanup = Effect.fn("Shell.cleanup")(function* () { const fs = yield* FSUtil.Service const global = yield* Global.Service + const pty = yield* PersistentPty.Service const directory = path.join(global.data, DIRECTORY) const projects = yield* fs.readDirectoryEntries(directory).pipe( Effect.map((entries) => entries.filter((entry) => entry.type === "Directory")), @@ -124,7 +178,32 @@ export const cleanup = Effect.fn("Shell.cleanup")(function* () { ), { concurrency: 8 }, ) - yield* FileRetention.cleanup(fs, files.flat(), RETENTION) + // Never starts the daemon. A command it still runs keeps its file; a finished one is forgotten before deletion. + const processes = new Map( + (yield* pty.listPipes().pipe(Effect.orElseSucceed(() => []))).map((process) => [process.outputPath, process]), + ) + yield* FileRetention.cleanup( + fs, + files.flat().filter((file) => processes.get(file)?.lifecycle.status !== "running"), + RETENTION, + (file) => { + const process = processes.get(file) + return process ? pty.terminatePipe(process.id).pipe(Effect.ignore) : Effect.void + }, + ) + // Finished processes whose file is gone, or that ended long ago, are of no use to any Location: forget them. + const cutoff = Date.now() - Duration.toMillis(RETENTION) + yield* Effect.forEach( + processes.values(), + (process) => + Effect.gen(function* () { + if (process.lifecycle.status === "running") return + const stale = (process.exitedAt ?? process.startedAt) < cutoff + if (!stale && (yield* fs.exists(process.outputPath).pipe(Effect.orElseSucceed(() => true)))) return + yield* pty.terminatePipe(process.id).pipe(Effect.ignore) + }), + { concurrency: 8, discard: true }, + ) }) const cleanupLayer = Layer.effectDiscard( @@ -134,7 +213,260 @@ const cleanupLayer = Layer.effectDiscard( const cleanupNode = makeGlobalNode({ name: "shell-output-cleanup", layer: cleanupLayer, - deps: [FSUtil.node, Global.node], + deps: [FSUtil.node, Global.node, PersistentPty.node], +}) + +/** One Location's daemon commands, as `Daemon.at` hands them to its `Shell`. */ +interface Runs { + readonly get: (id: Shell.ID) => Run | undefined + readonly list: () => Run[] + /** Fails with `UnavailableError` only when the daemon could not be reached or started: spawning instead is safe. */ + readonly create: (input: { + readonly info: Info + readonly args: readonly string[] + readonly env: Readonly> + readonly timeout: number + }) => Effect.Effect + readonly timeout: (run: Run, duration: number) => Effect.Effect + readonly stop: (run: Run, reason: StopReason) => Effect.Effect + readonly remove: (id: Shell.ID) => Effect.Effect +} + +export interface DaemonInterface { + /** A Location's daemon commands. */ + readonly at: (location: Location.Interface) => Effect.Effect + /** Runs `effect` until it ends or the server stops, outliving the caller's Location (e.g. a job's completion notice). */ + readonly watch: (effect: Effect.Effect) => Effect.Effect +} + +/** The process daemon's shell commands, owned by the server and grouped by Location. */ +export class Daemon extends Context.Service()("@opencode/Shell/Daemon") {} + +const daemonLayer = Layer.effect( + Daemon, + Effect.gen(function* () { + const bus = yield* Bus.Service + const pty = yield* PersistentPty.Service + const scope = yield* Effect.scope + const context = yield* Effect.context() + const { unlink } = yield* Effect.promise(() => import("fs/promises")) + const places = new Map() + // Fibers run with the server's context in its scope, so a Location closing never interrupts them. + const fork = (effect: Effect.Effect) => + effect.pipe(Effect.forkIn(scope, { startImmediately: true }), Effect.setContext(context)) + + const place = (location: Location.Interface): Runs => { + const runs = new Map() + const exitOrder: Shell.ID[] = [] + const ref = Location.Ref.make({ directory: location.directory, workspaceID: location.workspaceID }) + + const persisted = (run: Run): Stored => ({ + id: run.info.id, + command: run.info.command, + shell: run.info.shell, + location: { + directory: location.directory, + ...(location.workspaceID !== undefined ? { workspaceID: location.workspaceID } : {}), + }, + metadata: run.info.metadata, + timeoutDeadline: run.timeoutDeadline, + stopReason: run.stopReason, + }) + + // Records the command's latest timeout and stop reason in the daemon. + const store = (run: Run) => + pty + .setPipeMetadata(run.process, persisted(run)) + .pipe( + Effect.catch((error) => + Effect.logWarning("failed to update shell command metadata", { id: run.info.id, error: error.message }), + ), + ) + + const remove = Effect.fnUntraced(function* (id: Shell.ID) { + const run = runs.get(id) + const index = exitOrder.indexOf(id) + if (index !== -1) exitOrder.splice(index, 1) + if (!run) return + runs.delete(id) + if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber) + if (run.stopping) yield* Fiber.interrupt(run.stopping) + // Stop waiting first, so forgetting the process is not mistaken for losing it. + if (run.wait) yield* Fiber.interrupt(run.wait) + // Forget the process before deleting its file. This also kills what is left of its group, even after a + // successful exit, which a spawned command would leave running: the daemon cannot forget a process otherwise. + yield* pty.terminatePipe(run.process).pipe(Effect.ignore) + yield* Effect.promise(() => unlink(run.info.file).catch(() => {})) + yield* Deferred.fail(run.done, new NotFoundError({ id })) + // Every shell reports exactly one exit; removing a running one ends it as killed. + if (run.info.status === "running") + yield* bus.publish(Shell.Event.Exited, { id, status: "killed" }, { location: ref }) + yield* bus.publish(Shell.Event.Deleted, { id }, { location: ref }) + }) + + const evict = Effect.fnUntraced(function* () { + while (exitOrder.length > EXITED_LIMIT) { + const oldest = exitOrder[0] + if (!oldest) break + yield* remove(oldest) + } + }) + + const finish = Effect.fnUntraced(function* (run: Run, outcome: Ended) { + const id = run.info.id + // A removed command already failed its waiters and published its exit and deletion. + if (runs.get(id) !== run) return + run.info = produce(run.info, (draft) => { + draft.status = outcome.status + if (outcome.exit !== undefined) draft.exit = outcome.exit + if (outcome.signal !== undefined) draft.signal = outcome.signal + draft.time.completed = outcome.completed + }) + if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber) + run.timeoutFiber = undefined + // Resolve waiters before retention eviction, so an evicted command still reports its + // terminal Info rather than the removal NotFoundError. + yield* Deferred.succeed(run.done, run.info) + yield* bus.publish( + Shell.Event.Exited, + { id, ...(outcome.exit !== undefined ? { exit: outcome.exit } : {}), status: outcome.status }, + { location: ref }, + ) + exitOrder.push(id) + yield* evict() + }) + + // Graceful, like spawned commands: SIGTERM, then SIGKILL if it has not exited 3 s later. It runs on its own, + // so interrupting the caller (a timer being retimed, removed or torn down) never cancels a recorded stop. + const stop = (run: Run, reason: StopReason) => + Effect.gen(function* () { + if (run.stopReason !== undefined || run.info.status !== "running") return + run.stopReason = reason + run.timeoutDeadline = undefined + run.stopping = yield* fork( + Effect.gen(function* () { + yield* store(run) + yield* pty.signalPipe(run.process, "SIGTERM").pipe(Effect.ignore) + const ended = yield* Deferred.await(run.done).pipe(Effect.timeoutOption(STOP_GRACE), Effect.exit) + if (Exit.isSuccess(ended) && Option.isNone(ended.value)) + yield* pty.signalPipe(run.process, "SIGKILL").pipe(Effect.ignore) + }), + ) + }) + + const arm = Effect.fnUntraced(function* (run: Run, duration: number) { + run.timeoutFiber = yield* fork( + Effect.sleep(Duration.millis(duration)).pipe(Effect.andThen(stop(run, "timeout"))), + ) + }) + + const timeout = Effect.fnUntraced(function* (run: Run, duration: number) { + if (run.stopReason !== undefined || run.info.status !== "running") return + if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber) + run.timeoutFiber = undefined + run.timeoutDeadline = duration === 0 ? undefined : Date.now() + duration + yield* store(run) + if (duration !== 0) yield* arm(run, duration) + }) + + // Watches the exit, and the timeout (in ms from now, none if undefined). + const attach = Effect.fnUntraced(function* (run: Run, duration: number | undefined) { + run.wait = yield* fork( + pty.waitPipe(run.process).pipe( + Effect.map((exit): Ended => { + run.outputLines = exit.outputLines + run.outputError = exit.outputError + // A stopped command reports its reason, and how the stopped process ended, as spawned ones do. + return { + status: run.stopReason ?? "exited", + ...(exit.exitCode !== undefined ? { exit: exit.exitCode } : {}), + ...(exit.signal !== undefined ? { signal: exit.signal } : {}), + completed: exit.exitedAt, + } + }), + // The daemon went away (and its processes with it), or forgot the process: it was killed. + Effect.catch((error) => + Effect.logWarning("lost a shell command's process", { id: run.info.id, error: error.message }).pipe( + Effect.andThen( + Effect.sync((): Ended => { + run.killed = LOST + return { status: "killed", completed: Date.now() } + }), + ), + ), + ), + // Like spawned commands: what a failed command left running in its group is stopped with it, while a + // successful one may leave background processes (`cmd &`) behind. + Effect.tap((outcome) => + outcome.status === "exited" && outcome.exit !== undefined && outcome.exit !== 0 + ? pty.signalPipe(run.process, "SIGTERM").pipe(Effect.ignore) + : Effect.void, + ), + Effect.flatMap((outcome) => finish(run, outcome)), + ), + ) + if (duration === undefined) return + if (duration > 0) return yield* arm(run, duration) + yield* stop(run, "timeout") + }) + + const create: Runs["create"] = Effect.fnUntraced(function* (input) { + const run: Run = { + kind: "daemon", + info: input.info, + process: 0, + timeoutDeadline: input.timeout > 0 ? input.info.time.started + input.timeout : undefined, + done: Deferred.makeUnsafe(), + } + const created = yield* pty.createPipe({ + program: input.info.shell, + args: input.args, + cwd: input.info.cwd, + env: input.env, + groupID: location.project.id, + outputPath: input.info.file, + metadata: persisted(run), + }) + run.process = created.id + run.info = produce(input.info, (draft) => { + if (created.pid !== undefined) draft.pid = created.pid + // The daemon's clock, so the command reports the same start wherever it is read. + draft.time.started = created.startedAt + }) + runs.set(run.info.id, run) + // Timed from now: starting the daemon may have taken a while. + yield* attach(run, input.timeout > 0 ? input.timeout : undefined) + yield* bus.publish(Shell.Event.Created, { info: run.info }, { location: ref }) + return run.info + }) + + return { get: (id) => runs.get(id), list: () => Array.from(runs.values()), create, timeout, stop, remove } + } + + const at = (location: Location.Interface) => + Effect.sync(() => { + const key = `${location.directory}\0${location.workspaceID ?? ""}` + const existing = places.get(key) + if (existing) return existing + const runs = place(location) + places.set(key, runs) + return runs + }) + + return Daemon.of({ + at, + watch: (effect) => + fork(effect.pipe(Effect.catch((error) => Effect.logError("shell watcher failed", { error })))).pipe( + Effect.asVoid, + ), + }) + }), +) + +export const daemonNode = makeGlobalNode({ + service: Daemon, + layer: daemonLayer, + deps: [Bus.node, PersistentPty.node], }) const layer = () => @@ -149,13 +481,20 @@ const layer = () => const hooks = yield* PluginHooks.Service const environments = yield* SessionEnvironment.Service const config = yield* Config.Service + const daemon = yield* Daemon const context = yield* Effect.context() const runFork = Effect.runForkWith(context) - const commands = new Map() + // Commands this server spawned for the Location; daemon commands live in `runs`, owned by the server. + const commands = new Map() const exitOrder: Shell.ID[] = [] + // Remote workspaces and Windows keep spawning commands from the server. + const runs = + location.workspaceID === undefined && process.platform !== "win32" && PersistentPty.available + ? yield* daemon.at(location) + : undefined const outputDir = path.join(global.data, DIRECTORY, location.project.id) - const { mkdir, unlink } = yield* Effect.promise(() => import("fs/promises")) + const { mkdir, unlink, stat } = yield* Effect.promise(() => import("fs/promises")) const { createWriteStream, createReadStream } = yield* Effect.promise(() => import("fs")) yield* Effect.promise(() => mkdir(outputDir, { recursive: true })) @@ -163,7 +502,8 @@ const layer = () => Effect.gen(function* () { for (const command of commands.values()) { if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) - // Teardown interrupts pending commands; it is not a terminal command failure. + // Teardown interrupts pending commands; it is not a terminal command failure. Daemon commands are left + // to the server. yield* Deferred.interrupt(command.done) } commands.clear() @@ -172,7 +512,7 @@ const layer = () => ) const require = Effect.fnUntraced(function* (id: Shell.ID) { - const command = commands.get(id) + const command: Command | undefined = commands.get(id) ?? runs?.get(id) if (!command) return yield* new NotFoundError({ id }) return command }) @@ -186,19 +526,21 @@ const layer = () => if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) // Unblock any wait still pending when the command is removed before it terminated. yield* Deferred.fail(command.done, new NotFoundError({ id })) - yield* Effect.promise(() => unlink(command.file).catch(() => {})) + yield* Effect.promise(() => unlink(command.info.file).catch(() => {})) // Every shell reports exactly one exit; removing a running one ends it as killed. if (command.info.status === "running") yield* bus.publish(Shell.Event.Exited, { id, status: "killed" }) yield* bus.publish(Shell.Event.Deleted, { id }) }) const remove = Effect.fn("Shell.remove")(function* (id: Shell.ID) { - yield* annotate((yield* require(id)).info) + const command = yield* require(id) + yield* annotate(command.info) + if (command.kind === "daemon") return yield* runs?.remove(id) ?? Effect.void yield* removeCommand(id) }) const list = Effect.fn("Shell.list")(function* () { - return Array.from(commands.values()) + return [...commands.values(), ...(runs?.list() ?? [])] .filter((command) => command.info.status === "running") .map((command) => command.info) }) @@ -220,16 +562,17 @@ const layer = () => const timeout = Effect.fn("Shell.timeout")(function* (id: Shell.ID, duration: number) { const command = yield* require(id) yield* annotate(command.info) - if (command.info.status !== "running" || !command.timeout) return command.info - yield* command.timeout(duration) + if (command.kind === "daemon") yield* runs?.timeout(command, duration) ?? Effect.void + if (command.kind === "spawned" && command.info.status === "running" && command.timeout) + yield* command.timeout(duration) return command.info }) const stop = Effect.fn("Shell.stop")(function* (id: Shell.ID, reason: StopReason) { const command = yield* require(id) yield* annotate(command.info) - if (command.info.status !== "running" || !command.stop) return command.info - yield* command.stop(reason) + if (command.kind === "daemon") yield* runs?.stop(command, reason) ?? Effect.void + if (command.kind === "spawned" && command.info.status === "running" && command.stop) yield* command.stop(reason) return command.info }) @@ -252,21 +595,66 @@ const layer = () => }), ) + // A daemon command's file is the only record of its output, so its size is the file's; a missing file is empty. + const sizeOf = (command: Command) => + command.kind === "spawned" + ? Effect.succeed(command.size) + : Effect.promise(() => + stat(command.info.file).then( + (info) => info.size, + () => 0, + ), + ) + const output = Effect.fnUntraced(function* (id: Shell.ID, input?: Shell.OutputInput) { const command = yield* require(id) const cursor = input?.cursor ?? 0 const limit = input?.limit ?? 65536 - if (cursor >= command.size) return { output: "", cursor: command.size, size: command.size, truncated: false } + const size = yield* sizeOf(command) + if (cursor >= size) return { output: "", cursor: size, size, truncated: false } const start = Math.max(0, cursor) - const bytes = yield* readRange(command.file, start, Math.min(limit, command.size - start)) + const bytes = yield* readRange(command.info.file, start, Math.min(limit, size - start)) return { output: bytes.toString("utf8"), cursor: start + bytes.length, - size: command.size, + size, truncated: false, } }) + // Only for a command whose exit report is missing (still running, or lost with the daemon); the daemon counts + // newlines otherwise. + const countNewlines = (file: string, size: number) => + Effect.promise( + () => + new Promise((resolve) => { + if (size === 0) return resolve(0) + let count = 0 + const stream = createReadStream(file, { end: size - 1 }) + stream.on("data", (chunk: string | Buffer) => { + const bytes = Buffer.from(chunk) + for (let index = bytes.indexOf(10); index !== -1; index = bytes.indexOf(10, index + 1)) count++ + }) + stream.on("end", () => resolve(count)) + stream.on("error", () => resolve(count)) + }), + ) + + // Where the last two lines of a daemon command's output start, found by reading back from the end. Only needed + // to size a single cut-off line, which the spawner tracks as it writes. + const lineStartsOf = Effect.fnUntraced(function* (file: string, size: number) { + const starts: number[] = [] + let end = size + while (end > 0 && starts.length < 2) { + const from = Math.max(0, end - 65536) + const chunk = yield* readRange(file, from, end - from) + for (let index = chunk.length - 1; index >= 0 && starts.length < 2; index--) + if (chunk[index] === 10) starts.push(from + index + 1) + end = from + } + return [starts[1] ?? 0, starts[0] ?? 0] as [number, number] + }) + const result = Effect.fn("Shell.result")(function* ( started: Shell.Info, options?: { readonly wait?: Duration.Input }, @@ -291,35 +679,49 @@ const layer = () => const maxLines = limits?.max_lines ?? ToolOutput.MAX_LINES const maxBytes = limits?.max_bytes ?? ToolOutput.MAX_BYTES const command = yield* require(info.id) - const start = Math.max(0, command.size - maxBytes) + const size = yield* sizeOf(command) + const start = Math.max(0, size - maxBytes) // Read the byte before the window too, to tell whether the window starts on a line boundary. const before = Math.max(0, start - 1) - const window = yield* readRange(command.file, before, command.size - before) + const window = yield* readRange(info.file, before, size - before) const page = window.subarray(charBoundary(window, start - before)).toString("utf8") const partial = start > 0 && window[0] !== 10 const endsWithNewline = page.endsWith("\n") const lines = page.split("\n") if (endsWithNewline) lines.pop() - const total = command.newlines + (endsWithNewline ? 0 : 1) + const newlines = + command.kind === "spawned" + ? command.newlines + : (command.outputLines ?? (yield* countNewlines(info.file, size))) + const total = newlines + (endsWithNewline ? 0 : 1) const suffix = `full output saved to ${info.file}]` + // After a write error the daemon kept running the command, but its file stopped early. + const stoppedEarly = + command.kind === "daemon" && command.outputError + ? `\n\n[output file ${info.file} stopped early: ${command.outputError}]` + : "" + const killed = command.kind === "daemon" && command.killed ? { killed: command.killed } : {} if (partial && lines.length === 1) { - const lineStart = endsWithNewline ? command.lineStarts[0] : command.lineStarts[1] - const lineSize = command.size - (endsWithNewline ? 1 : 0) - lineStart + const lineStarts = command.kind === "spawned" ? command.lineStarts : yield* lineStartsOf(info.file, size) + const lineStart = endsWithNewline ? lineStarts[0] : lineStarts[1] + const lineSize = size - (endsWithNewline ? 1 : 0) - lineStart const shownSize = Buffer.byteLength(lines[0]) return { - output: `${lines[0]}\n\n[showing the last ${formatSize(shownSize)} of line ${total} (${formatSize(lineSize)}); ${suffix}`, + output: `${lines[0]}\n\n[showing the last ${formatSize(shownSize)} of line ${total} (${formatSize(lineSize)}); ${suffix}${stoppedEarly}`, truncated: true, + ...killed, } } if (start > 0 || lines.length > maxLines) { // A window that starts mid-line drops that partial line so the line numbers stay exact. const kept = lines.slice(-Math.min(maxLines, lines.length - (partial ? 1 : 0))) return { - output: `${kept.join("\n")}\n\n[showing lines ${total - kept.length + 1}-${total} of ${total}; ${suffix}`, + output: `${kept.join("\n")}\n\n[showing lines ${total - kept.length + 1}-${total} of ${total}; ${suffix}${stoppedEarly}`, truncated: true, + ...killed, } } - return { output: page || "(no output)", truncated: false } + return { output: `${page || "(no output)"}${stoppedEarly}`, truncated: false, ...killed } }).pipe(Effect.catchTag("Shell.NotFoundError", () => Effect.succeed(undefined))) return { info, capture } }) @@ -346,6 +748,8 @@ const layer = () => } yield* hooks.trigger("shell", "create.before", invocation) if (before) yield* before(invocation) + // Absolute for local commands, so the command reports the same cwd wherever it runs. + if (location.workspaceID === undefined) invocation.cwd = path.resolve(location.directory, invocation.cwd) const id = Shell.ID.ascending() const args = ShellSelect.args(invocation.shell, invocation.command) @@ -363,10 +767,36 @@ const layer = () => } yield* annotate(info) + if (runs) { + const created = yield* runs + .create({ + info, + args, + // The daemon replaces the environment, so variables removed by hooks stay removed, like the spawner. + env: Object.fromEntries( + Object.entries(invocation.env).filter((entry): entry is [string, string] => entry[1] !== undefined), + ), + timeout: invocation.timeout, + }) + .pipe( + Effect.map(Option.some), + Effect.catchTags({ + "PersistentPty.UnavailableError": (error) => + Effect.logWarning("process daemon unavailable; spawning shell command instead", { + error: error.message, + }).pipe(Effect.as(Option.none())), + // The daemon may have run it: never run it again here. + "PersistentPty.RequestError": (cause) => + Effect.fail(new AppProcess.AppProcessError({ command: invocation.command, cause })), + }), + ) + if (Option.isSome(created)) return created.value + } + // Spawn through the Environment and stream combined output to the file. The handle is scope-bound, so // the managing fiber keeps its scope open until the command terminates (it awaits `done` at the // end). `create` returns once `ready` resolves with the registered command. - const ready = Deferred.makeUnsafe() + const ready = Deferred.makeUnsafe() runFork( Effect.scoped( Effect.gen(function* () { @@ -377,17 +807,17 @@ const layer = () => env: invocation.env, stdin: "ignore", detached: process.platform !== "win32", - forceKillAfter: Duration.seconds(3), + forceKillAfter: STOP_GRACE, }), ) .pipe( Effect.mapError((cause) => new AppProcess.AppProcessError({ command: invocation.command, cause })), ) - const command: Active = { + const command: Spawned = { + kind: "spawned", info: produce(info, (draft) => { draft.pid = handle.pid }), - file, size: 0, newlines: 0, lineStarts: [0, 0], @@ -441,7 +871,7 @@ const layer = () => command.stop = (reason) => Effect.gen(function* () { if (!(yield* end({ status: reason }))) return - runFork(handle.kill({ forceKillAfter: Duration.seconds(3) }).pipe(Effect.catch(() => Effect.void))) + runFork(handle.kill({ forceKillAfter: STOP_GRACE }).pipe(Effect.catch(() => Effect.void))) }).pipe(Effect.uninterruptible) command.timeout = (duration) => @@ -570,6 +1000,7 @@ export const node = makeLocationNode({ PluginHooks.node, SessionEnvironment.node, Config.node, + daemonNode, cleanupNode, ], }) diff --git a/packages/core/src/shell/result.ts b/packages/core/src/shell/result.ts index 558d1b8c607d..2c25349fcccf 100644 --- a/packages/core/src/shell/result.ts +++ b/packages/core/src/shell/result.ts @@ -5,10 +5,11 @@ import type { Shell } from "@opencode/schema/shell" export type Result = { info: Shell.Info - capture: { output: string; truncated: boolean } | undefined + // `killed` says why a command was killed when no exit code or signal of its own tells, e.g. the process daemon stopped. + capture: { output: string; truncated: boolean; killed?: string } | undefined } -type Output = { output: string; truncated: boolean; exit?: number; signal?: string; timeout?: boolean } +type Output = { output: string; truncated: boolean; exit?: number; signal?: string; timeout?: boolean; killed?: string } const missing = "Shell command output is no longer available." export const unavailable: Shell.Output = { @@ -25,6 +26,7 @@ export function output(result: Result): Output { ...(result.info.exit !== undefined ? { exit: result.info.exit } : {}), ...(result.info.signal !== undefined ? { signal: result.info.signal } : {}), ...(result.info.status === "timeout" ? { timeout: true } : {}), + ...(result.capture?.killed !== undefined ? { killed: result.capture.killed } : {}), } } @@ -45,7 +47,8 @@ export function interrupted(result: Result) { return { output, notice, message: output ? `${output}\n\n${notice}` : notice } } -export function notice(output: Pick, timeout?: number) { +export function notice(output: Pick, timeout?: number) { + if (output.killed !== undefined) return output.killed if (output.timeout) return `Command timed out${timeout ? ` after ${timeout / 1_000} seconds` : ""}. If it needs longer and isn't waiting for input, rerun with a larger timeout.\nExit code: ${exitCode(output)}` if (output.signal !== undefined || (output.exit !== undefined && output.exit !== 0)) @@ -65,6 +68,7 @@ export function metadata(output: Output) { ...(output.exit !== undefined ? { exit: output.exit } : {}), ...(output.signal !== undefined ? { signal: output.signal } : {}), ...(output.timeout !== undefined ? { timeout: output.timeout } : {}), + ...(output.killed !== undefined ? { killed: true } : {}), } } diff --git a/packages/core/src/tool/plugin/shell.ts b/packages/core/src/tool/plugin/shell.ts index b2fbd18dc557..0c08343bb4e1 100644 --- a/packages/core/src/tool/plugin/shell.ts +++ b/packages/core/src/tool/plugin/shell.ts @@ -6,7 +6,7 @@ import type { SessionHooks } from "@opencode/plugin/effect/session" import type { ShellCreateBefore } from "@opencode/plugin/effect/shell" import type { Shell as ShellSchema } from "@opencode/schema/shell" import type { Tool } from "@opencode/schema/tool" -import { Deferred, Effect, Fiber, Option, Schema, Scope } from "effect" +import { Deferred, Effect, Fiber, Option, Schema } from "effect" import { Environment } from "../../environment/index.js" import { Job } from "../../job.js" import { FileAccess } from "../../file-access.js" @@ -72,6 +72,7 @@ const StructuredOutput = Schema.Struct({ pid: Schema.optionalKey(NonNegativeInt), truncated: Schema.Boolean, timeout: Schema.optionalKey(Schema.Boolean), + killed: Schema.optionalKey(Schema.String), }) const Output = Schema.Struct({ @@ -123,10 +124,10 @@ export const Plugin = { effect: Effect.fn("ShellTool.Plugin")(function* (ctx: Context) { const sessions = yield* Session.Service const jobs = yield* Job.Service - const scope = yield* Scope.Scope const environment = yield* Environment.Service const access = yield* FileAccess.Service const shell = yield* Shell.Service + const daemon = yield* Shell.Daemon const shellSelect = yield* ShellSelect.Service const compatibleShell = shellSelect.resolve({ priority: "compat" }) const permission = yield* Permission.Service @@ -193,7 +194,9 @@ export const Plugin = { }) if (info.notificationID) yield* jobs.completeBackground(info.notificationID) }, - Effect.forkIn(scope, { startImmediately: true }), + // Not in this Location's scope: a daemon command outlives a Location that closes while the server runs, and + // its session must still hear how it ended. + (effect) => daemon.watch(effect), ) // Stops the command of a call the user asked to stop, and fails the call with the output so far. The command diff --git a/packages/core/test/fixture/persistent-pty.ts b/packages/core/test/fixture/persistent-pty.ts new file mode 100644 index 000000000000..c81232f00e08 --- /dev/null +++ b/packages/core/test/fixture/persistent-pty.ts @@ -0,0 +1,8 @@ +import { Effect, Layer } from "effect" +import { PersistentPty } from "@opencode/core/persistent-pty" + +// No process daemon: shell commands fall back to the Environment's spawner, and nothing is ever running in it. +export const disabledPersistentPtyLayer = Layer.mock(PersistentPty.Service, { + createPipe: () => Effect.fail(new PersistentPty.UnavailableError({ message: "process daemon disabled" })), + listPipes: () => Effect.succeed([]), +}) diff --git a/packages/core/test/fixture/shell-daemon.ts b/packages/core/test/fixture/shell-daemon.ts new file mode 100644 index 000000000000..fabb399779bb --- /dev/null +++ b/packages/core/test/fixture/shell-daemon.ts @@ -0,0 +1,122 @@ +import path from "node:path" +import { Context, Effect, Exit, Layer, Scope } from "effect" +import { ChildProcessSpawner } from "effect/process" +import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" +import { Config } from "@opencode/core/config" +import { Environment } from "@opencode/core/environment/index" +import { Location } from "@opencode/core/location" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { AbsolutePath } from "@opencode/core/schema" +import { Shell } from "@opencode/core/shell" +import { Bus } from "@opencode/core/bus" +import { LayerNode } from "@opencode/util/effect/layer-node" +import { LocationLifecycle } from "@opencode/core/location-lifecycle" +import { Global } from "@opencode/util/global" +import { hostEnvironmentLayer } from "./environment" +import { location } from "./location" +import { fakeDaemon } from "./persistent-pty-daemon" +import { tmpdirScoped } from "./tmpdir" + +// One fake daemon and data directory shared by every Location (and server) a test builds. +export const daemonShells = Effect.fn(function* () { + const daemon = yield* fakeDaemon() + const data = (yield* tmpdirScoped()).path + const global = Global.layerWith({ data, state: path.join(data, "state") }) + const context = yield* Layer.build( + AppNodeBuilder.build(PersistentPty.configured(daemon.options), [Global.node.replace(global)]), + ) + const pty = Context.get(context, PersistentPty.Service) + let spawned = 0 + const environment = Layer.effect( + Environment.Service, + Effect.gen(function* () { + const host = yield* Environment.Service + return Environment.Service.of({ + ...host, + spawner: ChildProcessSpawner.make(() => + Effect.sync(() => spawned++).pipe(Effect.andThen(Effect.die("the spawner must not be used"))), + ), + }) + }), + ).pipe(Layer.provide(hostEnvironmentLayer)) + // `server` shares one server's daemon commands between Locations; without it the Location gets a server of its own. + const shellLayer = (directory: string, server?: Shell.DaemonInterface) => + AppNodeBuilder.build(LayerNode.group([Shell.node, LocationLifecycle.node, Bus.node]), [ + ...(server ? [Shell.daemonNode.replace(Layer.succeed(Shell.Daemon, server))] : []), + Location.node.replace( + Layer.succeed( + Location.Service, + Location.Service.of(location(Location.Ref.make({ directory: AbsolutePath.make(directory) }))), + ), + ), + Global.node.replace(global), + Config.node.replace(Config.testLayer()), + Environment.node.replace(environment), + PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, pty)), + ]) + // The server that `openLocation` Locations share, as long as the test runs. + const serverContext = yield* Layer.build( + AppNodeBuilder.build(LayerNode.group([Shell.daemonNode, Bus.node]), [ + Global.node.replace(global), + PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, pty)), + ]), + ) + const server = Context.get(serverContext, Shell.Daemon) + // A Location's Shell, on a server of its own, that stays open for the rest of the test. + const openShell = (directory: string) => + Effect.gen(function* () { + const context = yield* Layer.build(shellLayer(directory)) + return Context.get(context, Shell.Service) + }) + // A Location's Shell and its services, closed by `close` as a reload or an eviction does while the server runs. + const openLocation = (directory: string) => + Effect.gen(function* () { + const scope = yield* Scope.make() + const context = yield* Layer.buildWithScope(shellLayer(directory, server), scope) + return { + shell: Context.get(context, Shell.Service), + bus: Context.get(context, Bus.Service), + close: Context.get(context, LocationLifecycle.Service).shutdown.pipe( + Effect.andThen(Scope.close(scope, Exit.void)), + ), + } + }) + // Runs `body` against the Location's Shell on a server of its own, then stops that server, as a restart does. + const withShell = ( + directory: string, + body: (shell: Shell.Interface) => Effect.Effect, + ) => + Effect.scoped( + Effect.gen(function* () { + const shell = yield* Shell.Service + return yield* body(shell) + }).pipe(Effect.provide(shellLayer(directory))), + ) + const directory = (yield* tmpdirScoped()).path + return { + daemon, + data, + directory, + global, + pty, + withShell, + openShell, + openLocation, + // Where the server's daemon commands report. + serverBus: Context.get(serverContext, Bus.Service), + spawned: () => spawned, + created: () => daemon.requests.filter((request) => request.op === "create_pipe"), + signals: (id: number) => + daemon.requests.flatMap((request) => (request.op === "signal" && request.id === id ? [request.signal] : [])), + } +}) + +// Polls in real time until `check` holds: the fake daemon answers over a socket, which the test clock does not drive. +export const until = (check: (() => boolean) | Effect.Effect) => + Effect.gen(function* () { + for (let attempt = 0; attempt < 2_000; attempt++) { + if (typeof check === "function" ? check() : yield* check) return + yield* Effect.promise(() => Bun.sleep(2)) + } + return yield* Effect.die(new Error("condition never held")) + }) diff --git a/packages/core/test/shell-cleanup.test.ts b/packages/core/test/shell-cleanup.test.ts index 9af9ca64af26..a807b69d8307 100644 --- a/packages/core/test/shell-cleanup.test.ts +++ b/packages/core/test/shell-cleanup.test.ts @@ -6,13 +6,16 @@ import { Shell } from "@opencode/core/shell" import { LayerNode } from "@opencode/util/effect/layer-node" import { FSUtil } from "@opencode/util/fs-util" import { Global } from "@opencode/util/global" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { disabledPersistentPtyLayer } from "./fixture/persistent-pty" import { withTempDir } from "./fixture/tmpdir" import { it } from "./lib/effect" const withStore = (body: (fs: FSUtil.Interface, root: string) => Effect.Effect) => withTempDir((tmp) => { - const layer = AppNodeBuilder.build(LayerNode.group([FSUtil.node, Global.node]), [ + const layer = AppNodeBuilder.build(LayerNode.group([FSUtil.node, Global.node, PersistentPty.node]), [ Global.node.replace(Global.layerWith({ data: tmp.path })), + PersistentPty.node.replace(disabledPersistentPtyLayer), ]) return Effect.gen(function* () { const fs = yield* FSUtil.Service diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts new file mode 100644 index 000000000000..4db4594595f3 --- /dev/null +++ b/packages/core/test/shell-daemon.test.ts @@ -0,0 +1,413 @@ +import { describe, expect } from "bun:test" +import { existsSync, readdirSync, realpathSync } from "node:fs" +import { mkdir, rm, utimes, writeFile } from "node:fs/promises" +import path from "node:path" +import { Effect, Layer } from "effect" +import { TestClock } from "effect/testing" +import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { Config } from "@opencode/core/config" +import { Environment } from "@opencode/core/environment/index" +import { Location } from "@opencode/core/location" +import { AbsolutePath } from "@opencode/core/schema" +import { Shell } from "@opencode/core/shell" +import { Global } from "@opencode/util/global" +import { FSUtil } from "@opencode/util/fs-util" +import { LayerNode } from "@opencode/util/effect/layer-node" +import { AppProcess } from "@opencode/util/process" +import { hostEnvironmentLayer } from "./fixture/environment" +import { location } from "./fixture/location" +import { daemonShells, until } from "./fixture/shell-daemon" +import { tmpdirScoped } from "./fixture/tmpdir" +import { binaryPath } from "@opencode/pty" +import { it } from "./lib/effect" + +const unix = process.platform !== "win32" +const test = unix ? it.live : it.live.skip +// On the test clock, for timers. +const timed = unix ? it.effect : it.effect.skip + +const lines = (count: number) => Array.from({ length: count }, (_, index) => `line ${index + 1}\n`).join("") + +describe("Shell on the process daemon", () => { + test("runs a command with the server's environment, an absolute cwd and its metadata", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* Effect.promise(() => mkdir(path.join(env.directory, "sub"))) + process.env.OPENCODE_TEST_REMOVED = "1" + yield* Effect.addFinalizer(() => Effect.sync(() => delete process.env.OPENCODE_TEST_REMOVED)) + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const info = yield* shell.create( + { + shell: "/bin/sh", + command: "echo hi", + cwd: "sub", + timeout: 60_000, + metadata: { sessionID: "ses_test" }, + }, + (invocation) => + Effect.sync(() => { + delete invocation.env.OPENCODE_TEST_REMOVED + invocation.env.OPENCODE_TEST_UNSET = undefined + }), + ) + const [request] = env.created() + expect(request).toMatchObject({ + program: "/bin/sh", + args: ["-c", "echo hi"], + cwd: path.join(env.directory, "sub"), + output_path: info.file, + metadata: { + id: info.id, + command: "echo hi", + shell: "/bin/sh", + location: { directory: env.directory }, + metadata: { sessionID: "ses_test" }, + timeoutDeadline: expect.any(Number), + }, + }) + expect(request?.env).toMatchObject({ TERM: "xterm-256color", OPENCODE_TERMINAL: "1" }) + expect(request?.env).not.toHaveProperty("OPENCODE_TEST_REMOVED") + expect(request?.env).not.toHaveProperty("OPENCODE_TEST_UNSET") + expect(info).toMatchObject({ status: "running", pid: 1001, cwd: path.join(env.directory, "sub") }) + expect(env.spawned()).toBe(0) + + env.daemon.output(1, "first\nsecond") + expect(yield* shell.output(info.id)).toEqual({ + output: "first\nsecond", + cursor: 12, + size: 12, + truncated: false, + }) + expect(yield* shell.output(info.id, { cursor: 6, limit: 3 })).toEqual({ + output: "sec", + cursor: 9, + size: 12, + truncated: false, + }) + expect(yield* shell.output(info.id, { cursor: Number.MAX_SAFE_INTEGER })).toEqual({ + output: "", + cursor: 12, + size: 12, + truncated: false, + }) + expect((yield* shell.list()).map((item) => item.id)).toEqual([info.id]) + + env.daemon.exit(1, 0) + expect(yield* shell.result(info)).toMatchObject({ + info: { status: "exited", exit: 0, time: { completed: expect.any(Number) } }, + capture: { output: "first\nsecond", truncated: false }, + }) + expect(yield* shell.list()).toEqual([]) + }), + ) + })) + + test("counts lines like the spawner and reads only what reached the file after a write error", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const long = yield* shell.create({ shell: "/bin/sh", command: "long" }) + env.daemon.output(1, lines(3000)) + env.daemon.exit(1, 0) + const result = yield* shell.result(long) + expect(result.capture?.output).toEndWith( + `line 3000\n\n[showing lines 1001-3000 of 3000; full output saved to ${long.file}]`, + ) + + const failed = yield* shell.create({ shell: "/bin/sh", command: "failed" }) + env.daemon.output(2, "kept\n") + env.daemon.fail(2, "No space left on device") + env.daemon.output(2, "lost\n") + env.daemon.exit(2, 1) + expect(yield* shell.result(failed)).toMatchObject({ + info: { status: "exited", exit: 1 }, + capture: { + output: `kept\n\n\n[output file ${failed.file} stopped early: No space left on device]`, + }, + }) + + // A single line longer than the window is sized from the file, which the daemon does not track. + const wide = yield* shell.create({ shell: "/bin/sh", command: "wide" }) + env.daemon.output(3, `short\n${"x".repeat(60 * 1024)}\n`) + env.daemon.exit(3, 0) + expect((yield* shell.result(wide)).capture?.output).toEndWith( + `\n\n[showing the last 50 KB of line 2 (60 KB); full output saved to ${wide.file}]`, + ) + }), + ) + })) + + test("reports a command lost with the daemon as killed, with what it wrote", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const info = yield* shell.create({ shell: "/bin/sh", command: "work" }) + env.daemon.output(1, "partial\n") + yield* until(() => env.daemon.waiters(1) > 0) + env.daemon.drop(1) + expect(yield* shell.result(info)).toMatchObject({ + info: { status: "killed" }, + capture: { output: "partial\n", killed: "Command was killed: the process daemon stopped" }, + }) + }), + ) + })) + + test("stops what a failed command left in its group, and leaves a successful one's", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const succeeded = yield* shell.create({ shell: "/bin/sh", command: "serve & exit 0" }) + const failed = yield* shell.create({ shell: "/bin/sh", command: "serve & exit 3" }) + yield* until(() => env.daemon.waiters(1) > 0 && env.daemon.waiters(2) > 0) + env.daemon.exit(1, 0) + env.daemon.exit(2, 3) + expect(yield* shell.wait(failed.id)).toMatchObject({ status: "exited", exit: 3 }) + expect(yield* shell.wait(succeeded.id)).toMatchObject({ status: "exited", exit: 0 }) + expect(env.signals(2)).toEqual(["SIGTERM"]) + expect(env.signals(1)).toEqual([]) + }), + ) + })) + + test("fails a command the daemon rejected instead of spawning it", () => + Effect.gen(function* () { + const env = yield* daemonShells() + env.daemon.failCreate.message = "output path is a symlink" + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const error = yield* shell.create({ shell: "/bin/sh", command: "echo hi" }).pipe(Effect.flip) + expect(error).toBeInstanceOf(AppProcess.AppProcessError) + expect(env.spawned()).toBe(0) + expect(yield* shell.list()).toEqual([]) + }), + ) + })) + + // On the test clock: the timeout and the grace period pass without waiting for them. + timed("stops a timed-out command with SIGTERM, then SIGKILL after the grace period, and keeps it", () => + Effect.gen(function* () { + const env = yield* daemonShells() + env.daemon.trap.add(1) + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const info = yield* shell.create({ shell: "/bin/sh", command: "trap '' TERM; sleep 60", timeout: 10 }) + yield* TestClock.adjust("10 millis") + yield* until(() => env.signals(1).length > 0) + expect(env.signals(1)).toEqual(["SIGTERM"]) + expect((yield* shell.get(info.id)).status).toBe("running") + // The grace period starts once the daemon acknowledged SIGTERM, so keep the clock moving until it ends. + yield* until(TestClock.adjust("1 second").pipe(Effect.map(() => env.signals(1).length > 1))) + expect((yield* shell.wait(info.id)).status).toBe("timeout") + expect(env.signals(1)).toEqual(["SIGTERM", "SIGKILL"]) + expect(env.daemon.processes[0]?.pipe?.metadata).toMatchObject({ stopReason: "timeout" }) + expect(env.daemon.processes[0]?.pipe?.metadata).not.toHaveProperty("timeoutDeadline") + expect(yield* shell.result(info)).toMatchObject({ + info: { status: "timeout" }, + capture: { output: "(no output)" }, + }) + // Like a spawned command, it records how the stopped process ended. + expect((yield* shell.result(info)).info).toMatchObject({ signal: "SIGKILL" }) + expect(env.daemon.ops).not.toContain("terminate") + expect(existsSync(info.file)).toBe(true) + }), + ) + }), + ) + + test("forgets a command in the daemon before deleting its file, on remove and on eviction", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const running = yield* shell.create({ shell: "/bin/sh", command: "sleep 60" }) + yield* shell.remove(running.id) + expect(env.daemon.requests.filter((request) => request.op === "terminate")).toEqual([ + expect.objectContaining({ id: 1 }), + ]) + expect(existsSync(running.file)).toBe(false) + expect(env.daemon.processes).toEqual([]) + expect(yield* shell.result(running)).toMatchObject({ info: { status: "killed" }, capture: undefined }) + + const finished = yield* Effect.forEach(Array.from({ length: 26 }), (_, index) => + Effect.gen(function* () { + const info = yield* shell.create({ shell: "/bin/sh", command: `echo ${index}` }) + env.daemon.exit(index + 2, 0) + yield* shell.wait(info.id) + return info + }), + ) + // Waiters resolve before retention runs, which forgets the process and then deletes its file. + yield* until(() => !existsSync(finished[0]!.file)) + expect(yield* shell.get(finished[0]!.id).pipe(Effect.flip)).toBeInstanceOf(Shell.NotFoundError) + expect(existsSync(finished[0]!.file)).toBe(false) + expect(env.daemon.processes.map((process) => process.id)).not.toContain(2) + expect(existsSync(finished[1]!.file)).toBe(true) + }), + ) + })) + + timed("a Location closed while the server runs keeps enforcing its commands' timeouts", () => + Effect.gen(function* () { + const env = yield* daemonShells() + const closing = yield* env.openLocation(env.directory) + const info = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60", timeout: 50 }) + yield* closing.close + expect(env.signals(1)).toEqual([]) + yield* TestClock.adjust("50 millis") + yield* until(() => env.signals(1).length > 0) + expect(env.signals(1)).toEqual(["SIGTERM"]) + expect(yield* closing.shell.wait(info.id)).toMatchObject({ status: "timeout" }) + }), + ) + + timed("a closed Location and its replacement see the same commands, with one timer and one report each", () => + Effect.gen(function* () { + const env = yield* daemonShells() + const closing = yield* env.openLocation(env.directory) + const backgrounded = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60", timeout: 100 }) + const stopped = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60" }) + const removed = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60" }) + yield* closing.close + let exits = 0 + yield* env.serverBus.listen((event) => + Effect.sync(() => { + if (event.type === "shell.exited") exits++ + }), + ) + const replacement = yield* env.openLocation(env.directory) + expect((yield* replacement.shell.list()).map((item) => item.id)).toEqual([ + backgrounded.id, + stopped.id, + removed.id, + ]) + + // The call that started it, still in the closed Location, moves it to the background and clears its timeout. + yield* closing.shell.timeout(backgrounded.id, 0) + yield* TestClock.adjust("100 millis") + expect(env.signals(1)).toEqual([]) + + // Cancelling its job in the closed Location stops it; both see why it ended, and it is reported once. + yield* closing.shell.stop(stopped.id, "killed") + expect(yield* replacement.shell.wait(stopped.id)).toMatchObject({ status: "killed" }) + expect(yield* closing.shell.wait(stopped.id)).toMatchObject({ status: "killed" }) + yield* until(() => exits === 1) + + // Removed through one, it is gone from both. + yield* replacement.shell.remove(removed.id) + expect(yield* closing.shell.get(removed.id).pipe(Effect.flip)).toBeInstanceOf(Shell.NotFoundError) + // The server watched them all along: the replacement took nothing back and waited on nothing more. + expect(env.daemon.ops.filter((op) => op === "wait")).toHaveLength(3) + yield* replacement.close + }), + ) + + test("the sweep keeps a running command's file and forgets a finished one before deleting its file", () => + Effect.gen(function* () { + const env = yield* daemonShells() + const ids = yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const running = yield* shell.create({ shell: "/bin/sh", command: "running" }) + const finished = yield* shell.create({ shell: "/bin/sh", command: "finished" }) + env.daemon.exit(2, 0) + yield* shell.wait(finished.id) + const lostFile = yield* shell.create({ shell: "/bin/sh", command: "lost file" }) + env.daemon.exit(3, 0) + yield* shell.wait(lostFile.id) + const old = yield* shell.create({ shell: "/bin/sh", command: "old" }) + env.daemon.exit(4, 0) + yield* shell.wait(old.id) + return { running, finished, lostFile, old } + }), + ) + // A finished process whose file was deleted elsewhere, and one that ended more than 7 days ago. + yield* Effect.promise(() => rm(ids.lostFile.file)) + const ended = env.daemon.processes.find((process) => process.id === 4) + if (ended) ended.exited_at = Date.now() - 8 * 24 * 60 * 60 * 1_000 + const expired = new Date(Date.now() - 8 * 24 * 60 * 60 * 1_000) + yield* Effect.promise(() => utimes(ids.running.file, expired, expired)) + yield* Effect.promise(() => utimes(ids.finished.file, expired, expired)) + const orphan = path.join(path.dirname(ids.running.file), "sh_0123456789abORPHANORPHANOR.out") + yield* Effect.promise(() => writeFile(orphan, "orphan")) + yield* Effect.promise(() => utimes(orphan, expired, expired)) + + yield* Shell.cleanup().pipe( + Effect.provide( + AppNodeBuilder.build(LayerNode.group([FSUtil.node, Global.node, PersistentPty.node]), [ + Global.node.replace(env.global), + PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, env.pty)), + ]), + ), + ) + + expect(existsSync(ids.running.file)).toBe(true) + expect(existsSync(ids.finished.file)).toBe(false) + expect(existsSync(orphan)).toBe(false) + expect(env.daemon.processes.map((process) => process.id)).toEqual([1]) + // The old process's file is recent, so it stays. + expect(existsSync(ids.old.file)).toBe(true) + })) +}) + +// Against the real opencode-pty, like server/test/persistent-pty.test.ts. +const binary = unix ? (process.env.OPENCODE_PTY_BIN ?? binaryPath) : undefined +const real = binary !== undefined && existsSync(binary) ? it.live : it.live.skip + +describe("Shell on a real process daemon", () => { + real( + "starts the daemon on the first command, never on boot or sweep, and runs it with the server's environment", + () => + Effect.gen(function* () { + const data = (yield* tmpdirScoped()).path + const directory = (yield* tmpdirScoped()).path + const state = path.join(data, "state") + const global = Global.layerWith({ data, state }) + const started = () => existsSync(path.join(state, "pty")) && readdirSync(path.join(state, "pty")).length > 0 + process.env.OPENCODE_TEST_REMOVED = "1" + yield* Effect.addFinalizer(() => Effect.sync(() => delete process.env.OPENCODE_TEST_REMOVED)) + yield* Effect.gen(function* () { + const shell = yield* Shell.Service + yield* Shell.cleanup() + expect(started()).toBe(false) + const info = yield* shell.create( + { + shell: "/bin/sh", + command: 'printf "%s|%s|%s\\n" "${OPENCODE_TEST_REMOVED:-unset}" "$TERM" "$(pwd)"; printf tail', + }, + (invocation) => + Effect.sync(() => { + delete invocation.env.OPENCODE_TEST_REMOVED + }), + ) + expect(started()).toBe(true) + expect(yield* shell.result(info)).toMatchObject({ + info: { status: "exited", exit: 0 }, + capture: { + output: `unset|xterm-256color|${realpathSync(directory)}\ntail`, + }, + }) + }).pipe( + Effect.scoped, + Effect.provide( + AppNodeBuilder.build(LayerNode.group([Shell.node, FSUtil.node, Global.node, PersistentPty.node]), [ + Location.node.replace( + Layer.succeed( + Location.Service, + Location.Service.of(location(Location.Ref.make({ directory: AbsolutePath.make(directory) }))), + ), + ), + Global.node.replace(global), + Config.node.replace(Config.testLayer()), + Environment.node.replace(hostEnvironmentLayer), + ]), + ), + ) + }), + { timeout: 20_000 }, + ) +}) diff --git a/packages/core/test/shell-result.test.ts b/packages/core/test/shell-result.test.ts index 480028db69ab..25cc8133985d 100644 --- a/packages/core/test/shell-result.test.ts +++ b/packages/core/test/shell-result.test.ts @@ -46,3 +46,20 @@ describe("ShellResult.interrupted", () => { }) }) }) + +describe("ShellResult.output", () => { + test("reports a command killed without an exit code or signal of its own", () => { + const lost = ShellResult.output({ + info: info("killed"), + capture: { output: "partial\n", truncated: false, killed: "Command was killed: the process daemon stopped" }, + }) + expect(ShellResult.notice(lost)).toBe("Command was killed: the process daemon stopped") + expect(ShellResult.metadata(lost)).toMatchObject({ killed: true }) + }) + + test("keeps a removed command's result unchanged", () => { + const removed = ShellResult.output({ info: info("killed"), capture: undefined }) + expect(removed).toEqual({ output: ShellResult.unavailable.output, truncated: false }) + expect(ShellResult.notice(removed)).toBeUndefined() + }) +}) diff --git a/packages/core/test/shell-retention.test.ts b/packages/core/test/shell-retention.test.ts index 3f776e4bb275..e650568496d1 100644 --- a/packages/core/test/shell-retention.test.ts +++ b/packages/core/test/shell-retention.test.ts @@ -10,6 +10,8 @@ import { AbsolutePath } from "@opencode/core/schema" import { Shell } from "@opencode/core/shell" import { Workspace } from "@opencode/core/workspace" import { Global } from "@opencode/util/global" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { disabledPersistentPtyLayer } from "./fixture/persistent-pty" import { hostEnvironmentLayer } from "./fixture/environment" import { tempGlobalLayer } from "./fixture/global" import { location, tempLocationLayer } from "./fixture/location" @@ -95,6 +97,7 @@ it.live("eviction makes progress past an already-removed shell", () => Global.node.replace(tempGlobalLayer), Config.node.replace(Config.testLayer()), Environment.node.replace(environment), + PersistentPty.node.replace(disabledPersistentPtyLayer), ]), ), ) diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index e6b5d38f8198..3cc8354675cf 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -42,6 +42,9 @@ import { ShellTool } from "@opencode/core/tool/plugin/shell" import { ToolOutput } from "@opencode/core/tool-output" import { Tool } from "@opencode/core/tool" import { PluginHooks } from "@opencode/core/plugin/hooks" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { disabledPersistentPtyLayer } from "./fixture/persistent-pty" +import { fakeDaemon } from "./fixture/persistent-pty-daemon" import { tmpdir, tmpdirScoped, withTempDir } from "./fixture/tmpdir" import { tempGlobalLayer } from "./fixture/global" import { offlineModels } from "./fixture/models" @@ -142,12 +145,15 @@ const shellPluginSupervisor = makeLocationNode({ Session.node, Job.node, Shell.node, + Shell.daemonNode, ShellSelect.node, Tool.node, ], }) const nodes = LayerNode.group([ + PersistentPty.node, + Shell.daemonNode, Database.node, Bus.node, Job.node, @@ -158,21 +164,61 @@ const nodes = LayerNode.group([ FSUtil.node, Global.node, ]) -const replacements = [ +const daemonReplacements = [ SessionExecution.node.replace(executionNode), Permission.node.replace(permission), Global.node.replace(tempGlobalLayer), offlineModels, ] satisfies LayerNode.Replacements +// Most suites spawn commands from the server, so CI starts no daemon for them; `parity` runs a subset on both. +const replacements = [ + ...daemonReplacements, + PersistentPty.node.replace(disabledPersistentPtyLayer), +] satisfies LayerNode.Replacements const productionIt = testEffect(AppNodeBuilder.build(nodes, replacements)) const it = testEffect( AppNodeBuilder.build(nodes, [...replacements, PluginSupervisor.node.replace(shellPluginSupervisor)]), ) +// On the real process daemon (when its binary is compatible; otherwise this falls back to spawning). +const daemonIt = testEffect( + AppNodeBuilder.build(nodes, [...daemonReplacements, PluginSupervisor.node.replace(shellPluginSupervisor)]), +) +const parity = { + live: (name: string, body: Parameters[1], options?: Parameters[2]) => { + it.live(`${name} (spawned)`, body, options) + daemonIt.live(`${name} (daemon)`, body, options) + }, + skip: (name: string, body: Parameters[1], options?: Parameters[2]) => + it.live.skip(name, body, options), +} +// On an in-process fake daemon, so daemon-only behavior is covered without the real binary. +let fake: Effect.Success> | undefined +const fakeDaemonIt = testEffect( + AppNodeBuilder.build(nodes, [ + ...daemonReplacements, + PluginSupervisor.node.replace(shellPluginSupervisor), + PersistentPty.node.replace( + makeGlobalNode({ + service: PersistentPty.Service, + layer: Layer.unwrap( + fakeDaemon().pipe( + Effect.map((daemon) => { + fake = daemon + return PersistentPty.layerWith(daemon.options) + }), + ), + ), + deps: [Bus.node, Global.node], + }), + ), + ]), +) const permissionIt = testEffect( AppNodeBuilder.build(LayerNode.group([nodes, PermissionSaved.node]), [ SessionExecution.node.replace(executionNode), Global.node.replace(tempGlobalLayer), PluginSupervisor.node.replace(shellPluginSupervisor), + PersistentPty.node.replace(disabledPersistentPtyLayer), offlineModels, ]), ) @@ -184,6 +230,19 @@ const call = (input: typeof ShellTool.Input.Type, id = "call-shell") => ({ }) const isWindows = process.platform === "win32" + +// The process daemon keeps a finished command's zombie until the command is forgotten. +const running = (pid: number) => { + try { + process.kill(pid, 0) + } catch { + return false + } + return !Bun.spawnSync(["ps", "-o", "stat=", "-p", String(pid)]) + .stdout.toString() + .trim() + .startsWith("Z") +} const cwdCommand = isWindows ? "(Get-Location).Path; Start-Sleep -Milliseconds 100" : "pwd" const helloCommand = isWindows ? "[Console]::Out.Write('hello'); Start-Sleep -Milliseconds 100" : "printf hello" const stderrCommand = isWindows @@ -1199,6 +1258,7 @@ describe("ShellTool", () => { Session.node, Job.node, Shell.node, + Shell.daemonNode, ShellSelect.node, Tool.node, ], @@ -1268,7 +1328,7 @@ describe("ShellTool", () => { ) } - it.live("keeps non-zero exits useful", () => + parity.live("keeps non-zero exits useful", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => @@ -1558,7 +1618,7 @@ describe("ShellTool", () => { { timeout: 15_000 }, ) - it.live("uses configured line limits", () => + parity.live("uses configured line limits", () => withTempDir((tmp) => { reset() return Effect.gen(function* () { @@ -1742,7 +1802,79 @@ describe("ShellTool", () => { }), ) - it.live("returns the shell id and PID for a background command", () => + fakeDaemonIt.live("keeps a background daemon command's job across a Location reload and reports how it ended", () => + withTempDir((tmp) => + Effect.gen(function* () { + reset() + const sessions = yield* Session.Service + const jobs = yield* Job.Service + const bus = yield* Bus.Service + const locations = yield* LocationServiceMap.Service + const location = Location.Ref.make({ directory: AbsolutePath.make(tmp.path) }) + yield* sessions.create({ id: sessionID, title: "shell test", location, model: sessionModel }) + const admitted = yield* bus.subscribe(SessionEvent.InboxEnqueued).pipe( + Stream.filter((event) => event.data.sessionID === sessionID && event.data.item.type === "synthetic"), + Stream.take(2), + Stream.runCollect, + Effect.forkScoped({ startImmediately: true }), + ) + const started = yield* Effect.gen(function* () { + const plugins = yield* Plugin.Service + yield* plugins.awaitActivation + const registry = yield* Tool.Service + const shell = yield* Shell.Service + const run = (id: string) => + Effect.gen(function* () { + const settled = yield* executeTool(registry, call({ command: "work", background: true }, id)) + return yield* shell.get(ID.make(String(settled.metadata?.shellID))) + }) + return { finishing: yield* run("call-finishing"), cancelled: yield* run("call-cancelled") } + }).pipe(Effect.provide(locations.get(location))) + const process = (info: { file: string }) => { + const found = fake?.processes.find((item) => item.pipe?.output_path === info.file) + if (!found) throw new Error("no daemon process for the command") + return found.id + } + + // A reload closes the Location and boots its replacement, which takes the commands back. + yield* locations.invalidate(location) + const replacement = yield* Effect.gen(function* () { + return yield* Shell.Service + }).pipe(Effect.provide(locations.get(location))) + expect((yield* replacement.list()).map((info) => info.id).toSorted()).toEqual( + [started.finishing.id, started.cancelled.id].toSorted(), + ) + expect((yield* jobs.get(started.finishing.id))?.status).toBe("running") + + // Cancelling the session's job still stops its command. + yield* jobs.cancel(started.cancelled.id) + while (!fake?.ops.includes("signal")) yield* Effect.promise(() => Bun.sleep(2)) + expect(fake?.requests).toContainEqual( + expect.objectContaining({ op: "signal", id: process(started.cancelled), signal: "SIGTERM" }), + ) + + fake?.output(process(started.finishing), "done\n") + fake?.exit(process(started.finishing), 0) + const notices = Array.from(yield* Fiber.join(admitted)).map((event) => event.data.item.payload) + expect(notices).toContainEqual( + expect.objectContaining({ + text: expect.stringContaining("done"), + metadata: expect.objectContaining({ shellID: started.finishing.id, state: "completed", exit: 0 }), + }), + ) + expect(notices).toContainEqual( + expect.objectContaining({ + metadata: expect.objectContaining({ shellID: started.cancelled.id, state: "cancelled" }), + }), + ) + expect((yield* jobs.get(started.finishing.id))?.status).toBe("completed") + expect((yield* replacement.get(started.finishing.id)).status).toBe("exited") + yield* locations.invalidate(location) + }), + ), + ) + + parity.live("returns the shell id and PID for a background command", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => @@ -1910,8 +2042,8 @@ describe("ShellTool", () => { { timeout: 15_000 }, ) - const signalTest = isWindows ? it.live.skip : it.live - signalTest( + const signalParity = isWindows ? parity.skip : parity.live + signalParity( "escalates a shell timeout when the ready process ignores SIGTERM", () => Effect.gen(function* () { @@ -1946,7 +2078,7 @@ describe("ShellTool", () => { }) const pid = info.pid if (pid === undefined) throw new Error("Expected shell PID") - expect(() => process.kill(pid, 0)).toThrow() + expect(running(pid)).toBe(false) }), (info) => Effect.try(() => { @@ -2009,7 +2141,7 @@ describe("ShellTool", () => { ) } - it.live("backgrounds a foreground command when the session is signaled", () => + parity.live("backgrounds a foreground command when the session is signaled", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => @@ -2089,7 +2221,7 @@ describe("ShellTool stopped", () => { return { shellID, exit, result: Exit.isSuccess(exit) ? exit.value : undefined } }) - const test = isWindows ? it.live.skip : it.live + const test = isWindows ? parity.skip : parity.live test("stops a command the user stopped, keeps its file and fails the call with the tail, file and shell ID", () => withTempDir((tmp) => { diff --git a/packages/server/src/routes.ts b/packages/server/src/routes.ts index d987cd479716..b32b1be42149 100644 --- a/packages/server/src/routes.ts +++ b/packages/server/src/routes.ts @@ -20,6 +20,7 @@ import { Instance } from "@opencode/core/instance/service" import { SessionTransfer } from "@opencode/core/session/transfer" import { InstructionEntry } from "@opencode/core/session/instruction-entry" import { ShellSelect } from "@opencode/core/shell/select" +import { Shell } from "@opencode/core/shell" import { Job } from "@opencode/core/job" import { KV } from "@opencode/core/kv" import { Mcp } from "@opencode/core/mcp/index" @@ -75,6 +76,8 @@ const applicationServiceNodes = [ PermissionSaved.node, PtyTicket.node, PersistentPty.node, + // Daemon-run shell commands belong to the server, so a Location closing never takes them down. + Shell.daemonNode, Credential.node, WellKnown.node, ServerPairing.node, From 24711616725a7158ae7a43212b046f64f3a1028a Mon Sep 17 00:00:00 2001 From: James Long Date: Mon, 9 Nov 2026 20:44:25 +0000 Subject: [PATCH 02/12] feat(core): keep daemon shell commands running across a planned restart A planned restart (daemon handoff) now keeps local shell commands running, foreground calls included, with one boot-time path instead of shutdown-time work: - On shutdown nothing happens to commands. Only an explicit cancel (the user's interrupt, Job.cancel, cancelSession) stops one, through a new job cancel hook; teardown just interrupts the job. A closing job registry keeps a background job's row running for the next server. - A step interrupted by a server shutdown is left as a crash would leave it: unsettled calls, no step end. The session's interrupt reason tells a shutdown from the user. - The daemon keeps each command's tool call. The new server takes its Location's commands back when the Location boots, reattaches background jobs, and settles an unsettled foreground call as moved to the background with a background job, before recovered child sessions resume. - A foreground command's deadline is never written to the daemon, so the new server cannot time out a call it moves to the background. A quit without a handoff still ends every command with the daemon. User decisions: - Commands survive a planned restart, foreground calls included - One boot-time path; no shutdown-time backgrounding, no detaching service - Stop a command only on an explicit cancel, never because it was interrupted - Background jobs keep their row on shutdown --- packages/core/src/job.ts | 19 +- packages/core/src/session/execution.ts | 3 +- .../core/src/session/execution/restart.ts | 161 +++++++- packages/core/src/session/runner/step.ts | 8 + packages/core/src/shell.ts | 181 +++++++-- packages/core/src/tool/plugin/shell.ts | 200 ++++++---- packages/core/test/job.test.ts | 53 +++ packages/core/test/session-execution.test.ts | 344 +++++++++++++++++- packages/core/test/session-step.test.ts | 10 +- packages/core/test/shell-daemon.test.ts | 67 ++++ packages/core/test/tool-shell.test.ts | 32 +- 11 files changed, 968 insertions(+), 110 deletions(-) diff --git a/packages/core/src/job.ts b/packages/core/src/job.ts index fd5d391b396b..86605e0de8a3 100644 --- a/packages/core/src/job.ts +++ b/packages/core/src/job.ts @@ -49,6 +49,7 @@ type Active = { isBackgrounded: boolean consumed: boolean recovery?: Recovery + cancel?: Effect.Effect } type State = { @@ -59,6 +60,9 @@ type State = { type FinishResult = { info?: Info done?: Deferred.Deferred + // Teardown interrupted the work: `done` is interrupted instead of settled. + interrupted?: boolean + cancel?: Effect.Effect scope?: Scope.Closeable generation?: Scope.Closeable } @@ -90,6 +94,9 @@ export type StartInput = { recovery?: Recovery notificationID?: SessionMessage.ID run: Effect.Effect + // Runs when the job is cancelled (`cancel`, `cancelSession`), before its work is interrupted. Teardown, such as + // a server shutdown, interrupts the work without it. + cancel?: Effect.Effect } export type WaitInput = { @@ -248,6 +255,11 @@ export const make = Effect.gen(function* () { if (!job) return [{}, jobs] if (job.scope !== scope) return [{}, jobs] if (job.info.status !== "running") return [{ info: snapshot(job) }, jobs] + // Teardown (this registry closing) is not a terminal outcome: leave the job and its durable row running for + // the next server, and interrupt rather than settle its observers. A cancel has already settled the job + // as cancelled before interrupting its work. + if (Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause) && scope.state._tag === "Closed") + return [{ done: job.done, scope: job.scope, interrupted: true }, jobs] const status: Exclude = Exit.isSuccess(exit) ? "completed" : Cause.hasInterruptsOnly(exit.cause) @@ -268,6 +280,7 @@ export const make = Effect.gen(function* () { return [{ info: snapshot(next), done: job.done, scope: job.scope }, new Map(jobs).set(id, next)] }), ) + if (result.interrupted && result.done) yield* Deferred.interrupt(result.done) if (result.info && result.done) yield* Deferred.succeed(result.done, result.info) if (result.scope) { yield* Scope.close(result.scope, Exit.void).pipe(Effect.forkIn(state.scope, { startImmediately: true })) @@ -295,6 +308,8 @@ export const make = Effect.gen(function* () { return [{ info: snapshot(existing) }, jobs] } const scope = yield* Scope.fork(state.scope, "parallel") + // Closing the registry interrupts the work without settling it; interrupt its observers too. + yield* Scope.addFinalizer(scope, Deferred.interrupt(done)) const job = { info: { id: input.id, @@ -312,6 +327,7 @@ export const make = Effect.gen(function* () { isBackgrounded: false, consumed: false, recovery: input.recovery, + cancel: input.cancel, } return [{ info: snapshot(job), scope }, new Map(jobs).set(input.id, job)] }), @@ -459,11 +475,12 @@ export const make = Effect.gen(function* () { } if (persist) yield* persistBackground(next) return [ - { info: snapshot(next), done: job.done, scope: job.scope, generation: job.scope }, + { info: snapshot(next), done: job.done, scope: job.scope, generation: job.scope, cancel: job.cancel }, new Map(jobs).set(id, next), ] }), ) + if (result.cancel) yield* result.cancel if (result.info && result.done) yield* Deferred.succeed(result.done, result.info) if (result.scope) yield* Scope.close(result.scope, Exit.void) if (result.generation) yield* consume(id, result.generation) diff --git a/packages/core/src/session/execution.ts b/packages/core/src/session/execution.ts index f2867c4f7e3d..b70a90b3df68 100644 --- a/packages/core/src/session/execution.ts +++ b/packages/core/src/session/execution.ts @@ -154,7 +154,8 @@ export const layer = Layer.effect( ), }) - interruption = coordinator.interruption + // Teardown interrupts without calling `interrupt`, so an interrupt with no recorded reason is a shutdown. + interruption = (sessionID) => coordinator.interruption(sessionID) ?? "shutdown" return Service.of({ active: coordinator.active, isActive: coordinator.isActive, diff --git a/packages/core/src/session/execution/restart.ts b/packages/core/src/session/execution/restart.ts index 3bf4bae98888..8db6e1e5bb54 100644 --- a/packages/core/src/session/execution/restart.ts +++ b/packages/core/src/session/execution/restart.ts @@ -1,12 +1,20 @@ export * as SessionRestart from "./restart.js" -import { Context, Effect, Layer } from "effect" +import { Context, Effect, Layer, Option, Schema } from "effect" +import { isReadonlyArrayNonEmpty } from "effect/Array" import { makeGlobalNode } from "@opencode/util/effect/app-node" +import { Shell as ShellSchema } from "@opencode/schema/shell" import { Bus } from "../../bus.js" +import { Database } from "../../database/database.js" +import { Instance } from "../../instance/service.js" import { Job } from "../../job.js" import { Session } from "../../session.js" +import { Shell } from "../../shell.js" +import { ShellTool } from "../../tool/plugin/shell.js" import { SessionEvent } from "../event.js" import { SessionExecution } from "../execution.js" +import { SessionHistory } from "../history.js" +import { SessionMessage } from "../message.js" import { SessionSchema } from "../schema.js" import { SessionStore } from "../store.js" import { ImageNotice } from "../../image/notice.js" @@ -72,6 +80,9 @@ export const layer = (options?: Options) => const bus = yield* Bus.Service const jobs = yield* Job.Service const sessions = yield* Session.Service + const instances = yield* Instance.Service + const db = (yield* Database.Service).db + const daemon = yield* Shell.Daemon const scope = yield* Effect.scope const maxAttempts = options?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS @@ -99,10 +110,7 @@ export const layer = (options?: Options) => return true }) - const restartMessage = ( - background: Job.Background, - recovery: Exclude, - ) => { + const restartMessage = (background: Job.Background, recovery: Exclude) => { const state = background.status === "running" ? "cancelled" : background.status if (recovery.kind === "shell") { const text = @@ -164,6 +172,106 @@ export const layer = (options?: Options) => yield* jobs.completeBackground(background.notificationID) }) + // The Shell of a session's Location, booted so it takes back the commands the daemon still runs for it. + const shellOf = Effect.fnUntraced(function* (sessionID: SessionSchema.ID) { + const session = yield* store.get(sessionID) + if (!session) return undefined + return yield* Effect.serviceOption(Shell.Service).pipe( + instances.provide(session), + Effect.map(Option.getOrUndefined), + // The session's directory is gone: its commands are reported like any lost background work. + Effect.catchTag("FileSystem.DirectoryNotFoundError", () => Effect.succeed(undefined)), + ) + }) + + const reattach = ( + shell: Shell.Interface, + sessionID: SessionSchema.ID, + info: ShellSchema.Info, + suspended: ReadonlySet, + notificationID?: SessionMessage.ID, + ) => + ShellTool.reattach({ + shell, + jobs, + sessions, + watch: daemon.watch, + sessionID, + info, + notificationID, + resume: () => (suspended.has(sessionID) ? false : undefined), + }) + + // A shell job the previous server left running is watched again if its command is still known (running or + // finished during the restart); otherwise it is reported as lost. + const recoverShellJob = Effect.fnUntraced(function* ( + background: Job.Background, + recovery: Extract, + suspended: ReadonlySet, + reattached: Set, + ) { + const shell = background.status === "running" ? yield* shellOf(recovery.sessionID) : undefined + const info = + shell && Schema.is(ShellSchema.ID)(recovery.shellID) + ? yield* shell.get(recovery.shellID).pipe(Effect.option) + : Option.none() + if (!shell || Option.isNone(info)) return yield* recoverToolJob(background, recovery) + reattached.add(info.value.id) + yield* reattach(shell, recovery.sessionID, info.value, suspended, background.notificationID) + }) + + // A server that stopped (shut down or killed) during a foreground shell call leaves the call unsettled, its + // command still running in the daemon. Settle the call as moved to the background and watch the command. + const settleBackgroundedCalls = Effect.fnUntraced(function* ( + sessionID: SessionSchema.ID, + suspended: ReadonlySet, + reattached: ReadonlySet, + ) { + const calls = (yield* SessionHistory.incompleteAssistants(db, sessionID)).flatMap((message) => + message.type === "assistant" + ? message.content.flatMap((part) => + part.type === "tool" && part.name === ShellTool.name && part.state.status === "running" + ? [{ assistantMessageID: message.id, part }] + : [], + ) + : [], + ) + if (calls.length === 0) return + const shell = yield* shellOf(sessionID) + if (!shell) return + yield* Effect.forEach( + calls, + Effect.fnUntraced(function* (call) { + const info = yield* shell.byToolCall({ + sessionID, + assistantMessageID: call.assistantMessageID, + callID: call.part.id, + }) + if (!info) return + const result = ShellTool.backgroundToolResult(info) + if (!isReadonlyArrayNonEmpty(result.content)) return + yield* bus.publish(SessionEvent.Tool.Success, { + sessionID, + assistantMessageID: call.assistantMessageID, + id: call.part.id, + content: result.content, + metadata: { + shellID: info.id, + status: "running", + truncated: false, + ...(info.pid === undefined ? {} : { pid: info.pid }), + }, + executed: call.part.executed === true, + }) + // Backgrounded but stopped before the call was settled: its job row was already reattached (and, for a + // command that already finished, may even be done). + if (reattached.has(info.id)) return + yield* reattach(shell, sessionID, info, suspended) + }), + { discard: true }, + ) + }) + const recoverSubagent = Effect.fnUntraced(function* ( background: Job.Background, recovery: Extract, @@ -232,13 +340,37 @@ export const layer = (options?: Options) => [...(yield* store.listSuspended()), ...children].filter((sessionID) => !active.has(sessionID)), ) yield* store.releaseChildClaims(children) + const recoverable = Effect.fnUntraced(function* (background: Job.Background) { + return (yield* jobs.get(background.id))?.status !== "running" + }) + // Shell and tool outcomes first, then calls a killed server left unsettled, and only then recovered children, + // whose first resume would otherwise fail their own unsettled shell calls. + const reattached = new Set() + yield* Effect.forEach( + pending, + Effect.fnUntraced(function* (background) { + const recovery = background.recovery + if (recovery.kind === "subagent" || !(yield* recoverable(background))) return + if (recovery.kind === "shell") return yield* recoverShellJob(background, recovery, suspended, reattached) + yield* recoverToolJob(background, recovery) + }), + { discard: true }, + ) + yield* Effect.forEach( + suspended, + (sessionID) => + settleBackgroundedCalls(sessionID, suspended, reattached).pipe( + Effect.catch((error) => + Effect.logWarning("failed to settle backgrounded shell calls", { sessionID, error }), + ), + ), + { discard: true }, + ) yield* Effect.forEach( - // Admit shell outcomes before a recovered child can start its first model request. - pending.toSorted((a, b) => Number(a.recovery.kind === "subagent") - Number(b.recovery.kind === "subagent")), + pending, Effect.fnUntraced(function* (background) { - if ((yield* jobs.get(background.id))?.status === "running") return const recovery = background.recovery - if (recovery.kind !== "subagent") return yield* recoverToolJob(background, recovery) + if (recovery.kind !== "subagent" || !(yield* recoverable(background))) return yield* recoverSubagent(background, recovery, suspended) }), { discard: true }, @@ -264,5 +396,14 @@ export const layer = (options?: Options) => export const node = makeGlobalNode({ service: Service, layer: layer(), - deps: [SessionStore.node, SessionExecution.node, Bus.node, Job.node, Session.node], + deps: [ + SessionStore.node, + SessionExecution.node, + Bus.node, + Job.node, + Session.node, + Instance.node, + Database.node, + Shell.daemonNode, + ], }) diff --git a/packages/core/src/session/runner/step.ts b/packages/core/src/session/runner/step.ts index d8096d740fd0..30ca3cd3118c 100644 --- a/packages/core/src/session/runner/step.ts +++ b/packages/core/src/session/runner/step.ts @@ -182,6 +182,14 @@ export const make = Effect.gen(function* () { // Read each tool's exit once all have ended, so what they raised while stopping (a decline, a defect) counts. const exits = Exit.isSuccess(joined) ? joined.value : yield* Fiber.awaitAll(fibers) const tools = classifyToolExits(exits, toolRuns, Exit.hasInterrupts(joined)) + // A server shutting down leaves the step as a crash would: unsettled calls and no step end. The next server + // settles them when the Session resumes, and takes over the shell commands still running for them. + if ( + (streamInterrupted || tools.interrupted) && + tools.declines.length === 0 && + input.interruption?.() === "shutdown" + ) + return yield* Effect.interrupt if ( !publisher.record().outputStarted && diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index c8980db9cacc..72fa2e93ad76 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -52,21 +52,35 @@ const LOST = "Command was killed: the process daemon stopped" type Info = Shell.Info export type StopReason = "timeout" | "killed" +/** The tool call that started a command, so a server that took over can settle it. */ +export const ToolCall = Schema.Struct({ + sessionID: Schema.String, + assistantMessageID: Schema.String, + callID: Schema.String, +}) +export type ToolCall = typeof ToolCall.Type + type CreateInput = Shell.CreateInput & { shell?: string + toolCall?: ToolCall + // A call waits on the command. A server that takes the command over moves it to the background, which clears + // its timeout, so its deadline is not kept for a restart. + waited?: boolean } -// What the server records about a daemon-run command in the daemon itself. +// What a server needs to take a daemon-run command back after a restart, kept in the daemon's metadata. const Stored = Schema.Struct({ id: Shell.ID, command: Schema.String, shell: Schema.String, location: Schema.Struct({ directory: Schema.String, workspaceID: Schema.optionalKey(Schema.String) }), metadata: Shell.Metadata, + toolCall: Schema.optional(ToolCall), timeoutDeadline: Schema.optional(Schema.Number), stopReason: Schema.optional(Schema.Literals(["timeout", "killed"])), }) type Stored = typeof Stored.Type +const decodeStored = Schema.decodeUnknownOption(Stored) // A command this server spawned itself. It dies with its Location. type Spawned = { @@ -91,6 +105,9 @@ type Run = { kind: "daemon" info: Info process: number + toolCall?: ToolCall + // A call waits on it, until it moves to the background. + waited: boolean timeoutDeadline?: number stopReason?: StopReason // From the exit: newlines in the file, and why the daemon stopped writing it early. @@ -121,8 +138,9 @@ type Ended = { * file-backed output by cursor. No session, message, or permission state lives * here; callers (e.g. `ShellTool`) own that association and store the shell ID. * - * Local commands run in the process daemon when it is available (`Daemon`); remote workspaces, Windows and an - * unavailable daemon keep spawning them from the server. + * Local commands run in the process daemon when it is available (`Daemon`), so they outlive a planned server + * restart: the next server takes them back. Remote workspaces, Windows and an unavailable daemon keep spawning + * them from the server. */ export interface Interface { readonly create: ( @@ -148,6 +166,8 @@ export interface Interface { // with the given status and stays observable, file included, until retention evicts it. Returns at // once; `wait` observes the exit. Ignored once the command has ended or is already stopping. readonly stop: (id: Shell.ID, reason: StopReason) => Effect.Effect + // The daemon command a tool call started, while it is known here (including one taken back after a restart). + readonly byToolCall: (call: ToolCall) => Effect.Effect readonly output: (id: Shell.ID, input?: Shell.OutputInput) => Effect.Effect readonly remove: (id: Shell.ID) => Effect.Effect } @@ -226,6 +246,8 @@ interface Runs { readonly args: readonly string[] readonly env: Readonly> readonly timeout: number + readonly toolCall: ToolCall | undefined + readonly waited: boolean }) => Effect.Effect readonly timeout: (run: Run, duration: number) => Effect.Effect readonly stop: (run: Run, reason: StopReason) => Effect.Effect @@ -233,7 +255,7 @@ interface Runs { } export interface DaemonInterface { - /** A Location's daemon commands. */ + /** A Location's daemon commands. The first call for a Location in this server takes its commands back from the daemon. */ readonly at: (location: Location.Interface) => Effect.Effect /** Runs `effect` until it ends or the server stops, outliving the caller's Location (e.g. a job's completion notice). */ readonly watch: (effect: Effect.Effect) => Effect.Effect @@ -249,16 +271,21 @@ const daemonLayer = Layer.effect( const pty = yield* PersistentPty.Service const scope = yield* Effect.scope const context = yield* Effect.context() + const global = yield* Global.Service const { unlink } = yield* Effect.promise(() => import("fs/promises")) - const places = new Map() - // Fibers run with the server's context in its scope, so a Location closing never interrupts them. + // Keyed by canonical Location. `recovered` resolves once the Location's commands were taken back. + const places = new Map }>() + // Fibers run with the server's context in its scope, so a Location closing never interrupts them. On shutdown + // they end without settling anything: the commands keep running for the next server, and the jobs waiting on + // them are left to the job registry, which keeps their rows for it. const fork = (effect: Effect.Effect) => effect.pipe(Effect.forkIn(scope, { startImmediately: true }), Effect.setContext(context)) - const place = (location: Location.Interface): Runs => { + const place = (location: Location.Interface) => { const runs = new Map() const exitOrder: Shell.ID[] = [] const ref = Location.Ref.make({ directory: location.directory, workspaceID: location.workspaceID }) + const outputDir = path.join(global.data, DIRECTORY, location.project.id) const persisted = (run: Run): Stored => ({ id: run.info.id, @@ -269,11 +296,12 @@ const daemonLayer = Layer.effect( ...(location.workspaceID !== undefined ? { workspaceID: location.workspaceID } : {}), }, metadata: run.info.metadata, - timeoutDeadline: run.timeoutDeadline, + toolCall: run.toolCall, + timeoutDeadline: run.waited ? undefined : run.timeoutDeadline, stopReason: run.stopReason, }) - // Records the command's latest timeout and stop reason in the daemon. + // Records the command's latest timeout and stop reason in the daemon, so a restart sees them. const store = (run: Run) => pty .setPipeMetadata(run.process, persisted(run)) @@ -365,6 +393,8 @@ const daemonLayer = Layer.effect( if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber) run.timeoutFiber = undefined run.timeoutDeadline = duration === 0 ? undefined : Date.now() + duration + // Clearing the timeout is how a call moves its command to the background. + if (duration === 0) run.waited = false yield* store(run) if (duration !== 0) yield* arm(run, duration) }) @@ -405,6 +435,11 @@ const daemonLayer = Layer.effect( Effect.flatMap((outcome) => finish(run, outcome)), ), ) + // A stop interrupted by a restart lost its grace period: kill now. + if (run.stopReason !== undefined) { + run.stopping = yield* fork(pty.signalPipe(run.process, "SIGKILL").pipe(Effect.ignore)) + return + } if (duration === undefined) return if (duration > 0) return yield* arm(run, duration) yield* stop(run, "timeout") @@ -415,6 +450,8 @@ const daemonLayer = Layer.effect( kind: "daemon", info: input.info, process: 0, + toolCall: input.toolCall, + waited: input.waited, timeoutDeadline: input.timeout > 0 ? input.info.time.started + input.timeout : undefined, done: Deferred.makeUnsafe(), } @@ -440,18 +477,104 @@ const daemonLayer = Layer.effect( return run.info }) - return { get: (id) => runs.get(id), list: () => Array.from(runs.values()), create, timeout, stop, remove } + // Takes back this Location's commands, left by a previous server. Two Locations can share a project's output + // directory, so match the Location itself. + const recover = Effect.fnUntraced(function* () { + const processes = yield* pty + .listPipes() + .pipe( + Effect.catch((error) => + Effect.logWarning("failed to list shell commands to recover", { error: error.message }).pipe( + Effect.as([]), + ), + ), + ) + const finished: Run[] = [] + for (const process of processes) { + const stored = Option.getOrUndefined(decodeStored(process.metadata)) + if ( + !stored || + stored.location.directory !== location.directory || + stored.location.workspaceID !== location.workspaceID || + path.dirname(process.outputPath) !== outputDir || + runs.has(stored.id) + ) + continue + const running = process.lifecycle.status === "running" + const exited = process.lifecycle.status === "exited" ? process.lifecycle : undefined + const status = running ? "running" : (stored.stopReason ?? (exited ? "exited" : "killed")) + const run: Run = { + kind: "daemon", + info: { + id: stored.id, + status, + command: stored.command, + cwd: process.cwd, + shell: stored.shell, + file: process.outputPath, + ...(process.pid !== undefined ? { pid: process.pid } : {}), + ...(exited?.exitCode !== undefined ? { exit: exited.exitCode } : {}), + ...(exited?.signal !== undefined ? { signal: exited.signal } : {}), + metadata: stored.metadata, + time: { + started: process.startedAt, + ...(running ? {} : { completed: process.exitedAt ?? process.startedAt }), + }, + }, + process: process.id, + toolCall: stored.toolCall, + // No call waits on it any more: whatever waited died with the previous server. + waited: false, + timeoutDeadline: stored.timeoutDeadline, + stopReason: stored.stopReason, + ...(running ? {} : { outputLines: process.outputLines, outputError: process.outputError }), + ...(process.lifecycle.status === "failed" && !stored.stopReason + ? { killed: `Command was killed: ${process.lifecycle.message}` } + : {}), + done: Deferred.makeUnsafe(), + } + runs.set(stored.id, run) + if (running) { + // A deadline that passed while no server ran fires at once. + const deadline = stored.timeoutDeadline + yield* attach(run, deadline === undefined ? undefined : Math.max(0, deadline - Date.now())) + continue + } + yield* Deferred.succeed(run.done, run.info) + finished.push(run) + } + exitOrder.push( + ...finished + .toSorted((a, b) => (a.info.time.completed ?? 0) - (b.info.time.completed ?? 0)) + .map((run) => run.info.id), + ) + yield* evict() + }) + + const handle: Runs = { + get: (id) => runs.get(id), + list: () => Array.from(runs.values()), + create, + timeout, + stop, + remove, + } + return { runs: handle, recover } } - const at = (location: Location.Interface) => - Effect.sync(() => { - const key = `${location.directory}\0${location.workspaceID ?? ""}` - const existing = places.get(key) - if (existing) return existing - const runs = place(location) - places.set(key, runs) - return runs - }) + const at = Effect.fnUntraced(function* (location: Location.Interface) { + const key = `${location.directory}\0${location.workspaceID ?? ""}` + const existing = places.get(key) + if (existing) { + yield* Deferred.await(existing.recovered) + return existing.runs + } + const placed = place(location) + const entry = { runs: placed.runs, recovered: Deferred.makeUnsafe() } + places.set(key, entry) + yield* placed.recover().pipe(Effect.ensuring(Deferred.succeed(entry.recovered, undefined))) + return placed.runs + }) return Daemon.of({ at, @@ -466,7 +589,7 @@ const daemonLayer = Layer.effect( export const daemonNode = makeGlobalNode({ service: Daemon, layer: daemonLayer, - deps: [Bus.node, PersistentPty.node], + deps: [Bus.node, Global.node, PersistentPty.node], }) const layer = () => @@ -487,7 +610,8 @@ const layer = () => // Commands this server spawned for the Location; daemon commands live in `runs`, owned by the server. const commands = new Map() const exitOrder: Shell.ID[] = [] - // Remote workspaces and Windows keep spawning commands from the server. + // Remote workspaces and Windows keep spawning commands from the server. Taken back before the service is + // returned, so session restart recovery sees them. const runs = location.workspaceID === undefined && process.platform !== "win32" && PersistentPty.available ? yield* daemon.at(location) @@ -576,6 +700,17 @@ const layer = () => return command.info }) + const byToolCall = Effect.fn("Shell.byToolCall")(function* (call: ToolCall) { + return runs + ?.list() + .find( + (run) => + run.toolCall?.sessionID === call.sessionID && + run.toolCall.assistantMessageID === call.assistantMessageID && + run.toolCall.callID === call.callID, + )?.info + }) + const readRange = (file: string, start: number, length: number) => length <= 0 ? Effect.succeed(Buffer.alloc(0)) @@ -777,6 +912,8 @@ const layer = () => Object.entries(invocation.env).filter((entry): entry is [string, string] => entry[1] !== undefined), ), timeout: invocation.timeout, + toolCall: input.toolCall, + waited: input.waited === true, }) .pipe( Effect.map(Option.some), @@ -952,7 +1089,7 @@ const layer = () => return command.info }) - return Service.of({ create, list, get, wait, result, timeout, stop, output, remove }) + return Service.of({ create, list, get, wait, result, timeout, stop, byToolCall, output, remove }) }), ) diff --git a/packages/core/src/tool/plugin/shell.ts b/packages/core/src/tool/plugin/shell.ts index 0c08343bb4e1..330e5c23eeff 100644 --- a/packages/core/src/tool/plugin/shell.ts +++ b/packages/core/src/tool/plugin/shell.ts @@ -14,6 +14,7 @@ import { Flag } from "../../flag.js" import { Permission } from "../../permission.js" import { NonNegativeInt } from "../../schema.js" import { Session } from "../../session.js" +import { SessionMessage } from "../../session/message.js" import { SessionSchema } from "../../session/schema.js" import { Shell } from "../../shell.js" import { ShellParse } from "../../shell/parse.js" @@ -119,6 +120,122 @@ const backgroundResult = (info: ShellResult.Result["info"], timeout?: number) => status: "running" as const, }) +// Tells the session how a background command ended, once its job settles. +const notify = Effect.fnUntraced(function* ( + sessions: Session.Interface, + jobs: Job.Interface, + input: { + readonly sessionID: SessionSchema.ID + readonly id: string + readonly shellID: string + readonly command: string + readonly settled: Deferred.Deferred + // Read at delivery; `false` admits the notice without waking the session. + readonly resume?: () => false | undefined + }, +) { + const info = (yield* jobs.wait({ id: input.id })).info + if (!info || info.status === "running") return + const output = info.status === "completed" ? yield* Deferred.await(input.settled) : undefined + const text = output ? (info.output ?? "") : info.status === "error" ? (info.error ?? "Command failed") : "Cancelled" + const resume = input.resume?.() + yield* sessions.synthetic({ + ...(info.notificationID ? { id: info.notificationID } : {}), + sessionID: input.sessionID, + description: input.command, + ...ShellResult.notification({ + jobID: input.id, + shellID: input.shellID, + command: input.command, + state: info.status, + text, + output, + }), + ...(resume === false ? { resume } : {}), + }) + if (info.notificationID) yield* jobs.completeBackground(info.notificationID) +}) + +// The job that waits on a command for its call (or its session, once backgrounded). +const startJob = Effect.fnUntraced(function* ( + jobs: Job.Interface, + shell: Shell.Interface, + input: { + readonly sessionID: SessionSchema.ID + readonly info: ShellSchema.Info + readonly notificationID?: SessionMessage.ID + readonly timeout?: () => number + }, +) { + const info = input.info + const settled = yield* Deferred.make() + const job = yield* jobs.start({ + // CodeMode children share a tool-call ID, but each shell must own its job. + id: info.id, + type: name, + title: info.command, + metadata: { sessionID: input.sessionID, shellID: info.id }, + recovery: { + kind: "shell", + sessionID: input.sessionID, + shellID: info.id, + command: info.command, + }, + ...(input.notificationID ? { notificationID: input.notificationID } : {}), + run: Effect.gen(function* () { + const result = yield* shell.result(info) + if (!result.capture) return yield* new Shell.NotFoundError({ id: info.id }) + return { ...ShellResult.output(result), status: "completed" as const } + }).pipe( + Effect.tap((output) => Deferred.succeed(settled, output)), + Effect.map((output) => resultMessages(output, input.timeout?.()).join("\n\n")), + ), + // Only cancelling the job stops the command, keeping it and its output file. A server shutdown interrupts + // the job without stopping it, so the command keeps running for the next server. + cancel: shell.stop(info.id, "killed").pipe(Effect.ignore, Effect.asVoid), + }) + return { job, settled } +}) + +/** The tool result of a call whose command moved to the background. */ +export const backgroundToolResult = (info: ShellSchema.Info) => toolResult(backgroundResult(info)) + +/** + * Watches a command taken over from a previous server as a background job of its session, and notifies the + * session when it finishes, as for a command the model ran in the background. A finished command's notice is + * admitted before this returns. + */ +export const reattach = Effect.fnUntraced(function* (input: { + readonly shell: Shell.Interface + readonly jobs: Job.Interface + readonly sessions: Session.Interface + readonly watch: Shell.DaemonInterface["watch"] + readonly sessionID: SessionSchema.ID + readonly info: ShellSchema.Info + // The job's existing durable row, if the previous server had backgrounded it. + readonly notificationID?: SessionMessage.ID + readonly resume?: () => false | undefined +}) { + const { job, settled } = yield* startJob(input.jobs, input.shell, input) + yield* input.jobs.background(job.id) + const notifying = notify(input.sessions, input.jobs, { + sessionID: input.sessionID, + id: job.id, + shellID: input.info.id, + command: input.info.command, + settled, + resume: input.resume, + }) + const current = yield* input.shell.get(input.info.id).pipe(Effect.option) + if (Option.isSome(current) && current.value.status !== "running") + return yield* notifying.pipe( + Effect.catch((error) => + Effect.logWarning("failed to report a recovered shell command", { id: input.info.id, error }), + ), + ) + yield* input.watch(notifying) +}) + export const Plugin = { id: "opencode.tool.shell", effect: Effect.fn("ShellTool.Plugin")(function* (ctx: Context) { @@ -163,41 +280,15 @@ export const Plugin = { return timeout }) - const notifyWhenDone = Effect.fn("ShellTool.notifyWhenDone")( - function* ( - sessionID: SessionSchema.ID, - id: string, - shellID: string, - command: string, - settled: Deferred.Deferred, - ) { - const info = (yield* jobs.wait({ id })).info - if (!info || info.status === "running") return - const output = info.status === "completed" ? yield* Deferred.await(settled) : undefined - const text = output - ? (info.output ?? "") - : info.status === "error" - ? (info.error ?? "Command failed") - : "Cancelled" - yield* sessions.synthetic({ - ...(info.notificationID ? { id: info.notificationID } : {}), - sessionID, - description: command, - ...ShellResult.notification({ - jobID: id, - shellID, - command, - state: info.status, - text, - output, - }), - }) - if (info.notificationID) yield* jobs.completeBackground(info.notificationID) - }, - // Not in this Location's scope: a daemon command outlives a Location that closes while the server runs, and - // its session must still hear how it ended. - (effect) => daemon.watch(effect), - ) + // Not in this Location's scope: a daemon command outlives a Location that closes while the server runs, and + // its session must still hear how it ended. + const notifyWhenDone = ( + sessionID: SessionSchema.ID, + id: string, + shellID: string, + command: string, + settled: Deferred.Deferred, + ) => daemon.watch(notify(sessions, jobs, { sessionID, id, shellID, command, settled })) // Stops the command of a call the user asked to stop, and fails the call with the output so far. The command // stays observable, output file included. @@ -252,6 +343,12 @@ export const Plugin = { timeout, shell: yield* compatibleShell, metadata: { sessionID: context.sessionID }, + waited: input.background !== true, + toolCall: { + sessionID: context.sessionID, + assistantMessageID: context.messageID, + callID: context.id, + }, }, (invocation) => Effect.gen(function* () { @@ -277,30 +374,10 @@ export const Plugin = { ) yield* context.progress({ shellID: info.id }) - const settled = yield* Deferred.make() - const run = Effect.gen(function* () { - const result = yield* shell.result(info) - if (!result.capture) return yield* new Shell.NotFoundError({ id: info.id }) - return { ...ShellResult.output(result), status: "completed" as const } - }).pipe( - Effect.tap((output) => Deferred.succeed(settled, output)), - Effect.map((output) => resultMessages(output, finalTimeout).join("\n\n")), - // Cancelling the job stops the command but keeps it, output file included. - Effect.onInterrupt(() => shell.stop(info.id, "killed").pipe(Effect.ignore)), - ) - const job = yield* jobs.start({ - // CodeMode children share a tool-call ID, but each shell must own its job. - id: info.id, - type: name, - title: info.command, - metadata: { sessionID: context.sessionID, shellID: info.id }, - recovery: { - kind: "shell", - sessionID: context.sessionID, - shellID: info.id, - command: info.command, - }, - run, + const { job, settled } = yield* startJob(jobs, shell, { + sessionID: context.sessionID, + info, + timeout: () => finalTimeout, }) // Stopped while the command was starting: stop it now. if (yield* Deferred.isDone(stopped)) return yield* cancelled(info, job.id) @@ -329,13 +406,14 @@ export const Plugin = { ), ) : waiting - // A user's stop arrives through `cancel`; an interrupt (a server shutdown) cancels the job as before. + // A user's stop arrives through `cancel`. An interrupt (a server shutdown) leaves the command and its + // job running: the next server takes the call over and moves the command to the background. const outcome = yield* restore( Effect.raceFirst( blocked.pipe(Effect.map((result) => ({ type: "settled" as const, result }))), Deferred.await(stopped).pipe(Effect.as({ type: "cancelled" as const })), ), - ).pipe(Effect.onInterrupt(() => jobs.cancel(job.id).pipe(Effect.ignore))) + ) if (outcome.type === "cancelled") return yield* cancelled(info, job.id) const result = outcome.result if (result?.type === "backgrounded") { diff --git a/packages/core/test/job.test.ts b/packages/core/test/job.test.ts index 044159bf471a..aea0cdcb99b0 100644 --- a/packages/core/test/job.test.ts +++ b/packages/core/test/job.test.ts @@ -457,6 +457,59 @@ describe("Job", () => { }), ) + it.live("interrupts observers without settling a background job, or running its cancel hook, on teardown", () => + Effect.gen(function* () { + const scope = yield* Scope.make() + const jobs = yield* Job.make.pipe(Scope.provide(scope)) + let cancels = 0 + const job = yield* jobs.start({ + id: "shell_teardown", + type: "shell", + recovery: { + kind: "shell", + sessionID: SessionSchema.ID.make("ses_teardown"), + shellID: "shell_teardown", + command: "sleep 60", + }, + run: Effect.never, + cancel: Effect.sync(() => void cancels++), + }) + const background = yield* jobs.background(job.id) + const observer = yield* jobs.wait({ id: job.id }).pipe(Effect.forkChild({ startImmediately: true })) + + yield* Scope.close(scope, Exit.void) + + const observed = yield* Fiber.await(observer).pipe(Effect.timeout("1 second")) + expect(Exit.isFailure(observed) && Cause.hasInterruptsOnly(observed.cause)).toBe(true) + expect(cancels).toBe(0) + const current = yield* Job.make + expect(yield* current.pendingBackground).toMatchObject([ + { id: job.id, notificationID: background?.notificationID, status: "running" }, + ]) + if (background?.notificationID) yield* current.completeBackground(background.notificationID) + }), + ) + + it.live("runs a job's cancel hook when it is cancelled, alone or with its session", () => + Effect.gen(function* () { + const jobs = yield* Job.Service + const cancelled: string[] = [] + const start = (id: string) => + jobs.start({ + id, + type: "shell", + recovery: { kind: "shell", sessionID: SessionSchema.ID.make("ses_cancel_hook"), shellID: id, command: "x" }, + run: Effect.never, + cancel: Effect.sync(() => void cancelled.push(id)), + }) + yield* start("shell_cancel_one") + yield* start("shell_cancel_session") + yield* jobs.cancel("shell_cancel_one") + yield* jobs.cancelSession(SessionSchema.ID.make("ses_cancel_hook")) + expect(cancelled).toEqual(["shell_cancel_one", "shell_cancel_session"]) + }), + ) + it.live("cancels running session jobs and removes unacknowledged SQLite job rows on cancelSession", () => Effect.gen(function* () { const jobs = yield* Job.Service diff --git a/packages/core/test/session-execution.test.ts b/packages/core/test/session-execution.test.ts index efc08062535d..06e392e898f1 100644 --- a/packages/core/test/session-execution.test.ts +++ b/packages/core/test/session-execution.test.ts @@ -22,6 +22,11 @@ import { SessionMessage } from "@opencode/core/session/message" import { SessionRunner } from "@opencode/core/session/runner/index" import { SessionInboxTable, SessionTable } from "@opencode/core/session/sql" import { SessionStore } from "@opencode/core/session/store" +import { Shell } from "@opencode/core/shell" +import { Agent } from "@opencode/core/agent" +import { ID } from "@opencode/core/model" +import { Provider } from "@opencode/core/provider" +import { daemonShells } from "./fixture/shell-daemon" import { Cause, Context, Deferred, Effect, Exit, Fiber, Layer, LayerMap, Scope } from "effect" import { eq } from "drizzle-orm" import { testEffect } from "./lib/effect" @@ -142,7 +147,7 @@ describe("SessionExecution lifecycle", () => { }), ) - it.effect("tells the drain once the user stopped it", () => + it.effect("tells the drain whether the user stopped it or the server is shutting down", () => Effect.gen(function* () { const database = yield* Database.Service const sessionID = Session.ID.make("ses_interruption_user") @@ -156,8 +161,8 @@ describe("SessionExecution lifecycle", () => { const execution = Context.get(context, SessionExecution.Service) yield* execution.resume(sessionID).pipe(Effect.forkScoped) const reason = yield* Deferred.await(draining) - // Teardown interrupts without a reason, so only a user's stop reads as one. - expect(reason?.()).toBeUndefined() + // Until someone stops it, an interrupt can only be the server shutting down. + expect(reason?.()).toBe("shutdown") yield* execution.interrupt(sessionID) expect(reason?.()).toBe("user") yield* execution.awaitIdle(sessionID) @@ -486,6 +491,276 @@ describe("SessionRestart background recovery", () => { }), ) + it.live("watches shell jobs whose commands survived the restart", () => + Effect.gen(function* () { + const database = yield* Database.Service + const store = yield* SessionStore.Service + const jobs = yield* Job.Service + const bus = yield* Bus.Service + const parent = Session.ID.make("ses_background_shell_survived") + yield* seedSessions(database, [parent]) + const daemon = yield* daemonShells() + const before = yield* daemon.withShell("/project", (shell) => + Effect.gen(function* () { + const running = yield* shell.create({ shell: "/bin/sh", command: "running", cwd: "/tmp" }) + const finished = yield* shell.create({ shell: "/bin/sh", command: "finished", cwd: "/tmp" }) + return { running, finished } + }), + ) + yield* seedBackground(jobs, parent, [ + { id: before.running.id, shellID: before.running.id, command: "running" }, + { id: before.finished.id, shellID: before.finished.id, command: "finished" }, + ]) + // The finished command ended while no server watched it. + daemon.daemon.output(2, "done\n") + daemon.daemon.exit(2, 0) + + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope)) + const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope)) + const context = yield* buildExecution( + scope, + ({ sessionID }) => SessionInbox.promote(database.db, bus, sessionID, "steer").pipe(Effect.asVoid), + undefined, + restarted, + shell, + ) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + + const notices = Effect.gen(function* () { + return (yield* store.context(parent)).filter( + (message) => message.type === "synthetic" && message.metadata?.source === "shell", + ) + }) + while ((yield* notices).length < 1) yield* Effect.sleep(5) + expect(yield* notices).toMatchObject([ + { + text: expect.stringContaining("done"), + metadata: { source: "shell", shellID: before.finished.id, state: "completed", exit: 0 }, + }, + ]) + expect((yield* restarted.get(before.running.id))?.status).toBe("running") + expect((yield* restarted.pendingBackground).map((item) => item.id)).toEqual([before.running.id]) + + daemon.daemon.exit(1, 2) + while ((yield* notices).length < 2) yield* Effect.sleep(5) + expect((yield* notices)[1]).toMatchObject({ + text: expect.stringContaining("Exit code: 2"), + metadata: { source: "shell", shellID: before.running.id, state: "completed", exit: 2 }, + }) + expect(yield* restarted.pendingBackground).toEqual([]) + }), + ) + + it.live("settles a shell call a killed server left running as moved to the background", () => + Effect.gen(function* () { + const database = yield* Database.Service + const store = yield* SessionStore.Service + const bus = yield* Bus.Service + const parent = Session.ID.make("ses_background_shell_killed") + yield* seedSessions(database, [parent], { time_suspended: Date.now() }) + const assistantMessageID = SessionMessage.ID.create() + yield* bus.publish(SessionEvent.Step.Started, { + sessionID: parent, + assistantMessageID, + agent: Agent.ID.make("build"), + model: { id: ID.make("fake-model"), providerID: Provider.ID.make("fake") }, + started: 0, + }) + yield* bus.publish(SessionEvent.Tool.Input.Started, { + sessionID: parent, + assistantMessageID, + id: "call-shell", + name: "shell", + }) + yield* bus.publish(SessionEvent.Tool.Called, { + sessionID: parent, + assistantMessageID, + id: "call-shell", + input: { command: "sleep 60" }, + executed: false, + }) + const daemon = yield* daemonShells() + const info = yield* daemon.withShell("/project", (shell) => + shell.create({ + shell: "/bin/sh", + command: "sleep 60", + cwd: "/tmp", + timeout: 120_000, + waited: true, + toolCall: { sessionID: parent, assistantMessageID, callID: "call-shell" }, + }), + ) + // A call waits on it, so its deadline is not kept: taking it over never times it out. + expect(daemon.daemon.processes[0]?.pipe?.metadata).not.toHaveProperty("timeoutDeadline") + + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope)) + const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope)) + const context = yield* buildExecution( + scope, + ({ sessionID }) => SessionInbox.promote(database.db, bus, sessionID, "steer").pipe(Effect.asVoid), + undefined, + restarted, + shell, + ) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + + const [assistant] = (yield* store.context(parent)).filter((message) => message.type === "assistant") + expect(assistant?.type === "assistant" ? assistant.content : []).toMatchObject([ + { + type: "tool", + id: "call-shell", + state: { + status: "completed", + content: [ + { + type: "text", + text: `Command moved to the background (shell ID: ${info.id}).\nPID: ${info.pid}\nOutput is streaming to: ${info.file}`, + }, + { type: "text", text: expect.stringContaining("You will be notified automatically") }, + ], + }, + }, + ]) + expect((yield* restarted.get(info.id))?.status).toBe("running") + expect(daemon.signals(1)).toEqual([]) + + daemon.daemon.exit(1, 0) + const notices = Effect.gen(function* () { + return (yield* store.context(parent)).filter( + (message) => message.type === "synthetic" && message.metadata?.source === "shell", + ) + }) + while ((yield* notices).length < 1) yield* Effect.sleep(5) + expect((yield* notices)[0]).toMatchObject({ metadata: { source: "shell", shellID: info.id, state: "completed" } }) + }), + ) + + for (const exited of [false, true]) + it.live( + `reattaches once a ${exited ? "finished" : "running"} shell a killed server backgrounded without settling its call`, + () => + Effect.gen(function* () { + const database = yield* Database.Service + const store = yield* SessionStore.Service + const bus = yield* Bus.Service + const jobs = yield* Job.Service + const parent = Session.ID.make(`ses_background_shell_unsettled_${exited}`) + yield* seedSessions(database, [parent], { time_suspended: Date.now() }) + const assistantMessageID = yield* seedShellCall(bus, parent, "call-shell") + const daemon = yield* daemonShells() + const info = yield* daemon.withShell("/project", (shell) => + shell.create({ + shell: "/bin/sh", + command: "work", + cwd: "/tmp", + waited: true, + toolCall: { sessionID: parent, assistantMessageID, callID: "call-shell" }, + }), + ) + // The old server persisted the background job, then died before the step recorded the call's result. + yield* seedBackground(jobs, parent, [{ id: info.id, shellID: info.id, command: "work" }]) + const [row] = yield* jobs.pendingBackground + if (exited) { + daemon.daemon.output(1, "finished output\n") + daemon.daemon.exit(1, 0) + } + + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope)) + const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope)) + const context = yield* buildExecution( + scope, + ({ sessionID }) => SessionInbox.promote(database.db, bus, sessionID, "steer").pipe(Effect.asVoid), + undefined, + restarted, + shell, + ) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + + expect(yield* shellCallState(store, parent)).toMatchObject({ status: "completed" }) + const notices = shellNotices(store, parent) + // A finished command's result is admitted during recovery, before any session resumes. + if (exited) + expect((yield* SessionInbox.list(database.db, parent)).length + (yield* notices).length).toBeGreaterThan(0) + if (!exited) { + expect((yield* restarted.pendingBackground).map((item) => item.notificationID)).toEqual([ + row?.notificationID, + ]) + daemon.daemon.exit(1, 0) + } + while ((yield* notices).length < 1) yield* Effect.sleep(5) + yield* Effect.sleep(50) + expect(yield* notices).toMatchObject([{ id: row?.notificationID, metadata: { shellID: info.id } }]) + if (exited) expect(yield* notices).toMatchObject([{ text: expect.stringContaining("finished output") }]) + expect(yield* restarted.pendingBackground).toEqual([]) + }), + ) + + it.live("settles a recovered child's unsettled shell call before the child resumes", () => + Effect.gen(function* () { + const database = yield* Database.Service + const store = yield* SessionStore.Service + const bus = yield* Bus.Service + const jobs = yield* Job.Service + const parent = Session.ID.make("ses_background_shell_child_parent") + const child = Session.ID.make("ses_background_shell_child") + yield* seedSessions(database, [parent]) + yield* seedSessions(database, [child], { parent_id: parent, time_suspended: Date.now() }) + yield* jobs.start({ + id: child, + type: "subagent", + recovery: { + kind: "subagent", + parentSessionID: parent, + childSessionID: child, + agent: "explore", + description: "Inspect recovery", + }, + run: Effect.never, + }) + yield* jobs.background(child) + const assistantMessageID = yield* seedShellCall(bus, child, "call-child-shell") + const daemon = yield* daemonShells() + const info = yield* daemon.withShell("/project", (shell) => + shell.create({ + shell: "/bin/sh", + command: "child work", + cwd: "/tmp", + waited: true, + toolCall: { sessionID: child, assistantMessageID, callID: "call-child-shell" }, + }), + ) + + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope)) + const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope)) + const seen = yield* Deferred.make() + const context = yield* buildExecution( + scope, + ({ sessionID }) => + sessionID === child + ? shellCallState(store, child).pipe(Effect.flatMap((state) => Deferred.succeed(seen, state))) + : Effect.void, + undefined, + restarted, + shell, + ) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + expect(yield* Deferred.await(seen).pipe(Effect.timeout("2 seconds"))).toMatchObject({ status: "completed" }) + expect((yield* restarted.get(info.id))?.status).toBe("running") + // Cancelling a reattached job stops its command. + yield* restarted.cancel(info.id) + while (daemon.signals(1).length === 0) yield* Effect.sleep(5) + expect(daemon.signals(1)).toEqual(["SIGTERM"]) + }), + ) + it.effect("preserves locally running background work", () => Effect.gen(function* () { const database = yield* Database.Service @@ -1400,6 +1675,7 @@ function buildExecution( drain: (input: Parameters[0]) => Effect.Effect, options?: SessionRestart.Options, overrideJobs?: Job.Interface, + shell?: Shell.Interface, ) { return Effect.gen(function* () { const database = yield* Database.Service @@ -1430,9 +1706,12 @@ function buildExecution( LocationServiceMap.Service, LayerMap.make( () => - // The local execution test only needs the Session runner from the Location graph. + // The local execution test only needs the Session runner (and the Shell, if given) from the Location graph. // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion - runner as unknown as Layer.Layer, + (shell ? Layer.merge(runner, Layer.succeed(Shell.Service, shell)) : runner) as unknown as Layer.Layer< + LocationServices, + FileSystem.DirectoryNotFoundError + >, ), ) return yield* Layer.buildWithScope( @@ -1443,6 +1722,15 @@ function buildExecution( Layer.provide(Layer.succeed(Bus.Service, bus)), Layer.provide(Layer.succeed(SessionStore.Service, store)), Layer.provide(Layer.succeed(Job.Service, jobs)), + Layer.provide( + Layer.succeed( + Shell.Daemon, + Shell.Daemon.of({ + at: () => Effect.die("the harness has no daemon registry"), + watch: (effect) => effect.pipe(Effect.ignore, Effect.forkIn(scope), Effect.asVoid), + }), + ), + ), // Do not reuse the outer harness's selector with its already-captured Location map. Layer.provide( AppNodeBuilder.build(Instance.node, [LocationServiceMap.node.replace(locations)]).pipe(Layer.fresh), @@ -1452,3 +1740,49 @@ function buildExecution( ) }) } + +/** Records a shell call the model made that is still running, as a killed server leaves it. */ +function seedShellCall(bus: Bus.Interface, sessionID: Session.ID, callID: string) { + return Effect.gen(function* () { + const assistantMessageID = SessionMessage.ID.create() + yield* bus.publish(SessionEvent.Step.Started, { + sessionID, + assistantMessageID, + agent: Agent.ID.make("build"), + model: { id: ID.make("fake-model"), providerID: Provider.ID.make("fake") }, + started: 0, + }) + yield* bus.publish(SessionEvent.Tool.Input.Started, { sessionID, assistantMessageID, id: callID, name: "shell" }) + yield* bus.publish(SessionEvent.Tool.Called, { + sessionID, + assistantMessageID, + id: callID, + input: { command: "work" }, + executed: false, + }) + return assistantMessageID + }) +} + +function shellCallState(store: SessionStore.Interface, sessionID: Session.ID) { + return store.context(sessionID).pipe( + Effect.map((messages) => + messages.flatMap((message) => + message.type === "assistant" + ? message.content.flatMap((part) => (part.type === "tool" ? [part.state] : [])) + : [], + ), + ), + Effect.map((states) => states[0]), + ) +} + +function shellNotices(store: SessionStore.Interface, sessionID: Session.ID) { + return store + .context(sessionID) + .pipe( + Effect.map((messages) => + messages.filter((message) => message.type === "synthetic" && message.metadata?.source === "shell"), + ), + ) +} diff --git a/packages/core/test/session-step.test.ts b/packages/core/test/session-step.test.ts index 18a98273861c..d288204e61a0 100644 --- a/packages/core/test/session-step.test.ts +++ b/packages/core/test/session-step.test.ts @@ -392,17 +392,17 @@ it.live("classifies what a tool raises while the user's stop settles it: a defec }), ) -it.live("interrupts every tool on a server shutdown", () => +it.live("leaves an interrupted step as a crash would when the server shuts down", () => Effect.gen(function* () { const { exit, data, requests } = yield* stopTwoTools({ reason: "shutdown", locationClosed: false }) expect(Exit.hasInterrupts(exit)).toBe(true) expect(requests).toBe(1) - const generic = { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } } + // No step end and no settled call: the next server finds the calls unsettled. + expect(data).not.toHaveProperty("error") expect(data).toMatchObject({ - error: { type: "aborted", message: "Step interrupted" }, content: [ - { type: "tool", id: "call-cooperative", state: generic }, - { type: "tool", id: "call-stuck", state: generic }, + { type: "tool", id: "call-cooperative", state: { status: "running" } }, + { type: "tool", id: "call-stuck", state: { status: "running" } }, ], }) }), diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts index 4db4594595f3..5ba6658e1fae 100644 --- a/packages/core/test/shell-daemon.test.ts +++ b/packages/core/test/shell-daemon.test.ts @@ -252,6 +252,73 @@ describe("Shell on the process daemon", () => { ) })) + test("takes its commands back after a restart and leaves other Locations' alone", () => + Effect.gen(function* () { + const env = yield* daemonShells() + env.daemon.trap.add(4) + const before = yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const running = yield* shell.create({ shell: "/bin/sh", command: "running", metadata: { tag: "a" } }) + const finished = yield* shell.create({ shell: "/bin/sh", command: "finished" }) + env.daemon.output(2, "done\n") + env.daemon.exit(2, 3) + const exited = yield* shell.wait(finished.id) + const expiring = yield* shell.create({ shell: "/bin/sh", command: "expiring", timeout: 60_000 }) + const stopping = yield* shell.create({ shell: "/bin/sh", command: "stopping" }) + yield* shell.stop(stopping.id, "killed") + yield* until(() => env.signals(4).length > 0) + // A call waits on this one: a restart moves it to the background, so its timeout is not kept. + const waited = yield* shell.create({ shell: "/bin/sh", command: "waited", timeout: 60_000, waited: true }) + return { running, exited, expiring, stopping, waited } + }), + ) + // Closing the Location only detached from its commands. + expect(env.daemon.ops).not.toContain("terminate") + expect(env.signals(1)).toEqual([]) + expect(env.signals(4)).toEqual(["SIGTERM"]) + // The expiring command's deadline passes while no server runs. + const expiring = env.daemon.processes.find((process) => process.id === 3)?.pipe + if (expiring) expiring.metadata = { ...expiring.metadata, timeoutDeadline: Date.now() - 1 } + + const other = (yield* tmpdirScoped()).path + yield* env.withShell(other, (shell) => + Effect.gen(function* () { + expect(yield* shell.get(before.running.id).pipe(Effect.flip)).toBeInstanceOf(Shell.NotFoundError) + expect(yield* shell.list()).toEqual([]) + }), + ) + + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + expect(yield* shell.get(before.running.id)).toMatchObject({ + status: "running", + command: "running", + shell: "/bin/sh", + file: before.running.file, + metadata: { tag: "a" }, + pid: before.running.pid, + }) + expect(yield* shell.get(before.exited.id)).toEqual({ ...before.exited, cwd: env.directory }) + expect((yield* shell.result(before.exited)).capture).toMatchObject({ output: "done\n" }) + // The deadline passed during the restart, and the interrupted stop goes straight to SIGKILL. + expect((yield* shell.wait(before.expiring.id)).status).toBe("timeout") + expect(env.signals(3)).toEqual(["SIGTERM"]) + expect((yield* shell.wait(before.stopping.id)).status).toBe("killed") + expect(env.signals(4)).toEqual(["SIGTERM", "SIGKILL"]) + + expect(env.daemon.processes.find((process) => process.id === 5)?.pipe?.metadata).not.toHaveProperty( + "timeoutDeadline", + ) + expect((yield* shell.get(before.waited.id)).status).toBe("running") + expect(env.signals(5)).toEqual([]) + + env.daemon.exit(1, 0) + expect(yield* shell.wait(before.running.id)).toMatchObject({ status: "exited", exit: 0 }) + }), + ) + })) + + // A reload or an eviction closes a Location while the server keeps running. timed("a Location closed while the server runs keeps enforcing its commands' timeouts", () => Effect.gen(function* () { const env = yield* daemonShells() diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index 3cc8354675cf..5b330879ea43 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -2313,6 +2313,25 @@ describe("ShellTool stopped", () => { { timeout: 15_000 }, ) + fakeDaemonIt.live("records which call waits on a daemon command, and keeps no timeout for a restart", () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const { shellID } = yield* stopCall(registry, "sleep 60", "call-recorded", "interrupt", false) + const request = fake?.requests.find((item) => item.op === "create_pipe") + expect(request?.metadata).toMatchObject({ + id: shellID, + toolCall: { sessionID, callID: "call-recorded" }, + }) + // The call's timeout applies to this server only: a restart moves the command to the background. + expect(request?.metadata).not.toHaveProperty("timeoutDeadline") + expect(fake?.ops).not.toContain("signal") + }), + ) + }), + ) + test("lets an execute.after hook replace a stopped call's failure", () => withTempDir((tmp) => { reset() @@ -2341,18 +2360,21 @@ describe("ShellTool stopped", () => { ) })) - test("leaves a call a shutdown interrupted to the step", () => + test("leaves a call a shutdown interrupted, and its command, for the next server", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => Effect.gen(function* () { const shell = yield* Shell.Service + const jobs = yield* Job.Service const { shellID, exit } = yield* stopCall(registry, "printf ready; sleep 60", "call-teardown", "interrupt") expect(Exit.hasInterrupts(exit)).toBe(true) - // The job is cancelled as before, which stops the command; its file stays. - const info = yield* shell.wait(shellID) - expect(info.status).toBe("killed") - expect(yield* Effect.promise(() => fs.readFile(info.file, "utf8"))).toBe("ready") + // Nothing stops the command or ends its job: the next server takes the call over. + expect((yield* shell.get(shellID)).status).toBe("running") + expect((yield* jobs.get(shellID))?.status).toBe("running") + // Only an explicit cancel stops it. + yield* jobs.cancel(shellID) + expect((yield* shell.wait(shellID)).status).toBe("killed") }), ) })) From 05df583adc7aa47f5ec10cd68b882a8b85276d3f Mon Sep 17 00:00:00 2001 From: James Long Date: Tue, 10 Nov 2026 17:24:26 +0000 Subject: [PATCH 03/12] fix(cli): let service restart and stop end the process daemon through owner loss `opencode service restart` and `stop` asked the server to shut its process daemon down before stopping it. That call predates handoff (#45551): a stop without a handoff now drops the daemon's owner, which stops the daemon and its processes anyway, and the next server resumes the sessions and settles their calls. `uninstall` still shuts the daemon down explicitly. User decisions: - Remove the up-front daemon shutdown from service restart and stop; keep it in uninstall --- packages/cli/src/commands/handlers/service/restart.ts | 4 +--- packages/cli/src/commands/handlers/service/stop.ts | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/commands/handlers/service/restart.ts b/packages/cli/src/commands/handlers/service/restart.ts index afc7bb457c17..fbd2c3b98269 100644 --- a/packages/cli/src/commands/handlers/service/restart.ts +++ b/packages/cli/src/commands/handlers/service/restart.ts @@ -4,14 +4,12 @@ import { Service } from "@opencode/client/effect/service" import { Commands } from "../../commands" import { Runtime } from "../../../framework/runtime" import { ServiceConfig } from "../../../services/service-config" -import { ServerConnection } from "../../../services/server-connection" export default Runtime.handler( Commands.commands.service.commands.restart, Effect.fn("cli.service.restart")(function* () { const options = yield* ServiceConfig.options() - // Keep this explicit: automatic service replacement must preserve terminals. - yield* ServerConnection.shutdownPersistentPty(options).pipe(Effect.ignore) + // No explicit daemon shutdown: stopping without a handoff drops the daemon's owner, which stops it with its processes. yield* Service.stop(options) const transport = yield* Service.ensure(options) process.stdout.write(transport.url + EOL) diff --git a/packages/cli/src/commands/handlers/service/stop.ts b/packages/cli/src/commands/handlers/service/stop.ts index d29f33275b44..c0dd64ad4187 100644 --- a/packages/cli/src/commands/handlers/service/stop.ts +++ b/packages/cli/src/commands/handlers/service/stop.ts @@ -3,13 +3,12 @@ import { Service } from "@opencode/client/effect/service" import { Commands } from "../../commands" import { Runtime } from "../../../framework/runtime" import { ServiceConfig } from "../../../services/service-config" -import { ServerConnection } from "../../../services/server-connection" export default Runtime.handler( Commands.commands.service.commands.stop, Effect.fn("cli.service.stop")(function* () { const options = yield* ServiceConfig.options() - yield* ServerConnection.shutdownPersistentPty(options).pipe(Effect.ignore) + // No explicit daemon shutdown: stopping without a handoff drops the daemon's owner, which stops it with its processes. yield* Service.stop(options) }), ) From b311ebb8d3a91fe5f3becd15706535afc7b4b0a0 Mon Sep 17 00:00:00 2001 From: James Long Date: Tue, 10 Nov 2026 17:26:33 +0000 Subject: [PATCH 04/12] fix(core): harden daemon shell recovery, notices and result reads - A Location's command recovery runs uninterruptibly, so a caller's interrupt cannot leave the shared entry half recovered. - A background command's notice for a deleted session is dropped instead of logged as a watcher failure. - Reading a result no longer streams a whole large file to number lines the daemon has not counted (over 16 MiB the line numbers are left out), and looks back at most 1 MiB for the start of a cut-off line. User decisions: - Make Location recovery uninterruptible - Ignore a deleted session when notifying about a background command - Bound the reads a result needs, so a stop stays quick --- packages/core/src/shell.ts | 40 ++++++++++++++++++------- packages/core/src/tool/plugin/shell.ts | 31 ++++++++++--------- packages/core/test/shell-daemon.test.ts | 29 ++++++++++++++++++ 3 files changed, 75 insertions(+), 25 deletions(-) diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index 72fa2e93ad76..037a37db08e0 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -49,6 +49,9 @@ export const DIRECTORY = "shell" const SPAN_COMMAND_LIMIT = 200 const STOP_GRACE = Duration.seconds(3) const LOST = "Command was killed: the process daemon stopped" +// Bounds the reads a result needs when the daemon did not report the line count, so stopping a call stays quick. +const COUNT_NEWLINES_LIMIT = 16 * 1024 * 1024 +const LINE_START_SCAN_LIMIT = 1024 * 1024 type Info = Shell.Info export type StopReason = "timeout" | "killed" @@ -572,7 +575,10 @@ const daemonLayer = Layer.effect( const placed = place(location) const entry = { runs: placed.runs, recovered: Deferred.makeUnsafe() } places.set(key, entry) - yield* placed.recover().pipe(Effect.ensuring(Deferred.succeed(entry.recovered, undefined))) + // Uninterruptible: the entry is shared, so a caller's interrupt must not leave it half recovered. + yield* placed + .recover() + .pipe(Effect.ensuring(Deferred.succeed(entry.recovered, undefined)), Effect.uninterruptible) return placed.runs }) @@ -758,12 +764,13 @@ const layer = () => }) // Only for a command whose exit report is missing (still running, or lost with the daemon); the daemon counts - // newlines otherwise. + // newlines otherwise. Gives up on large files, whose line numbers then go unreported, to keep a result quick. const countNewlines = (file: string, size: number) => Effect.promise( () => - new Promise((resolve) => { + new Promise((resolve) => { if (size === 0) return resolve(0) + if (size > COUNT_NEWLINES_LIMIT) return resolve(undefined) let count = 0 const stream = createReadStream(file, { end: size - 1 }) stream.on("data", (chunk: string | Buffer) => { @@ -776,18 +783,21 @@ const layer = () => ) // Where the last two lines of a daemon command's output start, found by reading back from the end. Only needed - // to size a single cut-off line, which the spawner tracks as it writes. + // to size a single cut-off line, which the spawner tracks as it writes. A start further back than the scan + // limit is reported as unknown. const lineStartsOf = Effect.fnUntraced(function* (file: string, size: number) { const starts: number[] = [] let end = size - while (end > 0 && starts.length < 2) { - const from = Math.max(0, end - 65536) + const floor = Math.max(0, size - LINE_START_SCAN_LIMIT) + while (end > floor && starts.length < 2) { + const from = Math.max(floor, end - 65536) const chunk = yield* readRange(file, from, end - from) for (let index = chunk.length - 1; index >= 0 && starts.length < 2; index--) if (chunk[index] === 10) starts.push(from + index + 1) end = from } - return [starts[1] ?? 0, starts[0] ?? 0] as [number, number] + const unknown = floor > 0 ? undefined : 0 + return [starts[1] ?? unknown, starts[0] ?? unknown] as [number | undefined, number | undefined] }) const result = Effect.fn("Shell.result")(function* ( @@ -828,7 +838,7 @@ const layer = () => command.kind === "spawned" ? command.newlines : (command.outputLines ?? (yield* countNewlines(info.file, size))) - const total = newlines + (endsWithNewline ? 0 : 1) + const total = newlines === undefined ? undefined : newlines + (endsWithNewline ? 0 : 1) const suffix = `full output saved to ${info.file}]` // After a write error the daemon kept running the command, but its file stopped early. const stoppedEarly = @@ -839,10 +849,15 @@ const layer = () => if (partial && lines.length === 1) { const lineStarts = command.kind === "spawned" ? command.lineStarts : yield* lineStartsOf(info.file, size) const lineStart = endsWithNewline ? lineStarts[0] : lineStarts[1] - const lineSize = size - (endsWithNewline ? 1 : 0) - lineStart + const end = size - (endsWithNewline ? 1 : 0) + const lineSize = + lineStart === undefined + ? `over ${formatSize(Math.min(end, LINE_START_SCAN_LIMIT))}` + : formatSize(end - lineStart) const shownSize = Buffer.byteLength(lines[0]) + const line = total === undefined ? "the last line" : `line ${total}` return { - output: `${lines[0]}\n\n[showing the last ${formatSize(shownSize)} of line ${total} (${formatSize(lineSize)}); ${suffix}${stoppedEarly}`, + output: `${lines[0]}\n\n[showing the last ${formatSize(shownSize)} of ${line} (${lineSize}); ${suffix}${stoppedEarly}`, truncated: true, ...killed, } @@ -851,7 +866,10 @@ const layer = () => // A window that starts mid-line drops that partial line so the line numbers stay exact. const kept = lines.slice(-Math.min(maxLines, lines.length - (partial ? 1 : 0))) return { - output: `${kept.join("\n")}\n\n[showing lines ${total - kept.length + 1}-${total} of ${total}; ${suffix}${stoppedEarly}`, + output: + total === undefined + ? `${kept.join("\n")}\n\n[showing the last ${kept.length} lines; ${suffix}${stoppedEarly}` + : `${kept.join("\n")}\n\n[showing lines ${total - kept.length + 1}-${total} of ${total}; ${suffix}${stoppedEarly}`, truncated: true, ...killed, } diff --git a/packages/core/src/tool/plugin/shell.ts b/packages/core/src/tool/plugin/shell.ts index 330e5c23eeff..4adda793be92 100644 --- a/packages/core/src/tool/plugin/shell.ts +++ b/packages/core/src/tool/plugin/shell.ts @@ -139,20 +139,23 @@ const notify = Effect.fnUntraced(function* ( const output = info.status === "completed" ? yield* Deferred.await(input.settled) : undefined const text = output ? (info.output ?? "") : info.status === "error" ? (info.error ?? "Command failed") : "Cancelled" const resume = input.resume?.() - yield* sessions.synthetic({ - ...(info.notificationID ? { id: info.notificationID } : {}), - sessionID: input.sessionID, - description: input.command, - ...ShellResult.notification({ - jobID: input.id, - shellID: input.shellID, - command: input.command, - state: info.status, - text, - output, - }), - ...(resume === false ? { resume } : {}), - }) + // A session deleted while its command ran has no one left to tell. + yield* sessions + .synthetic({ + ...(info.notificationID ? { id: info.notificationID } : {}), + sessionID: input.sessionID, + description: input.command, + ...ShellResult.notification({ + jobID: input.id, + shellID: input.shellID, + command: input.command, + state: info.status, + text, + output, + }), + ...(resume === false ? { resume } : {}), + }) + .pipe(Effect.catchTag("Session.NotFoundError", () => Effect.void)) if (info.notificationID) yield* jobs.completeBackground(info.notificationID) }) diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts index 5ba6658e1fae..2736f8c2437c 100644 --- a/packages/core/test/shell-daemon.test.ts +++ b/packages/core/test/shell-daemon.test.ts @@ -140,6 +140,35 @@ describe("Shell on the process daemon", () => { ) })) + test("bounds what it reads to number lines the daemon has not counted yet", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + // Still running, so the daemon has not reported a line count, and the file is too big to count quickly. + const big = yield* shell.create({ shell: "/bin/sh", command: "big" }) + env.daemon.output(1, `${"y".repeat(17 * 1024 * 1024)}\n${lines(10)}`) + expect((yield* shell.result(big, { wait: 0 })).capture?.output).toEndWith( + `line 10\n\n[showing the last 10 lines; full output saved to ${big.file}]`, + ) + + // A cut-off line that started too far back to look for. + const wide = yield* shell.create({ shell: "/bin/sh", command: "wide" }) + env.daemon.output(2, `short\n${"x".repeat(2 * 1024 * 1024)}`) + expect((yield* shell.result(wide, { wait: 0 })).capture?.output).toEndWith( + `\n\n[showing the last 50 KB of line 2 (over 1024 KB); full output saved to ${wide.file}]`, + ) + + // The same, ending with a newline: the line's start is still too far back to know. + const ended = yield* shell.create({ shell: "/bin/sh", command: "ended" }) + env.daemon.output(3, `short\n${"x".repeat(2 * 1024 * 1024)}\n`) + expect((yield* shell.result(ended, { wait: 0 })).capture?.output).toEndWith( + `\n\n[showing the last 50 KB of line 2 (over 1024 KB); full output saved to ${ended.file}]`, + ) + }), + ) + })) + test("reports a command lost with the daemon as killed, with what it wrote", () => Effect.gen(function* () { const env = yield* daemonShells() From 1b84513b6b52570a200eec61f332e43e326347e7 Mon Sep 17 00:00:00 2001 From: James Long Date: Tue, 10 Nov 2026 17:29:27 +0000 Subject: [PATCH 05/12] refactor(core): simplify daemon shell plumbing - `watch` is on Shell.Interface, so the shell tool and restart recovery no longer reach for Shell.Daemon. - A daemon command's restart deadline is set when it is created (none while a call waits on it), replacing the `waited` flag. - The retention sweep no longer forgets a finished command before deleting its file; the missing-file pass right after does that. - A job torn down with its registry relies on its scope's finalizer to interrupt its observers. User decisions: - Put watch on Shell.Interface and keep Shell.Daemon internal - Drop run.waited and compute the restart deadline at create - Drop FileRetention.beforeRemove and the redundant teardown interrupt --- packages/core/src/file-retention.ts | 3 -- packages/core/src/job.ts | 7 ++-- packages/core/src/plugin/internal.ts | 1 - .../core/src/session/execution/restart.ts | 2 -- packages/core/src/shell.ts | 35 +++++++++++-------- packages/core/src/tool/plugin/shell.ts | 6 ++-- 6 files changed, 24 insertions(+), 30 deletions(-) diff --git a/packages/core/src/file-retention.ts b/packages/core/src/file-retention.ts index ee9f486e219c..a325f2165fc9 100644 --- a/packages/core/src/file-retention.ts +++ b/packages/core/src/file-retention.ts @@ -7,8 +7,6 @@ export const cleanup = Effect.fn("FileRetention.cleanup")(function* ( fs: FSUtil.Interface, files: ReadonlyArray, retention: Duration.Input, - // Runs before an expired file is removed. - beforeRemove?: (file: string) => Effect.Effect, ) { const cutoff = Date.now() - Duration.toMillis(retention) yield* Effect.forEach( @@ -18,7 +16,6 @@ export const cleanup = Effect.fn("FileRetention.cleanup")(function* ( const info = yield* fs.stat(file).pipe(Effect.orElseSucceed(() => undefined)) const mtime = info && Option.getOrUndefined(info.mtime) if (!mtime || mtime.getTime() >= cutoff) return - if (beforeRemove) yield* beforeRemove(file) yield* fs.remove(file).pipe(Effect.ignore) }), { concurrency: 8, discard: true }, diff --git a/packages/core/src/job.ts b/packages/core/src/job.ts index 86605e0de8a3..ec26953c7e68 100644 --- a/packages/core/src/job.ts +++ b/packages/core/src/job.ts @@ -60,8 +60,6 @@ type State = { type FinishResult = { info?: Info done?: Deferred.Deferred - // Teardown interrupted the work: `done` is interrupted instead of settled. - interrupted?: boolean cancel?: Effect.Effect scope?: Scope.Closeable generation?: Scope.Closeable @@ -256,10 +254,10 @@ export const make = Effect.gen(function* () { if (job.scope !== scope) return [{}, jobs] if (job.info.status !== "running") return [{ info: snapshot(job) }, jobs] // Teardown (this registry closing) is not a terminal outcome: leave the job and its durable row running for - // the next server, and interrupt rather than settle its observers. A cancel has already settled the job + // the next server. The scope's own finalizer interrupts its observers. A cancel has already settled the job // as cancelled before interrupting its work. if (Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause) && scope.state._tag === "Closed") - return [{ done: job.done, scope: job.scope, interrupted: true }, jobs] + return [{}, jobs] const status: Exclude = Exit.isSuccess(exit) ? "completed" : Cause.hasInterruptsOnly(exit.cause) @@ -280,7 +278,6 @@ export const make = Effect.gen(function* () { return [{ info: snapshot(next), done: job.done, scope: job.scope }, new Map(jobs).set(id, next)] }), ) - if (result.interrupted && result.done) yield* Deferred.interrupt(result.done) if (result.info && result.done) yield* Deferred.succeed(result.done, result.info) if (result.scope) { yield* Scope.close(result.scope, Exit.void).pipe(Effect.forkIn(state.scope, { startImmediately: true })) diff --git a/packages/core/src/plugin/internal.ts b/packages/core/src/plugin/internal.ts index 2d714ab172f2..91ba0645a412 100644 --- a/packages/core/src/plugin/internal.ts +++ b/packages/core/src/plugin/internal.ts @@ -166,7 +166,6 @@ const services = [ SubagentJob.Service, SharedCache.Service, Shell.Service, - Shell.Daemon, ShellSelect.Service, Snapshot.Service, Skill.Service, diff --git a/packages/core/src/session/execution/restart.ts b/packages/core/src/session/execution/restart.ts index 8db6e1e5bb54..6f0cb37be4a3 100644 --- a/packages/core/src/session/execution/restart.ts +++ b/packages/core/src/session/execution/restart.ts @@ -82,7 +82,6 @@ export const layer = (options?: Options) => const sessions = yield* Session.Service const instances = yield* Instance.Service const db = (yield* Database.Service).db - const daemon = yield* Shell.Daemon const scope = yield* Effect.scope const maxAttempts = options?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS @@ -195,7 +194,6 @@ export const layer = (options?: Options) => shell, jobs, sessions, - watch: daemon.watch, sessionID, info, notificationID, diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index 037a37db08e0..80fa0b953aa8 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -109,8 +109,7 @@ type Run = { info: Info process: number toolCall?: ToolCall - // A call waits on it, until it moves to the background. - waited: boolean + // When a restart should stop it. Unset while a call waits on it: that call's timeout dies with this server. timeoutDeadline?: number stopReason?: StopReason // From the exit: newlines in the file, and why the daemon stopped writing it early. @@ -173,6 +172,8 @@ export interface Interface { readonly byToolCall: (call: ToolCall) => Effect.Effect readonly output: (id: Shell.ID, input?: Shell.OutputInput) => Effect.Effect readonly remove: (id: Shell.ID) => Effect.Effect + /** Runs `effect` until it ends or the server stops, outliving this Location (e.g. a job's completion notice). */ + readonly watch: DaemonInterface["watch"] } export class Service extends Context.Service()("@opencode/Shell") {} @@ -201,7 +202,8 @@ export const cleanup = Effect.fn("Shell.cleanup")(function* () { ), { concurrency: 8 }, ) - // Never starts the daemon. A command it still runs keeps its file; a finished one is forgotten before deletion. + // Never starts the daemon. A command it still runs keeps its file; a finished one whose file expired is forgotten + // below. const processes = new Map( (yield* pty.listPipes().pipe(Effect.orElseSucceed(() => []))).map((process) => [process.outputPath, process]), ) @@ -209,10 +211,6 @@ export const cleanup = Effect.fn("Shell.cleanup")(function* () { fs, files.flat().filter((file) => processes.get(file)?.lifecycle.status !== "running"), RETENTION, - (file) => { - const process = processes.get(file) - return process ? pty.terminatePipe(process.id).pipe(Effect.ignore) : Effect.void - }, ) // Finished processes whose file is gone, or that ended long ago, are of no use to any Location: forget them. const cutoff = Date.now() - Duration.toMillis(RETENTION) @@ -300,7 +298,7 @@ const daemonLayer = Layer.effect( }, metadata: run.info.metadata, toolCall: run.toolCall, - timeoutDeadline: run.waited ? undefined : run.timeoutDeadline, + timeoutDeadline: run.timeoutDeadline, stopReason: run.stopReason, }) @@ -396,8 +394,6 @@ const daemonLayer = Layer.effect( if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber) run.timeoutFiber = undefined run.timeoutDeadline = duration === 0 ? undefined : Date.now() + duration - // Clearing the timeout is how a call moves its command to the background. - if (duration === 0) run.waited = false yield* store(run) if (duration !== 0) yield* arm(run, duration) }) @@ -454,8 +450,7 @@ const daemonLayer = Layer.effect( info: input.info, process: 0, toolCall: input.toolCall, - waited: input.waited, - timeoutDeadline: input.timeout > 0 ? input.info.time.started + input.timeout : undefined, + timeoutDeadline: !input.waited && input.timeout > 0 ? input.info.time.started + input.timeout : undefined, done: Deferred.makeUnsafe(), } const created = yield* pty.createPipe({ @@ -526,8 +521,6 @@ const daemonLayer = Layer.effect( }, process: process.id, toolCall: stored.toolCall, - // No call waits on it any more: whatever waited died with the previous server. - waited: false, timeoutDeadline: stored.timeoutDeadline, stopReason: stored.stopReason, ...(running ? {} : { outputLines: process.outputLines, outputError: process.outputError }), @@ -1107,7 +1100,19 @@ const layer = () => return command.info }) - return Service.of({ create, list, get, wait, result, timeout, stop, byToolCall, output, remove }) + return Service.of({ + create, + list, + get, + wait, + result, + timeout, + stop, + byToolCall, + output, + remove, + watch: daemon.watch, + }) }), ) diff --git a/packages/core/src/tool/plugin/shell.ts b/packages/core/src/tool/plugin/shell.ts index 4adda793be92..1a5271f3e975 100644 --- a/packages/core/src/tool/plugin/shell.ts +++ b/packages/core/src/tool/plugin/shell.ts @@ -212,7 +212,6 @@ export const reattach = Effect.fnUntraced(function* (input: { readonly shell: Shell.Interface readonly jobs: Job.Interface readonly sessions: Session.Interface - readonly watch: Shell.DaemonInterface["watch"] readonly sessionID: SessionSchema.ID readonly info: ShellSchema.Info // The job's existing durable row, if the previous server had backgrounded it. @@ -236,7 +235,7 @@ export const reattach = Effect.fnUntraced(function* (input: { Effect.logWarning("failed to report a recovered shell command", { id: input.info.id, error }), ), ) - yield* input.watch(notifying) + yield* input.shell.watch(notifying) }) export const Plugin = { @@ -247,7 +246,6 @@ export const Plugin = { const environment = yield* Environment.Service const access = yield* FileAccess.Service const shell = yield* Shell.Service - const daemon = yield* Shell.Daemon const shellSelect = yield* ShellSelect.Service const compatibleShell = shellSelect.resolve({ priority: "compat" }) const permission = yield* Permission.Service @@ -291,7 +289,7 @@ export const Plugin = { shellID: string, command: string, settled: Deferred.Deferred, - ) => daemon.watch(notify(sessions, jobs, { sessionID, id, shellID, command, settled })) + ) => shell.watch(notify(sessions, jobs, { sessionID, id, shellID, command, settled })) // Stops the command of a call the user asked to stop, and fails the call with the output so far. The command // stays observable, output file included. From a8a3c5b9e4a9c8dc04a1a78f69b9d8a7407b0127 Mon Sep 17 00:00:00 2001 From: James Long Date: Tue, 10 Nov 2026 17:32:56 +0000 Subject: [PATCH 06/12] test(core): carry a foreground shell call across a graceful restart end to end Server 1 runs a turn that waits on a daemon command and is torn down mid call; server 2, on the same daemon and database, settles the call as moved to the background, resumes the turn and reports how the command ended, both for a command still running and for one that finished during the restart. User decisions: - One integrated graceful-restart test, and one for a foreground command that finished during the restart --- packages/core/test/shell-restart.test.ts | 183 +++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 packages/core/test/shell-restart.test.ts diff --git a/packages/core/test/shell-restart.test.ts b/packages/core/test/shell-restart.test.ts new file mode 100644 index 000000000000..c93b837b81aa --- /dev/null +++ b/packages/core/test/shell-restart.test.ts @@ -0,0 +1,183 @@ +// A foreground shell call across a graceful restart, through the whole app: server 1 runs the turn and is torn down +// mid-call, server 2 recovers the command from the same process daemon and resumes the session. +import { describe, expect } from "bun:test" +import path from "path" +import { Context, Effect, Exit, Layer, Scope } from "effect" +import { LanguageModel, LLMClient } from "@opencode/ai/effect" +import { OpenAIChat } from "@opencode/ai/protocols/openai-chat" +import { TestLLM } from "@opencode/ai/testing" +import { Agent } from "@opencode/core/agent" +import { Bus } from "@opencode/core/bus" +import { Database } from "@opencode/core/database/database" +import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" +import { LayerNodePlatform } from "@opencode/core/effect/app-node-platform" +import { Watcher } from "@opencode/core/filesystem/watcher" +import { LocationServiceMap } from "@opencode/core/location-service-map" +import { Model } from "@opencode/core/model" +import { Permission } from "@opencode/core/permission" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { Plugin } from "@opencode/core/plugin" +import { Provider } from "@opencode/core/provider" +import { AbsolutePath } from "@opencode/core/schema" +import { Session } from "@opencode/core/session" +import { SessionRestart } from "@opencode/core/session/execution/restart" +import { SessionRunnerModel } from "@opencode/core/session/runner/model" +import { makeGlobalNode } from "@opencode/util/effect/app-node" +import { LayerNode } from "@opencode/util/effect/layer-node" +import { Global } from "@opencode/util/global" +import { fakeDaemon } from "./fixture/persistent-pty-daemon" +import { tempGlobalLayer } from "./fixture/global" +import { offlineModels } from "./fixture/models" +import { tmpdirScoped } from "./fixture/tmpdir" +import { testEffect } from "./lib/effect" +import { permissionLayer } from "./lib/permission" + +const it = testEffect(tempGlobalLayer) + +const model = SessionRunnerModel.node.replace( + Layer.succeed(SessionRunnerModel.Service, { + resolve: () => + Effect.succeed( + SessionRunnerModel.resolved(LanguageModel.make({ id: "restart", provider: "test", route: OpenAIChat.route }), { + capabilities: { tools: true, input: ["text"], output: ["text"] }, + cost: [], + limit: { context: 200_000, output: 32_000 }, + }), + ), + }), +) + +// Everything both servers share: the process daemon, the data directory, the database and the model. +const world = Effect.gen(function* () { + const daemon = yield* fakeDaemon() + const root = (yield* tmpdirScoped()).path + const data = path.join(root, "data") + const global = Global.layerWith({ + data, + cache: path.join(root, "cache"), + config: path.join(root, "config"), + state: path.join(root, "state"), + tmp: path.join(root, "tmp"), + bin: path.join(root, "cache", "bin"), + log: path.join(data, "log"), + repos: path.join(data, "repos"), + }) + const database = yield* Layer.build(Database.layer()) + const llm = yield* Layer.build(TestLLM.testLayer({ fallback: TestLLM.text("Done", "done") })) + const project = (yield* tmpdirScoped()).path + // A server: the whole app on the shared pieces, torn down when its scope closes. + const server = (scope: Scope.Scope) => + Layer.buildWithScope( + AppNodeBuilder.build(LayerNode.group([Session.node, LocationServiceMap.node, SessionRestart.node]), [ + Global.node.replace(global), + Database.node.replace(Layer.succeedContext(database)), + LayerNodePlatform.llmClient.replace(Layer.succeedContext(Context.pick(LLMClient.Service)(llm))), + PersistentPty.node.replace( + makeGlobalNode({ + service: PersistentPty.Service, + layer: PersistentPty.layerWith(daemon.options), + deps: [Bus.node, Global.node], + }), + ), + Permission.node.replace(permissionLayer({ assert: () => Effect.void })), + Watcher.node.replace(Watcher.configured({ enabled: false })), + offlineModels, + model, + ]), + scope, + ) + return { daemon, llm: Context.get(llm, TestLLM.Test), project, server } +}) + +const shellCall = (sessions: Session.Interface, sessionID: Session.ID) => + sessions + .context(sessionID) + .pipe( + Effect.map((messages) => + messages.flatMap((message) => + message.type === "assistant" ? message.content.filter((part) => part.type === "tool") : [], + ), + ), + ) + +const until = (check: () => boolean) => + Effect.gen(function* () { + while (!check()) yield* Effect.sleep(5) + }) + +describe("shell commands across a graceful restart", () => { + for (const finished of [false, true]) + it.live( + `settles a foreground call ${finished ? "whose command finished during the restart" : "whose command still runs"} and resumes its session`, + () => + Effect.gen(function* () { + const { daemon, llm, project, server } = yield* world + // The turn's first request runs the command; anything else (a title, the resumed turn) just answers. + let turns = 0 + yield* llm.serve((request) => + request.tools.some((tool) => tool.name === "shell") && turns++ === 0 + ? TestLLM.toolCall("call-restart", "shell", { command: "sleep 60" }) + : TestLLM.text("Done", "done"), + ) + + // Server 1 starts the turn, which waits on the command, then shuts down. + const first = yield* Scope.make() + const one = yield* server(first) + const sessions = Context.get(one, Session.Service) + const session = yield* sessions.create({ + location: { directory: AbsolutePath.make(project) }, + agent: Agent.ID.make("build"), + model: Model.Ref.make({ id: Model.ID.make("restart"), providerID: Provider.ID.make("test") }), + }) + yield* Plugin.Service.use((plugins) => plugins.awaitActivation).pipe( + Effect.provide(Context.get(one, LocationServiceMap.Service).get(session.location)), + ) + yield* sessions.prompt({ sessionID: session.id, text: "run it" }) + yield* until(() => daemon.processes.length === 1 && daemon.waiters(daemon.processes[0]!.id) > 0) + const command = daemon.processes[0]! + yield* Scope.close(first, Exit.void) + + // The shutdown stopped nothing and settled nothing. + expect(command.lifecycle.status).toBe("running") + expect(daemon.ops).not.toContain("signal") + expect(turns).toBe(1) + if (finished) { + daemon.output(command.id, "finished output\n") + daemon.exit(command.id, 0) + } + + // Server 2 takes the command back, settles the call and resumes the turn. + const second = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(second, Exit.void)) + const two = yield* server(second) + const resumed = Context.get(two, Session.Service) + yield* Context.get(two, SessionRestart.Service).resumeSuspendedSessions + yield* until(() => turns >= 2) + const shellID = String(command.pipe?.metadata.id) + expect(yield* shellCall(resumed, session.id)).toMatchObject([ + { + id: "call-restart", + state: { + status: "completed", + content: [ + { + type: "text", + text: expect.stringContaining(`Command moved to the background (shell ID: ${shellID})`), + }, + { type: "text", text: expect.stringContaining("You will be notified automatically") }, + ], + }, + }, + ]) + if (!finished) daemon.exit(command.id, 0) + // Either way the session hears how it ended. + const notices = Effect.map(resumed.context(session.id), (messages) => + messages.filter((message) => message.type === "synthetic" && message.metadata?.source === "shell"), + ) + while ((yield* notices).length === 0) yield* Effect.sleep(5) + expect((yield* notices)[0]).toMatchObject({ metadata: { shellID, state: "completed" } }) + if (finished) expect((yield* notices)[0]).toMatchObject({ text: expect.stringContaining("finished output") }) + yield* resumed.wait(session.id) + }), + ) +}) From 8b83b53a9f2d4ddb0b0ceaaf8a1639aa44a40628 Mon Sep 17 00:00:00 2001 From: James Long Date: Tue, 10 Nov 2026 17:21:18 +0000 Subject: [PATCH 07/12] test(core): cover a user stop while a daemon command starts or after a reload The fake daemon can hold `create_pipe`, like a daemon slow to start a command. With the step's real cancel gate: a stop while the command is being created reaches the tool cooperatively, which stops the command and ends its job; a stop after a Location reload does the same; a shutdown while the command starts leaves it running for restart recovery. User decisions: - The shell tool listens for the user's stop from the start of the call - A user stop after a Location reload also goes through cooperative cancel - A server shutdown stays a plain interrupt that leaves the command running --- packages/core/src/session/runner/step.ts | 2 +- .../test/fixture/persistent-pty-daemon.ts | 26 +++-- packages/core/test/tool-shell.test.ts | 108 ++++++++++++++++++ 3 files changed, 128 insertions(+), 8 deletions(-) diff --git a/packages/core/src/session/runner/step.ts b/packages/core/src/session/runner/step.ts index 30ca3cd3118c..65cd7cdec205 100644 --- a/packages/core/src/session/runner/step.ts +++ b/packages/core/src/session/runner/step.ts @@ -371,7 +371,7 @@ type Cancel = ReturnType * A tool call's `cancel`. `trigger` completes it, and reports whether the tool was awaiting it, so would stop on its * own; a tool that was not is interrupted instead. */ -const makeCancel = () => { +export const makeCancel = () => { let listeners = 0 let cancelled = false const gate = Deferred.makeUnsafe() diff --git a/packages/core/test/fixture/persistent-pty-daemon.ts b/packages/core/test/fixture/persistent-pty-daemon.ts index 38413c9e4a18..afca80e2d351 100644 --- a/packages/core/test/fixture/persistent-pty-daemon.ts +++ b/packages/core/test/fixture/persistent-pty-daemon.ts @@ -16,7 +16,11 @@ export function temporaryDirectory() { export function listen( socketPath: string, - handle: (socket: net.Socket, request: Record, token: string) => object | undefined, + handle: ( + socket: net.Socket, + request: Record, + token: string, + ) => object | undefined | Promise, ) { return Effect.acquireRelease( Effect.promise(async () => { @@ -33,8 +37,10 @@ export function listen( const owning = envelope.request.op === "own" if (owning) owner = true const response = owning ? { type: "owned" } : handle(socket, envelope.request, envelope.token) - if (response) socket.write(frame(response)) - if (owner) return serve() + return Promise.resolve(response).then((response) => { + if (response) socket.write(frame(response)) + if (owner) return serve() + }) }) void serve().catch(() => socket.destroy()) }) @@ -132,6 +138,8 @@ export const fakeDaemon = Effect.fn(function* () { const failTerminate = new Set() const trap = new Set() const failCreate = { message: undefined as string | undefined } + // While set, `create_pipe` answers only once it resolves, like a daemon slow to start a command. + const holdCreate: { until?: Promise } = {} const ops: string[] = [] const requests: Record[] = [] let nextID = 1 @@ -187,10 +195,13 @@ export const fakeDaemon = Effect.fn(function* () { if (typeof request.cwd !== "string" || !path.isAbsolute(request.cwd)) return { type: "error", message: "cwd must be absolute" } if (failCreate.message !== undefined) return { type: "error", message: failCreate.message } - const created = pipe(nextID++, request) - writeFileSync(String(request.output_path), "") - processes.push(created) - return { type: "created", process: created } + const create = () => { + const created = pipe(nextID++, request) + writeFileSync(String(request.output_path), "") + processes.push(created) + return { type: "created", process: created } + } + return holdCreate.until ? holdCreate.until.then(create) : create() } if (request.op === "control") return { type: "ok" } if (request.op === "prepare_handoff") return { type: "handoff", ticket: "next", expires_at: Date.now() + 60_000 } @@ -245,6 +256,7 @@ export const fakeDaemon = Effect.fn(function* () { return { failTerminate, failCreate, + holdCreate, trap, ops, requests, diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index 5b330879ea43..0d16d25fe37b 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -42,6 +42,7 @@ import { ShellTool } from "@opencode/core/tool/plugin/shell" import { ToolOutput } from "@opencode/core/tool-output" import { Tool } from "@opencode/core/tool" import { PluginHooks } from "@opencode/core/plugin/hooks" +import { SessionStep } from "@opencode/core/session/runner/step" import { PersistentPty } from "@opencode/core/persistent-pty" import { disabledPersistentPtyLayer } from "./fixture/persistent-pty" import { fakeDaemon } from "./fixture/persistent-pty-daemon" @@ -2313,6 +2314,113 @@ describe("ShellTool stopped", () => { { timeout: 15_000 }, ) + // Starts a call whose daemon command is slow to be created, and returns once the daemon was asked for it. + const startSlowCall = (registry: Tool.Interface, id: string) => + Effect.gen(function* () { + let release = () => {} + if (fake) fake.holdCreate.until = new Promise((resolve) => (release = resolve)) + // The step's own gate: a stop goes through `cancel` only if the tool is awaiting it. + const cancel = SessionStep.makeCancel() + const fiber = yield* executeTool(registry, { + ...call({ command: "sleep 60" }, id), + cancel: cancel.effect, + }).pipe(Effect.forkChild({ startImmediately: true })) + while (!fake?.ops.includes("create_pipe")) yield* Effect.promise(() => Bun.sleep(2)) + return { + fiber, + stop: Effect.sync(() => cancel.trigger()), + release: Effect.sync(() => { + if (fake) fake.holdCreate.until = undefined + release() + }), + } + }) + + fakeDaemonIt.live("stops a command the user stopped while it was being created", () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const jobs = yield* Job.Service + const shell = yield* Shell.Service + const call = yield* startSlowCall(registry, "call-slow-create") + // The tool already listens while its command is being created, so the step would not interrupt it. + expect(yield* call.stop).toBe(true) + yield* call.release + const result = yield* Fiber.join(call.fiber) + expect(result.error?.message).toContain("Command interrupted by the user") + const process = fake?.processes[0] + expect(fake?.requests).toContainEqual( + expect.objectContaining({ op: "signal", id: process?.id, signal: "SIGTERM" }), + ) + expect(process?.lifecycle.status).toBe("exited") + expect(yield* shell.list()).toEqual([]) + expect((yield* jobs.get(String(process?.pipe?.metadata.id)))?.status).not.toBe("running") + }), + ) + }), + ) + + fakeDaemonIt.live("stops a command the user stopped after its Location reloaded", () => + withTempDir((tmp) => + Effect.gen(function* () { + reset() + const sessions = yield* Session.Service + const locations = yield* LocationServiceMap.Service + const location = Location.Ref.make({ directory: AbsolutePath.make(tmp.path) }) + yield* sessions.create({ id: sessionID, title: "shell test", location, model: sessionModel }) + yield* Effect.gen(function* () { + const plugins = yield* Plugin.Service + yield* plugins.awaitActivation + const registry = yield* Tool.Service + const jobs = yield* Job.Service + const cancel = SessionStep.makeCancel() + const started = yield* Deferred.make() + const fiber = yield* executeTool(registry, { + ...call({ command: "sleep 60" }, "call-reloaded"), + progress: (update) => + typeof update.shellID === "string" + ? Deferred.succeed(started, update.shellID).pipe(Effect.asVoid) + : Effect.void, + cancel: cancel.effect, + }).pipe(Effect.forkChild({ startImmediately: true })) + const shellID = yield* Deferred.await(started) + // The call still holds the old Location when a reload closes it. + yield* locations.invalidate(location) + expect(cancel.trigger()).toBe(true) + const result = yield* Fiber.join(fiber) + expect(result.error?.message).toContain(`Command interrupted by the user (shell ID: ${shellID})`) + expect(fake?.processes[0]?.lifecycle.status).toBe("exited") + expect((yield* jobs.get(shellID))?.status).not.toBe("running") + }).pipe(Effect.provide(locations.get(location))) + yield* locations.invalidate(location) + }), + ), + ) + + fakeDaemonIt.live("leaves a command running when a shutdown interrupts its call while it starts", () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const jobs = yield* Job.Service + const call = yield* startSlowCall(registry, "call-shutdown-start") + const interrupting = yield* Fiber.interrupt(call.fiber).pipe(Effect.forkChild({ startImmediately: true })) + yield* call.release + yield* Fiber.join(interrupting) + const process = fake?.processes[0] + // Nothing stopped it: a restart takes it over through its recorded tool call. + expect(process?.lifecycle.status).toBe("running") + expect(process?.pipe?.metadata).toMatchObject({ toolCall: { callID: "call-shutdown-start" } }) + expect(fake?.ops).not.toContain("signal") + const shellID = String(process?.pipe?.metadata.id) + expect((yield* jobs.get(shellID))?.status).toBe("running") + yield* jobs.cancel(shellID) + }), + ) + }), + ) + fakeDaemonIt.live("records which call waits on a daemon command, and keeps no timeout for a restart", () => withTempDir((tmp) => { reset() From 478c75fd56188be1ef8c45592788b91473d310bc Mon Sep 17 00:00:00 2001 From: James Long Date: Tue, 10 Nov 2026 17:48:28 +0000 Subject: [PATCH 08/12] feat(core): stop interrupted daemon commands unless a handoff is prepared A prepared handoff is now the one signal that this server is being replaced and its daemon commands should outlive it. PersistentPty records it when `handoff()` prepares a ticket, and forgets it on `shutdown()` or when the ticket expires, so an abandoned handoff cannot leave it set. - The shell tool cancels its command's job when its call is interrupted (and a job's run stops its command when interrupted), unless a handoff is prepared: then the command keeps running for the next server. - The step leaves its tool calls unsettled, with no step end, only while a handoff is prepared; any other interrupt without a user reason settles them as "Tool execution interrupted", like before. The session-level `reason ?? "shutdown"` that keeps the claim is unchanged, and the execution no longer guesses "shutdown" for the step. User decisions: - "Handoff prepared" is the single detach signal, cleared by a shutdown or an expired ticket - Any interrupt stops the command unless a handoff is prepared, including reattached jobs - The step leaves calls unsettled only while a handoff is prepared - Keep the session-level reason ?? "shutdown" so service restart resumes sessions --- packages/core/src/persistent-pty/index.ts | 16 +++- packages/core/src/session/execution.ts | 3 +- packages/core/src/session/runner/llm.ts | 4 + packages/core/src/session/runner/step.ts | 10 ++- packages/core/src/shell.ts | 5 ++ packages/core/src/tool/plugin/shell.ts | 17 ++++- .../test/fixture/persistent-pty-daemon.ts | 22 +++++- packages/core/test/fixture/persistent-pty.ts | 1 + .../core/test/persistent-pty-session.test.ts | 28 +++++++ packages/core/test/session-execution.test.ts | 7 +- packages/core/test/session-owned.test.ts | 3 +- packages/core/test/session-step.test.ts | 33 ++++++++- packages/core/test/shell-restart.test.ts | 41 +++++----- packages/core/test/tool-shell.test.ts | 74 ++++++++++++------- 14 files changed, 198 insertions(+), 66 deletions(-) diff --git a/packages/core/src/persistent-pty/index.ts b/packages/core/src/persistent-pty/index.ts index 99c99e4f3c75..ac3db0abeb61 100644 --- a/packages/core/src/persistent-pty/index.ts +++ b/packages/core/src/persistent-pty/index.ts @@ -171,6 +171,11 @@ export interface Interface { readonly remove: (id: Pty.ID) => Effect.Effect readonly shutdown: () => Effect.Effect readonly handoff: () => Effect.Effect + /** + * Whether a handoff to the next server is prepared and its ticket still valid: this server is being replaced, so + * the daemon's processes should be left running for the next one. A shutdown clears it. + */ + readonly handoffPrepared: () => boolean readonly attach: (id: Pty.ID, input: SubscribeInput) => Effect.Effect /** Starts the daemon if needed. Fall back to another spawner only on `UnavailableError`. */ readonly createPipe: (input: CreatePipeInput) => Effect.Effect @@ -361,14 +366,22 @@ const makeLayer = (options: Options = {}) => return undefined }) + // When the prepared handoff's ticket expires; 0 when none is prepared. + let handoffExpiresAt = 0 + const shutdown = Effect.fn("PersistentPty.shutdown")(function* () { + handoffExpiresAt = 0 const response = yield* daemon.shutdown.pipe(Effect.mapError(unavailable)) current.clear() if (!response) return if (response.type !== "ok") return yield* unexpected(response) }) - const handoff = Effect.fn("PersistentPty.handoff")(() => daemon.handoff.pipe(Effect.mapError(unavailable))) + const handoff = Effect.fn("PersistentPty.handoff")(function* () { + const prepared = yield* daemon.handoff.pipe(Effect.mapError(unavailable)) + if (prepared) handoffExpiresAt = prepared.expiresAt + return prepared + }) const removeVisibleExit = (id: Pty.ID) => { if (removing.has(id)) return @@ -512,6 +525,7 @@ const makeLayer = (options: Options = {}) => remove, shutdown, handoff, + handoffPrepared: () => Date.now() < handoffExpiresAt, attach, createPipe, waitPipe, diff --git a/packages/core/src/session/execution.ts b/packages/core/src/session/execution.ts index b70a90b3df68..f2867c4f7e3d 100644 --- a/packages/core/src/session/execution.ts +++ b/packages/core/src/session/execution.ts @@ -154,8 +154,7 @@ export const layer = Layer.effect( ), }) - // Teardown interrupts without calling `interrupt`, so an interrupt with no recorded reason is a shutdown. - interruption = (sessionID) => coordinator.interruption(sessionID) ?? "shutdown" + interruption = coordinator.interruption return Service.of({ active: coordinator.active, isActive: coordinator.isActive, diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 637b30b76e6c..77ecc97d971f 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -29,6 +29,7 @@ import { llmClient } from "../../effect/app-node-platform.js" import { StepFailedError } from "../error.js" import { SessionRunnerRetry } from "./retry.js" import { SessionStep } from "./step.js" +import { PersistentPty } from "../../persistent-pty/index.js" import { ToolOutput } from "../../tool-output.js" import { Plugin } from "../../plugin.js" import { PluginHooks } from "../../plugin/hooks.js" @@ -54,6 +55,7 @@ const layer = Layer.effect( const jobs = yield* Job.Service const title = yield* SessionTitle.Service const steps = yield* SessionStep.make + const pty = yield* PersistentPty.Service // Title generation starts once input is visible and must not delay model execution. const titles = yield* FiberMap.make() @@ -293,6 +295,7 @@ const layer = Layer.effect( const outcome = yield* steps.attempt({ isLocationClosed: lifecycle.isClosed, interruption, + handoffPrepared: pty.handoffPrepared, sessionID, assistantMessageID, agent: loaded.agent.id, @@ -460,6 +463,7 @@ export const node = makeLocationNode({ Job.node, SessionTitle.node, Snapshot.node, + PersistentPty.node, ToolOutput.node, Database.node, ], diff --git a/packages/core/src/session/runner/step.ts b/packages/core/src/session/runner/step.ts index 65cd7cdec205..da69c87ff06d 100644 --- a/packages/core/src/session/runner/step.ts +++ b/packages/core/src/session/runner/step.ts @@ -44,6 +44,8 @@ interface Input { readonly isLocationClosed: () => boolean /** Why the execution is being interrupted, once it is: `"user"` when someone stopped it. */ readonly interruption?: () => "user" | "shutdown" | undefined + /** Whether this server is handing its process daemon over to the next one (see `PersistentPty.handoffPrepared`). */ + readonly handoffPrepared?: () => boolean readonly sessionID: SessionSchema.ID readonly assistantMessageID: SessionMessage.ID readonly agent: Agent.ID @@ -182,12 +184,14 @@ export const make = Effect.gen(function* () { // Read each tool's exit once all have ended, so what they raised while stopping (a decline, a defect) counts. const exits = Exit.isSuccess(joined) ? joined.value : yield* Fiber.awaitAll(fibers) const tools = classifyToolExits(exits, toolRuns, Exit.hasInterrupts(joined)) - // A server shutting down leaves the step as a crash would: unsettled calls and no step end. The next server - // settles them when the Session resumes, and takes over the shell commands still running for them. + // A server handing its process daemon over leaves the step as a crash would: unsettled calls and no step end. + // The next server settles them when the Session resumes, and takes over the shell commands still running for + // them. Any other interrupt settles the calls here. if ( (streamInterrupted || tools.interrupted) && tools.declines.length === 0 && - input.interruption?.() === "shutdown" + input.interruption?.() !== "user" && + input.handoffPrepared?.() === true ) return yield* Effect.interrupt diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index 80fa0b953aa8..f535924ac8d3 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -174,6 +174,8 @@ export interface Interface { readonly remove: (id: Shell.ID) => Effect.Effect /** Runs `effect` until it ends or the server stops, outliving this Location (e.g. a job's completion notice). */ readonly watch: DaemonInterface["watch"] + /** Whether the server is handing its daemon commands over to the next one, which takes them over. */ + readonly handoffPrepared: () => boolean } export class Service extends Context.Service()("@opencode/Shell") {} @@ -260,6 +262,7 @@ export interface DaemonInterface { readonly at: (location: Location.Interface) => Effect.Effect /** Runs `effect` until it ends or the server stops, outliving the caller's Location (e.g. a job's completion notice). */ readonly watch: (effect: Effect.Effect) => Effect.Effect + readonly handoffPrepared: () => boolean } /** The process daemon's shell commands, owned by the server and grouped by Location. */ @@ -577,6 +580,7 @@ const daemonLayer = Layer.effect( return Daemon.of({ at, + handoffPrepared: pty.handoffPrepared, watch: (effect) => fork(effect.pipe(Effect.catch((error) => Effect.logError("shell watcher failed", { error })))).pipe( Effect.asVoid, @@ -1112,6 +1116,7 @@ const layer = () => output, remove, watch: daemon.watch, + handoffPrepared: daemon.handoffPrepared, }) }), ) diff --git a/packages/core/src/tool/plugin/shell.ts b/packages/core/src/tool/plugin/shell.ts index 1a5271f3e975..cc384580e407 100644 --- a/packages/core/src/tool/plugin/shell.ts +++ b/packages/core/src/tool/plugin/shell.ts @@ -192,9 +192,13 @@ const startJob = Effect.fnUntraced(function* ( }).pipe( Effect.tap((output) => Deferred.succeed(settled, output)), Effect.map((output) => resultMessages(output, input.timeout?.()).join("\n\n")), + // Interrupting the job (its registry closing) stops the command too, unless the server is handing its daemon + // over: then the command keeps running for the next server. + Effect.onInterrupt(() => + shell.handoffPrepared() ? Effect.void : shell.stop(info.id, "killed").pipe(Effect.ignore, Effect.asVoid), + ), ), - // Only cancelling the job stops the command, keeping it and its output file. A server shutdown interrupts - // the job without stopping it, so the command keeps running for the next server. + // Cancelling the job stops the command, keeping it and its output file. cancel: shell.stop(info.id, "killed").pipe(Effect.ignore, Effect.asVoid), }) return { job, settled } @@ -407,13 +411,18 @@ export const Plugin = { ), ) : waiting - // A user's stop arrives through `cancel`. An interrupt (a server shutdown) leaves the command and its - // job running: the next server takes the call over and moves the command to the background. + // A user's stop arrives through `cancel`. Any interrupt stops the command too, unless the server is + // handing its daemon over: then the command and its job keep running, and the next server takes the + // call over and moves the command to the background. const outcome = yield* restore( Effect.raceFirst( blocked.pipe(Effect.map((result) => ({ type: "settled" as const, result }))), Deferred.await(stopped).pipe(Effect.as({ type: "cancelled" as const })), ), + ).pipe( + Effect.onInterrupt(() => + shell.handoffPrepared() ? Effect.void : jobs.cancel(job.id).pipe(Effect.ignore), + ), ) if (outcome.type === "cancelled") return yield* cancelled(info, job.id) const result = outcome.result diff --git a/packages/core/test/fixture/persistent-pty-daemon.ts b/packages/core/test/fixture/persistent-pty-daemon.ts index afca80e2d351..e692d52411d9 100644 --- a/packages/core/test/fixture/persistent-pty-daemon.ts +++ b/packages/core/test/fixture/persistent-pty-daemon.ts @@ -1,4 +1,4 @@ -import { appendFileSync, writeFileSync } from "node:fs" +import { appendFileSync, rmSync, writeFileSync } from "node:fs" import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" import net from "node:net" import os from "node:os" @@ -138,8 +138,12 @@ export const fakeDaemon = Effect.fn(function* () { const failTerminate = new Set() const trap = new Set() const failCreate = { message: undefined as string | undefined } + // How long a prepared handoff's ticket stays valid. + const handoffTTL = { ms: 60_000 } // While set, `create_pipe` answers only once it resolves, like a daemon slow to start a command. const holdCreate: { until?: Promise } = {} + // While set, `list` answers only once it resolves, like a busy daemon. + const holdList: { until?: Promise } = {} const ops: string[] = [] const requests: Record[] = [] let nextID = 1 @@ -179,8 +183,10 @@ export const fakeDaemon = Effect.fn(function* () { ops.push(String(request.op)) requests.push(request) if (request.op === "ping") return pong - if (request.op === "list") - return { type: "processes", processes: processes.filter((item) => item.mode === request.mode) } + if (request.op === "list") { + const list = () => ({ type: "processes", processes: processes.filter((item) => item.mode === request.mode) }) + return holdList.until ? holdList.until.then(list) : list() + } if (request.op === "get") { const found = find(request.id) if (!found) return { type: "error", message: `terminal ${String(request.id)} not found` } @@ -204,7 +210,13 @@ export const fakeDaemon = Effect.fn(function* () { return holdCreate.until ? holdCreate.until.then(create) : create() } if (request.op === "control") return { type: "ok" } - if (request.op === "prepare_handoff") return { type: "handoff", ticket: "next", expires_at: Date.now() + 60_000 } + // Unregisters, so the server sees it stopped; it keeps serving connections already open. + if (request.op === "shutdown") { + rmSync(path.join(directory, "service.json"), { force: true }) + return { type: "ok" } + } + if (request.op === "prepare_handoff") + return { type: "handoff", ticket: "next", expires_at: Date.now() + handoffTTL.ms } if (request.op === "rename") { const found = find(request.id) if (!found?.pty) return { type: "error", message: "terminal not found" } @@ -257,6 +269,8 @@ export const fakeDaemon = Effect.fn(function* () { failTerminate, failCreate, holdCreate, + holdList, + handoffTTL, trap, ops, requests, diff --git a/packages/core/test/fixture/persistent-pty.ts b/packages/core/test/fixture/persistent-pty.ts index c81232f00e08..c37a4004aaea 100644 --- a/packages/core/test/fixture/persistent-pty.ts +++ b/packages/core/test/fixture/persistent-pty.ts @@ -5,4 +5,5 @@ import { PersistentPty } from "@opencode/core/persistent-pty" export const disabledPersistentPtyLayer = Layer.mock(PersistentPty.Service, { createPipe: () => Effect.fail(new PersistentPty.UnavailableError({ message: "process daemon disabled" })), listPipes: () => Effect.succeed([]), + handoffPrepared: () => false, }) diff --git a/packages/core/test/persistent-pty-session.test.ts b/packages/core/test/persistent-pty-session.test.ts index d199807f8c5d..cf305ceb2d15 100644 --- a/packages/core/test/persistent-pty-session.test.ts +++ b/packages/core/test/persistent-pty-session.test.ts @@ -122,6 +122,34 @@ describe.skipIf(process.platform === "win32")("PersistentPty pipe processes", () metadata: { id: "sh_test", command: "echo hi" }, } + it.live("knows a handoff is prepared until a shutdown or until its ticket expires", () => + Effect.gen(function* () { + const daemon = yield* fakeDaemon() + yield* Effect.gen(function* () { + const pty = yield* PersistentPty.Service + expect(pty.handoffPrepared()).toBe(false) + yield* pty.createPipe(input) + expect(yield* pty.handoff()).not.toBeNull() + expect(pty.handoffPrepared()).toBe(true) + // Abandoned by a shutdown. + yield* pty.shutdown() + expect(pty.handoffPrepared()).toBe(false) + }).pipe(Effect.provide(layer(daemon))) + + // Abandoned by letting the ticket expire. + const expiring = yield* fakeDaemon() + expiring.handoffTTL.ms = 50 + yield* Effect.gen(function* () { + const pty = yield* PersistentPty.Service + yield* pty.createPipe(input) + expect(yield* pty.handoff()).not.toBeNull() + expect(pty.handoffPrepared()).toBe(true) + yield* Effect.sleep(80) + expect(pty.handoffPrepared()).toBe(false) + }).pipe(Effect.provide(layer(expiring))) + }), + ) + it.live("creates, inspects, signals and waits for a pipe process", () => Effect.gen(function* () { const daemon = yield* fakeDaemon() diff --git a/packages/core/test/session-execution.test.ts b/packages/core/test/session-execution.test.ts index 06e392e898f1..8bb91bc4b628 100644 --- a/packages/core/test/session-execution.test.ts +++ b/packages/core/test/session-execution.test.ts @@ -147,7 +147,7 @@ describe("SessionExecution lifecycle", () => { }), ) - it.effect("tells the drain whether the user stopped it or the server is shutting down", () => + it.effect("tells the drain once the user stopped it", () => Effect.gen(function* () { const database = yield* Database.Service const sessionID = Session.ID.make("ses_interruption_user") @@ -161,8 +161,8 @@ describe("SessionExecution lifecycle", () => { const execution = Context.get(context, SessionExecution.Service) yield* execution.resume(sessionID).pipe(Effect.forkScoped) const reason = yield* Deferred.await(draining) - // Until someone stops it, an interrupt can only be the server shutting down. - expect(reason?.()).toBe("shutdown") + // Teardown interrupts without a reason, so only a user's stop reads as one. + expect(reason?.()).toBeUndefined() yield* execution.interrupt(sessionID) expect(reason?.()).toBe("user") yield* execution.awaitIdle(sessionID) @@ -1728,6 +1728,7 @@ function buildExecution( Shell.Daemon.of({ at: () => Effect.die("the harness has no daemon registry"), watch: (effect) => effect.pipe(Effect.ignore, Effect.forkIn(scope), Effect.asVoid), + handoffPrepared: () => false, }), ), ), diff --git a/packages/core/test/session-owned.test.ts b/packages/core/test/session-owned.test.ts index 8d68b5b49cfb..0c3c996286d3 100644 --- a/packages/core/test/session-owned.test.ts +++ b/packages/core/test/session-owned.test.ts @@ -125,7 +125,7 @@ const setup = Effect.fnUntraced(function* (options?: { const services = Layer.mergeAll( Layer.succeed(PluginHooks.Service, hooks), Layer.mock(Image.Service, {}), - options?.shell ?? Layer.mock(Shell.Service, {}), + options?.shell ?? Layer.mock(Shell.Service, { handoffPrepared: () => false }), ) const servicesFor = (ref: Location.Ref) => { locations.push(ref) @@ -508,6 +508,7 @@ describe("Session-owned handles", () => { }) const fixture = yield* setup({ shell: Layer.mock(Shell.Service, { + handoffPrepared: () => false, create: (input) => Effect.sync(() => { expect(input).toEqual({ diff --git a/packages/core/test/session-step.test.ts b/packages/core/test/session-step.test.ts index d288204e61a0..258c3923701c 100644 --- a/packages/core/test/session-step.test.ts +++ b/packages/core/test/session-step.test.ts @@ -233,8 +233,9 @@ it.effect("records the cost the endpoint reports instead of the catalog price", // Interrupts an attempt running a tool that stops on `cancel` and one that never does, and returns how the step and // its calls were recorded. const stopTwoTools = (how: { - reason: "user" | "shutdown" + reason: "user" | "shutdown" | undefined locationClosed: boolean + handoffPrepared?: boolean // What the cooperative tool does once asked to stop; by default it fails with its partial output. stopped?: Effect.Effect }) => @@ -284,6 +285,7 @@ const stopTwoTools = (how: { .attempt({ isLocationClosed: () => how.locationClosed, interruption: () => how.reason, + handoffPrepared: () => how.handoffPrepared === true, sessionID, assistantMessageID, agent: Agent.defaultID, @@ -392,9 +394,34 @@ it.live("classifies what a tool raises while the user's stop settles it: a defec }), ) -it.live("leaves an interrupted step as a crash would when the server shuts down", () => +// Without a handoff nothing will take the calls over, so they settle here however the step was interrupted. +for (const reason of ["shutdown", undefined] as const) + it.live( + `settles every call as interrupted on ${reason === "shutdown" ? "a server shutdown" : "an unexplained interrupt"} without a handoff`, + () => + Effect.gen(function* () { + const { exit, data, requests, interrupted } = yield* stopTwoTools({ reason, locationClosed: false }) + expect(Exit.hasInterrupts(exit)).toBe(true) + expect(requests).toBe(1) + expect(interrupted).toBe(true) + const generic = { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } } + expect(data).toMatchObject({ + error: { type: "aborted", message: "Step interrupted" }, + content: [ + { type: "tool", id: "call-cooperative", state: generic }, + { type: "tool", id: "call-stuck", state: generic }, + ], + }) + }), + ) + +it.live("leaves an interrupted step as a crash would while the server hands its daemon over", () => Effect.gen(function* () { - const { exit, data, requests } = yield* stopTwoTools({ reason: "shutdown", locationClosed: false }) + const { exit, data, requests } = yield* stopTwoTools({ + reason: "shutdown", + locationClosed: false, + handoffPrepared: true, + }) expect(Exit.hasInterrupts(exit)).toBe(true) expect(requests).toBe(1) // No step end and no settled call: the next server finds the calls unsettled. diff --git a/packages/core/test/shell-restart.test.ts b/packages/core/test/shell-restart.test.ts index c93b837b81aa..db1e66c25f26 100644 --- a/packages/core/test/shell-restart.test.ts +++ b/packages/core/test/shell-restart.test.ts @@ -1,5 +1,5 @@ -// A foreground shell call across a graceful restart, through the whole app: server 1 runs the turn and is torn down -// mid-call, server 2 recovers the command from the same process daemon and resumes the session. +// A foreground shell call across a graceful restart, through the whole app: server 1 runs the turn, prepares a +// handoff and is torn down mid-call, server 2 recovers the command from the same process daemon and resumes the session. import { describe, expect } from "bun:test" import path from "path" import { Context, Effect, Exit, Layer, Scope } from "effect" @@ -68,22 +68,25 @@ const world = Effect.gen(function* () { // A server: the whole app on the shared pieces, torn down when its scope closes. const server = (scope: Scope.Scope) => Layer.buildWithScope( - AppNodeBuilder.build(LayerNode.group([Session.node, LocationServiceMap.node, SessionRestart.node]), [ - Global.node.replace(global), - Database.node.replace(Layer.succeedContext(database)), - LayerNodePlatform.llmClient.replace(Layer.succeedContext(Context.pick(LLMClient.Service)(llm))), - PersistentPty.node.replace( - makeGlobalNode({ - service: PersistentPty.Service, - layer: PersistentPty.layerWith(daemon.options), - deps: [Bus.node, Global.node], - }), - ), - Permission.node.replace(permissionLayer({ assert: () => Effect.void })), - Watcher.node.replace(Watcher.configured({ enabled: false })), - offlineModels, - model, - ]), + AppNodeBuilder.build( + LayerNode.group([Session.node, LocationServiceMap.node, SessionRestart.node, PersistentPty.node]), + [ + Global.node.replace(global), + Database.node.replace(Layer.succeedContext(database)), + LayerNodePlatform.llmClient.replace(Layer.succeedContext(Context.pick(LLMClient.Service)(llm))), + PersistentPty.node.replace( + makeGlobalNode({ + service: PersistentPty.Service, + layer: PersistentPty.layerWith(daemon.options), + deps: [Bus.node, Global.node], + }), + ), + Permission.node.replace(permissionLayer({ assert: () => Effect.void })), + Watcher.node.replace(Watcher.configured({ enabled: false })), + offlineModels, + model, + ], + ), scope, ) return { daemon, llm: Context.get(llm, TestLLM.Test), project, server } @@ -135,6 +138,8 @@ describe("shell commands across a graceful restart", () => { yield* sessions.prompt({ sessionID: session.id, text: "run it" }) yield* until(() => daemon.processes.length === 1 && daemon.waiters(daemon.processes[0]!.id) > 0) const command = daemon.processes[0]! + // A graceful restart: the server hands its daemon over to the next one, then shuts down. + expect(yield* Context.get(one, PersistentPty.Service).handoff()).not.toBeNull() yield* Scope.close(first, Exit.void) // The shutdown stopped nothing and settled nothing. diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index 0d16d25fe37b..7de1a8b66d1f 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -2398,28 +2398,52 @@ describe("ShellTool stopped", () => { ), ) - fakeDaemonIt.live("leaves a command running when a shutdown interrupts its call while it starts", () => - withTempDir((tmp) => { - reset() - return withSession(tmp.path, (registry) => - Effect.gen(function* () { - const jobs = yield* Job.Service - const call = yield* startSlowCall(registry, "call-shutdown-start") - const interrupting = yield* Fiber.interrupt(call.fiber).pipe(Effect.forkChild({ startImmediately: true })) - yield* call.release - yield* Fiber.join(interrupting) - const process = fake?.processes[0] - // Nothing stopped it: a restart takes it over through its recorded tool call. - expect(process?.lifecycle.status).toBe("running") - expect(process?.pipe?.metadata).toMatchObject({ toolCall: { callID: "call-shutdown-start" } }) - expect(fake?.ops).not.toContain("signal") - const shellID = String(process?.pipe?.metadata.id) - expect((yield* jobs.get(shellID))?.status).toBe("running") - yield* jobs.cancel(shellID) - }), + for (const handoff of [false, true]) + for (const starting of [true, false]) + fakeDaemonIt.live( + `${handoff ? "leaves a command running" : "stops a command"} when an interrupt ${handoff ? "during a handoff " : ""}cuts its call off while it ${starting ? "starts" : "runs"}`, + () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const jobs = yield* Job.Service + const pty = yield* PersistentPty.Service + const call = yield* startSlowCall(registry, "call-interrupted") + if (handoff) expect(yield* pty.handoff()).not.toBeNull() + if (!starting) { + yield* call.release + while (fake!.waiters(fake!.processes[0]?.id ?? -1) === 0) yield* Effect.sleep(2) + } + const interrupting = yield* Fiber.interrupt(call.fiber).pipe( + Effect.forkChild({ startImmediately: true }), + ) + if (starting) yield* call.release + yield* Fiber.join(interrupting) + expect(Exit.hasInterrupts(yield* Fiber.await(call.fiber))).toBe(true) + const process = fake?.processes[0] + const shellID = String(process?.pipe?.metadata.id) + if (handoff) { + // Nothing stopped it: the next server takes it over through its recorded tool call. + expect(process?.lifecycle.status).toBe("running") + expect(process?.pipe?.metadata).toMatchObject({ toolCall: { callID: "call-interrupted" } }) + expect(fake?.ops).not.toContain("signal") + expect((yield* jobs.get(shellID))?.status).toBe("running") + yield* jobs.cancel(shellID) + return + } + // Nothing will take it over, so it is stopped and its job ends. + const shell = yield* Shell.Service + expect((yield* shell.wait(ID.make(shellID))).status).toBe("killed") + expect(fake?.requests).toContainEqual( + expect.objectContaining({ op: "signal", id: process?.id, signal: "SIGTERM" }), + ) + expect(process?.lifecycle.status).toBe("exited") + expect((yield* jobs.get(shellID))?.status).toBe("cancelled") + }), + ) + }), ) - }), - ) fakeDaemonIt.live("records which call waits on a daemon command, and keeps no timeout for a restart", () => withTempDir((tmp) => { @@ -2468,7 +2492,7 @@ describe("ShellTool stopped", () => { ) })) - test("leaves a call a shutdown interrupted, and its command, for the next server", () => + test("stops a command whose call an interrupt cut off, with no handoff to take it over", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => @@ -2477,12 +2501,8 @@ describe("ShellTool stopped", () => { const jobs = yield* Job.Service const { shellID, exit } = yield* stopCall(registry, "printf ready; sleep 60", "call-teardown", "interrupt") expect(Exit.hasInterrupts(exit)).toBe(true) - // Nothing stops the command or ends its job: the next server takes the call over. - expect((yield* shell.get(shellID)).status).toBe("running") - expect((yield* jobs.get(shellID))?.status).toBe("running") - // Only an explicit cancel stops it. - yield* jobs.cancel(shellID) expect((yield* shell.wait(shellID)).status).toBe("killed") + expect((yield* jobs.get(shellID))?.status).toBe("cancelled") }), ) })) From 50ab9c085261f6f8fbfb4fa4c67fd99fea964072 Mon Sep 17 00:00:00 2001 From: James Long Date: Tue, 10 Nov 2026 17:48:28 +0000 Subject: [PATCH 09/12] test(core): finish a Location's recovery when the call that started it is interrupted Covers the uninterruptible recovery: the first call for a Location is interrupted while the daemon is slow to list its processes; recovery still takes the command back, and a later call does not run it again. User decisions: - Make Location recovery uninterruptible, and test it with a slow listPipes --- packages/core/test/shell-daemon.test.ts | 38 +++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts index 2736f8c2437c..b6ed23ff34ed 100644 --- a/packages/core/test/shell-daemon.test.ts +++ b/packages/core/test/shell-daemon.test.ts @@ -2,7 +2,7 @@ import { describe, expect } from "bun:test" import { existsSync, readdirSync, realpathSync } from "node:fs" import { mkdir, rm, utimes, writeFile } from "node:fs/promises" import path from "node:path" -import { Effect, Layer } from "effect" +import { Context, Effect, Fiber, Layer } from "effect" import { TestClock } from "effect/testing" import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" import { PersistentPty } from "@opencode/core/persistent-pty" @@ -11,6 +11,7 @@ import { Environment } from "@opencode/core/environment/index" import { Location } from "@opencode/core/location" import { AbsolutePath } from "@opencode/core/schema" import { Shell } from "@opencode/core/shell" +import { Bus } from "@opencode/core/bus" import { Global } from "@opencode/util/global" import { FSUtil } from "@opencode/util/fs-util" import { LayerNode } from "@opencode/util/effect/layer-node" @@ -403,7 +404,40 @@ describe("Shell on the process daemon", () => { }), ) - test("the sweep keeps a running command's file and forgets a finished one before deleting its file", () => + test("finishes taking a Location's commands back even if the call that started it is interrupted", () => + Effect.gen(function* () { + const env = yield* daemonShells() + const created = yield* env.withShell(env.directory, (shell) => + shell.create({ shell: "/bin/sh", command: "running" }), + ) + // A new server, whose first request for the Location is interrupted while the daemon is slow to list. + const server = Context.get( + yield* Layer.build( + AppNodeBuilder.build(LayerNode.group([Shell.daemonNode, Bus.node]), [ + Global.node.replace(env.global), + PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, env.pty)), + ]), + ), + Shell.Daemon, + ) + const place = location(Location.Ref.make({ directory: AbsolutePath.make(env.directory) })) + let release = () => {} + env.daemon.holdList.until = new Promise((resolve) => (release = resolve)) + const lists = () => env.daemon.ops.filter((op) => op === "list").length + const before = lists() + const first = yield* server.at(place).pipe(Effect.forkChild({ startImmediately: true })) + yield* until(() => lists() > before) + const interrupting = yield* Fiber.interrupt(first).pipe(Effect.forkChild({ startImmediately: true })) + env.daemon.holdList.until = undefined + release() + yield* Fiber.join(interrupting) + // Recovery ran to the end, and is not run again. + const runs = yield* server.at(place) + expect(runs.list().map((run) => run.info.id)).toEqual([created.id]) + expect(lists()).toBe(before + 1) + })) + + test("the sweep keeps a running command's file, and deletes a finished one's file and forgets it", () => Effect.gen(function* () { const env = yield* daemonShells() const ids = yield* env.withShell(env.directory, (shell) => From 3b8ded7be51174857f633d1cd1ecf94ab7d21bd7 Mon Sep 17 00:00:00 2001 From: James Long Date: Tue, 10 Nov 2026 18:42:02 +0000 Subject: [PATCH 10/12] fix(server): release the process daemon after the sessions and jobs that use it The server acquired the jobs and sessions before the process daemon and its command registry, so a shutdown released the daemon first. Interrupted shell calls and jobs then could not stop their commands, a stop already under way was cut short, and a command being created fell back to spawning mid-shutdown. The daemon is now acquired first, so it is released last. A stop also records itself and sends SIGTERM before it returns; only the SIGKILL escalation runs on its own. Tests cover a shutdown without a handoff, for a running command and one still being created: the command is stopped in the daemon, nothing is spawned, and the next server resumes the session with the call settled as interrupted. The restart test now waits for the resumed turn to end before the command finishes, and checks that its completion wakes it. User decisions: - Acquire the process daemon before the jobs and sessions, in the server and the test graph - Test that a shutdown without a handoff stops the command and never spawns one --- packages/core/src/shell.ts | 12 +-- packages/core/test/shell-restart.test.ts | 106 ++++++++++++++++++++--- packages/core/test/tool-shell.test.ts | 1 - packages/server/src/routes.ts | 8 +- 4 files changed, 104 insertions(+), 23 deletions(-) diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index f535924ac8d3..8786c771d41f 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -368,23 +368,25 @@ const daemonLayer = Layer.effect( yield* evict() }) - // Graceful, like spawned commands: SIGTERM, then SIGKILL if it has not exited 3 s later. It runs on its own, - // so interrupting the caller (a timer being retimed, removed or torn down) never cancels a recorded stop. + // Graceful, like spawned commands: SIGTERM, then SIGKILL if it has not exited 3 s later. The stop is recorded + // and the SIGTERM sent before this returns, uninterruptibly, so neither interrupting the caller (a timer being + // retimed, removed or torn down) nor a server shutting down right after cancels it. The escalation runs on its + // own. const stop = (run: Run, reason: StopReason) => Effect.gen(function* () { if (run.stopReason !== undefined || run.info.status !== "running") return run.stopReason = reason run.timeoutDeadline = undefined + yield* store(run) + yield* pty.signalPipe(run.process, "SIGTERM").pipe(Effect.ignore) run.stopping = yield* fork( Effect.gen(function* () { - yield* store(run) - yield* pty.signalPipe(run.process, "SIGTERM").pipe(Effect.ignore) const ended = yield* Deferred.await(run.done).pipe(Effect.timeoutOption(STOP_GRACE), Effect.exit) if (Exit.isSuccess(ended) && Option.isNone(ended.value)) yield* pty.signalPipe(run.process, "SIGKILL").pipe(Effect.ignore) }), ) - }) + }).pipe(Effect.uninterruptible) const arm = Effect.fnUntraced(function* (run: Run, duration: number) { run.timeoutFiber = yield* fork( diff --git a/packages/core/test/shell-restart.test.ts b/packages/core/test/shell-restart.test.ts index db1e66c25f26..491b13f3d6c3 100644 --- a/packages/core/test/shell-restart.test.ts +++ b/packages/core/test/shell-restart.test.ts @@ -2,12 +2,12 @@ // handoff and is torn down mid-call, server 2 recovers the command from the same process daemon and resumes the session. import { describe, expect } from "bun:test" import path from "path" -import { Context, Effect, Exit, Layer, Scope } from "effect" +import { Context, Effect, Exit, Fiber, Layer, Scope } from "effect" +import type { ChildProcess } from "effect/process" import { LanguageModel, LLMClient } from "@opencode/ai/effect" import { OpenAIChat } from "@opencode/ai/protocols/openai-chat" import { TestLLM } from "@opencode/ai/testing" import { Agent } from "@opencode/core/agent" -import { Bus } from "@opencode/core/bus" import { Database } from "@opencode/core/database/database" import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" import { LayerNodePlatform } from "@opencode/core/effect/app-node-platform" @@ -22,10 +22,12 @@ import { AbsolutePath } from "@opencode/core/schema" import { Session } from "@opencode/core/session" import { SessionRestart } from "@opencode/core/session/execution/restart" import { SessionRunnerModel } from "@opencode/core/session/runner/model" -import { makeGlobalNode } from "@opencode/util/effect/app-node" +import { Shell } from "@opencode/core/shell" +import { Environment } from "@opencode/core/environment/index" import { LayerNode } from "@opencode/util/effect/layer-node" import { Global } from "@opencode/util/global" import { fakeDaemon } from "./fixture/persistent-pty-daemon" +import { recordingEnvironmentLayer } from "./fixture/environment" import { tempGlobalLayer } from "./fixture/global" import { offlineModels } from "./fixture/models" import { tmpdirScoped } from "./fixture/tmpdir" @@ -65,22 +67,26 @@ const world = Effect.gen(function* () { const database = yield* Layer.build(Database.layer()) const llm = yield* Layer.build(TestLLM.testLayer({ fallback: TestLLM.text("Done", "done") })) const project = (yield* tmpdirScoped()).path - // A server: the whole app on the shared pieces, torn down when its scope closes. + // Every command the servers spawn instead of handing to the daemon. + const spawns: ChildProcess.Command[] = [] + // A server: the whole app on the shared pieces, torn down when its scope closes. The daemon comes first, as in the + // server's routes, so it is released after the sessions and jobs. const server = (scope: Scope.Scope) => Layer.buildWithScope( AppNodeBuilder.build( - LayerNode.group([Session.node, LocationServiceMap.node, SessionRestart.node, PersistentPty.node]), + LayerNode.group([ + PersistentPty.node, + Shell.daemonNode, + Session.node, + LocationServiceMap.node, + SessionRestart.node, + ]), [ Global.node.replace(global), Database.node.replace(Layer.succeedContext(database)), LayerNodePlatform.llmClient.replace(Layer.succeedContext(Context.pick(LLMClient.Service)(llm))), - PersistentPty.node.replace( - makeGlobalNode({ - service: PersistentPty.Service, - layer: PersistentPty.layerWith(daemon.options), - deps: [Bus.node, Global.node], - }), - ), + PersistentPty.node.replace(PersistentPty.configured(daemon.options)), + Environment.node.replace(recordingEnvironmentLayer(spawns)), Permission.node.replace(permissionLayer({ assert: () => Effect.void })), Watcher.node.replace(Watcher.configured({ enabled: false })), offlineModels, @@ -89,7 +95,8 @@ const world = Effect.gen(function* () { ), scope, ) - return { daemon, llm: Context.get(llm, TestLLM.Test), project, server } + const shellSpawns = () => spawns.filter((command) => JSON.stringify(command).includes("sleep 60")) + return { daemon, llm: Context.get(llm, TestLLM.Test), project, server, shellSpawns } }) const shellCall = (sessions: Session.Interface, sessionID: Session.ID) => @@ -174,7 +181,13 @@ describe("shell commands across a graceful restart", () => { }, }, ]) - if (!finished) daemon.exit(command.id, 0) + if (!finished) { + // The resumed turn ends while the command still runs; its completion wakes the session again. + yield* resumed.wait(session.id) + expect(turns).toBe(2) + daemon.exit(command.id, 0) + yield* until(() => turns >= 3) + } // Either way the session hears how it ended. const notices = Effect.map(resumed.context(session.id), (messages) => messages.filter((message) => message.type === "synthetic" && message.metadata?.source === "shell"), @@ -185,4 +198,69 @@ describe("shell commands across a graceful restart", () => { yield* resumed.wait(session.id) }), ) + + // Without a handoff nothing takes the commands over: the shutdown stops them, while the daemon is still reachable. + for (const starting of [false, true]) + it.live( + `stops a command ${starting ? "still being created" : "a foreground call waits on"} when the server shuts down without a handoff`, + () => + Effect.gen(function* () { + const { daemon, llm, project, server, shellSpawns } = yield* world + let turns = 0 + yield* llm.serve((request) => + request.tools.some((tool) => tool.name === "shell") && turns++ === 0 + ? TestLLM.toolCall("call-shutdown", "shell", { command: "sleep 60" }) + : TestLLM.text("Done", "done"), + ) + let release = () => {} + if (starting) daemon.holdCreate.until = new Promise((resolve) => (release = resolve)) + + const first = yield* Scope.make() + const one = yield* server(first) + const sessions = Context.get(one, Session.Service) + const session = yield* sessions.create({ + location: { directory: AbsolutePath.make(project) }, + agent: Agent.ID.make("build"), + model: Model.Ref.make({ id: Model.ID.make("restart"), providerID: Provider.ID.make("test") }), + }) + yield* Plugin.Service.use((plugins) => plugins.awaitActivation).pipe( + Effect.provide(Context.get(one, LocationServiceMap.Service).get(session.location)), + ) + yield* sessions.prompt({ sessionID: session.id, text: "run it" }) + if (starting) yield* until(() => daemon.ops.includes("create_pipe")) + else yield* until(() => daemon.processes.length === 1 && daemon.waiters(daemon.processes[0]!.id) > 0) + + const closing = yield* Scope.close(first, Exit.void).pipe(Effect.forkChild({ startImmediately: true })) + if (starting) { + yield* Effect.sleep(20) + daemon.holdCreate.until = undefined + release() + } + yield* Fiber.join(closing) + + // The command reached the daemon, and the shutdown stopped it there; nothing was spawned instead. + const command = daemon.processes[0] + expect(command).toBeDefined() + expect(daemon.requests).toContainEqual( + expect.objectContaining({ op: "signal", id: command?.id, signal: "SIGTERM" }), + ) + expect(command?.lifecycle.status).toBe("exited") + expect(shellSpawns()).toEqual([]) + + // The call was settled on the way down, and the next server resumes the session. + const second = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(second, Exit.void)) + const two = yield* server(second) + const resumed = Context.get(two, Session.Service) + expect(yield* shellCall(resumed, session.id)).toMatchObject([ + { + id: "call-shutdown", + state: { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } }, + }, + ]) + yield* Context.get(two, SessionRestart.Service).resumeSuspendedSessions + yield* until(() => turns >= 2) + yield* resumed.wait(session.id) + }), + ) }) diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index 7de1a8b66d1f..a2ee52545aff 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -2458,7 +2458,6 @@ describe("ShellTool stopped", () => { }) // The call's timeout applies to this server only: a restart moves the command to the background. expect(request?.metadata).not.toHaveProperty("timeoutDeadline") - expect(fake?.ops).not.toContain("signal") }), ) }), diff --git a/packages/server/src/routes.ts b/packages/server/src/routes.ts index b32b1be42149..5cf2d3ffa6f4 100644 --- a/packages/server/src/routes.ts +++ b/packages/server/src/routes.ts @@ -61,6 +61,11 @@ const applicationServiceNodes = [ Database.node, KV.node, Bus.node, + // Before the jobs and sessions, so they are released after them: a shutdown's interrupted shell calls and jobs + // still reach the daemon to stop their commands, and a command being created never falls back to spawning. + PersistentPty.node, + // Daemon-run shell commands belong to the server, so a Location closing never takes them down. + Shell.daemonNode, EventLogger.node, httpClient, Job.node, @@ -75,9 +80,6 @@ const applicationServiceNodes = [ PluginUpdate.node, PermissionSaved.node, PtyTicket.node, - PersistentPty.node, - // Daemon-run shell commands belong to the server, so a Location closing never takes them down. - Shell.daemonNode, Credential.node, WellKnown.node, ServerPairing.node, From 33b5a30fa83c01444c4a70a187105dfe203d7a55 Mon Sep 17 00:00:00 2001 From: James Long Date: Tue, 10 Nov 2026 18:43:51 +0000 Subject: [PATCH 11/12] refactor(core): reuse the background tool result's metadata, and tidy a stopped-early result Restart recovery publishes the metadata `ShellTool.backgroundToolResult` builds instead of rebuilding it, and a result whose output file stopped early has one blank line before the notice, not two. User decisions: - Reuse backgroundToolResult's metadata in restart recovery - Avoid a triple newline before the stopped-early notice --- packages/core/src/session/execution/restart.ts | 7 +------ packages/core/src/shell.ts | 4 +++- packages/core/src/tool/plugin/shell.ts | 12 +++++++++++- packages/core/test/shell-daemon.test.ts | 2 +- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/packages/core/src/session/execution/restart.ts b/packages/core/src/session/execution/restart.ts index 6f0cb37be4a3..503957b99a27 100644 --- a/packages/core/src/session/execution/restart.ts +++ b/packages/core/src/session/execution/restart.ts @@ -253,12 +253,7 @@ export const layer = (options?: Options) => assistantMessageID: call.assistantMessageID, id: call.part.id, content: result.content, - metadata: { - shellID: info.id, - status: "running", - truncated: false, - ...(info.pid === undefined ? {} : { pid: info.pid }), - }, + metadata: result.metadata, executed: call.part.executed === true, }) // Backgrounded but stopped before the call was settled: its job row was already reattached (and, for a diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index 8786c771d41f..8359d0628ec1 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -873,7 +873,9 @@ const layer = () => ...killed, } } - return { output: `${page || "(no output)"}${stoppedEarly}`, truncated: false, ...killed } + // One blank line before the notice, whether or not the output ended with a newline. + const shown = stoppedEarly && page.endsWith("\n") ? page.slice(0, -1) : page + return { output: `${shown || "(no output)"}${stoppedEarly}`, truncated: false, ...killed } }).pipe(Effect.catchTag("Shell.NotFoundError", () => Effect.succeed(undefined))) return { info, capture } }) diff --git a/packages/core/src/tool/plugin/shell.ts b/packages/core/src/tool/plugin/shell.ts index cc384580e407..584ba46af650 100644 --- a/packages/core/src/tool/plugin/shell.ts +++ b/packages/core/src/tool/plugin/shell.ts @@ -205,7 +205,17 @@ const startJob = Effect.fnUntraced(function* ( }) /** The tool result of a call whose command moved to the background. */ -export const backgroundToolResult = (info: ShellSchema.Info) => toolResult(backgroundResult(info)) +export const backgroundToolResult = (info: ShellSchema.Info) => { + const result = toolResult(backgroundResult(info)) + // Narrowed to what a backgrounded command reports, so it can be published as is. + const metadata = { + shellID: info.id, + status: "running" as const, + truncated: result.metadata.truncated, + ...(info.pid === undefined ? {} : { pid: info.pid }), + } + return { content: result.content, metadata } +} /** * Watches a command taken over from a previous server as a background job of its session, and notifies the diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts index b6ed23ff34ed..73e1500d22d3 100644 --- a/packages/core/test/shell-daemon.test.ts +++ b/packages/core/test/shell-daemon.test.ts @@ -126,7 +126,7 @@ describe("Shell on the process daemon", () => { expect(yield* shell.result(failed)).toMatchObject({ info: { status: "exited", exit: 1 }, capture: { - output: `kept\n\n\n[output file ${failed.file} stopped early: No space left on device]`, + output: `kept\n\n[output file ${failed.file} stopped early: No space left on device]`, }, }) From 360a201e421f431586c8cafebbc832198e3a15bc Mon Sep 17 00:00:00 2001 From: James Long Date: Tue, 10 Nov 2026 20:54:02 +0000 Subject: [PATCH 12/12] test(core): skip daemon shell cases on Windows Shell commands only run in the process daemon off Windows (pipe mode is Unix-only), so these cases wait on a daemon that Windows never uses. --- packages/core/test/session-execution.test.ts | 11 +++++++---- packages/core/test/shell-restart.test.ts | 3 ++- packages/core/test/tool-shell.test.ts | 12 +++++++----- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/packages/core/test/session-execution.test.ts b/packages/core/test/session-execution.test.ts index 8bb91bc4b628..9569aa371373 100644 --- a/packages/core/test/session-execution.test.ts +++ b/packages/core/test/session-execution.test.ts @@ -402,6 +402,9 @@ describe("SessionExecution lifecycle", () => { ) }) +// Shell commands only run in the process daemon off Windows (pipe mode is Unix-only), so the daemon shell cases skip there. +const daemonLive = process.platform === "win32" ? it.live.skip : it.live + describe("SessionRestart background recovery", () => { it.effect("keeps shell owners idle until a user prompt delivers recovered notices exactly once", () => Effect.gen(function* () { @@ -491,7 +494,7 @@ describe("SessionRestart background recovery", () => { }), ) - it.live("watches shell jobs whose commands survived the restart", () => + daemonLive("watches shell jobs whose commands survived the restart", () => Effect.gen(function* () { const database = yield* Database.Service const store = yield* SessionStore.Service @@ -553,7 +556,7 @@ describe("SessionRestart background recovery", () => { }), ) - it.live("settles a shell call a killed server left running as moved to the background", () => + daemonLive("settles a shell call a killed server left running as moved to the background", () => Effect.gen(function* () { const database = yield* Database.Service const store = yield* SessionStore.Service @@ -640,7 +643,7 @@ describe("SessionRestart background recovery", () => { ) for (const exited of [false, true]) - it.live( + daemonLive( `reattaches once a ${exited ? "finished" : "running"} shell a killed server backgrounded without settling its call`, () => Effect.gen(function* () { @@ -701,7 +704,7 @@ describe("SessionRestart background recovery", () => { }), ) - it.live("settles a recovered child's unsettled shell call before the child resumes", () => + daemonLive("settles a recovered child's unsettled shell call before the child resumes", () => Effect.gen(function* () { const database = yield* Database.Service const store = yield* SessionStore.Service diff --git a/packages/core/test/shell-restart.test.ts b/packages/core/test/shell-restart.test.ts index 491b13f3d6c3..604371cfb18b 100644 --- a/packages/core/test/shell-restart.test.ts +++ b/packages/core/test/shell-restart.test.ts @@ -115,7 +115,8 @@ const until = (check: () => boolean) => while (!check()) yield* Effect.sleep(5) }) -describe("shell commands across a graceful restart", () => { +// Shell commands only run in the process daemon off Windows (pipe mode is Unix-only). +describe.skipIf(process.platform === "win32")("shell commands across a graceful restart", () => { for (const finished of [false, true]) it.live( `settles a foreground call ${finished ? "whose command finished during the restart" : "whose command still runs"} and resumes its session`, diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index a2ee52545aff..623823b73445 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -231,6 +231,8 @@ const call = (input: typeof ShellTool.Input.Type, id = "call-shell") => ({ }) const isWindows = process.platform === "win32" +// Shell commands only run in the process daemon off Windows (pipe mode is Unix-only), so these cases have no daemon there. +const daemonLive = isWindows ? fakeDaemonIt.live.skip : fakeDaemonIt.live // The process daemon keeps a finished command's zombie until the command is forgotten. const running = (pid: number) => { @@ -1803,7 +1805,7 @@ describe("ShellTool", () => { }), ) - fakeDaemonIt.live("keeps a background daemon command's job across a Location reload and reports how it ended", () => + daemonLive("keeps a background daemon command's job across a Location reload and reports how it ended", () => withTempDir((tmp) => Effect.gen(function* () { reset() @@ -2336,7 +2338,7 @@ describe("ShellTool stopped", () => { } }) - fakeDaemonIt.live("stops a command the user stopped while it was being created", () => + daemonLive("stops a command the user stopped while it was being created", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => @@ -2361,7 +2363,7 @@ describe("ShellTool stopped", () => { }), ) - fakeDaemonIt.live("stops a command the user stopped after its Location reloaded", () => + daemonLive("stops a command the user stopped after its Location reloaded", () => withTempDir((tmp) => Effect.gen(function* () { reset() @@ -2400,7 +2402,7 @@ describe("ShellTool stopped", () => { for (const handoff of [false, true]) for (const starting of [true, false]) - fakeDaemonIt.live( + daemonLive( `${handoff ? "leaves a command running" : "stops a command"} when an interrupt ${handoff ? "during a handoff " : ""}cuts its call off while it ${starting ? "starts" : "runs"}`, () => withTempDir((tmp) => { @@ -2445,7 +2447,7 @@ describe("ShellTool stopped", () => { }), ) - fakeDaemonIt.live("records which call waits on a daemon command, and keeps no timeout for a restart", () => + daemonLive("records which call waits on a daemon command, and keeps no timeout for a restart", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) =>