From 203505ac605799fadfa542e0933a23a7e2288a16 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 00:09:25 +0000 Subject: [PATCH 01/29] feat(core): speak opencode-pty protocol v8 and add pipe process API - Protocol 8: ProcessInfo with pty/pipe groups, create_pty, get, list { mode }, required features; drop migrate(), the features fallback and the rename feature check (UnsupportedError). - Keep daemon request errors (kind "request") apart from connect/startup failures; PersistentPty.get uses get { id } and maps unknown ids and pipe processes to NotFoundError. - daemon.wait on its own connection with no timeout; interrupting closes it. - Pipe process API for Shell: createPipe, waitPipe, getPipe, signalPipe, setPipeMetadata, terminatePipe, listPipes. - Fake daemon speaks v8 including pipe requests. --- packages/core/src/persistent-pty/daemon.ts | 141 ++++++---- packages/core/src/persistent-pty/index.ts | 250 +++++++++++++++--- .../test/fixture/persistent-pty-daemon.ts | 176 +++++++++--- .../core/test/persistent-pty-daemon.test.ts | 145 ++++++---- .../core/test/persistent-pty-session.test.ts | 101 ++++++- .../server/src/handlers/persistent-pty.ts | 15 +- packages/server/test/persistent-pty.test.ts | 74 ++---- 7 files changed, 649 insertions(+), 253 deletions(-) diff --git a/packages/core/src/persistent-pty/daemon.ts b/packages/core/src/persistent-pty/daemon.ts index 526dc59c1a55..0f7866c4727c 100644 --- a/packages/core/src/persistent-pty/daemon.ts +++ b/packages/core/src/persistent-pty/daemon.ts @@ -1,36 +1,59 @@ import { spawn } from "node:child_process" -import { mkdir, readFile, rename } from "node:fs/promises" +import { readFile } from "node:fs/promises" import net from "node:net" import path from "node:path" import { Cause, Data, Duration, Effect, Schema, Semaphore } from "effect" import type { Handoff } from "@opencode/schema/persistent-pty" -import { Session } from "@opencode/schema/session" -const ProtocolVersion = 7 +const ProtocolVersion = 8 const MaxFrameBytes = 8 * 1024 * 1024 const Lifecycle = Schema.Union([ Schema.Struct({ status: Schema.Literal("running") }), - Schema.Struct({ status: Schema.Literal("exited"), exit_code: Schema.NullOr(Schema.Number) }), + Schema.Struct({ + status: Schema.Literal("exited"), + exit_code: Schema.NullOr(Schema.Number), + signal: Schema.optional(Schema.NullOr(Schema.String)), + }), Schema.Struct({ status: Schema.Literal("failed"), message: Schema.String }), ]) -export const WireTerminal = Schema.Struct({ - id: Schema.Number, - pid: Schema.NullOr(Schema.Number), +const PtyInfo = Schema.Struct({ title: Schema.String, - user_title: Schema.optional(Schema.NullOr(Schema.String)), + user_title: Schema.NullOr(Schema.String), foreground_process: Schema.NullOr(Schema.String), - group_id: Session.ID, - command: Schema.Array(Schema.String), - cwd: Schema.String, cols: Schema.Number, rows: Schema.Number, +}) + +export const Metadata = Schema.Record(Schema.String, Schema.Unknown) +export type Metadata = typeof Metadata.Type + +const PipeInfo = Schema.Struct({ + output_path: Schema.String, + output_lines: Schema.Number, + output_error: Schema.optional(Schema.NullOr(Schema.String)), + metadata: Metadata, +}) + +export const ProcessInfo = Schema.Struct({ + id: Schema.Number, + mode: Schema.Literals(["pty", "pipe"]), + pid: Schema.NullOr(Schema.Number), + command: Schema.Array(Schema.String), + cwd: Schema.String, + group_id: Schema.String, lifecycle: Lifecycle, output_head: Schema.Number, output_tail: Schema.Number, + started_at: Schema.Number, + exited_at: Schema.optional(Schema.NullOr(Schema.Number)), + pty: Schema.optional(Schema.NullOr(PtyInfo)), + pipe: Schema.optional(Schema.NullOr(PipeInfo)), }) -export type WireTerminal = typeof WireTerminal.Type +export type ProcessInfo = typeof ProcessInfo.Type + +export type Signal = "SIGTERM" | "SIGINT" | "SIGKILL" const Registration = Schema.Struct({ instance_id: Schema.String, @@ -38,7 +61,7 @@ const Registration = Schema.Struct({ protocol: Schema.Number, socket: Schema.String, token: Schema.String, - features: Schema.optional(Schema.Array(Schema.String)), + features: Schema.Array(Schema.String), }) type Registration = typeof Registration.Type @@ -48,16 +71,17 @@ export const WireResponse = Schema.Union([ instance_id: Schema.String, pid: Schema.Number, protocol: Schema.Number, - features: Schema.optional(Schema.Array(Schema.String)), + features: Schema.Array(Schema.String), }), - Schema.Struct({ type: Schema.Literal("created"), terminal: WireTerminal }), - Schema.Struct({ type: Schema.Literal("terminals"), terminals: Schema.Array(WireTerminal) }), + Schema.Struct({ type: Schema.Literal("created"), process: ProcessInfo }), + Schema.Struct({ type: Schema.Literal("processes"), processes: Schema.Array(ProcessInfo) }), + Schema.Struct({ type: Schema.Literal("process"), process: ProcessInfo }), Schema.Struct({ type: Schema.Literal("ok") }), Schema.Struct({ type: Schema.Literal("owned") }), Schema.Struct({ type: Schema.Literal("handoff"), ticket: Schema.String, expires_at: Schema.Number }), Schema.Struct({ type: Schema.Literal("snapshot"), - terminal: WireTerminal, + process: ProcessInfo, text: Schema.String, checkpoint_base64: Schema.String, cursor_x: Schema.Number, @@ -65,14 +89,14 @@ export const WireResponse = Schema.Union([ }), Schema.Struct({ type: Schema.Literal("rows"), - terminal: WireTerminal, + process: ProcessInfo, lines: Schema.Array(Schema.String), cursor_x: Schema.Number, cursor_y: Schema.Number, }), Schema.Struct({ type: Schema.Literal("attached"), - terminal: WireTerminal, + process: ProcessInfo, role: Schema.Literals(["controller", "observer"]), generation: Schema.Number, requested_offset: Schema.Number, @@ -91,7 +115,12 @@ export const WireResponse = Schema.Union([ Schema.Struct({ type: Schema.Literal("exited"), exit_code: Schema.NullOr(Schema.Number), + signal: Schema.optional(Schema.NullOr(Schema.String)), final_offset: Schema.Number, + // Pipe processes only. + output_lines: Schema.optional(Schema.Number), + output_error: Schema.optional(Schema.NullOr(Schema.String)), + exited_at: Schema.Number, }), Schema.Struct({ type: Schema.Literal("controller_changed"), @@ -103,6 +132,7 @@ export const WireResponse = Schema.Union([ Schema.Struct({ type: Schema.Literal("error"), message: Schema.String }), ]) export type WireResponse = typeof WireResponse.Type +export type Exited = Extract export type Role = "controller" | "observer" @@ -130,7 +160,7 @@ export type SubscribeInput = { } export type DaemonAttachment = { - readonly terminal: WireTerminal + readonly terminal: ProcessInfo readonly role: Role readonly generation: number readonly replay: { @@ -144,8 +174,13 @@ export type DaemonAttachment = { readonly detach: () => void } +/** + * `connect`, `spawn`, `registration` and `protocol` mean the daemon could not be reached or started and the + * request never ran. `response` means the request was sent but its reply was lost. `request` is the daemon's + * own `error` reply. + */ export class DaemonError extends Data.TaggedError("PersistentPty.DaemonError")<{ - readonly kind: "connect" | "response" | "registration" | "protocol" | "spawn" + readonly kind: "connect" | "response" | "registration" | "protocol" | "spawn" | "request" readonly message: string readonly pid?: number }> {} @@ -153,7 +188,8 @@ export class DaemonError extends Data.TaggedError("PersistentPty.DaemonError")<{ export interface DaemonTransport { readonly request: (value: object, start?: boolean) => Effect.Effect readonly requestIfRunning: (value: object) => Effect.Effect - readonly supports: (feature: string) => Effect.Effect + /** Waits on its own connection, with no timeout, until the pipe process exits. Never starts the daemon. */ + readonly wait: (id: number) => Effect.Effect readonly shutdown: Effect.Effect readonly handoff: Effect.Effect readonly subscribe: ( @@ -229,8 +265,7 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" response.protocol !== ProtocolVersion ) return yield* Effect.fail(new DaemonError({ kind: "protocol", message: "opencode-pty registration mismatch" })) - // Daemons older than the features list omit it; trust the live daemon's answer over the file. - return { ...decoded, features: response.features ?? [] } + return decoded }) const spawnDaemon = Effect.fn("PersistentPty.daemon.spawn")(function* () { @@ -351,9 +386,14 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" Effect.catch((error) => (error.kind === "connect" ? Effect.succeed(undefined) : Effect.fail(error))), ) - const supports = Effect.fn("PersistentPty.daemon.supports")(function* (feature: string) { - const current = yield* connect(false) - return current.features?.includes(feature) ?? false + const wait = Effect.fn("PersistentPty.daemon.wait")(function* (processID: number) { + const attempt = withRegistration(false, (current) => + Effect.tryPromise({ + try: (signal) => waitPromise(current, processID, signal), + catch: (cause) => (cause instanceof DaemonError ? cause : failure("connect", cause)), + }), + ) + return yield* attempt.pipe(Effect.catch((error) => (error.kind === "registration" ? attempt : Effect.fail(error)))) }) const shutdown = Effect.gen(function* () { @@ -418,13 +458,10 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" yield* Effect.gen(function* () { if (inherited.expiresAt <= Date.now()) return yield* Effect.fail(new DaemonError({ kind: "registration", message: "PTY restart handoff expired" })) - // Older releases put the daemon directory outside the root; its directory name is still the ID. id = path.basename(inherited.directory) found = yield* discover(inherited.directory) if (found.instance_id !== inherited.instanceID) return yield* Effect.fail(new DaemonError({ kind: "registration", message: "PTY restart daemon changed" })) - if (path.resolve(inherited.directory) !== path.resolve(root, id)) - yield* migrate(inherited.directory, path.join(root, id)) yield* claim(found, inherited.ticket) }).pipe( Effect.catchCause((cause) => @@ -440,20 +477,7 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" ) } - return { request, requestIfRunning, supports, shutdown, handoff, subscribe } satisfies DaemonTransport -}) - -// Older releases kept registrations in purgeable temp directories. Renaming keeps the lock file's -// inode, so the still-running daemon keeps holding its lock in the new location. -const migrate = Effect.fn("PersistentPty.daemon.migrate")(function* (directory: string, target: string) { - yield* Effect.tryPromise({ - try: async () => { - await mkdir(target, { recursive: true, mode: 0o700 }) - await rename(path.join(directory, "service.lock"), path.join(target, "service.lock")) - await rename(path.join(directory, "service.json"), path.join(target, "service.json")) - }, - catch: (cause) => failure("registration", cause), - }) + return { request, requestIfRunning, wait, shutdown, handoff, subscribe } satisfies DaemonTransport }) async function openOwner(registration: Registration, ticket: string | undefined, signal: AbortSignal) { @@ -535,7 +559,7 @@ const oneShot = Effect.fn("PersistentPty.daemon.oneShot")(function* (registratio }) const response = yield* Effect.try({ try: () => decode(first), - catch: (cause) => failure("protocol", cause), + catch: (cause) => failure("response", cause), }) if (response.type === "error") return yield* Effect.fail(responseError(response.message)) return response @@ -632,7 +656,7 @@ async function subscribePromise( } let activated = false return { - terminal: response.terminal, + terminal: response.process, role: response.role, generation: response.generation, replay: { @@ -659,6 +683,29 @@ async function subscribePromise( } } +async function waitPromise(registration: Registration, id: number, signal: AbortSignal): Promise { + const socket = net.createConnection({ path: registration.socket }) + const abort = () => socket.destroy() + signal.addEventListener("abort", abort, { once: true }) + try { + await new Promise((resolve, reject) => { + socket.once("connect", resolve) + socket.once("error", reject) + }) + socket.write(encode({ token: registration.token, request: { op: "wait", id } })) + const frame = await decoder(socket).next() + // The connection was open, so a missing reply means the daemon went away while the process ran. + if (frame.done) throw failure("connect", "opencode-pty closed before the process exited") + const response = decode(frame.value) + if (response.type === "error") throw responseError(response.message) + if (response.type !== "exited") throw failure("response", `unexpected opencode-pty response: ${response.type}`) + return response + } finally { + signal.removeEventListener("abort", abort) + socket.destroy() + } +} + function encode(value: unknown) { const payload = Buffer.from(JSON.stringify(value)) if (payload.length > MaxFrameBytes) throw new Error("opencode-pty frame too large") @@ -691,7 +738,7 @@ function decode(payload: Uint8Array) { } function responseError(message: string) { - return new DaemonError({ kind: message === "authentication failed" ? "registration" : "protocol", message }) + return new DaemonError({ kind: message === "authentication failed" ? "registration" : "request", message }) } function failure(kind: DaemonError["kind"], cause: unknown) { diff --git a/packages/core/src/persistent-pty/index.ts b/packages/core/src/persistent-pty/index.ts index 1728996636db..368b65d570a1 100644 --- a/packages/core/src/persistent-pty/index.ts +++ b/packages/core/src/persistent-pty/index.ts @@ -22,15 +22,19 @@ import { ShellSelect } from "../shell/select.js" import { makeDaemonTransport, type DaemonAttachment, + type DaemonError, type DaemonTransport, + type Exited, + type Metadata, + type ProcessInfo, type Role, + type Signal, type SubscribeInput, type WireResponse, - type WireTerminal, } from "./daemon.js" import { resolveBinary } from "#persistent-pty-binary" -export type { Role, StreamEvent } from "./daemon.js" +export type { Metadata, Role, Signal, StreamEvent } from "./daemon.js" export { Handoff } from "@opencode/schema/persistent-pty" export { available } from "#persistent-pty-binary" @@ -54,22 +58,73 @@ export type Attachment = { readonly detach: () => void } +/** The daemon could not be reached or started; the request never ran. */ export class UnavailableError extends Schema.TaggedError()("PersistentPty.UnavailableError", { message: Schema.String, }) {} +/** The daemon rejected the request, or its reply was lost after it was sent, so it may have run. */ +export class RequestError extends Schema.TaggedError()("PersistentPty.RequestError", { + message: Schema.String, +}) {} + export class NotFoundError extends Schema.TaggedError()("PersistentPty.NotFoundError", { ptyID: Pty.ID, }) {} +export class ProcessNotFoundError extends Schema.TaggedError()( + "PersistentPty.ProcessNotFoundError", + { processID: Schema.Number }, +) {} + export class InvalidCwdError extends Schema.TaggedError()("PersistentPty.InvalidCwdError", { cwd: Schema.String, message: Schema.String, }) {} -export class UnsupportedError extends Schema.TaggedError()("PersistentPty.UnsupportedError", { - feature: Schema.String, -}) {} +/** A non-PTY process run by the daemon, its output written to `outputPath`. */ +export type PipeInfo = { + readonly id: number + readonly pid: number | undefined + readonly command: readonly string[] + readonly cwd: string + readonly groupID: string + readonly lifecycle: + | { readonly status: "running" } + | { readonly status: "exited"; readonly exitCode: number | undefined; readonly signal: string | undefined } + | { readonly status: "failed"; readonly message: string } + /** Bytes counted so far; the file can hold a little more. Pair only bytes up to here with `outputLines`. */ + readonly outputTail: number + readonly outputLines: number + readonly outputPath: string + readonly outputError: string | undefined + readonly metadata: Metadata + readonly startedAt: number + readonly exitedAt: number | undefined +} + +export type PipeExit = { + readonly exitCode: number | undefined + readonly signal: string | undefined + /** Runs past the file's size when `outputError` is set; use the file's real size instead. */ + readonly finalOffset: number + readonly outputLines: number + readonly outputError: string | undefined + readonly exitedAt: number +} + +export type CreatePipeInput = { + readonly program: string + readonly args: readonly string[] + /** Absolute. */ + readonly cwd: string + /** The child's whole environment; the daemon's own is not inherited. */ + readonly env: Readonly> + readonly groupID: string + /** Absolute; truncated on open, symlinks refused. */ + readonly outputPath: string + readonly metadata: Metadata +} export interface Interface { readonly list: (sessionID?: Session.ID) => Effect.Effect @@ -110,10 +165,7 @@ export interface Interface { rows: number, data: Uint8Array, ) => Effect.Effect - readonly rename: ( - id: Pty.ID, - title: string | null, - ) => Effect.Effect + readonly rename: (id: Pty.ID, title: string | null) => Effect.Effect readonly snapshot: (id: Pty.ID) => Effect.Effect readonly read: (sessionID: Session.ID, lines?: number) => Effect.Effect readonly remove: (id: Pty.ID) => Effect.Effect @@ -123,6 +175,20 @@ export interface Interface { id: Pty.ID, input: SubscribeInput, ) => Effect.Effect + /** Starts the daemon if needed. Fall back to another spawner only on `UnavailableError`. */ + readonly createPipe: (input: CreatePipeInput) => Effect.Effect + /** Resolves when the process exits, on its own connection with no timeout; interrupting closes it. */ + readonly waitPipe: (id: number) => Effect.Effect + /** `ProcessNotFoundError` for an unknown id, a PTY, or when no daemon is running. */ + readonly getPipe: (id: number) => Effect.Effect + /** Signals the process group without forgetting the process; `waitPipe` then reports the signal. */ + readonly signalPipe: (id: number, signal: Signal) => Effect.Effect + /** Replaces the stored metadata. */ + readonly setPipeMetadata: (id: number, metadata: Metadata) => Effect.Effect + /** Kills the process if it still runs and forgets it; the output file is left alone. */ + readonly terminatePipe: (id: number) => Effect.Effect + /** Never starts the daemon; empty when none is running. */ + readonly listPipes: () => Effect.Effect } export class Service extends Context.Service()("@opencode/PersistentPty") {} @@ -151,18 +217,18 @@ const makeLayer = (options: Options = {}) => const current = new Map() const list = Effect.fn("PersistentPty.list")(function* (sessionID?: Session.ID) { - const response = yield* optionalRequest(daemon, { op: "list" }) + const response = yield* optionalRequest(daemon, { op: "list", mode: "pty" }) if (!response) return [] - if (response.type !== "terminals") return yield* unexpected(response) - return response.terminals + if (response.type !== "processes") return yield* unexpected(response) + return response.processes .map(toInfo) .filter((terminal) => sessionID === undefined || terminal.sessionID === sessionID) }) const get = Effect.fn("PersistentPty.get")(function* (id: Pty.ID) { - const found = (yield* list()).find((terminal) => terminal.id === id) - if (!found) return yield* new NotFoundError({ ptyID: id }) - return found + const found = yield* lookup(daemon, fromID(id)).pipe(Effect.mapError(unavailable)) + if (found?.mode !== "pty") return yield* new NotFoundError({ ptyID: id }) + return toInfo(found) }) const create = Effect.fn("PersistentPty.create")(function* ( @@ -181,7 +247,7 @@ const makeLayer = (options: Options = {}) => const response = yield* request( daemon, { - op: "create", + op: "create_pty", program: input.command ?? ShellSelect.environment(global.bin), args: input.args, cwd, @@ -194,7 +260,7 @@ const makeLayer = (options: Options = {}) => true, ) if (response.type !== "created") return yield* unexpected(response) - const terminal = toInfo(response.terminal) + const terminal = toInfo(response.process) yield* bus.publish(Added, { sessionID, terminal }) return terminal }) @@ -238,9 +304,6 @@ const makeLayer = (options: Options = {}) => const rename = Effect.fn("PersistentPty.rename")(function* (id: Pty.ID, title: string | null) { const terminal = yield* get(id) - // Daemons without the feature drop the connection on unknown requests. - if (!(yield* daemon.supports("rename").pipe(Effect.mapError(unavailable)))) - return yield* new UnsupportedError({ feature: "rename" }) const response = yield* request(daemon, { op: "rename", id: fromID(id), title }) if (response.type !== "ok") return yield* unexpected(response) const renamed = yield* get(id) @@ -253,7 +316,7 @@ const makeLayer = (options: Options = {}) => const response = yield* request(daemon, { op: "snapshot", id: fromID(id) }) if (response.type !== "snapshot") return yield* unexpected(response) return { - info: toInfo(response.terminal), + info: toInfo(response.process), text: response.text, checkpoint: Buffer.from(response.checkpoint_base64, "base64"), cursor: { x: response.cursor_x, y: response.cursor_y }, @@ -273,7 +336,7 @@ const makeLayer = (options: Options = {}) => // Let the daemon choose the live height in the same snapshot when lines is omitted. const response = yield* request(daemon, { op: "read_rows", id: fromID(id), rows: lines }) if (response.type !== "rows") return yield* unexpected(response) - const info = toInfo(response.terminal) + const info = toInfo(response.process) return { ptyID: info.id, title: info.title, @@ -367,9 +430,10 @@ const makeLayer = (options: Options = {}) => }, }) .pipe(Effect.mapError(unavailable)) - if (attachment.role === "controller") current.set(attachment.terminal.group_id, id) + const info = toInfo(attachment.terminal) + if (attachment.role === "controller") current.set(info.sessionID, id) return { - info: toInfo(attachment.terminal), + info, role: attachment.role, generation: attachment.generation, replay: attachment.replay, @@ -378,6 +442,61 @@ const makeLayer = (options: Options = {}) => } }) + const createPipe = Effect.fn("PersistentPty.createPipe")(function* (input: CreatePipeInput) { + const response = yield* daemon + .request( + { + op: "create_pipe", + program: input.program, + args: input.args, + cwd: input.cwd, + env: input.env, + group_id: input.groupID, + output_path: input.outputPath, + metadata: input.metadata, + }, + true, + ) + .pipe(Effect.mapError(pipeError)) + if (response.type !== "created") return yield* unexpectedPipe(response) + return toPipeInfo(response.process) + }) + + const waitPipe = Effect.fn("PersistentPty.waitPipe")(function* (id: number) { + return toPipeExit(yield* daemon.wait(id).pipe(Effect.mapError(pipeError))) + }) + + const getPipe = Effect.fn("PersistentPty.getPipe")(function* (id: number) { + const found = yield* lookup(daemon, id).pipe(Effect.mapError(unavailable)) + if (found?.mode !== "pipe") return yield* new ProcessNotFoundError({ processID: id }) + return toPipeInfo(found) + }) + + const pipeCommand = (value: object) => + Effect.gen(function* () { + const response = yield* daemon.request(value).pipe(Effect.mapError(pipeError)) + if (response.type !== "ok") return yield* unexpectedPipe(response) + }) + + const signalPipe = Effect.fn("PersistentPty.signalPipe")((id: number, signal: Signal) => + pipeCommand({ op: "signal", id, signal }), + ) + + const setPipeMetadata = Effect.fn("PersistentPty.setPipeMetadata")((id: number, metadata: Metadata) => + pipeCommand({ op: "set_metadata", id, metadata }), + ) + + const terminatePipe = Effect.fn("PersistentPty.terminatePipe")((id: number) => + pipeCommand({ op: "terminate", id }), + ) + + const listPipes = Effect.fn("PersistentPty.listPipes")(function* () { + const response = yield* daemon.requestIfRunning({ op: "list", mode: "pipe" }).pipe(Effect.mapError(pipeError)) + if (!response) return [] + if (response.type !== "processes") return yield* unexpectedPipe(response) + return response.processes.map(toPipeInfo) + }) + return Service.of({ list, get, @@ -393,6 +512,13 @@ const makeLayer = (options: Options = {}) => shutdown, handoff, attach, + createPipe, + waitPipe, + getPipe, + signalPipe, + setPipeMetadata, + terminatePipe, + listPipes, }) }), ) @@ -412,33 +538,95 @@ const request = (daemon: DaemonTransport, value: object, start = false) => const optionalRequest = (daemon: DaemonTransport, value: object) => daemon.requestIfRunning(value).pipe(Effect.mapError(unavailable)) +// `get` fails only for an unknown id, and no running daemon has no processes. +const lookup = (daemon: DaemonTransport, id: number) => + daemon.requestIfRunning({ op: "get", id }).pipe( + Effect.catchIf( + (error) => error.kind === "request", + () => Effect.succeed(undefined), + ), + Effect.flatMap((response) => { + if (!response) return Effect.succeed(undefined) + if (response.type !== "process") return Effect.fail(new Error(`unexpected opencode-pty response: ${response.type}`)) + return Effect.succeed(response.process) + }), + ) + const unexpected = (response: WireResponse) => Effect.fail(new UnavailableError({ message: `unexpected opencode-pty response: ${response.type}` })) const unavailable = (error: unknown) => new UnavailableError({ message: error instanceof Error ? error.message : String(error) }) -function toInfo(value: WireTerminal): Info { +// Only a failure before the request was sent is safe to retry somewhere else. +const pipeError = (error: DaemonError) => + error.kind === "request" || error.kind === "response" + ? new RequestError({ message: error.message }) + : new UnavailableError({ message: error.message }) + +const unexpectedPipe = (response: WireResponse) => + Effect.fail(new RequestError({ message: `unexpected opencode-pty response: ${response.type}` })) + +function toInfo(value: ProcessInfo): Info { + if (!value.pty) throw new Error(`opencode-pty process ${value.id} is not a terminal`) const status = value.lifecycle.status return { ...Pty.Info.make({ id: toID(value.id), - title: value.title, + title: value.pty.title, command: value.command[0] || "", args: value.command.slice(1), cwd: value.cwd, status: status === "running" ? "running" : "exited", pid: value.pid ?? 0, - ...(status === "exited" ? { exitCode: value.lifecycle.exit_code ?? undefined } : {}), + ...(value.lifecycle.status === "exited" ? { exitCode: value.lifecycle.exit_code ?? undefined } : {}), }), - sessionID: value.group_id, - userTitle: value.user_title ?? null, - foregroundProcess: value.foreground_process, - size: { cols: value.cols, rows: value.rows }, + // Terminals are always created with a session ID as their group. + sessionID: Session.ID.make(value.group_id), + userTitle: value.pty.user_title, + foregroundProcess: value.pty.foreground_process, + size: { cols: value.pty.cols, rows: value.pty.rows }, output: { head: value.output_head, tail: value.output_tail }, } } +function toPipeInfo(value: ProcessInfo): PipeInfo { + if (!value.pipe) throw new Error(`opencode-pty process ${value.id} is not a pipe process`) + return { + id: value.id, + pid: value.pid ?? undefined, + command: value.command, + cwd: value.cwd, + groupID: value.group_id, + lifecycle: + value.lifecycle.status === "exited" + ? { + status: "exited", + exitCode: value.lifecycle.exit_code ?? undefined, + signal: value.lifecycle.signal ?? undefined, + } + : value.lifecycle, + outputTail: value.output_tail, + outputLines: value.pipe.output_lines, + outputPath: value.pipe.output_path, + outputError: value.pipe.output_error ?? undefined, + metadata: value.pipe.metadata, + startedAt: value.started_at, + exitedAt: value.exited_at ?? undefined, + } +} + +function toPipeExit(value: Exited): PipeExit { + return { + exitCode: value.exit_code ?? undefined, + signal: value.signal ?? undefined, + finalOffset: value.final_offset, + outputLines: value.output_lines ?? 0, + outputError: value.output_error ?? undefined, + exitedAt: value.exited_at, + } +} + function toID(value: number) { return Pty.ID.make(`pty_persistent_${value}`) } diff --git a/packages/core/test/fixture/persistent-pty-daemon.ts b/packages/core/test/fixture/persistent-pty-daemon.ts index c4a5e57794dc..ff5980fb882f 100644 --- a/packages/core/test/fixture/persistent-pty-daemon.ts +++ b/packages/core/test/fixture/persistent-pty-daemon.ts @@ -4,7 +4,7 @@ import os from "node:os" import path from "node:path" import { Effect } from "effect" -export const pong = { type: "pong", instance_id: "test", pid: process.pid, protocol: 7 } +export const pong = { type: "pong", instance_id: "test", pid: process.pid, protocol: 8, features: [] } export function temporaryDirectory() { return Effect.acquireRelease( @@ -81,7 +81,7 @@ function readRequest(socket: net.Socket) { }) } -function frame(value: object) { +export function frame(value: object) { const payload = Buffer.from(JSON.stringify(value)) const output = Buffer.allocUnsafe(payload.length + 4) output.writeUInt32BE(payload.length) @@ -89,70 +89,126 @@ function frame(value: object) { return output } -export function writeRegistration( - directory: string, - socket: string, - instance = "test", - token = "test", - features?: string[], -) { +export function writeRegistration(directory: string, socket: string, instance = "test", token = "test") { return writeFile( path.join(directory, "service.json"), - JSON.stringify({ instance_id: instance, pid: process.pid, protocol: 7, socket, token, features }), + JSON.stringify({ instance_id: instance, pid: process.pid, protocol: 8, socket, token, features: [] }), ) } -// An in-process opencode-pty that a server adopts through a handoff, so no daemon is spawned. Without -// features it answers like a daemon that predates the features list. -export const fakeDaemon = Effect.fn(function* (features?: string[]) { +type Process = { + id: number + mode: "pty" | "pipe" + pid: number | null + command: string[] + cwd: string + group_id: string + lifecycle: { status: "running" | "exited"; exit_code?: number | null; signal?: string } + output_head: number + output_tail: number + started_at: number + exited_at: number | null + pty?: { title: string; user_title: string | null; foreground_process: string | null; cols: number; rows: number } + pipe?: { output_path: string; output_lines: number; metadata: Record } +} + +// An in-process opencode-pty that a server adopts through a handoff, so no daemon is spawned. Pipe processes +// run nothing: they stay running until `exit` or a signal ends them. +export const fakeDaemon = Effect.fn(function* () { const base = yield* temporaryDirectory() const root = path.join(base, "root") const directory = path.join(root, "inherited") const socket = path.join(base, "daemon.sock") - const terminals: Array & { user_title?: string | null }> = [] + const processes: Process[] = [] + const waiters = new Map() const failTerminate = new Set() const ops: string[] = [] + const requests: Record[] = [] let nextID = 1 + + const find = (id: unknown) => processes.find((item) => item.id === id) + const exit = (id: number, exitCode: number | null, signal?: string) => { + const found = find(id) + if (!found || found.lifecycle.status !== "running") return + found.lifecycle = { status: "exited", exit_code: exitCode, signal } + found.exited_at = Date.now() + const reply = exited(found) + waiters.get(id)?.forEach((connection) => connection.end(frame(reply))) + waiters.delete(id) + } + yield* listen(socket, (connection, request) => { ops.push(String(request.op)) - if (request.op === "ping") return features ? { ...pong, features } : pong - if (request.op === "list") return { type: "terminals", terminals } - if (request.op === "create") { - const base = terminal(nextID++, String(request.group_id)) - // A daemon that advertises features always serializes user_title. - const created = features ? { ...base, user_title: null } : base - terminals.push(created) - return { type: "created", terminal: created } + requests.push(request) + if (request.op === "ping") return pong + if (request.op === "list") + return { type: "processes", processes: processes.filter((item) => item.mode === request.mode) } + if (request.op === "get") { + const found = find(request.id) + if (!found) return { type: "error", message: `terminal ${String(request.id)} not found` } + return { type: "process", process: found } + } + if (request.op === "create_pty") { + const created = terminal(nextID++, String(request.group_id)) + processes.push(created) + return { type: "created", process: created } + } + if (request.op === "create_pipe") { + if (typeof request.cwd !== "string" || !path.isAbsolute(request.cwd)) + return { type: "error", message: "cwd must be absolute" } + const created = pipe(nextID++, request) + processes.push(created) + return { type: "created", process: created } } if (request.op === "control") return { type: "ok" } - if (request.op === "terminate") { - if (failTerminate.has(`pty_persistent_${request.id}`)) return { type: "error", message: "terminate failed" } - const index = terminals.findIndex((item) => item.id === request.id) - if (index === -1) return { type: "error", message: `terminal not found: ${String(request.id)}` } - terminals.splice(index, 1) + if (request.op === "rename") { + const found = find(request.id) + if (!found?.pty) return { type: "error", message: "terminal not found" } + found.pty.user_title = typeof request.title === "string" && request.title !== "" ? request.title : null + found.pty.title = found.pty.user_title ?? "test" return { type: "ok" } } - if (request.op === "rename" && features?.includes("rename")) { - const found = terminals.find((item) => item.id === request.id) - if (!found) return { type: "error", message: "terminal not found" } - found.user_title = typeof request.title === "string" && request.title !== "" ? request.title : null - found.title = found.user_title ?? "test" + if (request.op === "wait") { + const found = find(request.id) + if (!found?.pipe) return { type: "error", message: `terminal ${String(request.id)} not found` } + if (found.lifecycle.status !== "running") return exited(found) + waiters.set(found.id, [...(waiters.get(found.id) ?? []), connection]) + return undefined + } + if (request.op === "signal") { + const found = find(request.id) + if (!found?.pipe) return { type: "error", message: `terminal ${String(request.id)} not found` } + exit(found.id, null, String(request.signal)) return { type: "ok" } } - // Old daemons cannot parse unknown requests and drop the connection. - if (request.op === "rename") { - connection.destroy() - return undefined + if (request.op === "set_metadata") { + const found = find(request.id) + if (!found?.pipe || !isRecord(request.metadata)) + return { type: "error", message: `terminal ${String(request.id)} not found` } + found.pipe.metadata = request.metadata + return { type: "ok" } } - return { type: "error", message: `unexpected request: ${String(request.op)}` } + if (request.op === "terminate") { + if (failTerminate.has(`pty_persistent_${request.id}`)) return { type: "error", message: "terminate failed" } + const index = processes.findIndex((item) => item.id === request.id) + if (index === -1) return { type: "error", message: `terminal ${String(request.id)} not found` } + const [removed] = processes.splice(index, 1) + waiters.get(removed.id)?.forEach((waiter) => waiter.end(frame({ type: "error", message: "terminated" }))) + waiters.delete(removed.id) + return { type: "ok" } + } + return { type: "error", message: `unsupported request: ${String(request.op)}` } }) yield* Effect.promise(async () => { await mkdir(directory, { recursive: true }) - await writeRegistration(directory, socket, "test", "test", features) + await writeRegistration(directory, socket) }) return { failTerminate, ops, + requests, + exit, + waiters: (id: number) => waiters.get(id)?.length ?? 0, options: { root, handoff: { directory, instanceID: "test", ticket: "ticket", expiresAt: Date.now() + 60_000 }, @@ -160,20 +216,52 @@ export const fakeDaemon = Effect.fn(function* (features?: string[]) { } }) -export function terminal(id: number, groupID = "ses_test") { +export function terminal(id: number, groupID = "ses_test"): Process { return { id, + mode: "pty", pid: null, - title: "test", - foreground_process: null, - group_id: groupID, command: ["test"], cwd: "/tmp", - cols: 80, - rows: 24, + group_id: groupID, lifecycle: { status: "running" }, output_head: 0, output_tail: 0, + started_at: Date.now(), + exited_at: null, + pty: { title: "test", user_title: null, foreground_process: null, cols: 80, rows: 24 }, + } +} + +function pipe(id: number, request: Record): Process { + return { + id, + mode: "pipe", + pid: 1_000 + id, + command: [String(request.program), ...(Array.isArray(request.args) ? request.args.map(String) : [])], + cwd: String(request.cwd), + group_id: String(request.group_id), + lifecycle: { status: "running" }, + output_head: 0, + output_tail: 0, + started_at: Date.now(), + exited_at: null, + pipe: { + output_path: String(request.output_path), + output_lines: 0, + metadata: isRecord(request.metadata) ? request.metadata : {}, + }, + } +} + +function exited(process: Process) { + return { + type: "exited", + exit_code: process.lifecycle.exit_code ?? null, + signal: process.lifecycle.signal, + final_offset: process.output_tail, + output_lines: process.pipe?.output_lines, + exited_at: process.exited_at ?? Date.now(), } } diff --git a/packages/core/test/persistent-pty-daemon.test.ts b/packages/core/test/persistent-pty-daemon.test.ts index 70eb640737c6..ad223935d0ab 100644 --- a/packages/core/test/persistent-pty-daemon.test.ts +++ b/packages/core/test/persistent-pty-daemon.test.ts @@ -1,11 +1,11 @@ import { expect } from "bun:test" import { spawn } from "node:child_process" -import { existsSync } from "node:fs" -import { mkdir, stat, writeFile } from "node:fs/promises" +import { mkdir, writeFile } from "node:fs/promises" +import net from "node:net" import path from "node:path" -import { Effect } from "effect" +import { Effect, Fiber } from "effect" import { makeDaemonTransport } from "../src/persistent-pty/daemon" -import { listen, pong, temporaryDirectory, terminal, writeRegistration } from "./fixture/persistent-pty-daemon" +import { frame, listen, pong, temporaryDirectory, terminal, writeRegistration } from "./fixture/persistent-pty-daemon" import { it } from "./lib/effect" it.live("rediscovers a same-protocol daemon after its registration rotates", () => @@ -19,7 +19,7 @@ it.live("rediscovers a same-protocol daemon after its registration rotates", () if (receivedToken !== token) return { type: "error", message: "authentication failed" } if (request.op === "ping") return { ...pong, instance_id: instance } if (request.op === "create") creates++ - return request.op === "list" ? { type: "terminals", terminals: [] } : { type: "ok" } + return request.op === "list" ? { type: "processes", processes: [] } : { type: "ok" } }) yield* Effect.promise(() => writeRegistration(directory, socketPath, instance, token)) const daemon = yield* makeDaemonTransport(root, "daemon") @@ -30,7 +30,7 @@ it.live("rediscovers a same-protocol daemon after its registration rotates", () yield* Effect.promise(() => writeRegistration(directory, socketPath, instance, token)) const running = yield* daemon.requestIfRunning({ op: "list" }) - expect(running).toEqual({ type: "terminals", terminals: [] }) + expect(running).toEqual({ type: "processes", processes: [] }) token = "newest-token" instance = "newest-instance" @@ -50,11 +50,11 @@ it.live("rediscovers a rotated registration when acquiring a subscription", () = yield* listen(socketPath, (_socket, request, receivedToken) => { if (receivedToken !== token) return { type: "error", message: "authentication failed" } if (request.op === "ping") return { ...pong, instance_id: instance } - if (request.op !== "subscribe") return { type: "terminals", terminals: [] } + if (request.op !== "subscribe") return { type: "processes", processes: [] } subscriptions++ return { type: "attached", - terminal: terminal(1), + process: terminal(1), role: "observer", generation: 1, requested_offset: 0, @@ -85,30 +85,75 @@ it.live("rediscovers a rotated registration when acquiring a subscription", () = }), ) -it.live("refreshes the daemon's features when its registration rotates", () => +it.live("waits on its own connection until the process exits", () => Effect.gen(function* () { const { root, directory } = yield* temporaryRuntime() const socketPath = path.join(directory, "daemon.sock") - let token = "old-token" - let instance = "old-instance" - let features: string[] | undefined - yield* listen(socketPath, (_socket, request, receivedToken) => { - if (receivedToken !== token) return { type: "error", message: "authentication failed" } - if (request.op === "ping") return { ...pong, instance_id: instance, ...(features ? { features } : {}) } - return { type: "terminals", terminals: [] } + const waiting: net.Socket[] = [] + yield* listen(socketPath, (socket, request) => { + if (request.op === "ping") return pong + if (request.op === "wait" && request.id === 1) { + waiting.push(socket) + return undefined + } + return { type: "error", message: "terminal 2 not found" } }) - yield* Effect.promise(() => writeRegistration(directory, socketPath, instance, token)) + yield* Effect.promise(() => writeRegistration(directory, socketPath)) const daemon = yield* makeDaemonTransport(root, "daemon") - expect(yield* daemon.supports("rename")).toBeFalse() - token = "new-token" - instance = "new-instance" - features = ["rename"] - yield* Effect.promise(() => writeRegistration(directory, socketPath, instance, token, features)) - // The cached registration is replaced once a request is rejected, as rename's lookup does first. - yield* daemon.request({ op: "list" }) + const missing = yield* Effect.flip(daemon.wait(2)) + expect(missing).toMatchObject({ kind: "request", message: "terminal 2 not found" }) + + const fiber = yield* daemon.wait(1).pipe(Effect.forkScoped({ startImmediately: true })) + yield* poll(() => waiting.length === 1) + waiting[0].end( + frame({ type: "exited", exit_code: null, signal: "SIGTERM", final_offset: 3, output_lines: 1, exited_at: 7 }), + ) + expect(yield* Fiber.join(fiber)).toEqual({ + type: "exited", + exit_code: null, + signal: "SIGTERM", + final_offset: 3, + output_lines: 1, + exited_at: 7, + }) + }), +) - expect(yield* daemon.supports("rename")).toBeTrue() +it.live("closes the wait connection when the wait is interrupted", () => + Effect.gen(function* () { + const { root, directory } = yield* temporaryRuntime() + const socketPath = path.join(directory, "daemon.sock") + const waiting: net.Socket[] = [] + yield* listen(socketPath, (socket, request) => { + if (request.op === "ping") return pong + waiting.push(socket) + return undefined + }) + yield* Effect.promise(() => writeRegistration(directory, socketPath)) + const daemon = yield* makeDaemonTransport(root, "daemon") + + const fiber = yield* daemon.wait(1).pipe(Effect.forkScoped({ startImmediately: true })) + yield* poll(() => waiting.length === 1) + const closed = new Promise((resolve) => waiting[0].once("close", () => resolve())) + yield* Fiber.interrupt(fiber) + yield* Effect.promise(() => closed) + }), +) + +it.live("reports a daemon that goes away during a wait as unreachable", () => + Effect.gen(function* () { + const { root, directory } = yield* temporaryRuntime() + const socketPath = path.join(directory, "daemon.sock") + yield* listen(socketPath, (socket, request) => { + if (request.op === "ping") return pong + socket.destroy() + return undefined + }) + yield* Effect.promise(() => writeRegistration(directory, socketPath)) + const daemon = yield* makeDaemonTransport(root, "daemon") + + expect((yield* Effect.flip(daemon.wait(1))).kind).toBe("connect") }), ) @@ -119,7 +164,7 @@ it.live("retries a start-required request when connection fails before dispatch" const secondSocket = path.join(directory, "second.sock") const first = yield* listen(firstSocket, (_socket, request) => { if (request.op === "ping") return pong - return { type: "terminals", terminals: [] } + return { type: "processes", processes: [] } }) yield* Effect.promise(() => writeRegistration(directory, firstSocket)) const daemon = yield* makeDaemonTransport(root, "daemon") @@ -175,7 +220,14 @@ it.live("rejects incompatible daemons without replacing or killing them", () => yield* Effect.promise(() => writeFile( path.join(directory, "service.json"), - JSON.stringify({ instance_id: "old", pid: existing.pid, protocol: 6, socket: "/unused", token: "old" }), + JSON.stringify({ + instance_id: "old", + pid: existing.pid, + protocol: 7, + socket: "/unused", + token: "old", + features: [], + }), ), ) const daemon = yield* makeDaemonTransport(root, "daemon", () => Promise.resolve("/missing/opencode-pty")) @@ -183,7 +235,7 @@ it.live("rejects incompatible daemons without replacing or killing them", () => const optional = yield* Effect.flip(daemon.requestIfRunning({ op: "list" })) expect(optional).toMatchObject({ kind: "protocol", - message: "opencode-pty protocol mismatch: daemon=6, client=7", + message: "opencode-pty protocol mismatch: daemon=7, client=8", pid: existing.pid, }) @@ -195,36 +247,6 @@ it.live("rejects incompatible daemons without replacing or killing them", () => }), ) -it.live("moves an inherited legacy registration into the runtime root and keeps its lock file", () => - Effect.gen(function* () { - const base = yield* temporaryDirectory() - const legacy = path.join(base, "legacy", "inherited") - const socketPath = path.join(base, "daemon.sock") - yield* listen(socketPath, (_socket, request) => { - if (request.op === "ping") return pong - return { type: "terminals", terminals: [] } - }) - yield* Effect.promise(async () => { - await mkdir(legacy, { recursive: true }) - await writeFile(path.join(legacy, "service.lock"), "") - await writeRegistration(legacy, socketPath) - }) - const lock = yield* Effect.promise(() => stat(path.join(legacy, "service.lock"))) - const daemon = yield* makeDaemonTransport(path.join(base, "root"), "fresh", undefined, { - directory: legacy, - instanceID: "test", - ticket: "ticket", - expiresAt: Date.now() + 60_000, - }) - - const migrated = path.join(base, "root", "inherited") - expect(existsSync(path.join(legacy, "service.json"))).toBeFalse() - expect((yield* Effect.promise(() => stat(path.join(migrated, "service.lock")))).ino).toBe(lock.ino) - expect(existsSync(path.join(migrated, "service.json"))).toBeTrue() - expect(yield* daemon.request({ op: "list" })).toEqual({ type: "terminals", terminals: [] }) - }), -) - it.live("starts fresh instead of failing when an inherited registration is gone or expired", () => Effect.gen(function* () { const base = yield* temporaryDirectory() @@ -264,6 +286,13 @@ it.live("stops a discovered inherited daemon when adoption fails", () => }), ) +function poll(ready: () => boolean) { + return Effect.gen(function* () { + for (let attempt = 0; attempt < 200 && !ready(); attempt++) yield* Effect.sleep(10) + expect(ready()).toBeTrue() + }) +} + function temporaryRuntime() { return Effect.gen(function* () { const root = yield* temporaryDirectory() diff --git a/packages/core/test/persistent-pty-session.test.ts b/packages/core/test/persistent-pty-session.test.ts index e154ebe85b5d..54e4d0ca2816 100644 --- a/packages/core/test/persistent-pty-session.test.ts +++ b/packages/core/test/persistent-pty-session.test.ts @@ -17,6 +17,7 @@ import { SessionStore } from "@opencode/core/session/store" import { SessionEnvironment } from "@opencode/core/session/environment" import { LocationServiceMap } from "@opencode/core/location-services" import { Removed, Updated } from "@opencode/schema/persistent-pty" +import { Pty } from "@opencode/schema/pty" import { fakeDaemon } from "./fixture/persistent-pty-daemon" import { it } from "./lib/effect" import { globalProjectNode } from "./lib/project" @@ -89,7 +90,7 @@ describe("PersistentPty session cleanup", () => { describe("PersistentPty rename", () => { it.live("renames a terminal, publishes the update and clears the user title", () => Effect.gen(function* () { - const daemon = yield* fakeDaemon(["rename"]) + const daemon = yield* fakeDaemon() yield* Effect.gen(function* () { const pty = yield* PersistentPty.Service const bus = yield* Bus.Service @@ -107,31 +108,115 @@ describe("PersistentPty rename", () => { }).pipe(Effect.provide(layer(daemon))) }), ) +}) + +describe("PersistentPty pipe processes", () => { + const input = { + program: "/bin/sh", + args: ["-c", "echo hi"], + cwd: "/tmp", + env: { PATH: "/usr/bin" }, + groupID: "proj_test", + outputPath: "/tmp/sh_test.out", + metadata: { id: "sh_test", command: "echo hi" }, + } + + it.live("creates, inspects, signals and waits for a pipe process", () => + Effect.gen(function* () { + const daemon = yield* fakeDaemon() + yield* Effect.gen(function* () { + const pty = yield* PersistentPty.Service + const created = yield* pty.createPipe(input) + expect(daemon.requests.at(-1)).toEqual({ + op: "create_pipe", + program: "/bin/sh", + args: ["-c", "echo hi"], + cwd: "/tmp", + env: { PATH: "/usr/bin" }, + group_id: "proj_test", + output_path: "/tmp/sh_test.out", + metadata: { id: "sh_test", command: "echo hi" }, + }) + expect(created).toMatchObject({ + groupID: "proj_test", + outputPath: "/tmp/sh_test.out", + lifecycle: { status: "running" }, + metadata: input.metadata, + exitedAt: undefined, + }) + + yield* pty.setPipeMetadata(created.id, { id: "sh_test", status: "killed" }) + expect((yield* pty.getPipe(created.id)).metadata).toEqual({ id: "sh_test", status: "killed" }) + expect((yield* pty.listPipes()).map((item) => item.id)).toEqual([created.id]) + + const waiting = yield* pty.waitPipe(created.id).pipe(Effect.forkScoped({ startImmediately: true })) + yield* pty.signalPipe(created.id, "SIGTERM") + expect(yield* Fiber.join(waiting)).toMatchObject({ exitCode: undefined, signal: "SIGTERM", outputLines: 0 }) + // A finished process answers a later wait at once and keeps its record until it is terminated. + expect((yield* pty.waitPipe(created.id)).signal).toBe("SIGTERM") + expect((yield* pty.getPipe(created.id)).lifecycle).toEqual({ + status: "exited", + exitCode: undefined, + signal: "SIGTERM", + }) + + yield* pty.terminatePipe(created.id) + expect(yield* Effect.flip(pty.getPipe(created.id))).toEqual( + new PersistentPty.ProcessNotFoundError({ processID: created.id }), + ) + expect(yield* pty.listPipes()).toEqual([]) + }).pipe(Effect.provide(layer(daemon))) + }), + ) - it.live("refuses to rename on a daemon without the rename feature and never sends it", () => + it.live("keeps pipe processes and terminals apart", () => Effect.gen(function* () { const daemon = yield* fakeDaemon() yield* Effect.gen(function* () { const pty = yield* PersistentPty.Service const directory = yield* tmpdirScoped() - const terminal = yield* pty.create(Session.ID.make("ses_rename"), { + const terminal = yield* pty.create(Session.ID.make("ses_pipe"), { args: [], cwd: directory.path, title: "test", env: {}, }) - expect(terminal.userTitle).toBeNull() + const pipe = yield* pty.createPipe(input) - const error = yield* Effect.flip(pty.rename(terminal.id, "build")) + expect((yield* pty.list()).map((item) => item.id)).toEqual([terminal.id]) + expect((yield* pty.listPipes()).map((item) => item.id)).toEqual([pipe.id]) + const ptyID = Pty.ID.make(`pty_persistent_${pipe.id}`) + expect(yield* Effect.flip(pty.get(ptyID))).toEqual(new PersistentPty.NotFoundError({ ptyID })) + }).pipe(Effect.provide(layer(daemon))) + }), + ) - expect(error).toEqual(new PersistentPty.UnsupportedError({ feature: "rename" })) - expect(daemon.ops).not.toContain("rename") - expect((yield* pty.get(terminal.id)).title).toBe("test") + it.live("reports daemon rejections as request errors and a terminated wait as one", () => + Effect.gen(function* () { + const daemon = yield* fakeDaemon() + yield* Effect.gen(function* () { + const pty = yield* PersistentPty.Service + expect(yield* Effect.flip(pty.createPipe({ ...input, cwd: "relative" }))).toEqual( + new PersistentPty.RequestError({ message: "cwd must be absolute" }), + ) + const created = yield* pty.createPipe(input) + const waiting = yield* pty.waitPipe(created.id).pipe(Effect.forkScoped({ startImmediately: true })) + yield* poll(() => daemon.waiters(created.id) === 1) + yield* pty.terminatePipe(created.id) + expect(yield* Fiber.join(waiting).pipe(Effect.flip)).toEqual( + new PersistentPty.RequestError({ message: "terminated" }), + ) }).pipe(Effect.provide(layer(daemon))) }), ) }) +const poll = (ready: () => boolean) => + Effect.gen(function* () { + for (let attempt = 0; attempt < 200 && !ready(); attempt++) yield* Effect.sleep(10) + expect(ready()).toBeTrue() + }) + const nextUpdated = (bus: Bus.Interface) => bus .subscribe(Updated) diff --git a/packages/server/src/handlers/persistent-pty.ts b/packages/server/src/handlers/persistent-pty.ts index e861cfdecc2d..d57340829e9e 100644 --- a/packages/server/src/handlers/persistent-pty.ts +++ b/packages/server/src/handlers/persistent-pty.ts @@ -3,7 +3,6 @@ import { PtyTicket } from "@opencode/core/pty/ticket" import { ForbiddenError, InvalidRequestError, - PtyFeatureUnsupportedError, PtyNotFoundError, ServiceUnavailableError, } from "@opencode/protocol/errors" @@ -113,19 +112,7 @@ export const PersistentPtyHandler = HttpApiBuilder.group(Api, "server.experiment .resize(ctx.params.ptyID, ctx.payload.size.cols, ctx.payload.size.rows, ctx.payload.attachmentID) .pipe(mapTerminalError) if (ctx.payload.title === undefined) return { data: yield* pty.get(ctx.params.ptyID).pipe(mapTerminalError) } - return { - data: yield* pty.rename(ctx.params.ptyID, ctx.payload.title).pipe( - Effect.catchTags({ - "PersistentPty.NotFoundError": notFound, - "PersistentPty.UnsupportedError": (error) => - new PtyFeatureUnsupportedError({ - feature: error.feature, - message: "Terminal renaming will be available after OpenCode's terminal service restarts", - }), - "PersistentPty.UnavailableError": unavailable, - }), - ), - } + return { data: yield* pty.rename(ctx.params.ptyID, ctx.payload.title).pipe(mapTerminalError) } }), ) .handle( diff --git a/packages/server/test/persistent-pty.test.ts b/packages/server/test/persistent-pty.test.ts index 2be137bb61bf..d4b728543342 100644 --- a/packages/server/test/persistent-pty.test.ts +++ b/packages/server/test/persistent-pty.test.ts @@ -208,7 +208,7 @@ smoke( 20_000, ) -it.live("reports renaming as unsupported on a daemon without the rename feature", () => +it.live("renames without a feature check and hides unknown terminals", () => Effect.gen(function* () { const fixture = yield* testDirectory() const daemon = yield* fakeDaemon() @@ -222,7 +222,7 @@ it.live("reports renaming as unsupported on a daemon without the rename feature" fs: { filewatcher: false }, }) const base = HttpServer.formatAddress(server.address) - const created = yield* request(base, "POST", "/api/experimental/session/ses_old_daemon/terminal", { + const created = yield* request(base, "POST", "/api/experimental/session/ses_v8_daemon/terminal", { args: [], cwd: fixture.root, title: "created", @@ -231,28 +231,19 @@ it.live("reports renaming as unsupported on a daemon without the rename feature" const terminal = Schema.decodeUnknownSync(PersistentPty.Info)(created.data) expect(terminal.userTitle).toBeNull() - const response = yield* Effect.promise(async () => { - const response = await fetch(new URL(`/api/experimental/persistent-pty/${terminal.id}`, base), { - method: "PUT", - headers: { authorization: `Basic ${btoa("opencode:secret")}`, "content-type": "application/json" }, - body: JSON.stringify({ title: "build" }), - }) - return { status: response.status, body: await response.json() } - }) + const renamed = yield* request(base, "PUT", `/api/experimental/persistent-pty/${terminal.id}`, { title: "build" }) + expect(renamed.data).toMatchObject({ title: "build", userTitle: "build" }) + expect(daemon.ops).toContain("rename") + expect(daemon.ops).not.toContain("list") - expect(response).toEqual({ - status: 501, - body: { - _tag: "PtyFeatureUnsupportedError", - feature: "rename", - message: "Terminal renaming will be available after OpenCode's terminal service restarts", - }, - }) - expect(daemon.ops).not.toContain("rename") - expect((yield* request(base, "GET", `/api/experimental/persistent-pty/${terminal.id}`)).data).toMatchObject({ - title: "test", - userTitle: null, + const missing = yield* Effect.promise(async () => { + const response = await fetch(new URL("/api/experimental/persistent-pty/pty_persistent_99", base), { + headers: { authorization: `Basic ${btoa("opencode:secret")}` }, + }) + await response.arrayBuffer() + return response.status }) + expect(missing).toBe(404) }), ) @@ -573,12 +564,11 @@ smoke( ) smoke( - "migrates a legacy runtime directory on restart and boots fresh when the registration is gone", + "boots fresh on restart when the handed-off registration is gone", () => Effect.gen(function* () { const fixture = yield* testDirectory() const scope = yield* Scope.Scope - const legacy = path.join(fixture.root, "legacy") const options = { hostname: "127.0.0.1", port: 0, @@ -586,50 +576,32 @@ smoke( app: { version: "test-version" }, database: { path: fixture.database }, fs: { filewatcher: false }, + pty: { root: fixture.directory }, } - const sessionID = Session.ID.make("ses_persistent_pty_migration") + const sessionID = Session.ID.make("ses_persistent_pty_gone") const originalScope = yield* Scope.fork(scope) - const original = yield* ServerProcess.start({ ...options, pty: { root: legacy } }).pipe( + const original = yield* ServerProcess.start(options).pipe( Effect.provideService(Scope.Scope, originalScope), ) const base = HttpServer.formatAddress(original.address) const first = Schema.decodeUnknownSync(PersistentPty.Info)( (yield* request(base, "POST", `/api/experimental/session/${sessionID}/terminal`, { command: "/bin/sh", - args: ["-c", "stty -echo; printf before-migration; exec cat"], + args: ["-c", "stty -echo; printf before-restart; exec cat"], cwd: process.cwd(), - title: "legacy", + title: "first", env: {}, })).data, ) - expect(yield* waitForText(base, first.id, "before-migration")).toContain("before-migration") - const legacyHandoff = Schema.decodeUnknownSync(PersistentPty.Handoff)( + expect(yield* waitForText(base, first.id, "before-restart")).toContain("before-restart") + const handoff = Schema.decodeUnknownSync(PersistentPty.Handoff)( (yield* request(base, "POST", "/api/experimental/persistent-pty/handoff")).handoff, ) yield* Scope.close(originalScope, Exit.void) - - const migratedScope = yield* Scope.fork(scope) - const migrated = yield* ServerProcess.start({ - ...options, - pty: { root: fixture.directory, handoff: legacyHandoff }, - }).pipe(Effect.provideService(Scope.Scope, migratedScope)) - const migratedBase = HttpServer.formatAddress(migrated.address) - const directory = path.join(fixture.directory, path.basename(legacyHandoff.directory)) - expect(existsSync(path.join(directory, "service.json"))).toBeTrue() - expect(existsSync(path.join(legacyHandoff.directory, "service.json"))).toBeFalse() - expect( - (yield* request(migratedBase, "GET", `/api/experimental/session/${sessionID}/terminal`)).data, - ).toMatchObject([{ id: first.id, pid: first.pid }]) - - const handoff = Schema.decodeUnknownSync(PersistentPty.Handoff)( - (yield* request(migratedBase, "POST", "/api/experimental/persistent-pty/handoff")).handoff, - ) - expect(handoff.directory).toBe(directory) - yield* Scope.close(migratedScope, Exit.void) // Simulate macOS purging the registration while the daemon waits for its successor. - yield* Effect.promise(() => fs.rm(path.join(directory, "service.json"))) + yield* Effect.promise(() => fs.rm(path.join(handoff.directory, "service.json"))) - const fresh = yield* ServerProcess.start({ ...options, pty: { root: fixture.directory, handoff } }) + const fresh = yield* ServerProcess.start({ ...options, pty: { ...options.pty, handoff } }) const freshBase = HttpServer.formatAddress(fresh.address) expect((yield* request(freshBase, "GET", `/api/experimental/session/${sessionID}/terminal`)).data).toEqual([]) const second = Schema.decodeUnknownSync(PersistentPty.Info)( From 8ecfa5caeff018185a079a0482f81e7313b6c0e5 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 00:11:05 +0000 Subject: [PATCH 02/29] feat(core): track planned-shutdown detaching state --- packages/core/src/detaching.ts | 41 +++++++++++++++++++++++ packages/core/src/persistent-pty/index.ts | 11 ++++-- packages/server/src/process.ts | 12 +++++++ packages/server/src/routes.ts | 2 ++ packages/server/test/process.test.ts | 24 ++++++++++++- 5 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 packages/core/src/detaching.ts diff --git a/packages/core/src/detaching.ts b/packages/core/src/detaching.ts new file mode 100644 index 000000000000..020fa8a57335 --- /dev/null +++ b/packages/core/src/detaching.ts @@ -0,0 +1,41 @@ +export * as Detaching from "./detaching.js" + +import { Context, Effect, Layer } from "effect" +import { makeGlobalNode } from "@opencode/util/effect/app-node" + +/** + * Planned-shutdown state of this server process. + * + * A prepared daemon handoff means a new server will take over the daemon's processes. If this server + * then shuts down, it is detaching: work that can outlive it in the daemon is left running and recorded + * for the new server instead of being stopped. Without a prepared handoff (a normal quit), shutdown + * stops everything as before. + */ +export interface Interface { + /** Records that a daemon handoff was prepared successfully. */ + readonly handoffPrepared: Effect.Effect + /** Called once shutdown starts, before any service is torn down. Returns whether this server is detaching. */ + readonly begin: Effect.Effect + readonly isDetaching: () => boolean +} + +export class Service extends Context.Service()("@opencode/Detaching") {} + +export const make = Effect.sync(() => { + let prepared = false + let detaching = false + return Service.of({ + handoffPrepared: Effect.sync(() => { + prepared = true + }), + begin: Effect.sync(() => { + detaching = prepared + return detaching + }), + isDetaching: () => detaching, + }) +}) + +export const layer = Layer.effect(Service, make) + +export const node = makeGlobalNode({ service: Service, layer, deps: [] }) diff --git a/packages/core/src/persistent-pty/index.ts b/packages/core/src/persistent-pty/index.ts index 368b65d570a1..c485b5804803 100644 --- a/packages/core/src/persistent-pty/index.ts +++ b/packages/core/src/persistent-pty/index.ts @@ -16,6 +16,7 @@ import { import { Session } from "@opencode/schema/session" import { SessionEvent } from "@opencode/schema/session-event" import { Bus } from "../bus.js" +import { Detaching } from "../detaching.js" import { Pty } from "@opencode/schema/pty" import { Global } from "@opencode/util/global" import { ShellSelect } from "../shell/select.js" @@ -199,6 +200,7 @@ const makeLayer = (options: Options = {}) => Effect.gen(function* () { const bus = yield* Bus.Service const global = yield* Global.Service + const detaching = yield* Detaching.Service const context = yield* Effect.context() const runFork = Effect.runForkWith(context) let binary: Promise | undefined @@ -367,7 +369,12 @@ const makeLayer = (options: Options = {}) => if (response.type !== "ok") return yield* unexpected(response) }) - const handoff = Effect.fn("PersistentPty.handoff")(() => daemon.handoff.pipe(Effect.mapError(unavailable))) + const handoff = Effect.fn("PersistentPty.handoff")(() => + daemon.handoff.pipe( + Effect.tap((prepared) => (prepared ? detaching.handoffPrepared : Effect.void)), + Effect.mapError(unavailable), + ), + ) const removeVisibleExit = (id: Pty.ID) => { if (removing.has(id)) return @@ -528,7 +535,7 @@ export const configured = (options?: Options) => makeGlobalNode({ service: Service, layer: options === undefined ? layer : makeLayer(options), - deps: [Bus.node, Global.node], + deps: [Bus.node, Global.node, Detaching.node], }) export const node = configured() diff --git a/packages/server/src/process.ts b/packages/server/src/process.ts index 71aff3ecb6ac..8f15371f3857 100644 --- a/packages/server/src/process.ts +++ b/packages/server/src/process.ts @@ -2,6 +2,7 @@ export * as ServerProcess from "./process" import { NodeHttpServer } from "@effect/platform-node" import { Bus } from "@opencode/core/bus" +import { Detaching } from "@opencode/core/detaching" import { SessionRestart } from "@opencode/core/session/execution/restart" import { Schedule } from "@opencode/core/schedule" import { InstallationEvent } from "@opencode/schema/installation-event" @@ -88,6 +89,8 @@ export const start = Effect.fn("ServerProcess.start")(function* ( const parentScope = yield* Scope.Scope const applicationScope = yield* Scope.fork(parentScope) + let detaching: Detaching.Interface | undefined + yield* enterDetachingOnShutdown(() => detaching) yield* Effect.addFinalizer(() => status.beginStopping.pipe( Effect.andThen(Ref.set(application, Option.none())), @@ -115,6 +118,7 @@ export const start = Effect.fn("ServerProcess.start")(function* ( } // Schedules fire from any server process (`service`, `opencode serve`, or `--standalone`); the table holds all their state. yield* Effect.forkIn(Context.get(context, Schedule.Service).run, applicationScope) + detaching = Context.get(context, Detaching.Service) yield* Ref.set(application, Option.some(Context.get(context, HttpRouter.HttpRouter).asHttpEffect())) yield* status.ready const bus = Context.get(context, Bus.Service) @@ -146,6 +150,14 @@ export const start = Effect.fn("ServerProcess.start")(function* ( return yield* Effect.raceFirst(boot, shutdown.await.pipe(Effect.andThen(Effect.interrupt))) }) +/** + * Decides on shutdown whether this server is detaching (a daemon handoff was prepared), before anything + * is torn down. Must be added to the parent scope after the application scope was forked from it: + * finalizers run in reverse order, so it then runs before the application scope closes. + */ +export const enterDetachingOnShutdown = (detaching: () => Detaching.Interface | undefined) => + Effect.addFinalizer(() => Effect.suspend(() => detaching()?.begin ?? Effect.void)) + function listen(options: { readonly hostname: string; readonly port: Option.Option }) { if (Option.isSome(options.port)) return bind(options.hostname, options.port.value) const next = (port: number): ReturnType => diff --git a/packages/server/src/routes.ts b/packages/server/src/routes.ts index f0430053f0b1..b5bdeba58989 100644 --- a/packages/server/src/routes.ts +++ b/packages/server/src/routes.ts @@ -20,6 +20,7 @@ import { Instance } from "@opencode/core/instance/service" import { SessionTransfer } from "@opencode/core/session/transfer" import { InstructionEntry } from "@opencode/core/session/instruction-entry" import { ShellSelect } from "@opencode/core/shell/select" +import { Detaching } from "@opencode/core/detaching" import { Job } from "@opencode/core/job" import { KV } from "@opencode/core/kv" import { Mcp } from "@opencode/core/mcp/index" @@ -62,6 +63,7 @@ const applicationServiceNodes = [ Bus.node, EventLogger.node, httpClient, + Detaching.node, Job.node, Project.node, Project.sweepNode, diff --git a/packages/server/test/process.test.ts b/packages/server/test/process.test.ts index 20ada8bae073..b797e01aee49 100644 --- a/packages/server/test/process.test.ts +++ b/packages/server/test/process.test.ts @@ -1,5 +1,6 @@ import { expect } from "bun:test" -import { Effect } from "effect" +import { Detaching } from "@opencode/core/detaching" +import { Effect, Exit, Scope } from "effect" import { HttpServer, HttpServerRequest, HttpServerResponse } from "effect/http" import { it } from "../../core/test/lib/effect" import { ServerProcess } from "../src/process" @@ -259,3 +260,24 @@ it.live("server info lists remote URLs alongside local addresses while they are expect((yield* info()).urls).toEqual([base]) }), ) + +it.live("enters detaching on shutdown before the application scope closes", () => + Effect.gen(function* () { + for (const prepared of [true, false]) { + const detaching = yield* Detaching.make + if (prepared) yield* detaching.handoffPrepared + const parent = yield* Scope.make() + // Same order as ServerProcess.start: fork the application scope, then register the hook. + const application = yield* Scope.fork(parent) + const seen: boolean[] = [] + yield* Scope.addFinalizer( + application, + Effect.sync(() => seen.push(detaching.isDetaching())), + ) + yield* ServerProcess.enterDetachingOnShutdown(() => detaching).pipe(Scope.provide(parent)) + expect(detaching.isDetaching()).toBe(false) + yield* Scope.close(parent, Exit.void) + expect(seen).toEqual([prepared]) + } + }), +) From 3a486ee46bb29f4772ff729c4d3333d0b6fb7b18 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 00:11:05 +0000 Subject: [PATCH 03/29] feat(core): stop shell commands without forgetting them --- packages/core/src/shell.ts | 61 ++++++++++++++++-------- packages/core/src/shell/result.ts | 24 +++++++++- packages/core/test/session-owned.test.ts | 3 +- 3 files changed, 65 insertions(+), 23 deletions(-) diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index e7cfea7b5176..6b118188d686 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -34,6 +34,7 @@ export const RETENTION = Duration.days(7) export const DIRECTORY = "shell" type Info = Shell.Info +export type StopReason = "timeout" | "killed" type CreateInput = Shell.CreateInput & { shell?: string } @@ -49,10 +50,13 @@ type Active = { done: Deferred.Deferred timeoutFiber?: Fiber.Fiber timeout?: (duration: number) => Effect.Effect + stop?: (reason: StopReason) => Effect.Effect + // Whether the process can outlive this server (run by the process daemon). Spawned commands cannot. + detachable: boolean } type Ended = { - status: "exited" | "timeout" + status: "exited" | StopReason exit?: number signal?: string completed: number @@ -82,6 +86,12 @@ export interface Interface { // Replaces the running command's timeout from now; zero clears it. Ignored once the command has // ended or timed out, even while its process is still being killed or its output captured. readonly timeout: (id: Shell.ID, duration: number) => Effect.Effect + // Stops a running command gracefully (SIGTERM, then SIGKILL after 3 s) without forgetting it: it ends + // with the given status and stays observable, file included, until retention evicts it. Returns at + // once; `wait` observes the exit. Ignored once the command has ended or is already stopping. + readonly stop: (id: Shell.ID, reason: StopReason) => Effect.Effect + // Whether the command's process can outlive this server, so a planned shutdown leaves it running. + readonly detachable: (id: Shell.ID) => boolean readonly output: (id: Shell.ID, input?: Shell.OutputInput) => Effect.Effect readonly remove: (id: Shell.ID) => Effect.Effect } @@ -150,7 +160,8 @@ const layer = () => Effect.gen(function* () { for (const command of commands.values()) { if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) - // Teardown interrupts pending commands; it is not a terminal command failure. + // Teardown interrupts pending commands; it is not a terminal command failure. It never stops or + // removes a command itself: a spawned process dies with its handle's scope once `done` settles. yield* Deferred.interrupt(command.done) } commands.clear() @@ -205,6 +216,15 @@ const layer = () => return command.info }) + const stop = Effect.fn("Shell.stop")(function* (id: Shell.ID, reason: StopReason) { + const command = yield* require(id) + if (command.info.status !== "running" || !command.stop) return command.info + yield* command.stop(reason) + return command.info + }) + + const detachable = (id: Shell.ID) => commands.get(id)?.detachable ?? false + const output = Effect.fnUntraced(function* (id: Shell.ID, input?: Shell.OutputInput) { const command = yield* require(id) const cursor = input?.cursor ?? 0 @@ -251,12 +271,11 @@ const layer = () => if (page.output.endsWith("\n")) lines.pop() const truncated = latest.size > maxBytes || lines.length > maxLines const text = lines.length > maxLines ? lines.slice(-maxLines).join("\n") : page.output - const total = (yield* require(info.id)).newlines + (page.output.endsWith("\n") ? 0 : 1) - const shown = Math.min(lines.length, maxLines) - const notice = truncated - ? `\n\n[showing lines ${total - shown + 1}-${total} of ${total}; full output saved to ${info.file}]` - : "" - return { output: `${text || "(no output)"}${notice}`, truncated } + const empty = latest.size === 0 + const total = empty ? 0 : (yield* require(info.id)).newlines + (page.output.endsWith("\n") ? 0 : 1) + const shown = empty ? 0 : Math.min(lines.length, maxLines) + const notice = truncated ? `\n\n${ShellResult.savedNotice({ shown, total }, info.file)}` : "" + return { output: `${text || "(no output)"}${notice}`, truncated, lines: { shown, total } } }).pipe(Effect.catchTag("Shell.NotFoundError", () => Effect.succeed(undefined))) return { info, capture } }) @@ -327,6 +346,7 @@ const layer = () => size: 0, newlines: 0, done: Deferred.makeUnsafe(), + detachable: false, } commands.set(id, command) @@ -368,6 +388,15 @@ const layer = () => const end = (outcome: Omit) => Effect.suspend(() => Deferred.succeed(ended, { ...outcome, completed: Date.now() })) + // A recorded stop must always kill. The kill runs detached, so interrupting the caller (a timer + // being retimed, removed or torn down) neither cancels it nor waits for the process to die. + const stop = (reason: StopReason) => + Effect.gen(function* () { + if (!(yield* end({ status: reason }))) return + runFork(handle.kill({ forceKillAfter: Duration.seconds(3) }).pipe(Effect.catch(() => Effect.void))) + }).pipe(Effect.uninterruptible) + command.stop = stop + command.timeout = (duration) => Effect.gen(function* () { if (Deferred.isDoneUnsafe(ended)) return @@ -375,18 +404,7 @@ const layer = () => command.timeoutFiber = undefined if (duration === 0) return command.timeoutFiber = runFork( - Effect.sleep(Duration.millis(duration)).pipe( - // A recorded timeout must always kill. The kill runs detached, so interrupting this - // timer (retime, remove, teardown) neither cancels it nor waits for the process to die. - Effect.andThen( - Effect.gen(function* () { - if (!(yield* end({ status: "timeout" }))) return - runFork( - handle.kill({ forceKillAfter: Duration.seconds(3) }).pipe(Effect.catch(() => Effect.void)), - ) - }).pipe(Effect.uninterruptible), - ), - ), + Effect.sleep(Duration.millis(duration)).pipe(Effect.andThen(stop("timeout"))), ) }) @@ -434,6 +452,7 @@ const layer = () => // Keep exited history data-only. const timeoutFiber = command.timeoutFiber command.timeout = undefined + command.stop = undefined command.timeoutFiber = undefined if (timeoutFiber) yield* Fiber.interrupt(timeoutFiber) }), @@ -452,7 +471,7 @@ const layer = () => return command.info }) - return Service.of({ create, list, get, wait, result, timeout, output, remove }) + return Service.of({ create, list, get, wait, result, timeout, stop, detachable, output, remove }) }), ) diff --git a/packages/core/src/shell/result.ts b/packages/core/src/shell/result.ts index b9dc352e86fa..73a22fd6bed2 100644 --- a/packages/core/src/shell/result.ts +++ b/packages/core/src/shell/result.ts @@ -2,9 +2,12 @@ export * as ShellResult from "./result.js" import type { Shell } from "@opencode/schema/shell" +export type Lines = { shown: number; total: number } + export type Result = { info: Shell.Info - capture: { output: string; truncated: boolean } | undefined + // `lines` counts the tail's lines (`shown`) and the file's (`total`), zero for an empty file. + capture: { output: string; truncated: boolean; lines?: Lines } | undefined } type Output = { output: string; truncated: boolean; exit?: number; signal?: string; timeout?: boolean } @@ -27,6 +30,25 @@ export function output(result: Result): Output { } } +export function savedNotice(lines: Lines, file: string) { + if (lines.total === 0) return `[no output; output file: ${file}]` + return `[showing lines ${lines.total - lines.shown + 1}-${lines.total} of ${lines.total}; full output saved to ${file}]` +} + +// The result of a command stopped because the user interrupted its call. It always names the shell +// and its output file, which `output` leaves out when the output fits. +export function interrupted(result: Result): Output { + const captured = output(result) + const lines = result.capture && !result.capture.truncated ? result.capture.lines : undefined + return { + ...captured, + output: [ + lines ? `${captured.output.replace(/\n$/, "")}\n\n${savedNotice(lines, result.info.file)}` : captured.output, + `Command interrupted by the user (shell ID: ${result.info.id}).`, + ].join("\n\n"), + } +} + export function notice(output: Pick) { if (output.timeout) return "Timed out before completion" if (output.signal !== undefined) return `Killed by ${output.signal}` diff --git a/packages/core/test/session-owned.test.ts b/packages/core/test/session-owned.test.ts index 8d68b5b49cfb..19393e5c3891 100644 --- a/packages/core/test/session-owned.test.ts +++ b/packages/core/test/session-owned.test.ts @@ -125,7 +125,7 @@ const setup = Effect.fnUntraced(function* (options?: { const services = Layer.mergeAll( Layer.succeed(PluginHooks.Service, hooks), Layer.mock(Image.Service, {}), - options?.shell ?? Layer.mock(Shell.Service, {}), + options?.shell ?? Layer.mock(Shell.Service, { detachable: () => false }), ) const servicesFor = (ref: Location.Ref) => { locations.push(ref) @@ -508,6 +508,7 @@ describe("Session-owned handles", () => { }) const fixture = yield* setup({ shell: Layer.mock(Shell.Service, { + detachable: () => false, create: (input) => Effect.sync(() => { expect(input).toEqual({ From 90f7884c74af63ac4f5a0c46d10ded51bbffbd29 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 00:11:05 +0000 Subject: [PATCH 04/29] fix(core): leave jobs unsettled when teardown interrupts them --- packages/core/src/job.ts | 34 ++++++++++++++++- packages/core/test/job.test.ts | 70 ++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/packages/core/src/job.ts b/packages/core/src/job.ts index fd5d391b396b..6d3dad6a2cef 100644 --- a/packages/core/src/job.ts +++ b/packages/core/src/job.ts @@ -1,9 +1,23 @@ export * as Job from "./job.js" import { asc, eq, or } from "drizzle-orm" -import { Array, Cause, Clock, Context, Deferred, Effect, Exit, Layer, Schema, Scope, SynchronizedRef } from "effect" +import { + Array, + Cause, + Clock, + Context, + Deferred, + Effect, + Exit, + Layer, + Option, + Schema, + Scope, + SynchronizedRef, +} from "effect" import { makeGlobalNode } from "@opencode/util/effect/app-node" import { Database } from "./database/database.js" +import { Detaching } from "./detaching.js" import { JobList } from "./job/list.js" import { type Recovery, RecoverySchema } from "./job/recovery.js" import { JobTable } from "./job/sql.js" @@ -59,6 +73,8 @@ type State = { type FinishResult = { info?: Info done?: Deferred.Deferred + // Teardown interrupted the work: `done` is interrupted instead of settled. + interrupted?: boolean scope?: Scope.Closeable generation?: Scope.Closeable } @@ -187,6 +203,8 @@ function decrementSession(input: Map, sessionID: Sessi */ export const make = Effect.gen(function* () { const db = (yield* Database.Service).db + // Optional so registries built outside the application graph (tests, tools) need not provide it. + const detaching = Option.getOrUndefined(yield* Effect.serviceOption(Detaching.Service)) const state: State = { jobs: yield* SynchronizedRef.make(new Map()), scope: yield* Scope.Scope, @@ -248,6 +266,15 @@ export const make = Effect.gen(function* () { if (!job) return [{}, jobs] if (job.scope !== scope) return [{}, jobs] if (job.info.status !== "running") return [{ info: snapshot(job) }, jobs] + // Teardown (this registry closing, or a planned shutdown) is not a terminal outcome: leave the job + // and its durable row running for the next server, and interrupt rather than settle its observers. + // A user cancel has already settled the job as cancelled before interrupting its work. + if ( + Exit.isFailure(exit) && + Cause.hasInterruptsOnly(exit.cause) && + (scope.state._tag === "Closed" || detaching?.isDetaching()) + ) + return [{ done: job.done, scope: job.scope, interrupted: true }, jobs] const status: Exclude = Exit.isSuccess(exit) ? "completed" : Cause.hasInterruptsOnly(exit.cause) @@ -268,6 +295,7 @@ export const make = Effect.gen(function* () { return [{ info: snapshot(next), done: job.done, scope: job.scope }, new Map(jobs).set(id, next)] }), ) + if (result.interrupted && result.done) yield* Deferred.interrupt(result.done) if (result.info && result.done) yield* Deferred.succeed(result.done, result.info) if (result.scope) { yield* Scope.close(result.scope, Exit.void).pipe(Effect.forkIn(state.scope, { startImmediately: true })) @@ -295,6 +323,8 @@ export const make = Effect.gen(function* () { return [{ info: snapshot(existing) }, jobs] } const scope = yield* Scope.fork(state.scope, "parallel") + // Closing the registry interrupts the work without settling it; interrupt its observers too. + yield* Scope.addFinalizer(scope, Deferred.interrupt(done)) const job = { info: { id: input.id, @@ -551,4 +581,4 @@ export const make = Effect.gen(function* () { const layer = Layer.effect(Service, make) -export const node = makeGlobalNode({ service: Service, layer, deps: [Database.node] }) +export const node = makeGlobalNode({ service: Service, layer, deps: [Database.node, Detaching.node] }) diff --git a/packages/core/test/job.test.ts b/packages/core/test/job.test.ts index 044159bf471a..c7616c15479d 100644 --- a/packages/core/test/job.test.ts +++ b/packages/core/test/job.test.ts @@ -1,5 +1,6 @@ import { describe, expect } from "bun:test" import { Database } from "@opencode/core/database/database" +import { Detaching } from "@opencode/core/detaching" import jobMigration from "@opencode/core/database/migration/20261030054400_job" import { Job } from "@opencode/core/job" import { JobTable } from "@opencode/core/job/sql" @@ -457,6 +458,75 @@ describe("Job", () => { }), ) + it.live("interrupts observers without settling a background job when its registry is torn down", () => + Effect.gen(function* () { + const scope = yield* Scope.make() + const jobs = yield* Job.make.pipe(Scope.provide(scope)) + const job = yield* jobs.start({ + id: "shell_teardown", + type: "shell", + recovery: { + kind: "shell", + sessionID: SessionSchema.ID.make("ses_teardown"), + shellID: "shell_teardown", + command: "sleep 60", + }, + run: Effect.never, + }) + const background = yield* jobs.background(job.id) + const observer = yield* jobs.wait({ id: job.id }).pipe(Effect.forkChild({ startImmediately: true })) + + yield* Scope.close(scope, Exit.void) + + const observed = yield* Fiber.await(observer).pipe(Effect.timeout("1 second")) + expect(Exit.isFailure(observed) && Cause.hasInterruptsOnly(observed.cause)).toBe(true) + const current = yield* Job.make + expect(yield* current.pendingBackground).toMatchObject([ + { id: job.id, notificationID: background?.notificationID, status: "running" }, + ]) + if (background?.notificationID) yield* current.completeBackground(background.notificationID) + }), + ) + + it.live("leaves interrupted work unsettled while detaching, but still cancels on request", () => + Effect.gen(function* () { + const detaching = yield* Detaching.make + const jobs = yield* Job.make.pipe(Effect.provideService(Detaching.Service, detaching)) + const recovery = (id: string) => ({ + kind: "shell" as const, + sessionID: SessionSchema.ID.make("ses_detaching"), + shellID: id, + command: "sleep 60", + }) + const interrupt = yield* Deferred.make() + const detached = yield* jobs.start({ + id: "shell_detached", + type: "shell", + recovery: recovery("shell_detached"), + run: Deferred.await(interrupt).pipe(Effect.andThen(Effect.interrupt)), + }) + const cancelled = yield* jobs.start({ + id: "shell_cancelled", + type: "shell", + recovery: recovery("shell_cancelled"), + run: Effect.never, + }) + const background = yield* jobs.background(detached.id) + const observer = yield* jobs.wait({ id: detached.id }).pipe(Effect.forkChild({ startImmediately: true })) + + yield* detaching.handoffPrepared + expect(yield* detaching.begin).toBe(true) + yield* Deferred.succeed(interrupt, undefined) + + const observed = yield* Fiber.await(observer).pipe(Effect.timeout("1 second")) + expect(Exit.isFailure(observed) && Cause.hasInterruptsOnly(observed.cause)).toBe(true) + expect(yield* jobs.pendingBackground).toMatchObject([{ id: detached.id, status: "running" }]) + expect(yield* jobs.cancel(cancelled.id)).toMatchObject({ status: "cancelled" }) + expect((yield* jobs.wait({ id: cancelled.id })).info?.status).toBe("cancelled") + if (background?.notificationID) yield* jobs.completeBackground(background.notificationID) + }), + ) + it.live("cancels running session jobs and removes unacknowledged SQLite job rows on cancelSession", () => Effect.gen(function* () { const jobs = yield* Job.Service From c0cc5e6fda3656c55c11332fd924948f3e291165 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 00:11:05 +0000 Subject: [PATCH 05/29] fix(core): keep a shell command's output when its call is interrupted --- packages/core/src/plugin/internal.ts | 3 + packages/core/src/session/runner/step.ts | 10 +- packages/core/src/tool/interrupted.ts | 22 +++ packages/core/src/tool/plugin/shell.ts | 191 +++++++++++++--------- packages/core/test/session-step.test.ts | 85 +++++++++- packages/core/test/tool-shell.test.ts | 193 +++++++++++++++++++++++ 6 files changed, 430 insertions(+), 74 deletions(-) create mode 100644 packages/core/src/tool/interrupted.ts diff --git a/packages/core/src/plugin/internal.ts b/packages/core/src/plugin/internal.ts index a5fed55d5344..cc8110b3051d 100644 --- a/packages/core/src/plugin/internal.ts +++ b/packages/core/src/plugin/internal.ts @@ -53,6 +53,7 @@ import { Global } from "@opencode/util/global" import { Image } from "../image.js" import { InstructionDiscovery } from "../instruction-discovery.js" import { Integration } from "../integration.js" +import { Detaching } from "../detaching.js" import { Job } from "../job.js" import { KV } from "../kv.js" import { Location } from "../location.js" @@ -146,6 +147,7 @@ const services = [ Image.Service, InstructionDiscovery.Service, Integration.Service, + Detaching.Service, Job.Service, KV.Service, LLMClient.Service, @@ -207,6 +209,7 @@ export const requirements = LayerNode.group([ Image.node, InstructionDiscovery.node, Integration.node, + Detaching.node, Job.node, KV.node, llmClient, diff --git a/packages/core/src/session/runner/step.ts b/packages/core/src/session/runner/step.ts index 9243c6702b58..14344b83d81f 100644 --- a/packages/core/src/session/runner/step.ts +++ b/packages/core/src/session/runner/step.ts @@ -15,6 +15,7 @@ import { Permission } from "../../permission.js" import { Snapshot } from "../../snapshot.js" import { Tool } from "../../tool.js" import { ToolOutput } from "../../tool-output.js" +import { ToolInterrupted } from "../../tool/interrupted.js" import { QuestionTool } from "../../tool/plugin/question.js" import { StepFailedError } from "../error.js" import { SessionEvent } from "../event.js" @@ -117,10 +118,17 @@ export const make = Effect.gen(function* () { if (firstOutputAt === undefined && publisher.record().outputStarted) firstOutputAt = yield* Clock.currentTimeMillis if (event.type !== "tool-call" || event.providerExecuted) return + const interrupted: ToolInterrupted.Slot = {} toolRuns.push({ call: event, fiber: yield* Effect.uninterruptibleMask((restore) => - restore(executeTool(event)).pipe( + restore(executeTool(event).pipe(Effect.provideService(ToolInterrupted.Slot, interrupted))).pipe( + // A tool can settle its call even when interrupted; the fiber still exits interrupted. + Effect.catchCause((cause) => + interrupted.result && Cause.hasInterruptsOnly(cause) + ? Effect.succeed(interrupted.result) + : Effect.failCause(cause), + ), Effect.flatMap(toolOutput.truncate), Effect.flatMap((outcome) => publisher.toolExecution(event.id, event.name, outcome)), Effect.catchTag("Tool.Error", (error) => diff --git a/packages/core/src/tool/interrupted.ts b/packages/core/src/tool/interrupted.ts new file mode 100644 index 000000000000..83c9811a8f2a --- /dev/null +++ b/packages/core/src/tool/interrupted.ts @@ -0,0 +1,22 @@ +export * as ToolInterrupted from "./interrupted.js" + +import { Context, Effect } from "effect" +import type { NormalizedResult } from "../tool.js" + +/** + * Lets a tool whose call is interrupted still settle it with a result. + * + * An interrupted fiber fails as soon as it re-enters an interruptible region, so a tool that finishes its + * work uninterruptibly cannot return a value through the caller. It reports the result here instead, + * before returning; the session step settles the call with it and still ends as interrupted. + */ +export type Slot = { result?: NormalizedResult } + +export const Slot = Context.Reference("@opencode/ToolInterrupted/Slot", { + defaultValue: () => undefined, +}) + +export const report = Effect.fnUntraced(function* (result: NormalizedResult) { + const slot = yield* Slot + if (slot) slot.result = result +}) diff --git a/packages/core/src/tool/plugin/shell.ts b/packages/core/src/tool/plugin/shell.ts index d433087033b5..1cd135acfb4e 100644 --- a/packages/core/src/tool/plugin/shell.ts +++ b/packages/core/src/tool/plugin/shell.ts @@ -4,8 +4,10 @@ import { ToolFailure } from "@opencode/ai/effect" import type { Context } from "@opencode/plugin/effect/plugin" import type { SessionHooks } from "@opencode/plugin/effect/session" import type { ShellCreateBefore } from "@opencode/plugin/effect/shell" +import type { Shell as ShellSchema } from "@opencode/schema/shell" import type { Tool } from "@opencode/schema/tool" -import { Deferred, Effect, Schema, Scope } from "effect" +import { Cause, Deferred, Effect, Exit, Schema, Scope } from "effect" +import { Detaching } from "../../detaching.js" import { Environment } from "../../environment/index.js" import { Job } from "../../job.js" import { FileAccess } from "../../file-access.js" @@ -17,6 +19,7 @@ import { Shell } from "../../shell.js" import { ShellParse } from "../../shell/parse.js" import { ShellSelect } from "../../shell/select.js" import { ShellResult } from "../../shell/result.js" +import { ToolInterrupted } from "../interrupted.js" export const name = "shell" export const DEFAULT_TIMEOUT_MS = 2 * 60 * 1_000 @@ -64,6 +67,7 @@ const StructuredOutput = Schema.Struct({ shellID: Schema.optionalKey(Schema.String), truncated: Schema.Boolean, timeout: Schema.optionalKey(Schema.Boolean), + interrupted: Schema.optionalKey(Schema.Boolean), }) const Output = Schema.Struct({ @@ -87,6 +91,7 @@ const toolResult = (output: Output) => { status: output.status, ...ShellResult.metadata(output), ...(output.shellID !== undefined ? { shellID: output.shellID } : {}), + ...(output.interrupted !== undefined ? { interrupted: output.interrupted } : {}), }, } } @@ -110,6 +115,10 @@ export const Plugin = { const shellSelect = yield* ShellSelect.Service const compatibleShell = shellSelect.resolve({ priority: "compat" }) const permission = yield* Permission.Service + const detaching = yield* Detaching.Service + + // A planned shutdown leaves commands that can outlive this server running for the next one. + const leaveRunning = (id: ShellSchema.ID) => detaching.isDetaching() && shell.detachable(id) const prepare = Effect.fn("ShellTool.prepare")(function* (invocation: ShellCreateBefore, context: Tool.Context) { const target = yield* access.resolve({ path: invocation.cwd, kind: "directory" }) @@ -149,7 +158,11 @@ export const Plugin = { command: string, settled: Deferred.Deferred, ) { - const info = (yield* jobs.wait({ id })).info + const waited = yield* jobs.wait({ id }).pipe(Effect.exit) + // Teardown interrupts the job without settling it; its durable row is reported by the next server. + if (Exit.isFailure(waited)) + return Cause.hasInterrupts(waited.cause) ? undefined : yield* Effect.failCause(waited.cause) + const info = waited.value.info if (!info || info.status === "running") return const output = info.status === "completed" ? yield* Deferred.await(settled) : undefined const text = output @@ -175,6 +188,29 @@ export const Plugin = { Effect.forkIn(scope, { startImmediately: true }), ) + // Settles an interrupted foreground call. The interrupt only reaches the step through ToolInterrupted. + const interrupted = Effect.fnUntraced(function* (info: ShellSchema.Info, jobID: string) { + if (leaveRunning(info.id)) { + yield* jobs.background(jobID) + yield* shell.timeout(info.id, 0).pipe(Effect.ignore) + const output = backgroundResult(info.id, info.file) + yield* ToolInterrupted.report(toolResult(output)) + return output + } + yield* shell.stop(info.id, "killed").pipe(Effect.ignore) + const result = yield* shell.result(info) + // The stopped job settles from the same exit; this only releases it. + yield* jobs.cancel(jobID) + const output = { + ...ShellResult.interrupted(result), + shellID: info.id, + interrupted: true, + status: "completed" as const, + } + yield* ToolInterrupted.report(toolResult(output)) + return output + }) + yield* ctx.tool .transform((editor) => editor.add({ @@ -184,80 +220,91 @@ export const Plugin = { input: Input, output: Output, execute: (input, context) => - Effect.gen(function* () { - const timeout = input.background === true ? (input.timeout ?? 0) : (input.timeout ?? DEFAULT_TIMEOUT_MS) - let finalTimeout = timeout - const info = yield* shell.create( - { - command: input.command, - cwd: input.workdir, - timeout, - shell: yield* compatibleShell, - metadata: { sessionID: context.sessionID }, - }, - (invocation) => - Effect.gen(function* () { - invocation.env.AGENT = "1" - invocation.env.OPENCODE = "1" - invocation.env.AI_AGENT ||= "opencode" - invocation.env.OPENCODE_SESSION_ID = context.sessionID - finalTimeout = yield* prepare(invocation, context) - }), - ) - yield* context.progress({ shellID: info.id }) + // Uninterruptible from spawning to blocking, so an interrupt cannot orphan a started command: + // it stops (or, while detaching, backgrounds) the command and settles the call with its output. + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const timeout = input.background === true ? (input.timeout ?? 0) : (input.timeout ?? DEFAULT_TIMEOUT_MS) + let finalTimeout = timeout + const info = yield* shell.create( + { + command: input.command, + cwd: input.workdir, + timeout, + shell: yield* compatibleShell, + metadata: { sessionID: context.sessionID }, + }, + (invocation) => + Effect.gen(function* () { + invocation.env.AGENT = "1" + invocation.env.OPENCODE = "1" + invocation.env.AI_AGENT ||= "opencode" + invocation.env.OPENCODE_SESSION_ID = context.sessionID + finalTimeout = yield* restore(prepare(invocation, context)) + }), + ) + yield* context.progress({ shellID: info.id }) - const settled = yield* Deferred.make() - const run = Effect.gen(function* () { - const result = yield* shell.result(info) - if (!result.capture) return yield* new Shell.NotFoundError({ id: info.id }) - const output = ShellResult.output(result) - return { - ...output, - output: output.timeout - ? `${output.output}\n\nCommand exceeded timeout of ${finalTimeout} ms. Retry with a larger timeout if the command is expected to take longer.` - : output.output, - status: "completed" as const, - } - }).pipe( - Effect.tap((output) => Deferred.succeed(settled, output)), - Effect.map((output) => resultMessages(output).join("\n\n")), - Effect.onInterrupt(() => shell.remove(info.id).pipe(Effect.ignore)), - ) - const job = yield* jobs.start({ - // CodeMode children share a tool-call ID, but each shell must own its job. - id: info.id, - type: name, - title: info.command, - metadata: { sessionID: context.sessionID, shellID: info.id }, - recovery: { - kind: "shell", - sessionID: context.sessionID, - shellID: info.id, - command: info.command, - }, - run, - }) + const settled = yield* Deferred.make() + const run = Effect.gen(function* () { + const result = yield* shell.result(info) + if (!result.capture) return yield* new Shell.NotFoundError({ id: info.id }) + const output = ShellResult.output(result) + return { + ...output, + output: output.timeout + ? `${output.output}\n\nCommand exceeded timeout of ${finalTimeout} ms. Retry with a larger timeout if the command is expected to take longer.` + : output.output, + status: "completed" as const, + } + }).pipe( + Effect.tap((output) => Deferred.succeed(settled, output)), + Effect.map((output) => resultMessages(output).join("\n\n")), + Effect.onInterrupt(() => + leaveRunning(info.id) ? Effect.void : shell.stop(info.id, "killed").pipe(Effect.ignore), + ), + ) + const job = yield* jobs.start({ + // CodeMode children share a tool-call ID, but each shell must own its job. + id: info.id, + type: name, + title: info.command, + metadata: { sessionID: context.sessionID, shellID: info.id }, + recovery: { + kind: "shell", + sessionID: context.sessionID, + shellID: info.id, + command: info.command, + }, + run, + }) - if (input.background === true) { - yield* jobs.background(job.id) - yield* notifyWhenDone(context.sessionID, job.id, info.id, info.command, settled) - return backgroundResult(info.id, info.file) - } + if (input.background === true) { + yield* jobs.background(job.id) + yield* notifyWhenDone(context.sessionID, job.id, info.id, info.command, settled) + return backgroundResult(info.id, info.file) + } - const result = yield* jobs - .block({ id: job.id, sessionID: context.sessionID }) - .pipe(Effect.onInterrupt(() => jobs.cancel(job.id).pipe(Effect.ignore))) - if (result?.type === "backgrounded") { - yield* shell.timeout(info.id, 0) - yield* notifyWhenDone(context.sessionID, job.id, info.id, info.command, settled) - return backgroundResult(info.id, info.file) - } - if (result?.info.status === "error") - return yield* Effect.fail(new Error(result.info.error ?? "Command failed")) - if (result?.info.status === "cancelled") return yield* Effect.fail(new Error("Command cancelled")) + const blocked = yield* restore(jobs.block({ id: job.id, sessionID: context.sessionID })).pipe( + Effect.exit, + ) + if (Exit.isFailure(blocked)) { + if (Cause.hasInterrupts(blocked.cause)) return yield* interrupted(info, job.id) + return yield* Effect.failCause(blocked.cause) + } + const result = blocked.value + if (result?.type === "backgrounded") { + yield* shell.timeout(info.id, 0) + yield* notifyWhenDone(context.sessionID, job.id, info.id, info.command, settled) + return backgroundResult(info.id, info.file) + } + if (result?.info.status === "error") + return yield* Effect.fail(new Error(result.info.error ?? "Command failed")) + if (result?.info.status === "cancelled") return yield* Effect.fail(new Error("Command cancelled")) - return yield* Deferred.await(settled) - }).pipe( + return yield* Deferred.await(settled) + }), + ).pipe( Effect.map(toolResult), Effect.mapError( (error) => new ToolFailure({ message: `Unable to execute command: ${input.command}`, error }), diff --git a/packages/core/test/session-step.test.ts b/packages/core/test/session-step.test.ts index 513263e59007..532c136cd66f 100644 --- a/packages/core/test/session-step.test.ts +++ b/packages/core/test/session-step.test.ts @@ -18,10 +18,11 @@ import { SessionStep } from "@opencode/core/session/runner/step" import { SessionMessageTable, SessionTable } from "@opencode/core/session/sql" import { Snapshot } from "@opencode/core/snapshot" import { ToolOutput } from "@opencode/core/tool-output" +import { ToolInterrupted } from "@opencode/core/tool/interrupted" import { Money } from "@opencode/schema/money" import { LayerNode } from "@opencode/util/effect/layer-node" import { asc, eq } from "drizzle-orm" -import { Effect, Exit, Layer } from "effect" +import { Deferred, Effect, Exit, Fiber, Layer } from "effect" import { testEffect } from "./lib/effect" const it = testEffect( @@ -227,3 +228,85 @@ it.effect("records the cost the endpoint reports instead of the catalog price", expect(yield* db.select().from(SessionTable).where(eq(SessionTable.id, sessionID)).get()).toMatchObject({ cost: 0 }) }), ) + +it.live("settles an interrupted tool with the result it reports and still ends interrupted", () => + Effect.gen(function* () { + const db = (yield* Database.Service).db + const llm = yield* TestLLM.Test + const sessionID = Session.ID.create() + const assistantMessageID = SessionMessage.ID.create() + const steps = yield* SessionStep.make.pipe( + Effect.provide( + Layer.mock(Snapshot.Service)({ + capture: () => Effect.succeed(undefined), + files: () => Effect.succeed([]), + }), + ), + ) + yield* db + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] }) + .run() + yield* db + .insert(SessionTable) + .values({ id: sessionID, project_id: Project.ID.global, slug: "step", directory: "/project", version: "test" }) + .run() + const model = SessionRunnerModel.resolved( + LanguageModel.make({ id: "test-model", provider: "test", route: OpenAIChat.route }), + { + capabilities: { tools: true, input: ["text"], output: ["text"] }, + limit: { context: 100_000, output: 1_000 }, + cost: [], + }, + ) + yield* llm.push( + TestLLM.complete( + { reason: { normalized: "tool-calls" }, usage: { inputTokens: 1, outputTokens: 1 } }, + LLMEvent.toolCall({ id: "call-interrupted", name: "test", input: {} }), + ), + ) + const started = yield* Deferred.make() + const attempt = yield* steps + .attempt({ + isLocationClosed: () => false, + sessionID, + assistantMessageID, + agent: Agent.defaultID, + model, + prepared: { + retry: () => Effect.void, + request: ai.request({ model: model.model, prompt: "Run one tool" }), + options: {}, + // Like the shell tool: finish the work uninterruptibly, then report the result. + executeTool: () => + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + yield* Deferred.succeed(started, undefined) + yield* restore(Effect.never).pipe(Effect.exit) + const result = { content: [{ type: "text" as const, text: "partial output" }] } + yield* ToolInterrupted.report(result) + return result + }), + ), + }, + retry: () => Effect.succeed({ retry: false }), + recoverContinuation: true, + recoverOverflow: Effect.succeed(false), + }) + .pipe(Effect.forkChild({ startImmediately: true })) + yield* Deferred.await(started) + yield* Fiber.interrupt(attempt) + const exit = yield* Fiber.await(attempt) + expect(Exit.hasInterrupts(exit)).toBe(true) + expect(yield* llm.requests()).toHaveLength(1) + const message = yield* db + .select() + .from(SessionMessageTable) + .where(eq(SessionMessageTable.id, assistantMessageID)) + .get() + expect(message?.data).toMatchObject({ + error: { type: "aborted", message: "Step interrupted" }, + content: [{ type: "tool", state: { status: "completed", content: [{ type: "text", text: "partial output" }] } }], + }) + }), +) diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index e4879804c580..08c90ba34ab5 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -22,6 +22,7 @@ import { Model } from "@opencode/core/model" import { Provider } from "@opencode/core/provider" import { AbsolutePath } from "@opencode/core/schema" import { Agent } from "@opencode/core/agent" +import { Detaching } from "@opencode/core/detaching" import { Job } from "@opencode/core/job" import { Session } from "@opencode/core/session" import { SessionEvent } from "@opencode/core/session/event" @@ -38,6 +39,7 @@ import { ID } from "@opencode/schema/shell" import { ShellTool } from "@opencode/core/tool/plugin/shell" import { ToolOutput } from "@opencode/core/tool-output" import { Tool } from "@opencode/core/tool" +import { ToolInterrupted } from "@opencode/core/tool/interrupted" import { tmpdir, tmpdirScoped, withTempDir } from "./fixture/tmpdir" import { tempGlobalLayer } from "./fixture/global" import { offlineModels } from "./fixture/models" @@ -135,6 +137,7 @@ const shellPluginSupervisor = makeLocationNode({ FileAccess.node, Permission.node, Session.node, + Detaching.node, Job.node, Shell.node, ShellSelect.node, @@ -145,6 +148,7 @@ const shellPluginSupervisor = makeLocationNode({ const nodes = LayerNode.group([ Database.node, Bus.node, + Detaching.node, Job.node, Session.node, SessionExecution.node, @@ -172,6 +176,31 @@ const permissionIt = testEffect( ]), ) +// Stands in for a daemon-backed shell: every command reports that it can outlive the server. +const detachableShellSupervisor = makeLocationNode({ + name: "test/shell-plugins-detachable", + layer: Layer.effectDiscard( + registerToolPlugin(ShellTool.Plugin).pipe( + Effect.updateService(Shell.Service, (shell) => Shell.Service.of({ ...shell, detachable: () => true })), + ), + ), + deps: [ + Config.node, + Environment.node, + FileAccess.node, + Permission.node, + Session.node, + Detaching.node, + Job.node, + Shell.node, + ShellSelect.node, + Tool.node, + ], +}) +const detachableIt = testEffect( + AppNodeBuilder.build(nodes, [...replacements, PluginSupervisor.node.replace(detachableShellSupervisor)]), +) + const call = (input: typeof ShellTool.Input.Type, id = "call-shell") => ({ sessionID, ...toolIdentity, @@ -1170,6 +1199,7 @@ describe("ShellTool", () => { FileAccess.node, Permission.node, Session.node, + Detaching.node, Job.node, Shell.node, ShellSelect.node, @@ -1560,6 +1590,51 @@ describe("ShellTool", () => { { timeout: 30_000 }, ) + signalTest( + "stops a timed-out command with SIGTERM first and SIGKILL after the grace period", + () => + withTempDir((tmp) => + withSession(tmp.path, () => + Effect.gen(function* () { + const shell = yield* Shell.Service + const info = yield* shell.create({ + shell: "/bin/sh", + command: "trap 'echo got TERM' TERM; echo ready; while :; do sleep 0.1; done", + timeout: 0, + }) + while ((yield* shell.output(info.id, { cursor: Number.MAX_SAFE_INTEGER })).size === 0) + yield* Effect.sleep(Duration.millis(10)) + const start = Date.now() + yield* shell.timeout(info.id, 1) + const ended = yield* shell.wait(info.id).pipe(Effect.timeout("10 seconds")) + expect(ended.status).toBe("timeout") + expect(Date.now() - start).toBeGreaterThanOrEqual(2_900) + expect((yield* shell.output(info.id)).output).toContain("got TERM") + }), + ), + ), + { timeout: 30_000 }, + ) + + signalTest("stops a command without forgetting it", () => + withTempDir((tmp) => + withSession(tmp.path, () => + Effect.gen(function* () { + const shell = yield* Shell.Service + const info = yield* shell.create({ shell: "/bin/sh", command: "echo kept; sleep 60", timeout: 0 }) + while ((yield* shell.output(info.id, { cursor: Number.MAX_SAFE_INTEGER })).size === 0) + yield* Effect.sleep(Duration.millis(10)) + yield* shell.stop(info.id, "killed") + expect((yield* shell.wait(info.id).pipe(Effect.timeout("5 seconds"))).status).toBe("killed") + expect(yield* shell.get(info.id)).toMatchObject({ status: "killed" }) + expect(yield* Effect.promise(() => fs.readFile(info.file, "utf8"))).toBe("kept\n") + // Stopping an ended command changes nothing. + expect((yield* shell.stop(info.id, "timeout")).status).toBe("killed") + }), + ), + ), + ) + if (!isWindows) { it.live("settles a shell terminated by an external signal", () => withTempDir((tmp) => { @@ -1590,6 +1665,85 @@ describe("ShellTool", () => { ) } + // Runs a foreground call the way a session step does, and interrupts it once its output is on disk. + const interruptCall = (registry: Tool.Interface, command: string, id: string, onStarted = Effect.void) => + Effect.gen(function* () { + const shell = yield* Shell.Service + const slot: ToolInterrupted.Slot = {} + const started = yield* Deferred.make() + const fiber = yield* executeTool(registry, { + ...call({ command }, id), + progress: (update) => + typeof update.shellID === "string" + ? Deferred.succeed(started, update.shellID).pipe(Effect.asVoid) + : Effect.void, + }).pipe(Effect.provideService(ToolInterrupted.Slot, slot), Effect.forkChild({ startImmediately: true })) + const shellID = ID.make(yield* Deferred.await(started)) + while ((yield* shell.output(shellID, { cursor: Number.MAX_SAFE_INTEGER })).size === 0) + yield* Effect.sleep(Duration.millis(10)) + yield* onStarted + yield* Fiber.interrupt(fiber) + const exit = yield* Fiber.await(fiber) + return { shellID, exit, result: slot.result } + }) + + if (!isWindows) { + it.live("keeps the output of an interrupted call and settles it with the tail, file and shell ID", () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const shell = yield* Shell.Service + const jobs = yield* Job.Service + const { shellID, exit, result } = yield* interruptCall( + registry, + "printf 'first\\nsecond\\n'; sleep 60", + "call-interrupted", + ) + // The interrupt still reaches the caller; the result travels through ToolInterrupted. + expect(Exit.hasInterrupts(exit)).toBe(true) + const info = yield* shell.get(shellID) + expect(info.status).toBe("killed") + expect(yield* Effect.promise(() => fs.readFile(info.file, "utf8"))).toBe("first\nsecond\n") + expect(result?.metadata).toMatchObject({ shellID, interrupted: true, truncated: false }) + expect(result?.content).toEqual([ + { + type: "text", + text: `first\nsecond\n\n[showing lines 1-2 of 2; full output saved to ${info.file}]\n\nCommand interrupted by the user (shell ID: ${shellID}).`, + }, + ]) + expect((yield* jobs.get(shellID))?.status).not.toBe("running") + expect(yield* jobs.pendingBackground).toEqual([]) + }), + ) + }), + ) + + it.live("leaves a spawned command's interrupt unchanged while detaching", () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const shell = yield* Shell.Service + const detaching = yield* Detaching.Service + const { shellID, result } = yield* interruptCall( + registry, + "printf ready; sleep 60", + "call-detaching-spawned", + Effect.gen(function* () { + yield* detaching.handoffPrepared + yield* detaching.begin + }), + ) + expect(shell.detachable(shellID)).toBe(false) + expect((yield* shell.get(shellID)).status).toBe("killed") + expect(result?.metadata).toMatchObject({ shellID, interrupted: true }) + }), + ) + }), + ) + } + it.live("backgrounds a foreground command when the session is signaled", () => withTempDir((tmp) => { reset() @@ -1638,3 +1792,42 @@ describe("ShellTool", () => { }), ) }) + +describe("ShellTool detaching", () => { + detachableIt.live("moves a detachable command to the background when interrupted while detaching", () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const shell = yield* Shell.Service + const jobs = yield* Job.Service + const detaching = yield* Detaching.Service + const slot: ToolInterrupted.Slot = {} + const started = yield* Deferred.make() + const fiber = yield* executeTool(registry, { + ...call({ command: idleCommand }, "call-detaching"), + progress: (update) => + typeof update.shellID === "string" + ? Deferred.succeed(started, update.shellID).pipe(Effect.asVoid) + : Effect.void, + }).pipe(Effect.provideService(ToolInterrupted.Slot, slot), Effect.forkChild({ startImmediately: true })) + const shellID = ID.make(yield* Deferred.await(started)) + yield* detaching.handoffPrepared + expect(yield* detaching.begin).toBe(true) + yield* Fiber.interrupt(fiber) + + const info = yield* shell.get(shellID) + expect(info.status).toBe("running") + expect(slot.result?.content[0]).toEqual({ + type: "text", + text: `Command moved to the background (shell ID: ${shellID}).\nOutput is streaming to: ${info.file}`, + }) + expect(slot.result?.metadata).toMatchObject({ status: "running", shellID }) + // Backgrounded durably for the next server, without a completion notification of its own. + expect(yield* jobs.pendingBackground).toMatchObject([{ id: shellID, status: "running" }]) + yield* shell.remove(shellID) + }), + ) + }), + ) +}) From 592a5ce0eded552a5f98d350a5aa49e2bf25048e Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 00:23:26 +0000 Subject: [PATCH 06/29] feat(core): run local shell commands in the process daemon --- packages/core/src/file-retention.ts | 3 + packages/core/src/shell.ts | 546 ++++++++++++++++--- packages/core/src/tool/plugin/shell.ts | 5 + packages/core/test/fixture/persistent-pty.ts | 8 + packages/core/test/shell-cleanup.test.ts | 5 +- packages/core/test/shell-retention.test.ts | 3 + packages/core/test/tool-shell.test.ts | 27 +- 7 files changed, 533 insertions(+), 64 deletions(-) create mode 100644 packages/core/test/fixture/persistent-pty.ts diff --git a/packages/core/src/file-retention.ts b/packages/core/src/file-retention.ts index a325f2165fc9..ee9f486e219c 100644 --- a/packages/core/src/file-retention.ts +++ b/packages/core/src/file-retention.ts @@ -7,6 +7,8 @@ export const cleanup = Effect.fn("FileRetention.cleanup")(function* ( fs: FSUtil.Interface, files: ReadonlyArray, retention: Duration.Input, + // Runs before an expired file is removed. + beforeRemove?: (file: string) => Effect.Effect, ) { const cutoff = Date.now() - Duration.toMillis(retention) yield* Effect.forEach( @@ -16,6 +18,7 @@ export const cleanup = Effect.fn("FileRetention.cleanup")(function* ( const info = yield* fs.stat(file).pipe(Effect.orElseSucceed(() => undefined)) const mtime = info && Option.getOrUndefined(info.mtime) if (!mtime || mtime.getTime() >= cutoff) return + if (beforeRemove) yield* beforeRemove(file) yield* fs.remove(file).pipe(Effect.ignore) }), { concurrency: 8, discard: true }, diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index 6b118188d686..d35445c16e4b 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -1,7 +1,20 @@ export * as Shell from "./shell.js" import path from "path" -import { Context, Deferred, Duration, Effect, Fiber, Latch, Layer, Schema, Schedule, Stream } from "effect" +import { + Context, + Deferred, + Duration, + Effect, + Exit, + Fiber, + Latch, + Layer, + Option, + Schema, + Schedule, + Stream, +} from "effect" import { ChildProcess } from "effect/process" import { produce } from "immer" import { Shell } from "@opencode/schema/shell" @@ -14,6 +27,7 @@ import { Environment } from "./environment/index.js" import { FileRetention } from "./file-retention.js" import { Location } from "./location.js" import { Global } from "@opencode/util/global" +import { PersistentPty } from "./persistent-pty.js" import { ShellSelect } from "./shell/select.js" import type { ShellCreateBefore } from "@opencode/plugin/effect/shell" import { PluginHooks } from "./plugin/hooks.js" @@ -32,17 +46,55 @@ export class NotFoundError extends Schema.TaggedError()("Shell.No const EXITED_LIMIT = 25 export const RETENTION = Duration.days(7) export const DIRECTORY = "shell" +const STOP_GRACE = Duration.seconds(3) type Info = Shell.Info export type StopReason = "timeout" | "killed" + +/** The tool call that started a command, so a server that took over can settle it. */ +export const ToolCall = Schema.Struct({ + sessionID: Schema.String, + assistantMessageID: Schema.String, + callID: Schema.String, +}) +export type ToolCall = typeof ToolCall.Type + type CreateInput = Shell.CreateInput & { shell?: string + toolCall?: ToolCall +} + +// What the server needs to take a daemon-run command back after a restart, kept in the daemon's metadata. +const Stored = Schema.Struct({ + id: Shell.ID, + command: Schema.String, + shell: Schema.String, + location: Schema.Struct({ directory: Schema.String, workspaceID: Schema.optionalKey(Schema.String) }), + metadata: Shell.Metadata, + toolCall: Schema.optional(ToolCall), + timeoutDeadline: Schema.optional(Schema.Number), + stopReason: Schema.optional(Schema.Literals(["timeout", "killed"])), + backgroundedAtShutdown: Schema.optional(Schema.Boolean), +}) +type Stored = typeof Stored.Type +const decodeStored = Schema.decodeUnknownOption(Stored) + +// A command run by the process daemon, which writes and counts its output file. +type Daemon = { + id: number + stored: Stored + // From the exit: newlines in the file, and why the daemon stopped writing it early. + outputLines?: number + outputError?: string + wait?: Fiber.Fiber + kill?: Fiber.Fiber } type Active = { // Immutable snapshot; lifecycle updates replace it via immer `produce`. info: Info file: string + // Spawned commands only: bytes and newlines written so far. The daemon's file is the only count for its commands. size: number newlines: number // Resolves with the terminal Info once the command exits, times out, or is killed. A wait @@ -53,6 +105,7 @@ type Active = { stop?: (reason: StopReason) => Effect.Effect // Whether the process can outlive this server (run by the process daemon). Spawned commands cannot. detachable: boolean + daemon?: Daemon } type Ended = { @@ -65,10 +118,13 @@ type Ended = { /** * Location-owned non-interactive shell command process service. * - * Each `create` spawns one shell command, captures combined stdout/stderr to a + * Each `create` starts one shell command, captures combined stdout/stderr to a * file, and returns an ID. Clients poll `get` for status and `output` for * file-backed output by cursor. No session, message, or permission state lives * here; callers (e.g. `ShellTool`) own that association and store the shell ID. + * + * Local commands run in the process daemon when it is available, so they survive a planned server + * restart; this service takes them back when it starts. Elsewhere they are spawned by the server. */ export interface Interface { readonly create: ( @@ -92,6 +148,11 @@ export interface Interface { readonly stop: (id: Shell.ID, reason: StopReason) => Effect.Effect // Whether the command's process can outlive this server, so a planned shutdown leaves it running. readonly detachable: (id: Shell.ID) => boolean + // Records in the daemon that a planned shutdown moved this running command to the background, clearing + // its timeout, so the next server neither times it out nor waits on it. Best effort; spawned commands ignore it. + readonly backgroundAtShutdown: (id: Shell.ID) => Effect.Effect + // The command a tool call started, while it is still known here (including one taken back after a restart). + readonly byToolCall: (call: ToolCall) => Effect.Effect readonly output: (id: Shell.ID, input?: Shell.OutputInput) => Effect.Effect readonly remove: (id: Shell.ID) => Effect.Effect } @@ -101,6 +162,7 @@ export class Service extends Context.Service()("@opencode/Sh export const cleanup = Effect.fn("Shell.cleanup")(function* () { const fs = yield* FSUtil.Service const global = yield* Global.Service + const pty = yield* PersistentPty.Service const directory = path.join(global.data, DIRECTORY) const projects = yield* fs.readDirectoryEntries(directory).pipe( Effect.map((entries) => entries.filter((entry) => entry.type === "Directory")), @@ -121,7 +183,19 @@ export const cleanup = Effect.fn("Shell.cleanup")(function* () { ), { concurrency: 8 }, ) - yield* FileRetention.cleanup(fs, files.flat(), RETENTION) + // Never starts the daemon. A command it still runs keeps its file; a finished one is forgotten before deletion. + const processes = new Map( + (yield* pty.listPipes().pipe(Effect.orElseSucceed(() => []))).map((process) => [process.outputPath, process]), + ) + yield* FileRetention.cleanup( + fs, + files.flat().filter((file) => processes.get(file)?.lifecycle.status !== "running"), + RETENTION, + (file) => { + const process = processes.get(file) + return process ? pty.terminatePipe(process.id).pipe(Effect.ignore) : Effect.void + }, + ) }) const cleanupLayer = Layer.effectDiscard( @@ -131,7 +205,7 @@ const cleanupLayer = Layer.effectDiscard( const cleanupNode = makeGlobalNode({ name: "shell-output-cleanup", layer: cleanupLayer, - deps: [FSUtil.node, Global.node], + deps: [FSUtil.node, Global.node, PersistentPty.node], }) const layer = () => @@ -146,13 +220,17 @@ const layer = () => const hooks = yield* PluginHooks.Service const environments = yield* SessionEnvironment.Service const config = yield* Config.Service + const pty = yield* PersistentPty.Service const context = yield* Effect.context() const runFork = Effect.runForkWith(context) const commands = new Map() const exitOrder: Shell.ID[] = [] + // Remote workspaces and Windows keep spawning commands from the server. + const daemonEnabled = + location.workspaceID === undefined && process.platform !== "win32" && PersistentPty.available const outputDir = path.join(global.data, DIRECTORY, location.project.id) - const { mkdir, unlink } = yield* Effect.promise(() => import("fs/promises")) + const { mkdir, unlink, open } = yield* Effect.promise(() => import("fs/promises")) const { createWriteStream, createReadStream } = yield* Effect.promise(() => import("fs")) yield* Effect.promise(() => mkdir(outputDir, { recursive: true })) @@ -160,6 +238,9 @@ const layer = () => Effect.gen(function* () { for (const command of commands.values()) { if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) + // Daemon commands are only detached from: their processes keep running for the next owner. + if (command.daemon?.kill) yield* Fiber.interrupt(command.daemon.kill) + if (command.daemon?.wait) yield* Fiber.interrupt(command.daemon.wait) // Teardown interrupts pending commands; it is not a terminal command failure. It never stops or // removes a command itself: a spawned process dies with its handle's scope once `done` settles. yield* Deferred.interrupt(command.done) @@ -182,14 +263,60 @@ const layer = () => if (!command) return commands.delete(id) if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) - // Unblock any wait still pending when the command is removed before it terminated. - yield* Deferred.fail(command.done, new NotFoundError({ id })) - yield* Effect.promise(() => unlink(command.file).catch(() => {})) + if (command.daemon) { + if (command.daemon.kill) yield* Fiber.interrupt(command.daemon.kill) + // Forget the process before deleting its file; its pending `wait` then ends as terminated. + yield* pty.terminatePipe(command.daemon.id).pipe(Effect.ignore) + yield* Effect.promise(() => unlink(command.file).catch(() => {})) + yield* Deferred.fail(command.done, new NotFoundError({ id })) + } + if (!command.daemon) { + // Unblock any wait still pending when the command is removed before it terminated. + yield* Deferred.fail(command.done, new NotFoundError({ id })) + yield* Effect.promise(() => unlink(command.file).catch(() => {})) + } // Every shell reports exactly one exit; removing a running one ends it as killed. if (command.info.status === "running") yield* bus.publish(Shell.Event.Exited, { id, status: "killed" }) yield* bus.publish(Shell.Event.Deleted, { id }) }) + const evict = Effect.fnUntraced(function* () { + while (exitOrder.length > EXITED_LIMIT) { + const oldest = exitOrder[0] + if (!oldest) break + yield* removeCommand(oldest) + } + }) + + // Records how a command ended, once, unless it was removed meanwhile. + const finish = Effect.fnUntraced(function* (command: Active, outcome: Ended) { + const id = command.info.id + // A removed command already failed its waiters and published its exit and deletion. + if (commands.get(id) !== command) return + command.info = produce(command.info, (draft) => { + draft.status = outcome.status + if (outcome.exit !== undefined) draft.exit = outcome.exit + if (outcome.signal !== undefined) draft.signal = outcome.signal + draft.time.completed = outcome.completed + }) + // Resolve waiters before retention eviction, so an evicted command still reports its + // terminal Info rather than the removal NotFoundError. + yield* Deferred.succeed(command.done, command.info) + yield* bus.publish(Shell.Event.Exited, { + id, + ...(outcome.exit !== undefined ? { exit: outcome.exit } : {}), + status: outcome.status, + }) + exitOrder.push(id) + yield* evict() + // Keep exited history data-only. + const timeoutFiber = command.timeoutFiber + command.timeout = undefined + command.stop = undefined + command.timeoutFiber = undefined + if (timeoutFiber) yield* Fiber.interrupt(timeoutFiber) + }) + const remove = Effect.fn("Shell.remove")(function* (id: Shell.ID) { yield* require(id) yield* removeCommand(id) @@ -225,33 +352,120 @@ const layer = () => const detachable = (id: Shell.ID) => commands.get(id)?.detachable ?? false + const backgroundAtShutdown = Effect.fn("Shell.backgroundAtShutdown")(function* (id: Shell.ID) { + const command = commands.get(id) + const daemon = command?.daemon + if (!command || !daemon || command.info.status !== "running") return + if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) + command.timeoutFiber = undefined + yield* store(daemon, { backgroundedAtShutdown: true, timeoutDeadline: undefined }) + }) + + const byToolCall = Effect.fn("Shell.byToolCall")(function* (call: ToolCall) { + return Array.from(commands.values()).find((command) => { + const stored = command.daemon?.stored.toolCall + return ( + stored?.sessionID === call.sessionID && + stored.assistantMessageID === call.assistantMessageID && + stored.callID === call.callID + ) + })?.info + }) + + const readSpawned = (file: string, start: number, length: number) => + length <= 0 + ? Effect.succeed(Buffer.alloc(0)) + : Effect.promise( + () => + new Promise((resolve) => { + const buffer = Buffer.alloc(length) + const stream = createReadStream(file, { start, end: start + length - 1 }) + let offset = 0 + stream.on("data", (chunk: string | Buffer) => { + const bytes = Buffer.from(chunk) + bytes.copy(buffer, offset) + offset += bytes.length + }) + stream.on("end", () => resolve(buffer.subarray(0, offset))) + stream.on("error", () => resolve(Buffer.alloc(0))) + }), + ) + + // Reads a daemon command's file, which is the only record of its output. `size` comes from the same open + // handle; a missing file reads as empty. + const readFile = (file: string, start: (size: number) => number, limit: number) => + Effect.promise(async () => { + const handle = await open(file, "r").catch(() => undefined) + if (!handle) return { size: 0, start: 0, bytes: Buffer.alloc(0) } + try { + const size = (await handle.stat()).size + const from = Math.max(0, start(size)) + if (from >= size) return { size, start: from, bytes: Buffer.alloc(0) } + const buffer = Buffer.alloc(Math.min(limit, size - from)) + const read = await handle.read(buffer, 0, buffer.length, from) + return { size, start: from, bytes: buffer.subarray(0, read.bytesRead) } + } catch { + return { size: 0, start: 0, bytes: Buffer.alloc(0) } + } finally { + await handle.close() + } + }) + const output = Effect.fnUntraced(function* (id: Shell.ID, input?: Shell.OutputInput) { const command = yield* require(id) const cursor = input?.cursor ?? 0 const limit = input?.limit ?? 65536 + if (command.daemon) { + const page = yield* readFile(command.file, () => cursor, limit) + if (cursor >= page.size) return { output: "", cursor: page.size, size: page.size, truncated: false } + return { + output: page.bytes.toString("utf8"), + cursor: page.start + page.bytes.length, + size: page.size, + truncated: false, + } + } if (cursor >= command.size) return { output: "", cursor: command.size, size: command.size, truncated: false } const start = Math.max(0, cursor) - const length = Math.min(limit, command.size - start) - const buffer = Buffer.alloc(length) - const bytesRead = yield* Effect.promise( + const bytes = yield* readSpawned(command.file, start, Math.min(limit, command.size - start)) + return { + output: bytes.toString("utf8"), + cursor: start + bytes.length, + size: command.size, + truncated: false, + } + }) + + // Only for a command whose exit report was lost; the daemon counts newlines otherwise. + const countNewlines = (file: string) => + Effect.promise( () => new Promise((resolve) => { - const stream = createReadStream(command.file, { start, end: start + length - 1 }) - let offset = 0 + let count = 0 + const stream = createReadStream(file) stream.on("data", (chunk: string | Buffer) => { const bytes = Buffer.from(chunk) - bytes.copy(buffer, offset) - offset += bytes.length + for (let index = bytes.indexOf(10); index !== -1; index = bytes.indexOf(10, index + 1)) count++ }) - stream.on("end", () => resolve(offset)) - stream.on("error", () => resolve(0)) + stream.on("end", () => resolve(count)) + stream.on("error", () => resolve(count)) }), ) + + // The last `maxBytes` of a finished command's output, the file's size and its newline count. + const tail = Effect.fnUntraced(function* (command: Active, maxBytes: number) { + if (!command.daemon) { + const start = Math.max(0, command.size - maxBytes) + const bytes = yield* readSpawned(command.file, start, command.size - start) + return { text: bytes.toString("utf8"), size: command.size, newlines: command.newlines, error: undefined } + } + const page = yield* readFile(command.file, (size) => size - maxBytes, maxBytes) return { - output: buffer.subarray(0, bytesRead).toString("utf8"), - cursor: start + bytesRead, - size: command.size, - truncated: false, + text: page.bytes.toString("utf8"), + size: page.size, + // The daemon's count, unless the command ended without its exit report (the daemon went away). + newlines: command.daemon.outputLines ?? (yield* countNewlines(command.file)), + error: command.daemon.outputError, } }) @@ -265,21 +479,225 @@ const layer = () => const limits = Config.latest(yield* config.entries(), "tool_output") const maxLines = limits?.max_lines ?? ToolOutput.MAX_LINES const maxBytes = limits?.max_bytes ?? ToolOutput.MAX_BYTES - const latest = yield* output(info.id, { cursor: Number.MAX_SAFE_INTEGER }) - const page = yield* output(info.id, { cursor: Math.max(0, latest.size - maxBytes), limit: maxBytes }) - const lines = page.output.split("\n") - if (page.output.endsWith("\n")) lines.pop() - const truncated = latest.size > maxBytes || lines.length > maxLines - const text = lines.length > maxLines ? lines.slice(-maxLines).join("\n") : page.output - const empty = latest.size === 0 - const total = empty ? 0 : (yield* require(info.id)).newlines + (page.output.endsWith("\n") ? 0 : 1) + const page = yield* tail(yield* require(info.id), maxBytes) + const lines = page.text.split("\n") + if (page.text.endsWith("\n")) lines.pop() + const truncated = page.size > maxBytes || lines.length > maxLines + const text = lines.length > maxLines ? lines.slice(-maxLines).join("\n") : page.text + const empty = page.size === 0 const shown = empty ? 0 : Math.min(lines.length, maxLines) + const total = empty ? 0 : Math.max(page.newlines + (page.text.endsWith("\n") ? 0 : 1), shown) const notice = truncated ? `\n\n${ShellResult.savedNotice({ shown, total }, info.file)}` : "" - return { output: `${text || "(no output)"}${notice}`, truncated, lines: { shown, total } } + const error = page.error ? `\n\n[output file ${info.file} stopped early: ${page.error}]` : "" + return { output: `${text || "(no output)"}${notice}${error}`, truncated, lines: { shown, total } } }).pipe(Effect.catchTag("Shell.NotFoundError", () => Effect.succeed(undefined))) return { info, capture } }) + // Replaces the command's daemon metadata, so a restart sees its latest timeout and stop reason. + // Cleared fields are left out of the JSON. + const store = (daemon: Daemon, patch: Partial) => { + daemon.stored = { ...daemon.stored, ...patch } + return pty + .setPipeMetadata(daemon.id, daemon.stored) + .pipe( + Effect.catch((error) => + Effect.logWarning("failed to update shell command metadata", { + id: daemon.stored.id, + error: error.message, + }), + ), + ) + } + + // Watches a command the daemon runs: its exit, timeout (in ms from now, none if undefined) and graceful + // stop. Used for new commands and for running ones taken back after a restart. + const attach = (command: Active, daemon: Daemon, timeout: number | undefined) => { + const stopping = (reason: StopReason) => + Effect.gen(function* () { + yield* store(daemon, { stopReason: reason, timeoutDeadline: undefined }) + yield* pty.signalPipe(daemon.id, "SIGTERM").pipe(Effect.ignore) + const ended = yield* Deferred.await(command.done).pipe(Effect.timeoutOption(STOP_GRACE), Effect.exit) + if (Exit.isSuccess(ended) && Option.isNone(ended.value)) + yield* pty.signalPipe(daemon.id, "SIGKILL").pipe(Effect.ignore) + }) + + // Detached, so interrupting the caller (a timer being retimed, removed or torn down) never cancels a + // recorded stop. Only removal and teardown interrupt it. + const stop = (reason: StopReason) => + Effect.sync(() => { + if (daemon.stored.stopReason !== undefined) return + daemon.stored = { ...daemon.stored, stopReason: reason } + daemon.kill = runFork(stopping(reason)) + }) + command.stop = stop + + const arm = (duration: number) => { + command.timeoutFiber = runFork(Effect.sleep(Duration.millis(duration)).pipe(Effect.andThen(stop("timeout")))) + } + + command.timeout = (duration) => + Effect.gen(function* () { + if (daemon.stored.stopReason !== undefined) return + if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) + command.timeoutFiber = undefined + yield* store(daemon, { timeoutDeadline: duration === 0 ? undefined : Date.now() + duration }) + if (duration !== 0) arm(duration) + }) + + daemon.wait = runFork( + pty.waitPipe(daemon.id).pipe( + Effect.map((exit): Ended => { + daemon.outputLines = exit.outputLines + daemon.outputError = exit.outputError + const reason = daemon.stored.stopReason + // A stopped command reports its reason, not the signal it was stopped with, as spawned ones do. + if (reason) return { status: reason, completed: exit.exitedAt } + return { + status: "exited", + ...(exit.exitCode !== undefined ? { exit: exit.exitCode } : {}), + ...(exit.signal !== undefined ? { signal: exit.signal } : {}), + completed: exit.exitedAt, + } + }), + // The daemon went away (and its processes with it), or forgot the process: it was killed. + Effect.catch((error) => + Effect.logWarning("lost a shell command's process", { id: command.info.id, error: error.message }).pipe( + Effect.as({ status: "killed", completed: Date.now() }), + ), + ), + Effect.flatMap((outcome) => finish(command, outcome)), + ), + ) + + // A stop interrupted by a restart lost its grace period: kill now. + if (daemon.stored.stopReason !== undefined) { + daemon.kill = runFork(pty.signalPipe(daemon.id, "SIGKILL").pipe(Effect.ignore)) + return + } + if (timeout === undefined) return + if (timeout > 0) return arm(timeout) + runFork(stop("timeout")) + } + + const daemonCommand = (info: Info, daemon: Daemon): Active => ({ + info, + file: info.file, + size: 0, + newlines: 0, + done: Deferred.makeUnsafe(), + detachable: true, + daemon, + }) + + // Runs the command in the daemon. Fails with `UnavailableError` only when the daemon could not be reached + // or started, so the request never ran and spawning it here instead is safe. + const createInDaemon = Effect.fnUntraced(function* ( + info: Info, + invocation: ShellCreateBefore, + args: string[], + toolCall: ToolCall | undefined, + ) { + const stored: Stored = { + id: info.id, + command: info.command, + shell: info.shell, + location: { + directory: location.directory, + ...(location.workspaceID !== undefined ? { workspaceID: location.workspaceID } : {}), + }, + metadata: info.metadata, + ...(toolCall ? { toolCall } : {}), + ...(invocation.timeout > 0 ? { timeoutDeadline: info.time.started + invocation.timeout } : {}), + } + const created = yield* pty.createPipe({ + program: invocation.shell, + args, + cwd: path.resolve(location.directory, invocation.cwd), + // The daemon replaces the environment, so variables removed by hooks stay removed, like the spawner. + env: Object.fromEntries( + Object.entries(invocation.env).filter((entry): entry is [string, string] => entry[1] !== undefined), + ), + groupID: location.project.id, + outputPath: info.file, + metadata: stored, + }) + const daemon: Daemon = { id: created.id, stored } + const command = daemonCommand( + produce(info, (draft) => { + if (created.pid !== undefined) draft.pid = created.pid + }), + daemon, + ) + commands.set(info.id, command) + // Timed from now: starting the daemon may have taken a while. + attach(command, daemon, invocation.timeout > 0 ? invocation.timeout : undefined) + yield* bus.publish(Shell.Event.Created, { info: command.info }) + return command.info + }) + + // Takes back this Location's daemon commands, left by a previous server or an evicted Location. + // Two Locations can share a project's output directory, so match the Location itself. + const recover = Effect.fnUntraced(function* () { + const processes = yield* pty + .listPipes() + .pipe( + Effect.catch((error) => + Effect.logWarning("failed to list shell commands to recover", { error: error.message }).pipe( + Effect.as([]), + ), + ), + ) + const finished: { id: Shell.ID; exitedAt: number }[] = [] + for (const process of processes) { + const stored = Option.getOrUndefined(decodeStored(process.metadata)) + if ( + !stored || + stored.location.directory !== location.directory || + stored.location.workspaceID !== location.workspaceID || + path.dirname(process.outputPath) !== outputDir || + commands.has(stored.id) + ) + continue + const running = process.lifecycle.status === "running" + const exited = process.lifecycle.status === "exited" ? process.lifecycle : undefined + const status = running ? "running" : (stored.stopReason ?? (exited ? "exited" : "killed")) + const info: Info = { + id: stored.id, + status, + command: stored.command, + cwd: process.cwd, + shell: stored.shell, + file: process.outputPath, + ...(process.pid !== undefined ? { pid: process.pid } : {}), + ...(status === "exited" && exited?.exitCode !== undefined ? { exit: exited.exitCode } : {}), + ...(status === "exited" && exited?.signal !== undefined ? { signal: exited.signal } : {}), + metadata: stored.metadata, + time: { + started: process.startedAt, + ...(running ? {} : { completed: process.exitedAt ?? process.startedAt }), + }, + } + const daemon: Daemon = { + id: process.id, + stored, + ...(running ? {} : { outputLines: process.outputLines, outputError: process.outputError }), + } + const command = daemonCommand(info, daemon) + commands.set(stored.id, command) + if (running) { + // A command moved to the background at shutdown has no timeout; an expired deadline fires at once. + const deadline = stored.backgroundedAtShutdown ? undefined : stored.timeoutDeadline + attach(command, daemon, deadline === undefined ? undefined : Math.max(0, deadline - Date.now())) + continue + } + yield* Deferred.succeed(command.done, info) + finished.push({ id: stored.id, exitedAt: info.time.completed ?? 0 }) + } + exitOrder.push(...finished.toSorted((a, b) => a.exitedAt - b.exitedAt).map((item) => item.id)) + yield* evict() + }) + const create = Effect.fn("Shell.create")(function* ( input: CreateInput, before?: (input: ShellCreateBefore) => Effect.Effect, @@ -318,6 +736,22 @@ const layer = () => time: { started: Date.now() }, } + if (daemonEnabled) { + const created = yield* createInDaemon(info, invocation, args, input.toolCall).pipe( + Effect.map(Option.some), + Effect.catchTags({ + "PersistentPty.UnavailableError": (error) => + Effect.logWarning("process daemon unavailable; spawning shell command instead", { + error: error.message, + }).pipe(Effect.as(Option.none())), + // The daemon may have run it: never run it again here. + "PersistentPty.RequestError": (cause) => + Effect.fail(new AppProcess.AppProcessError({ command: invocation.command, cause })), + }), + ) + if (Option.isSome(created)) return created.value + } + // Spawn through the Environment and stream combined output to the file. The handle is scope-bound, so // the managing fiber keeps its scope open until the command terminates (it awaits `done` at the // end). `create` returns once `ready` resolves with the registered command. @@ -332,7 +766,7 @@ const layer = () => env: invocation.env, stdin: "ignore", detached: process.platform !== "win32", - forceKillAfter: Duration.seconds(3), + forceKillAfter: STOP_GRACE, }), ) .pipe( @@ -393,7 +827,7 @@ const layer = () => const stop = (reason: StopReason) => Effect.gen(function* () { if (!(yield* end({ status: reason }))) return - runFork(handle.kill({ forceKillAfter: Duration.seconds(3) }).pipe(Effect.catch(() => Effect.void))) + runFork(handle.kill({ forceKillAfter: STOP_GRACE }).pipe(Effect.catch(() => Effect.void))) }).pipe(Effect.uninterruptible) command.stop = stop @@ -427,34 +861,7 @@ const layer = () => // Report the terminal status only once captured output is complete, so a reader that // sees it never reads a partial file. yield* outputDone.await - // A removed command already failed its waiters and published its exit and deletion. - if (commands.get(id) !== command) return - command.info = produce(command.info, (draft) => { - draft.status = outcome.status - if (outcome.exit !== undefined) draft.exit = outcome.exit - if (outcome.signal !== undefined) draft.signal = outcome.signal - draft.time.completed = outcome.completed - }) - // Resolve waiters before retention eviction, so an evicted command still reports its - // terminal Info rather than the removal NotFoundError. - yield* Deferred.succeed(command.done, command.info) - yield* bus.publish(Shell.Event.Exited, { - id, - ...(outcome.exit !== undefined ? { exit: outcome.exit } : {}), - status: outcome.status, - }) - exitOrder.push(id) - while (exitOrder.length > EXITED_LIMIT) { - const oldest = exitOrder[0] - if (!oldest) break - yield* removeCommand(oldest) - } - // Keep exited history data-only. - const timeoutFiber = command.timeoutFiber - command.timeout = undefined - command.stop = undefined - command.timeoutFiber = undefined - if (timeoutFiber) yield* Fiber.interrupt(timeoutFiber) + yield* finish(command, outcome) }), ) @@ -471,7 +878,23 @@ const layer = () => return command.info }) - return Service.of({ create, list, get, wait, result, timeout, stop, detachable, output, remove }) + // Taken back before the service is returned, so session restart recovery sees them. + if (daemonEnabled) yield* recover() + + return Service.of({ + create, + list, + get, + wait, + result, + timeout, + stop, + detachable, + backgroundAtShutdown, + byToolCall, + output, + remove, + }) }), ) @@ -487,6 +910,7 @@ export const node = makeLocationNode({ PluginHooks.node, SessionEnvironment.node, Config.node, + PersistentPty.node, cleanupNode, ], }) diff --git a/packages/core/src/tool/plugin/shell.ts b/packages/core/src/tool/plugin/shell.ts index 1cd135acfb4e..6441924ebc19 100644 --- a/packages/core/src/tool/plugin/shell.ts +++ b/packages/core/src/tool/plugin/shell.ts @@ -233,6 +233,11 @@ export const Plugin = { timeout, shell: yield* compatibleShell, metadata: { sessionID: context.sessionID }, + toolCall: { + sessionID: context.sessionID, + assistantMessageID: context.messageID, + callID: context.id, + }, }, (invocation) => Effect.gen(function* () { diff --git a/packages/core/test/fixture/persistent-pty.ts b/packages/core/test/fixture/persistent-pty.ts new file mode 100644 index 000000000000..c81232f00e08 --- /dev/null +++ b/packages/core/test/fixture/persistent-pty.ts @@ -0,0 +1,8 @@ +import { Effect, Layer } from "effect" +import { PersistentPty } from "@opencode/core/persistent-pty" + +// No process daemon: shell commands fall back to the Environment's spawner, and nothing is ever running in it. +export const disabledPersistentPtyLayer = Layer.mock(PersistentPty.Service, { + createPipe: () => Effect.fail(new PersistentPty.UnavailableError({ message: "process daemon disabled" })), + listPipes: () => Effect.succeed([]), +}) diff --git a/packages/core/test/shell-cleanup.test.ts b/packages/core/test/shell-cleanup.test.ts index 9af9ca64af26..a807b69d8307 100644 --- a/packages/core/test/shell-cleanup.test.ts +++ b/packages/core/test/shell-cleanup.test.ts @@ -6,13 +6,16 @@ import { Shell } from "@opencode/core/shell" import { LayerNode } from "@opencode/util/effect/layer-node" import { FSUtil } from "@opencode/util/fs-util" import { Global } from "@opencode/util/global" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { disabledPersistentPtyLayer } from "./fixture/persistent-pty" import { withTempDir } from "./fixture/tmpdir" import { it } from "./lib/effect" const withStore = (body: (fs: FSUtil.Interface, root: string) => Effect.Effect) => withTempDir((tmp) => { - const layer = AppNodeBuilder.build(LayerNode.group([FSUtil.node, Global.node]), [ + const layer = AppNodeBuilder.build(LayerNode.group([FSUtil.node, Global.node, PersistentPty.node]), [ Global.node.replace(Global.layerWith({ data: tmp.path })), + PersistentPty.node.replace(disabledPersistentPtyLayer), ]) return Effect.gen(function* () { const fs = yield* FSUtil.Service diff --git a/packages/core/test/shell-retention.test.ts b/packages/core/test/shell-retention.test.ts index 3f776e4bb275..e650568496d1 100644 --- a/packages/core/test/shell-retention.test.ts +++ b/packages/core/test/shell-retention.test.ts @@ -10,6 +10,8 @@ import { AbsolutePath } from "@opencode/core/schema" import { Shell } from "@opencode/core/shell" import { Workspace } from "@opencode/core/workspace" import { Global } from "@opencode/util/global" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { disabledPersistentPtyLayer } from "./fixture/persistent-pty" import { hostEnvironmentLayer } from "./fixture/environment" import { tempGlobalLayer } from "./fixture/global" import { location, tempLocationLayer } from "./fixture/location" @@ -95,6 +97,7 @@ it.live("eviction makes progress past an already-removed shell", () => Global.node.replace(tempGlobalLayer), Config.node.replace(Config.testLayer()), Environment.node.replace(environment), + PersistentPty.node.replace(disabledPersistentPtyLayer), ]), ), ) diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index 08c90ba34ab5..f2429986c75e 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -37,6 +37,8 @@ import { Shell } from "@opencode/core/shell" import { ShellSelect } from "@opencode/core/shell/select" import { ID } from "@opencode/schema/shell" import { ShellTool } from "@opencode/core/tool/plugin/shell" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { disabledPersistentPtyLayer } from "./fixture/persistent-pty" import { ToolOutput } from "@opencode/core/tool-output" import { Tool } from "@opencode/core/tool" import { ToolInterrupted } from "@opencode/core/tool/interrupted" @@ -167,6 +169,14 @@ const productionIt = testEffect(AppNodeBuilder.build(nodes, replacements)) const it = testEffect( AppNodeBuilder.build(nodes, [...replacements, PluginSupervisor.node.replace(shellPluginSupervisor)]), ) +// Without the process daemon, so commands are spawned by the server. +const spawnerIt = testEffect( + AppNodeBuilder.build(nodes, [ + ...replacements, + PluginSupervisor.node.replace(shellPluginSupervisor), + PersistentPty.node.replace(disabledPersistentPtyLayer), + ]), +) const permissionIt = testEffect( AppNodeBuilder.build(LayerNode.group([nodes, PermissionSaved.node]), [ SessionExecution.node.replace(executionNode), @@ -208,6 +218,19 @@ const call = (input: typeof ShellTool.Input.Type, id = "call-shell") => ({ }) const isWindows = process.platform === "win32" + +// The process daemon keeps a finished command's zombie until the command is forgotten. +const running = (pid: number) => { + try { + process.kill(pid, 0) + } catch { + return false + } + return !Bun.spawnSync(["ps", "-o", "stat=", "-p", String(pid)]) + .stdout.toString() + .trim() + .startsWith("Z") +} const cwdCommand = isWindows ? "(Get-Location).Path; Start-Sleep -Milliseconds 100" : "pwd" const helloCommand = isWindows ? "[Console]::Out.Write('hello'); Start-Sleep -Milliseconds 100" : "printf hello" const stderrCommand = isWindows @@ -1574,7 +1597,7 @@ describe("ShellTool", () => { expect((yield* shell.wait(info.id).pipe(Effect.timeout("10 seconds"))).status).toBe("timeout") const pid = info.pid if (pid === undefined) throw new Error("Expected shell PID") - expect(() => process.kill(pid, 0)).toThrow() + expect(running(pid)).toBe(false) }), (info) => Effect.try(() => { @@ -1719,7 +1742,7 @@ describe("ShellTool", () => { }), ) - it.live("leaves a spawned command's interrupt unchanged while detaching", () => + spawnerIt.live("leaves a spawned command's interrupt unchanged while detaching", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => From 1b02e477774e2355e4d9e229d310675db0746e76 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 00:26:46 +0000 Subject: [PATCH 07/29] test(core): cover daemon-backed shell commands with the fake daemon --- packages/core/src/shell.ts | 2 + .../test/fixture/persistent-pty-daemon.ts | 39 +- packages/core/test/shell-daemon.test.ts | 383 ++++++++++++++++++ 3 files changed, 421 insertions(+), 3 deletions(-) create mode 100644 packages/core/test/shell-daemon.test.ts diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index d35445c16e4b..7a91190df60a 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -626,6 +626,8 @@ const layer = () => const command = daemonCommand( produce(info, (draft) => { if (created.pid !== undefined) draft.pid = created.pid + // The daemon's clock, so the command reports the same start after a restart. + draft.time.started = created.startedAt }), daemon, ) diff --git a/packages/core/test/fixture/persistent-pty-daemon.ts b/packages/core/test/fixture/persistent-pty-daemon.ts index ff5980fb882f..dcb1b46b5739 100644 --- a/packages/core/test/fixture/persistent-pty-daemon.ts +++ b/packages/core/test/fixture/persistent-pty-daemon.ts @@ -1,3 +1,4 @@ +import { appendFileSync, writeFileSync } from "node:fs" import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" import net from "node:net" import os from "node:os" @@ -109,11 +110,12 @@ type Process = { started_at: number exited_at: number | null pty?: { title: string; user_title: string | null; foreground_process: string | null; cols: number; rows: number } - pipe?: { output_path: string; output_lines: number; metadata: Record } + pipe?: { output_path: string; output_lines: number; output_error?: string; metadata: Record } } // An in-process opencode-pty that a server adopts through a handoff, so no daemon is spawned. Pipe processes -// run nothing: they stay running until `exit` or a signal ends them. +// run nothing: they stay running until `exit` or a signal ends them (SIGTERM spares those in `trap`), and +// write only what `output` gives them. export const fakeDaemon = Effect.fn(function* () { const base = yield* temporaryDirectory() const root = path.join(base, "root") @@ -122,6 +124,8 @@ export const fakeDaemon = Effect.fn(function* () { const processes: Process[] = [] const waiters = new Map() const failTerminate = new Set() + const trap = new Set() + const failCreate = { message: undefined as string | undefined } const ops: string[] = [] const requests: Record[] = [] let nextID = 1 @@ -137,6 +141,21 @@ export const fakeDaemon = Effect.fn(function* () { waiters.delete(id) } + // Appends to the output file and counts it, as the daemon's reader does; after `fail`, the file stops growing. + const output = (id: number, text: string) => { + const found = find(id) + if (!found?.pipe) return + if (found.pipe.output_error === undefined) { + appendFileSync(found.pipe.output_path, text) + found.pipe.output_lines += text.split("\n").length - 1 + } + found.output_tail += Buffer.byteLength(text) + } + const fail = (id: number, message: string) => { + const found = find(id) + if (found?.pipe) found.pipe.output_error = message + } + yield* listen(socket, (connection, request) => { ops.push(String(request.op)) requests.push(request) @@ -156,7 +175,9 @@ export const fakeDaemon = Effect.fn(function* () { if (request.op === "create_pipe") { if (typeof request.cwd !== "string" || !path.isAbsolute(request.cwd)) return { type: "error", message: "cwd must be absolute" } + if (failCreate.message !== undefined) return { type: "error", message: failCreate.message } const created = pipe(nextID++, request) + writeFileSync(String(request.output_path), "") processes.push(created) return { type: "created", process: created } } @@ -173,12 +194,18 @@ export const fakeDaemon = Effect.fn(function* () { if (!found?.pipe) return { type: "error", message: `terminal ${String(request.id)} not found` } if (found.lifecycle.status !== "running") return exited(found) waiters.set(found.id, [...(waiters.get(found.id) ?? []), connection]) + connection.once("close", () => + waiters.set( + found.id, + (waiters.get(found.id) ?? []).filter((waiter) => waiter !== connection), + ), + ) return undefined } if (request.op === "signal") { const found = find(request.id) if (!found?.pipe) return { type: "error", message: `terminal ${String(request.id)} not found` } - exit(found.id, null, String(request.signal)) + if (request.signal !== "SIGTERM" || !trap.has(found.id)) exit(found.id, null, String(request.signal)) return { type: "ok" } } if (request.op === "set_metadata") { @@ -205,9 +232,14 @@ export const fakeDaemon = Effect.fn(function* () { }) return { failTerminate, + failCreate, + trap, ops, requests, exit, + output, + fail, + processes, waiters: (id: number) => waiters.get(id)?.length ?? 0, options: { root, @@ -261,6 +293,7 @@ function exited(process: Process) { signal: process.lifecycle.signal, final_offset: process.output_tail, output_lines: process.pipe?.output_lines, + output_error: process.pipe?.output_error, exited_at: process.exited_at ?? Date.now(), } } diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts new file mode 100644 index 000000000000..940c0ee40dfe --- /dev/null +++ b/packages/core/test/shell-daemon.test.ts @@ -0,0 +1,383 @@ +import { describe, expect } from "bun:test" +import { existsSync } from "node:fs" +import { mkdir, utimes, writeFile } from "node:fs/promises" +import path from "node:path" +import { Context, Effect, Layer } from "effect" +import { ChildProcessSpawner } from "effect/process" +import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" +import { Config } from "@opencode/core/config" +import { Environment } from "@opencode/core/environment/index" +import { Location } from "@opencode/core/location" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { AbsolutePath } from "@opencode/core/schema" +import { Shell } from "@opencode/core/shell" +import { Global } from "@opencode/util/global" +import { FSUtil } from "@opencode/util/fs-util" +import { LayerNode } from "@opencode/util/effect/layer-node" +import { AppProcess } from "@opencode/util/process" +import { hostEnvironmentLayer } from "./fixture/environment" +import { location } from "./fixture/location" +import { fakeDaemon } from "./fixture/persistent-pty-daemon" +import { tmpdirScoped } from "./fixture/tmpdir" +import { it } from "./lib/effect" + +const unix = process.platform !== "win32" +const test = unix ? it.live : it.live.skip + +// One fake daemon and data directory shared by every Location (and server) a test builds. +const setup = Effect.fn(function* () { + const daemon = yield* fakeDaemon() + const data = (yield* tmpdirScoped()).path + const global = Global.layerWith({ data, state: path.join(data, "state") }) + const context = yield* Layer.build( + AppNodeBuilder.build(PersistentPty.configured(daemon.options), [Global.node.replace(global)]), + ) + const pty = Context.get(context, PersistentPty.Service) + let spawned = 0 + const environment = Layer.effect( + Environment.Service, + Effect.gen(function* () { + const host = yield* Environment.Service + return Environment.Service.of({ + ...host, + spawner: ChildProcessSpawner.make(() => + Effect.sync(() => spawned++).pipe(Effect.andThen(Effect.die("the spawner must not be used"))), + ), + }) + }), + ).pipe(Layer.provide(hostEnvironmentLayer)) + const shellLayer = (directory: string) => + AppNodeBuilder.build(Shell.node, [ + Location.node.replace( + Layer.succeed( + Location.Service, + Location.Service.of(location(Location.Ref.make({ directory: AbsolutePath.make(directory) }))), + ), + ), + Global.node.replace(global), + Config.node.replace(Config.testLayer()), + Environment.node.replace(environment), + PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, pty)), + ]) + // Runs `body` against the Location's Shell, then closes it as a restart or an eviction would. + const withShell = (directory: string, body: (shell: Shell.Interface) => Effect.Effect) => + Effect.scoped( + Effect.gen(function* () { + const shell = yield* Shell.Service + return yield* body(shell) + }).pipe(Effect.provide(shellLayer(directory))), + ) + const directory = (yield* tmpdirScoped()).path + return { + daemon, + data, + directory, + global, + pty, + withShell, + spawned: () => spawned, + created: () => daemon.requests.filter((request) => request.op === "create_pipe"), + signals: (id: number) => + daemon.requests.flatMap((request) => (request.op === "signal" && request.id === id ? [request.signal] : [])), + } +}) + +const lines = (count: number) => Array.from({ length: count }, (_, index) => `line ${index + 1}\n`).join("") + +describe("Shell on the process daemon", () => { + test("runs a command with the server's environment, an absolute cwd and its recovery metadata", () => + Effect.gen(function* () { + const env = yield* setup() + yield* Effect.promise(() => mkdir(path.join(env.directory, "sub"))) + process.env.OPENCODE_TEST_REMOVED = "1" + yield* Effect.addFinalizer(() => Effect.sync(() => delete process.env.OPENCODE_TEST_REMOVED)) + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const info = yield* shell.create( + { + shell: "/bin/sh", + command: "echo hi", + cwd: "sub", + timeout: 60_000, + metadata: { sessionID: "ses_test" }, + toolCall: { sessionID: "ses_test", assistantMessageID: "msg_test", callID: "call_test" }, + }, + (invocation) => + Effect.sync(() => { + delete invocation.env.OPENCODE_TEST_REMOVED + invocation.env.OPENCODE_TEST_UNSET = undefined + }), + ) + const [request] = env.created() + expect(request).toMatchObject({ + program: "/bin/sh", + args: ["-c", "echo hi"], + cwd: path.join(env.directory, "sub"), + output_path: info.file, + metadata: { + id: info.id, + command: "echo hi", + shell: "/bin/sh", + location: { directory: env.directory }, + metadata: { sessionID: "ses_test" }, + toolCall: { sessionID: "ses_test", assistantMessageID: "msg_test", callID: "call_test" }, + timeoutDeadline: expect.any(Number), + }, + }) + expect(request?.env).toMatchObject({ TERM: "xterm-256color", OPENCODE_TERMINAL: "1" }) + expect(request?.env).not.toHaveProperty("OPENCODE_TEST_REMOVED") + expect(request?.env).not.toHaveProperty("OPENCODE_TEST_UNSET") + expect(info).toMatchObject({ status: "running", pid: 1001 }) + expect(shell.detachable(info.id)).toBe(true) + expect(env.spawned()).toBe(0) + expect( + yield* shell.byToolCall({ sessionID: "ses_test", assistantMessageID: "msg_test", callID: "call_test" }), + ).toMatchObject({ id: info.id }) + + env.daemon.output(1, "first\nsecond") + expect(yield* shell.output(info.id)).toEqual({ + output: "first\nsecond", + cursor: 12, + size: 12, + truncated: false, + }) + expect(yield* shell.output(info.id, { cursor: 6, limit: 3 })).toEqual({ + output: "sec", + cursor: 9, + size: 12, + truncated: false, + }) + expect(yield* shell.output(info.id, { cursor: Number.MAX_SAFE_INTEGER })).toEqual({ + output: "", + cursor: 12, + size: 12, + truncated: false, + }) + expect((yield* shell.list()).map((item) => item.id)).toEqual([info.id]) + + env.daemon.exit(1, 0) + expect(yield* shell.result(info)).toMatchObject({ + info: { status: "exited", exit: 0, time: { completed: expect.any(Number) } }, + capture: { output: "first\nsecond", truncated: false, lines: { shown: 2, total: 2 } }, + }) + expect(yield* shell.list()).toEqual([]) + }), + ) + })) + + test("counts lines like the spawner and reads only what reached the file after a write error", () => + Effect.gen(function* () { + const env = yield* setup() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const long = yield* shell.create({ shell: "/bin/sh", command: "long" }) + env.daemon.output(1, lines(3000)) + env.daemon.exit(1, 0) + const result = yield* shell.result(long) + expect(result.capture?.lines).toEqual({ shown: 2000, total: 3000 }) + expect(result.capture?.output).toEndWith( + `line 3000\n\n[showing lines 1001-3000 of 3000; full output saved to ${long.file}]`, + ) + + const failed = yield* shell.create({ shell: "/bin/sh", command: "failed" }) + env.daemon.output(2, "kept\n") + env.daemon.fail(2, "No space left on device") + env.daemon.output(2, "lost\n") + env.daemon.exit(2, 1) + expect(yield* shell.result(failed)).toMatchObject({ + info: { status: "exited", exit: 1 }, + capture: { + output: `kept\n\n\n[output file ${failed.file} stopped early: No space left on device]`, + lines: { shown: 1, total: 1 }, + }, + }) + }), + ) + })) + + test("fails a command the daemon rejected instead of spawning it", () => + Effect.gen(function* () { + const env = yield* setup() + env.daemon.failCreate.message = "output path is a symlink" + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const error = yield* shell.create({ shell: "/bin/sh", command: "echo hi" }).pipe(Effect.flip) + expect(error).toBeInstanceOf(AppProcess.AppProcessError) + expect(env.spawned()).toBe(0) + expect(yield* shell.list()).toEqual([]) + }), + ) + })) + + test( + "stops a timed-out command with SIGTERM, then SIGKILL after the grace period, and keeps it", + () => + Effect.gen(function* () { + const env = yield* setup() + env.daemon.trap.add(1) + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const info = yield* shell.create({ shell: "/bin/sh", command: "trap '' TERM; sleep 60", timeout: 10 }) + const started = Date.now() + expect((yield* shell.wait(info.id)).status).toBe("timeout") + expect(Date.now() - started).toBeGreaterThanOrEqual(2900) + expect(env.signals(1)).toEqual(["SIGTERM", "SIGKILL"]) + expect(env.daemon.processes[0]?.pipe?.metadata).toMatchObject({ stopReason: "timeout" }) + expect(env.daemon.processes[0]?.pipe?.metadata).not.toHaveProperty("timeoutDeadline") + expect(yield* shell.result(info)).toMatchObject({ + info: { status: "timeout" }, + capture: { output: "(no output)" }, + }) + expect((yield* shell.result(info)).info).not.toHaveProperty("signal") + expect(env.daemon.ops).not.toContain("terminate") + expect(existsSync(info.file)).toBe(true) + }), + ) + }), + { timeout: 10_000 }, + ) + + test("forgets a command in the daemon before deleting its file, on remove and on eviction", () => + Effect.gen(function* () { + const env = yield* setup() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const running = yield* shell.create({ shell: "/bin/sh", command: "sleep 60" }) + yield* shell.remove(running.id) + expect(env.daemon.requests.filter((request) => request.op === "terminate")).toEqual([ + expect.objectContaining({ id: 1 }), + ]) + expect(existsSync(running.file)).toBe(false) + expect(env.daemon.processes).toEqual([]) + expect(yield* shell.result(running)).toMatchObject({ info: { status: "killed" }, capture: undefined }) + + const finished = yield* Effect.forEach(Array.from({ length: 26 }), (_, index) => + Effect.gen(function* () { + const info = yield* shell.create({ shell: "/bin/sh", command: `echo ${index}` }) + env.daemon.exit(index + 2, 0) + yield* shell.wait(info.id) + return info + }), + ) + // Waiters resolve before retention runs. + yield* Effect.sleep(20) + expect(yield* shell.get(finished[0]!.id).pipe(Effect.flip)).toBeInstanceOf(Shell.NotFoundError) + expect(existsSync(finished[0]!.file)).toBe(false) + expect(env.daemon.processes.map((process) => process.id)).not.toContain(2) + expect(existsSync(finished[1]!.file)).toBe(true) + }), + ) + })) + + test("takes its commands back after a restart and leaves other Locations' alone", () => + Effect.gen(function* () { + const env = yield* setup() + env.daemon.trap.add(4) + const before = yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const running = yield* shell.create({ shell: "/bin/sh", command: "running", metadata: { tag: "a" } }) + const finished = yield* shell.create({ shell: "/bin/sh", command: "finished" }) + env.daemon.output(2, "done\n") + env.daemon.exit(2, 3) + const exited = yield* shell.wait(finished.id) + const expiring = yield* shell.create({ shell: "/bin/sh", command: "expiring", timeout: 100 }) + const stopping = yield* shell.create({ shell: "/bin/sh", command: "stopping" }) + yield* shell.stop(stopping.id, "killed") + while (env.signals(4).length === 0) yield* Effect.sleep(5) + return { running, exited, expiring, stopping } + }), + ) + // Closing the Location only detached from its commands. + expect(env.daemon.ops).not.toContain("terminate") + expect(env.signals(1)).toEqual([]) + expect(env.signals(4)).toEqual(["SIGTERM"]) + yield* Effect.sleep(150) + + const other = (yield* tmpdirScoped()).path + yield* env.withShell(other, (shell) => + Effect.gen(function* () { + expect(yield* shell.get(before.running.id).pipe(Effect.flip)).toBeInstanceOf(Shell.NotFoundError) + expect(yield* shell.list()).toEqual([]) + }), + ) + + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + expect(yield* shell.get(before.running.id)).toMatchObject({ + status: "running", + command: "running", + shell: "/bin/sh", + file: before.running.file, + metadata: { tag: "a" }, + pid: before.running.pid, + }) + expect(yield* shell.get(before.exited.id)).toEqual({ ...before.exited, cwd: env.directory }) + expect((yield* shell.result(before.exited)).capture).toMatchObject({ output: "done\n", lines: { total: 1 } }) + // The deadline passed during the restart, and the interrupted stop goes straight to SIGKILL. + expect((yield* shell.wait(before.expiring.id)).status).toBe("timeout") + expect(env.signals(3)).toEqual(["SIGTERM"]) + expect((yield* shell.wait(before.stopping.id)).status).toBe("killed") + expect(env.signals(4)).toEqual(["SIGTERM", "SIGKILL"]) + + env.daemon.exit(1, 0) + expect(yield* shell.wait(before.running.id)).toMatchObject({ status: "exited", exit: 0 }) + }), + ) + })) + + test("records a command moved to the background at shutdown so the next server never times it out", () => + Effect.gen(function* () { + const env = yield* setup() + const info = yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const info = yield* shell.create({ shell: "/bin/sh", command: "sleep 60", timeout: 100 }) + yield* shell.backgroundAtShutdown(info.id) + return info + }), + ) + expect(env.daemon.processes[0]?.pipe?.metadata).toMatchObject({ backgroundedAtShutdown: true }) + expect(env.daemon.processes[0]?.pipe?.metadata).not.toHaveProperty("timeoutDeadline") + yield* Effect.sleep(150) + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + yield* Effect.sleep(50) + expect((yield* shell.get(info.id)).status).toBe("running") + expect(env.signals(1)).toEqual([]) + }), + ) + })) + + test("the sweep keeps a running command's file and forgets a finished one before deleting its file", () => + Effect.gen(function* () { + const env = yield* setup() + const ids = yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const running = yield* shell.create({ shell: "/bin/sh", command: "running" }) + const finished = yield* shell.create({ shell: "/bin/sh", command: "finished" }) + env.daemon.exit(2, 0) + yield* shell.wait(finished.id) + return { running, finished } + }), + ) + const expired = new Date(Date.now() - 8 * 24 * 60 * 60 * 1_000) + yield* Effect.promise(() => utimes(ids.running.file, expired, expired)) + yield* Effect.promise(() => utimes(ids.finished.file, expired, expired)) + const orphan = path.join(path.dirname(ids.running.file), "sh_0123456789abORPHANORPHANOR.out") + yield* Effect.promise(() => writeFile(orphan, "orphan")) + yield* Effect.promise(() => utimes(orphan, expired, expired)) + + yield* Shell.cleanup().pipe( + Effect.provide( + AppNodeBuilder.build(LayerNode.group([FSUtil.node, Global.node, PersistentPty.node]), [ + Global.node.replace(env.global), + PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, env.pty)), + ]), + ), + ) + + expect(existsSync(ids.running.file)).toBe(true) + expect(existsSync(ids.finished.file)).toBe(false) + expect(existsSync(orphan)).toBe(false) + expect(env.daemon.processes.map((process) => process.id)).toEqual([1]) + })) +}) From 9467bccceb554061b5e77f23113b4774f31caae1 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 00:32:26 +0000 Subject: [PATCH 08/29] feat(core): reattach shell commands after a planned restart --- packages/core/src/detaching.ts | 32 ++- .../core/src/session/execution/restart.ts | 118 ++++++++++- packages/core/src/tool/plugin/shell.ts | 200 ++++++++++++------ packages/core/test/fixture/shell-daemon.ts | 80 +++++++ packages/core/test/session-execution.test.ts | 158 +++++++++++++- packages/core/test/shell-daemon.test.ts | 85 +------- 6 files changed, 526 insertions(+), 147 deletions(-) create mode 100644 packages/core/test/fixture/shell-daemon.ts diff --git a/packages/core/src/detaching.ts b/packages/core/src/detaching.ts index 020fa8a57335..9d3159cbe409 100644 --- a/packages/core/src/detaching.ts +++ b/packages/core/src/detaching.ts @@ -1,8 +1,11 @@ export * as Detaching from "./detaching.js" -import { Context, Effect, Layer } from "effect" +import { Context, Duration, Effect, Layer, Scope } from "effect" import { makeGlobalNode } from "@opencode/util/effect/app-node" +// What detaching hooks may take together: shutdown should not wait on them. +const HOOK_BUDGET = Duration.millis(500) + /** * Planned-shutdown state of this server process. * @@ -14,9 +17,14 @@ import { makeGlobalNode } from "@opencode/util/effect/app-node" export interface Interface { /** Records that a daemon handoff was prepared successfully. */ readonly handoffPrepared: Effect.Effect - /** Called once shutdown starts, before any service is torn down. Returns whether this server is detaching. */ + /** + * Called once shutdown starts, before any service is torn down. Returns whether this server is detaching, + * after running the detaching hooks (best effort, bounded) when it is. + */ readonly begin: Effect.Effect readonly isDetaching: () => boolean + /** Runs `hook` when this server starts detaching, while every service is still open. Removed with the scope. */ + readonly onDetach: (hook: Effect.Effect) => Effect.Effect } export class Service extends Context.Service()("@opencode/Detaching") {} @@ -24,15 +32,31 @@ export class Service extends Context.Service()("@opencode/De export const make = Effect.sync(() => { let prepared = false let detaching = false + const hooks = new Set>() return Service.of({ handoffPrepared: Effect.sync(() => { prepared = true }), - begin: Effect.sync(() => { + begin: Effect.gen(function* () { detaching = prepared - return detaching + if (!detaching) return false + yield* Effect.forEach(hooks, (hook) => hook.pipe(Effect.catchCause(() => Effect.void)), { + concurrency: "unbounded", + discard: true, + }).pipe(Effect.timeoutOption(HOOK_BUDGET)) + return true }), isDetaching: () => detaching, + onDetach: (hook) => + Effect.acquireRelease( + Effect.sync(() => { + // A fresh reference per registration, so the same effect can be registered twice. + const entry = Effect.suspend(() => hook) + hooks.add(entry) + return entry + }), + (entry) => Effect.sync(() => hooks.delete(entry)), + ).pipe(Effect.asVoid), }) }) diff --git a/packages/core/src/session/execution/restart.ts b/packages/core/src/session/execution/restart.ts index 3bf4bae98888..9a0f4dd5055e 100644 --- a/packages/core/src/session/execution/restart.ts +++ b/packages/core/src/session/execution/restart.ts @@ -1,12 +1,19 @@ export * as SessionRestart from "./restart.js" -import { Context, Effect, Layer } from "effect" +import { Context, Effect, Layer, Option, Schema } from "effect" +import { isReadonlyArrayNonEmpty } from "effect/Array" import { makeGlobalNode } from "@opencode/util/effect/app-node" +import { Shell as ShellSchema } from "@opencode/schema/shell" import { Bus } from "../../bus.js" +import { Database } from "../../database/database.js" +import { Instance } from "../../instance/service.js" import { Job } from "../../job.js" import { Session } from "../../session.js" +import { Shell } from "../../shell.js" +import { ShellTool } from "../../tool/plugin/shell.js" import { SessionEvent } from "../event.js" import { SessionExecution } from "../execution.js" +import { SessionHistory } from "../history.js" import { SessionSchema } from "../schema.js" import { SessionStore } from "../store.js" import { ImageNotice } from "../../image/notice.js" @@ -72,6 +79,8 @@ export const layer = (options?: Options) => const bus = yield* Bus.Service const jobs = yield* Job.Service const sessions = yield* Session.Service + const instances = yield* Instance.Service + const db = (yield* Database.Service).db const scope = yield* Effect.scope const maxAttempts = options?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS @@ -99,10 +108,7 @@ export const layer = (options?: Options) => return true }) - const restartMessage = ( - background: Job.Background, - recovery: Exclude, - ) => { + const restartMessage = (background: Job.Background, recovery: Exclude) => { const state = background.status === "running" ? "cancelled" : background.status if (recovery.kind === "shell") { const text = @@ -164,6 +170,95 @@ export const layer = (options?: Options) => yield* jobs.completeBackground(background.notificationID) }) + // The Shell of a session's Location, booted so it takes back the commands the daemon still runs for it. + const shellOf = Effect.fnUntraced(function* (sessionID: SessionSchema.ID) { + const session = yield* store.get(sessionID) + if (!session) return undefined + return yield* Effect.serviceOption(Shell.Service).pipe( + instances.provide(session), + Effect.map(Option.getOrUndefined), + // The session's directory is gone: its commands are reported like any lost background work. + Effect.catchTag("FileSystem.DirectoryNotFoundError", () => Effect.succeed(undefined)), + ) + }) + + // A shell job the previous server left running is watched again if its command is still known (running or + // finished during the restart); otherwise it is reported as lost. + const recoverShellJob = Effect.fnUntraced(function* ( + background: Job.Background, + recovery: Extract, + suspended: ReadonlySet, + ) { + const shell = background.status === "running" ? yield* shellOf(recovery.sessionID) : undefined + const info = + shell && Schema.is(ShellSchema.ID)(recovery.shellID) + ? yield* shell.get(recovery.shellID).pipe(Effect.option) + : Option.none() + if (!shell || Option.isNone(info)) return yield* recoverToolJob(background, recovery) + yield* ShellTool.reattach({ + shell, + jobs, + sessions, + scope, + sessionID: recovery.sessionID, + info: info.value, + notificationID: background.notificationID, + resume: () => (suspended.has(recovery.sessionID) ? false : undefined), + }) + }) + + // A server killed while shutting down may leave a foreground shell call unsettled, its command still running + // in the daemon. Settle the call as moved to the background and watch the command, as the old server would have. + const settleBackgroundedCalls = Effect.fnUntraced(function* ( + sessionID: SessionSchema.ID, + suspended: ReadonlySet, + ) { + const calls = (yield* SessionHistory.incompleteAssistants(db, sessionID)).flatMap((message) => + message.type === "assistant" + ? message.content.flatMap((part) => + part.type === "tool" && part.name === ShellTool.name && part.state.status === "running" + ? [{ assistantMessageID: message.id, part }] + : [], + ) + : [], + ) + if (calls.length === 0) return + const shell = yield* shellOf(sessionID) + if (!shell) return + yield* Effect.forEach( + calls, + Effect.fnUntraced(function* (call) { + const info = yield* shell.byToolCall({ + sessionID, + assistantMessageID: call.assistantMessageID, + callID: call.part.id, + }) + if (!info || (yield* jobs.get(info.id))) return + const result = ShellTool.backgroundToolResult(info.id, info.file) + if (!isReadonlyArrayNonEmpty(result.content)) return + yield* bus.publish(SessionEvent.Tool.Success, { + sessionID, + assistantMessageID: call.assistantMessageID, + id: call.part.id, + content: result.content, + metadata: { shellID: info.id, status: "running", truncated: false }, + executed: call.part.executed === true, + }) + yield* shell.timeout(info.id, 0).pipe(Effect.ignore) + yield* ShellTool.reattach({ + shell, + jobs, + sessions, + scope, + sessionID, + info, + resume: () => (suspended.has(sessionID) ? false : undefined), + }) + }), + { discard: true }, + ) + }) + const recoverSubagent = Effect.fnUntraced(function* ( background: Job.Background, recovery: Extract, @@ -238,11 +333,22 @@ export const layer = (options?: Options) => Effect.fnUntraced(function* (background) { if ((yield* jobs.get(background.id))?.status === "running") return const recovery = background.recovery + if (recovery.kind === "shell") return yield* recoverShellJob(background, recovery, suspended) if (recovery.kind !== "subagent") return yield* recoverToolJob(background, recovery) yield* recoverSubagent(background, recovery, suspended) }), { discard: true }, ) + yield* Effect.forEach( + suspended, + (sessionID) => + settleBackgroundedCalls(sessionID, suspended).pipe( + Effect.catch((error) => + Effect.logWarning("failed to settle backgrounded shell calls", { sessionID, error }), + ), + ), + { discard: true }, + ) // Background completion can wake a parent, so inspect local ownership only after recovery. const resumed = yield* execution.active @@ -264,5 +370,5 @@ export const layer = (options?: Options) => export const node = makeGlobalNode({ service: Service, layer: layer(), - deps: [SessionStore.node, SessionExecution.node, Bus.node, Job.node, Session.node], + deps: [SessionStore.node, SessionExecution.node, Bus.node, Job.node, Session.node, Instance.node, Database.node], }) diff --git a/packages/core/src/tool/plugin/shell.ts b/packages/core/src/tool/plugin/shell.ts index 6441924ebc19..0a8b61f00c7d 100644 --- a/packages/core/src/tool/plugin/shell.ts +++ b/packages/core/src/tool/plugin/shell.ts @@ -14,6 +14,7 @@ import { FileAccess } from "../../file-access.js" import { Permission } from "../../permission.js" import { NonNegativeInt } from "../../schema.js" import { Session } from "../../session.js" +import { SessionMessage } from "../../session/message.js" import { SessionSchema } from "../../session/schema.js" import { Shell } from "../../shell.js" import { ShellParse } from "../../shell/parse.js" @@ -103,6 +104,130 @@ const backgroundResult = (shellID: string, file: string) => ({ status: "running" as const, }) +// The model's result for a finished command. +const completed = (shell: Shell.Interface, info: ShellSchema.Info, timeout?: number) => + Effect.gen(function* () { + const result = yield* shell.result(info) + if (!result.capture) return yield* new Shell.NotFoundError({ id: info.id }) + const output = ShellResult.output(result) + return { + ...output, + output: + output.timeout && timeout !== undefined + ? `${output.output}\n\nCommand exceeded timeout of ${timeout} ms. Retry with a larger timeout if the command is expected to take longer.` + : output.output, + status: "completed" as const, + } + }) + +// Tells the session how a background command ended, once its job settles. +const notify = Effect.fnUntraced(function* ( + sessions: Session.Interface, + jobs: Job.Interface, + input: { + readonly sessionID: SessionSchema.ID + readonly id: string + readonly shellID: string + readonly command: string + readonly settled: Deferred.Deferred + // Evaluated at delivery; `false` admits the notice without waking the session. + readonly resume?: () => false | undefined + }, +) { + const waited = yield* jobs.wait({ id: input.id }).pipe(Effect.exit) + // Teardown interrupts the job without settling it; its durable row is reported by the next server. + if (Exit.isFailure(waited)) + return Cause.hasInterrupts(waited.cause) ? undefined : yield* Effect.failCause(waited.cause) + const info = waited.value.info + if (!info || info.status === "running") return + const output = info.status === "completed" ? yield* Deferred.await(input.settled) : undefined + const text = output + ? resultMessages(output).join("\n\n") + : info.status === "error" + ? (info.error ?? "Command failed") + : "Cancelled" + const resume = input.resume?.() + yield* sessions.synthetic({ + ...(info.notificationID ? { id: info.notificationID } : {}), + sessionID: input.sessionID, + description: input.command, + ...ShellResult.notification({ + jobID: input.id, + shellID: input.shellID, + command: input.command, + state: info.status, + text, + output, + }), + ...(resume === false ? { resume } : {}), + }) + if (info.notificationID) yield* jobs.completeBackground(info.notificationID) +}) + +const startJob = ( + jobs: Job.Interface, + input: { + readonly sessionID: SessionSchema.ID + readonly info: ShellSchema.Info + readonly notificationID?: SessionMessage.ID + readonly run: Effect.Effect + }, +) => + jobs.start({ + // CodeMode children share a tool-call ID, but each shell must own its job. + id: input.info.id, + type: name, + title: input.info.command, + metadata: { sessionID: input.sessionID, shellID: input.info.id }, + recovery: { + kind: "shell", + sessionID: input.sessionID, + shellID: input.info.id, + command: input.info.command, + }, + ...(input.notificationID ? { notificationID: input.notificationID } : {}), + run: input.run, + }) + +/** The tool result of a call whose command moved to the background. */ +export const backgroundToolResult = (shellID: string, file: string) => toolResult(backgroundResult(shellID, file)) + +/** + * Watches a command taken over from a previous server as a background job of its session, and notifies the + * session when it finishes, as for a command the model ran in the background. A finished command notifies at once. + */ +export const reattach = Effect.fnUntraced(function* (input: { + readonly shell: Shell.Interface + readonly jobs: Job.Interface + readonly sessions: Session.Interface + readonly scope: Scope.Scope + readonly sessionID: SessionSchema.ID + readonly info: ShellSchema.Info + // The job's existing durable row, if the previous server had backgrounded it. + readonly notificationID?: SessionMessage.ID + readonly resume?: () => false | undefined +}) { + const settled = yield* Deferred.make() + yield* startJob(input.jobs, { + sessionID: input.sessionID, + info: input.info, + notificationID: input.notificationID, + run: completed(input.shell, input.info).pipe( + Effect.tap((output) => Deferred.succeed(settled, output)), + Effect.map((output) => resultMessages(output).join("\n\n")), + ), + }) + yield* input.jobs.background(input.info.id) + yield* notify(input.sessions, input.jobs, { + sessionID: input.sessionID, + id: input.info.id, + shellID: input.info.id, + command: input.info.command, + settled, + resume: input.resume, + }).pipe(Effect.forkIn(input.scope, { startImmediately: true })) +}) + export const Plugin = { id: "opencode.tool.shell", effect: Effect.fn("ShellTool.Plugin")(function* (ctx: Context) { @@ -151,43 +276,20 @@ export const Plugin = { }) const notifyWhenDone = Effect.fn("ShellTool.notifyWhenDone")( - function* ( - sessionID: SessionSchema.ID, - id: string, - shellID: string, - command: string, - settled: Deferred.Deferred, - ) { - const waited = yield* jobs.wait({ id }).pipe(Effect.exit) - // Teardown interrupts the job without settling it; its durable row is reported by the next server. - if (Exit.isFailure(waited)) - return Cause.hasInterrupts(waited.cause) ? undefined : yield* Effect.failCause(waited.cause) - const info = waited.value.info - if (!info || info.status === "running") return - const output = info.status === "completed" ? yield* Deferred.await(settled) : undefined - const text = output - ? resultMessages(output).join("\n\n") - : info.status === "error" - ? (info.error ?? "Command failed") - : "Cancelled" - yield* sessions.synthetic({ - ...(info.notificationID ? { id: info.notificationID } : {}), - sessionID, - description: command, - ...ShellResult.notification({ - jobID: id, - shellID, - command, - state: info.status, - text, - output, - }), - }) - if (info.notificationID) yield* jobs.completeBackground(info.notificationID) - }, + (sessionID: SessionSchema.ID, id: string, shellID: string, command: string, settled: Deferred.Deferred) => + notify(sessions, jobs, { sessionID, id, shellID, command, settled }), Effect.forkIn(scope, { startImmediately: true }), ) + // Foreground commands this tool is blocked on. A planned shutdown moves them to the background, so it records + // that in the daemon while it still can, for a next server that finds them without a background job. + const foreground = new Set() + yield* detaching + .onDetach( + Effect.forEach(foreground, (id) => shell.backgroundAtShutdown(id), { concurrency: "unbounded", discard: true }), + ) + .pipe(Effect.provideService(Scope.Scope, scope)) + // Settles an interrupted foreground call. The interrupt only reaches the step through ToolInterrupted. const interrupted = Effect.fnUntraced(function* (info: ShellSchema.Info, jobID: string) { if (leaveRunning(info.id)) { @@ -251,38 +353,14 @@ export const Plugin = { yield* context.progress({ shellID: info.id }) const settled = yield* Deferred.make() - const run = Effect.gen(function* () { - const result = yield* shell.result(info) - if (!result.capture) return yield* new Shell.NotFoundError({ id: info.id }) - const output = ShellResult.output(result) - return { - ...output, - output: output.timeout - ? `${output.output}\n\nCommand exceeded timeout of ${finalTimeout} ms. Retry with a larger timeout if the command is expected to take longer.` - : output.output, - status: "completed" as const, - } - }).pipe( + const run = completed(shell, info, finalTimeout).pipe( Effect.tap((output) => Deferred.succeed(settled, output)), Effect.map((output) => resultMessages(output).join("\n\n")), Effect.onInterrupt(() => leaveRunning(info.id) ? Effect.void : shell.stop(info.id, "killed").pipe(Effect.ignore), ), ) - const job = yield* jobs.start({ - // CodeMode children share a tool-call ID, but each shell must own its job. - id: info.id, - type: name, - title: info.command, - metadata: { sessionID: context.sessionID, shellID: info.id }, - recovery: { - kind: "shell", - sessionID: context.sessionID, - shellID: info.id, - command: info.command, - }, - run, - }) + const job = yield* startJob(jobs, { sessionID: context.sessionID, info, run }) if (input.background === true) { yield* jobs.background(job.id) @@ -290,8 +368,10 @@ export const Plugin = { return backgroundResult(info.id, info.file) } + foreground.add(info.id) const blocked = yield* restore(jobs.block({ id: job.id, sessionID: context.sessionID })).pipe( Effect.exit, + Effect.ensuring(Effect.sync(() => foreground.delete(info.id))), ) if (Exit.isFailure(blocked)) { if (Cause.hasInterrupts(blocked.cause)) return yield* interrupted(info, job.id) diff --git a/packages/core/test/fixture/shell-daemon.ts b/packages/core/test/fixture/shell-daemon.ts new file mode 100644 index 000000000000..5d420f37e968 --- /dev/null +++ b/packages/core/test/fixture/shell-daemon.ts @@ -0,0 +1,80 @@ +import path from "node:path" +import { Context, Effect, Layer } from "effect" +import { ChildProcessSpawner } from "effect/process" +import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" +import { Config } from "@opencode/core/config" +import { Environment } from "@opencode/core/environment/index" +import { Location } from "@opencode/core/location" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { AbsolutePath } from "@opencode/core/schema" +import { Shell } from "@opencode/core/shell" +import { Global } from "@opencode/util/global" +import { hostEnvironmentLayer } from "./environment" +import { location } from "./location" +import { fakeDaemon } from "./persistent-pty-daemon" +import { tmpdirScoped } from "./tmpdir" + +// One fake daemon and data directory shared by every Location (and server) a test builds. +export const daemonShells = Effect.fn(function* () { + const daemon = yield* fakeDaemon() + const data = (yield* tmpdirScoped()).path + const global = Global.layerWith({ data, state: path.join(data, "state") }) + const context = yield* Layer.build( + AppNodeBuilder.build(PersistentPty.configured(daemon.options), [Global.node.replace(global)]), + ) + const pty = Context.get(context, PersistentPty.Service) + let spawned = 0 + const environment = Layer.effect( + Environment.Service, + Effect.gen(function* () { + const host = yield* Environment.Service + return Environment.Service.of({ + ...host, + spawner: ChildProcessSpawner.make(() => + Effect.sync(() => spawned++).pipe(Effect.andThen(Effect.die("the spawner must not be used"))), + ), + }) + }), + ).pipe(Layer.provide(hostEnvironmentLayer)) + const shellLayer = (directory: string) => + AppNodeBuilder.build(Shell.node, [ + Location.node.replace( + Layer.succeed( + Location.Service, + Location.Service.of(location(Location.Ref.make({ directory: AbsolutePath.make(directory) }))), + ), + ), + Global.node.replace(global), + Config.node.replace(Config.testLayer()), + Environment.node.replace(environment), + PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, pty)), + ]) + // Runs `body` against the Location's Shell, then closes it as a restart or an eviction would. + // A Location's Shell that stays open for the rest of the test. + const openShell = (directory: string) => + Effect.gen(function* () { + const context = yield* Layer.build(shellLayer(directory)) + return Context.get(context, Shell.Service) + }) + const withShell = (directory: string, body: (shell: Shell.Interface) => Effect.Effect) => + Effect.scoped( + Effect.gen(function* () { + const shell = yield* Shell.Service + return yield* body(shell) + }).pipe(Effect.provide(shellLayer(directory))), + ) + const directory = (yield* tmpdirScoped()).path + return { + daemon, + data, + directory, + global, + pty, + withShell, + openShell, + spawned: () => spawned, + created: () => daemon.requests.filter((request) => request.op === "create_pipe"), + signals: (id: number) => + daemon.requests.flatMap((request) => (request.op === "signal" && request.id === id ? [request.signal] : [])), + } +}) diff --git a/packages/core/test/session-execution.test.ts b/packages/core/test/session-execution.test.ts index edc90d4495cd..2005b2639df1 100644 --- a/packages/core/test/session-execution.test.ts +++ b/packages/core/test/session-execution.test.ts @@ -22,6 +22,11 @@ import { SessionMessage } from "@opencode/core/session/message" import { SessionRunner } from "@opencode/core/session/runner/index" import { SessionInboxTable, SessionTable } from "@opencode/core/session/sql" import { SessionStore } from "@opencode/core/session/store" +import { Shell } from "@opencode/core/shell" +import { Agent } from "@opencode/core/agent" +import { ID } from "@opencode/core/model" +import { Provider } from "@opencode/core/provider" +import { daemonShells } from "./fixture/shell-daemon" import { Cause, Context, Deferred, Effect, Exit, Fiber, Layer, LayerMap, Scope } from "effect" import { eq } from "drizzle-orm" import { testEffect } from "./lib/effect" @@ -464,6 +469,151 @@ describe("SessionRestart background recovery", () => { }), ) + it.live("watches shell jobs whose commands survived the restart", () => + Effect.gen(function* () { + const database = yield* Database.Service + const store = yield* SessionStore.Service + const jobs = yield* Job.Service + const bus = yield* Bus.Service + const parent = Session.ID.make("ses_background_shell_survived") + yield* seedSessions(database, [parent]) + const daemon = yield* daemonShells() + const before = yield* daemon.withShell("/project", (shell) => + Effect.gen(function* () { + const running = yield* shell.create({ shell: "/bin/sh", command: "running", cwd: "/tmp" }) + const finished = yield* shell.create({ shell: "/bin/sh", command: "finished", cwd: "/tmp" }) + return { running, finished } + }), + ) + yield* seedBackground(jobs, parent, [ + { id: before.running.id, shellID: before.running.id, command: "running" }, + { id: before.finished.id, shellID: before.finished.id, command: "finished" }, + ]) + // The finished command ended while no server watched it. + daemon.daemon.output(2, "done\n") + daemon.daemon.exit(2, 0) + + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope)) + const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope)) + const context = yield* buildExecution( + scope, + ({ sessionID }) => SessionInbox.promote(database.db, bus, sessionID, "steer").pipe(Effect.asVoid), + undefined, + restarted, + shell, + ) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + + const notices = Effect.gen(function* () { + return (yield* store.context(parent)).filter( + (message) => message.type === "synthetic" && message.metadata?.source === "shell", + ) + }) + while ((yield* notices).length < 1) yield* Effect.sleep(5) + expect(yield* notices).toMatchObject([ + { + text: expect.stringContaining("done"), + metadata: { source: "shell", shellID: before.finished.id, state: "completed", exit: 0 }, + }, + ]) + expect((yield* restarted.get(before.running.id))?.status).toBe("running") + expect((yield* restarted.pendingBackground).map((item) => item.id)).toEqual([before.running.id]) + + daemon.daemon.exit(1, 2) + while ((yield* notices).length < 2) yield* Effect.sleep(5) + expect((yield* notices)[1]).toMatchObject({ + text: expect.stringContaining("Exited with code 2"), + metadata: { source: "shell", shellID: before.running.id, state: "completed", exit: 2 }, + }) + expect(yield* restarted.pendingBackground).toEqual([]) + }), + ) + + it.live("settles a shell call a killed server left running as moved to the background", () => + Effect.gen(function* () { + const database = yield* Database.Service + const store = yield* SessionStore.Service + const bus = yield* Bus.Service + const parent = Session.ID.make("ses_background_shell_killed") + yield* seedSessions(database, [parent], { time_suspended: Date.now() }) + const assistantMessageID = SessionMessage.ID.create() + yield* bus.publish(SessionEvent.Step.Started, { + sessionID: parent, + assistantMessageID, + agent: Agent.ID.make("build"), + model: { id: ID.make("fake-model"), providerID: Provider.ID.make("fake") }, + started: 0, + }) + yield* bus.publish(SessionEvent.Tool.Input.Started, { + sessionID: parent, + assistantMessageID, + id: "call-shell", + name: "shell", + }) + yield* bus.publish(SessionEvent.Tool.Called, { + sessionID: parent, + assistantMessageID, + id: "call-shell", + input: { command: "sleep 60" }, + executed: false, + }) + const daemon = yield* daemonShells() + const info = yield* daemon.withShell("/project", (shell) => + shell.create({ + shell: "/bin/sh", + command: "sleep 60", + cwd: "/tmp", + timeout: 120_000, + toolCall: { sessionID: parent, assistantMessageID, callID: "call-shell" }, + }), + ) + + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope)) + const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope)) + const context = yield* buildExecution( + scope, + ({ sessionID }) => SessionInbox.promote(database.db, bus, sessionID, "steer").pipe(Effect.asVoid), + undefined, + restarted, + shell, + ) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + + const [assistant] = (yield* store.context(parent)).filter((message) => message.type === "assistant") + expect(assistant?.type === "assistant" ? assistant.content : []).toMatchObject([ + { + type: "tool", + id: "call-shell", + state: { + status: "completed", + content: [ + { + type: "text", + text: `Command moved to the background (shell ID: ${info.id}).\nOutput is streaming to: ${info.file}`, + }, + { type: "text", text: expect.stringContaining("You will be notified automatically") }, + ], + }, + }, + ]) + expect((yield* restarted.get(info.id))?.status).toBe("running") + expect(daemon.daemon.processes[0]?.pipe?.metadata).not.toHaveProperty("timeoutDeadline") + + daemon.daemon.exit(1, 0) + const notices = Effect.gen(function* () { + return (yield* store.context(parent)).filter( + (message) => message.type === "synthetic" && message.metadata?.source === "shell", + ) + }) + while ((yield* notices).length < 1) yield* Effect.sleep(5) + expect((yield* notices)[0]).toMatchObject({ metadata: { source: "shell", shellID: info.id, state: "completed" } }) + }), + ) + it.effect("preserves locally running background work", () => Effect.gen(function* () { const database = yield* Database.Service @@ -1378,6 +1528,7 @@ function buildExecution( drain: (input: Parameters[0]) => Effect.Effect, options?: SessionRestart.Options, overrideJobs?: Job.Interface, + shell?: Shell.Interface, ) { return Effect.gen(function* () { const database = yield* Database.Service @@ -1408,9 +1559,12 @@ function buildExecution( LocationServiceMap.Service, LayerMap.make( () => - // The local execution test only needs the Session runner from the Location graph. + // The local execution test only needs the Session runner (and the Shell, if given) from the Location graph. // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion - runner as unknown as Layer.Layer, + (shell ? Layer.merge(runner, Layer.succeed(Shell.Service, shell)) : runner) as unknown as Layer.Layer< + LocationServices, + FileSystem.DirectoryNotFoundError + >, ), ) return yield* Layer.buildWithScope( diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts index 940c0ee40dfe..f5bffe1be198 100644 --- a/packages/core/test/shell-daemon.test.ts +++ b/packages/core/test/shell-daemon.test.ts @@ -2,92 +2,27 @@ import { describe, expect } from "bun:test" import { existsSync } from "node:fs" import { mkdir, utimes, writeFile } from "node:fs/promises" import path from "node:path" -import { Context, Effect, Layer } from "effect" -import { ChildProcessSpawner } from "effect/process" +import { Effect, Layer } from "effect" import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" -import { Config } from "@opencode/core/config" -import { Environment } from "@opencode/core/environment/index" -import { Location } from "@opencode/core/location" import { PersistentPty } from "@opencode/core/persistent-pty" -import { AbsolutePath } from "@opencode/core/schema" import { Shell } from "@opencode/core/shell" import { Global } from "@opencode/util/global" import { FSUtil } from "@opencode/util/fs-util" import { LayerNode } from "@opencode/util/effect/layer-node" import { AppProcess } from "@opencode/util/process" -import { hostEnvironmentLayer } from "./fixture/environment" -import { location } from "./fixture/location" -import { fakeDaemon } from "./fixture/persistent-pty-daemon" +import { daemonShells } from "./fixture/shell-daemon" import { tmpdirScoped } from "./fixture/tmpdir" import { it } from "./lib/effect" const unix = process.platform !== "win32" const test = unix ? it.live : it.live.skip -// One fake daemon and data directory shared by every Location (and server) a test builds. -const setup = Effect.fn(function* () { - const daemon = yield* fakeDaemon() - const data = (yield* tmpdirScoped()).path - const global = Global.layerWith({ data, state: path.join(data, "state") }) - const context = yield* Layer.build( - AppNodeBuilder.build(PersistentPty.configured(daemon.options), [Global.node.replace(global)]), - ) - const pty = Context.get(context, PersistentPty.Service) - let spawned = 0 - const environment = Layer.effect( - Environment.Service, - Effect.gen(function* () { - const host = yield* Environment.Service - return Environment.Service.of({ - ...host, - spawner: ChildProcessSpawner.make(() => - Effect.sync(() => spawned++).pipe(Effect.andThen(Effect.die("the spawner must not be used"))), - ), - }) - }), - ).pipe(Layer.provide(hostEnvironmentLayer)) - const shellLayer = (directory: string) => - AppNodeBuilder.build(Shell.node, [ - Location.node.replace( - Layer.succeed( - Location.Service, - Location.Service.of(location(Location.Ref.make({ directory: AbsolutePath.make(directory) }))), - ), - ), - Global.node.replace(global), - Config.node.replace(Config.testLayer()), - Environment.node.replace(environment), - PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, pty)), - ]) - // Runs `body` against the Location's Shell, then closes it as a restart or an eviction would. - const withShell = (directory: string, body: (shell: Shell.Interface) => Effect.Effect) => - Effect.scoped( - Effect.gen(function* () { - const shell = yield* Shell.Service - return yield* body(shell) - }).pipe(Effect.provide(shellLayer(directory))), - ) - const directory = (yield* tmpdirScoped()).path - return { - daemon, - data, - directory, - global, - pty, - withShell, - spawned: () => spawned, - created: () => daemon.requests.filter((request) => request.op === "create_pipe"), - signals: (id: number) => - daemon.requests.flatMap((request) => (request.op === "signal" && request.id === id ? [request.signal] : [])), - } -}) - const lines = (count: number) => Array.from({ length: count }, (_, index) => `line ${index + 1}\n`).join("") describe("Shell on the process daemon", () => { test("runs a command with the server's environment, an absolute cwd and its recovery metadata", () => Effect.gen(function* () { - const env = yield* setup() + const env = yield* daemonShells() yield* Effect.promise(() => mkdir(path.join(env.directory, "sub"))) process.env.OPENCODE_TEST_REMOVED = "1" yield* Effect.addFinalizer(() => Effect.sync(() => delete process.env.OPENCODE_TEST_REMOVED)) @@ -167,7 +102,7 @@ describe("Shell on the process daemon", () => { test("counts lines like the spawner and reads only what reached the file after a write error", () => Effect.gen(function* () { - const env = yield* setup() + const env = yield* daemonShells() yield* env.withShell(env.directory, (shell) => Effect.gen(function* () { const long = yield* shell.create({ shell: "/bin/sh", command: "long" }) @@ -197,7 +132,7 @@ describe("Shell on the process daemon", () => { test("fails a command the daemon rejected instead of spawning it", () => Effect.gen(function* () { - const env = yield* setup() + const env = yield* daemonShells() env.daemon.failCreate.message = "output path is a symlink" yield* env.withShell(env.directory, (shell) => Effect.gen(function* () { @@ -213,7 +148,7 @@ describe("Shell on the process daemon", () => { "stops a timed-out command with SIGTERM, then SIGKILL after the grace period, and keeps it", () => Effect.gen(function* () { - const env = yield* setup() + const env = yield* daemonShells() env.daemon.trap.add(1) yield* env.withShell(env.directory, (shell) => Effect.gen(function* () { @@ -239,7 +174,7 @@ describe("Shell on the process daemon", () => { test("forgets a command in the daemon before deleting its file, on remove and on eviction", () => Effect.gen(function* () { - const env = yield* setup() + const env = yield* daemonShells() yield* env.withShell(env.directory, (shell) => Effect.gen(function* () { const running = yield* shell.create({ shell: "/bin/sh", command: "sleep 60" }) @@ -271,7 +206,7 @@ describe("Shell on the process daemon", () => { test("takes its commands back after a restart and leaves other Locations' alone", () => Effect.gen(function* () { - const env = yield* setup() + const env = yield* daemonShells() env.daemon.trap.add(4) const before = yield* env.withShell(env.directory, (shell) => Effect.gen(function* () { @@ -327,7 +262,7 @@ describe("Shell on the process daemon", () => { test("records a command moved to the background at shutdown so the next server never times it out", () => Effect.gen(function* () { - const env = yield* setup() + const env = yield* daemonShells() const info = yield* env.withShell(env.directory, (shell) => Effect.gen(function* () { const info = yield* shell.create({ shell: "/bin/sh", command: "sleep 60", timeout: 100 }) @@ -349,7 +284,7 @@ describe("Shell on the process daemon", () => { test("the sweep keeps a running command's file and forgets a finished one before deleting its file", () => Effect.gen(function* () { - const env = yield* setup() + const env = yield* daemonShells() const ids = yield* env.withShell(env.directory, (shell) => Effect.gen(function* () { const running = yield* shell.create({ shell: "/bin/sh", command: "running" }) From 81c2b7ebfa2102fe6ce877104d651127d1dff7b5 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 00:34:24 +0000 Subject: [PATCH 09/29] test(core): cover planned-shutdown recording and the real daemon path --- packages/core/test/detaching.test.ts | 41 +++++++++++++++ packages/core/test/shell-daemon.test.ts | 67 ++++++++++++++++++++++++- packages/core/test/tool-shell.test.ts | 36 +++++++++++++ 3 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 packages/core/test/detaching.test.ts diff --git a/packages/core/test/detaching.test.ts b/packages/core/test/detaching.test.ts new file mode 100644 index 000000000000..3bbea9c85be6 --- /dev/null +++ b/packages/core/test/detaching.test.ts @@ -0,0 +1,41 @@ +import { describe, expect } from "bun:test" +import { Effect, Exit, Scope } from "effect" +import { Detaching } from "@opencode/core/detaching" +import { it } from "./lib/effect" + +describe("Detaching", () => { + it.live("runs its hooks only when a prepared handoff makes shutdown detach", () => + Effect.gen(function* () { + const quit = yield* Detaching.make + let ran = 0 + yield* quit.onDetach(Effect.sync(() => ran++)) + expect(yield* quit.begin).toBe(false) + expect(ran).toBe(0) + + const detaching = yield* Detaching.make + yield* detaching.onDetach(Effect.sync(() => ran++)) + yield* detaching.onDetach(Effect.die("a failing hook does not stop the others")) + yield* detaching.onDetach(Effect.sync(() => ran++)) + yield* detaching.handoffPrepared + expect(yield* detaching.begin).toBe(true) + expect(detaching.isDetaching()).toBe(true) + expect(ran).toBe(2) + }), + ) + + it.live("bounds slow hooks and forgets hooks whose scope closed", () => + Effect.gen(function* () { + const detaching = yield* Detaching.make + let ran = 0 + const scope = yield* Scope.make() + yield* detaching.onDetach(Effect.sync(() => ran++)).pipe(Effect.provideService(Scope.Scope, scope)) + yield* Scope.close(scope, Exit.void) + yield* detaching.onDetach(Effect.never) + yield* detaching.handoffPrepared + const started = Date.now() + expect(yield* detaching.begin).toBe(true) + expect(Date.now() - started).toBeLessThan(2_000) + expect(ran).toBe(0) + }), + ) +}) diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts index f5bffe1be198..3a1dc14c8349 100644 --- a/packages/core/test/shell-daemon.test.ts +++ b/packages/core/test/shell-daemon.test.ts @@ -1,15 +1,21 @@ import { describe, expect } from "bun:test" -import { existsSync } from "node:fs" +import { existsSync, readdirSync, realpathSync } from "node:fs" import { mkdir, utimes, writeFile } from "node:fs/promises" import path from "node:path" import { Effect, Layer } from "effect" import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" import { PersistentPty } from "@opencode/core/persistent-pty" +import { Config } from "@opencode/core/config" +import { Environment } from "@opencode/core/environment/index" +import { Location } from "@opencode/core/location" +import { AbsolutePath } from "@opencode/core/schema" import { Shell } from "@opencode/core/shell" import { Global } from "@opencode/util/global" import { FSUtil } from "@opencode/util/fs-util" import { LayerNode } from "@opencode/util/effect/layer-node" import { AppProcess } from "@opencode/util/process" +import { hostEnvironmentLayer } from "./fixture/environment" +import { location } from "./fixture/location" import { daemonShells } from "./fixture/shell-daemon" import { tmpdirScoped } from "./fixture/tmpdir" import { it } from "./lib/effect" @@ -316,3 +322,62 @@ describe("Shell on the process daemon", () => { expect(env.daemon.processes.map((process) => process.id)).toEqual([1]) })) }) + +// Against the real opencode-pty, when OPENCODE_PTY_BIN names a v8 build. +const real = unix && process.env.OPENCODE_PTY_BIN ? it.live : it.live.skip + +describe("Shell on a real process daemon", () => { + real( + "starts the daemon on the first command, never on boot or sweep, and runs it with the server's environment", + () => + Effect.gen(function* () { + const data = (yield* tmpdirScoped()).path + const directory = (yield* tmpdirScoped()).path + const state = path.join(data, "state") + const global = Global.layerWith({ data, state }) + const started = () => existsSync(path.join(state, "pty")) && readdirSync(path.join(state, "pty")).length > 0 + process.env.OPENCODE_TEST_REMOVED = "1" + yield* Effect.addFinalizer(() => Effect.sync(() => delete process.env.OPENCODE_TEST_REMOVED)) + yield* Effect.gen(function* () { + const shell = yield* Shell.Service + yield* Shell.cleanup() + expect(started()).toBe(false) + const info = yield* shell.create( + { + shell: "/bin/sh", + command: 'printf "%s|%s|%s\\n" "${OPENCODE_TEST_REMOVED:-unset}" "$TERM" "$(pwd)"; printf tail', + }, + (invocation) => + Effect.sync(() => { + delete invocation.env.OPENCODE_TEST_REMOVED + }), + ) + expect(started()).toBe(true) + expect(shell.detachable(info.id)).toBe(true) + expect(yield* shell.result(info)).toMatchObject({ + info: { status: "exited", exit: 0 }, + capture: { + output: `unset|xterm-256color|${realpathSync(directory)}\ntail`, + lines: { shown: 2, total: 2 }, + }, + }) + }).pipe( + Effect.scoped, + Effect.provide( + AppNodeBuilder.build(LayerNode.group([Shell.node, FSUtil.node, Global.node, PersistentPty.node]), [ + Location.node.replace( + Layer.succeed( + Location.Service, + Location.Service.of(location(Location.Ref.make({ directory: AbsolutePath.make(directory) }))), + ), + ), + Global.node.replace(global), + Config.node.replace(Config.testLayer()), + Environment.node.replace(hostEnvironmentLayer), + ]), + ), + ) + }), + { timeout: 20_000 }, + ) +}) diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index f2429986c75e..8e9860f25ab8 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -148,6 +148,7 @@ const shellPluginSupervisor = makeLocationNode({ }) const nodes = LayerNode.group([ + PersistentPty.node, Database.node, Bus.node, Detaching.node, @@ -1742,6 +1743,41 @@ describe("ShellTool", () => { }), ) + it.live("records a daemon-run foreground command as backgrounded when a planned shutdown starts", () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const shell = yield* Shell.Service + const jobs = yield* Job.Service + const pty = yield* PersistentPty.Service + const detaching = yield* Detaching.Service + const { shellID, result } = yield* interruptCall( + registry, + "printf ready; sleep 60", + "call-detaching-daemon", + Effect.gen(function* () { + yield* detaching.handoffPrepared + yield* detaching.begin + }), + ) + // Only the process daemon lets a command outlive the server. + if (!shell.detachable(shellID)) return + const info = yield* shell.get(shellID) + expect(info.status).toBe("running") + expect(result?.metadata).toMatchObject({ shellID, status: "running" }) + const process = (yield* pty.listPipes()).find((item) => item.outputPath === info.file) + expect(process?.lifecycle.status).toBe("running") + expect(process?.metadata).toMatchObject({ id: shellID, backgroundedAtShutdown: true }) + expect(process?.metadata).not.toHaveProperty("timeoutDeadline") + expect(process?.metadata).not.toHaveProperty("stopReason") + expect((yield* jobs.pendingBackground).map((item) => item.id)).toEqual([shellID]) + yield* shell.remove(shellID) + }), + ) + }), + ) + spawnerIt.live("leaves a spawned command's interrupt unchanged while detaching", () => withTempDir((tmp) => { reset() From c13fa9b1791a79ddf03ec89e8cf121654e66bbf8 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 01:02:05 +0000 Subject: [PATCH 10/29] fix(core): give up at once on an incompatible opencode-pty binary --- packages/core/src/persistent-pty/daemon.ts | 34 +++++++++++-------- .../core/test/persistent-pty-daemon.test.ts | 33 ++++++++++++++++++ 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/packages/core/src/persistent-pty/daemon.ts b/packages/core/src/persistent-pty/daemon.ts index 0f7866c4727c..95382daf0a41 100644 --- a/packages/core/src/persistent-pty/daemon.ts +++ b/packages/core/src/persistent-pty/daemon.ts @@ -192,10 +192,7 @@ export interface DaemonTransport { readonly wait: (id: number) => Effect.Effect readonly shutdown: Effect.Effect readonly handoff: Effect.Effect - readonly subscribe: ( - id: number, - input: SubscribeInput, - ) => Effect.Effect + readonly subscribe: (id: number, input: SubscribeInput) => Effect.Effect } export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport")(function* ( @@ -210,6 +207,8 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" let registration: Registration | undefined let owner: Awaited> | undefined let closed = false + // Set once a daemon we spawned turned out to speak another protocol. + let incompatible: DaemonError | undefined yield* Effect.addFinalizer(() => Effect.sync(() => { closed = true @@ -273,11 +272,11 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" const child = yield* Effect.tryPromise({ try: () => new Promise>((resolve, reject) => { - const child = spawn( - executable, - ["daemon", "--runtime-dir", root, "--name", id], - { detached: true, windowsHide: true, stdio: "ignore" }, - ) + const child = spawn(executable, ["daemon", "--runtime-dir", root, "--name", id], { + detached: true, + windowsHide: true, + stdio: "ignore", + }) child.once("spawn", () => { child.unref() resolve(child) @@ -293,6 +292,11 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" const found = yield* discover().pipe( Effect.map((value) => ({ value })), Effect.catch((error) => { + // The binary we just started speaks another protocol: waiting longer cannot help. + if (error.kind === "protocol" && error.pid !== undefined && error.pid === child.pid) { + incompatible = error + return Effect.fail(error) + } last = error return Effect.succeed(undefined) }), @@ -315,11 +319,13 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" ) }) - // An unreachable daemon can still hold this directory's lock, so retry once in a fresh directory. + // An unreachable daemon, or one left by another release, can still hold this directory's lock, so retry once in a + // fresh directory. A binary that cannot be spawned, or that speaks another protocol, fails the same way again. const start = () => spawnDaemon().pipe( Effect.catch((error) => Effect.gen(function* () { + if (error.kind === "spawn" || error === incompatible) return yield* Effect.fail(error) yield* Effect.logWarning("Starting opencode-pty in a fresh runtime directory", { cause: error.message }) abandon() return yield* spawnDaemon() @@ -342,7 +348,8 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" if (registration) return registration const found = yield* discover().pipe( Effect.catch((error) => { - if (shouldStart && error.kind === "connect") return start() + // Starting the same incompatible binary again would only fail again. + if (shouldStart && error.kind === "connect") return incompatible ? Effect.fail(incompatible) : start() return Effect.fail(error) }), ) @@ -438,10 +445,7 @@ export const makeDaemonTransport = Effect.fn("PersistentPty.makeDaemonTransport" }), ) - const subscribe = Effect.fn("PersistentPty.daemon.subscribe")(function* ( - terminalID: number, - input: SubscribeInput, - ) { + const subscribe = Effect.fn("PersistentPty.daemon.subscribe")(function* (terminalID: number, input: SubscribeInput) { const attempt = withRegistration(false, (current) => Effect.tryPromise({ try: () => subscribePromise(current, terminalID, input), diff --git a/packages/core/test/persistent-pty-daemon.test.ts b/packages/core/test/persistent-pty-daemon.test.ts index ad223935d0ab..f1f03a45cb03 100644 --- a/packages/core/test/persistent-pty-daemon.test.ts +++ b/packages/core/test/persistent-pty-daemon.test.ts @@ -301,3 +301,36 @@ function temporaryRuntime() { return { root, directory } }) } + +if (process.platform !== "win32") + it.live("gives up at once on a daemon binary that speaks another protocol, and never starts it again", () => + Effect.gen(function* () { + const root = yield* temporaryDirectory() + const runtime = path.join(root, "runtime") + const spawns = path.join(root, "spawns") + const binary = path.join(root, "old-opencode-pty") + // Registers like an older release would, then idles. + yield* Effect.promise(() => + writeFile( + binary, + [ + "#!/bin/sh", + `echo spawn >> ${JSON.stringify(spawns)}`, + 'mkdir -p "$3/$5"', + `printf '{"instance_id":"old","pid":%s,"protocol":7,"socket":"none","token":"t","features":[]}' $$ > "$3/$5/service.json"`, + "exec sleep 30", + ].join("\n"), + { mode: 0o755 }, + ), + ) + const daemon = yield* makeDaemonTransport(runtime, "daemon", () => Promise.resolve(binary)) + const started = Date.now() + const first = yield* daemon.request({ op: "list", mode: "pipe" }, true).pipe(Effect.flip) + expect(first.kind).toBe("protocol") + const second = yield* daemon.request({ op: "list", mode: "pipe" }, true).pipe(Effect.flip) + expect(second.kind).toBe("protocol") + expect(Date.now() - started).toBeLessThan(2_000) + const count = yield* Effect.promise(() => Bun.file(spawns).text()) + expect(count.trim().split("\n")).toHaveLength(1) + }), + ) From 17ce2b31f56eb09771d31da6541a8a72485c629a Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 01:03:37 +0000 Subject: [PATCH 11/29] fix(core): keep an interrupted tool's settled call interrupted --- packages/core/src/session/runner/step.ts | 16 ++++++++++------ packages/core/src/tool.ts | 5 ++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/packages/core/src/session/runner/step.ts b/packages/core/src/session/runner/step.ts index 14344b83d81f..bfc101132983 100644 --- a/packages/core/src/session/runner/step.ts +++ b/packages/core/src/session/runner/step.ts @@ -123,14 +123,18 @@ export const make = Effect.gen(function* () { call: event, fiber: yield* Effect.uninterruptibleMask((restore) => restore(executeTool(event).pipe(Effect.provideService(ToolInterrupted.Slot, interrupted))).pipe( - // A tool can settle its call even when interrupted; the fiber still exits interrupted. - Effect.catchCause((cause) => - interrupted.result && Cause.hasInterruptsOnly(cause) - ? Effect.succeed(interrupted.result) - : Effect.failCause(cause), - ), Effect.flatMap(toolOutput.truncate), Effect.flatMap((outcome) => publisher.toolExecution(event.id, event.name, outcome)), + // A tool can settle its call even when interrupted; the fiber still exits interrupted, so the + // step ends as interrupted instead of starting another request. + Effect.catchCause((cause) => { + const result = interrupted.result + if (!result || !Cause.hasInterruptsOnly(cause)) return Effect.failCause(cause) + return toolOutput.truncate(result).pipe( + Effect.flatMap((outcome) => publisher.toolExecution(event.id, event.name, outcome)), + Effect.andThen(Effect.failCause(cause)), + ) + }), Effect.catchTag("Tool.Error", (error) => publisher.failTool(event.id, toSessionError(error), error.metadata).pipe(Effect.asVoid), ), diff --git a/packages/core/src/tool.ts b/packages/core/src/tool.ts index 463723972d6f..2219d1a915ec 100644 --- a/packages/core/src/tool.ts +++ b/packages/core/src/tool.ts @@ -18,6 +18,7 @@ import { SessionSchema } from "./session/schema.js" import { State } from "./state.js" import { definition, effectiveName, execute, normalizedName, normalizeContent } from "./tool/runtime.js" import { Wildcard } from "./util/wildcard.js" +import { ToolInterrupted } from "./tool/interrupted.js" export class RegistrationError extends Schema.TaggedError()("Tool.RegistrationError", { name: Schema.String, @@ -272,7 +273,9 @@ const layer = Layer.effect( const codeModeTool = codeModeEnabled ? CodeModeTool.create( codeModeInventory, - (name, tool, input, context) => run(tool, name, input, context), + // A child call cannot settle the outer CodeMode call with its own interrupted result. + (name, tool, input, context) => + run(tool, name, input, context).pipe(Effect.provideService(ToolInterrupted.Slot, undefined)), permission, ) : undefined From 805b9f1d82f9e1ecc317721856596ace9db07f67 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 01:07:29 +0000 Subject: [PATCH 12/29] fix(core): tell shutdown and Location close apart from a user interrupt --- packages/core/src/detaching.ts | 5 +++ packages/core/src/plugin/internal.ts | 2 + packages/core/src/tool/plugin/shell.ts | 49 +++++++++++++++++++------ packages/core/test/tool-shell.test.ts | 51 ++++++++++++++++++++------ 4 files changed, 85 insertions(+), 22 deletions(-) diff --git a/packages/core/src/detaching.ts b/packages/core/src/detaching.ts index 9d3159cbe409..f6b7f0769d07 100644 --- a/packages/core/src/detaching.ts +++ b/packages/core/src/detaching.ts @@ -23,6 +23,8 @@ export interface Interface { */ readonly begin: Effect.Effect readonly isDetaching: () => boolean + /** Whether shutdown has started, detaching or not. Interrupts from then on are not the user's. */ + readonly isStopping: () => boolean /** Runs `hook` when this server starts detaching, while every service is still open. Removed with the scope. */ readonly onDetach: (hook: Effect.Effect) => Effect.Effect } @@ -32,12 +34,14 @@ export class Service extends Context.Service()("@opencode/De export const make = Effect.sync(() => { let prepared = false let detaching = false + let stopping = false const hooks = new Set>() return Service.of({ handoffPrepared: Effect.sync(() => { prepared = true }), begin: Effect.gen(function* () { + stopping = true detaching = prepared if (!detaching) return false yield* Effect.forEach(hooks, (hook) => hook.pipe(Effect.catchCause(() => Effect.void)), { @@ -47,6 +51,7 @@ export const make = Effect.sync(() => { return true }), isDetaching: () => detaching, + isStopping: () => stopping, onDetach: (hook) => Effect.acquireRelease( Effect.sync(() => { diff --git a/packages/core/src/plugin/internal.ts b/packages/core/src/plugin/internal.ts index cc8110b3051d..c7f0d7894ec1 100644 --- a/packages/core/src/plugin/internal.ts +++ b/packages/core/src/plugin/internal.ts @@ -54,6 +54,7 @@ import { Image } from "../image.js" import { InstructionDiscovery } from "../instruction-discovery.js" import { Integration } from "../integration.js" import { Detaching } from "../detaching.js" +import { LocationLifecycle } from "../location-lifecycle.js" import { Job } from "../job.js" import { KV } from "../kv.js" import { Location } from "../location.js" @@ -211,6 +212,7 @@ export const requirements = LayerNode.group([ Integration.node, Detaching.node, Job.node, + LocationLifecycle.node, KV.node, llmClient, Location.node, diff --git a/packages/core/src/tool/plugin/shell.ts b/packages/core/src/tool/plugin/shell.ts index 0a8b61f00c7d..d042eb8028cd 100644 --- a/packages/core/src/tool/plugin/shell.ts +++ b/packages/core/src/tool/plugin/shell.ts @@ -6,10 +6,11 @@ import type { SessionHooks } from "@opencode/plugin/effect/session" import type { ShellCreateBefore } from "@opencode/plugin/effect/shell" import type { Shell as ShellSchema } from "@opencode/schema/shell" import type { Tool } from "@opencode/schema/tool" -import { Cause, Deferred, Effect, Exit, Schema, Scope } from "effect" +import { Cause, Deferred, Effect, Exit, Option, Schema, Scope } from "effect" import { Detaching } from "../../detaching.js" import { Environment } from "../../environment/index.js" import { Job } from "../../job.js" +import { LocationLifecycle } from "../../location-lifecycle.js" import { FileAccess } from "../../file-access.js" import { Permission } from "../../permission.js" import { NonNegativeInt } from "../../schema.js" @@ -206,8 +207,11 @@ export const reattach = Effect.fnUntraced(function* (input: { // The job's existing durable row, if the previous server had backgrounded it. readonly notificationID?: SessionMessage.ID readonly resume?: () => false | undefined + // Whether a planned shutdown leaves the command running; otherwise cancelling the job stops it. + readonly leaveRunning: () => boolean }) { const settled = yield* Deferred.make() + const id = input.info.id yield* startJob(input.jobs, { sessionID: input.sessionID, info: input.info, @@ -215,17 +219,27 @@ export const reattach = Effect.fnUntraced(function* (input: { run: completed(input.shell, input.info).pipe( Effect.tap((output) => Deferred.succeed(settled, output)), Effect.map((output) => resultMessages(output).join("\n\n")), + Effect.onInterrupt(() => + input.leaveRunning() ? Effect.void : input.shell.stop(id, "killed").pipe(Effect.ignore), + ), ), }) - yield* input.jobs.background(input.info.id) - yield* notify(input.sessions, input.jobs, { + yield* input.jobs.background(id) + const notifying = notify(input.sessions, input.jobs, { sessionID: input.sessionID, - id: input.info.id, - shellID: input.info.id, + id, + shellID: id, command: input.info.command, settled, resume: input.resume, - }).pipe(Effect.forkIn(input.scope, { startImmediately: true })) + }) + // A finished command's result is admitted before recovery moves on, like a recovered subagent's. + const current = yield* input.shell.get(id).pipe(Effect.option) + if (Option.isSome(current) && current.value.status !== "running") + return yield* notifying.pipe( + Effect.catch((error) => Effect.logWarning("failed to report a recovered shell command", { id, error })), + ) + yield* notifying.pipe(Effect.forkIn(input.scope, { startImmediately: true })) }) export const Plugin = { @@ -241,9 +255,12 @@ export const Plugin = { const compatibleShell = shellSelect.resolve({ priority: "compat" }) const permission = yield* Permission.Service const detaching = yield* Detaching.Service + const lifecycle = Option.getOrUndefined(yield* Effect.serviceOption(LocationLifecycle.Service)) // A planned shutdown leaves commands that can outlive this server running for the next one. - const leaveRunning = (id: ShellSchema.ID) => detaching.isDetaching() && shell.detachable(id) + const leaveRunning = (detachable: boolean) => detaching.isDetaching() && detachable + // Shutting down or closing the Location interrupts calls too, but that is not the user cancelling them. + const closing = () => detaching.isStopping() || lifecycle?.isClosed() === true const prepare = Effect.fn("ShellTool.prepare")(function* (invocation: ShellCreateBefore, context: Tool.Context) { const target = yield* access.resolve({ path: invocation.cwd, kind: "directory" }) @@ -291,14 +308,21 @@ export const Plugin = { .pipe(Effect.provideService(Scope.Scope, scope)) // Settles an interrupted foreground call. The interrupt only reaches the step through ToolInterrupted. - const interrupted = Effect.fnUntraced(function* (info: ShellSchema.Info, jobID: string) { - if (leaveRunning(info.id)) { + const interrupted = Effect.fnUntraced(function* ( + info: ShellSchema.Info, + jobID: string, + detachable: boolean, + cause: Cause.Cause, + ) { + if (leaveRunning(detachable)) { yield* jobs.background(jobID) yield* shell.timeout(info.id, 0).pipe(Effect.ignore) const output = backgroundResult(info.id, info.file) yield* ToolInterrupted.report(toolResult(output)) return output } + // Settled as interrupted by the step, as before; the command ends with the job's scope. + if (closing()) return yield* Effect.failCause(cause) yield* shell.stop(info.id, "killed").pipe(Effect.ignore) const result = yield* shell.result(info) // The stopped job settles from the same exit; this only releases it. @@ -350,6 +374,8 @@ export const Plugin = { finalTimeout = yield* restore(prepare(invocation, context)) }), ) + // Read once: teardown may forget the command before an interrupt is handled. + const detachable = shell.detachable(info.id) yield* context.progress({ shellID: info.id }) const settled = yield* Deferred.make() @@ -357,7 +383,7 @@ export const Plugin = { Effect.tap((output) => Deferred.succeed(settled, output)), Effect.map((output) => resultMessages(output).join("\n\n")), Effect.onInterrupt(() => - leaveRunning(info.id) ? Effect.void : shell.stop(info.id, "killed").pipe(Effect.ignore), + leaveRunning(detachable) ? Effect.void : shell.stop(info.id, "killed").pipe(Effect.ignore), ), ) const job = yield* startJob(jobs, { sessionID: context.sessionID, info, run }) @@ -374,7 +400,8 @@ export const Plugin = { Effect.ensuring(Effect.sync(() => foreground.delete(info.id))), ) if (Exit.isFailure(blocked)) { - if (Cause.hasInterrupts(blocked.cause)) return yield* interrupted(info, job.id) + if (Cause.hasInterrupts(blocked.cause)) + return yield* interrupted(info, job.id, detachable, blocked.cause) return yield* Effect.failCause(blocked.cause) } const result = blocked.value diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index 8e9860f25ab8..8de325982ec3 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -18,6 +18,7 @@ import { Global } from "@opencode/util/global" import { Location } from "@opencode/core/location" import { FileAccess } from "@opencode/core/file-access" import { LocationServiceMap } from "@opencode/core/location-service-map" +import { LocationLifecycle } from "@opencode/core/location-lifecycle" import { Model } from "@opencode/core/model" import { Provider } from "@opencode/core/provider" import { AbsolutePath } from "@opencode/core/schema" @@ -1778,29 +1779,57 @@ describe("ShellTool", () => { }), ) - spawnerIt.live("leaves a spawned command's interrupt unchanged while detaching", () => + it.live("leaves a call interrupted, not cancelled by the user, when its Location closes", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => Effect.gen(function* () { const shell = yield* Shell.Service - const detaching = yield* Detaching.Service - const { shellID, result } = yield* interruptCall( + const lifecycle = yield* LocationLifecycle.Service + const { shellID, exit, result } = yield* interruptCall( registry, "printf ready; sleep 60", - "call-detaching-spawned", - Effect.gen(function* () { - yield* detaching.handoffPrepared - yield* detaching.begin - }), + "call-location-closed", + lifecycle.shutdown, ) - expect(shell.detachable(shellID)).toBe(false) - expect((yield* shell.get(shellID)).status).toBe("killed") - expect(result?.metadata).toMatchObject({ shellID, interrupted: true }) + expect(Exit.hasInterrupts(exit)).toBe(true) + expect(result).toBeUndefined() + expect((yield* shell.get(shellID)).status).toBe("running") + yield* shell.remove(shellID) }), ) }), ) + + for (const handoff of [true, false]) + spawnerIt.live( + `leaves a spawned command's call interrupted, not cancelled by the user, on ${handoff ? "a planned restart" : "a normal quit"}`, + () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const shell = yield* Shell.Service + const detaching = yield* Detaching.Service + const { shellID, exit, result } = yield* interruptCall( + registry, + "printf ready; sleep 60", + `call-stopping-spawned-${handoff}`, + Effect.gen(function* () { + if (handoff) yield* detaching.handoffPrepared + yield* detaching.begin + }), + ) + expect(shell.detachable(shellID)).toBe(false) + // The step settles it as interrupted; no "interrupted by the user" result, no stop to wait for. + expect(Exit.hasInterrupts(exit)).toBe(true) + expect(result).toBeUndefined() + expect((yield* shell.get(shellID)).status).toBe("running") + yield* shell.remove(shellID) + }), + ) + }), + ) } it.live("backgrounds a foreground command when the session is signaled", () => From 1d4455bd73858062b2d4809cf4827e503ddd4f90 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 01:07:29 +0000 Subject: [PATCH 13/29] fix(core): settle killed shell calls before recovered children resume --- .../core/src/session/execution/restart.ts | 89 +++++++++++++------ 1 file changed, 61 insertions(+), 28 deletions(-) diff --git a/packages/core/src/session/execution/restart.ts b/packages/core/src/session/execution/restart.ts index 9a0f4dd5055e..a7557fde90f9 100644 --- a/packages/core/src/session/execution/restart.ts +++ b/packages/core/src/session/execution/restart.ts @@ -6,6 +6,7 @@ import { makeGlobalNode } from "@opencode/util/effect/app-node" import { Shell as ShellSchema } from "@opencode/schema/shell" import { Bus } from "../../bus.js" import { Database } from "../../database/database.js" +import { Detaching } from "../../detaching.js" import { Instance } from "../../instance/service.js" import { Job } from "../../job.js" import { Session } from "../../session.js" @@ -14,6 +15,7 @@ import { ShellTool } from "../../tool/plugin/shell.js" import { SessionEvent } from "../event.js" import { SessionExecution } from "../execution.js" import { SessionHistory } from "../history.js" +import { SessionMessage } from "../message.js" import { SessionSchema } from "../schema.js" import { SessionStore } from "../store.js" import { ImageNotice } from "../../image/notice.js" @@ -81,6 +83,8 @@ export const layer = (options?: Options) => const sessions = yield* Session.Service const instances = yield* Instance.Service const db = (yield* Database.Service).db + // Optional so restart built outside the application graph (tests) need not provide it. + const detaching = Option.getOrUndefined(yield* Effect.serviceOption(Detaching.Service)) const scope = yield* Effect.scope const maxAttempts = options?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS @@ -182,12 +186,32 @@ export const layer = (options?: Options) => ) }) + const reattach = ( + shell: Shell.Interface, + sessionID: SessionSchema.ID, + info: ShellSchema.Info, + suspended: ReadonlySet, + notificationID?: SessionMessage.ID, + ) => + ShellTool.reattach({ + shell, + jobs, + sessions, + scope, + sessionID, + info, + notificationID, + resume: () => (suspended.has(sessionID) ? false : undefined), + leaveRunning: () => detaching?.isDetaching() === true && shell.detachable(info.id), + }) + // A shell job the previous server left running is watched again if its command is still known (running or // finished during the restart); otherwise it is reported as lost. const recoverShellJob = Effect.fnUntraced(function* ( background: Job.Background, recovery: Extract, suspended: ReadonlySet, + reattached: Set, ) { const shell = background.status === "running" ? yield* shellOf(recovery.sessionID) : undefined const info = @@ -195,16 +219,8 @@ export const layer = (options?: Options) => ? yield* shell.get(recovery.shellID).pipe(Effect.option) : Option.none() if (!shell || Option.isNone(info)) return yield* recoverToolJob(background, recovery) - yield* ShellTool.reattach({ - shell, - jobs, - sessions, - scope, - sessionID: recovery.sessionID, - info: info.value, - notificationID: background.notificationID, - resume: () => (suspended.has(recovery.sessionID) ? false : undefined), - }) + reattached.add(info.value.id) + yield* reattach(shell, recovery.sessionID, info.value, suspended, background.notificationID) }) // A server killed while shutting down may leave a foreground shell call unsettled, its command still running @@ -212,6 +228,7 @@ export const layer = (options?: Options) => const settleBackgroundedCalls = Effect.fnUntraced(function* ( sessionID: SessionSchema.ID, suspended: ReadonlySet, + reattached: ReadonlySet, ) { const calls = (yield* SessionHistory.incompleteAssistants(db, sessionID)).flatMap((message) => message.type === "assistant" @@ -233,7 +250,7 @@ export const layer = (options?: Options) => assistantMessageID: call.assistantMessageID, callID: call.part.id, }) - if (!info || (yield* jobs.get(info.id))) return + if (!info) return const result = ShellTool.backgroundToolResult(info.id, info.file) if (!isReadonlyArrayNonEmpty(result.content)) return yield* bus.publish(SessionEvent.Tool.Success, { @@ -245,15 +262,9 @@ export const layer = (options?: Options) => executed: call.part.executed === true, }) yield* shell.timeout(info.id, 0).pipe(Effect.ignore) - yield* ShellTool.reattach({ - shell, - jobs, - sessions, - scope, - sessionID, - info, - resume: () => (suspended.has(sessionID) ? false : undefined), - }) + // Killed after backgrounding the job but before settling the call: the job was already reattached. + if (reattached.has(info.id) || (yield* jobs.get(info.id))) return + yield* reattach(shell, sessionID, info, suspended) }), { discard: true }, ) @@ -327,28 +338,41 @@ export const layer = (options?: Options) => [...(yield* store.listSuspended()), ...children].filter((sessionID) => !active.has(sessionID)), ) yield* store.releaseChildClaims(children) + const recoverable = Effect.fnUntraced(function* (background: Job.Background) { + return (yield* jobs.get(background.id))?.status !== "running" + }) + // Shell and tool outcomes first, then calls a killed server left unsettled, and only then recovered children, + // whose first resume would otherwise fail their own unsettled shell calls. + const reattached = new Set() yield* Effect.forEach( - // Admit shell outcomes before a recovered child can start its first model request. - pending.toSorted((a, b) => Number(a.recovery.kind === "subagent") - Number(b.recovery.kind === "subagent")), + pending, Effect.fnUntraced(function* (background) { - if ((yield* jobs.get(background.id))?.status === "running") return const recovery = background.recovery - if (recovery.kind === "shell") return yield* recoverShellJob(background, recovery, suspended) - if (recovery.kind !== "subagent") return yield* recoverToolJob(background, recovery) - yield* recoverSubagent(background, recovery, suspended) + if (recovery.kind === "subagent" || !(yield* recoverable(background))) return + if (recovery.kind === "shell") return yield* recoverShellJob(background, recovery, suspended, reattached) + yield* recoverToolJob(background, recovery) }), { discard: true }, ) yield* Effect.forEach( suspended, (sessionID) => - settleBackgroundedCalls(sessionID, suspended).pipe( + settleBackgroundedCalls(sessionID, suspended, reattached).pipe( Effect.catch((error) => Effect.logWarning("failed to settle backgrounded shell calls", { sessionID, error }), ), ), { discard: true }, ) + yield* Effect.forEach( + pending, + Effect.fnUntraced(function* (background) { + const recovery = background.recovery + if (recovery.kind !== "subagent" || !(yield* recoverable(background))) return + yield* recoverSubagent(background, recovery, suspended) + }), + { discard: true }, + ) // Background completion can wake a parent, so inspect local ownership only after recovery. const resumed = yield* execution.active @@ -370,5 +394,14 @@ export const layer = (options?: Options) => export const node = makeGlobalNode({ service: Service, layer: layer(), - deps: [SessionStore.node, SessionExecution.node, Bus.node, Job.node, Session.node, Instance.node, Database.node], + deps: [ + SessionStore.node, + SessionExecution.node, + Bus.node, + Job.node, + Session.node, + Instance.node, + Database.node, + Detaching.node, + ], }) From 1efd16a37cc22b876709c897c9d8a17afd54074b Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 01:07:29 +0000 Subject: [PATCH 14/29] fix(core): leave a closed Location's daemon commands to its replacement --- packages/core/src/shell.ts | 56 +++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index 7a91190df60a..d13dafdfd553 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -26,6 +26,7 @@ import { Bus } from "./bus.js" import { Environment } from "./environment/index.js" import { FileRetention } from "./file-retention.js" import { Location } from "./location.js" +import { LocationLifecycle } from "./location-lifecycle.js" import { Global } from "@opencode/util/global" import { PersistentPty } from "./persistent-pty.js" import { ShellSelect } from "./shell/select.js" @@ -221,6 +222,7 @@ const layer = () => const environments = yield* SessionEnvironment.Service const config = yield* Config.Service const pty = yield* PersistentPty.Service + const lifecycle = yield* LocationLifecycle.Service const context = yield* Effect.context() const runFork = Effect.runForkWith(context) const commands = new Map() @@ -250,6 +252,25 @@ const layer = () => }), ) + // A reload boots the Location's replacement while calls may still hold this one, and the replacement takes + // the daemon's commands back. Once closed, this one leaves their timers, stops and reporting to it, and only + // still answers its own waiters. + let closed = false + yield* lifecycle.awaitShutdown.pipe( + Effect.andThen( + Effect.gen(function* () { + closed = true + for (const command of commands.values()) { + if (!command.daemon) continue + if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) + command.timeoutFiber = undefined + if (command.daemon.kill) yield* Fiber.interrupt(command.daemon.kill) + } + }), + ), + Effect.forkScoped, + ) + const require = Effect.fnUntraced(function* (id: Shell.ID) { const command = commands.get(id) if (!command) return yield* new NotFoundError({ id }) @@ -265,7 +286,10 @@ const layer = () => if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) if (command.daemon) { if (command.daemon.kill) yield* Fiber.interrupt(command.daemon.kill) - // Forget the process before deleting its file; its pending `wait` then ends as terminated. + // Stop waiting first, so forgetting the process is not mistaken for losing it. Removal never runs on the + // wait's own fiber: it settles the command through `finish`, which only evicts other commands. + if (command.daemon.wait) yield* Fiber.interrupt(command.daemon.wait) + // Forget the process before deleting its file. yield* pty.terminatePipe(command.daemon.id).pipe(Effect.ignore) yield* Effect.promise(() => unlink(command.file).catch(() => {})) yield* Deferred.fail(command.done, new NotFoundError({ id })) @@ -299,6 +323,11 @@ const layer = () => if (outcome.signal !== undefined) draft.signal = outcome.signal draft.time.completed = outcome.completed }) + // The Location's replacement reports it and applies retention. + if (closed && command.daemon) { + yield* Deferred.succeed(command.done, command.info) + return + } // Resolve waiters before retention eviction, so an evicted command still reports its // terminal Info rather than the removal NotFoundError. yield* Deferred.succeed(command.done, command.info) @@ -498,16 +527,14 @@ const layer = () => // Cleared fields are left out of the JSON. const store = (daemon: Daemon, patch: Partial) => { daemon.stored = { ...daemon.stored, ...patch } - return pty - .setPipeMetadata(daemon.id, daemon.stored) - .pipe( - Effect.catch((error) => - Effect.logWarning("failed to update shell command metadata", { - id: daemon.stored.id, - error: error.message, - }), - ), - ) + return pty.setPipeMetadata(daemon.id, daemon.stored).pipe( + Effect.catch((error) => + Effect.logWarning("failed to update shell command metadata", { + id: daemon.stored.id, + error: error.message, + }), + ), + ) } // Watches a command the daemon runs: its exit, timeout (in ms from now, none if undefined) and graceful @@ -526,7 +553,7 @@ const layer = () => // recorded stop. Only removal and teardown interrupt it. const stop = (reason: StopReason) => Effect.sync(() => { - if (daemon.stored.stopReason !== undefined) return + if (closed || daemon.stored.stopReason !== undefined) return daemon.stored = { ...daemon.stored, stopReason: reason } daemon.kill = runFork(stopping(reason)) }) @@ -613,7 +640,7 @@ const layer = () => const created = yield* pty.createPipe({ program: invocation.shell, args, - cwd: path.resolve(location.directory, invocation.cwd), + cwd: invocation.cwd, // The daemon replaces the environment, so variables removed by hooks stay removed, like the spawner. env: Object.fromEntries( Object.entries(invocation.env).filter((entry): entry is [string, string] => entry[1] !== undefined), @@ -722,6 +749,8 @@ const layer = () => } yield* hooks.trigger("shell", "create.before", invocation) if (before) yield* before(invocation) + // Absolute for local commands, so the command reports the same cwd after a restart, wherever it runs. + if (location.workspaceID === undefined) invocation.cwd = path.resolve(location.directory, invocation.cwd) const id = Shell.ID.ascending() const args = ShellSelect.args(invocation.shell, invocation.command) @@ -913,6 +942,7 @@ export const node = makeLocationNode({ SessionEnvironment.node, Config.node, PersistentPty.node, + LocationLifecycle.node, cleanupNode, ], }) From f94671a1d9be608ae53bc1024fdc6d01a1a3bf29 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 01:08:34 +0000 Subject: [PATCH 15/29] test(core): cover shell recovery ordering after a killed server --- packages/core/test/session-execution.test.ts | 166 +++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/packages/core/test/session-execution.test.ts b/packages/core/test/session-execution.test.ts index 2005b2639df1..8a85255a2d5d 100644 --- a/packages/core/test/session-execution.test.ts +++ b/packages/core/test/session-execution.test.ts @@ -614,6 +614,126 @@ describe("SessionRestart background recovery", () => { }), ) + for (const exited of [false, true]) + it.live( + `reattaches once a ${exited ? "finished" : "running"} shell a killed server backgrounded without settling its call`, + () => + Effect.gen(function* () { + const database = yield* Database.Service + const store = yield* SessionStore.Service + const bus = yield* Bus.Service + const jobs = yield* Job.Service + const parent = Session.ID.make(`ses_background_shell_unsettled_${exited}`) + yield* seedSessions(database, [parent], { time_suspended: Date.now() }) + const assistantMessageID = yield* seedShellCall(bus, parent, "call-shell") + const daemon = yield* daemonShells() + const info = yield* daemon.withShell("/project", (shell) => + shell.create({ + shell: "/bin/sh", + command: "work", + cwd: "/tmp", + toolCall: { sessionID: parent, assistantMessageID, callID: "call-shell" }, + }), + ) + // The old server persisted the background job, then died before the step recorded the call's result. + yield* seedBackground(jobs, parent, [{ id: info.id, shellID: info.id, command: "work" }]) + const [row] = yield* jobs.pendingBackground + if (exited) { + daemon.daemon.output(1, "finished output\n") + daemon.daemon.exit(1, 0) + } + + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope)) + const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope)) + const context = yield* buildExecution( + scope, + ({ sessionID }) => SessionInbox.promote(database.db, bus, sessionID, "steer").pipe(Effect.asVoid), + undefined, + restarted, + shell, + ) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + + expect(yield* shellCallState(store, parent)).toMatchObject({ status: "completed" }) + const notices = shellNotices(store, parent) + // A finished command's result is admitted during recovery, before any session resumes. + if (exited) + expect((yield* SessionInbox.list(database.db, parent)).length + (yield* notices).length).toBeGreaterThan(0) + if (!exited) { + expect((yield* restarted.pendingBackground).map((item) => item.notificationID)).toEqual([ + row?.notificationID, + ]) + daemon.daemon.exit(1, 0) + } + while ((yield* notices).length < 1) yield* Effect.sleep(5) + yield* Effect.sleep(50) + expect(yield* notices).toMatchObject([{ id: row?.notificationID, metadata: { shellID: info.id } }]) + if (exited) expect(yield* notices).toMatchObject([{ text: expect.stringContaining("finished output") }]) + expect(yield* restarted.pendingBackground).toEqual([]) + }), + ) + + it.live("settles a recovered child's unsettled shell call before the child resumes", () => + Effect.gen(function* () { + const database = yield* Database.Service + const store = yield* SessionStore.Service + const bus = yield* Bus.Service + const jobs = yield* Job.Service + const parent = Session.ID.make("ses_background_shell_child_parent") + const child = Session.ID.make("ses_background_shell_child") + yield* seedSessions(database, [parent]) + yield* seedSessions(database, [child], { parent_id: parent, time_suspended: Date.now() }) + yield* jobs.start({ + id: child, + type: "subagent", + recovery: { + kind: "subagent", + parentSessionID: parent, + childSessionID: child, + agent: "explore", + description: "Inspect recovery", + }, + run: Effect.never, + }) + yield* jobs.background(child) + const assistantMessageID = yield* seedShellCall(bus, child, "call-child-shell") + const daemon = yield* daemonShells() + const info = yield* daemon.withShell("/project", (shell) => + shell.create({ + shell: "/bin/sh", + command: "child work", + cwd: "/tmp", + toolCall: { sessionID: child, assistantMessageID, callID: "call-child-shell" }, + }), + ) + + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope)) + const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope)) + const seen = yield* Deferred.make() + const context = yield* buildExecution( + scope, + ({ sessionID }) => + sessionID === child + ? shellCallState(store, child).pipe(Effect.flatMap((state) => Deferred.succeed(seen, state))) + : Effect.void, + undefined, + restarted, + shell, + ) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + expect(yield* Deferred.await(seen).pipe(Effect.timeout("2 seconds"))).toMatchObject({ status: "completed" }) + expect((yield* restarted.get(info.id))?.status).toBe("running") + // Cancelling a reattached job stops its command. + yield* restarted.cancel(info.id) + while (daemon.signals(1).length === 0) yield* Effect.sleep(5) + expect(daemon.signals(1)).toEqual(["SIGTERM"]) + }), + ) + it.effect("preserves locally running background work", () => Effect.gen(function* () { const database = yield* Database.Service @@ -1584,3 +1704,49 @@ function buildExecution( ) }) } + +/** Records a shell call the model made that is still running, as a killed server leaves it. */ +function seedShellCall(bus: Bus.Interface, sessionID: Session.ID, callID: string) { + return Effect.gen(function* () { + const assistantMessageID = SessionMessage.ID.create() + yield* bus.publish(SessionEvent.Step.Started, { + sessionID, + assistantMessageID, + agent: Agent.ID.make("build"), + model: { id: ID.make("fake-model"), providerID: Provider.ID.make("fake") }, + started: 0, + }) + yield* bus.publish(SessionEvent.Tool.Input.Started, { sessionID, assistantMessageID, id: callID, name: "shell" }) + yield* bus.publish(SessionEvent.Tool.Called, { + sessionID, + assistantMessageID, + id: callID, + input: { command: "work" }, + executed: false, + }) + return assistantMessageID + }) +} + +function shellCallState(store: SessionStore.Interface, sessionID: Session.ID) { + return store.context(sessionID).pipe( + Effect.map((messages) => + messages.flatMap((message) => + message.type === "assistant" + ? message.content.flatMap((part) => (part.type === "tool" ? [part.state] : [])) + : [], + ), + ), + Effect.map((states) => states[0]), + ) +} + +function shellNotices(store: SessionStore.Interface, sessionID: Session.ID) { + return store + .context(sessionID) + .pipe( + Effect.map((messages) => + messages.filter((message) => message.type === "synthetic" && message.metadata?.source === "shell"), + ), + ) +} From d9868362e9e43dba76d071eaca2292ca64ec3782 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 01:12:57 +0000 Subject: [PATCH 16/29] test(core): run shell daemon tests on the fake daemon and the test clock --- packages/core/src/persistent-pty/index.ts | 14 +-- packages/core/test/detaching.test.ts | 23 ++++- .../test/fixture/persistent-pty-daemon.ts | 13 ++- packages/core/test/fixture/shell-daemon.ts | 10 +- packages/core/test/shell-daemon.test.ts | 54 +++++++++- packages/core/test/tool-shell.test.ts | 98 +++++++++++++------ 6 files changed, 160 insertions(+), 52 deletions(-) diff --git a/packages/core/src/persistent-pty/index.ts b/packages/core/src/persistent-pty/index.ts index c485b5804803..5e9e7227b511 100644 --- a/packages/core/src/persistent-pty/index.ts +++ b/packages/core/src/persistent-pty/index.ts @@ -172,10 +172,7 @@ export interface Interface { readonly remove: (id: Pty.ID) => Effect.Effect readonly shutdown: () => Effect.Effect readonly handoff: () => Effect.Effect - readonly attach: ( - id: Pty.ID, - input: SubscribeInput, - ) => Effect.Effect + readonly attach: (id: Pty.ID, input: SubscribeInput) => Effect.Effect /** Starts the daemon if needed. Fall back to another spawner only on `UnavailableError`. */ readonly createPipe: (input: CreatePipeInput) => Effect.Effect /** Resolves when the process exits, on its own connection with no timeout; interrupting closes it. */ @@ -285,8 +282,9 @@ const makeLayer = (options: Options = {}) => ), ) - const resize = Effect.fn("PersistentPty.resize")((id: Pty.ID, cols: number, rows: number, attachmentID?: string) => - command(id, "resize", { attachment_id: attachmentID ?? null, cols, rows }, true), + const resize = Effect.fn("PersistentPty.resize")( + (id: Pty.ID, cols: number, rows: number, attachmentID?: string) => + command(id, "resize", { attachment_id: attachmentID ?? null, cols, rows }, true), ) const control = Effect.fn("PersistentPty.control")( @@ -531,6 +529,7 @@ const makeLayer = (options: Options = {}) => ) export const layer = makeLayer() +export const layerWith = (options: Options) => makeLayer(options) export const configured = (options?: Options) => makeGlobalNode({ service: Service, @@ -554,7 +553,8 @@ const lookup = (daemon: DaemonTransport, id: number) => ), Effect.flatMap((response) => { if (!response) return Effect.succeed(undefined) - if (response.type !== "process") return Effect.fail(new Error(`unexpected opencode-pty response: ${response.type}`)) + if (response.type !== "process") + return Effect.fail(new Error(`unexpected opencode-pty response: ${response.type}`)) return Effect.succeed(response.process) }), ) diff --git a/packages/core/test/detaching.test.ts b/packages/core/test/detaching.test.ts index 3bbea9c85be6..40e237c51c10 100644 --- a/packages/core/test/detaching.test.ts +++ b/packages/core/test/detaching.test.ts @@ -1,5 +1,6 @@ import { describe, expect } from "bun:test" -import { Effect, Exit, Scope } from "effect" +import { Effect, Exit, Fiber, Scope } from "effect" +import { TestClock } from "effect/testing" import { Detaching } from "@opencode/core/detaching" import { it } from "./lib/effect" @@ -23,7 +24,7 @@ describe("Detaching", () => { }), ) - it.live("bounds slow hooks and forgets hooks whose scope closed", () => + it.effect("bounds slow hooks and forgets hooks whose scope closed", () => Effect.gen(function* () { const detaching = yield* Detaching.make let ran = 0 @@ -32,10 +33,22 @@ describe("Detaching", () => { yield* Scope.close(scope, Exit.void) yield* detaching.onDetach(Effect.never) yield* detaching.handoffPrepared - const started = Date.now() - expect(yield* detaching.begin).toBe(true) - expect(Date.now() - started).toBeLessThan(2_000) + const begin = yield* detaching.begin.pipe(Effect.forkChild({ startImmediately: true })) + yield* TestClock.adjust("499 millis") + expect(begin.pollUnsafe()).toBeUndefined() + yield* TestClock.adjust("1 millis") + expect(yield* Fiber.join(begin)).toBe(true) expect(ran).toBe(0) }), ) + + it.effect("starts stopping on any shutdown, detaching or not", () => + Effect.gen(function* () { + const detaching = yield* Detaching.make + expect(detaching.isStopping()).toBe(false) + expect(yield* detaching.begin).toBe(false) + expect(detaching.isStopping()).toBe(true) + expect(detaching.isDetaching()).toBe(false) + }), + ) }) diff --git a/packages/core/test/fixture/persistent-pty-daemon.ts b/packages/core/test/fixture/persistent-pty-daemon.ts index dcb1b46b5739..ce8882541125 100644 --- a/packages/core/test/fixture/persistent-pty-daemon.ts +++ b/packages/core/test/fixture/persistent-pty-daemon.ts @@ -24,13 +24,15 @@ export function listen( const server = net.createServer((socket) => { sockets.add(socket) socket.once("close", () => sockets.delete(socket)) - void readRequest(socket) - .then((envelope) => { - const response = - envelope.request.op === "own" ? { type: "owned" } : handle(socket, envelope.request, envelope.token) + const serve = (): Promise => + readRequest(socket).then((envelope) => { + const owning = envelope.request.op === "own" + const response = owning ? { type: "owned" } : handle(socket, envelope.request, envelope.token) if (response) socket.write(frame(response)) + // The ownership connection stays open for later requests, such as handoff preparation. + if (owning) return serve() }) - .catch(() => socket.destroy()) + void serve().catch(() => socket.destroy()) }) await new Promise((resolve, reject) => { server.once("error", reject) @@ -182,6 +184,7 @@ export const fakeDaemon = Effect.fn(function* () { return { type: "created", process: created } } if (request.op === "control") return { type: "ok" } + if (request.op === "prepare_handoff") return { type: "handoff", ticket: "next", expires_at: Date.now() + 60_000 } if (request.op === "rename") { const found = find(request.id) if (!found?.pty) return { type: "error", message: "terminal not found" } diff --git a/packages/core/test/fixture/shell-daemon.ts b/packages/core/test/fixture/shell-daemon.ts index 5d420f37e968..00eb71fe7db4 100644 --- a/packages/core/test/fixture/shell-daemon.ts +++ b/packages/core/test/fixture/shell-daemon.ts @@ -8,6 +8,9 @@ import { Location } from "@opencode/core/location" import { PersistentPty } from "@opencode/core/persistent-pty" import { AbsolutePath } from "@opencode/core/schema" import { Shell } from "@opencode/core/shell" +import { Bus } from "@opencode/core/bus" +import { LayerNode } from "@opencode/util/effect/layer-node" +import { LocationLifecycle } from "@opencode/core/location-lifecycle" import { Global } from "@opencode/util/global" import { hostEnvironmentLayer } from "./environment" import { location } from "./location" @@ -37,7 +40,7 @@ export const daemonShells = Effect.fn(function* () { }), ).pipe(Layer.provide(hostEnvironmentLayer)) const shellLayer = (directory: string) => - AppNodeBuilder.build(Shell.node, [ + AppNodeBuilder.build(LayerNode.group([Shell.node, LocationLifecycle.node, Bus.node]), [ Location.node.replace( Layer.succeed( Location.Service, @@ -56,7 +59,10 @@ export const daemonShells = Effect.fn(function* () { const context = yield* Layer.build(shellLayer(directory)) return Context.get(context, Shell.Service) }) - const withShell = (directory: string, body: (shell: Shell.Interface) => Effect.Effect) => + const withShell = ( + directory: string, + body: (shell: Shell.Interface) => Effect.Effect, + ) => Effect.scoped( Effect.gen(function* () { const shell = yield* Shell.Service diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts index 3a1dc14c8349..9ff4e87bb678 100644 --- a/packages/core/test/shell-daemon.test.ts +++ b/packages/core/test/shell-daemon.test.ts @@ -3,6 +3,7 @@ import { existsSync, readdirSync, realpathSync } from "node:fs" import { mkdir, utimes, writeFile } from "node:fs/promises" import path from "node:path" import { Effect, Layer } from "effect" +import { TestClock } from "effect/testing" import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" import { PersistentPty } from "@opencode/core/persistent-pty" import { Config } from "@opencode/core/config" @@ -10,6 +11,8 @@ import { Environment } from "@opencode/core/environment/index" import { Location } from "@opencode/core/location" import { AbsolutePath } from "@opencode/core/schema" import { Shell } from "@opencode/core/shell" +import { Bus } from "@opencode/core/bus" +import { LocationLifecycle } from "@opencode/core/location-lifecycle" import { Global } from "@opencode/util/global" import { FSUtil } from "@opencode/util/fs-util" import { LayerNode } from "@opencode/util/effect/layer-node" @@ -150,18 +153,31 @@ describe("Shell on the process daemon", () => { ) })) - test( + // On the test clock: the timeout and the grace period pass without waiting for them. + ;(unix ? it.effect : it.effect.skip)( "stops a timed-out command with SIGTERM, then SIGKILL after the grace period, and keeps it", () => Effect.gen(function* () { const env = yield* daemonShells() env.daemon.trap.add(1) + // Real time for the fake daemon's socket round trips. + const until = (done: () => boolean) => + Effect.promise(async () => { + while (!done()) await Bun.sleep(5) + }) yield* env.withShell(env.directory, (shell) => Effect.gen(function* () { const info = yield* shell.create({ shell: "/bin/sh", command: "trap '' TERM; sleep 60", timeout: 10 }) - const started = Date.now() + yield* TestClock.adjust("10 millis") + yield* until(() => env.signals(1).length > 0) + expect(env.signals(1)).toEqual(["SIGTERM"]) + expect((yield* shell.get(info.id)).status).toBe("running") + // The grace period starts once the daemon acknowledged SIGTERM. + while (env.signals(1).length < 2) { + yield* TestClock.adjust("1 second") + yield* Effect.promise(() => Bun.sleep(5)) + } expect((yield* shell.wait(info.id)).status).toBe("timeout") - expect(Date.now() - started).toBeGreaterThanOrEqual(2900) expect(env.signals(1)).toEqual(["SIGTERM", "SIGKILL"]) expect(env.daemon.processes[0]?.pipe?.metadata).toMatchObject({ stopReason: "timeout" }) expect(env.daemon.processes[0]?.pipe?.metadata).not.toHaveProperty("timeoutDeadline") @@ -175,7 +191,6 @@ describe("Shell on the process daemon", () => { }), ) }), - { timeout: 10_000 }, ) test("forgets a command in the daemon before deleting its file, on remove and on eviction", () => @@ -266,6 +281,33 @@ describe("Shell on the process daemon", () => { ) })) + test("a closed Location leaves its daemon commands' timers and reporting to its replacement", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const lifecycle = yield* LocationLifecycle.Service + const bus = yield* Bus.Service + const exits: string[] = [] + yield* bus.listen((event) => + Effect.sync(() => { + if (event.type === "shell.exited") exits.push(event.type) + }), + ) + const info = yield* shell.create({ shell: "/bin/sh", command: "sleep 60", timeout: 50 }) + yield* lifecycle.shutdown + yield* Effect.sleep(100) + // The replacement owns the timeout now. + expect(env.signals(1)).toEqual([]) + env.daemon.exit(1, 0) + // A call still holding this Location gets its result, but nothing is reported twice. + expect(yield* shell.wait(info.id)).toMatchObject({ status: "exited", exit: 0 }) + yield* Effect.sleep(20) + expect(exits).toEqual([]) + }), + ) + })) + test("records a command moved to the background at shutdown so the next server never times it out", () => Effect.gen(function* () { const env = yield* daemonShells() @@ -323,7 +365,9 @@ describe("Shell on the process daemon", () => { })) }) -// Against the real opencode-pty, when OPENCODE_PTY_BIN names a v8 build. +// Against the real opencode-pty. Opt-in through OPENCODE_PTY_BIN on purpose: the installed @opencode/pty is still +// 0.4 (protocol v7) until the release bumps it, so its binaryPath would only exercise the spawner fallback. Once it +// speaks v8, fall back to binaryPath here like server/test/persistent-pty.test.ts does. const real = unix && process.env.OPENCODE_PTY_BIN ? it.live : it.live.skip describe("Shell on a real process daemon", () => { diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index 8de325982ec3..1e2757b80879 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -40,6 +40,7 @@ import { ID } from "@opencode/schema/shell" import { ShellTool } from "@opencode/core/tool/plugin/shell" import { PersistentPty } from "@opencode/core/persistent-pty" import { disabledPersistentPtyLayer } from "./fixture/persistent-pty" +import { fakeDaemon } from "./fixture/persistent-pty-daemon" import { ToolOutput } from "@opencode/core/tool-output" import { Tool } from "@opencode/core/tool" import { ToolInterrupted } from "@opencode/core/tool/interrupted" @@ -161,29 +162,62 @@ const nodes = LayerNode.group([ FSUtil.node, Global.node, ]) -const replacements = [ +const daemonReplacements = [ SessionExecution.node.replace(executionNode), Permission.node.replace(permission), Global.node.replace(tempGlobalLayer), offlineModels, ] satisfies LayerNode.Replacements +// Most suites spawn commands from the server, so CI starts no daemon for them; `parity` runs a subset on both. +const replacements = [ + ...daemonReplacements, + PersistentPty.node.replace(disabledPersistentPtyLayer), +] satisfies LayerNode.Replacements const productionIt = testEffect(AppNodeBuilder.build(nodes, replacements)) const it = testEffect( AppNodeBuilder.build(nodes, [...replacements, PluginSupervisor.node.replace(shellPluginSupervisor)]), ) -// Without the process daemon, so commands are spawned by the server. -const spawnerIt = testEffect( +// On the real process daemon (when its binary is compatible; otherwise this falls back to spawning). +const daemonIt = testEffect( + AppNodeBuilder.build(nodes, [...daemonReplacements, PluginSupervisor.node.replace(shellPluginSupervisor)]), +) +const parity = { + live: (name: string, body: Parameters[1], options?: Parameters[2]) => { + it.live(`${name} (spawned)`, body, options) + daemonIt.live(`${name} (daemon)`, body, options) + }, + skip: (name: string, body: Parameters[1], options?: Parameters[2]) => + it.live.skip(name, body, options), +} +// On an in-process fake daemon, so daemon-only behavior is covered without the real binary. +let fake: Effect.Success> | undefined +const fakeDaemonIt = testEffect( AppNodeBuilder.build(nodes, [ - ...replacements, + ...daemonReplacements, PluginSupervisor.node.replace(shellPluginSupervisor), - PersistentPty.node.replace(disabledPersistentPtyLayer), + PersistentPty.node.replace( + makeGlobalNode({ + service: PersistentPty.Service, + layer: Layer.unwrap( + fakeDaemon().pipe( + Effect.map((daemon) => { + fake = daemon + return PersistentPty.layerWith(daemon.options) + }), + ), + ), + deps: [Bus.node, Global.node, Detaching.node], + }), + ), ]), ) + const permissionIt = testEffect( AppNodeBuilder.build(LayerNode.group([nodes, PermissionSaved.node]), [ SessionExecution.node.replace(executionNode), Global.node.replace(tempGlobalLayer), PluginSupervisor.node.replace(shellPluginSupervisor), + PersistentPty.node.replace(disabledPersistentPtyLayer), offlineModels, ]), ) @@ -1296,7 +1330,7 @@ describe("ShellTool", () => { ) } - it.live("keeps non-zero exits useful", () => + parity.live("keeps non-zero exits useful", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => @@ -1338,7 +1372,7 @@ describe("ShellTool", () => { { timeout: 15_000 }, ) - it.live("uses configured line limits", () => + parity.live("uses configured line limits", () => withTempDir((tmp) => { reset() return Effect.gen(function* () { @@ -1409,7 +1443,7 @@ describe("ShellTool", () => { { timeout: 15_000 }, ) - it.live("returns the shell id for a background command", () => + parity.live("returns the shell id for a background command", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => @@ -1567,7 +1601,8 @@ describe("ShellTool", () => { ) const signalTest = isWindows ? it.live.skip : it.live - signalTest( + const signalParity = isWindows ? parity.skip : parity.live + signalParity( "escalates a shell timeout when the ready process ignores SIGTERM", () => Effect.gen(function* () { @@ -1615,7 +1650,7 @@ describe("ShellTool", () => { { timeout: 30_000 }, ) - signalTest( + signalParity( "stops a timed-out command with SIGTERM first and SIGKILL after the grace period", () => withTempDir((tmp) => @@ -1691,7 +1726,13 @@ describe("ShellTool", () => { } // Runs a foreground call the way a session step does, and interrupts it once its output is on disk. - const interruptCall = (registry: Tool.Interface, command: string, id: string, onStarted = Effect.void) => + const interruptCall = ( + registry: Tool.Interface, + command: string, + id: string, + onStarted: Effect.Effect = Effect.void, + waitForOutput = true, + ) => Effect.gen(function* () { const shell = yield* Shell.Service const slot: ToolInterrupted.Slot = {} @@ -1704,7 +1745,7 @@ describe("ShellTool", () => { : Effect.void, }).pipe(Effect.provideService(ToolInterrupted.Slot, slot), Effect.forkChild({ startImmediately: true })) const shellID = ID.make(yield* Deferred.await(started)) - while ((yield* shell.output(shellID, { cursor: Number.MAX_SAFE_INTEGER })).size === 0) + while (waitForOutput && (yield* shell.output(shellID, { cursor: Number.MAX_SAFE_INTEGER })).size === 0) yield* Effect.sleep(Duration.millis(10)) yield* onStarted yield* Fiber.interrupt(fiber) @@ -1713,7 +1754,7 @@ describe("ShellTool", () => { }) if (!isWindows) { - it.live("keeps the output of an interrupted call and settles it with the tail, file and shell ID", () => + parity.live("keeps the output of an interrupted call and settles it with the tail, file and shell ID", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => @@ -1744,7 +1785,7 @@ describe("ShellTool", () => { }), ) - it.live("records a daemon-run foreground command as backgrounded when a planned shutdown starts", () => + fakeDaemonIt.live("a prepared handoff moves a daemon-run foreground call to the background at shutdown", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => @@ -1753,33 +1794,34 @@ describe("ShellTool", () => { const jobs = yield* Job.Service const pty = yield* PersistentPty.Service const detaching = yield* Detaching.Service - const { shellID, result } = yield* interruptCall( + const { shellID, exit, result } = yield* interruptCall( registry, - "printf ready; sleep 60", + "sleep 60", "call-detaching-daemon", Effect.gen(function* () { - yield* detaching.handoffPrepared - yield* detaching.begin + expect(yield* pty.handoff()).toMatchObject({ ticket: "next" }) + expect(yield* detaching.begin).toBe(true) }), + false, ) - // Only the process daemon lets a command outlive the server. - if (!shell.detachable(shellID)) return + expect(shell.detachable(shellID)).toBe(true) + expect(Exit.hasInterrupts(exit)).toBe(true) const info = yield* shell.get(shellID) expect(info.status).toBe("running") expect(result?.metadata).toMatchObject({ shellID, status: "running" }) - const process = (yield* pty.listPipes()).find((item) => item.outputPath === info.file) + const process = fake?.processes.find((item) => item.pipe?.output_path === info.file) expect(process?.lifecycle.status).toBe("running") - expect(process?.metadata).toMatchObject({ id: shellID, backgroundedAtShutdown: true }) - expect(process?.metadata).not.toHaveProperty("timeoutDeadline") - expect(process?.metadata).not.toHaveProperty("stopReason") + expect(process?.pipe?.metadata).toMatchObject({ id: shellID, backgroundedAtShutdown: true }) + expect(process?.pipe?.metadata).not.toHaveProperty("timeoutDeadline") + expect(process?.pipe?.metadata).not.toHaveProperty("stopReason") + expect(fake?.ops).not.toContain("signal") expect((yield* jobs.pendingBackground).map((item) => item.id)).toEqual([shellID]) - yield* shell.remove(shellID) }), ) }), ) - it.live("leaves a call interrupted, not cancelled by the user, when its Location closes", () => + parity.live("leaves a call interrupted, not cancelled by the user, when its Location closes", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => @@ -1802,7 +1844,7 @@ describe("ShellTool", () => { ) for (const handoff of [true, false]) - spawnerIt.live( + it.live( `leaves a spawned command's call interrupted, not cancelled by the user, on ${handoff ? "a planned restart" : "a normal quit"}`, () => withTempDir((tmp) => { @@ -1832,7 +1874,7 @@ describe("ShellTool", () => { ) } - it.live("backgrounds a foreground command when the session is signaled", () => + parity.live("backgrounds a foreground command when the session is signaled", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => From 1ef8f938d6beff5f2b5f8948adb2d2202357889b Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 01:13:04 +0000 Subject: [PATCH 17/29] test(core): check a relative cwd resolves against the Location --- packages/core/test/shell-daemon.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts index 9ff4e87bb678..bb5f5b4a575d 100644 --- a/packages/core/test/shell-daemon.test.ts +++ b/packages/core/test/shell-daemon.test.ts @@ -71,7 +71,7 @@ describe("Shell on the process daemon", () => { expect(request?.env).toMatchObject({ TERM: "xterm-256color", OPENCODE_TERMINAL: "1" }) expect(request?.env).not.toHaveProperty("OPENCODE_TEST_REMOVED") expect(request?.env).not.toHaveProperty("OPENCODE_TEST_UNSET") - expect(info).toMatchObject({ status: "running", pid: 1001 }) + expect(info).toMatchObject({ status: "running", pid: 1001, cwd: path.join(env.directory, "sub") }) expect(shell.detachable(info.id)).toBe(true) expect(env.spawned()).toBe(0) expect( From 3050c9b44e3313807e0808e4d81f789ec67e0a02 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 17:55:46 +0000 Subject: [PATCH 18/29] chore: use @opencode/pty 0.5.0 --- bun.lock | 24 ++++++++++++------------ packages/cli/package.json | 2 +- packages/core/package.json | 2 +- packages/server/package.json | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/bun.lock b/bun.lock index 60b5189f7596..b4367a1889fd 100644 --- a/bun.lock +++ b/bun.lock @@ -119,7 +119,7 @@ "@effect/platform-node": "catalog:", "@opencode/client": "workspace:*", "@opencode/plugin": "workspace:*", - "@opencode/pty": "0.4.0", + "@opencode/pty": "0.5.0", "@opencode/schema": "workspace:*", "@opencode/server": "workspace:*", "@opencode/tui": "workspace:*", @@ -267,7 +267,7 @@ "@opencode/codemode": "workspace:*", "@opencode/plugin": "workspace:*", "@opencode/plugin-browser": "workspace:*", - "@opencode/pty": "0.4.0", + "@opencode/pty": "0.5.0", "@opencode/schema": "workspace:*", "@opencode/util": "workspace:*", "@parcel/watcher": "2.5.1", @@ -666,7 +666,7 @@ "devDependencies": { "@opencode/client": "workspace:*", "@opencode/plugin": "workspace:*", - "@opencode/pty": "0.4.0", + "@opencode/pty": "0.5.0", "@tsconfig/bun": "catalog:", "@types/bun": "catalog:", "@typescript/native": "catalog:", @@ -1957,23 +1957,23 @@ "@opencode/protocol": ["@opencode/protocol@workspace:packages/protocol"], - "@opencode/pty": ["@opencode/pty@0.4.0", "", { "optionalDependencies": { "@opencode/pty-darwin-arm64": "0.4.0", "@opencode/pty-darwin-x64": "0.4.0", "@opencode/pty-linux-arm64-gnu": "0.4.0", "@opencode/pty-linux-arm64-musl": "0.4.0", "@opencode/pty-linux-x64-gnu": "0.4.0", "@opencode/pty-linux-x64-musl": "0.4.0", "@opencode/pty-win32-arm64": "0.4.0", "@opencode/pty-win32-x64": "0.4.0" }, "bin": { "opencode-pty": "bin/opencode-pty.js" } }, "sha512-v7cLT71yfoNM5FlyPGwMjeqIJbr0DneGq4NpoOY5iAKpsevzOIcsI2E0fiDWajuk5xAFRxwyv4CpGqow7cu79Q=="], + "@opencode/pty": ["@opencode/pty@0.5.0", "", { "optionalDependencies": { "@opencode/pty-darwin-arm64": "0.5.0", "@opencode/pty-darwin-x64": "0.5.0", "@opencode/pty-linux-arm64-gnu": "0.5.0", "@opencode/pty-linux-arm64-musl": "0.5.0", "@opencode/pty-linux-x64-gnu": "0.5.0", "@opencode/pty-linux-x64-musl": "0.5.0", "@opencode/pty-win32-arm64": "0.5.0", "@opencode/pty-win32-x64": "0.5.0" }, "bin": { "opencode-pty": "bin/opencode-pty.js" } }, "sha512-2U0RfT5U1lp1BcegXLoSBYfBx9sBNw3uJ/pao4UWdNV6UDfmDNGzdc8m4K1Q4ByBkXhbh8hexa0JS7HZ0vSsgg=="], - "@opencode/pty-darwin-arm64": ["@opencode/pty-darwin-arm64@0.4.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-XzZZJcQeR42IRpxHbbPEo4AK7jdp9SpgPx234DZ6uLnl6yclXcixH70G2U88EhKtGAMyCPIXdBfvQf4OQTDvbQ=="], + "@opencode/pty-darwin-arm64": ["@opencode/pty-darwin-arm64@0.5.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-D+L/TnNUc35airWTBlLussnuDbZpKPQ2/uBHVkcycw6KrLYDb4hVoUJn8N5PXmLTnhmU64B4MUsdH9JoOZIi8w=="], - "@opencode/pty-darwin-x64": ["@opencode/pty-darwin-x64@0.4.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-NvQfygGcsGmDCIUJ/VHLcS6SduC93Y/qP2hrHjtBpiL2sSQyeCDcifNhLvqY/wxGFNNZ4GeZwwZqe7rMEuBJbQ=="], + "@opencode/pty-darwin-x64": ["@opencode/pty-darwin-x64@0.5.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-/JMtOIAY9WWszsauWnCwjl2Qr8PBihS94tLsxa8prZBVhbbpOzcFslQsqR8JnJ1gAVlYINxEVS79+QQPQ4NC9g=="], - "@opencode/pty-linux-arm64-gnu": ["@opencode/pty-linux-arm64-gnu@0.4.0", "", { "os": "linux", "cpu": "arm64", "libc": "glibc" }, "sha512-VZXkYD+CT+zdodmTnKn9/VlmswksLPBLJryeBsG4IXUFXTsATr7+PVygJk96FxCh5nq68H3a/UJgyiIUMl1XCQ=="], + "@opencode/pty-linux-arm64-gnu": ["@opencode/pty-linux-arm64-gnu@0.5.0", "", { "os": "linux", "cpu": "arm64", "libc": "glibc" }, "sha512-vNiHKopfKrX6vj8OmDXYxZ+5JEZwf5djDzt3LOqeYuwStQRX2rxcl2/zjrvd5i8rAJw30RpJ+PTdS1t08vqtLA=="], - "@opencode/pty-linux-arm64-musl": ["@opencode/pty-linux-arm64-musl@0.4.0", "", { "os": "linux", "cpu": "arm64", "libc": "musl" }, "sha512-hCx25Hn9sALdnkx2Y4aPGVw3glTbX61XRiJs+XETPPIO9QTaav4OS4k2JR2uHC6/u161xKdN82iJ+OYfwcGUSg=="], + "@opencode/pty-linux-arm64-musl": ["@opencode/pty-linux-arm64-musl@0.5.0", "", { "os": "linux", "cpu": "arm64", "libc": "musl" }, "sha512-aW3lNGjit3FNAfpEskJgvTrtE2Lfnukpn8KmUVPUBN/EbRhT56h/N3ghzHYc5jIExETy06Xpy9i2xZxZQy0McQ=="], - "@opencode/pty-linux-x64-gnu": ["@opencode/pty-linux-x64-gnu@0.4.0", "", { "os": "linux", "cpu": "x64", "libc": "glibc" }, "sha512-Xal/d0+xp6XE4iCGUAw4JTQ4uf5l8V0EIbeKq+e03cHJewOMKZk4fooQjxpZvQIToBHRtnCJAmd+EEhVxXiDDw=="], + "@opencode/pty-linux-x64-gnu": ["@opencode/pty-linux-x64-gnu@0.5.0", "", { "os": "linux", "cpu": "x64", "libc": "glibc" }, "sha512-XaMnBSh1QvJmcYKSOtx1fVvldZ+x212zAHaq8ovyafufqp6/Xl3AuqPCXwPFnycW2PfbUDwfZ7dBCXThQtOt3Q=="], - "@opencode/pty-linux-x64-musl": ["@opencode/pty-linux-x64-musl@0.4.0", "", { "os": "linux", "cpu": "x64", "libc": "musl" }, "sha512-dBVtKZoVKAN6/E+65GrJksgRKzPkPHUjWZBzf3mRANXt2adpdbV+/M1AMfPCZSju2mN1j9rNu8MCrxwr+ugEig=="], + "@opencode/pty-linux-x64-musl": ["@opencode/pty-linux-x64-musl@0.5.0", "", { "os": "linux", "cpu": "x64", "libc": "musl" }, "sha512-30LVc91GrJP8YD6chLic4+5nMsLz3hCYu2cxPQczSKyRWMyBjuwvCf040UzlRAQiGllp8KBb7nYa8SCtq2/49Q=="], - "@opencode/pty-win32-arm64": ["@opencode/pty-win32-arm64@0.4.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-IKJMUnS2aluWqRj13enMWomYtwbXq8rStcf/mUbxBoAGTDMtWR/ZrRmKb2ptR7KUqDjdSEa+8C8rczS7uGFl9g=="], + "@opencode/pty-win32-arm64": ["@opencode/pty-win32-arm64@0.5.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-BYVSytysZvXmIQcP9VbbXbOV417RLW/BATiAK8MMbHBF6fzqQlpAMTi4cQmWJKOlHgrREKeMBw786qZ8BES3Vg=="], - "@opencode/pty-win32-x64": ["@opencode/pty-win32-x64@0.4.0", "", { "os": "win32", "cpu": "x64" }, "sha512-P1QJLUX918/y8he8VEBSYoiN5bYhUnJtEaEpiE5fxoYsDP9nqAHXqIrGxTNNnda5jOBcCalxxy0WhL8qgeKOqQ=="], + "@opencode/pty-win32-x64": ["@opencode/pty-win32-x64@0.5.0", "", { "os": "win32", "cpu": "x64" }, "sha512-42OEc6vvlL/gXAairdWf4DLX6juPrpOycES9sp3iE3UfQ5KsRsuQ6F3HFFmSBkiZ/yUhJtKoJdJbquBTNEL+Ow=="], "@opencode/schema": ["@opencode/schema@workspace:packages/schema"], diff --git a/packages/cli/package.json b/packages/cli/package.json index ecc5abf437b5..00e249bb2a56 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -33,7 +33,7 @@ "@effect/platform-node": "catalog:", "@opencode/client": "workspace:*", "@opencode/plugin": "workspace:*", - "@opencode/pty": "0.4.0", + "@opencode/pty": "0.5.0", "@opencode/schema": "workspace:*", "@opencode/server": "workspace:*", "@opencode/tui": "workspace:*", diff --git a/packages/core/package.json b/packages/core/package.json index 07ab612afbab..055fc1979082 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -105,7 +105,7 @@ "@modelcontextprotocol/client": "2.3.1", "@modelcontextprotocol/core": "2.3.1", "@modelcontextprotocol/server": "2.3.1", - "@opencode/pty": "0.4.0", + "@opencode/pty": "0.5.0", "@opencode/ai": "workspace:*", "@opencode/codemode": "workspace:*", "@opencode/plugin": "workspace:*", diff --git a/packages/server/package.json b/packages/server/package.json index efdc624ac1ad..da5fcf020ea9 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -37,7 +37,7 @@ "effect": "catalog:" }, "devDependencies": { - "@opencode/pty": "0.4.0", + "@opencode/pty": "0.5.0", "@opencode/client": "workspace:*", "@opencode/plugin": "workspace:*", "@tsconfig/bun": "catalog:", From 923b92688cfea762d607e2a06d0c6b2e06cae0b0 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 18:26:11 +0000 Subject: [PATCH 19/29] fix(core): keep watching daemon commands when their Location closes --- packages/core/src/shell.ts | 90 +++++++++++++++++--------- packages/core/src/tool/plugin/shell.ts | 4 +- packages/core/test/tool-shell.test.ts | 72 +++++++++++++++++++++ 3 files changed, 133 insertions(+), 33 deletions(-) diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index d13dafdfd553..39e33f67a1c5 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -27,6 +27,7 @@ import { Environment } from "./environment/index.js" import { FileRetention } from "./file-retention.js" import { Location } from "./location.js" import { LocationLifecycle } from "./location-lifecycle.js" +import { Detaching } from "./detaching.js" import { Global } from "@opencode/util/global" import { PersistentPty } from "./persistent-pty.js" import { ShellSelect } from "./shell/select.js" @@ -223,6 +224,7 @@ const layer = () => const config = yield* Config.Service const pty = yield* PersistentPty.Service const lifecycle = yield* LocationLifecycle.Service + const detaching = yield* Detaching.Service const context = yield* Effect.context() const runFork = Effect.runForkWith(context) const commands = new Map() @@ -236,36 +238,32 @@ const layer = () => const { createWriteStream, createReadStream } = yield* Effect.promise(() => import("fs")) yield* Effect.promise(() => mkdir(outputDir, { recursive: true })) + // A Location that closes while the server keeps running (a reload, an eviction) keeps watching the daemon's + // commands it started: their exit, timeout and stop, so the jobs waiting on them still hear how they ended. + // Its replacement, if one boots, takes the commands back too and is the one that reports and evicts them. yield* Effect.addFinalizer(() => Effect.gen(function* () { - for (const command of commands.values()) { + const stopping = detaching.isStopping() + for (const command of [...commands.values()]) { + if (command.daemon && !stopping) continue if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) - // Daemon commands are only detached from: their processes keep running for the next owner. + // On shutdown daemon commands are only detached from: their processes keep running for the next owner. if (command.daemon?.kill) yield* Fiber.interrupt(command.daemon.kill) if (command.daemon?.wait) yield* Fiber.interrupt(command.daemon.wait) // Teardown interrupts pending commands; it is not a terminal command failure. It never stops or // removes a command itself: a spawned process dies with its handle's scope once `done` settles. yield* Deferred.interrupt(command.done) + commands.delete(command.info.id) } - commands.clear() exitOrder.length = 0 }), ) - // A reload boots the Location's replacement while calls may still hold this one, and the replacement takes - // the daemon's commands back. Once closed, this one leaves their timers, stops and reporting to it, and only - // still answers its own waiters. let closed = false yield* lifecycle.awaitShutdown.pipe( Effect.andThen( - Effect.gen(function* () { + Effect.sync(() => { closed = true - for (const command of commands.values()) { - if (!command.daemon) continue - if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) - command.timeoutFiber = undefined - if (command.daemon.kill) yield* Fiber.interrupt(command.daemon.kill) - } }), ), Effect.forkScoped, @@ -537,6 +535,18 @@ const layer = () => ) } + // The command's metadata as the daemon has it. A closed Location and its replacement can both watch one + // command, so a timeout or stop recorded by either is read back from there. + const latest = (daemon: Daemon) => + pty.getPipe(daemon.id).pipe( + Effect.map((process) => Option.getOrUndefined(decodeStored(process.metadata))), + Effect.catchTags({ + // Forgotten, or no daemon (which takes its processes with it). + "PersistentPty.ProcessNotFoundError": () => Effect.succeed(undefined), + "PersistentPty.UnavailableError": () => Effect.succeed(daemon.stored), + }), + ) + // Watches a command the daemon runs: its exit, timeout (in ms from now, none if undefined) and graceful // stop. Used for new commands and for running ones taken back after a restart. const attach = (command: Active, daemon: Daemon, timeout: number | undefined) => { @@ -553,14 +563,25 @@ const layer = () => // recorded stop. Only removal and teardown interrupt it. const stop = (reason: StopReason) => Effect.sync(() => { - if (closed || daemon.stored.stopReason !== undefined) return + if (daemon.stored.stopReason !== undefined) return daemon.stored = { ...daemon.stored, stopReason: reason } daemon.kill = runFork(stopping(reason)) }) command.stop = stop - const arm = (duration: number) => { - command.timeoutFiber = runFork(Effect.sleep(Duration.millis(duration)).pipe(Effect.andThen(stop("timeout")))) + // Times out at `deadline`, unless the daemon's metadata says the timeout was cleared, moved or already hit. + const arm = (duration: number, deadline: number | undefined) => { + const expire: Effect.Effect = Effect.gen(function* () { + const current = yield* latest(daemon) + if (!current) return + daemon.stored = current + if (current.stopReason !== undefined || current.backgroundedAtShutdown) return + const moved = current.timeoutDeadline + if (moved === undefined) return + if (deadline !== undefined && moved > deadline) return arm(moved - deadline, moved) + yield* stop("timeout") + }) + command.timeoutFiber = runFork(Effect.sleep(Duration.millis(duration)).pipe(Effect.andThen(expire))) } command.timeout = (duration) => @@ -568,25 +589,29 @@ const layer = () => if (daemon.stored.stopReason !== undefined) return if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) command.timeoutFiber = undefined - yield* store(daemon, { timeoutDeadline: duration === 0 ? undefined : Date.now() + duration }) - if (duration !== 0) arm(duration) + const deadline = duration === 0 ? undefined : Date.now() + duration + yield* store(daemon, { timeoutDeadline: deadline }) + if (deadline !== undefined) arm(duration, deadline) }) daemon.wait = runFork( pty.waitPipe(daemon.id).pipe( - Effect.map((exit): Ended => { - daemon.outputLines = exit.outputLines - daemon.outputError = exit.outputError - const reason = daemon.stored.stopReason - // A stopped command reports its reason, not the signal it was stopped with, as spawned ones do. - if (reason) return { status: reason, completed: exit.exitedAt } - return { - status: "exited", - ...(exit.exitCode !== undefined ? { exit: exit.exitCode } : {}), - ...(exit.signal !== undefined ? { signal: exit.signal } : {}), - completed: exit.exitedAt, - } - }), + Effect.flatMap( + Effect.fnUntraced(function* (exit): Effect.fn.Return { + daemon.outputLines = exit.outputLines + daemon.outputError = exit.outputError + // Another watcher of the command may have stopped it. + const reason = daemon.stored.stopReason ?? (yield* latest(daemon))?.stopReason + // A stopped command reports its reason, not the signal it was stopped with, as spawned ones do. + if (reason) return { status: reason, completed: exit.exitedAt } + return { + status: "exited", + ...(exit.exitCode !== undefined ? { exit: exit.exitCode } : {}), + ...(exit.signal !== undefined ? { signal: exit.signal } : {}), + completed: exit.exitedAt, + } + }), + ), // The daemon went away (and its processes with it), or forgot the process: it was killed. Effect.catch((error) => Effect.logWarning("lost a shell command's process", { id: command.info.id, error: error.message }).pipe( @@ -603,7 +628,7 @@ const layer = () => return } if (timeout === undefined) return - if (timeout > 0) return arm(timeout) + if (timeout > 0) return arm(timeout, daemon.stored.timeoutDeadline) runFork(stop("timeout")) } @@ -943,6 +968,7 @@ export const node = makeLocationNode({ Config.node, PersistentPty.node, LocationLifecycle.node, + Detaching.node, cleanupNode, ], }) diff --git a/packages/core/src/tool/plugin/shell.ts b/packages/core/src/tool/plugin/shell.ts index d042eb8028cd..dd710c992363 100644 --- a/packages/core/src/tool/plugin/shell.ts +++ b/packages/core/src/tool/plugin/shell.ts @@ -295,7 +295,9 @@ export const Plugin = { const notifyWhenDone = Effect.fn("ShellTool.notifyWhenDone")( (sessionID: SessionSchema.ID, id: string, shellID: string, command: string, settled: Deferred.Deferred) => notify(sessions, jobs, { sessionID, id, shellID, command, settled }), - Effect.forkIn(scope, { startImmediately: true }), + // Not in this Location's scope: a daemon command outlives a Location that closes while the server runs, and + // its session must still hear how it ended. The watch ends with the job, or when the job registry closes. + Effect.forkDetach({ startImmediately: true }), ) // Foreground commands this tool is blocked on. A planned shutdown moves them to the background, so it records diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index 1e2757b80879..9e4be19c533b 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -1491,6 +1491,78 @@ describe("ShellTool", () => { }), ) + fakeDaemonIt.live("keeps a background daemon command's job across a Location reload and reports how it ended", () => + withTempDir((tmp) => + Effect.gen(function* () { + reset() + const sessions = yield* Session.Service + const jobs = yield* Job.Service + const bus = yield* Bus.Service + const locations = yield* LocationServiceMap.Service + const location = Location.Ref.make({ directory: AbsolutePath.make(tmp.path) }) + yield* sessions.create({ id: sessionID, title: "shell test", location, model: sessionModel }) + const admitted = yield* bus.subscribe(SessionEvent.InboxEnqueued).pipe( + Stream.filter((event) => event.data.sessionID === sessionID && event.data.item.type === "synthetic"), + Stream.take(2), + Stream.runCollect, + Effect.forkScoped({ startImmediately: true }), + ) + const started = yield* Effect.gen(function* () { + const plugins = yield* Plugin.Service + yield* plugins.awaitActivation + const registry = yield* Tool.Service + const shell = yield* Shell.Service + const run = (id: string) => + Effect.gen(function* () { + const settled = yield* executeTool(registry, call({ command: "work", background: true }, id)) + return yield* shell.get(ID.make(String(settled.metadata?.shellID))) + }) + return { finishing: yield* run("call-finishing"), cancelled: yield* run("call-cancelled") } + }).pipe(Effect.provide(locations.get(location))) + const process = (info: { file: string }) => { + const found = fake?.processes.find((item) => item.pipe?.output_path === info.file) + if (!found) throw new Error("no daemon process for the command") + return found.id + } + + // A reload closes the Location and boots its replacement, which takes the commands back. + yield* locations.invalidate(location) + const replacement = yield* Effect.gen(function* () { + return yield* Shell.Service + }).pipe(Effect.provide(locations.get(location))) + expect((yield* replacement.list()).map((info) => info.id).toSorted()).toEqual( + [started.finishing.id, started.cancelled.id].toSorted(), + ) + expect((yield* jobs.get(started.finishing.id))?.status).toBe("running") + + // Cancelling the session's job still stops its command. + yield* jobs.cancel(started.cancelled.id) + while (!fake?.ops.includes("signal")) yield* Effect.promise(() => Bun.sleep(2)) + expect(fake?.requests).toContainEqual( + expect.objectContaining({ op: "signal", id: process(started.cancelled), signal: "SIGTERM" }), + ) + + fake?.output(process(started.finishing), "done\n") + fake?.exit(process(started.finishing), 0) + const notices = Array.from(yield* Fiber.join(admitted)).map((event) => event.data.item.payload) + expect(notices).toContainEqual( + expect.objectContaining({ + text: expect.stringContaining("done"), + metadata: expect.objectContaining({ shellID: started.finishing.id, state: "completed", exit: 0 }), + }), + ) + expect(notices).toContainEqual( + expect.objectContaining({ + metadata: expect.objectContaining({ shellID: started.cancelled.id, state: "cancelled" }), + }), + ) + expect((yield* jobs.get(started.finishing.id))?.status).toBe("completed") + expect((yield* replacement.get(started.finishing.id)).status).toBe("exited") + yield* locations.invalidate(location) + }), + ), + ) + it.live("preserves a background command's non-zero exit", () => withTempDir((tmp) => { reset() From 0d93fd732cbd7aaf48c065c27959b88d7fb6f198 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 18:26:11 +0000 Subject: [PATCH 20/29] test(core): drive shell daemon timers with the test clock and fake-daemon ops --- .../test/fixture/persistent-pty-daemon.ts | 2 + packages/core/test/fixture/shell-daemon.ts | 36 +++- packages/core/test/shell-daemon.test.ts | 168 ++++++++++-------- 3 files changed, 129 insertions(+), 77 deletions(-) diff --git a/packages/core/test/fixture/persistent-pty-daemon.ts b/packages/core/test/fixture/persistent-pty-daemon.ts index ce8882541125..480618f8c3a7 100644 --- a/packages/core/test/fixture/persistent-pty-daemon.ts +++ b/packages/core/test/fixture/persistent-pty-daemon.ts @@ -23,6 +23,8 @@ export function listen( const sockets = new Set() const server = net.createServer((socket) => { sockets.add(socket) + // A client may close its end (an interrupted wait) just before a reply is written. + socket.on("error", () => {}) socket.once("close", () => sockets.delete(socket)) const serve = (): Promise => readRequest(socket).then((envelope) => { diff --git a/packages/core/test/fixture/shell-daemon.ts b/packages/core/test/fixture/shell-daemon.ts index 00eb71fe7db4..945abb9192f9 100644 --- a/packages/core/test/fixture/shell-daemon.ts +++ b/packages/core/test/fixture/shell-daemon.ts @@ -1,5 +1,5 @@ import path from "node:path" -import { Context, Effect, Layer } from "effect" +import { Context, Effect, Exit, Layer, Scope } from "effect" import { ChildProcessSpawner } from "effect/process" import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" import { Config } from "@opencode/core/config" @@ -8,6 +8,7 @@ import { Location } from "@opencode/core/location" import { PersistentPty } from "@opencode/core/persistent-pty" import { AbsolutePath } from "@opencode/core/schema" import { Shell } from "@opencode/core/shell" +import { Detaching } from "@opencode/core/detaching" import { Bus } from "@opencode/core/bus" import { LayerNode } from "@opencode/util/effect/layer-node" import { LocationLifecycle } from "@opencode/core/location-lifecycle" @@ -40,7 +41,7 @@ export const daemonShells = Effect.fn(function* () { }), ).pipe(Layer.provide(hostEnvironmentLayer)) const shellLayer = (directory: string) => - AppNodeBuilder.build(LayerNode.group([Shell.node, LocationLifecycle.node, Bus.node]), [ + AppNodeBuilder.build(LayerNode.group([Shell.node, LocationLifecycle.node, Bus.node, Detaching.node]), [ Location.node.replace( Layer.succeed( Location.Service, @@ -52,13 +53,26 @@ export const daemonShells = Effect.fn(function* () { Environment.node.replace(environment), PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, pty)), ]) - // Runs `body` against the Location's Shell, then closes it as a restart or an eviction would. // A Location's Shell that stays open for the rest of the test. const openShell = (directory: string) => Effect.gen(function* () { const context = yield* Layer.build(shellLayer(directory)) return Context.get(context, Shell.Service) }) + // A Location's Shell and its services, closed by `close` as a reload or an eviction does while the server runs. + const openLocation = (directory: string) => + Effect.gen(function* () { + const scope = yield* Scope.make() + const context = yield* Layer.buildWithScope(shellLayer(directory), scope) + return { + shell: Context.get(context, Shell.Service), + bus: Context.get(context, Bus.Service), + close: Context.get(context, LocationLifecycle.Service).shutdown.pipe( + Effect.andThen(Scope.close(scope, Exit.void)), + ), + } + }) + // Runs `body` against the Location's Shell, then closes it as the server shutting down for a restart does. const withShell = ( directory: string, body: (shell: Shell.Interface) => Effect.Effect, @@ -66,7 +80,10 @@ export const daemonShells = Effect.fn(function* () { Effect.scoped( Effect.gen(function* () { const shell = yield* Shell.Service - return yield* body(shell) + const detaching = yield* Detaching.Service + const result = yield* body(shell) + yield* detaching.begin + return result }).pipe(Effect.provide(shellLayer(directory))), ) const directory = (yield* tmpdirScoped()).path @@ -78,9 +95,20 @@ export const daemonShells = Effect.fn(function* () { pty, withShell, openShell, + openLocation, spawned: () => spawned, created: () => daemon.requests.filter((request) => request.op === "create_pipe"), signals: (id: number) => daemon.requests.flatMap((request) => (request.op === "signal" && request.id === id ? [request.signal] : [])), } }) + +// Polls in real time until `check` holds: the fake daemon answers over a socket, which the test clock does not drive. +export const until = (check: (() => boolean) | Effect.Effect) => + Effect.gen(function* () { + for (let attempt = 0; attempt < 2_000; attempt++) { + if (typeof check === "function" ? check() : yield* check) return + yield* Effect.promise(() => Bun.sleep(2)) + } + return yield* Effect.die(new Error("condition never held")) + }) diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts index bb5f5b4a575d..c4bd5cdde276 100644 --- a/packages/core/test/shell-daemon.test.ts +++ b/packages/core/test/shell-daemon.test.ts @@ -11,20 +11,21 @@ import { Environment } from "@opencode/core/environment/index" import { Location } from "@opencode/core/location" import { AbsolutePath } from "@opencode/core/schema" import { Shell } from "@opencode/core/shell" -import { Bus } from "@opencode/core/bus" -import { LocationLifecycle } from "@opencode/core/location-lifecycle" import { Global } from "@opencode/util/global" import { FSUtil } from "@opencode/util/fs-util" import { LayerNode } from "@opencode/util/effect/layer-node" import { AppProcess } from "@opencode/util/process" import { hostEnvironmentLayer } from "./fixture/environment" import { location } from "./fixture/location" -import { daemonShells } from "./fixture/shell-daemon" +import { daemonShells, until } from "./fixture/shell-daemon" import { tmpdirScoped } from "./fixture/tmpdir" +import { binaryPath } from "@opencode/pty" import { it } from "./lib/effect" const unix = process.platform !== "win32" const test = unix ? it.live : it.live.skip +// On the test clock, for timers. +const timed = unix ? it.effect : it.effect.skip const lines = (count: number) => Array.from({ length: count }, (_, index) => `line ${index + 1}\n`).join("") @@ -154,43 +155,33 @@ describe("Shell on the process daemon", () => { })) // On the test clock: the timeout and the grace period pass without waiting for them. - ;(unix ? it.effect : it.effect.skip)( - "stops a timed-out command with SIGTERM, then SIGKILL after the grace period, and keeps it", - () => - Effect.gen(function* () { - const env = yield* daemonShells() - env.daemon.trap.add(1) - // Real time for the fake daemon's socket round trips. - const until = (done: () => boolean) => - Effect.promise(async () => { - while (!done()) await Bun.sleep(5) + timed("stops a timed-out command with SIGTERM, then SIGKILL after the grace period, and keeps it", () => + Effect.gen(function* () { + const env = yield* daemonShells() + env.daemon.trap.add(1) + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const info = yield* shell.create({ shell: "/bin/sh", command: "trap '' TERM; sleep 60", timeout: 10 }) + yield* TestClock.adjust("10 millis") + yield* until(() => env.signals(1).length > 0) + expect(env.signals(1)).toEqual(["SIGTERM"]) + expect((yield* shell.get(info.id)).status).toBe("running") + // The grace period starts once the daemon acknowledged SIGTERM, so keep the clock moving until it ends. + yield* until(TestClock.adjust("1 second").pipe(Effect.map(() => env.signals(1).length > 1))) + expect((yield* shell.wait(info.id)).status).toBe("timeout") + expect(env.signals(1)).toEqual(["SIGTERM", "SIGKILL"]) + expect(env.daemon.processes[0]?.pipe?.metadata).toMatchObject({ stopReason: "timeout" }) + expect(env.daemon.processes[0]?.pipe?.metadata).not.toHaveProperty("timeoutDeadline") + expect(yield* shell.result(info)).toMatchObject({ + info: { status: "timeout" }, + capture: { output: "(no output)" }, }) - yield* env.withShell(env.directory, (shell) => - Effect.gen(function* () { - const info = yield* shell.create({ shell: "/bin/sh", command: "trap '' TERM; sleep 60", timeout: 10 }) - yield* TestClock.adjust("10 millis") - yield* until(() => env.signals(1).length > 0) - expect(env.signals(1)).toEqual(["SIGTERM"]) - expect((yield* shell.get(info.id)).status).toBe("running") - // The grace period starts once the daemon acknowledged SIGTERM. - while (env.signals(1).length < 2) { - yield* TestClock.adjust("1 second") - yield* Effect.promise(() => Bun.sleep(5)) - } - expect((yield* shell.wait(info.id)).status).toBe("timeout") - expect(env.signals(1)).toEqual(["SIGTERM", "SIGKILL"]) - expect(env.daemon.processes[0]?.pipe?.metadata).toMatchObject({ stopReason: "timeout" }) - expect(env.daemon.processes[0]?.pipe?.metadata).not.toHaveProperty("timeoutDeadline") - expect(yield* shell.result(info)).toMatchObject({ - info: { status: "timeout" }, - capture: { output: "(no output)" }, - }) - expect((yield* shell.result(info)).info).not.toHaveProperty("signal") - expect(env.daemon.ops).not.toContain("terminate") - expect(existsSync(info.file)).toBe(true) - }), - ) - }), + expect((yield* shell.result(info)).info).not.toHaveProperty("signal") + expect(env.daemon.ops).not.toContain("terminate") + expect(existsSync(info.file)).toBe(true) + }), + ) + }), ) test("forgets a command in the daemon before deleting its file, on remove and on eviction", () => @@ -215,8 +206,8 @@ describe("Shell on the process daemon", () => { return info }), ) - // Waiters resolve before retention runs. - yield* Effect.sleep(20) + // Waiters resolve before retention runs, which forgets the process and then deletes its file. + yield* until(() => !existsSync(finished[0]!.file)) expect(yield* shell.get(finished[0]!.id).pipe(Effect.flip)).toBeInstanceOf(Shell.NotFoundError) expect(existsSync(finished[0]!.file)).toBe(false) expect(env.daemon.processes.map((process) => process.id)).not.toContain(2) @@ -239,7 +230,7 @@ describe("Shell on the process daemon", () => { const expiring = yield* shell.create({ shell: "/bin/sh", command: "expiring", timeout: 100 }) const stopping = yield* shell.create({ shell: "/bin/sh", command: "stopping" }) yield* shell.stop(stopping.id, "killed") - while (env.signals(4).length === 0) yield* Effect.sleep(5) + yield* until(() => env.signals(4).length > 0) return { running, exited, expiring, stopping } }), ) @@ -247,7 +238,9 @@ describe("Shell on the process daemon", () => { expect(env.daemon.ops).not.toContain("terminate") expect(env.signals(1)).toEqual([]) expect(env.signals(4)).toEqual(["SIGTERM"]) - yield* Effect.sleep(150) + // The expiring command's deadline passes while no server runs. + const expiring = env.daemon.processes.find((process) => process.id === 3)?.pipe + if (expiring) expiring.metadata = { ...expiring.metadata, timeoutDeadline: Date.now() - 1 } const other = (yield* tmpdirScoped()).path yield* env.withShell(other, (shell) => @@ -281,34 +274,62 @@ describe("Shell on the process daemon", () => { ) })) - test("a closed Location leaves its daemon commands' timers and reporting to its replacement", () => + // A reload or an eviction closes a Location while the server keeps running. + timed("a Location closed while the server runs keeps enforcing its commands' timeouts", () => Effect.gen(function* () { const env = yield* daemonShells() - yield* env.withShell(env.directory, (shell) => - Effect.gen(function* () { - const lifecycle = yield* LocationLifecycle.Service - const bus = yield* Bus.Service - const exits: string[] = [] - yield* bus.listen((event) => - Effect.sync(() => { - if (event.type === "shell.exited") exits.push(event.type) - }), - ) - const info = yield* shell.create({ shell: "/bin/sh", command: "sleep 60", timeout: 50 }) - yield* lifecycle.shutdown - yield* Effect.sleep(100) - // The replacement owns the timeout now. - expect(env.signals(1)).toEqual([]) - env.daemon.exit(1, 0) - // A call still holding this Location gets its result, but nothing is reported twice. - expect(yield* shell.wait(info.id)).toMatchObject({ status: "exited", exit: 0 }) - yield* Effect.sleep(20) - expect(exits).toEqual([]) + const closing = yield* env.openLocation(env.directory) + const info = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60", timeout: 50 }) + yield* closing.close + expect(env.signals(1)).toEqual([]) + yield* TestClock.adjust("50 millis") + yield* until(() => env.signals(1).length > 0) + expect(env.signals(1)).toEqual(["SIGTERM"]) + expect(yield* closing.shell.wait(info.id)).toMatchObject({ status: "timeout" }) + }), + ) + + timed("a closed Location and its replacement share a command's timeout and stop through the daemon", () => + Effect.gen(function* () { + const env = yield* daemonShells() + const closing = yield* env.openLocation(env.directory) + const backgrounded = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60", timeout: 100 }) + const stopped = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60" }) + yield* closing.close + const exits = { closed: 0, replacement: 0 } + yield* closing.bus.listen((event) => + Effect.sync(() => { + if (event.type === "shell.exited") exits.closed++ }), ) - })) + const replacement = yield* env.openLocation(env.directory) + yield* replacement.bus.listen((event) => + Effect.sync(() => { + if (event.type === "shell.exited") exits.replacement++ + }), + ) + expect((yield* replacement.shell.list()).map((item) => item.id)).toEqual([backgrounded.id, stopped.id]) + + // The call that started it, still in the closed Location, moves it to the background and clears its timeout. + yield* closing.shell.timeout(backgrounded.id, 0) + const gets = () => env.daemon.ops.filter((op) => op === "get").length + const before = gets() + yield* TestClock.adjust("100 millis") + // The replacement's timer checked the daemon and left the command alone. + yield* until(() => gets() > before) + expect(env.signals(1)).toEqual([]) - test("records a command moved to the background at shutdown so the next server never times it out", () => + // Cancelling its job in the closed Location stops it; the replacement reports why it ended. + yield* closing.shell.stop(stopped.id, "killed") + expect(yield* replacement.shell.wait(stopped.id)).toMatchObject({ status: "killed" }) + expect(yield* closing.shell.wait(stopped.id)).toMatchObject({ status: "killed" }) + yield* until(() => exits.replacement === 1) + expect(exits.closed).toBe(0) + yield* replacement.close + }), + ) + + timed("records a command moved to the background at shutdown so the next server never times it out", () => Effect.gen(function* () { const env = yield* daemonShells() const info = yield* env.withShell(env.directory, (shell) => @@ -320,15 +341,17 @@ describe("Shell on the process daemon", () => { ) expect(env.daemon.processes[0]?.pipe?.metadata).toMatchObject({ backgroundedAtShutdown: true }) expect(env.daemon.processes[0]?.pipe?.metadata).not.toHaveProperty("timeoutDeadline") - yield* Effect.sleep(150) yield* env.withShell(env.directory, (shell) => Effect.gen(function* () { - yield* Effect.sleep(50) + yield* TestClock.adjust("1 second") expect((yield* shell.get(info.id)).status).toBe("running") expect(env.signals(1)).toEqual([]) + // No timer was armed, so none consulted the daemon. + expect(env.daemon.ops).not.toContain("get") }), ) - })) + }), + ) test("the sweep keeps a running command's file and forgets a finished one before deleting its file", () => Effect.gen(function* () { @@ -365,10 +388,9 @@ describe("Shell on the process daemon", () => { })) }) -// Against the real opencode-pty. Opt-in through OPENCODE_PTY_BIN on purpose: the installed @opencode/pty is still -// 0.4 (protocol v7) until the release bumps it, so its binaryPath would only exercise the spawner fallback. Once it -// speaks v8, fall back to binaryPath here like server/test/persistent-pty.test.ts does. -const real = unix && process.env.OPENCODE_PTY_BIN ? it.live : it.live.skip +// Against the real opencode-pty, like server/test/persistent-pty.test.ts. +const binary = unix ? (process.env.OPENCODE_PTY_BIN ?? binaryPath) : undefined +const real = binary !== undefined && existsSync(binary) ? it.live : it.live.skip describe("Shell on a real process daemon", () => { real( From 992ff961b7fd66b45ae21e7945703c5282c5b4b5 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 20:02:35 +0000 Subject: [PATCH 21/29] fix(core): keep an interrupted shell call's output visible in clients --- packages/core/src/tool/plugin/shell.ts | 3 ++- packages/core/test/tool-shell.test.ts | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/core/src/tool/plugin/shell.ts b/packages/core/src/tool/plugin/shell.ts index dd710c992363..e3b96d71c0b2 100644 --- a/packages/core/src/tool/plugin/shell.ts +++ b/packages/core/src/tool/plugin/shell.ts @@ -329,9 +329,10 @@ export const Plugin = { const result = yield* shell.result(info) // The stopped job settles from the same exit; this only releases it. yield* jobs.cancel(jobID) + // No `shellID`: clients read it as a background call and show the shell's live output instead of this result. + // The text names the shell and its output file. const output = { ...ShellResult.interrupted(result), - shellID: info.id, interrupted: true, status: "completed" as const, } diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index 9e4be19c533b..47ea434ce798 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -1843,7 +1843,9 @@ describe("ShellTool", () => { const info = yield* shell.get(shellID) expect(info.status).toBe("killed") expect(yield* Effect.promise(() => fs.readFile(info.file, "utf8"))).toBe("first\nsecond\n") - expect(result?.metadata).toMatchObject({ shellID, interrupted: true, truncated: false }) + expect(result?.metadata).toMatchObject({ interrupted: true, truncated: false, status: "completed" }) + // A shell ID marks a background call, whose live output clients show instead of the result. + expect(result?.metadata).not.toHaveProperty("shellID") expect(result?.content).toEqual([ { type: "text", From e2f1dab89e93828b94d992266d91910e5de545e7 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 20:03:59 +0000 Subject: [PATCH 22/29] fix(core): report a killed shell command as killed --- packages/core/src/shell.ts | 20 ++++++- packages/core/src/shell/result.ts | 30 ++++++++-- packages/core/src/tool/plugin/shell.ts | 1 + .../test/fixture/persistent-pty-daemon.ts | 6 ++ packages/core/test/shell-daemon.test.ts | 17 ++++++ packages/core/test/shell-result.test.ts | 56 +++++++++++++++++++ 6 files changed, 122 insertions(+), 8 deletions(-) create mode 100644 packages/core/test/shell-result.test.ts diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index 39e33f67a1c5..2980d6650662 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -49,6 +49,7 @@ const EXITED_LIMIT = 25 export const RETENTION = Duration.days(7) export const DIRECTORY = "shell" const STOP_GRACE = Duration.seconds(3) +const LOST = "Command was killed: the process daemon stopped" type Info = Shell.Info export type StopReason = "timeout" | "killed" @@ -108,6 +109,8 @@ type Active = { // Whether the process can outlive this server (run by the process daemon). Spawned commands cannot. detachable: boolean daemon?: Daemon + // Why a command was killed without a signal of its own. + killed?: string } type Ended = { @@ -516,7 +519,13 @@ const layer = () => const total = empty ? 0 : Math.max(page.newlines + (page.text.endsWith("\n") ? 0 : 1), shown) const notice = truncated ? `\n\n${ShellResult.savedNotice({ shown, total }, info.file)}` : "" const error = page.error ? `\n\n[output file ${info.file} stopped early: ${page.error}]` : "" - return { output: `${text || "(no output)"}${notice}${error}`, truncated, lines: { shown, total } } + const killed = (yield* require(info.id)).killed + return { + output: `${text || "(no output)"}${notice}${error}`, + truncated, + lines: { shown, total }, + ...(killed ? { killed } : {}), + } }).pipe(Effect.catchTag("Shell.NotFoundError", () => Effect.succeed(undefined))) return { info, capture } }) @@ -615,7 +624,12 @@ const layer = () => // The daemon went away (and its processes with it), or forgot the process: it was killed. Effect.catch((error) => Effect.logWarning("lost a shell command's process", { id: command.info.id, error: error.message }).pipe( - Effect.as({ status: "killed", completed: Date.now() }), + Effect.andThen( + Effect.sync((): Ended => { + command.killed = LOST + return { status: "killed", completed: Date.now() } + }), + ), ), ), Effect.flatMap((outcome) => finish(command, outcome)), @@ -738,6 +752,8 @@ const layer = () => ...(running ? {} : { outputLines: process.outputLines, outputError: process.outputError }), } const command = daemonCommand(info, daemon) + if (process.lifecycle.status === "failed" && !stored.stopReason) + command.killed = `Command was killed: ${process.lifecycle.message}` commands.set(stored.id, command) if (running) { // A command moved to the background at shutdown has no timeout; an expired deadline fires at once. diff --git a/packages/core/src/shell/result.ts b/packages/core/src/shell/result.ts index 73a22fd6bed2..cfa4baf30a5d 100644 --- a/packages/core/src/shell/result.ts +++ b/packages/core/src/shell/result.ts @@ -7,10 +7,19 @@ export type Lines = { shown: number; total: number } export type Result = { info: Shell.Info // `lines` counts the tail's lines (`shown`) and the file's (`total`), zero for an empty file. - capture: { output: string; truncated: boolean; lines?: Lines } | undefined + // `killed` says why a command was killed when no signal of its own tells, e.g. the process daemon stopped. + capture: { output: string; truncated: boolean; lines?: Lines; killed?: string } | undefined } -type Output = { output: string; truncated: boolean; exit?: number; signal?: string; timeout?: boolean } +type Output = { + output: string + truncated: boolean + exit?: number + signal?: string + timeout?: boolean + // Killed without an exit code or signal of its own: stopped, or lost with the process daemon. + killed?: string +} const missing = "Shell command output is no longer available." export const unavailable: Shell.Output = { @@ -20,13 +29,17 @@ export const unavailable: Shell.Output = { truncated: false, } -export function output(result: Result): Output { +export function output(result: Result, reportKilled = true): Output { return { output: result.capture?.output ?? unavailable.output, truncated: result.capture?.truncated ?? false, ...(result.info.exit !== undefined ? { exit: result.info.exit } : {}), ...(result.info.signal !== undefined ? { signal: result.info.signal } : {}), ...(result.info.status === "timeout" ? { timeout: true } : {}), + // A removed command has no capture and keeps today's plain "unavailable" result. + ...(reportKilled && result.info.status === "killed" && result.capture && result.info.signal === undefined + ? { killed: result.capture.killed ?? "Command was killed" } + : {}), } } @@ -38,19 +51,23 @@ export function savedNotice(lines: Lines, file: string) { // The result of a command stopped because the user interrupted its call. It always names the shell // and its output file, which `output` leaves out when the output fits. export function interrupted(result: Result): Output { - const captured = output(result) + // Saying it was interrupted already says it was killed. + const captured = output(result, false) const lines = result.capture && !result.capture.truncated ? result.capture.lines : undefined + // An empty capture is all notice: no "(no output)" before "[no output; …]". + const text = lines?.total === 0 ? "" : `${captured.output.replace(/\n$/, "")}\n\n` return { ...captured, output: [ - lines ? `${captured.output.replace(/\n$/, "")}\n\n${savedNotice(lines, result.info.file)}` : captured.output, + lines ? `${text}${savedNotice(lines, result.info.file)}` : captured.output, `Command interrupted by the user (shell ID: ${result.info.id}).`, ].join("\n\n"), } } -export function notice(output: Pick) { +export function notice(output: Pick) { if (output.timeout) return "Timed out before completion" + if (output.killed !== undefined) return output.killed if (output.signal !== undefined) return `Killed by ${output.signal}` if (output.exit !== undefined && output.exit !== 0) return `Exited with code ${output.exit}` } @@ -61,6 +78,7 @@ export function metadata(output: Output) { ...(output.exit !== undefined ? { exit: output.exit } : {}), ...(output.signal !== undefined ? { signal: output.signal } : {}), ...(output.timeout !== undefined ? { timeout: output.timeout } : {}), + ...(output.killed !== undefined ? { killed: true } : {}), } } diff --git a/packages/core/src/tool/plugin/shell.ts b/packages/core/src/tool/plugin/shell.ts index e3b96d71c0b2..9429d5a44f81 100644 --- a/packages/core/src/tool/plugin/shell.ts +++ b/packages/core/src/tool/plugin/shell.ts @@ -70,6 +70,7 @@ const StructuredOutput = Schema.Struct({ truncated: Schema.Boolean, timeout: Schema.optionalKey(Schema.Boolean), interrupted: Schema.optionalKey(Schema.Boolean), + killed: Schema.optionalKey(Schema.String), }) const Output = Schema.Struct({ diff --git a/packages/core/test/fixture/persistent-pty-daemon.ts b/packages/core/test/fixture/persistent-pty-daemon.ts index 480618f8c3a7..605f5fb28c59 100644 --- a/packages/core/test/fixture/persistent-pty-daemon.ts +++ b/packages/core/test/fixture/persistent-pty-daemon.ts @@ -155,6 +155,11 @@ export const fakeDaemon = Effect.fn(function* () { } found.output_tail += Buffer.byteLength(text) } + // The daemon going away under a waiting server: the wait connection closes without a reply. + const drop = (id: number) => { + waiters.get(id)?.forEach((connection) => connection.destroy()) + waiters.delete(id) + } const fail = (id: number, message: string) => { const found = find(id) if (found?.pipe) found.pipe.output_error = message @@ -244,6 +249,7 @@ export const fakeDaemon = Effect.fn(function* () { exit, output, fail, + drop, processes, waiters: (id: number) => waiters.get(id)?.length ?? 0, options: { diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts index c4bd5cdde276..9c8cf328be69 100644 --- a/packages/core/test/shell-daemon.test.ts +++ b/packages/core/test/shell-daemon.test.ts @@ -140,6 +140,23 @@ describe("Shell on the process daemon", () => { ) })) + test("reports a command lost with the daemon as killed, with what it wrote", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const info = yield* shell.create({ shell: "/bin/sh", command: "work" }) + env.daemon.output(1, "partial\n") + yield* until(() => env.daemon.waiters(1) > 0) + env.daemon.drop(1) + expect(yield* shell.result(info)).toMatchObject({ + info: { status: "killed" }, + capture: { output: "partial\n", killed: "Command was killed: the process daemon stopped" }, + }) + }), + ) + })) + test("fails a command the daemon rejected instead of spawning it", () => Effect.gen(function* () { const env = yield* daemonShells() diff --git a/packages/core/test/shell-result.test.ts b/packages/core/test/shell-result.test.ts new file mode 100644 index 000000000000..a9ba2d6706e9 --- /dev/null +++ b/packages/core/test/shell-result.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, test } from "bun:test" +import { ShellResult } from "@opencode/core/shell/result" +import { Shell } from "@opencode/schema/shell" + +const info = (status: Shell.Status, extra: Partial = {}): Shell.Info => ({ + id: Shell.ID.make("sh_result"), + status, + command: "work", + cwd: "/tmp", + shell: "/bin/sh", + file: "/tmp/sh_result.out", + metadata: {}, + time: { started: 0, completed: 1 }, + ...extra, +}) +const capture = { output: "partial\n", truncated: false, lines: { shown: 1, total: 1 } } + +describe("ShellResult", () => { + test("reports a command killed without a signal of its own", () => { + const killed = ShellResult.output({ info: info("killed"), capture }) + expect(killed).toMatchObject({ killed: "Command was killed" }) + expect(ShellResult.notice(killed)).toBe("Command was killed") + expect(ShellResult.metadata(killed)).toMatchObject({ killed: true }) + + const lost = ShellResult.output({ + info: info("killed"), + capture: { ...capture, killed: "Command was killed: the process daemon stopped" }, + }) + expect(ShellResult.notice(lost)).toBe("Command was killed: the process daemon stopped") + }) + + test("keeps a removed command's result unchanged", () => { + const removed = ShellResult.output({ info: info("killed"), capture: undefined }) + expect(removed).toEqual({ output: ShellResult.unavailable.output, truncated: false }) + expect(ShellResult.notice(removed)).toBeUndefined() + }) + + test("an interrupted result says it was interrupted, not also that it was killed", () => { + const result = ShellResult.interrupted({ info: info("killed"), capture }) + expect(result).not.toHaveProperty("killed") + expect(ShellResult.notice(result)).toBeUndefined() + expect(result.output).toBe( + "partial\n\n[showing lines 1-1 of 1; full output saved to /tmp/sh_result.out]\n\nCommand interrupted by the user (shell ID: sh_result).", + ) + }) + + test("an interrupted command without output says so once", () => { + const result = ShellResult.interrupted({ + info: info("killed"), + capture: { output: "(no output)", truncated: false, lines: { shown: 0, total: 0 } }, + }) + expect(result.output).toBe( + "[no output; output file: /tmp/sh_result.out]\n\nCommand interrupted by the user (shell ID: sh_result).", + ) + }) +}) From 11c99974ea7cb716b142b8b7263fe541ddb835bb Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 20:04:50 +0000 Subject: [PATCH 23/29] fix(core): keep background jobs running for the next server on any shutdown --- packages/core/src/job.ts | 4 +- packages/core/test/job.test.ts | 80 ++++++++++++++------------- packages/core/test/tool-shell.test.ts | 36 ++++++++++++ 3 files changed, 80 insertions(+), 40 deletions(-) diff --git a/packages/core/src/job.ts b/packages/core/src/job.ts index 6d3dad6a2cef..c579e1c1c212 100644 --- a/packages/core/src/job.ts +++ b/packages/core/src/job.ts @@ -266,13 +266,13 @@ export const make = Effect.gen(function* () { if (!job) return [{}, jobs] if (job.scope !== scope) return [{}, jobs] if (job.info.status !== "running") return [{ info: snapshot(job) }, jobs] - // Teardown (this registry closing, or a planned shutdown) is not a terminal outcome: leave the job + // Teardown (this registry closing, or any server shutdown) is not a terminal outcome: leave the job // and its durable row running for the next server, and interrupt rather than settle its observers. // A user cancel has already settled the job as cancelled before interrupting its work. if ( Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause) && - (scope.state._tag === "Closed" || detaching?.isDetaching()) + (scope.state._tag === "Closed" || detaching?.isStopping()) ) return [{ done: job.done, scope: job.scope, interrupted: true }, jobs] const status: Exclude = Exit.isSuccess(exit) diff --git a/packages/core/test/job.test.ts b/packages/core/test/job.test.ts index c7616c15479d..17e350902525 100644 --- a/packages/core/test/job.test.ts +++ b/packages/core/test/job.test.ts @@ -488,44 +488,48 @@ describe("Job", () => { }), ) - it.live("leaves interrupted work unsettled while detaching, but still cancels on request", () => - Effect.gen(function* () { - const detaching = yield* Detaching.make - const jobs = yield* Job.make.pipe(Effect.provideService(Detaching.Service, detaching)) - const recovery = (id: string) => ({ - kind: "shell" as const, - sessionID: SessionSchema.ID.make("ses_detaching"), - shellID: id, - command: "sleep 60", - }) - const interrupt = yield* Deferred.make() - const detached = yield* jobs.start({ - id: "shell_detached", - type: "shell", - recovery: recovery("shell_detached"), - run: Deferred.await(interrupt).pipe(Effect.andThen(Effect.interrupt)), - }) - const cancelled = yield* jobs.start({ - id: "shell_cancelled", - type: "shell", - recovery: recovery("shell_cancelled"), - run: Effect.never, - }) - const background = yield* jobs.background(detached.id) - const observer = yield* jobs.wait({ id: detached.id }).pipe(Effect.forkChild({ startImmediately: true })) - - yield* detaching.handoffPrepared - expect(yield* detaching.begin).toBe(true) - yield* Deferred.succeed(interrupt, undefined) - - const observed = yield* Fiber.await(observer).pipe(Effect.timeout("1 second")) - expect(Exit.isFailure(observed) && Cause.hasInterruptsOnly(observed.cause)).toBe(true) - expect(yield* jobs.pendingBackground).toMatchObject([{ id: detached.id, status: "running" }]) - expect(yield* jobs.cancel(cancelled.id)).toMatchObject({ status: "cancelled" }) - expect((yield* jobs.wait({ id: cancelled.id })).info?.status).toBe("cancelled") - if (background?.notificationID) yield* jobs.completeBackground(background.notificationID) - }), - ) + for (const handoff of [true, false]) + it.live( + `leaves interrupted work unsettled on ${handoff ? "a planned restart" : "a normal quit"}, but still cancels on request`, + () => + Effect.gen(function* () { + const detaching = yield* Detaching.make + const jobs = yield* Job.make.pipe(Effect.provideService(Detaching.Service, detaching)) + const recovery = (id: string) => ({ + kind: "shell" as const, + sessionID: SessionSchema.ID.make("ses_detaching"), + shellID: id, + command: "sleep 60", + }) + const interrupt = yield* Deferred.make() + const detached = yield* jobs.start({ + id: "shell_detached", + type: "shell", + recovery: recovery("shell_detached"), + run: Deferred.await(interrupt).pipe(Effect.andThen(Effect.interrupt)), + }) + const cancelled = yield* jobs.start({ + id: "shell_cancelled", + type: "shell", + recovery: recovery("shell_cancelled"), + run: Effect.never, + }) + const background = yield* jobs.background(detached.id) + const observer = yield* jobs.wait({ id: detached.id }).pipe(Effect.forkChild({ startImmediately: true })) + + if (handoff) yield* detaching.handoffPrepared + expect(yield* detaching.begin).toBe(handoff) + expect(detaching.isStopping()).toBe(true) + yield* Deferred.succeed(interrupt, undefined) + + const observed = yield* Fiber.await(observer).pipe(Effect.timeout("1 second")) + expect(Exit.isFailure(observed) && Cause.hasInterruptsOnly(observed.cause)).toBe(true) + expect(yield* jobs.pendingBackground).toMatchObject([{ id: detached.id, status: "running" }]) + expect(yield* jobs.cancel(cancelled.id)).toMatchObject({ status: "cancelled" }) + expect((yield* jobs.wait({ id: cancelled.id })).info?.status).toBe("cancelled") + if (background?.notificationID) yield* jobs.completeBackground(background.notificationID) + }), + ) it.live("cancels running session jobs and removes unacknowledged SQLite job rows on cancelSession", () => Effect.gen(function* () { diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index 47ea434ce798..763133993401 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -1563,6 +1563,42 @@ describe("ShellTool", () => { ), ) + it.live("leaves a background job running for the next server, without a notice, on a normal quit", () => + withTempDir((tmp) => + Effect.gen(function* () { + reset() + const sessions = yield* Session.Service + const jobs = yield* Job.Service + const bus = yield* Bus.Service + const detaching = yield* Detaching.Service + const locations = yield* LocationServiceMap.Service + const location = Location.Ref.make({ directory: AbsolutePath.make(tmp.path) }) + yield* sessions.create({ id: sessionID, title: "shell test", location, model: sessionModel }) + const notices: unknown[] = [] + yield* bus.listen((event) => + Effect.sync(() => { + if (event.type === SessionEvent.InboxEnqueued.type) notices.push(event) + }), + ) + const shellID = yield* Effect.gen(function* () { + const plugins = yield* Plugin.Service + yield* plugins.awaitActivation + const registry = yield* Tool.Service + const settled = yield* executeTool(registry, call({ command: "sleep 60", background: true }, "call-quit")) + return String(settled.metadata?.shellID) + }).pipe(Effect.provide(locations.get(location))) + // The server quits without a handoff: shutdown starts, then the Location closes. + expect(yield* detaching.begin).toBe(false) + yield* locations.invalidate(location) + // The job's observers are interrupted, not told it was cancelled. + expect(Exit.hasInterrupts(yield* jobs.wait({ id: shellID }).pipe(Effect.exit))).toBe(true) + yield* Effect.yieldNow + expect((yield* jobs.pendingBackground).map((item) => [item.id, item.status])).toEqual([[shellID, "running"]]) + expect(notices).toEqual([]) + }), + ), + ) + it.live("preserves a background command's non-zero exit", () => withTempDir((tmp) => { reset() From 0475da6918cd194b4660d7e6aacb231ff0426e2b Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 20:05:24 +0000 Subject: [PATCH 24/29] fix(core): forget a prepared handoff when the daemon is shut down --- packages/core/src/detaching.ts | 5 +++++ packages/core/src/persistent-pty/index.ts | 1 + packages/core/test/detaching.test.ts | 10 ++++++++++ 3 files changed, 16 insertions(+) diff --git a/packages/core/src/detaching.ts b/packages/core/src/detaching.ts index f6b7f0769d07..f1652287b7b8 100644 --- a/packages/core/src/detaching.ts +++ b/packages/core/src/detaching.ts @@ -17,6 +17,8 @@ const HOOK_BUDGET = Duration.millis(500) export interface Interface { /** Records that a daemon handoff was prepared successfully. */ readonly handoffPrepared: Effect.Effect + /** Records that the prepared handoff no longer holds: the daemon was shut down, so nothing can outlive this server. */ + readonly handoffDropped: Effect.Effect /** * Called once shutdown starts, before any service is torn down. Returns whether this server is detaching, * after running the detaching hooks (best effort, bounded) when it is. @@ -40,6 +42,9 @@ export const make = Effect.sync(() => { handoffPrepared: Effect.sync(() => { prepared = true }), + handoffDropped: Effect.sync(() => { + prepared = false + }), begin: Effect.gen(function* () { stopping = true detaching = prepared diff --git a/packages/core/src/persistent-pty/index.ts b/packages/core/src/persistent-pty/index.ts index 5e9e7227b511..32536b1451cc 100644 --- a/packages/core/src/persistent-pty/index.ts +++ b/packages/core/src/persistent-pty/index.ts @@ -363,6 +363,7 @@ const makeLayer = (options: Options = {}) => const shutdown = Effect.fn("PersistentPty.shutdown")(function* () { const response = yield* daemon.shutdown.pipe(Effect.mapError(unavailable)) current.clear() + yield* detaching.handoffDropped if (!response) return if (response.type !== "ok") return yield* unexpected(response) }) diff --git a/packages/core/test/detaching.test.ts b/packages/core/test/detaching.test.ts index 40e237c51c10..9bf401d90a0d 100644 --- a/packages/core/test/detaching.test.ts +++ b/packages/core/test/detaching.test.ts @@ -42,6 +42,16 @@ describe("Detaching", () => { }), ) + it.effect("does not detach once the prepared handoff was dropped", () => + Effect.gen(function* () { + const detaching = yield* Detaching.make + yield* detaching.handoffPrepared + yield* detaching.handoffDropped + expect(yield* detaching.begin).toBe(false) + expect(detaching.isStopping()).toBe(true) + }), + ) + it.effect("starts stopping on any shutdown, detaching or not", () => Effect.gen(function* () { const detaching = yield* Detaching.make From 71005e765c349f3a6927ab4061ba3f02b2252676 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 20:15:04 +0000 Subject: [PATCH 25/29] refactor(core): keep daemon shell commands in one server-wide registry --- packages/core/src/plugin/internal.ts | 2 + packages/core/src/shell.ts | 856 ++++++++++++--------- packages/core/src/tool/plugin/shell.ts | 8 +- packages/core/test/fixture/shell-daemon.ts | 18 +- packages/core/test/shell-daemon.test.ts | 35 +- packages/core/test/tool-shell.test.ts | 4 + packages/server/src/routes.ts | 3 + 7 files changed, 526 insertions(+), 400 deletions(-) diff --git a/packages/core/src/plugin/internal.ts b/packages/core/src/plugin/internal.ts index c7f0d7894ec1..a77e460c529a 100644 --- a/packages/core/src/plugin/internal.ts +++ b/packages/core/src/plugin/internal.ts @@ -169,6 +169,7 @@ const services = [ SubagentJob.Service, SharedCache.Service, Shell.Service, + Shell.Daemon, ShellSelect.Service, Snapshot.Service, Skill.Service, @@ -213,6 +214,7 @@ export const requirements = LayerNode.group([ Detaching.node, Job.node, LocationLifecycle.node, + Shell.daemonNode, KV.node, llmClient, Location.node, diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index 2980d6650662..6d2cea8c270e 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -26,8 +26,6 @@ import { Bus } from "./bus.js" import { Environment } from "./environment/index.js" import { FileRetention } from "./file-retention.js" import { Location } from "./location.js" -import { LocationLifecycle } from "./location-lifecycle.js" -import { Detaching } from "./detaching.js" import { Global } from "@opencode/util/global" import { PersistentPty } from "./persistent-pty.js" import { ShellSelect } from "./shell/select.js" @@ -82,22 +80,12 @@ const Stored = Schema.Struct({ type Stored = typeof Stored.Type const decodeStored = Schema.decodeUnknownOption(Stored) -// A command run by the process daemon, which writes and counts its output file. -type Daemon = { - id: number - stored: Stored - // From the exit: newlines in the file, and why the daemon stopped writing it early. - outputLines?: number - outputError?: string - wait?: Fiber.Fiber - kill?: Fiber.Fiber -} - -type Active = { +// A command the server spawned itself. It dies with its Location. +type Spawned = { + kind: "spawned" // Immutable snapshot; lifecycle updates replace it via immer `produce`. info: Info - file: string - // Spawned commands only: bytes and newlines written so far. The daemon's file is the only count for its commands. + // Bytes and newlines written so far. size: number newlines: number // Resolves with the terminal Info once the command exits, times out, or is killed. A wait @@ -106,13 +94,31 @@ type Active = { timeoutFiber?: Fiber.Fiber timeout?: (duration: number) => Effect.Effect stop?: (reason: StopReason) => Effect.Effect - // Whether the process can outlive this server (run by the process daemon). Spawned commands cannot. - detachable: boolean - daemon?: Daemon - // Why a command was killed without a signal of its own. +} + +// A command the process daemon runs, which writes and counts its output file. It belongs to the server, not to +// the Location that started it, so it outlives a Location that closes or reloads. +type Run = { + kind: "daemon" + info: Info + process: number + toolCall?: ToolCall + timeoutDeadline?: number + stopReason?: StopReason + backgroundedAtShutdown?: boolean + // From the exit: newlines in the file, and why the daemon stopped writing it early. + outputLines?: number + outputError?: string + // Why it was killed without a signal of its own. killed?: string + done: Deferred.Deferred + timeoutFiber?: Fiber.Fiber + stopping?: Fiber.Fiber + wait?: Fiber.Fiber } +type Command = Spawned | Run + type Ended = { status: "exited" | StopReason exit?: number @@ -213,6 +219,369 @@ const cleanupNode = makeGlobalNode({ deps: [FSUtil.node, Global.node, PersistentPty.node], }) +/** One Location's daemon commands, as `Daemon.at` hands them to its `Shell`. */ +interface Runs { + readonly get: (id: Shell.ID) => Run | undefined + readonly list: () => Run[] + /** Fails with `UnavailableError` only when the daemon could not be reached or started: spawning instead is safe. */ + readonly create: (input: { + readonly info: Info + readonly args: readonly string[] + readonly env: Readonly> + readonly timeout: number + readonly toolCall: ToolCall | undefined + }) => Effect.Effect + readonly timeout: (run: Run, duration: number) => Effect.Effect + readonly stop: (run: Run, reason: StopReason) => Effect.Effect + readonly backgroundAtShutdown: (run: Run) => Effect.Effect + readonly remove: (id: Shell.ID) => Effect.Effect +} + +export interface DaemonInterface { + /** A Location's daemon commands. The first call for a Location in this server takes its commands back from the daemon. */ + readonly at: (location: Location.Interface) => Effect.Effect + /** Runs `effect` until it ends or the server stops, outliving the caller's Location (e.g. a job's completion notice). */ + readonly watch: (effect: Effect.Effect) => Effect.Effect +} + +/** The process daemon's shell commands, owned by the server and grouped by Location. */ +export class Daemon extends Context.Service()("@opencode/Shell/Daemon") {} + +const daemonLayer = Layer.effect( + Daemon, + Effect.gen(function* () { + const bus = yield* Bus.Service + const global = yield* Global.Service + const pty = yield* PersistentPty.Service + const scope = yield* Effect.scope + const context = yield* Effect.context() + const { unlink } = yield* Effect.promise(() => import("fs/promises")) + // Keyed by canonical Location. `recovered` resolves once the Location's commands were taken back. + const places = new Map }>() + // Fibers run with the server's context in its scope, so a Location closing never interrupts them. + const fork = (effect: Effect.Effect) => + effect.pipe(Effect.forkIn(scope, { startImmediately: true }), Effect.setContext(context)) + + // On shutdown the processes keep running for the next server: only stop waiting on them. + yield* Effect.addFinalizer(() => + Effect.forEach( + places.values(), + (place) => Effect.forEach(place.runs.list(), (run) => Deferred.interrupt(run.done), { discard: true }), + { discard: true }, + ), + ) + + const place = (location: Location.Interface) => { + const runs = new Map() + const exitOrder: Shell.ID[] = [] + const ref = Location.Ref.make({ directory: location.directory, workspaceID: location.workspaceID }) + const outputDir = path.join(global.data, DIRECTORY, location.project.id) + + const persisted = (run: Run): Stored => ({ + id: run.info.id, + command: run.info.command, + shell: run.info.shell, + location: { + directory: location.directory, + ...(location.workspaceID !== undefined ? { workspaceID: location.workspaceID } : {}), + }, + metadata: run.info.metadata, + toolCall: run.toolCall, + timeoutDeadline: run.timeoutDeadline, + stopReason: run.stopReason, + backgroundedAtShutdown: run.backgroundedAtShutdown, + }) + + // Records the command's latest timeout and stop reason in the daemon, so a restart sees them. + const store = (run: Run) => + pty + .setPipeMetadata(run.process, persisted(run)) + .pipe( + Effect.catch((error) => + Effect.logWarning("failed to update shell command metadata", { id: run.info.id, error: error.message }), + ), + ) + + const removeRun = Effect.fnUntraced(function* (id: Shell.ID) { + const run = runs.get(id) + const index = exitOrder.indexOf(id) + if (index !== -1) exitOrder.splice(index, 1) + if (!run) return + runs.delete(id) + if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber) + if (run.stopping) yield* Fiber.interrupt(run.stopping) + // Stop waiting first, so forgetting the process is not mistaken for losing it. + if (run.wait) yield* Fiber.interrupt(run.wait) + // Forget the process before deleting its file. + yield* pty.terminatePipe(run.process).pipe(Effect.ignore) + yield* Effect.promise(() => unlink(run.info.file).catch(() => {})) + yield* Deferred.fail(run.done, new NotFoundError({ id })) + // Every shell reports exactly one exit; removing a running one ends it as killed. + if (run.info.status === "running") + yield* bus.publish(Shell.Event.Exited, { id, status: "killed" }, { location: ref }) + yield* bus.publish(Shell.Event.Deleted, { id }, { location: ref }) + }) + + const evict = Effect.fnUntraced(function* () { + while (exitOrder.length > EXITED_LIMIT) { + const oldest = exitOrder[0] + if (!oldest) break + yield* removeRun(oldest) + } + }) + + const finish = Effect.fnUntraced(function* (run: Run, outcome: Ended) { + const id = run.info.id + // A removed command already failed its waiters and published its exit and deletion. + if (runs.get(id) !== run) return + run.info = produce(run.info, (draft) => { + draft.status = outcome.status + if (outcome.exit !== undefined) draft.exit = outcome.exit + if (outcome.signal !== undefined) draft.signal = outcome.signal + draft.time.completed = outcome.completed + }) + if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber) + run.timeoutFiber = undefined + // Resolve waiters before retention eviction, so an evicted command still reports its + // terminal Info rather than the removal NotFoundError. + yield* Deferred.succeed(run.done, run.info) + yield* bus.publish( + Shell.Event.Exited, + { id, ...(outcome.exit !== undefined ? { exit: outcome.exit } : {}), status: outcome.status }, + { location: ref }, + ) + exitOrder.push(id) + yield* evict() + }) + + // Graceful, like spawned commands: SIGTERM, then SIGKILL if it has not exited 3 s later. It runs on its own, + // so interrupting the caller (a timer being retimed, removed or torn down) never cancels a recorded stop. + const stop = (run: Run, reason: StopReason) => + Effect.gen(function* () { + if (run.stopReason !== undefined || run.info.status !== "running") return + run.stopReason = reason + run.timeoutDeadline = undefined + run.stopping = yield* fork( + Effect.gen(function* () { + yield* store(run) + yield* pty.signalPipe(run.process, "SIGTERM").pipe(Effect.ignore) + const ended = yield* Deferred.await(run.done).pipe(Effect.timeoutOption(STOP_GRACE), Effect.exit) + if (Exit.isSuccess(ended) && Option.isNone(ended.value)) + yield* pty.signalPipe(run.process, "SIGKILL").pipe(Effect.ignore) + }), + ) + }) + + const arm = Effect.fnUntraced(function* (run: Run, duration: number) { + run.timeoutFiber = yield* fork( + Effect.sleep(Duration.millis(duration)).pipe(Effect.andThen(stop(run, "timeout"))), + ) + }) + + const timeout = Effect.fnUntraced(function* (run: Run, duration: number) { + if (run.stopReason !== undefined || run.info.status !== "running") return + if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber) + run.timeoutFiber = undefined + run.timeoutDeadline = duration === 0 ? undefined : Date.now() + duration + yield* store(run) + if (duration !== 0) yield* arm(run, duration) + }) + + const backgroundAtShutdown = Effect.fnUntraced(function* (run: Run) { + if (run.info.status !== "running") return + if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber) + run.timeoutFiber = undefined + run.timeoutDeadline = undefined + run.backgroundedAtShutdown = true + yield* store(run) + }) + + // Watches the exit, and the timeout (in ms from now, none if undefined). + const attach = Effect.fnUntraced(function* (run: Run, duration: number | undefined) { + run.wait = yield* fork( + pty.waitPipe(run.process).pipe( + Effect.map((exit): Ended => { + run.outputLines = exit.outputLines + run.outputError = exit.outputError + // A stopped command reports its reason, not the signal it was stopped with, as spawned ones do. + if (run.stopReason) return { status: run.stopReason, completed: exit.exitedAt } + return { + status: "exited", + ...(exit.exitCode !== undefined ? { exit: exit.exitCode } : {}), + ...(exit.signal !== undefined ? { signal: exit.signal } : {}), + completed: exit.exitedAt, + } + }), + // The daemon went away (and its processes with it), or forgot the process: it was killed. + Effect.catch((error) => + Effect.logWarning("lost a shell command's process", { id: run.info.id, error: error.message }).pipe( + Effect.andThen( + Effect.sync((): Ended => { + run.killed = LOST + return { status: "killed", completed: Date.now() } + }), + ), + ), + ), + Effect.flatMap((outcome) => finish(run, outcome)), + ), + ) + // A stop interrupted by a restart lost its grace period: kill now. + if (run.stopReason !== undefined) { + run.stopping = yield* fork(pty.signalPipe(run.process, "SIGKILL").pipe(Effect.ignore)) + return + } + if (duration === undefined) return + if (duration > 0) return yield* arm(run, duration) + yield* stop(run, "timeout") + }) + + const create: Runs["create"] = Effect.fnUntraced(function* (input) { + const run: Run = { + kind: "daemon", + info: input.info, + process: 0, + toolCall: input.toolCall, + timeoutDeadline: input.timeout > 0 ? input.info.time.started + input.timeout : undefined, + done: Deferred.makeUnsafe(), + } + const created = yield* pty.createPipe({ + program: input.info.shell, + args: input.args, + cwd: input.info.cwd, + env: input.env, + groupID: location.project.id, + outputPath: input.info.file, + metadata: persisted(run), + }) + run.process = created.id + run.info = produce(input.info, (draft) => { + if (created.pid !== undefined) draft.pid = created.pid + // The daemon's clock, so the command reports the same start after a restart. + draft.time.started = created.startedAt + }) + runs.set(run.info.id, run) + // Timed from now: starting the daemon may have taken a while. + yield* attach(run, input.timeout > 0 ? input.timeout : undefined) + yield* bus.publish(Shell.Event.Created, { info: run.info }, { location: ref }) + return run.info + }) + + // Takes back this Location's commands, left by a previous server. Two Locations can share a project's output + // directory, so match the Location itself. + const recover = Effect.fnUntraced(function* () { + const processes = yield* pty + .listPipes() + .pipe( + Effect.catch((error) => + Effect.logWarning("failed to list shell commands to recover", { error: error.message }).pipe( + Effect.as([]), + ), + ), + ) + const finished: Run[] = [] + for (const process of processes) { + const stored = Option.getOrUndefined(decodeStored(process.metadata)) + if ( + !stored || + stored.location.directory !== location.directory || + stored.location.workspaceID !== location.workspaceID || + path.dirname(process.outputPath) !== outputDir || + runs.has(stored.id) + ) + continue + const running = process.lifecycle.status === "running" + const exited = process.lifecycle.status === "exited" ? process.lifecycle : undefined + const status = running ? "running" : (stored.stopReason ?? (exited ? "exited" : "killed")) + const run: Run = { + kind: "daemon", + info: { + id: stored.id, + status, + command: stored.command, + cwd: process.cwd, + shell: stored.shell, + file: process.outputPath, + ...(process.pid !== undefined ? { pid: process.pid } : {}), + ...(status === "exited" && exited?.exitCode !== undefined ? { exit: exited.exitCode } : {}), + ...(status === "exited" && exited?.signal !== undefined ? { signal: exited.signal } : {}), + metadata: stored.metadata, + time: { + started: process.startedAt, + ...(running ? {} : { completed: process.exitedAt ?? process.startedAt }), + }, + }, + process: process.id, + toolCall: stored.toolCall, + timeoutDeadline: stored.timeoutDeadline, + stopReason: stored.stopReason, + backgroundedAtShutdown: stored.backgroundedAtShutdown, + ...(running ? {} : { outputLines: process.outputLines, outputError: process.outputError }), + ...(process.lifecycle.status === "failed" && !stored.stopReason + ? { killed: `Command was killed: ${process.lifecycle.message}` } + : {}), + done: Deferred.makeUnsafe(), + } + runs.set(stored.id, run) + if (running) { + // A command moved to the background at shutdown has no timeout; an expired deadline fires at once. + const deadline = stored.backgroundedAtShutdown ? undefined : stored.timeoutDeadline + yield* attach(run, deadline === undefined ? undefined : Math.max(0, deadline - Date.now())) + continue + } + yield* Deferred.succeed(run.done, run.info) + finished.push(run) + } + exitOrder.push( + ...finished + .toSorted((a, b) => (a.info.time.completed ?? 0) - (b.info.time.completed ?? 0)) + .map((run) => run.info.id), + ) + yield* evict() + }) + + const handle: Runs = { + get: (id) => runs.get(id), + list: () => Array.from(runs.values()), + create, + timeout, + stop, + backgroundAtShutdown, + remove: removeRun, + } + return { runs: handle, recover } + } + + const at = Effect.fnUntraced(function* (location: Location.Interface) { + const key = `${location.directory}\0${location.workspaceID ?? ""}` + const existing = places.get(key) + if (existing) { + yield* Deferred.await(existing.recovered) + return existing.runs + } + const placed = place(location) + const entry = { runs: placed.runs, recovered: Deferred.makeUnsafe() } + places.set(key, entry) + yield* placed.recover().pipe(Effect.ensuring(Deferred.succeed(entry.recovered, undefined))) + return placed.runs + }) + + return Daemon.of({ + at, + watch: (effect) => + fork(effect.pipe(Effect.catch((error) => Effect.logError("shell watcher failed", { error })))).pipe( + Effect.asVoid, + ), + }) + }), +) + +export const daemonNode = makeGlobalNode({ + service: Daemon, + layer: daemonLayer, + deps: [Bus.node, Global.node, PersistentPty.node], +}) + const layer = () => Layer.effect( Service, @@ -225,110 +594,70 @@ const layer = () => const hooks = yield* PluginHooks.Service const environments = yield* SessionEnvironment.Service const config = yield* Config.Service - const pty = yield* PersistentPty.Service - const lifecycle = yield* LocationLifecycle.Service - const detaching = yield* Detaching.Service + const daemon = yield* Daemon const context = yield* Effect.context() const runFork = Effect.runForkWith(context) - const commands = new Map() + // Commands this server spawned for the Location. Daemon commands live in `runs`, owned by the server. + const spawned = new Map() const exitOrder: Shell.ID[] = [] // Remote workspaces and Windows keep spawning commands from the server. const daemonEnabled = location.workspaceID === undefined && process.platform !== "win32" && PersistentPty.available + // Taken back before the service is returned, so session restart recovery sees them. + const runs = daemonEnabled ? yield* daemon.at(location) : undefined const outputDir = path.join(global.data, DIRECTORY, location.project.id) const { mkdir, unlink, open } = yield* Effect.promise(() => import("fs/promises")) const { createWriteStream, createReadStream } = yield* Effect.promise(() => import("fs")) yield* Effect.promise(() => mkdir(outputDir, { recursive: true })) - // A Location that closes while the server keeps running (a reload, an eviction) keeps watching the daemon's - // commands it started: their exit, timeout and stop, so the jobs waiting on them still hear how they ended. - // Its replacement, if one boots, takes the commands back too and is the one that reports and evicts them. + // Spawned commands end with their Location; daemon commands are left to the server. yield* Effect.addFinalizer(() => Effect.gen(function* () { - const stopping = detaching.isStopping() - for (const command of [...commands.values()]) { - if (command.daemon && !stopping) continue + for (const command of spawned.values()) { if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) - // On shutdown daemon commands are only detached from: their processes keep running for the next owner. - if (command.daemon?.kill) yield* Fiber.interrupt(command.daemon.kill) - if (command.daemon?.wait) yield* Fiber.interrupt(command.daemon.wait) // Teardown interrupts pending commands; it is not a terminal command failure. It never stops or // removes a command itself: a spawned process dies with its handle's scope once `done` settles. yield* Deferred.interrupt(command.done) - commands.delete(command.info.id) } + spawned.clear() exitOrder.length = 0 }), ) - let closed = false - yield* lifecycle.awaitShutdown.pipe( - Effect.andThen( - Effect.sync(() => { - closed = true - }), - ), - Effect.forkScoped, - ) + const find = (id: Shell.ID): Command | undefined => spawned.get(id) ?? runs?.get(id) const require = Effect.fnUntraced(function* (id: Shell.ID) { - const command = commands.get(id) + const command = find(id) if (!command) return yield* new NotFoundError({ id }) return command }) - const removeCommand = Effect.fnUntraced(function* (id: Shell.ID) { - const command = commands.get(id) + const removeSpawned = Effect.fnUntraced(function* (id: Shell.ID) { + const command = spawned.get(id) const index = exitOrder.indexOf(id) if (index !== -1) exitOrder.splice(index, 1) if (!command) return - commands.delete(id) + spawned.delete(id) if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) - if (command.daemon) { - if (command.daemon.kill) yield* Fiber.interrupt(command.daemon.kill) - // Stop waiting first, so forgetting the process is not mistaken for losing it. Removal never runs on the - // wait's own fiber: it settles the command through `finish`, which only evicts other commands. - if (command.daemon.wait) yield* Fiber.interrupt(command.daemon.wait) - // Forget the process before deleting its file. - yield* pty.terminatePipe(command.daemon.id).pipe(Effect.ignore) - yield* Effect.promise(() => unlink(command.file).catch(() => {})) - yield* Deferred.fail(command.done, new NotFoundError({ id })) - } - if (!command.daemon) { - // Unblock any wait still pending when the command is removed before it terminated. - yield* Deferred.fail(command.done, new NotFoundError({ id })) - yield* Effect.promise(() => unlink(command.file).catch(() => {})) - } + // Unblock any wait still pending when the command is removed before it terminated. + yield* Deferred.fail(command.done, new NotFoundError({ id })) + yield* Effect.promise(() => unlink(command.info.file).catch(() => {})) // Every shell reports exactly one exit; removing a running one ends it as killed. if (command.info.status === "running") yield* bus.publish(Shell.Event.Exited, { id, status: "killed" }) yield* bus.publish(Shell.Event.Deleted, { id }) }) - const evict = Effect.fnUntraced(function* () { - while (exitOrder.length > EXITED_LIMIT) { - const oldest = exitOrder[0] - if (!oldest) break - yield* removeCommand(oldest) - } - }) - - // Records how a command ended, once, unless it was removed meanwhile. - const finish = Effect.fnUntraced(function* (command: Active, outcome: Ended) { + // Records how a spawned command ended, once, unless it was removed meanwhile. + const finish = Effect.fnUntraced(function* (command: Spawned, outcome: Ended) { const id = command.info.id - // A removed command already failed its waiters and published its exit and deletion. - if (commands.get(id) !== command) return + if (spawned.get(id) !== command) return command.info = produce(command.info, (draft) => { draft.status = outcome.status if (outcome.exit !== undefined) draft.exit = outcome.exit if (outcome.signal !== undefined) draft.signal = outcome.signal draft.time.completed = outcome.completed }) - // The Location's replacement reports it and applies retention. - if (closed && command.daemon) { - yield* Deferred.succeed(command.done, command.info) - return - } // Resolve waiters before retention eviction, so an evicted command still reports its // terminal Info rather than the removal NotFoundError. yield* Deferred.succeed(command.done, command.info) @@ -338,7 +667,11 @@ const layer = () => status: outcome.status, }) exitOrder.push(id) - yield* evict() + while (exitOrder.length > EXITED_LIMIT) { + const oldest = exitOrder[0] + if (!oldest) break + yield* removeSpawned(oldest) + } // Keep exited history data-only. const timeoutFiber = command.timeoutFiber command.timeout = undefined @@ -348,12 +681,13 @@ const layer = () => }) const remove = Effect.fn("Shell.remove")(function* (id: Shell.ID) { - yield* require(id) - yield* removeCommand(id) + const command = yield* require(id) + if (command.kind === "daemon") return yield* runs!.remove(id) + yield* removeSpawned(id) }) const list = Effect.fn("Shell.list")(function* () { - return Array.from(commands.values()) + return [...spawned.values(), ...(runs?.list() ?? [])] .filter((command) => command.info.status === "running") .map((command) => command.info) }) @@ -368,38 +702,35 @@ const layer = () => const timeout = Effect.fn("Shell.timeout")(function* (id: Shell.ID, duration: number) { const command = yield* require(id) - if (command.info.status !== "running" || !command.timeout) return command.info - yield* command.timeout(duration) + if (command.kind === "daemon") yield* runs!.timeout(command, duration) + if (command.kind === "spawned" && command.info.status === "running" && command.timeout) + yield* command.timeout(duration) return command.info }) const stop = Effect.fn("Shell.stop")(function* (id: Shell.ID, reason: StopReason) { const command = yield* require(id) - if (command.info.status !== "running" || !command.stop) return command.info - yield* command.stop(reason) + if (command.kind === "daemon") yield* runs!.stop(command, reason) + if (command.kind === "spawned" && command.info.status === "running" && command.stop) yield* command.stop(reason) return command.info }) - const detachable = (id: Shell.ID) => commands.get(id)?.detachable ?? false + const detachable = (id: Shell.ID) => find(id)?.kind === "daemon" const backgroundAtShutdown = Effect.fn("Shell.backgroundAtShutdown")(function* (id: Shell.ID) { - const command = commands.get(id) - const daemon = command?.daemon - if (!command || !daemon || command.info.status !== "running") return - if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) - command.timeoutFiber = undefined - yield* store(daemon, { backgroundedAtShutdown: true, timeoutDeadline: undefined }) + const command = find(id) + if (command?.kind === "daemon") yield* runs!.backgroundAtShutdown(command) }) const byToolCall = Effect.fn("Shell.byToolCall")(function* (call: ToolCall) { - return Array.from(commands.values()).find((command) => { - const stored = command.daemon?.stored.toolCall - return ( - stored?.sessionID === call.sessionID && - stored.assistantMessageID === call.assistantMessageID && - stored.callID === call.callID - ) - })?.info + return runs + ?.list() + .find( + (run) => + run.toolCall?.sessionID === call.sessionID && + run.toolCall.assistantMessageID === call.assistantMessageID && + run.toolCall.callID === call.callID, + )?.info }) const readSpawned = (file: string, start: number, length: number) => @@ -445,8 +776,8 @@ const layer = () => const command = yield* require(id) const cursor = input?.cursor ?? 0 const limit = input?.limit ?? 65536 - if (command.daemon) { - const page = yield* readFile(command.file, () => cursor, limit) + if (command.kind === "daemon") { + const page = yield* readFile(command.info.file, () => cursor, limit) if (cursor >= page.size) return { output: "", cursor: page.size, size: page.size, truncated: false } return { output: page.bytes.toString("utf8"), @@ -457,7 +788,7 @@ const layer = () => } if (cursor >= command.size) return { output: "", cursor: command.size, size: command.size, truncated: false } const start = Math.max(0, cursor) - const bytes = yield* readSpawned(command.file, start, Math.min(limit, command.size - start)) + const bytes = yield* readSpawned(command.info.file, start, Math.min(limit, command.size - start)) return { output: bytes.toString("utf8"), cursor: start + bytes.length, @@ -483,19 +814,26 @@ const layer = () => ) // The last `maxBytes` of a finished command's output, the file's size and its newline count. - const tail = Effect.fnUntraced(function* (command: Active, maxBytes: number) { - if (!command.daemon) { + const tail = Effect.fnUntraced(function* (command: Command, maxBytes: number) { + if (command.kind === "spawned") { const start = Math.max(0, command.size - maxBytes) - const bytes = yield* readSpawned(command.file, start, command.size - start) - return { text: bytes.toString("utf8"), size: command.size, newlines: command.newlines, error: undefined } + const bytes = yield* readSpawned(command.info.file, start, command.size - start) + return { + text: bytes.toString("utf8"), + size: command.size, + newlines: command.newlines, + error: undefined, + killed: undefined, + } } - const page = yield* readFile(command.file, (size) => size - maxBytes, maxBytes) + const page = yield* readFile(command.info.file, (size) => size - maxBytes, maxBytes) return { text: page.bytes.toString("utf8"), size: page.size, // The daemon's count, unless the command ended without its exit report (the daemon went away). - newlines: command.daemon.outputLines ?? (yield* countNewlines(command.file)), - error: command.daemon.outputError, + newlines: command.outputLines ?? (yield* countNewlines(command.info.file)), + error: command.outputError, + killed: command.killed, } }) @@ -519,255 +857,16 @@ const layer = () => const total = empty ? 0 : Math.max(page.newlines + (page.text.endsWith("\n") ? 0 : 1), shown) const notice = truncated ? `\n\n${ShellResult.savedNotice({ shown, total }, info.file)}` : "" const error = page.error ? `\n\n[output file ${info.file} stopped early: ${page.error}]` : "" - const killed = (yield* require(info.id)).killed return { output: `${text || "(no output)"}${notice}${error}`, truncated, lines: { shown, total }, - ...(killed ? { killed } : {}), + ...(page.killed ? { killed: page.killed } : {}), } }).pipe(Effect.catchTag("Shell.NotFoundError", () => Effect.succeed(undefined))) return { info, capture } }) - // Replaces the command's daemon metadata, so a restart sees its latest timeout and stop reason. - // Cleared fields are left out of the JSON. - const store = (daemon: Daemon, patch: Partial) => { - daemon.stored = { ...daemon.stored, ...patch } - return pty.setPipeMetadata(daemon.id, daemon.stored).pipe( - Effect.catch((error) => - Effect.logWarning("failed to update shell command metadata", { - id: daemon.stored.id, - error: error.message, - }), - ), - ) - } - - // The command's metadata as the daemon has it. A closed Location and its replacement can both watch one - // command, so a timeout or stop recorded by either is read back from there. - const latest = (daemon: Daemon) => - pty.getPipe(daemon.id).pipe( - Effect.map((process) => Option.getOrUndefined(decodeStored(process.metadata))), - Effect.catchTags({ - // Forgotten, or no daemon (which takes its processes with it). - "PersistentPty.ProcessNotFoundError": () => Effect.succeed(undefined), - "PersistentPty.UnavailableError": () => Effect.succeed(daemon.stored), - }), - ) - - // Watches a command the daemon runs: its exit, timeout (in ms from now, none if undefined) and graceful - // stop. Used for new commands and for running ones taken back after a restart. - const attach = (command: Active, daemon: Daemon, timeout: number | undefined) => { - const stopping = (reason: StopReason) => - Effect.gen(function* () { - yield* store(daemon, { stopReason: reason, timeoutDeadline: undefined }) - yield* pty.signalPipe(daemon.id, "SIGTERM").pipe(Effect.ignore) - const ended = yield* Deferred.await(command.done).pipe(Effect.timeoutOption(STOP_GRACE), Effect.exit) - if (Exit.isSuccess(ended) && Option.isNone(ended.value)) - yield* pty.signalPipe(daemon.id, "SIGKILL").pipe(Effect.ignore) - }) - - // Detached, so interrupting the caller (a timer being retimed, removed or torn down) never cancels a - // recorded stop. Only removal and teardown interrupt it. - const stop = (reason: StopReason) => - Effect.sync(() => { - if (daemon.stored.stopReason !== undefined) return - daemon.stored = { ...daemon.stored, stopReason: reason } - daemon.kill = runFork(stopping(reason)) - }) - command.stop = stop - - // Times out at `deadline`, unless the daemon's metadata says the timeout was cleared, moved or already hit. - const arm = (duration: number, deadline: number | undefined) => { - const expire: Effect.Effect = Effect.gen(function* () { - const current = yield* latest(daemon) - if (!current) return - daemon.stored = current - if (current.stopReason !== undefined || current.backgroundedAtShutdown) return - const moved = current.timeoutDeadline - if (moved === undefined) return - if (deadline !== undefined && moved > deadline) return arm(moved - deadline, moved) - yield* stop("timeout") - }) - command.timeoutFiber = runFork(Effect.sleep(Duration.millis(duration)).pipe(Effect.andThen(expire))) - } - - command.timeout = (duration) => - Effect.gen(function* () { - if (daemon.stored.stopReason !== undefined) return - if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) - command.timeoutFiber = undefined - const deadline = duration === 0 ? undefined : Date.now() + duration - yield* store(daemon, { timeoutDeadline: deadline }) - if (deadline !== undefined) arm(duration, deadline) - }) - - daemon.wait = runFork( - pty.waitPipe(daemon.id).pipe( - Effect.flatMap( - Effect.fnUntraced(function* (exit): Effect.fn.Return { - daemon.outputLines = exit.outputLines - daemon.outputError = exit.outputError - // Another watcher of the command may have stopped it. - const reason = daemon.stored.stopReason ?? (yield* latest(daemon))?.stopReason - // A stopped command reports its reason, not the signal it was stopped with, as spawned ones do. - if (reason) return { status: reason, completed: exit.exitedAt } - return { - status: "exited", - ...(exit.exitCode !== undefined ? { exit: exit.exitCode } : {}), - ...(exit.signal !== undefined ? { signal: exit.signal } : {}), - completed: exit.exitedAt, - } - }), - ), - // The daemon went away (and its processes with it), or forgot the process: it was killed. - Effect.catch((error) => - Effect.logWarning("lost a shell command's process", { id: command.info.id, error: error.message }).pipe( - Effect.andThen( - Effect.sync((): Ended => { - command.killed = LOST - return { status: "killed", completed: Date.now() } - }), - ), - ), - ), - Effect.flatMap((outcome) => finish(command, outcome)), - ), - ) - - // A stop interrupted by a restart lost its grace period: kill now. - if (daemon.stored.stopReason !== undefined) { - daemon.kill = runFork(pty.signalPipe(daemon.id, "SIGKILL").pipe(Effect.ignore)) - return - } - if (timeout === undefined) return - if (timeout > 0) return arm(timeout, daemon.stored.timeoutDeadline) - runFork(stop("timeout")) - } - - const daemonCommand = (info: Info, daemon: Daemon): Active => ({ - info, - file: info.file, - size: 0, - newlines: 0, - done: Deferred.makeUnsafe(), - detachable: true, - daemon, - }) - - // Runs the command in the daemon. Fails with `UnavailableError` only when the daemon could not be reached - // or started, so the request never ran and spawning it here instead is safe. - const createInDaemon = Effect.fnUntraced(function* ( - info: Info, - invocation: ShellCreateBefore, - args: string[], - toolCall: ToolCall | undefined, - ) { - const stored: Stored = { - id: info.id, - command: info.command, - shell: info.shell, - location: { - directory: location.directory, - ...(location.workspaceID !== undefined ? { workspaceID: location.workspaceID } : {}), - }, - metadata: info.metadata, - ...(toolCall ? { toolCall } : {}), - ...(invocation.timeout > 0 ? { timeoutDeadline: info.time.started + invocation.timeout } : {}), - } - const created = yield* pty.createPipe({ - program: invocation.shell, - args, - cwd: invocation.cwd, - // The daemon replaces the environment, so variables removed by hooks stay removed, like the spawner. - env: Object.fromEntries( - Object.entries(invocation.env).filter((entry): entry is [string, string] => entry[1] !== undefined), - ), - groupID: location.project.id, - outputPath: info.file, - metadata: stored, - }) - const daemon: Daemon = { id: created.id, stored } - const command = daemonCommand( - produce(info, (draft) => { - if (created.pid !== undefined) draft.pid = created.pid - // The daemon's clock, so the command reports the same start after a restart. - draft.time.started = created.startedAt - }), - daemon, - ) - commands.set(info.id, command) - // Timed from now: starting the daemon may have taken a while. - attach(command, daemon, invocation.timeout > 0 ? invocation.timeout : undefined) - yield* bus.publish(Shell.Event.Created, { info: command.info }) - return command.info - }) - - // Takes back this Location's daemon commands, left by a previous server or an evicted Location. - // Two Locations can share a project's output directory, so match the Location itself. - const recover = Effect.fnUntraced(function* () { - const processes = yield* pty - .listPipes() - .pipe( - Effect.catch((error) => - Effect.logWarning("failed to list shell commands to recover", { error: error.message }).pipe( - Effect.as([]), - ), - ), - ) - const finished: { id: Shell.ID; exitedAt: number }[] = [] - for (const process of processes) { - const stored = Option.getOrUndefined(decodeStored(process.metadata)) - if ( - !stored || - stored.location.directory !== location.directory || - stored.location.workspaceID !== location.workspaceID || - path.dirname(process.outputPath) !== outputDir || - commands.has(stored.id) - ) - continue - const running = process.lifecycle.status === "running" - const exited = process.lifecycle.status === "exited" ? process.lifecycle : undefined - const status = running ? "running" : (stored.stopReason ?? (exited ? "exited" : "killed")) - const info: Info = { - id: stored.id, - status, - command: stored.command, - cwd: process.cwd, - shell: stored.shell, - file: process.outputPath, - ...(process.pid !== undefined ? { pid: process.pid } : {}), - ...(status === "exited" && exited?.exitCode !== undefined ? { exit: exited.exitCode } : {}), - ...(status === "exited" && exited?.signal !== undefined ? { signal: exited.signal } : {}), - metadata: stored.metadata, - time: { - started: process.startedAt, - ...(running ? {} : { completed: process.exitedAt ?? process.startedAt }), - }, - } - const daemon: Daemon = { - id: process.id, - stored, - ...(running ? {} : { outputLines: process.outputLines, outputError: process.outputError }), - } - const command = daemonCommand(info, daemon) - if (process.lifecycle.status === "failed" && !stored.stopReason) - command.killed = `Command was killed: ${process.lifecycle.message}` - commands.set(stored.id, command) - if (running) { - // A command moved to the background at shutdown has no timeout; an expired deadline fires at once. - const deadline = stored.backgroundedAtShutdown ? undefined : stored.timeoutDeadline - attach(command, daemon, deadline === undefined ? undefined : Math.max(0, deadline - Date.now())) - continue - } - yield* Deferred.succeed(command.done, info) - finished.push({ id: stored.id, exitedAt: info.time.completed ?? 0 }) - } - exitOrder.push(...finished.toSorted((a, b) => a.exitedAt - b.exitedAt).map((item) => item.id)) - yield* evict() - }) - const create = Effect.fn("Shell.create")(function* ( input: CreateInput, before?: (input: ShellCreateBefore) => Effect.Effect, @@ -808,26 +907,37 @@ const layer = () => time: { started: Date.now() }, } - if (daemonEnabled) { - const created = yield* createInDaemon(info, invocation, args, input.toolCall).pipe( - Effect.map(Option.some), - Effect.catchTags({ - "PersistentPty.UnavailableError": (error) => - Effect.logWarning("process daemon unavailable; spawning shell command instead", { - error: error.message, - }).pipe(Effect.as(Option.none())), - // The daemon may have run it: never run it again here. - "PersistentPty.RequestError": (cause) => - Effect.fail(new AppProcess.AppProcessError({ command: invocation.command, cause })), - }), - ) + if (runs) { + const created = yield* runs + .create({ + info, + args, + // The daemon replaces the environment, so variables removed by hooks stay removed, like the spawner. + env: Object.fromEntries( + Object.entries(invocation.env).filter((entry): entry is [string, string] => entry[1] !== undefined), + ), + timeout: invocation.timeout, + toolCall: input.toolCall, + }) + .pipe( + Effect.map(Option.some), + Effect.catchTags({ + "PersistentPty.UnavailableError": (error) => + Effect.logWarning("process daemon unavailable; spawning shell command instead", { + error: error.message, + }).pipe(Effect.as(Option.none())), + // The daemon may have run it: never run it again here. + "PersistentPty.RequestError": (cause) => + Effect.fail(new AppProcess.AppProcessError({ command: invocation.command, cause })), + }), + ) if (Option.isSome(created)) return created.value } // Spawn through the Environment and stream combined output to the file. The handle is scope-bound, so // the managing fiber keeps its scope open until the command terminates (it awaits `done` at the // end). `create` returns once `ready` resolves with the registered command. - const ready = Deferred.makeUnsafe() + const ready = Deferred.makeUnsafe() runFork( Effect.scoped( Effect.gen(function* () { @@ -844,17 +954,16 @@ const layer = () => .pipe( Effect.mapError((cause) => new AppProcess.AppProcessError({ command: invocation.command, cause })), ) - const command: Active = { + const command: Spawned = { + kind: "spawned", info: produce(info, (draft) => { draft.pid = handle.pid }), - file, size: 0, newlines: 0, done: Deferred.makeUnsafe(), - detachable: false, } - commands.set(id, command) + spawned.set(id, command) const stream = createWriteStream(file) const outputDone = Latch.makeUnsafe() @@ -950,9 +1059,6 @@ const layer = () => return command.info }) - // Taken back before the service is returned, so session restart recovery sees them. - if (daemonEnabled) yield* recover() - return Service.of({ create, list, @@ -982,9 +1088,7 @@ export const node = makeLocationNode({ PluginHooks.node, SessionEnvironment.node, Config.node, - PersistentPty.node, - LocationLifecycle.node, - Detaching.node, + daemonNode, cleanupNode, ], }) diff --git a/packages/core/src/tool/plugin/shell.ts b/packages/core/src/tool/plugin/shell.ts index 9429d5a44f81..9fce7a410410 100644 --- a/packages/core/src/tool/plugin/shell.ts +++ b/packages/core/src/tool/plugin/shell.ts @@ -252,6 +252,7 @@ export const Plugin = { const environment = yield* Environment.Service const access = yield* FileAccess.Service const shell = yield* Shell.Service + const daemon = yield* Shell.Daemon const shellSelect = yield* ShellSelect.Service const compatibleShell = shellSelect.resolve({ priority: "compat" }) const permission = yield* Permission.Service @@ -294,11 +295,10 @@ export const Plugin = { }) const notifyWhenDone = Effect.fn("ShellTool.notifyWhenDone")( - (sessionID: SessionSchema.ID, id: string, shellID: string, command: string, settled: Deferred.Deferred) => - notify(sessions, jobs, { sessionID, id, shellID, command, settled }), // Not in this Location's scope: a daemon command outlives a Location that closes while the server runs, and - // its session must still hear how it ended. The watch ends with the job, or when the job registry closes. - Effect.forkDetach({ startImmediately: true }), + // its session must still hear how it ended. + (sessionID: SessionSchema.ID, id: string, shellID: string, command: string, settled: Deferred.Deferred) => + daemon.watch(notify(sessions, jobs, { sessionID, id, shellID, command, settled })), ) // Foreground commands this tool is blocked on. A planned shutdown moves them to the background, so it records diff --git a/packages/core/test/fixture/shell-daemon.ts b/packages/core/test/fixture/shell-daemon.ts index 945abb9192f9..f874a7c071bc 100644 --- a/packages/core/test/fixture/shell-daemon.ts +++ b/packages/core/test/fixture/shell-daemon.ts @@ -40,8 +40,10 @@ export const daemonShells = Effect.fn(function* () { }) }), ).pipe(Layer.provide(hostEnvironmentLayer)) - const shellLayer = (directory: string) => + // `server` shares one server's daemon commands between Locations; without it the Location gets a server of its own. + const shellLayer = (directory: string, server?: Shell.DaemonInterface) => AppNodeBuilder.build(LayerNode.group([Shell.node, LocationLifecycle.node, Bus.node, Detaching.node]), [ + ...(server ? [Shell.daemonNode.replace(Layer.succeed(Shell.Daemon, server))] : []), Location.node.replace( Layer.succeed( Location.Service, @@ -53,7 +55,15 @@ export const daemonShells = Effect.fn(function* () { Environment.node.replace(environment), PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, pty)), ]) - // A Location's Shell that stays open for the rest of the test. + // The server that `openLocation` Locations share, as long as the test runs. + const serverContext = yield* Layer.build( + AppNodeBuilder.build(LayerNode.group([Shell.daemonNode, Bus.node]), [ + Global.node.replace(global), + PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, pty)), + ]), + ) + const server = Context.get(serverContext, Shell.Daemon) + // A Location's Shell, on a server of its own, that stays open for the rest of the test. const openShell = (directory: string) => Effect.gen(function* () { const context = yield* Layer.build(shellLayer(directory)) @@ -63,7 +73,7 @@ export const daemonShells = Effect.fn(function* () { const openLocation = (directory: string) => Effect.gen(function* () { const scope = yield* Scope.make() - const context = yield* Layer.buildWithScope(shellLayer(directory), scope) + const context = yield* Layer.buildWithScope(shellLayer(directory, server), scope) return { shell: Context.get(context, Shell.Service), bus: Context.get(context, Bus.Service), @@ -96,6 +106,8 @@ export const daemonShells = Effect.fn(function* () { withShell, openShell, openLocation, + // Where the server's daemon commands report. + serverBus: Context.get(serverContext, Bus.Service), spawned: () => spawned, created: () => daemon.requests.filter((request) => request.op === "create_pipe"), signals: (id: number) => diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts index 9c8cf328be69..ed77de8434e2 100644 --- a/packages/core/test/shell-daemon.test.ts +++ b/packages/core/test/shell-daemon.test.ts @@ -306,42 +306,43 @@ describe("Shell on the process daemon", () => { }), ) - timed("a closed Location and its replacement share a command's timeout and stop through the daemon", () => + timed("a closed Location and its replacement see the same commands, with one timer and one report each", () => Effect.gen(function* () { const env = yield* daemonShells() const closing = yield* env.openLocation(env.directory) const backgrounded = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60", timeout: 100 }) const stopped = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60" }) + const removed = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60" }) yield* closing.close - const exits = { closed: 0, replacement: 0 } - yield* closing.bus.listen((event) => + let exits = 0 + yield* env.serverBus.listen((event) => Effect.sync(() => { - if (event.type === "shell.exited") exits.closed++ + if (event.type === "shell.exited") exits++ }), ) const replacement = yield* env.openLocation(env.directory) - yield* replacement.bus.listen((event) => - Effect.sync(() => { - if (event.type === "shell.exited") exits.replacement++ - }), - ) - expect((yield* replacement.shell.list()).map((item) => item.id)).toEqual([backgrounded.id, stopped.id]) + expect((yield* replacement.shell.list()).map((item) => item.id)).toEqual([ + backgrounded.id, + stopped.id, + removed.id, + ]) // The call that started it, still in the closed Location, moves it to the background and clears its timeout. yield* closing.shell.timeout(backgrounded.id, 0) - const gets = () => env.daemon.ops.filter((op) => op === "get").length - const before = gets() yield* TestClock.adjust("100 millis") - // The replacement's timer checked the daemon and left the command alone. - yield* until(() => gets() > before) expect(env.signals(1)).toEqual([]) - // Cancelling its job in the closed Location stops it; the replacement reports why it ended. + // Cancelling its job in the closed Location stops it; both see why it ended, and it is reported once. yield* closing.shell.stop(stopped.id, "killed") expect(yield* replacement.shell.wait(stopped.id)).toMatchObject({ status: "killed" }) expect(yield* closing.shell.wait(stopped.id)).toMatchObject({ status: "killed" }) - yield* until(() => exits.replacement === 1) - expect(exits.closed).toBe(0) + yield* until(() => exits === 1) + + // Removed through one, it is gone from both. + yield* replacement.shell.remove(removed.id) + expect(yield* closing.shell.get(removed.id).pipe(Effect.flip)).toBeInstanceOf(Shell.NotFoundError) + // The server watched them all along: the replacement took nothing back and waited on nothing more. + expect(env.daemon.ops.filter((op) => op === "wait")).toHaveLength(3) yield* replacement.close }), ) diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index 763133993401..ca7dccdaecfa 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -144,6 +144,7 @@ const shellPluginSupervisor = makeLocationNode({ Detaching.node, Job.node, Shell.node, + Shell.daemonNode, ShellSelect.node, Tool.node, ], @@ -151,6 +152,7 @@ const shellPluginSupervisor = makeLocationNode({ const nodes = LayerNode.group([ PersistentPty.node, + Shell.daemonNode, Database.node, Bus.node, Detaching.node, @@ -239,6 +241,7 @@ const detachableShellSupervisor = makeLocationNode({ Detaching.node, Job.node, Shell.node, + Shell.daemonNode, ShellSelect.node, Tool.node, ], @@ -1261,6 +1264,7 @@ describe("ShellTool", () => { Detaching.node, Job.node, Shell.node, + Shell.daemonNode, ShellSelect.node, Tool.node, ], diff --git a/packages/server/src/routes.ts b/packages/server/src/routes.ts index b5bdeba58989..dec20ac21b60 100644 --- a/packages/server/src/routes.ts +++ b/packages/server/src/routes.ts @@ -20,6 +20,7 @@ import { Instance } from "@opencode/core/instance/service" import { SessionTransfer } from "@opencode/core/session/transfer" import { InstructionEntry } from "@opencode/core/session/instruction-entry" import { ShellSelect } from "@opencode/core/shell/select" +import { Shell } from "@opencode/core/shell" import { Detaching } from "@opencode/core/detaching" import { Job } from "@opencode/core/job" import { KV } from "@opencode/core/kv" @@ -77,6 +78,8 @@ const applicationServiceNodes = [ PermissionSaved.node, PtyTicket.node, PersistentPty.node, + // Daemon-run shell commands belong to the server, so a Location closing never takes them down. + Shell.daemonNode, Credential.node, WellKnown.node, ServerPairing.node, From 6739eb588bf65274a4ff5c4da50c5ce0ce653cb3 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 20:18:21 +0000 Subject: [PATCH 26/29] refactor(core): record a foreground shell command in the daemon instead of at shutdown --- packages/core/src/artifact.ts | 4 +- packages/core/src/detaching.ts | 38 ++------ packages/core/src/generate.ts | 92 +++++++++++--------- packages/core/src/schedule.ts | 41 +++++---- packages/core/src/shell.ts | 41 ++++----- packages/core/src/tool/plugin/shell.ts | 13 +-- packages/core/test/detaching.test.ts | 32 +------ packages/core/test/fixture/shell-daemon.ts | 10 +-- packages/core/test/session-execution.test.ts | 8 ++ packages/core/test/shell-daemon.test.ts | 38 +++++--- packages/core/test/tool-shell.test.ts | 3 +- 11 files changed, 140 insertions(+), 180 deletions(-) diff --git a/packages/core/src/artifact.ts b/packages/core/src/artifact.ts index d59263696c39..5c184d291bc0 100644 --- a/packages/core/src/artifact.ts +++ b/packages/core/src/artifact.ts @@ -169,7 +169,9 @@ export interface Interface extends State.Transformable { readonly provider?: ProviderID }) => Effect.Effect readonly create: (input: CreateInput) => Effect.Effect - readonly get: (id: ID) => Effect.Effect + readonly get: ( + id: ID, + ) => Effect.Effect readonly info: (id: ID) => Effect.Effect readonly update: (id: ID, input: UpdateInput) => Effect.Effect readonly remove: (id: ID) => Effect.Effect diff --git a/packages/core/src/detaching.ts b/packages/core/src/detaching.ts index f1652287b7b8..e15ad9967e34 100644 --- a/packages/core/src/detaching.ts +++ b/packages/core/src/detaching.ts @@ -1,34 +1,26 @@ export * as Detaching from "./detaching.js" -import { Context, Duration, Effect, Layer, Scope } from "effect" +import { Context, Effect, Layer } from "effect" import { makeGlobalNode } from "@opencode/util/effect/app-node" -// What detaching hooks may take together: shutdown should not wait on them. -const HOOK_BUDGET = Duration.millis(500) - /** * Planned-shutdown state of this server process. * * A prepared daemon handoff means a new server will take over the daemon's processes. If this server - * then shuts down, it is detaching: work that can outlive it in the daemon is left running and recorded - * for the new server instead of being stopped. Without a prepared handoff (a normal quit), shutdown - * stops everything as before. + * then shuts down, it is detaching: work that can outlive it in the daemon is left running for the new + * server instead of being stopped. Without a prepared handoff (a normal quit), shutdown stops everything + * as before. */ export interface Interface { /** Records that a daemon handoff was prepared successfully. */ readonly handoffPrepared: Effect.Effect /** Records that the prepared handoff no longer holds: the daemon was shut down, so nothing can outlive this server. */ readonly handoffDropped: Effect.Effect - /** - * Called once shutdown starts, before any service is torn down. Returns whether this server is detaching, - * after running the detaching hooks (best effort, bounded) when it is. - */ + /** Called once shutdown starts, before any service is torn down. Returns whether this server is detaching. */ readonly begin: Effect.Effect readonly isDetaching: () => boolean /** Whether shutdown has started, detaching or not. Interrupts from then on are not the user's. */ readonly isStopping: () => boolean - /** Runs `hook` when this server starts detaching, while every service is still open. Removed with the scope. */ - readonly onDetach: (hook: Effect.Effect) => Effect.Effect } export class Service extends Context.Service()("@opencode/Detaching") {} @@ -37,7 +29,6 @@ export const make = Effect.sync(() => { let prepared = false let detaching = false let stopping = false - const hooks = new Set>() return Service.of({ handoffPrepared: Effect.sync(() => { prepared = true @@ -45,28 +36,13 @@ export const make = Effect.sync(() => { handoffDropped: Effect.sync(() => { prepared = false }), - begin: Effect.gen(function* () { + begin: Effect.sync(() => { stopping = true detaching = prepared - if (!detaching) return false - yield* Effect.forEach(hooks, (hook) => hook.pipe(Effect.catchCause(() => Effect.void)), { - concurrency: "unbounded", - discard: true, - }).pipe(Effect.timeoutOption(HOOK_BUDGET)) - return true + return detaching }), isDetaching: () => detaching, isStopping: () => stopping, - onDetach: (hook) => - Effect.acquireRelease( - Effect.sync(() => { - // A fresh reference per registration, so the same effect can be registered twice. - const entry = Effect.suspend(() => hook) - hooks.add(entry) - return entry - }), - (entry) => Effect.sync(() => hooks.delete(entry)), - ).pipe(Effect.asVoid), }) }) diff --git a/packages/core/src/generate.ts b/packages/core/src/generate.ts index a28ab6b32646..bb1c3e7dcbad 100644 --- a/packages/core/src/generate.ts +++ b/packages/core/src/generate.ts @@ -90,24 +90,26 @@ export const layer = Layer.effect( const decisionClient = yield* DecisionClient.Service const runText = Effect.fn("Generate.text")(function* (input: TextInput) { - const resolved = yield* resolver.resolve(input.model).pipe( - Effect.catchTag( - [ - "SessionRunnerModel.VariantUnavailableError", - "SessionRunnerModel.UnsupportedPackageError", - "SessionRunnerModel.ModelConfigurationError", - "SessionRunnerModel.ModelInitializationError", - "SessionRunnerModel.UnresolvedProviderVariablesError", - "SessionRunnerModel.UnsupportedCompactionError", - ], - (error) => - Effect.fail( - input.model - ? resolutionError(input.model, "chat", error) - : new UnavailableError({ message: error.message, service: error.providerID }), - ), - ), - ) + const resolved = yield* resolver + .resolve(input.model) + .pipe( + Effect.catchTag( + [ + "SessionRunnerModel.VariantUnavailableError", + "SessionRunnerModel.UnsupportedPackageError", + "SessionRunnerModel.ModelConfigurationError", + "SessionRunnerModel.ModelInitializationError", + "SessionRunnerModel.UnresolvedProviderVariablesError", + "SessionRunnerModel.UnsupportedCompactionError", + ], + (error) => + Effect.fail( + input.model + ? resolutionError(input.model, "chat", error) + : new UnavailableError({ message: error.message, service: error.providerID }), + ), + ), + ) if (!resolved && input.model) return yield* missing(input.model, "chat") if (!resolved) return yield* new UnavailableError({ message: "No model specified and no supported model is available" }) @@ -152,18 +154,20 @@ export const layer = Layer.effect( .pipe(Effect.mapError((error) => resolutionError(input.model, "image", error))) if (!model) return yield* missing(input.model, "image") // `ai.image.generate` turns an invalid request into a typed error; building it with `ai.image.request` would throw. - const response = yield* ai.image.generate({ - model, - prompt: input.prompt, - images: input.images?.map((file) => Media.base64(file.data, file.mime)), - n: input.n, - aspect: input.aspect, - resolution: input.resolution, - }).pipe( - Effect.provideService(ImageClient.Service, images), - ModelRetry.retryOneShot, - Effect.mapError((error) => providerError(input.model, error)), - ) + const response = yield* ai.image + .generate({ + model, + prompt: input.prompt, + images: input.images?.map((file) => Media.base64(file.data, file.mime)), + n: input.n, + aspect: input.aspect, + resolution: input.resolution, + }) + .pipe( + Effect.provideService(ImageClient.Service, images), + ModelRetry.retryOneShot, + Effect.mapError((error) => providerError(input.model, error)), + ) // URL results are downloaded so every image comes back inline, like the request's own files. // Materializing also detects the type of a downloaded image from its bytes when the URL didn't say. const outputs = yield* Effect.forEach(response.images, (asset) => @@ -190,19 +194,21 @@ export const layer = Layer.effect( .resolveDecision(input.model) .pipe(Effect.mapError((error) => resolutionError(input.model, "decision", error))) if (!model) return yield* missing(input.model, "decision") - const response = yield* ai.decision.generate({ - model, - input: input.input.map((part) => - part.type === "file" - ? { type: "media" as const, media: Media.base64(part.data, part.mime), filename: part.name } - : part, - ), - questions: input.questions, - }).pipe( - Effect.provideService(DecisionClient.Service, decisionClient), - ModelRetry.retryOneShot, - Effect.mapError((error) => providerError(input.model, error)), - ) + const response = yield* ai.decision + .generate({ + model, + input: input.input.map((part) => + part.type === "file" + ? { type: "media" as const, media: Media.base64(part.data, part.mime), filename: part.name } + : part, + ), + questions: input.questions, + }) + .pipe( + Effect.provideService(DecisionClient.Service, decisionClient), + ModelRetry.retryOneShot, + Effect.mapError((error) => providerError(input.model, error)), + ) const usage = response.usage return { answers: response.answers, diff --git a/packages/core/src/schedule.ts b/packages/core/src/schedule.ts index 16dad0ac5ba0..8182768a02d1 100644 --- a/packages/core/src/schedule.ts +++ b/packages/core/src/schedule.ts @@ -2,7 +2,20 @@ export * as Schedule from "./schedule.js" import { randomUUID } from "node:crypto" import { and, asc, eq, gt, isNull, lte, min, or, sql } from "drizzle-orm" -import { Cause, Clock, Context, Cron, DateTime, Effect, Latch, Layer, Option, Result, Schedule as EffectSchedule, Schema } from "effect" +import { + Cause, + Clock, + Context, + Cron, + DateTime, + Effect, + Latch, + Layer, + Option, + Result, + Schedule as EffectSchedule, + Schema, +} from "effect" import { Node, makeGlobalNode } from "@opencode/util/effect/app-node" import { LayerNode } from "@opencode/util/effect/layer-node" import { @@ -317,15 +330,10 @@ const layer = Layer.effect( const note = outcome.invalid ? `[Schedule paused after invalid check: ${outcome.error}]` : `[Schedule paused after ${strikes} failed checks: ${outcome.error}]` - const id = yield* admitWake( - row, - session.id, - run, - due, - late, - `${row.action.prompt}\n\n${note}`, - { error: outcome.error, failures: strikes }, - ) + const id = yield* admitWake(row, session.id, run, due, late, `${row.action.prompt}\n\n${note}`, { + error: outcome.error, + failures: strikes, + }) yield* db .update(ScheduleTable) .set({ @@ -624,12 +632,7 @@ const layer = Layer.effect( }) } - const firstDue = Effect.fnUntraced(function* ( - when: When, - hasCheck: boolean, - zone: DateTime.TimeZone, - now: number, - ) { + const firstDue = Effect.fnUntraced(function* (when: When, hasCheck: boolean, zone: DateTime.TimeZone, now: number) { switch (when.type) { case "in": return now + (yield* duration(when.in)) @@ -688,8 +691,7 @@ const layer = Layer.effect( if (!config.enabled) return yield* new DisabledError() const check = yield* resolveStoredCheck(input) const when = input.when ?? (check?.mode === "until" ? DEFAULT_UNTIL_EVERY : undefined) - if (when === undefined) - return yield* new InvalidError({ reason: "when is required unless until is set" }) + if (when === undefined) return yield* new InvalidError({ reason: "when is required unless until is set" }) const zone = yield* resolveZone(when.tz ?? config.timezone) const now = yield* Clock.currentTimeMillis const due = yield* firstDue(when, check !== undefined, zone, now) @@ -724,7 +726,8 @@ const layer = Layer.effect( ) if (!init.ok) { // A code check that throws or fails at creation is broken (e.g. wrong MCP response field); fail right away. - if (init.invalid || check.condition.type === "code") return yield* new InvalidError({ reason: init.error }) + if (init.invalid || check.condition.type === "code") + return yield* new InvalidError({ reason: init.error }) return UNSEEDED } if (check.mode === "until" && init.wake) diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index 6d2cea8c270e..213c9e8e3e81 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -63,6 +63,8 @@ export type ToolCall = typeof ToolCall.Type type CreateInput = Shell.CreateInput & { shell?: string toolCall?: ToolCall + // A call is waiting on it. A server that takes it over after a restart moves it to the background instead. + foreground?: boolean } // What the server needs to take a daemon-run command back after a restart, kept in the daemon's metadata. @@ -75,7 +77,7 @@ const Stored = Schema.Struct({ toolCall: Schema.optional(ToolCall), timeoutDeadline: Schema.optional(Schema.Number), stopReason: Schema.optional(Schema.Literals(["timeout", "killed"])), - backgroundedAtShutdown: Schema.optional(Schema.Boolean), + foreground: Schema.optional(Schema.Boolean), }) type Stored = typeof Stored.Type const decodeStored = Schema.decodeUnknownOption(Stored) @@ -105,7 +107,8 @@ type Run = { toolCall?: ToolCall timeoutDeadline?: number stopReason?: StopReason - backgroundedAtShutdown?: boolean + // A call waits on it, until it moves to the background (its timeout cleared). + foreground: boolean // From the exit: newlines in the file, and why the daemon stopped writing it early. outputLines?: number outputError?: string @@ -159,9 +162,6 @@ export interface Interface { readonly stop: (id: Shell.ID, reason: StopReason) => Effect.Effect // Whether the command's process can outlive this server, so a planned shutdown leaves it running. readonly detachable: (id: Shell.ID) => boolean - // Records in the daemon that a planned shutdown moved this running command to the background, clearing - // its timeout, so the next server neither times it out nor waits on it. Best effort; spawned commands ignore it. - readonly backgroundAtShutdown: (id: Shell.ID) => Effect.Effect // The command a tool call started, while it is still known here (including one taken back after a restart). readonly byToolCall: (call: ToolCall) => Effect.Effect readonly output: (id: Shell.ID, input?: Shell.OutputInput) => Effect.Effect @@ -230,10 +230,10 @@ interface Runs { readonly env: Readonly> readonly timeout: number readonly toolCall: ToolCall | undefined + readonly foreground: boolean }) => Effect.Effect readonly timeout: (run: Run, duration: number) => Effect.Effect readonly stop: (run: Run, reason: StopReason) => Effect.Effect - readonly backgroundAtShutdown: (run: Run) => Effect.Effect readonly remove: (id: Shell.ID) => Effect.Effect } @@ -289,7 +289,7 @@ const daemonLayer = Layer.effect( toolCall: run.toolCall, timeoutDeadline: run.timeoutDeadline, stopReason: run.stopReason, - backgroundedAtShutdown: run.backgroundedAtShutdown, + foreground: run.foreground, }) // Records the command's latest timeout and stop reason in the daemon, so a restart sees them. @@ -383,19 +383,12 @@ const daemonLayer = Layer.effect( if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber) run.timeoutFiber = undefined run.timeoutDeadline = duration === 0 ? undefined : Date.now() + duration + // Clearing the timeout is how a call moves its command to the background. + if (duration === 0) run.foreground = false yield* store(run) if (duration !== 0) yield* arm(run, duration) }) - const backgroundAtShutdown = Effect.fnUntraced(function* (run: Run) { - if (run.info.status !== "running") return - if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber) - run.timeoutFiber = undefined - run.timeoutDeadline = undefined - run.backgroundedAtShutdown = true - yield* store(run) - }) - // Watches the exit, and the timeout (in ms from now, none if undefined). const attach = Effect.fnUntraced(function* (run: Run, duration: number | undefined) { run.wait = yield* fork( @@ -442,6 +435,7 @@ const daemonLayer = Layer.effect( info: input.info, process: 0, toolCall: input.toolCall, + foreground: input.foreground, timeoutDeadline: input.timeout > 0 ? input.info.time.started + input.timeout : undefined, done: Deferred.makeUnsafe(), } @@ -515,7 +509,7 @@ const daemonLayer = Layer.effect( toolCall: stored.toolCall, timeoutDeadline: stored.timeoutDeadline, stopReason: stored.stopReason, - backgroundedAtShutdown: stored.backgroundedAtShutdown, + foreground: stored.foreground === true, ...(running ? {} : { outputLines: process.outputLines, outputError: process.outputError }), ...(process.lifecycle.status === "failed" && !stored.stopReason ? { killed: `Command was killed: ${process.lifecycle.message}` } @@ -524,8 +518,9 @@ const daemonLayer = Layer.effect( } runs.set(stored.id, run) if (running) { - // A command moved to the background at shutdown has no timeout; an expired deadline fires at once. - const deadline = stored.backgroundedAtShutdown ? undefined : stored.timeoutDeadline + // Its call died with the previous server, so taking it over moves it to the background, which clears + // its timeout. Otherwise an expired deadline fires at once. + const deadline = stored.foreground ? undefined : stored.timeoutDeadline yield* attach(run, deadline === undefined ? undefined : Math.max(0, deadline - Date.now())) continue } @@ -546,7 +541,6 @@ const daemonLayer = Layer.effect( create, timeout, stop, - backgroundAtShutdown, remove: removeRun, } return { runs: handle, recover } @@ -717,11 +711,6 @@ const layer = () => const detachable = (id: Shell.ID) => find(id)?.kind === "daemon" - const backgroundAtShutdown = Effect.fn("Shell.backgroundAtShutdown")(function* (id: Shell.ID) { - const command = find(id) - if (command?.kind === "daemon") yield* runs!.backgroundAtShutdown(command) - }) - const byToolCall = Effect.fn("Shell.byToolCall")(function* (call: ToolCall) { return runs ?.list() @@ -918,6 +907,7 @@ const layer = () => ), timeout: invocation.timeout, toolCall: input.toolCall, + foreground: input.foreground === true, }) .pipe( Effect.map(Option.some), @@ -1068,7 +1058,6 @@ const layer = () => timeout, stop, detachable, - backgroundAtShutdown, byToolCall, output, remove, diff --git a/packages/core/src/tool/plugin/shell.ts b/packages/core/src/tool/plugin/shell.ts index 9fce7a410410..906f40751875 100644 --- a/packages/core/src/tool/plugin/shell.ts +++ b/packages/core/src/tool/plugin/shell.ts @@ -248,7 +248,6 @@ export const Plugin = { effect: Effect.fn("ShellTool.Plugin")(function* (ctx: Context) { const sessions = yield* Session.Service const jobs = yield* Job.Service - const scope = yield* Scope.Scope const environment = yield* Environment.Service const access = yield* FileAccess.Service const shell = yield* Shell.Service @@ -301,15 +300,6 @@ export const Plugin = { daemon.watch(notify(sessions, jobs, { sessionID, id, shellID, command, settled })), ) - // Foreground commands this tool is blocked on. A planned shutdown moves them to the background, so it records - // that in the daemon while it still can, for a next server that finds them without a background job. - const foreground = new Set() - yield* detaching - .onDetach( - Effect.forEach(foreground, (id) => shell.backgroundAtShutdown(id), { concurrency: "unbounded", discard: true }), - ) - .pipe(Effect.provideService(Scope.Scope, scope)) - // Settles an interrupted foreground call. The interrupt only reaches the step through ToolInterrupted. const interrupted = Effect.fnUntraced(function* ( info: ShellSchema.Info, @@ -363,6 +353,7 @@ export const Plugin = { timeout, shell: yield* compatibleShell, metadata: { sessionID: context.sessionID }, + foreground: input.background !== true, toolCall: { sessionID: context.sessionID, assistantMessageID: context.messageID, @@ -398,10 +389,8 @@ export const Plugin = { return backgroundResult(info.id, info.file) } - foreground.add(info.id) const blocked = yield* restore(jobs.block({ id: job.id, sessionID: context.sessionID })).pipe( Effect.exit, - Effect.ensuring(Effect.sync(() => foreground.delete(info.id))), ) if (Exit.isFailure(blocked)) { if (Cause.hasInterrupts(blocked.cause)) diff --git a/packages/core/test/detaching.test.ts b/packages/core/test/detaching.test.ts index 9bf401d90a0d..1143668d7f7c 100644 --- a/packages/core/test/detaching.test.ts +++ b/packages/core/test/detaching.test.ts @@ -1,44 +1,18 @@ import { describe, expect } from "bun:test" -import { Effect, Exit, Fiber, Scope } from "effect" -import { TestClock } from "effect/testing" +import { Effect } from "effect" import { Detaching } from "@opencode/core/detaching" import { it } from "./lib/effect" describe("Detaching", () => { - it.live("runs its hooks only when a prepared handoff makes shutdown detach", () => + it.effect("detaches only on a shutdown after a prepared handoff", () => Effect.gen(function* () { const quit = yield* Detaching.make - let ran = 0 - yield* quit.onDetach(Effect.sync(() => ran++)) expect(yield* quit.begin).toBe(false) - expect(ran).toBe(0) - const detaching = yield* Detaching.make - yield* detaching.onDetach(Effect.sync(() => ran++)) - yield* detaching.onDetach(Effect.die("a failing hook does not stop the others")) - yield* detaching.onDetach(Effect.sync(() => ran++)) yield* detaching.handoffPrepared + expect(detaching.isDetaching()).toBe(false) expect(yield* detaching.begin).toBe(true) expect(detaching.isDetaching()).toBe(true) - expect(ran).toBe(2) - }), - ) - - it.effect("bounds slow hooks and forgets hooks whose scope closed", () => - Effect.gen(function* () { - const detaching = yield* Detaching.make - let ran = 0 - const scope = yield* Scope.make() - yield* detaching.onDetach(Effect.sync(() => ran++)).pipe(Effect.provideService(Scope.Scope, scope)) - yield* Scope.close(scope, Exit.void) - yield* detaching.onDetach(Effect.never) - yield* detaching.handoffPrepared - const begin = yield* detaching.begin.pipe(Effect.forkChild({ startImmediately: true })) - yield* TestClock.adjust("499 millis") - expect(begin.pollUnsafe()).toBeUndefined() - yield* TestClock.adjust("1 millis") - expect(yield* Fiber.join(begin)).toBe(true) - expect(ran).toBe(0) }), ) diff --git a/packages/core/test/fixture/shell-daemon.ts b/packages/core/test/fixture/shell-daemon.ts index f874a7c071bc..fabb399779bb 100644 --- a/packages/core/test/fixture/shell-daemon.ts +++ b/packages/core/test/fixture/shell-daemon.ts @@ -8,7 +8,6 @@ import { Location } from "@opencode/core/location" import { PersistentPty } from "@opencode/core/persistent-pty" import { AbsolutePath } from "@opencode/core/schema" import { Shell } from "@opencode/core/shell" -import { Detaching } from "@opencode/core/detaching" import { Bus } from "@opencode/core/bus" import { LayerNode } from "@opencode/util/effect/layer-node" import { LocationLifecycle } from "@opencode/core/location-lifecycle" @@ -42,7 +41,7 @@ export const daemonShells = Effect.fn(function* () { ).pipe(Layer.provide(hostEnvironmentLayer)) // `server` shares one server's daemon commands between Locations; without it the Location gets a server of its own. const shellLayer = (directory: string, server?: Shell.DaemonInterface) => - AppNodeBuilder.build(LayerNode.group([Shell.node, LocationLifecycle.node, Bus.node, Detaching.node]), [ + AppNodeBuilder.build(LayerNode.group([Shell.node, LocationLifecycle.node, Bus.node]), [ ...(server ? [Shell.daemonNode.replace(Layer.succeed(Shell.Daemon, server))] : []), Location.node.replace( Layer.succeed( @@ -82,7 +81,7 @@ export const daemonShells = Effect.fn(function* () { ), } }) - // Runs `body` against the Location's Shell, then closes it as the server shutting down for a restart does. + // Runs `body` against the Location's Shell on a server of its own, then stops that server, as a restart does. const withShell = ( directory: string, body: (shell: Shell.Interface) => Effect.Effect, @@ -90,10 +89,7 @@ export const daemonShells = Effect.fn(function* () { Effect.scoped( Effect.gen(function* () { const shell = yield* Shell.Service - const detaching = yield* Detaching.Service - const result = yield* body(shell) - yield* detaching.begin - return result + return yield* body(shell) }).pipe(Effect.provide(shellLayer(directory))), ) const directory = (yield* tmpdirScoped()).path diff --git a/packages/core/test/session-execution.test.ts b/packages/core/test/session-execution.test.ts index 8a85255a2d5d..823f95fe7a94 100644 --- a/packages/core/test/session-execution.test.ts +++ b/packages/core/test/session-execution.test.ts @@ -566,9 +566,13 @@ describe("SessionRestart background recovery", () => { command: "sleep 60", cwd: "/tmp", timeout: 120_000, + foreground: true, toolCall: { sessionID: parent, assistantMessageID, callID: "call-shell" }, }), ) + // Its deadline passed while no server ran: taking it over must not time it out before settling the call. + const pipe = daemon.daemon.processes[0]?.pipe + if (pipe) pipe.metadata = { ...pipe.metadata, timeoutDeadline: Date.now() - 1 } const scope = yield* Scope.make() yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) @@ -602,6 +606,8 @@ describe("SessionRestart background recovery", () => { ]) expect((yield* restarted.get(info.id))?.status).toBe("running") expect(daemon.daemon.processes[0]?.pipe?.metadata).not.toHaveProperty("timeoutDeadline") + expect(daemon.daemon.processes[0]?.pipe?.metadata).toMatchObject({ foreground: false }) + expect(daemon.signals(1)).toEqual([]) daemon.daemon.exit(1, 0) const notices = Effect.gen(function* () { @@ -632,6 +638,7 @@ describe("SessionRestart background recovery", () => { shell: "/bin/sh", command: "work", cwd: "/tmp", + foreground: true, toolCall: { sessionID: parent, assistantMessageID, callID: "call-shell" }, }), ) @@ -705,6 +712,7 @@ describe("SessionRestart background recovery", () => { shell: "/bin/sh", command: "child work", cwd: "/tmp", + foreground: true, toolCall: { sessionID: child, assistantMessageID, callID: "call-child-shell" }, }), ) diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts index ed77de8434e2..d087f4de8339 100644 --- a/packages/core/test/shell-daemon.test.ts +++ b/packages/core/test/shell-daemon.test.ts @@ -347,25 +347,41 @@ describe("Shell on the process daemon", () => { }), ) - timed("records a command moved to the background at shutdown so the next server never times it out", () => + timed("a server that takes over a foreground command times out only what moved to the background", () => Effect.gen(function* () { const env = yield* daemonShells() - const info = yield* env.withShell(env.directory, (shell) => + // The old server dies without a shutdown: one call still waits on its command, another backgrounded its own. + const before = yield* env.withShell(env.directory, (shell) => Effect.gen(function* () { - const info = yield* shell.create({ shell: "/bin/sh", command: "sleep 60", timeout: 100 }) - yield* shell.backgroundAtShutdown(info.id) - return info + const waited = yield* shell.create({ shell: "/bin/sh", command: "sleep 60", timeout: 100, foreground: true }) + const backgrounded = yield* shell.create({ + shell: "/bin/sh", + command: "sleep 60", + timeout: 100, + foreground: true, + }) + yield* shell.timeout(backgrounded.id, 30_000) + yield* shell.timeout(backgrounded.id, 0) + yield* shell.timeout(backgrounded.id, 100) + return { waited, backgrounded } }), ) - expect(env.daemon.processes[0]?.pipe?.metadata).toMatchObject({ backgroundedAtShutdown: true }) - expect(env.daemon.processes[0]?.pipe?.metadata).not.toHaveProperty("timeoutDeadline") + expect(env.daemon.processes[0]?.pipe?.metadata).toMatchObject({ + foreground: true, + timeoutDeadline: expect.any(Number), + }) + expect(env.daemon.processes[1]?.pipe?.metadata).toMatchObject({ foreground: false }) + // Both deadlines pass while no server runs. + for (const process of env.daemon.processes) + if (process.pipe) process.pipe.metadata = { ...process.pipe.metadata, timeoutDeadline: Date.now() - 1 } yield* env.withShell(env.directory, (shell) => Effect.gen(function* () { - yield* TestClock.adjust("1 second") - expect((yield* shell.get(info.id)).status).toBe("running") + yield* until(() => env.signals(2).length > 0) + expect((yield* shell.wait(before.backgrounded.id)).status).toBe("timeout") + // The restart moves the waited-on command to the background, so its old timeout no longer applies. + yield* TestClock.adjust("1 hour") + expect((yield* shell.get(before.waited.id)).status).toBe("running") expect(env.signals(1)).toEqual([]) - // No timer was armed, so none consulted the daemon. - expect(env.daemon.ops).not.toContain("get") }), ) }), diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index ca7dccdaecfa..0100788b5a9e 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -1925,7 +1925,8 @@ describe("ShellTool", () => { expect(result?.metadata).toMatchObject({ shellID, status: "running" }) const process = fake?.processes.find((item) => item.pipe?.output_path === info.file) expect(process?.lifecycle.status).toBe("running") - expect(process?.pipe?.metadata).toMatchObject({ id: shellID, backgroundedAtShutdown: true }) + // Moved to the background: a next server neither waits on it nor times it out. + expect(process?.pipe?.metadata).toMatchObject({ id: shellID, foreground: false }) expect(process?.pipe?.metadata).not.toHaveProperty("timeoutDeadline") expect(process?.pipe?.metadata).not.toHaveProperty("stopReason") expect(fake?.ops).not.toContain("signal") From 703d6279db69e54455d65fcd3163349162dc278f Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 20:20:03 +0000 Subject: [PATCH 27/29] fix(core): settle an interrupted shell call without waiting for the stop --- packages/core/src/shell.ts | 21 ++++++++++++++--- packages/core/src/shell/result.ts | 4 +++- packages/core/src/tool/plugin/shell.ts | 5 ++++- packages/core/test/tool-shell.test.ts | 31 ++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 5 deletions(-) diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index 213c9e8e3e81..be20ae8b8ae4 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -152,7 +152,11 @@ export interface Interface { // NotFoundError if the command is unknown or is removed before it terminates. readonly wait: (id: Shell.ID) => Effect.Effect // A known shell's terminal state and bounded tail. Missing capture remains distinct from its exit status. - readonly result: (started: Shell.Info) => Effect.Effect + // With `wait`, gives up waiting for the exit after that long and returns the output so far, still running. + readonly result: ( + started: Shell.Info, + options?: { readonly wait?: Duration.Input }, + ) => Effect.Effect // Replaces the running command's timeout from now; zero clears it. Ignored once the command has // ended or timed out, even while its process is still being killed or its output captured. readonly timeout: (id: Shell.ID, duration: number) => Effect.Effect @@ -826,8 +830,19 @@ const layer = () => } }) - const result = Effect.fn("Shell.result")(function* (started: Shell.Info) { - const info = yield* wait(started.id).pipe( + const result = Effect.fn("Shell.result")(function* ( + started: Shell.Info, + options?: { readonly wait?: Duration.Input }, + ) { + const ended = wait(started.id).pipe(Effect.map(Option.some)) + const info = yield* ( + options?.wait === undefined + ? ended + : ended.pipe( + Effect.timeoutOrElse({ duration: options.wait, orElse: () => Effect.succeed(Option.none()) }), + ) + ).pipe( + Effect.flatMap((info) => (Option.isSome(info) ? Effect.succeed(info.value) : get(started.id))), Effect.catchTag("Shell.NotFoundError", () => Effect.succeed({ ...started, status: "killed" as const, time: { ...started.time, completed: Date.now() } }), ), diff --git a/packages/core/src/shell/result.ts b/packages/core/src/shell/result.ts index cfa4baf30a5d..484f81297c19 100644 --- a/packages/core/src/shell/result.ts +++ b/packages/core/src/shell/result.ts @@ -60,7 +60,9 @@ export function interrupted(result: Result): Output { ...captured, output: [ lines ? `${text}${savedNotice(lines, result.info.file)}` : captured.output, - `Command interrupted by the user (shell ID: ${result.info.id}).`, + result.info.status === "running" + ? `Command interrupted by the user (shell ID: ${result.info.id}); it is still being stopped.` + : `Command interrupted by the user (shell ID: ${result.info.id}).`, ].join("\n\n"), } } diff --git a/packages/core/src/tool/plugin/shell.ts b/packages/core/src/tool/plugin/shell.ts index 906f40751875..2385bc668044 100644 --- a/packages/core/src/tool/plugin/shell.ts +++ b/packages/core/src/tool/plugin/shell.ts @@ -25,6 +25,8 @@ import { ToolInterrupted } from "../interrupted.js" export const name = "shell" export const DEFAULT_TIMEOUT_MS = 2 * 60 * 1_000 +// How long an interrupted call waits for its stopped command to exit before settling with the output so far. +const INTERRUPT_WAIT = "500 millis" const BACKGROUND_INSTRUCTION = "You will be notified automatically when the command finishes. The notification will include the command's output. Unless the user explicitly asks otherwise, DO NOT poll for completion, even if you need the final result to continue. Repeatedly sleeping and reading or searching the output file is polling, not useful work. You may read the current output if it lets you do useful work now, but do not repeatedly check it while waiting for the command to finish. Keep working on anything that does not depend on the result. If you have nothing else to do, end your response; you will be resumed automatically when the command finishes." @@ -317,7 +319,8 @@ export const Plugin = { // Settled as interrupted by the step, as before; the command ends with the job's scope. if (closing()) return yield* Effect.failCause(cause) yield* shell.stop(info.id, "killed").pipe(Effect.ignore) - const result = yield* shell.result(info) + // The stop escalates to SIGKILL on its own; the session does not wait for that to settle the call. + const result = yield* shell.result(info, { wait: INTERRUPT_WAIT }) // The stopped job settles from the same exit; this only releases it. yield* jobs.cancel(jobID) // No `shellID`: clients read it as a background call and show the shell's live output instead of this result. diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index 0100788b5a9e..ad1e4d353d3f 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -1899,6 +1899,37 @@ describe("ShellTool", () => { }), ) + parity.live( + "settles an interrupted call without waiting for a command that ignores SIGTERM", + () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const shell = yield* Shell.Service + const started = Date.now() + const { shellID, result } = yield* interruptCall( + registry, + "trap '' TERM; printf 'kept\\n'; while :; do sleep 1; done", + "call-interrupted-stubborn", + ) + // Bounded by the interrupt's short wait, not by the 3 s grace before SIGKILL. + expect(Date.now() - started).toBeLessThan(2_500) + const info = yield* shell.get(shellID) + expect(result?.content).toEqual([ + { + type: "text", + text: `kept\n\n[showing lines 1-1 of 1; full output saved to ${info.file}]\n\nCommand interrupted by the user (shell ID: ${shellID}); it is still being stopped.`, + }, + ]) + // The stop still ends it. + expect((yield* shell.wait(shellID)).status).toBe("killed") + }), + ) + }), + { timeout: 15_000 }, + ) + fakeDaemonIt.live("a prepared handoff moves a daemon-run foreground call to the background at shutdown", () => withTempDir((tmp) => { reset() From 9c5e408e824c81cefda50bcceae80ab420435e68 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 20:20:28 +0000 Subject: [PATCH 28/29] fix(core): stop what a failed daemon shell command left in its group --- packages/core/src/shell.ts | 10 +++++++++- packages/core/test/shell-daemon.test.ts | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index be20ae8b8ae4..f3e8b0e032b6 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -316,7 +316,8 @@ const daemonLayer = Layer.effect( if (run.stopping) yield* Fiber.interrupt(run.stopping) // Stop waiting first, so forgetting the process is not mistaken for losing it. if (run.wait) yield* Fiber.interrupt(run.wait) - // Forget the process before deleting its file. + // Forget the process before deleting its file. This also kills what is left of its group, even after a + // successful exit, which a spawned command would leave running: the daemon cannot forget a process otherwise. yield* pty.terminatePipe(run.process).pipe(Effect.ignore) yield* Effect.promise(() => unlink(run.info.file).catch(() => {})) yield* Deferred.fail(run.done, new NotFoundError({ id })) @@ -420,6 +421,13 @@ const daemonLayer = Layer.effect( ), ), ), + // Like spawned commands: what a failed command left running in its group is stopped with it, while a + // successful one may leave background processes (`cmd &`) behind. + Effect.tap((outcome) => + outcome.status === "exited" && outcome.exit !== undefined && outcome.exit !== 0 + ? pty.signalPipe(run.process, "SIGTERM").pipe(Effect.ignore) + : Effect.void, + ), Effect.flatMap((outcome) => finish(run, outcome)), ), ) diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts index d087f4de8339..d18f5b897b06 100644 --- a/packages/core/test/shell-daemon.test.ts +++ b/packages/core/test/shell-daemon.test.ts @@ -157,6 +157,24 @@ describe("Shell on the process daemon", () => { ) })) + test("stops what a failed command left in its group, and leaves a successful one's", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const succeeded = yield* shell.create({ shell: "/bin/sh", command: "serve & exit 0" }) + const failed = yield* shell.create({ shell: "/bin/sh", command: "serve & exit 3" }) + yield* until(() => env.daemon.waiters(1) > 0 && env.daemon.waiters(2) > 0) + env.daemon.exit(1, 0) + env.daemon.exit(2, 3) + expect(yield* shell.wait(failed.id)).toMatchObject({ status: "exited", exit: 3 }) + expect(yield* shell.wait(succeeded.id)).toMatchObject({ status: "exited", exit: 0 }) + expect(env.signals(2)).toEqual(["SIGTERM"]) + expect(env.signals(1)).toEqual([]) + }), + ) + })) + test("fails a command the daemon rejected instead of spawning it", () => Effect.gen(function* () { const env = yield* daemonShells() From 744907b928f4f03888724338b7657e544da5a221 Mon Sep 17 00:00:00 2001 From: James Long Date: Thu, 5 Nov 2026 20:21:22 +0000 Subject: [PATCH 29/29] fix(core): forget stale finished daemon processes in the shell sweep --- packages/core/src/shell.ts | 13 +++++++++++++ packages/core/test/shell-daemon.test.ts | 16 ++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index f3e8b0e032b6..2844a7f05a67 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -211,6 +211,19 @@ export const cleanup = Effect.fn("Shell.cleanup")(function* () { return process ? pty.terminatePipe(process.id).pipe(Effect.ignore) : Effect.void }, ) + // Finished processes whose file is gone, or that ended long ago, are of no use to any Location: forget them. + const cutoff = Date.now() - Duration.toMillis(RETENTION) + yield* Effect.forEach( + processes.values(), + (process) => + Effect.gen(function* () { + if (process.lifecycle.status === "running") return + const stale = (process.exitedAt ?? process.startedAt) < cutoff + if (!stale && (yield* fs.exists(process.outputPath).pipe(Effect.orElseSucceed(() => true)))) return + yield* pty.terminatePipe(process.id).pipe(Effect.ignore) + }), + { concurrency: 8, discard: true }, + ) }) const cleanupLayer = Layer.effectDiscard( diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts index d18f5b897b06..35b6d32a9428 100644 --- a/packages/core/test/shell-daemon.test.ts +++ b/packages/core/test/shell-daemon.test.ts @@ -1,6 +1,6 @@ import { describe, expect } from "bun:test" import { existsSync, readdirSync, realpathSync } from "node:fs" -import { mkdir, utimes, writeFile } from "node:fs/promises" +import { mkdir, rm, utimes, writeFile } from "node:fs/promises" import path from "node:path" import { Effect, Layer } from "effect" import { TestClock } from "effect/testing" @@ -414,9 +414,19 @@ describe("Shell on the process daemon", () => { const finished = yield* shell.create({ shell: "/bin/sh", command: "finished" }) env.daemon.exit(2, 0) yield* shell.wait(finished.id) - return { running, finished } + const lostFile = yield* shell.create({ shell: "/bin/sh", command: "lost file" }) + env.daemon.exit(3, 0) + yield* shell.wait(lostFile.id) + const old = yield* shell.create({ shell: "/bin/sh", command: "old" }) + env.daemon.exit(4, 0) + yield* shell.wait(old.id) + return { running, finished, lostFile, old } }), ) + // A finished process whose file was deleted elsewhere, and one that ended more than 7 days ago. + yield* Effect.promise(() => rm(ids.lostFile.file)) + const ended = env.daemon.processes.find((process) => process.id === 4) + if (ended) ended.exited_at = Date.now() - 8 * 24 * 60 * 60 * 1_000 const expired = new Date(Date.now() - 8 * 24 * 60 * 60 * 1_000) yield* Effect.promise(() => utimes(ids.running.file, expired, expired)) yield* Effect.promise(() => utimes(ids.finished.file, expired, expired)) @@ -437,6 +447,8 @@ describe("Shell on the process daemon", () => { expect(existsSync(ids.finished.file)).toBe(false) expect(existsSync(orphan)).toBe(false) expect(env.daemon.processes.map((process) => process.id)).toEqual([1]) + // The old process's file is recent, so it stays. + expect(existsSync(ids.old.file)).toBe(true) })) })