diff --git a/packages/cli/src/commands/handlers/service/restart.ts b/packages/cli/src/commands/handlers/service/restart.ts index afc7bb457c17..fbd2c3b98269 100644 --- a/packages/cli/src/commands/handlers/service/restart.ts +++ b/packages/cli/src/commands/handlers/service/restart.ts @@ -4,14 +4,12 @@ import { Service } from "@opencode/client/effect/service" import { Commands } from "../../commands" import { Runtime } from "../../../framework/runtime" import { ServiceConfig } from "../../../services/service-config" -import { ServerConnection } from "../../../services/server-connection" export default Runtime.handler( Commands.commands.service.commands.restart, Effect.fn("cli.service.restart")(function* () { const options = yield* ServiceConfig.options() - // Keep this explicit: automatic service replacement must preserve terminals. - yield* ServerConnection.shutdownPersistentPty(options).pipe(Effect.ignore) + // No explicit daemon shutdown: stopping without a handoff drops the daemon's owner, which stops it with its processes. yield* Service.stop(options) const transport = yield* Service.ensure(options) process.stdout.write(transport.url + EOL) diff --git a/packages/cli/src/commands/handlers/service/stop.ts b/packages/cli/src/commands/handlers/service/stop.ts index d29f33275b44..c0dd64ad4187 100644 --- a/packages/cli/src/commands/handlers/service/stop.ts +++ b/packages/cli/src/commands/handlers/service/stop.ts @@ -3,13 +3,12 @@ import { Service } from "@opencode/client/effect/service" import { Commands } from "../../commands" import { Runtime } from "../../../framework/runtime" import { ServiceConfig } from "../../../services/service-config" -import { ServerConnection } from "../../../services/server-connection" export default Runtime.handler( Commands.commands.service.commands.stop, Effect.fn("cli.service.stop")(function* () { const options = yield* ServiceConfig.options() - yield* ServerConnection.shutdownPersistentPty(options).pipe(Effect.ignore) + // No explicit daemon shutdown: stopping without a handoff drops the daemon's owner, which stops it with its processes. yield* Service.stop(options) }), ) diff --git a/packages/core/src/job.ts b/packages/core/src/job.ts index fd5d391b396b..ec26953c7e68 100644 --- a/packages/core/src/job.ts +++ b/packages/core/src/job.ts @@ -49,6 +49,7 @@ type Active = { isBackgrounded: boolean consumed: boolean recovery?: Recovery + cancel?: Effect.Effect } type State = { @@ -59,6 +60,7 @@ type State = { type FinishResult = { info?: Info done?: Deferred.Deferred + cancel?: Effect.Effect scope?: Scope.Closeable generation?: Scope.Closeable } @@ -90,6 +92,9 @@ export type StartInput = { recovery?: Recovery notificationID?: SessionMessage.ID run: Effect.Effect + // Runs when the job is cancelled (`cancel`, `cancelSession`), before its work is interrupted. Teardown, such as + // a server shutdown, interrupts the work without it. + cancel?: Effect.Effect } export type WaitInput = { @@ -248,6 +253,11 @@ export const make = Effect.gen(function* () { if (!job) return [{}, jobs] if (job.scope !== scope) return [{}, jobs] if (job.info.status !== "running") return [{ info: snapshot(job) }, jobs] + // Teardown (this registry closing) is not a terminal outcome: leave the job and its durable row running for + // the next server. The scope's own finalizer interrupts its observers. A cancel has already settled the job + // as cancelled before interrupting its work. + if (Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause) && scope.state._tag === "Closed") + return [{}, jobs] const status: Exclude = Exit.isSuccess(exit) ? "completed" : Cause.hasInterruptsOnly(exit.cause) @@ -295,6 +305,8 @@ export const make = Effect.gen(function* () { return [{ info: snapshot(existing) }, jobs] } const scope = yield* Scope.fork(state.scope, "parallel") + // Closing the registry interrupts the work without settling it; interrupt its observers too. + yield* Scope.addFinalizer(scope, Deferred.interrupt(done)) const job = { info: { id: input.id, @@ -312,6 +324,7 @@ export const make = Effect.gen(function* () { isBackgrounded: false, consumed: false, recovery: input.recovery, + cancel: input.cancel, } return [{ info: snapshot(job), scope }, new Map(jobs).set(input.id, job)] }), @@ -459,11 +472,12 @@ export const make = Effect.gen(function* () { } if (persist) yield* persistBackground(next) return [ - { info: snapshot(next), done: job.done, scope: job.scope, generation: job.scope }, + { info: snapshot(next), done: job.done, scope: job.scope, generation: job.scope, cancel: job.cancel }, new Map(jobs).set(id, next), ] }), ) + if (result.cancel) yield* result.cancel if (result.info && result.done) yield* Deferred.succeed(result.done, result.info) if (result.scope) yield* Scope.close(result.scope, Exit.void) if (result.generation) yield* consume(id, result.generation) diff --git a/packages/core/src/persistent-pty/index.ts b/packages/core/src/persistent-pty/index.ts index 99c99e4f3c75..ac3db0abeb61 100644 --- a/packages/core/src/persistent-pty/index.ts +++ b/packages/core/src/persistent-pty/index.ts @@ -171,6 +171,11 @@ export interface Interface { readonly remove: (id: Pty.ID) => Effect.Effect readonly shutdown: () => Effect.Effect readonly handoff: () => Effect.Effect + /** + * Whether a handoff to the next server is prepared and its ticket still valid: this server is being replaced, so + * the daemon's processes should be left running for the next one. A shutdown clears it. + */ + readonly handoffPrepared: () => boolean readonly attach: (id: Pty.ID, input: SubscribeInput) => Effect.Effect /** Starts the daemon if needed. Fall back to another spawner only on `UnavailableError`. */ readonly createPipe: (input: CreatePipeInput) => Effect.Effect @@ -361,14 +366,22 @@ const makeLayer = (options: Options = {}) => return undefined }) + // When the prepared handoff's ticket expires; 0 when none is prepared. + let handoffExpiresAt = 0 + const shutdown = Effect.fn("PersistentPty.shutdown")(function* () { + handoffExpiresAt = 0 const response = yield* daemon.shutdown.pipe(Effect.mapError(unavailable)) current.clear() if (!response) return if (response.type !== "ok") return yield* unexpected(response) }) - const handoff = Effect.fn("PersistentPty.handoff")(() => daemon.handoff.pipe(Effect.mapError(unavailable))) + const handoff = Effect.fn("PersistentPty.handoff")(function* () { + const prepared = yield* daemon.handoff.pipe(Effect.mapError(unavailable)) + if (prepared) handoffExpiresAt = prepared.expiresAt + return prepared + }) const removeVisibleExit = (id: Pty.ID) => { if (removing.has(id)) return @@ -512,6 +525,7 @@ const makeLayer = (options: Options = {}) => remove, shutdown, handoff, + handoffPrepared: () => Date.now() < handoffExpiresAt, attach, createPipe, waitPipe, diff --git a/packages/core/src/plugin/internal.ts b/packages/core/src/plugin/internal.ts index 180aa4661a81..91ba0645a412 100644 --- a/packages/core/src/plugin/internal.ts +++ b/packages/core/src/plugin/internal.ts @@ -208,6 +208,7 @@ export const requirements = LayerNode.group([ InstructionDiscovery.node, Integration.node, Job.node, + Shell.daemonNode, KV.node, llmClient, Location.node, diff --git a/packages/core/src/session/execution/restart.ts b/packages/core/src/session/execution/restart.ts index 3bf4bae98888..503957b99a27 100644 --- a/packages/core/src/session/execution/restart.ts +++ b/packages/core/src/session/execution/restart.ts @@ -1,12 +1,20 @@ export * as SessionRestart from "./restart.js" -import { Context, Effect, Layer } from "effect" +import { Context, Effect, Layer, Option, Schema } from "effect" +import { isReadonlyArrayNonEmpty } from "effect/Array" import { makeGlobalNode } from "@opencode/util/effect/app-node" +import { Shell as ShellSchema } from "@opencode/schema/shell" import { Bus } from "../../bus.js" +import { Database } from "../../database/database.js" +import { Instance } from "../../instance/service.js" import { Job } from "../../job.js" import { Session } from "../../session.js" +import { Shell } from "../../shell.js" +import { ShellTool } from "../../tool/plugin/shell.js" import { SessionEvent } from "../event.js" import { SessionExecution } from "../execution.js" +import { SessionHistory } from "../history.js" +import { SessionMessage } from "../message.js" import { SessionSchema } from "../schema.js" import { SessionStore } from "../store.js" import { ImageNotice } from "../../image/notice.js" @@ -72,6 +80,8 @@ export const layer = (options?: Options) => const bus = yield* Bus.Service const jobs = yield* Job.Service const sessions = yield* Session.Service + const instances = yield* Instance.Service + const db = (yield* Database.Service).db const scope = yield* Effect.scope const maxAttempts = options?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS @@ -99,10 +109,7 @@ export const layer = (options?: Options) => return true }) - const restartMessage = ( - background: Job.Background, - recovery: Exclude, - ) => { + const restartMessage = (background: Job.Background, recovery: Exclude) => { const state = background.status === "running" ? "cancelled" : background.status if (recovery.kind === "shell") { const text = @@ -164,6 +171,100 @@ export const layer = (options?: Options) => yield* jobs.completeBackground(background.notificationID) }) + // The Shell of a session's Location, booted so it takes back the commands the daemon still runs for it. + const shellOf = Effect.fnUntraced(function* (sessionID: SessionSchema.ID) { + const session = yield* store.get(sessionID) + if (!session) return undefined + return yield* Effect.serviceOption(Shell.Service).pipe( + instances.provide(session), + Effect.map(Option.getOrUndefined), + // The session's directory is gone: its commands are reported like any lost background work. + Effect.catchTag("FileSystem.DirectoryNotFoundError", () => Effect.succeed(undefined)), + ) + }) + + const reattach = ( + shell: Shell.Interface, + sessionID: SessionSchema.ID, + info: ShellSchema.Info, + suspended: ReadonlySet, + notificationID?: SessionMessage.ID, + ) => + ShellTool.reattach({ + shell, + jobs, + sessions, + sessionID, + info, + notificationID, + resume: () => (suspended.has(sessionID) ? false : undefined), + }) + + // A shell job the previous server left running is watched again if its command is still known (running or + // finished during the restart); otherwise it is reported as lost. + const recoverShellJob = Effect.fnUntraced(function* ( + background: Job.Background, + recovery: Extract, + suspended: ReadonlySet, + reattached: Set, + ) { + const shell = background.status === "running" ? yield* shellOf(recovery.sessionID) : undefined + const info = + shell && Schema.is(ShellSchema.ID)(recovery.shellID) + ? yield* shell.get(recovery.shellID).pipe(Effect.option) + : Option.none() + if (!shell || Option.isNone(info)) return yield* recoverToolJob(background, recovery) + reattached.add(info.value.id) + yield* reattach(shell, recovery.sessionID, info.value, suspended, background.notificationID) + }) + + // A server that stopped (shut down or killed) during a foreground shell call leaves the call unsettled, its + // command still running in the daemon. Settle the call as moved to the background and watch the command. + const settleBackgroundedCalls = Effect.fnUntraced(function* ( + sessionID: SessionSchema.ID, + suspended: ReadonlySet, + reattached: ReadonlySet, + ) { + const calls = (yield* SessionHistory.incompleteAssistants(db, sessionID)).flatMap((message) => + message.type === "assistant" + ? message.content.flatMap((part) => + part.type === "tool" && part.name === ShellTool.name && part.state.status === "running" + ? [{ assistantMessageID: message.id, part }] + : [], + ) + : [], + ) + if (calls.length === 0) return + const shell = yield* shellOf(sessionID) + if (!shell) return + yield* Effect.forEach( + calls, + Effect.fnUntraced(function* (call) { + const info = yield* shell.byToolCall({ + sessionID, + assistantMessageID: call.assistantMessageID, + callID: call.part.id, + }) + if (!info) return + const result = ShellTool.backgroundToolResult(info) + if (!isReadonlyArrayNonEmpty(result.content)) return + yield* bus.publish(SessionEvent.Tool.Success, { + sessionID, + assistantMessageID: call.assistantMessageID, + id: call.part.id, + content: result.content, + metadata: result.metadata, + executed: call.part.executed === true, + }) + // Backgrounded but stopped before the call was settled: its job row was already reattached (and, for a + // command that already finished, may even be done). + if (reattached.has(info.id)) return + yield* reattach(shell, sessionID, info, suspended) + }), + { discard: true }, + ) + }) + const recoverSubagent = Effect.fnUntraced(function* ( background: Job.Background, recovery: Extract, @@ -232,13 +333,37 @@ export const layer = (options?: Options) => [...(yield* store.listSuspended()), ...children].filter((sessionID) => !active.has(sessionID)), ) yield* store.releaseChildClaims(children) + const recoverable = Effect.fnUntraced(function* (background: Job.Background) { + return (yield* jobs.get(background.id))?.status !== "running" + }) + // Shell and tool outcomes first, then calls a killed server left unsettled, and only then recovered children, + // whose first resume would otherwise fail their own unsettled shell calls. + const reattached = new Set() + yield* Effect.forEach( + pending, + Effect.fnUntraced(function* (background) { + const recovery = background.recovery + if (recovery.kind === "subagent" || !(yield* recoverable(background))) return + if (recovery.kind === "shell") return yield* recoverShellJob(background, recovery, suspended, reattached) + yield* recoverToolJob(background, recovery) + }), + { discard: true }, + ) + yield* Effect.forEach( + suspended, + (sessionID) => + settleBackgroundedCalls(sessionID, suspended, reattached).pipe( + Effect.catch((error) => + Effect.logWarning("failed to settle backgrounded shell calls", { sessionID, error }), + ), + ), + { discard: true }, + ) yield* Effect.forEach( - // Admit shell outcomes before a recovered child can start its first model request. - pending.toSorted((a, b) => Number(a.recovery.kind === "subagent") - Number(b.recovery.kind === "subagent")), + pending, Effect.fnUntraced(function* (background) { - if ((yield* jobs.get(background.id))?.status === "running") return const recovery = background.recovery - if (recovery.kind !== "subagent") return yield* recoverToolJob(background, recovery) + if (recovery.kind !== "subagent" || !(yield* recoverable(background))) return yield* recoverSubagent(background, recovery, suspended) }), { discard: true }, @@ -264,5 +389,14 @@ export const layer = (options?: Options) => export const node = makeGlobalNode({ service: Service, layer: layer(), - deps: [SessionStore.node, SessionExecution.node, Bus.node, Job.node, Session.node], + deps: [ + SessionStore.node, + SessionExecution.node, + Bus.node, + Job.node, + Session.node, + Instance.node, + Database.node, + Shell.daemonNode, + ], }) diff --git a/packages/core/src/session/runner/llm.ts b/packages/core/src/session/runner/llm.ts index 637b30b76e6c..77ecc97d971f 100644 --- a/packages/core/src/session/runner/llm.ts +++ b/packages/core/src/session/runner/llm.ts @@ -29,6 +29,7 @@ import { llmClient } from "../../effect/app-node-platform.js" import { StepFailedError } from "../error.js" import { SessionRunnerRetry } from "./retry.js" import { SessionStep } from "./step.js" +import { PersistentPty } from "../../persistent-pty/index.js" import { ToolOutput } from "../../tool-output.js" import { Plugin } from "../../plugin.js" import { PluginHooks } from "../../plugin/hooks.js" @@ -54,6 +55,7 @@ const layer = Layer.effect( const jobs = yield* Job.Service const title = yield* SessionTitle.Service const steps = yield* SessionStep.make + const pty = yield* PersistentPty.Service // Title generation starts once input is visible and must not delay model execution. const titles = yield* FiberMap.make() @@ -293,6 +295,7 @@ const layer = Layer.effect( const outcome = yield* steps.attempt({ isLocationClosed: lifecycle.isClosed, interruption, + handoffPrepared: pty.handoffPrepared, sessionID, assistantMessageID, agent: loaded.agent.id, @@ -460,6 +463,7 @@ export const node = makeLocationNode({ Job.node, SessionTitle.node, Snapshot.node, + PersistentPty.node, ToolOutput.node, Database.node, ], diff --git a/packages/core/src/session/runner/step.ts b/packages/core/src/session/runner/step.ts index d8096d740fd0..da69c87ff06d 100644 --- a/packages/core/src/session/runner/step.ts +++ b/packages/core/src/session/runner/step.ts @@ -44,6 +44,8 @@ interface Input { readonly isLocationClosed: () => boolean /** Why the execution is being interrupted, once it is: `"user"` when someone stopped it. */ readonly interruption?: () => "user" | "shutdown" | undefined + /** Whether this server is handing its process daemon over to the next one (see `PersistentPty.handoffPrepared`). */ + readonly handoffPrepared?: () => boolean readonly sessionID: SessionSchema.ID readonly assistantMessageID: SessionMessage.ID readonly agent: Agent.ID @@ -182,6 +184,16 @@ export const make = Effect.gen(function* () { // Read each tool's exit once all have ended, so what they raised while stopping (a decline, a defect) counts. const exits = Exit.isSuccess(joined) ? joined.value : yield* Fiber.awaitAll(fibers) const tools = classifyToolExits(exits, toolRuns, Exit.hasInterrupts(joined)) + // A server handing its process daemon over leaves the step as a crash would: unsettled calls and no step end. + // The next server settles them when the Session resumes, and takes over the shell commands still running for + // them. Any other interrupt settles the calls here. + if ( + (streamInterrupted || tools.interrupted) && + tools.declines.length === 0 && + input.interruption?.() !== "user" && + input.handoffPrepared?.() === true + ) + return yield* Effect.interrupt if ( !publisher.record().outputStarted && @@ -363,7 +375,7 @@ type Cancel = ReturnType * A tool call's `cancel`. `trigger` completes it, and reports whether the tool was awaiting it, so would stop on its * own; a tool that was not is interrupted instead. */ -const makeCancel = () => { +export const makeCancel = () => { let listeners = 0 let cancelled = false const gate = Deferred.makeUnsafe() diff --git a/packages/core/src/shell.ts b/packages/core/src/shell.ts index 054237074457..8359d0628ec1 100644 --- a/packages/core/src/shell.ts +++ b/packages/core/src/shell.ts @@ -1,7 +1,20 @@ export * as Shell from "./shell.js" import path from "path" -import { Context, Deferred, Duration, Effect, Fiber, Latch, Layer, Option, Schema, Schedule, Stream } from "effect" +import { + Context, + Deferred, + Duration, + Effect, + Exit, + Fiber, + Latch, + Layer, + Option, + Schema, + Schedule, + Stream, +} from "effect" import { ChildProcess } from "effect/process" import { produce } from "immer" import { Shell } from "@opencode/schema/shell" @@ -14,6 +27,7 @@ import { Environment } from "./environment/index.js" import { FileRetention } from "./file-retention.js" import { Location } from "./location.js" import { Global } from "@opencode/util/global" +import { PersistentPty } from "./persistent-pty.js" import { ShellSelect } from "./shell/select.js" import type { ShellCreateBefore } from "@opencode/plugin/effect/shell" import { PluginHooks } from "./plugin/hooks.js" @@ -33,17 +47,49 @@ const EXITED_LIMIT = 25 export const RETENTION = Duration.days(7) export const DIRECTORY = "shell" const SPAN_COMMAND_LIMIT = 200 +const STOP_GRACE = Duration.seconds(3) +const LOST = "Command was killed: the process daemon stopped" +// Bounds the reads a result needs when the daemon did not report the line count, so stopping a call stays quick. +const COUNT_NEWLINES_LIMIT = 16 * 1024 * 1024 +const LINE_START_SCAN_LIMIT = 1024 * 1024 type Info = Shell.Info export type StopReason = "timeout" | "killed" +/** The tool call that started a command, so a server that took over can settle it. */ +export const ToolCall = Schema.Struct({ + sessionID: Schema.String, + assistantMessageID: Schema.String, + callID: Schema.String, +}) +export type ToolCall = typeof ToolCall.Type + type CreateInput = Shell.CreateInput & { shell?: string + toolCall?: ToolCall + // A call waits on the command. A server that takes the command over moves it to the background, which clears + // its timeout, so its deadline is not kept for a restart. + waited?: boolean } -type Active = { +// What a server needs to take a daemon-run command back after a restart, kept in the daemon's metadata. +const Stored = Schema.Struct({ + id: Shell.ID, + command: Schema.String, + shell: Schema.String, + location: Schema.Struct({ directory: Schema.String, workspaceID: Schema.optionalKey(Schema.String) }), + metadata: Shell.Metadata, + toolCall: Schema.optional(ToolCall), + timeoutDeadline: Schema.optional(Schema.Number), + stopReason: Schema.optional(Schema.Literals(["timeout", "killed"])), +}) +type Stored = typeof Stored.Type +const decodeStored = Schema.decodeUnknownOption(Stored) + +// A command this server spawned itself. It dies with its Location. +type Spawned = { + kind: "spawned" // Immutable snapshot; lifecycle updates replace it via immer `produce`. info: Info - file: string size: number newlines: number // Byte offsets where the last two lines start, so a truncation notice can size a cut-off line. @@ -56,6 +102,29 @@ type Active = { stop?: (reason: StopReason) => Effect.Effect } +// A command the process daemon runs. The daemon writes and counts its output file. It belongs to the server, not +// to the Location that started it, so a Location closing or reloading never stops or forgets it. +type Run = { + kind: "daemon" + info: Info + process: number + toolCall?: ToolCall + // When a restart should stop it. Unset while a call waits on it: that call's timeout dies with this server. + timeoutDeadline?: number + stopReason?: StopReason + // From the exit: newlines in the file, and why the daemon stopped writing it early. + outputLines?: number + outputError?: string + // Why it was killed without a signal of its own. + killed?: string + done: Deferred.Deferred + timeoutFiber?: Fiber.Fiber + stopping?: Fiber.Fiber + wait?: Fiber.Fiber +} + +type Command = Spawned | Run + type Ended = { status: "exited" | StopReason exit?: number @@ -66,10 +135,14 @@ type Ended = { /** * Location-owned non-interactive shell command process service. * - * Each `create` spawns one shell command, captures combined stdout/stderr to a + * Each `create` starts one shell command, captures combined stdout/stderr to a * file, and returns an ID. Clients poll `get` for status and `output` for * file-backed output by cursor. No session, message, or permission state lives * here; callers (e.g. `ShellTool`) own that association and store the shell ID. + * + * Local commands run in the process daemon when it is available (`Daemon`), so they outlive a planned server + * restart: the next server takes them back. Remote workspaces, Windows and an unavailable daemon keep spawning + * them from the server. */ export interface Interface { readonly create: ( @@ -95,8 +168,14 @@ export interface Interface { // with the given status and stays observable, file included, until retention evicts it. Returns at // once; `wait` observes the exit. Ignored once the command has ended or is already stopping. readonly stop: (id: Shell.ID, reason: StopReason) => Effect.Effect + // The daemon command a tool call started, while it is known here (including one taken back after a restart). + readonly byToolCall: (call: ToolCall) => Effect.Effect readonly output: (id: Shell.ID, input?: Shell.OutputInput) => Effect.Effect readonly remove: (id: Shell.ID) => Effect.Effect + /** Runs `effect` until it ends or the server stops, outliving this Location (e.g. a job's completion notice). */ + readonly watch: DaemonInterface["watch"] + /** Whether the server is handing its daemon commands over to the next one, which takes them over. */ + readonly handoffPrepared: () => boolean } export class Service extends Context.Service()("@opencode/Shell") {} @@ -104,6 +183,7 @@ export class Service extends Context.Service()("@opencode/Sh export const cleanup = Effect.fn("Shell.cleanup")(function* () { const fs = yield* FSUtil.Service const global = yield* Global.Service + const pty = yield* PersistentPty.Service const directory = path.join(global.data, DIRECTORY) const projects = yield* fs.readDirectoryEntries(directory).pipe( Effect.map((entries) => entries.filter((entry) => entry.type === "Directory")), @@ -124,7 +204,29 @@ export const cleanup = Effect.fn("Shell.cleanup")(function* () { ), { concurrency: 8 }, ) - yield* FileRetention.cleanup(fs, files.flat(), RETENTION) + // Never starts the daemon. A command it still runs keeps its file; a finished one whose file expired is forgotten + // below. + const processes = new Map( + (yield* pty.listPipes().pipe(Effect.orElseSucceed(() => []))).map((process) => [process.outputPath, process]), + ) + yield* FileRetention.cleanup( + fs, + files.flat().filter((file) => processes.get(file)?.lifecycle.status !== "running"), + RETENTION, + ) + // Finished processes whose file is gone, or that ended long ago, are of no use to any Location: forget them. + const cutoff = Date.now() - Duration.toMillis(RETENTION) + yield* Effect.forEach( + processes.values(), + (process) => + Effect.gen(function* () { + if (process.lifecycle.status === "running") return + const stale = (process.exitedAt ?? process.startedAt) < cutoff + if (!stale && (yield* fs.exists(process.outputPath).pipe(Effect.orElseSucceed(() => true)))) return + yield* pty.terminatePipe(process.id).pipe(Effect.ignore) + }), + { concurrency: 8, discard: true }, + ) }) const cleanupLayer = Layer.effectDiscard( @@ -134,7 +236,365 @@ const cleanupLayer = Layer.effectDiscard( const cleanupNode = makeGlobalNode({ name: "shell-output-cleanup", layer: cleanupLayer, - deps: [FSUtil.node, Global.node], + deps: [FSUtil.node, Global.node, PersistentPty.node], +}) + +/** One Location's daemon commands, as `Daemon.at` hands them to its `Shell`. */ +interface Runs { + readonly get: (id: Shell.ID) => Run | undefined + readonly list: () => Run[] + /** Fails with `UnavailableError` only when the daemon could not be reached or started: spawning instead is safe. */ + readonly create: (input: { + readonly info: Info + readonly args: readonly string[] + readonly env: Readonly> + readonly timeout: number + readonly toolCall: ToolCall | undefined + readonly waited: boolean + }) => Effect.Effect + readonly timeout: (run: Run, duration: number) => Effect.Effect + readonly stop: (run: Run, reason: StopReason) => Effect.Effect + readonly remove: (id: Shell.ID) => Effect.Effect +} + +export interface DaemonInterface { + /** A Location's daemon commands. The first call for a Location in this server takes its commands back from the daemon. */ + readonly at: (location: Location.Interface) => Effect.Effect + /** Runs `effect` until it ends or the server stops, outliving the caller's Location (e.g. a job's completion notice). */ + readonly watch: (effect: Effect.Effect) => Effect.Effect + readonly handoffPrepared: () => boolean +} + +/** The process daemon's shell commands, owned by the server and grouped by Location. */ +export class Daemon extends Context.Service()("@opencode/Shell/Daemon") {} + +const daemonLayer = Layer.effect( + Daemon, + Effect.gen(function* () { + const bus = yield* Bus.Service + const pty = yield* PersistentPty.Service + const scope = yield* Effect.scope + const context = yield* Effect.context() + const global = yield* Global.Service + const { unlink } = yield* Effect.promise(() => import("fs/promises")) + // Keyed by canonical Location. `recovered` resolves once the Location's commands were taken back. + const places = new Map }>() + // Fibers run with the server's context in its scope, so a Location closing never interrupts them. On shutdown + // they end without settling anything: the commands keep running for the next server, and the jobs waiting on + // them are left to the job registry, which keeps their rows for it. + const fork = (effect: Effect.Effect) => + effect.pipe(Effect.forkIn(scope, { startImmediately: true }), Effect.setContext(context)) + + const place = (location: Location.Interface) => { + const runs = new Map() + const exitOrder: Shell.ID[] = [] + const ref = Location.Ref.make({ directory: location.directory, workspaceID: location.workspaceID }) + const outputDir = path.join(global.data, DIRECTORY, location.project.id) + + const persisted = (run: Run): Stored => ({ + id: run.info.id, + command: run.info.command, + shell: run.info.shell, + location: { + directory: location.directory, + ...(location.workspaceID !== undefined ? { workspaceID: location.workspaceID } : {}), + }, + metadata: run.info.metadata, + toolCall: run.toolCall, + timeoutDeadline: run.timeoutDeadline, + stopReason: run.stopReason, + }) + + // Records the command's latest timeout and stop reason in the daemon, so a restart sees them. + const store = (run: Run) => + pty + .setPipeMetadata(run.process, persisted(run)) + .pipe( + Effect.catch((error) => + Effect.logWarning("failed to update shell command metadata", { id: run.info.id, error: error.message }), + ), + ) + + const remove = Effect.fnUntraced(function* (id: Shell.ID) { + const run = runs.get(id) + const index = exitOrder.indexOf(id) + if (index !== -1) exitOrder.splice(index, 1) + if (!run) return + runs.delete(id) + if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber) + if (run.stopping) yield* Fiber.interrupt(run.stopping) + // Stop waiting first, so forgetting the process is not mistaken for losing it. + if (run.wait) yield* Fiber.interrupt(run.wait) + // Forget the process before deleting its file. This also kills what is left of its group, even after a + // successful exit, which a spawned command would leave running: the daemon cannot forget a process otherwise. + yield* pty.terminatePipe(run.process).pipe(Effect.ignore) + yield* Effect.promise(() => unlink(run.info.file).catch(() => {})) + yield* Deferred.fail(run.done, new NotFoundError({ id })) + // Every shell reports exactly one exit; removing a running one ends it as killed. + if (run.info.status === "running") + yield* bus.publish(Shell.Event.Exited, { id, status: "killed" }, { location: ref }) + yield* bus.publish(Shell.Event.Deleted, { id }, { location: ref }) + }) + + const evict = Effect.fnUntraced(function* () { + while (exitOrder.length > EXITED_LIMIT) { + const oldest = exitOrder[0] + if (!oldest) break + yield* remove(oldest) + } + }) + + const finish = Effect.fnUntraced(function* (run: Run, outcome: Ended) { + const id = run.info.id + // A removed command already failed its waiters and published its exit and deletion. + if (runs.get(id) !== run) return + run.info = produce(run.info, (draft) => { + draft.status = outcome.status + if (outcome.exit !== undefined) draft.exit = outcome.exit + if (outcome.signal !== undefined) draft.signal = outcome.signal + draft.time.completed = outcome.completed + }) + if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber) + run.timeoutFiber = undefined + // Resolve waiters before retention eviction, so an evicted command still reports its + // terminal Info rather than the removal NotFoundError. + yield* Deferred.succeed(run.done, run.info) + yield* bus.publish( + Shell.Event.Exited, + { id, ...(outcome.exit !== undefined ? { exit: outcome.exit } : {}), status: outcome.status }, + { location: ref }, + ) + exitOrder.push(id) + yield* evict() + }) + + // Graceful, like spawned commands: SIGTERM, then SIGKILL if it has not exited 3 s later. The stop is recorded + // and the SIGTERM sent before this returns, uninterruptibly, so neither interrupting the caller (a timer being + // retimed, removed or torn down) nor a server shutting down right after cancels it. The escalation runs on its + // own. + const stop = (run: Run, reason: StopReason) => + Effect.gen(function* () { + if (run.stopReason !== undefined || run.info.status !== "running") return + run.stopReason = reason + run.timeoutDeadline = undefined + yield* store(run) + yield* pty.signalPipe(run.process, "SIGTERM").pipe(Effect.ignore) + run.stopping = yield* fork( + Effect.gen(function* () { + const ended = yield* Deferred.await(run.done).pipe(Effect.timeoutOption(STOP_GRACE), Effect.exit) + if (Exit.isSuccess(ended) && Option.isNone(ended.value)) + yield* pty.signalPipe(run.process, "SIGKILL").pipe(Effect.ignore) + }), + ) + }).pipe(Effect.uninterruptible) + + const arm = Effect.fnUntraced(function* (run: Run, duration: number) { + run.timeoutFiber = yield* fork( + Effect.sleep(Duration.millis(duration)).pipe(Effect.andThen(stop(run, "timeout"))), + ) + }) + + const timeout = Effect.fnUntraced(function* (run: Run, duration: number) { + if (run.stopReason !== undefined || run.info.status !== "running") return + if (run.timeoutFiber) yield* Fiber.interrupt(run.timeoutFiber) + run.timeoutFiber = undefined + run.timeoutDeadline = duration === 0 ? undefined : Date.now() + duration + yield* store(run) + if (duration !== 0) yield* arm(run, duration) + }) + + // Watches the exit, and the timeout (in ms from now, none if undefined). + const attach = Effect.fnUntraced(function* (run: Run, duration: number | undefined) { + run.wait = yield* fork( + pty.waitPipe(run.process).pipe( + Effect.map((exit): Ended => { + run.outputLines = exit.outputLines + run.outputError = exit.outputError + // A stopped command reports its reason, and how the stopped process ended, as spawned ones do. + return { + status: run.stopReason ?? "exited", + ...(exit.exitCode !== undefined ? { exit: exit.exitCode } : {}), + ...(exit.signal !== undefined ? { signal: exit.signal } : {}), + completed: exit.exitedAt, + } + }), + // The daemon went away (and its processes with it), or forgot the process: it was killed. + Effect.catch((error) => + Effect.logWarning("lost a shell command's process", { id: run.info.id, error: error.message }).pipe( + Effect.andThen( + Effect.sync((): Ended => { + run.killed = LOST + return { status: "killed", completed: Date.now() } + }), + ), + ), + ), + // Like spawned commands: what a failed command left running in its group is stopped with it, while a + // successful one may leave background processes (`cmd &`) behind. + Effect.tap((outcome) => + outcome.status === "exited" && outcome.exit !== undefined && outcome.exit !== 0 + ? pty.signalPipe(run.process, "SIGTERM").pipe(Effect.ignore) + : Effect.void, + ), + Effect.flatMap((outcome) => finish(run, outcome)), + ), + ) + // A stop interrupted by a restart lost its grace period: kill now. + if (run.stopReason !== undefined) { + run.stopping = yield* fork(pty.signalPipe(run.process, "SIGKILL").pipe(Effect.ignore)) + return + } + if (duration === undefined) return + if (duration > 0) return yield* arm(run, duration) + yield* stop(run, "timeout") + }) + + const create: Runs["create"] = Effect.fnUntraced(function* (input) { + const run: Run = { + kind: "daemon", + info: input.info, + process: 0, + toolCall: input.toolCall, + timeoutDeadline: !input.waited && input.timeout > 0 ? input.info.time.started + input.timeout : undefined, + done: Deferred.makeUnsafe(), + } + const created = yield* pty.createPipe({ + program: input.info.shell, + args: input.args, + cwd: input.info.cwd, + env: input.env, + groupID: location.project.id, + outputPath: input.info.file, + metadata: persisted(run), + }) + run.process = created.id + run.info = produce(input.info, (draft) => { + if (created.pid !== undefined) draft.pid = created.pid + // The daemon's clock, so the command reports the same start wherever it is read. + draft.time.started = created.startedAt + }) + runs.set(run.info.id, run) + // Timed from now: starting the daemon may have taken a while. + yield* attach(run, input.timeout > 0 ? input.timeout : undefined) + yield* bus.publish(Shell.Event.Created, { info: run.info }, { location: ref }) + return run.info + }) + + // Takes back this Location's commands, left by a previous server. Two Locations can share a project's output + // directory, so match the Location itself. + const recover = Effect.fnUntraced(function* () { + const processes = yield* pty + .listPipes() + .pipe( + Effect.catch((error) => + Effect.logWarning("failed to list shell commands to recover", { error: error.message }).pipe( + Effect.as([]), + ), + ), + ) + const finished: Run[] = [] + for (const process of processes) { + const stored = Option.getOrUndefined(decodeStored(process.metadata)) + if ( + !stored || + stored.location.directory !== location.directory || + stored.location.workspaceID !== location.workspaceID || + path.dirname(process.outputPath) !== outputDir || + runs.has(stored.id) + ) + continue + const running = process.lifecycle.status === "running" + const exited = process.lifecycle.status === "exited" ? process.lifecycle : undefined + const status = running ? "running" : (stored.stopReason ?? (exited ? "exited" : "killed")) + const run: Run = { + kind: "daemon", + info: { + id: stored.id, + status, + command: stored.command, + cwd: process.cwd, + shell: stored.shell, + file: process.outputPath, + ...(process.pid !== undefined ? { pid: process.pid } : {}), + ...(exited?.exitCode !== undefined ? { exit: exited.exitCode } : {}), + ...(exited?.signal !== undefined ? { signal: exited.signal } : {}), + metadata: stored.metadata, + time: { + started: process.startedAt, + ...(running ? {} : { completed: process.exitedAt ?? process.startedAt }), + }, + }, + process: process.id, + toolCall: stored.toolCall, + timeoutDeadline: stored.timeoutDeadline, + stopReason: stored.stopReason, + ...(running ? {} : { outputLines: process.outputLines, outputError: process.outputError }), + ...(process.lifecycle.status === "failed" && !stored.stopReason + ? { killed: `Command was killed: ${process.lifecycle.message}` } + : {}), + done: Deferred.makeUnsafe(), + } + runs.set(stored.id, run) + if (running) { + // A deadline that passed while no server ran fires at once. + const deadline = stored.timeoutDeadline + yield* attach(run, deadline === undefined ? undefined : Math.max(0, deadline - Date.now())) + continue + } + yield* Deferred.succeed(run.done, run.info) + finished.push(run) + } + exitOrder.push( + ...finished + .toSorted((a, b) => (a.info.time.completed ?? 0) - (b.info.time.completed ?? 0)) + .map((run) => run.info.id), + ) + yield* evict() + }) + + const handle: Runs = { + get: (id) => runs.get(id), + list: () => Array.from(runs.values()), + create, + timeout, + stop, + remove, + } + return { runs: handle, recover } + } + + const at = Effect.fnUntraced(function* (location: Location.Interface) { + const key = `${location.directory}\0${location.workspaceID ?? ""}` + const existing = places.get(key) + if (existing) { + yield* Deferred.await(existing.recovered) + return existing.runs + } + const placed = place(location) + const entry = { runs: placed.runs, recovered: Deferred.makeUnsafe() } + places.set(key, entry) + // Uninterruptible: the entry is shared, so a caller's interrupt must not leave it half recovered. + yield* placed + .recover() + .pipe(Effect.ensuring(Deferred.succeed(entry.recovered, undefined)), Effect.uninterruptible) + return placed.runs + }) + + return Daemon.of({ + at, + handoffPrepared: pty.handoffPrepared, + watch: (effect) => + fork(effect.pipe(Effect.catch((error) => Effect.logError("shell watcher failed", { error })))).pipe( + Effect.asVoid, + ), + }) + }), +) + +export const daemonNode = makeGlobalNode({ + service: Daemon, + layer: daemonLayer, + deps: [Bus.node, Global.node, PersistentPty.node], }) const layer = () => @@ -149,13 +609,21 @@ const layer = () => const hooks = yield* PluginHooks.Service const environments = yield* SessionEnvironment.Service const config = yield* Config.Service + const daemon = yield* Daemon const context = yield* Effect.context() const runFork = Effect.runForkWith(context) - const commands = new Map() + // Commands this server spawned for the Location; daemon commands live in `runs`, owned by the server. + const commands = new Map() const exitOrder: Shell.ID[] = [] + // Remote workspaces and Windows keep spawning commands from the server. Taken back before the service is + // returned, so session restart recovery sees them. + const runs = + location.workspaceID === undefined && process.platform !== "win32" && PersistentPty.available + ? yield* daemon.at(location) + : undefined const outputDir = path.join(global.data, DIRECTORY, location.project.id) - const { mkdir, unlink } = yield* Effect.promise(() => import("fs/promises")) + const { mkdir, unlink, stat } = yield* Effect.promise(() => import("fs/promises")) const { createWriteStream, createReadStream } = yield* Effect.promise(() => import("fs")) yield* Effect.promise(() => mkdir(outputDir, { recursive: true })) @@ -163,7 +631,8 @@ const layer = () => Effect.gen(function* () { for (const command of commands.values()) { if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) - // Teardown interrupts pending commands; it is not a terminal command failure. + // Teardown interrupts pending commands; it is not a terminal command failure. Daemon commands are left + // to the server. yield* Deferred.interrupt(command.done) } commands.clear() @@ -172,7 +641,7 @@ const layer = () => ) const require = Effect.fnUntraced(function* (id: Shell.ID) { - const command = commands.get(id) + const command: Command | undefined = commands.get(id) ?? runs?.get(id) if (!command) return yield* new NotFoundError({ id }) return command }) @@ -186,19 +655,21 @@ const layer = () => if (command.timeoutFiber) yield* Fiber.interrupt(command.timeoutFiber) // Unblock any wait still pending when the command is removed before it terminated. yield* Deferred.fail(command.done, new NotFoundError({ id })) - yield* Effect.promise(() => unlink(command.file).catch(() => {})) + yield* Effect.promise(() => unlink(command.info.file).catch(() => {})) // Every shell reports exactly one exit; removing a running one ends it as killed. if (command.info.status === "running") yield* bus.publish(Shell.Event.Exited, { id, status: "killed" }) yield* bus.publish(Shell.Event.Deleted, { id }) }) const remove = Effect.fn("Shell.remove")(function* (id: Shell.ID) { - yield* annotate((yield* require(id)).info) + const command = yield* require(id) + yield* annotate(command.info) + if (command.kind === "daemon") return yield* runs?.remove(id) ?? Effect.void yield* removeCommand(id) }) const list = Effect.fn("Shell.list")(function* () { - return Array.from(commands.values()) + return [...commands.values(), ...(runs?.list() ?? [])] .filter((command) => command.info.status === "running") .map((command) => command.info) }) @@ -220,19 +691,31 @@ const layer = () => const timeout = Effect.fn("Shell.timeout")(function* (id: Shell.ID, duration: number) { const command = yield* require(id) yield* annotate(command.info) - if (command.info.status !== "running" || !command.timeout) return command.info - yield* command.timeout(duration) + if (command.kind === "daemon") yield* runs?.timeout(command, duration) ?? Effect.void + if (command.kind === "spawned" && command.info.status === "running" && command.timeout) + yield* command.timeout(duration) return command.info }) const stop = Effect.fn("Shell.stop")(function* (id: Shell.ID, reason: StopReason) { const command = yield* require(id) yield* annotate(command.info) - if (command.info.status !== "running" || !command.stop) return command.info - yield* command.stop(reason) + if (command.kind === "daemon") yield* runs?.stop(command, reason) ?? Effect.void + if (command.kind === "spawned" && command.info.status === "running" && command.stop) yield* command.stop(reason) return command.info }) + const byToolCall = Effect.fn("Shell.byToolCall")(function* (call: ToolCall) { + return runs + ?.list() + .find( + (run) => + run.toolCall?.sessionID === call.sessionID && + run.toolCall.assistantMessageID === call.assistantMessageID && + run.toolCall.callID === call.callID, + )?.info + }) + const readRange = (file: string, start: number, length: number) => length <= 0 ? Effect.succeed(Buffer.alloc(0)) @@ -252,21 +735,70 @@ const layer = () => }), ) + // A daemon command's file is the only record of its output, so its size is the file's; a missing file is empty. + const sizeOf = (command: Command) => + command.kind === "spawned" + ? Effect.succeed(command.size) + : Effect.promise(() => + stat(command.info.file).then( + (info) => info.size, + () => 0, + ), + ) + const output = Effect.fnUntraced(function* (id: Shell.ID, input?: Shell.OutputInput) { const command = yield* require(id) const cursor = input?.cursor ?? 0 const limit = input?.limit ?? 65536 - if (cursor >= command.size) return { output: "", cursor: command.size, size: command.size, truncated: false } + const size = yield* sizeOf(command) + if (cursor >= size) return { output: "", cursor: size, size, truncated: false } const start = Math.max(0, cursor) - const bytes = yield* readRange(command.file, start, Math.min(limit, command.size - start)) + const bytes = yield* readRange(command.info.file, start, Math.min(limit, size - start)) return { output: bytes.toString("utf8"), cursor: start + bytes.length, - size: command.size, + size, truncated: false, } }) + // Only for a command whose exit report is missing (still running, or lost with the daemon); the daemon counts + // newlines otherwise. Gives up on large files, whose line numbers then go unreported, to keep a result quick. + const countNewlines = (file: string, size: number) => + Effect.promise( + () => + new Promise((resolve) => { + if (size === 0) return resolve(0) + if (size > COUNT_NEWLINES_LIMIT) return resolve(undefined) + let count = 0 + const stream = createReadStream(file, { end: size - 1 }) + stream.on("data", (chunk: string | Buffer) => { + const bytes = Buffer.from(chunk) + for (let index = bytes.indexOf(10); index !== -1; index = bytes.indexOf(10, index + 1)) count++ + }) + stream.on("end", () => resolve(count)) + stream.on("error", () => resolve(count)) + }), + ) + + // Where the last two lines of a daemon command's output start, found by reading back from the end. Only needed + // to size a single cut-off line, which the spawner tracks as it writes. A start further back than the scan + // limit is reported as unknown. + const lineStartsOf = Effect.fnUntraced(function* (file: string, size: number) { + const starts: number[] = [] + let end = size + const floor = Math.max(0, size - LINE_START_SCAN_LIMIT) + while (end > floor && starts.length < 2) { + const from = Math.max(floor, end - 65536) + const chunk = yield* readRange(file, from, end - from) + for (let index = chunk.length - 1; index >= 0 && starts.length < 2; index--) + if (chunk[index] === 10) starts.push(from + index + 1) + end = from + } + const unknown = floor > 0 ? undefined : 0 + return [starts[1] ?? unknown, starts[0] ?? unknown] as [number | undefined, number | undefined] + }) + const result = Effect.fn("Shell.result")(function* ( started: Shell.Info, options?: { readonly wait?: Duration.Input }, @@ -291,35 +823,59 @@ const layer = () => const maxLines = limits?.max_lines ?? ToolOutput.MAX_LINES const maxBytes = limits?.max_bytes ?? ToolOutput.MAX_BYTES const command = yield* require(info.id) - const start = Math.max(0, command.size - maxBytes) + const size = yield* sizeOf(command) + const start = Math.max(0, size - maxBytes) // Read the byte before the window too, to tell whether the window starts on a line boundary. const before = Math.max(0, start - 1) - const window = yield* readRange(command.file, before, command.size - before) + const window = yield* readRange(info.file, before, size - before) const page = window.subarray(charBoundary(window, start - before)).toString("utf8") const partial = start > 0 && window[0] !== 10 const endsWithNewline = page.endsWith("\n") const lines = page.split("\n") if (endsWithNewline) lines.pop() - const total = command.newlines + (endsWithNewline ? 0 : 1) + const newlines = + command.kind === "spawned" + ? command.newlines + : (command.outputLines ?? (yield* countNewlines(info.file, size))) + const total = newlines === undefined ? undefined : newlines + (endsWithNewline ? 0 : 1) const suffix = `full output saved to ${info.file}]` + // After a write error the daemon kept running the command, but its file stopped early. + const stoppedEarly = + command.kind === "daemon" && command.outputError + ? `\n\n[output file ${info.file} stopped early: ${command.outputError}]` + : "" + const killed = command.kind === "daemon" && command.killed ? { killed: command.killed } : {} if (partial && lines.length === 1) { - const lineStart = endsWithNewline ? command.lineStarts[0] : command.lineStarts[1] - const lineSize = command.size - (endsWithNewline ? 1 : 0) - lineStart + const lineStarts = command.kind === "spawned" ? command.lineStarts : yield* lineStartsOf(info.file, size) + const lineStart = endsWithNewline ? lineStarts[0] : lineStarts[1] + const end = size - (endsWithNewline ? 1 : 0) + const lineSize = + lineStart === undefined + ? `over ${formatSize(Math.min(end, LINE_START_SCAN_LIMIT))}` + : formatSize(end - lineStart) const shownSize = Buffer.byteLength(lines[0]) + const line = total === undefined ? "the last line" : `line ${total}` return { - output: `${lines[0]}\n\n[showing the last ${formatSize(shownSize)} of line ${total} (${formatSize(lineSize)}); ${suffix}`, + output: `${lines[0]}\n\n[showing the last ${formatSize(shownSize)} of ${line} (${lineSize}); ${suffix}${stoppedEarly}`, truncated: true, + ...killed, } } if (start > 0 || lines.length > maxLines) { // A window that starts mid-line drops that partial line so the line numbers stay exact. const kept = lines.slice(-Math.min(maxLines, lines.length - (partial ? 1 : 0))) return { - output: `${kept.join("\n")}\n\n[showing lines ${total - kept.length + 1}-${total} of ${total}; ${suffix}`, + output: + total === undefined + ? `${kept.join("\n")}\n\n[showing the last ${kept.length} lines; ${suffix}${stoppedEarly}` + : `${kept.join("\n")}\n\n[showing lines ${total - kept.length + 1}-${total} of ${total}; ${suffix}${stoppedEarly}`, truncated: true, + ...killed, } } - return { output: page || "(no output)", truncated: false } + // One blank line before the notice, whether or not the output ended with a newline. + const shown = stoppedEarly && page.endsWith("\n") ? page.slice(0, -1) : page + return { output: `${shown || "(no output)"}${stoppedEarly}`, truncated: false, ...killed } }).pipe(Effect.catchTag("Shell.NotFoundError", () => Effect.succeed(undefined))) return { info, capture } }) @@ -346,6 +902,8 @@ const layer = () => } yield* hooks.trigger("shell", "create.before", invocation) if (before) yield* before(invocation) + // Absolute for local commands, so the command reports the same cwd wherever it runs. + if (location.workspaceID === undefined) invocation.cwd = path.resolve(location.directory, invocation.cwd) const id = Shell.ID.ascending() const args = ShellSelect.args(invocation.shell, invocation.command) @@ -363,10 +921,38 @@ const layer = () => } yield* annotate(info) + if (runs) { + const created = yield* runs + .create({ + info, + args, + // The daemon replaces the environment, so variables removed by hooks stay removed, like the spawner. + env: Object.fromEntries( + Object.entries(invocation.env).filter((entry): entry is [string, string] => entry[1] !== undefined), + ), + timeout: invocation.timeout, + toolCall: input.toolCall, + waited: input.waited === true, + }) + .pipe( + Effect.map(Option.some), + Effect.catchTags({ + "PersistentPty.UnavailableError": (error) => + Effect.logWarning("process daemon unavailable; spawning shell command instead", { + error: error.message, + }).pipe(Effect.as(Option.none())), + // The daemon may have run it: never run it again here. + "PersistentPty.RequestError": (cause) => + Effect.fail(new AppProcess.AppProcessError({ command: invocation.command, cause })), + }), + ) + if (Option.isSome(created)) return created.value + } + // Spawn through the Environment and stream combined output to the file. The handle is scope-bound, so // the managing fiber keeps its scope open until the command terminates (it awaits `done` at the // end). `create` returns once `ready` resolves with the registered command. - const ready = Deferred.makeUnsafe() + const ready = Deferred.makeUnsafe() runFork( Effect.scoped( Effect.gen(function* () { @@ -377,17 +963,17 @@ const layer = () => env: invocation.env, stdin: "ignore", detached: process.platform !== "win32", - forceKillAfter: Duration.seconds(3), + forceKillAfter: STOP_GRACE, }), ) .pipe( Effect.mapError((cause) => new AppProcess.AppProcessError({ command: invocation.command, cause })), ) - const command: Active = { + const command: Spawned = { + kind: "spawned", info: produce(info, (draft) => { draft.pid = handle.pid }), - file, size: 0, newlines: 0, lineStarts: [0, 0], @@ -441,7 +1027,7 @@ const layer = () => command.stop = (reason) => Effect.gen(function* () { if (!(yield* end({ status: reason }))) return - runFork(handle.kill({ forceKillAfter: Duration.seconds(3) }).pipe(Effect.catch(() => Effect.void))) + runFork(handle.kill({ forceKillAfter: STOP_GRACE }).pipe(Effect.catch(() => Effect.void))) }).pipe(Effect.uninterruptible) command.timeout = (duration) => @@ -522,7 +1108,20 @@ const layer = () => return command.info }) - return Service.of({ create, list, get, wait, result, timeout, stop, output, remove }) + return Service.of({ + create, + list, + get, + wait, + result, + timeout, + stop, + byToolCall, + output, + remove, + watch: daemon.watch, + handoffPrepared: daemon.handoffPrepared, + }) }), ) @@ -570,6 +1169,7 @@ export const node = makeLocationNode({ PluginHooks.node, SessionEnvironment.node, Config.node, + daemonNode, cleanupNode, ], }) diff --git a/packages/core/src/shell/result.ts b/packages/core/src/shell/result.ts index 558d1b8c607d..2c25349fcccf 100644 --- a/packages/core/src/shell/result.ts +++ b/packages/core/src/shell/result.ts @@ -5,10 +5,11 @@ import type { Shell } from "@opencode/schema/shell" export type Result = { info: Shell.Info - capture: { output: string; truncated: boolean } | undefined + // `killed` says why a command was killed when no exit code or signal of its own tells, e.g. the process daemon stopped. + capture: { output: string; truncated: boolean; killed?: string } | undefined } -type Output = { output: string; truncated: boolean; exit?: number; signal?: string; timeout?: boolean } +type Output = { output: string; truncated: boolean; exit?: number; signal?: string; timeout?: boolean; killed?: string } const missing = "Shell command output is no longer available." export const unavailable: Shell.Output = { @@ -25,6 +26,7 @@ export function output(result: Result): Output { ...(result.info.exit !== undefined ? { exit: result.info.exit } : {}), ...(result.info.signal !== undefined ? { signal: result.info.signal } : {}), ...(result.info.status === "timeout" ? { timeout: true } : {}), + ...(result.capture?.killed !== undefined ? { killed: result.capture.killed } : {}), } } @@ -45,7 +47,8 @@ export function interrupted(result: Result) { return { output, notice, message: output ? `${output}\n\n${notice}` : notice } } -export function notice(output: Pick, timeout?: number) { +export function notice(output: Pick, timeout?: number) { + if (output.killed !== undefined) return output.killed if (output.timeout) return `Command timed out${timeout ? ` after ${timeout / 1_000} seconds` : ""}. If it needs longer and isn't waiting for input, rerun with a larger timeout.\nExit code: ${exitCode(output)}` if (output.signal !== undefined || (output.exit !== undefined && output.exit !== 0)) @@ -65,6 +68,7 @@ export function metadata(output: Output) { ...(output.exit !== undefined ? { exit: output.exit } : {}), ...(output.signal !== undefined ? { signal: output.signal } : {}), ...(output.timeout !== undefined ? { timeout: output.timeout } : {}), + ...(output.killed !== undefined ? { killed: true } : {}), } } diff --git a/packages/core/src/tool/plugin/shell.ts b/packages/core/src/tool/plugin/shell.ts index b2fbd18dc557..584ba46af650 100644 --- a/packages/core/src/tool/plugin/shell.ts +++ b/packages/core/src/tool/plugin/shell.ts @@ -6,7 +6,7 @@ import type { SessionHooks } from "@opencode/plugin/effect/session" import type { ShellCreateBefore } from "@opencode/plugin/effect/shell" import type { Shell as ShellSchema } from "@opencode/schema/shell" import type { Tool } from "@opencode/schema/tool" -import { Deferred, Effect, Fiber, Option, Schema, Scope } from "effect" +import { Deferred, Effect, Fiber, Option, Schema } from "effect" import { Environment } from "../../environment/index.js" import { Job } from "../../job.js" import { FileAccess } from "../../file-access.js" @@ -14,6 +14,7 @@ import { Flag } from "../../flag.js" import { Permission } from "../../permission.js" import { NonNegativeInt } from "../../schema.js" import { Session } from "../../session.js" +import { SessionMessage } from "../../session/message.js" import { SessionSchema } from "../../session/schema.js" import { Shell } from "../../shell.js" import { ShellParse } from "../../shell/parse.js" @@ -72,6 +73,7 @@ const StructuredOutput = Schema.Struct({ pid: Schema.optionalKey(NonNegativeInt), truncated: Schema.Boolean, timeout: Schema.optionalKey(Schema.Boolean), + killed: Schema.optionalKey(Schema.String), }) const Output = Schema.Struct({ @@ -118,12 +120,143 @@ const backgroundResult = (info: ShellResult.Result["info"], timeout?: number) => status: "running" as const, }) +// Tells the session how a background command ended, once its job settles. +const notify = Effect.fnUntraced(function* ( + sessions: Session.Interface, + jobs: Job.Interface, + input: { + readonly sessionID: SessionSchema.ID + readonly id: string + readonly shellID: string + readonly command: string + readonly settled: Deferred.Deferred + // Read at delivery; `false` admits the notice without waking the session. + readonly resume?: () => false | undefined + }, +) { + const info = (yield* jobs.wait({ id: input.id })).info + if (!info || info.status === "running") return + const output = info.status === "completed" ? yield* Deferred.await(input.settled) : undefined + const text = output ? (info.output ?? "") : info.status === "error" ? (info.error ?? "Command failed") : "Cancelled" + const resume = input.resume?.() + // A session deleted while its command ran has no one left to tell. + yield* sessions + .synthetic({ + ...(info.notificationID ? { id: info.notificationID } : {}), + sessionID: input.sessionID, + description: input.command, + ...ShellResult.notification({ + jobID: input.id, + shellID: input.shellID, + command: input.command, + state: info.status, + text, + output, + }), + ...(resume === false ? { resume } : {}), + }) + .pipe(Effect.catchTag("Session.NotFoundError", () => Effect.void)) + if (info.notificationID) yield* jobs.completeBackground(info.notificationID) +}) + +// The job that waits on a command for its call (or its session, once backgrounded). +const startJob = Effect.fnUntraced(function* ( + jobs: Job.Interface, + shell: Shell.Interface, + input: { + readonly sessionID: SessionSchema.ID + readonly info: ShellSchema.Info + readonly notificationID?: SessionMessage.ID + readonly timeout?: () => number + }, +) { + const info = input.info + const settled = yield* Deferred.make() + const job = yield* jobs.start({ + // CodeMode children share a tool-call ID, but each shell must own its job. + id: info.id, + type: name, + title: info.command, + metadata: { sessionID: input.sessionID, shellID: info.id }, + recovery: { + kind: "shell", + sessionID: input.sessionID, + shellID: info.id, + command: info.command, + }, + ...(input.notificationID ? { notificationID: input.notificationID } : {}), + run: Effect.gen(function* () { + const result = yield* shell.result(info) + if (!result.capture) return yield* new Shell.NotFoundError({ id: info.id }) + return { ...ShellResult.output(result), status: "completed" as const } + }).pipe( + Effect.tap((output) => Deferred.succeed(settled, output)), + Effect.map((output) => resultMessages(output, input.timeout?.()).join("\n\n")), + // Interrupting the job (its registry closing) stops the command too, unless the server is handing its daemon + // over: then the command keeps running for the next server. + Effect.onInterrupt(() => + shell.handoffPrepared() ? Effect.void : shell.stop(info.id, "killed").pipe(Effect.ignore, Effect.asVoid), + ), + ), + // Cancelling the job stops the command, keeping it and its output file. + cancel: shell.stop(info.id, "killed").pipe(Effect.ignore, Effect.asVoid), + }) + return { job, settled } +}) + +/** The tool result of a call whose command moved to the background. */ +export const backgroundToolResult = (info: ShellSchema.Info) => { + const result = toolResult(backgroundResult(info)) + // Narrowed to what a backgrounded command reports, so it can be published as is. + const metadata = { + shellID: info.id, + status: "running" as const, + truncated: result.metadata.truncated, + ...(info.pid === undefined ? {} : { pid: info.pid }), + } + return { content: result.content, metadata } +} + +/** + * Watches a command taken over from a previous server as a background job of its session, and notifies the + * session when it finishes, as for a command the model ran in the background. A finished command's notice is + * admitted before this returns. + */ +export const reattach = Effect.fnUntraced(function* (input: { + readonly shell: Shell.Interface + readonly jobs: Job.Interface + readonly sessions: Session.Interface + readonly sessionID: SessionSchema.ID + readonly info: ShellSchema.Info + // The job's existing durable row, if the previous server had backgrounded it. + readonly notificationID?: SessionMessage.ID + readonly resume?: () => false | undefined +}) { + const { job, settled } = yield* startJob(input.jobs, input.shell, input) + yield* input.jobs.background(job.id) + const notifying = notify(input.sessions, input.jobs, { + sessionID: input.sessionID, + id: job.id, + shellID: input.info.id, + command: input.info.command, + settled, + resume: input.resume, + }) + const current = yield* input.shell.get(input.info.id).pipe(Effect.option) + if (Option.isSome(current) && current.value.status !== "running") + return yield* notifying.pipe( + Effect.catch((error) => + Effect.logWarning("failed to report a recovered shell command", { id: input.info.id, error }), + ), + ) + yield* input.shell.watch(notifying) +}) + export const Plugin = { id: "opencode.tool.shell", effect: Effect.fn("ShellTool.Plugin")(function* (ctx: Context) { const sessions = yield* Session.Service const jobs = yield* Job.Service - const scope = yield* Scope.Scope const environment = yield* Environment.Service const access = yield* FileAccess.Service const shell = yield* Shell.Service @@ -162,39 +295,15 @@ export const Plugin = { return timeout }) - const notifyWhenDone = Effect.fn("ShellTool.notifyWhenDone")( - function* ( - sessionID: SessionSchema.ID, - id: string, - shellID: string, - command: string, - settled: Deferred.Deferred, - ) { - const info = (yield* jobs.wait({ id })).info - if (!info || info.status === "running") return - const output = info.status === "completed" ? yield* Deferred.await(settled) : undefined - const text = output - ? (info.output ?? "") - : info.status === "error" - ? (info.error ?? "Command failed") - : "Cancelled" - yield* sessions.synthetic({ - ...(info.notificationID ? { id: info.notificationID } : {}), - sessionID, - description: command, - ...ShellResult.notification({ - jobID: id, - shellID, - command, - state: info.status, - text, - output, - }), - }) - if (info.notificationID) yield* jobs.completeBackground(info.notificationID) - }, - Effect.forkIn(scope, { startImmediately: true }), - ) + // Not in this Location's scope: a daemon command outlives a Location that closes while the server runs, and + // its session must still hear how it ended. + const notifyWhenDone = ( + sessionID: SessionSchema.ID, + id: string, + shellID: string, + command: string, + settled: Deferred.Deferred, + ) => shell.watch(notify(sessions, jobs, { sessionID, id, shellID, command, settled })) // Stops the command of a call the user asked to stop, and fails the call with the output so far. The command // stays observable, output file included. @@ -249,6 +358,12 @@ export const Plugin = { timeout, shell: yield* compatibleShell, metadata: { sessionID: context.sessionID }, + waited: input.background !== true, + toolCall: { + sessionID: context.sessionID, + assistantMessageID: context.messageID, + callID: context.id, + }, }, (invocation) => Effect.gen(function* () { @@ -274,30 +389,10 @@ export const Plugin = { ) yield* context.progress({ shellID: info.id }) - const settled = yield* Deferred.make() - const run = Effect.gen(function* () { - const result = yield* shell.result(info) - if (!result.capture) return yield* new Shell.NotFoundError({ id: info.id }) - return { ...ShellResult.output(result), status: "completed" as const } - }).pipe( - Effect.tap((output) => Deferred.succeed(settled, output)), - Effect.map((output) => resultMessages(output, finalTimeout).join("\n\n")), - // Cancelling the job stops the command but keeps it, output file included. - Effect.onInterrupt(() => shell.stop(info.id, "killed").pipe(Effect.ignore)), - ) - const job = yield* jobs.start({ - // CodeMode children share a tool-call ID, but each shell must own its job. - id: info.id, - type: name, - title: info.command, - metadata: { sessionID: context.sessionID, shellID: info.id }, - recovery: { - kind: "shell", - sessionID: context.sessionID, - shellID: info.id, - command: info.command, - }, - run, + const { job, settled } = yield* startJob(jobs, shell, { + sessionID: context.sessionID, + info, + timeout: () => finalTimeout, }) // Stopped while the command was starting: stop it now. if (yield* Deferred.isDone(stopped)) return yield* cancelled(info, job.id) @@ -326,13 +421,19 @@ export const Plugin = { ), ) : waiting - // A user's stop arrives through `cancel`; an interrupt (a server shutdown) cancels the job as before. + // A user's stop arrives through `cancel`. Any interrupt stops the command too, unless the server is + // handing its daemon over: then the command and its job keep running, and the next server takes the + // call over and moves the command to the background. const outcome = yield* restore( Effect.raceFirst( blocked.pipe(Effect.map((result) => ({ type: "settled" as const, result }))), Deferred.await(stopped).pipe(Effect.as({ type: "cancelled" as const })), ), - ).pipe(Effect.onInterrupt(() => jobs.cancel(job.id).pipe(Effect.ignore))) + ).pipe( + Effect.onInterrupt(() => + shell.handoffPrepared() ? Effect.void : jobs.cancel(job.id).pipe(Effect.ignore), + ), + ) if (outcome.type === "cancelled") return yield* cancelled(info, job.id) const result = outcome.result if (result?.type === "backgrounded") { diff --git a/packages/core/test/fixture/persistent-pty-daemon.ts b/packages/core/test/fixture/persistent-pty-daemon.ts index 38413c9e4a18..e692d52411d9 100644 --- a/packages/core/test/fixture/persistent-pty-daemon.ts +++ b/packages/core/test/fixture/persistent-pty-daemon.ts @@ -1,4 +1,4 @@ -import { appendFileSync, writeFileSync } from "node:fs" +import { appendFileSync, rmSync, writeFileSync } from "node:fs" import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises" import net from "node:net" import os from "node:os" @@ -16,7 +16,11 @@ export function temporaryDirectory() { export function listen( socketPath: string, - handle: (socket: net.Socket, request: Record, token: string) => object | undefined, + handle: ( + socket: net.Socket, + request: Record, + token: string, + ) => object | undefined | Promise, ) { return Effect.acquireRelease( Effect.promise(async () => { @@ -33,8 +37,10 @@ export function listen( const owning = envelope.request.op === "own" if (owning) owner = true const response = owning ? { type: "owned" } : handle(socket, envelope.request, envelope.token) - if (response) socket.write(frame(response)) - if (owner) return serve() + return Promise.resolve(response).then((response) => { + if (response) socket.write(frame(response)) + if (owner) return serve() + }) }) void serve().catch(() => socket.destroy()) }) @@ -132,6 +138,12 @@ export const fakeDaemon = Effect.fn(function* () { const failTerminate = new Set() const trap = new Set() const failCreate = { message: undefined as string | undefined } + // How long a prepared handoff's ticket stays valid. + const handoffTTL = { ms: 60_000 } + // While set, `create_pipe` answers only once it resolves, like a daemon slow to start a command. + const holdCreate: { until?: Promise } = {} + // While set, `list` answers only once it resolves, like a busy daemon. + const holdList: { until?: Promise } = {} const ops: string[] = [] const requests: Record[] = [] let nextID = 1 @@ -171,8 +183,10 @@ export const fakeDaemon = Effect.fn(function* () { ops.push(String(request.op)) requests.push(request) if (request.op === "ping") return pong - if (request.op === "list") - return { type: "processes", processes: processes.filter((item) => item.mode === request.mode) } + if (request.op === "list") { + const list = () => ({ type: "processes", processes: processes.filter((item) => item.mode === request.mode) }) + return holdList.until ? holdList.until.then(list) : list() + } if (request.op === "get") { const found = find(request.id) if (!found) return { type: "error", message: `terminal ${String(request.id)} not found` } @@ -187,13 +201,22 @@ export const fakeDaemon = Effect.fn(function* () { if (typeof request.cwd !== "string" || !path.isAbsolute(request.cwd)) return { type: "error", message: "cwd must be absolute" } if (failCreate.message !== undefined) return { type: "error", message: failCreate.message } - const created = pipe(nextID++, request) - writeFileSync(String(request.output_path), "") - processes.push(created) - return { type: "created", process: created } + const create = () => { + const created = pipe(nextID++, request) + writeFileSync(String(request.output_path), "") + processes.push(created) + return { type: "created", process: created } + } + return holdCreate.until ? holdCreate.until.then(create) : create() } if (request.op === "control") return { type: "ok" } - if (request.op === "prepare_handoff") return { type: "handoff", ticket: "next", expires_at: Date.now() + 60_000 } + // Unregisters, so the server sees it stopped; it keeps serving connections already open. + if (request.op === "shutdown") { + rmSync(path.join(directory, "service.json"), { force: true }) + return { type: "ok" } + } + if (request.op === "prepare_handoff") + return { type: "handoff", ticket: "next", expires_at: Date.now() + handoffTTL.ms } if (request.op === "rename") { const found = find(request.id) if (!found?.pty) return { type: "error", message: "terminal not found" } @@ -245,6 +268,9 @@ export const fakeDaemon = Effect.fn(function* () { return { failTerminate, failCreate, + holdCreate, + holdList, + handoffTTL, trap, ops, requests, diff --git a/packages/core/test/fixture/persistent-pty.ts b/packages/core/test/fixture/persistent-pty.ts new file mode 100644 index 000000000000..c37a4004aaea --- /dev/null +++ b/packages/core/test/fixture/persistent-pty.ts @@ -0,0 +1,9 @@ +import { Effect, Layer } from "effect" +import { PersistentPty } from "@opencode/core/persistent-pty" + +// No process daemon: shell commands fall back to the Environment's spawner, and nothing is ever running in it. +export const disabledPersistentPtyLayer = Layer.mock(PersistentPty.Service, { + createPipe: () => Effect.fail(new PersistentPty.UnavailableError({ message: "process daemon disabled" })), + listPipes: () => Effect.succeed([]), + handoffPrepared: () => false, +}) diff --git a/packages/core/test/fixture/shell-daemon.ts b/packages/core/test/fixture/shell-daemon.ts new file mode 100644 index 000000000000..fabb399779bb --- /dev/null +++ b/packages/core/test/fixture/shell-daemon.ts @@ -0,0 +1,122 @@ +import path from "node:path" +import { Context, Effect, Exit, Layer, Scope } from "effect" +import { ChildProcessSpawner } from "effect/process" +import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" +import { Config } from "@opencode/core/config" +import { Environment } from "@opencode/core/environment/index" +import { Location } from "@opencode/core/location" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { AbsolutePath } from "@opencode/core/schema" +import { Shell } from "@opencode/core/shell" +import { Bus } from "@opencode/core/bus" +import { LayerNode } from "@opencode/util/effect/layer-node" +import { LocationLifecycle } from "@opencode/core/location-lifecycle" +import { Global } from "@opencode/util/global" +import { hostEnvironmentLayer } from "./environment" +import { location } from "./location" +import { fakeDaemon } from "./persistent-pty-daemon" +import { tmpdirScoped } from "./tmpdir" + +// One fake daemon and data directory shared by every Location (and server) a test builds. +export const daemonShells = Effect.fn(function* () { + const daemon = yield* fakeDaemon() + const data = (yield* tmpdirScoped()).path + const global = Global.layerWith({ data, state: path.join(data, "state") }) + const context = yield* Layer.build( + AppNodeBuilder.build(PersistentPty.configured(daemon.options), [Global.node.replace(global)]), + ) + const pty = Context.get(context, PersistentPty.Service) + let spawned = 0 + const environment = Layer.effect( + Environment.Service, + Effect.gen(function* () { + const host = yield* Environment.Service + return Environment.Service.of({ + ...host, + spawner: ChildProcessSpawner.make(() => + Effect.sync(() => spawned++).pipe(Effect.andThen(Effect.die("the spawner must not be used"))), + ), + }) + }), + ).pipe(Layer.provide(hostEnvironmentLayer)) + // `server` shares one server's daemon commands between Locations; without it the Location gets a server of its own. + const shellLayer = (directory: string, server?: Shell.DaemonInterface) => + AppNodeBuilder.build(LayerNode.group([Shell.node, LocationLifecycle.node, Bus.node]), [ + ...(server ? [Shell.daemonNode.replace(Layer.succeed(Shell.Daemon, server))] : []), + Location.node.replace( + Layer.succeed( + Location.Service, + Location.Service.of(location(Location.Ref.make({ directory: AbsolutePath.make(directory) }))), + ), + ), + Global.node.replace(global), + Config.node.replace(Config.testLayer()), + Environment.node.replace(environment), + PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, pty)), + ]) + // The server that `openLocation` Locations share, as long as the test runs. + const serverContext = yield* Layer.build( + AppNodeBuilder.build(LayerNode.group([Shell.daemonNode, Bus.node]), [ + Global.node.replace(global), + PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, pty)), + ]), + ) + const server = Context.get(serverContext, Shell.Daemon) + // A Location's Shell, on a server of its own, that stays open for the rest of the test. + const openShell = (directory: string) => + Effect.gen(function* () { + const context = yield* Layer.build(shellLayer(directory)) + return Context.get(context, Shell.Service) + }) + // A Location's Shell and its services, closed by `close` as a reload or an eviction does while the server runs. + const openLocation = (directory: string) => + Effect.gen(function* () { + const scope = yield* Scope.make() + const context = yield* Layer.buildWithScope(shellLayer(directory, server), scope) + return { + shell: Context.get(context, Shell.Service), + bus: Context.get(context, Bus.Service), + close: Context.get(context, LocationLifecycle.Service).shutdown.pipe( + Effect.andThen(Scope.close(scope, Exit.void)), + ), + } + }) + // Runs `body` against the Location's Shell on a server of its own, then stops that server, as a restart does. + const withShell = ( + directory: string, + body: (shell: Shell.Interface) => Effect.Effect, + ) => + Effect.scoped( + Effect.gen(function* () { + const shell = yield* Shell.Service + return yield* body(shell) + }).pipe(Effect.provide(shellLayer(directory))), + ) + const directory = (yield* tmpdirScoped()).path + return { + daemon, + data, + directory, + global, + pty, + withShell, + openShell, + openLocation, + // Where the server's daemon commands report. + serverBus: Context.get(serverContext, Bus.Service), + spawned: () => spawned, + created: () => daemon.requests.filter((request) => request.op === "create_pipe"), + signals: (id: number) => + daemon.requests.flatMap((request) => (request.op === "signal" && request.id === id ? [request.signal] : [])), + } +}) + +// Polls in real time until `check` holds: the fake daemon answers over a socket, which the test clock does not drive. +export const until = (check: (() => boolean) | Effect.Effect) => + Effect.gen(function* () { + for (let attempt = 0; attempt < 2_000; attempt++) { + if (typeof check === "function" ? check() : yield* check) return + yield* Effect.promise(() => Bun.sleep(2)) + } + return yield* Effect.die(new Error("condition never held")) + }) diff --git a/packages/core/test/job.test.ts b/packages/core/test/job.test.ts index 044159bf471a..aea0cdcb99b0 100644 --- a/packages/core/test/job.test.ts +++ b/packages/core/test/job.test.ts @@ -457,6 +457,59 @@ describe("Job", () => { }), ) + it.live("interrupts observers without settling a background job, or running its cancel hook, on teardown", () => + Effect.gen(function* () { + const scope = yield* Scope.make() + const jobs = yield* Job.make.pipe(Scope.provide(scope)) + let cancels = 0 + const job = yield* jobs.start({ + id: "shell_teardown", + type: "shell", + recovery: { + kind: "shell", + sessionID: SessionSchema.ID.make("ses_teardown"), + shellID: "shell_teardown", + command: "sleep 60", + }, + run: Effect.never, + cancel: Effect.sync(() => void cancels++), + }) + const background = yield* jobs.background(job.id) + const observer = yield* jobs.wait({ id: job.id }).pipe(Effect.forkChild({ startImmediately: true })) + + yield* Scope.close(scope, Exit.void) + + const observed = yield* Fiber.await(observer).pipe(Effect.timeout("1 second")) + expect(Exit.isFailure(observed) && Cause.hasInterruptsOnly(observed.cause)).toBe(true) + expect(cancels).toBe(0) + const current = yield* Job.make + expect(yield* current.pendingBackground).toMatchObject([ + { id: job.id, notificationID: background?.notificationID, status: "running" }, + ]) + if (background?.notificationID) yield* current.completeBackground(background.notificationID) + }), + ) + + it.live("runs a job's cancel hook when it is cancelled, alone or with its session", () => + Effect.gen(function* () { + const jobs = yield* Job.Service + const cancelled: string[] = [] + const start = (id: string) => + jobs.start({ + id, + type: "shell", + recovery: { kind: "shell", sessionID: SessionSchema.ID.make("ses_cancel_hook"), shellID: id, command: "x" }, + run: Effect.never, + cancel: Effect.sync(() => void cancelled.push(id)), + }) + yield* start("shell_cancel_one") + yield* start("shell_cancel_session") + yield* jobs.cancel("shell_cancel_one") + yield* jobs.cancelSession(SessionSchema.ID.make("ses_cancel_hook")) + expect(cancelled).toEqual(["shell_cancel_one", "shell_cancel_session"]) + }), + ) + it.live("cancels running session jobs and removes unacknowledged SQLite job rows on cancelSession", () => Effect.gen(function* () { const jobs = yield* Job.Service diff --git a/packages/core/test/persistent-pty-session.test.ts b/packages/core/test/persistent-pty-session.test.ts index d199807f8c5d..cf305ceb2d15 100644 --- a/packages/core/test/persistent-pty-session.test.ts +++ b/packages/core/test/persistent-pty-session.test.ts @@ -122,6 +122,34 @@ describe.skipIf(process.platform === "win32")("PersistentPty pipe processes", () metadata: { id: "sh_test", command: "echo hi" }, } + it.live("knows a handoff is prepared until a shutdown or until its ticket expires", () => + Effect.gen(function* () { + const daemon = yield* fakeDaemon() + yield* Effect.gen(function* () { + const pty = yield* PersistentPty.Service + expect(pty.handoffPrepared()).toBe(false) + yield* pty.createPipe(input) + expect(yield* pty.handoff()).not.toBeNull() + expect(pty.handoffPrepared()).toBe(true) + // Abandoned by a shutdown. + yield* pty.shutdown() + expect(pty.handoffPrepared()).toBe(false) + }).pipe(Effect.provide(layer(daemon))) + + // Abandoned by letting the ticket expire. + const expiring = yield* fakeDaemon() + expiring.handoffTTL.ms = 50 + yield* Effect.gen(function* () { + const pty = yield* PersistentPty.Service + yield* pty.createPipe(input) + expect(yield* pty.handoff()).not.toBeNull() + expect(pty.handoffPrepared()).toBe(true) + yield* Effect.sleep(80) + expect(pty.handoffPrepared()).toBe(false) + }).pipe(Effect.provide(layer(expiring))) + }), + ) + it.live("creates, inspects, signals and waits for a pipe process", () => Effect.gen(function* () { const daemon = yield* fakeDaemon() diff --git a/packages/core/test/session-execution.test.ts b/packages/core/test/session-execution.test.ts index efc08062535d..9569aa371373 100644 --- a/packages/core/test/session-execution.test.ts +++ b/packages/core/test/session-execution.test.ts @@ -22,6 +22,11 @@ import { SessionMessage } from "@opencode/core/session/message" import { SessionRunner } from "@opencode/core/session/runner/index" import { SessionInboxTable, SessionTable } from "@opencode/core/session/sql" import { SessionStore } from "@opencode/core/session/store" +import { Shell } from "@opencode/core/shell" +import { Agent } from "@opencode/core/agent" +import { ID } from "@opencode/core/model" +import { Provider } from "@opencode/core/provider" +import { daemonShells } from "./fixture/shell-daemon" import { Cause, Context, Deferred, Effect, Exit, Fiber, Layer, LayerMap, Scope } from "effect" import { eq } from "drizzle-orm" import { testEffect } from "./lib/effect" @@ -397,6 +402,9 @@ describe("SessionExecution lifecycle", () => { ) }) +// Shell commands only run in the process daemon off Windows (pipe mode is Unix-only), so the daemon shell cases skip there. +const daemonLive = process.platform === "win32" ? it.live.skip : it.live + describe("SessionRestart background recovery", () => { it.effect("keeps shell owners idle until a user prompt delivers recovered notices exactly once", () => Effect.gen(function* () { @@ -486,6 +494,276 @@ describe("SessionRestart background recovery", () => { }), ) + daemonLive("watches shell jobs whose commands survived the restart", () => + Effect.gen(function* () { + const database = yield* Database.Service + const store = yield* SessionStore.Service + const jobs = yield* Job.Service + const bus = yield* Bus.Service + const parent = Session.ID.make("ses_background_shell_survived") + yield* seedSessions(database, [parent]) + const daemon = yield* daemonShells() + const before = yield* daemon.withShell("/project", (shell) => + Effect.gen(function* () { + const running = yield* shell.create({ shell: "/bin/sh", command: "running", cwd: "/tmp" }) + const finished = yield* shell.create({ shell: "/bin/sh", command: "finished", cwd: "/tmp" }) + return { running, finished } + }), + ) + yield* seedBackground(jobs, parent, [ + { id: before.running.id, shellID: before.running.id, command: "running" }, + { id: before.finished.id, shellID: before.finished.id, command: "finished" }, + ]) + // The finished command ended while no server watched it. + daemon.daemon.output(2, "done\n") + daemon.daemon.exit(2, 0) + + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope)) + const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope)) + const context = yield* buildExecution( + scope, + ({ sessionID }) => SessionInbox.promote(database.db, bus, sessionID, "steer").pipe(Effect.asVoid), + undefined, + restarted, + shell, + ) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + + const notices = Effect.gen(function* () { + return (yield* store.context(parent)).filter( + (message) => message.type === "synthetic" && message.metadata?.source === "shell", + ) + }) + while ((yield* notices).length < 1) yield* Effect.sleep(5) + expect(yield* notices).toMatchObject([ + { + text: expect.stringContaining("done"), + metadata: { source: "shell", shellID: before.finished.id, state: "completed", exit: 0 }, + }, + ]) + expect((yield* restarted.get(before.running.id))?.status).toBe("running") + expect((yield* restarted.pendingBackground).map((item) => item.id)).toEqual([before.running.id]) + + daemon.daemon.exit(1, 2) + while ((yield* notices).length < 2) yield* Effect.sleep(5) + expect((yield* notices)[1]).toMatchObject({ + text: expect.stringContaining("Exit code: 2"), + metadata: { source: "shell", shellID: before.running.id, state: "completed", exit: 2 }, + }) + expect(yield* restarted.pendingBackground).toEqual([]) + }), + ) + + daemonLive("settles a shell call a killed server left running as moved to the background", () => + Effect.gen(function* () { + const database = yield* Database.Service + const store = yield* SessionStore.Service + const bus = yield* Bus.Service + const parent = Session.ID.make("ses_background_shell_killed") + yield* seedSessions(database, [parent], { time_suspended: Date.now() }) + const assistantMessageID = SessionMessage.ID.create() + yield* bus.publish(SessionEvent.Step.Started, { + sessionID: parent, + assistantMessageID, + agent: Agent.ID.make("build"), + model: { id: ID.make("fake-model"), providerID: Provider.ID.make("fake") }, + started: 0, + }) + yield* bus.publish(SessionEvent.Tool.Input.Started, { + sessionID: parent, + assistantMessageID, + id: "call-shell", + name: "shell", + }) + yield* bus.publish(SessionEvent.Tool.Called, { + sessionID: parent, + assistantMessageID, + id: "call-shell", + input: { command: "sleep 60" }, + executed: false, + }) + const daemon = yield* daemonShells() + const info = yield* daemon.withShell("/project", (shell) => + shell.create({ + shell: "/bin/sh", + command: "sleep 60", + cwd: "/tmp", + timeout: 120_000, + waited: true, + toolCall: { sessionID: parent, assistantMessageID, callID: "call-shell" }, + }), + ) + // A call waits on it, so its deadline is not kept: taking it over never times it out. + expect(daemon.daemon.processes[0]?.pipe?.metadata).not.toHaveProperty("timeoutDeadline") + + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope)) + const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope)) + const context = yield* buildExecution( + scope, + ({ sessionID }) => SessionInbox.promote(database.db, bus, sessionID, "steer").pipe(Effect.asVoid), + undefined, + restarted, + shell, + ) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + + const [assistant] = (yield* store.context(parent)).filter((message) => message.type === "assistant") + expect(assistant?.type === "assistant" ? assistant.content : []).toMatchObject([ + { + type: "tool", + id: "call-shell", + state: { + status: "completed", + content: [ + { + type: "text", + text: `Command moved to the background (shell ID: ${info.id}).\nPID: ${info.pid}\nOutput is streaming to: ${info.file}`, + }, + { type: "text", text: expect.stringContaining("You will be notified automatically") }, + ], + }, + }, + ]) + expect((yield* restarted.get(info.id))?.status).toBe("running") + expect(daemon.signals(1)).toEqual([]) + + daemon.daemon.exit(1, 0) + const notices = Effect.gen(function* () { + return (yield* store.context(parent)).filter( + (message) => message.type === "synthetic" && message.metadata?.source === "shell", + ) + }) + while ((yield* notices).length < 1) yield* Effect.sleep(5) + expect((yield* notices)[0]).toMatchObject({ metadata: { source: "shell", shellID: info.id, state: "completed" } }) + }), + ) + + for (const exited of [false, true]) + daemonLive( + `reattaches once a ${exited ? "finished" : "running"} shell a killed server backgrounded without settling its call`, + () => + Effect.gen(function* () { + const database = yield* Database.Service + const store = yield* SessionStore.Service + const bus = yield* Bus.Service + const jobs = yield* Job.Service + const parent = Session.ID.make(`ses_background_shell_unsettled_${exited}`) + yield* seedSessions(database, [parent], { time_suspended: Date.now() }) + const assistantMessageID = yield* seedShellCall(bus, parent, "call-shell") + const daemon = yield* daemonShells() + const info = yield* daemon.withShell("/project", (shell) => + shell.create({ + shell: "/bin/sh", + command: "work", + cwd: "/tmp", + waited: true, + toolCall: { sessionID: parent, assistantMessageID, callID: "call-shell" }, + }), + ) + // The old server persisted the background job, then died before the step recorded the call's result. + yield* seedBackground(jobs, parent, [{ id: info.id, shellID: info.id, command: "work" }]) + const [row] = yield* jobs.pendingBackground + if (exited) { + daemon.daemon.output(1, "finished output\n") + daemon.daemon.exit(1, 0) + } + + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope)) + const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope)) + const context = yield* buildExecution( + scope, + ({ sessionID }) => SessionInbox.promote(database.db, bus, sessionID, "steer").pipe(Effect.asVoid), + undefined, + restarted, + shell, + ) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + + expect(yield* shellCallState(store, parent)).toMatchObject({ status: "completed" }) + const notices = shellNotices(store, parent) + // A finished command's result is admitted during recovery, before any session resumes. + if (exited) + expect((yield* SessionInbox.list(database.db, parent)).length + (yield* notices).length).toBeGreaterThan(0) + if (!exited) { + expect((yield* restarted.pendingBackground).map((item) => item.notificationID)).toEqual([ + row?.notificationID, + ]) + daemon.daemon.exit(1, 0) + } + while ((yield* notices).length < 1) yield* Effect.sleep(5) + yield* Effect.sleep(50) + expect(yield* notices).toMatchObject([{ id: row?.notificationID, metadata: { shellID: info.id } }]) + if (exited) expect(yield* notices).toMatchObject([{ text: expect.stringContaining("finished output") }]) + expect(yield* restarted.pendingBackground).toEqual([]) + }), + ) + + daemonLive("settles a recovered child's unsettled shell call before the child resumes", () => + Effect.gen(function* () { + const database = yield* Database.Service + const store = yield* SessionStore.Service + const bus = yield* Bus.Service + const jobs = yield* Job.Service + const parent = Session.ID.make("ses_background_shell_child_parent") + const child = Session.ID.make("ses_background_shell_child") + yield* seedSessions(database, [parent]) + yield* seedSessions(database, [child], { parent_id: parent, time_suspended: Date.now() }) + yield* jobs.start({ + id: child, + type: "subagent", + recovery: { + kind: "subagent", + parentSessionID: parent, + childSessionID: child, + agent: "explore", + description: "Inspect recovery", + }, + run: Effect.never, + }) + yield* jobs.background(child) + const assistantMessageID = yield* seedShellCall(bus, child, "call-child-shell") + const daemon = yield* daemonShells() + const info = yield* daemon.withShell("/project", (shell) => + shell.create({ + shell: "/bin/sh", + command: "child work", + cwd: "/tmp", + waited: true, + toolCall: { sessionID: child, assistantMessageID, callID: "call-child-shell" }, + }), + ) + + const scope = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) + const restarted = yield* Job.make.pipe(Effect.provideService(Scope.Scope, scope)) + const shell = yield* daemon.openShell("/project").pipe(Effect.provideService(Scope.Scope, scope)) + const seen = yield* Deferred.make() + const context = yield* buildExecution( + scope, + ({ sessionID }) => + sessionID === child + ? shellCallState(store, child).pipe(Effect.flatMap((state) => Deferred.succeed(seen, state))) + : Effect.void, + undefined, + restarted, + shell, + ) + yield* Context.get(context, SessionRestart.Service).resumeSuspendedSessions + expect(yield* Deferred.await(seen).pipe(Effect.timeout("2 seconds"))).toMatchObject({ status: "completed" }) + expect((yield* restarted.get(info.id))?.status).toBe("running") + // Cancelling a reattached job stops its command. + yield* restarted.cancel(info.id) + while (daemon.signals(1).length === 0) yield* Effect.sleep(5) + expect(daemon.signals(1)).toEqual(["SIGTERM"]) + }), + ) + it.effect("preserves locally running background work", () => Effect.gen(function* () { const database = yield* Database.Service @@ -1400,6 +1678,7 @@ function buildExecution( drain: (input: Parameters[0]) => Effect.Effect, options?: SessionRestart.Options, overrideJobs?: Job.Interface, + shell?: Shell.Interface, ) { return Effect.gen(function* () { const database = yield* Database.Service @@ -1430,9 +1709,12 @@ function buildExecution( LocationServiceMap.Service, LayerMap.make( () => - // The local execution test only needs the Session runner from the Location graph. + // The local execution test only needs the Session runner (and the Shell, if given) from the Location graph. // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion - runner as unknown as Layer.Layer, + (shell ? Layer.merge(runner, Layer.succeed(Shell.Service, shell)) : runner) as unknown as Layer.Layer< + LocationServices, + FileSystem.DirectoryNotFoundError + >, ), ) return yield* Layer.buildWithScope( @@ -1443,6 +1725,16 @@ function buildExecution( Layer.provide(Layer.succeed(Bus.Service, bus)), Layer.provide(Layer.succeed(SessionStore.Service, store)), Layer.provide(Layer.succeed(Job.Service, jobs)), + Layer.provide( + Layer.succeed( + Shell.Daemon, + Shell.Daemon.of({ + at: () => Effect.die("the harness has no daemon registry"), + watch: (effect) => effect.pipe(Effect.ignore, Effect.forkIn(scope), Effect.asVoid), + handoffPrepared: () => false, + }), + ), + ), // Do not reuse the outer harness's selector with its already-captured Location map. Layer.provide( AppNodeBuilder.build(Instance.node, [LocationServiceMap.node.replace(locations)]).pipe(Layer.fresh), @@ -1452,3 +1744,49 @@ function buildExecution( ) }) } + +/** Records a shell call the model made that is still running, as a killed server leaves it. */ +function seedShellCall(bus: Bus.Interface, sessionID: Session.ID, callID: string) { + return Effect.gen(function* () { + const assistantMessageID = SessionMessage.ID.create() + yield* bus.publish(SessionEvent.Step.Started, { + sessionID, + assistantMessageID, + agent: Agent.ID.make("build"), + model: { id: ID.make("fake-model"), providerID: Provider.ID.make("fake") }, + started: 0, + }) + yield* bus.publish(SessionEvent.Tool.Input.Started, { sessionID, assistantMessageID, id: callID, name: "shell" }) + yield* bus.publish(SessionEvent.Tool.Called, { + sessionID, + assistantMessageID, + id: callID, + input: { command: "work" }, + executed: false, + }) + return assistantMessageID + }) +} + +function shellCallState(store: SessionStore.Interface, sessionID: Session.ID) { + return store.context(sessionID).pipe( + Effect.map((messages) => + messages.flatMap((message) => + message.type === "assistant" + ? message.content.flatMap((part) => (part.type === "tool" ? [part.state] : [])) + : [], + ), + ), + Effect.map((states) => states[0]), + ) +} + +function shellNotices(store: SessionStore.Interface, sessionID: Session.ID) { + return store + .context(sessionID) + .pipe( + Effect.map((messages) => + messages.filter((message) => message.type === "synthetic" && message.metadata?.source === "shell"), + ), + ) +} diff --git a/packages/core/test/session-owned.test.ts b/packages/core/test/session-owned.test.ts index 8d68b5b49cfb..0c3c996286d3 100644 --- a/packages/core/test/session-owned.test.ts +++ b/packages/core/test/session-owned.test.ts @@ -125,7 +125,7 @@ const setup = Effect.fnUntraced(function* (options?: { const services = Layer.mergeAll( Layer.succeed(PluginHooks.Service, hooks), Layer.mock(Image.Service, {}), - options?.shell ?? Layer.mock(Shell.Service, {}), + options?.shell ?? Layer.mock(Shell.Service, { handoffPrepared: () => false }), ) const servicesFor = (ref: Location.Ref) => { locations.push(ref) @@ -508,6 +508,7 @@ describe("Session-owned handles", () => { }) const fixture = yield* setup({ shell: Layer.mock(Shell.Service, { + handoffPrepared: () => false, create: (input) => Effect.sync(() => { expect(input).toEqual({ diff --git a/packages/core/test/session-step.test.ts b/packages/core/test/session-step.test.ts index 18a98273861c..258c3923701c 100644 --- a/packages/core/test/session-step.test.ts +++ b/packages/core/test/session-step.test.ts @@ -233,8 +233,9 @@ it.effect("records the cost the endpoint reports instead of the catalog price", // Interrupts an attempt running a tool that stops on `cancel` and one that never does, and returns how the step and // its calls were recorded. const stopTwoTools = (how: { - reason: "user" | "shutdown" + reason: "user" | "shutdown" | undefined locationClosed: boolean + handoffPrepared?: boolean // What the cooperative tool does once asked to stop; by default it fails with its partial output. stopped?: Effect.Effect }) => @@ -284,6 +285,7 @@ const stopTwoTools = (how: { .attempt({ isLocationClosed: () => how.locationClosed, interruption: () => how.reason, + handoffPrepared: () => how.handoffPrepared === true, sessionID, assistantMessageID, agent: Agent.defaultID, @@ -392,17 +394,42 @@ it.live("classifies what a tool raises while the user's stop settles it: a defec }), ) -it.live("interrupts every tool on a server shutdown", () => +// Without a handoff nothing will take the calls over, so they settle here however the step was interrupted. +for (const reason of ["shutdown", undefined] as const) + it.live( + `settles every call as interrupted on ${reason === "shutdown" ? "a server shutdown" : "an unexplained interrupt"} without a handoff`, + () => + Effect.gen(function* () { + const { exit, data, requests, interrupted } = yield* stopTwoTools({ reason, locationClosed: false }) + expect(Exit.hasInterrupts(exit)).toBe(true) + expect(requests).toBe(1) + expect(interrupted).toBe(true) + const generic = { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } } + expect(data).toMatchObject({ + error: { type: "aborted", message: "Step interrupted" }, + content: [ + { type: "tool", id: "call-cooperative", state: generic }, + { type: "tool", id: "call-stuck", state: generic }, + ], + }) + }), + ) + +it.live("leaves an interrupted step as a crash would while the server hands its daemon over", () => Effect.gen(function* () { - const { exit, data, requests } = yield* stopTwoTools({ reason: "shutdown", locationClosed: false }) + const { exit, data, requests } = yield* stopTwoTools({ + reason: "shutdown", + locationClosed: false, + handoffPrepared: true, + }) expect(Exit.hasInterrupts(exit)).toBe(true) expect(requests).toBe(1) - const generic = { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } } + // No step end and no settled call: the next server finds the calls unsettled. + expect(data).not.toHaveProperty("error") expect(data).toMatchObject({ - error: { type: "aborted", message: "Step interrupted" }, content: [ - { type: "tool", id: "call-cooperative", state: generic }, - { type: "tool", id: "call-stuck", state: generic }, + { type: "tool", id: "call-cooperative", state: { status: "running" } }, + { type: "tool", id: "call-stuck", state: { status: "running" } }, ], }) }), diff --git a/packages/core/test/shell-cleanup.test.ts b/packages/core/test/shell-cleanup.test.ts index 9af9ca64af26..a807b69d8307 100644 --- a/packages/core/test/shell-cleanup.test.ts +++ b/packages/core/test/shell-cleanup.test.ts @@ -6,13 +6,16 @@ import { Shell } from "@opencode/core/shell" import { LayerNode } from "@opencode/util/effect/layer-node" import { FSUtil } from "@opencode/util/fs-util" import { Global } from "@opencode/util/global" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { disabledPersistentPtyLayer } from "./fixture/persistent-pty" import { withTempDir } from "./fixture/tmpdir" import { it } from "./lib/effect" const withStore = (body: (fs: FSUtil.Interface, root: string) => Effect.Effect) => withTempDir((tmp) => { - const layer = AppNodeBuilder.build(LayerNode.group([FSUtil.node, Global.node]), [ + const layer = AppNodeBuilder.build(LayerNode.group([FSUtil.node, Global.node, PersistentPty.node]), [ Global.node.replace(Global.layerWith({ data: tmp.path })), + PersistentPty.node.replace(disabledPersistentPtyLayer), ]) return Effect.gen(function* () { const fs = yield* FSUtil.Service diff --git a/packages/core/test/shell-daemon.test.ts b/packages/core/test/shell-daemon.test.ts new file mode 100644 index 000000000000..73e1500d22d3 --- /dev/null +++ b/packages/core/test/shell-daemon.test.ts @@ -0,0 +1,543 @@ +import { describe, expect } from "bun:test" +import { existsSync, readdirSync, realpathSync } from "node:fs" +import { mkdir, rm, utimes, writeFile } from "node:fs/promises" +import path from "node:path" +import { Context, Effect, Fiber, Layer } from "effect" +import { TestClock } from "effect/testing" +import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { Config } from "@opencode/core/config" +import { Environment } from "@opencode/core/environment/index" +import { Location } from "@opencode/core/location" +import { AbsolutePath } from "@opencode/core/schema" +import { Shell } from "@opencode/core/shell" +import { Bus } from "@opencode/core/bus" +import { Global } from "@opencode/util/global" +import { FSUtil } from "@opencode/util/fs-util" +import { LayerNode } from "@opencode/util/effect/layer-node" +import { AppProcess } from "@opencode/util/process" +import { hostEnvironmentLayer } from "./fixture/environment" +import { location } from "./fixture/location" +import { daemonShells, until } from "./fixture/shell-daemon" +import { tmpdirScoped } from "./fixture/tmpdir" +import { binaryPath } from "@opencode/pty" +import { it } from "./lib/effect" + +const unix = process.platform !== "win32" +const test = unix ? it.live : it.live.skip +// On the test clock, for timers. +const timed = unix ? it.effect : it.effect.skip + +const lines = (count: number) => Array.from({ length: count }, (_, index) => `line ${index + 1}\n`).join("") + +describe("Shell on the process daemon", () => { + test("runs a command with the server's environment, an absolute cwd and its metadata", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* Effect.promise(() => mkdir(path.join(env.directory, "sub"))) + process.env.OPENCODE_TEST_REMOVED = "1" + yield* Effect.addFinalizer(() => Effect.sync(() => delete process.env.OPENCODE_TEST_REMOVED)) + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const info = yield* shell.create( + { + shell: "/bin/sh", + command: "echo hi", + cwd: "sub", + timeout: 60_000, + metadata: { sessionID: "ses_test" }, + }, + (invocation) => + Effect.sync(() => { + delete invocation.env.OPENCODE_TEST_REMOVED + invocation.env.OPENCODE_TEST_UNSET = undefined + }), + ) + const [request] = env.created() + expect(request).toMatchObject({ + program: "/bin/sh", + args: ["-c", "echo hi"], + cwd: path.join(env.directory, "sub"), + output_path: info.file, + metadata: { + id: info.id, + command: "echo hi", + shell: "/bin/sh", + location: { directory: env.directory }, + metadata: { sessionID: "ses_test" }, + timeoutDeadline: expect.any(Number), + }, + }) + expect(request?.env).toMatchObject({ TERM: "xterm-256color", OPENCODE_TERMINAL: "1" }) + expect(request?.env).not.toHaveProperty("OPENCODE_TEST_REMOVED") + expect(request?.env).not.toHaveProperty("OPENCODE_TEST_UNSET") + expect(info).toMatchObject({ status: "running", pid: 1001, cwd: path.join(env.directory, "sub") }) + expect(env.spawned()).toBe(0) + + env.daemon.output(1, "first\nsecond") + expect(yield* shell.output(info.id)).toEqual({ + output: "first\nsecond", + cursor: 12, + size: 12, + truncated: false, + }) + expect(yield* shell.output(info.id, { cursor: 6, limit: 3 })).toEqual({ + output: "sec", + cursor: 9, + size: 12, + truncated: false, + }) + expect(yield* shell.output(info.id, { cursor: Number.MAX_SAFE_INTEGER })).toEqual({ + output: "", + cursor: 12, + size: 12, + truncated: false, + }) + expect((yield* shell.list()).map((item) => item.id)).toEqual([info.id]) + + env.daemon.exit(1, 0) + expect(yield* shell.result(info)).toMatchObject({ + info: { status: "exited", exit: 0, time: { completed: expect.any(Number) } }, + capture: { output: "first\nsecond", truncated: false }, + }) + expect(yield* shell.list()).toEqual([]) + }), + ) + })) + + test("counts lines like the spawner and reads only what reached the file after a write error", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const long = yield* shell.create({ shell: "/bin/sh", command: "long" }) + env.daemon.output(1, lines(3000)) + env.daemon.exit(1, 0) + const result = yield* shell.result(long) + expect(result.capture?.output).toEndWith( + `line 3000\n\n[showing lines 1001-3000 of 3000; full output saved to ${long.file}]`, + ) + + const failed = yield* shell.create({ shell: "/bin/sh", command: "failed" }) + env.daemon.output(2, "kept\n") + env.daemon.fail(2, "No space left on device") + env.daemon.output(2, "lost\n") + env.daemon.exit(2, 1) + expect(yield* shell.result(failed)).toMatchObject({ + info: { status: "exited", exit: 1 }, + capture: { + output: `kept\n\n[output file ${failed.file} stopped early: No space left on device]`, + }, + }) + + // A single line longer than the window is sized from the file, which the daemon does not track. + const wide = yield* shell.create({ shell: "/bin/sh", command: "wide" }) + env.daemon.output(3, `short\n${"x".repeat(60 * 1024)}\n`) + env.daemon.exit(3, 0) + expect((yield* shell.result(wide)).capture?.output).toEndWith( + `\n\n[showing the last 50 KB of line 2 (60 KB); full output saved to ${wide.file}]`, + ) + }), + ) + })) + + test("bounds what it reads to number lines the daemon has not counted yet", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + // Still running, so the daemon has not reported a line count, and the file is too big to count quickly. + const big = yield* shell.create({ shell: "/bin/sh", command: "big" }) + env.daemon.output(1, `${"y".repeat(17 * 1024 * 1024)}\n${lines(10)}`) + expect((yield* shell.result(big, { wait: 0 })).capture?.output).toEndWith( + `line 10\n\n[showing the last 10 lines; full output saved to ${big.file}]`, + ) + + // A cut-off line that started too far back to look for. + const wide = yield* shell.create({ shell: "/bin/sh", command: "wide" }) + env.daemon.output(2, `short\n${"x".repeat(2 * 1024 * 1024)}`) + expect((yield* shell.result(wide, { wait: 0 })).capture?.output).toEndWith( + `\n\n[showing the last 50 KB of line 2 (over 1024 KB); full output saved to ${wide.file}]`, + ) + + // The same, ending with a newline: the line's start is still too far back to know. + const ended = yield* shell.create({ shell: "/bin/sh", command: "ended" }) + env.daemon.output(3, `short\n${"x".repeat(2 * 1024 * 1024)}\n`) + expect((yield* shell.result(ended, { wait: 0 })).capture?.output).toEndWith( + `\n\n[showing the last 50 KB of line 2 (over 1024 KB); full output saved to ${ended.file}]`, + ) + }), + ) + })) + + test("reports a command lost with the daemon as killed, with what it wrote", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const info = yield* shell.create({ shell: "/bin/sh", command: "work" }) + env.daemon.output(1, "partial\n") + yield* until(() => env.daemon.waiters(1) > 0) + env.daemon.drop(1) + expect(yield* shell.result(info)).toMatchObject({ + info: { status: "killed" }, + capture: { output: "partial\n", killed: "Command was killed: the process daemon stopped" }, + }) + }), + ) + })) + + test("stops what a failed command left in its group, and leaves a successful one's", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const succeeded = yield* shell.create({ shell: "/bin/sh", command: "serve & exit 0" }) + const failed = yield* shell.create({ shell: "/bin/sh", command: "serve & exit 3" }) + yield* until(() => env.daemon.waiters(1) > 0 && env.daemon.waiters(2) > 0) + env.daemon.exit(1, 0) + env.daemon.exit(2, 3) + expect(yield* shell.wait(failed.id)).toMatchObject({ status: "exited", exit: 3 }) + expect(yield* shell.wait(succeeded.id)).toMatchObject({ status: "exited", exit: 0 }) + expect(env.signals(2)).toEqual(["SIGTERM"]) + expect(env.signals(1)).toEqual([]) + }), + ) + })) + + test("fails a command the daemon rejected instead of spawning it", () => + Effect.gen(function* () { + const env = yield* daemonShells() + env.daemon.failCreate.message = "output path is a symlink" + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const error = yield* shell.create({ shell: "/bin/sh", command: "echo hi" }).pipe(Effect.flip) + expect(error).toBeInstanceOf(AppProcess.AppProcessError) + expect(env.spawned()).toBe(0) + expect(yield* shell.list()).toEqual([]) + }), + ) + })) + + // On the test clock: the timeout and the grace period pass without waiting for them. + timed("stops a timed-out command with SIGTERM, then SIGKILL after the grace period, and keeps it", () => + Effect.gen(function* () { + const env = yield* daemonShells() + env.daemon.trap.add(1) + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const info = yield* shell.create({ shell: "/bin/sh", command: "trap '' TERM; sleep 60", timeout: 10 }) + yield* TestClock.adjust("10 millis") + yield* until(() => env.signals(1).length > 0) + expect(env.signals(1)).toEqual(["SIGTERM"]) + expect((yield* shell.get(info.id)).status).toBe("running") + // The grace period starts once the daemon acknowledged SIGTERM, so keep the clock moving until it ends. + yield* until(TestClock.adjust("1 second").pipe(Effect.map(() => env.signals(1).length > 1))) + expect((yield* shell.wait(info.id)).status).toBe("timeout") + expect(env.signals(1)).toEqual(["SIGTERM", "SIGKILL"]) + expect(env.daemon.processes[0]?.pipe?.metadata).toMatchObject({ stopReason: "timeout" }) + expect(env.daemon.processes[0]?.pipe?.metadata).not.toHaveProperty("timeoutDeadline") + expect(yield* shell.result(info)).toMatchObject({ + info: { status: "timeout" }, + capture: { output: "(no output)" }, + }) + // Like a spawned command, it records how the stopped process ended. + expect((yield* shell.result(info)).info).toMatchObject({ signal: "SIGKILL" }) + expect(env.daemon.ops).not.toContain("terminate") + expect(existsSync(info.file)).toBe(true) + }), + ) + }), + ) + + test("forgets a command in the daemon before deleting its file, on remove and on eviction", () => + Effect.gen(function* () { + const env = yield* daemonShells() + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const running = yield* shell.create({ shell: "/bin/sh", command: "sleep 60" }) + yield* shell.remove(running.id) + expect(env.daemon.requests.filter((request) => request.op === "terminate")).toEqual([ + expect.objectContaining({ id: 1 }), + ]) + expect(existsSync(running.file)).toBe(false) + expect(env.daemon.processes).toEqual([]) + expect(yield* shell.result(running)).toMatchObject({ info: { status: "killed" }, capture: undefined }) + + const finished = yield* Effect.forEach(Array.from({ length: 26 }), (_, index) => + Effect.gen(function* () { + const info = yield* shell.create({ shell: "/bin/sh", command: `echo ${index}` }) + env.daemon.exit(index + 2, 0) + yield* shell.wait(info.id) + return info + }), + ) + // Waiters resolve before retention runs, which forgets the process and then deletes its file. + yield* until(() => !existsSync(finished[0]!.file)) + expect(yield* shell.get(finished[0]!.id).pipe(Effect.flip)).toBeInstanceOf(Shell.NotFoundError) + expect(existsSync(finished[0]!.file)).toBe(false) + expect(env.daemon.processes.map((process) => process.id)).not.toContain(2) + expect(existsSync(finished[1]!.file)).toBe(true) + }), + ) + })) + + test("takes its commands back after a restart and leaves other Locations' alone", () => + Effect.gen(function* () { + const env = yield* daemonShells() + env.daemon.trap.add(4) + const before = yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const running = yield* shell.create({ shell: "/bin/sh", command: "running", metadata: { tag: "a" } }) + const finished = yield* shell.create({ shell: "/bin/sh", command: "finished" }) + env.daemon.output(2, "done\n") + env.daemon.exit(2, 3) + const exited = yield* shell.wait(finished.id) + const expiring = yield* shell.create({ shell: "/bin/sh", command: "expiring", timeout: 60_000 }) + const stopping = yield* shell.create({ shell: "/bin/sh", command: "stopping" }) + yield* shell.stop(stopping.id, "killed") + yield* until(() => env.signals(4).length > 0) + // A call waits on this one: a restart moves it to the background, so its timeout is not kept. + const waited = yield* shell.create({ shell: "/bin/sh", command: "waited", timeout: 60_000, waited: true }) + return { running, exited, expiring, stopping, waited } + }), + ) + // Closing the Location only detached from its commands. + expect(env.daemon.ops).not.toContain("terminate") + expect(env.signals(1)).toEqual([]) + expect(env.signals(4)).toEqual(["SIGTERM"]) + // The expiring command's deadline passes while no server runs. + const expiring = env.daemon.processes.find((process) => process.id === 3)?.pipe + if (expiring) expiring.metadata = { ...expiring.metadata, timeoutDeadline: Date.now() - 1 } + + const other = (yield* tmpdirScoped()).path + yield* env.withShell(other, (shell) => + Effect.gen(function* () { + expect(yield* shell.get(before.running.id).pipe(Effect.flip)).toBeInstanceOf(Shell.NotFoundError) + expect(yield* shell.list()).toEqual([]) + }), + ) + + yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + expect(yield* shell.get(before.running.id)).toMatchObject({ + status: "running", + command: "running", + shell: "/bin/sh", + file: before.running.file, + metadata: { tag: "a" }, + pid: before.running.pid, + }) + expect(yield* shell.get(before.exited.id)).toEqual({ ...before.exited, cwd: env.directory }) + expect((yield* shell.result(before.exited)).capture).toMatchObject({ output: "done\n" }) + // The deadline passed during the restart, and the interrupted stop goes straight to SIGKILL. + expect((yield* shell.wait(before.expiring.id)).status).toBe("timeout") + expect(env.signals(3)).toEqual(["SIGTERM"]) + expect((yield* shell.wait(before.stopping.id)).status).toBe("killed") + expect(env.signals(4)).toEqual(["SIGTERM", "SIGKILL"]) + + expect(env.daemon.processes.find((process) => process.id === 5)?.pipe?.metadata).not.toHaveProperty( + "timeoutDeadline", + ) + expect((yield* shell.get(before.waited.id)).status).toBe("running") + expect(env.signals(5)).toEqual([]) + + env.daemon.exit(1, 0) + expect(yield* shell.wait(before.running.id)).toMatchObject({ status: "exited", exit: 0 }) + }), + ) + })) + + // A reload or an eviction closes a Location while the server keeps running. + timed("a Location closed while the server runs keeps enforcing its commands' timeouts", () => + Effect.gen(function* () { + const env = yield* daemonShells() + const closing = yield* env.openLocation(env.directory) + const info = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60", timeout: 50 }) + yield* closing.close + expect(env.signals(1)).toEqual([]) + yield* TestClock.adjust("50 millis") + yield* until(() => env.signals(1).length > 0) + expect(env.signals(1)).toEqual(["SIGTERM"]) + expect(yield* closing.shell.wait(info.id)).toMatchObject({ status: "timeout" }) + }), + ) + + timed("a closed Location and its replacement see the same commands, with one timer and one report each", () => + Effect.gen(function* () { + const env = yield* daemonShells() + const closing = yield* env.openLocation(env.directory) + const backgrounded = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60", timeout: 100 }) + const stopped = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60" }) + const removed = yield* closing.shell.create({ shell: "/bin/sh", command: "sleep 60" }) + yield* closing.close + let exits = 0 + yield* env.serverBus.listen((event) => + Effect.sync(() => { + if (event.type === "shell.exited") exits++ + }), + ) + const replacement = yield* env.openLocation(env.directory) + expect((yield* replacement.shell.list()).map((item) => item.id)).toEqual([ + backgrounded.id, + stopped.id, + removed.id, + ]) + + // The call that started it, still in the closed Location, moves it to the background and clears its timeout. + yield* closing.shell.timeout(backgrounded.id, 0) + yield* TestClock.adjust("100 millis") + expect(env.signals(1)).toEqual([]) + + // Cancelling its job in the closed Location stops it; both see why it ended, and it is reported once. + yield* closing.shell.stop(stopped.id, "killed") + expect(yield* replacement.shell.wait(stopped.id)).toMatchObject({ status: "killed" }) + expect(yield* closing.shell.wait(stopped.id)).toMatchObject({ status: "killed" }) + yield* until(() => exits === 1) + + // Removed through one, it is gone from both. + yield* replacement.shell.remove(removed.id) + expect(yield* closing.shell.get(removed.id).pipe(Effect.flip)).toBeInstanceOf(Shell.NotFoundError) + // The server watched them all along: the replacement took nothing back and waited on nothing more. + expect(env.daemon.ops.filter((op) => op === "wait")).toHaveLength(3) + yield* replacement.close + }), + ) + + test("finishes taking a Location's commands back even if the call that started it is interrupted", () => + Effect.gen(function* () { + const env = yield* daemonShells() + const created = yield* env.withShell(env.directory, (shell) => + shell.create({ shell: "/bin/sh", command: "running" }), + ) + // A new server, whose first request for the Location is interrupted while the daemon is slow to list. + const server = Context.get( + yield* Layer.build( + AppNodeBuilder.build(LayerNode.group([Shell.daemonNode, Bus.node]), [ + Global.node.replace(env.global), + PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, env.pty)), + ]), + ), + Shell.Daemon, + ) + const place = location(Location.Ref.make({ directory: AbsolutePath.make(env.directory) })) + let release = () => {} + env.daemon.holdList.until = new Promise((resolve) => (release = resolve)) + const lists = () => env.daemon.ops.filter((op) => op === "list").length + const before = lists() + const first = yield* server.at(place).pipe(Effect.forkChild({ startImmediately: true })) + yield* until(() => lists() > before) + const interrupting = yield* Fiber.interrupt(first).pipe(Effect.forkChild({ startImmediately: true })) + env.daemon.holdList.until = undefined + release() + yield* Fiber.join(interrupting) + // Recovery ran to the end, and is not run again. + const runs = yield* server.at(place) + expect(runs.list().map((run) => run.info.id)).toEqual([created.id]) + expect(lists()).toBe(before + 1) + })) + + test("the sweep keeps a running command's file, and deletes a finished one's file and forgets it", () => + Effect.gen(function* () { + const env = yield* daemonShells() + const ids = yield* env.withShell(env.directory, (shell) => + Effect.gen(function* () { + const running = yield* shell.create({ shell: "/bin/sh", command: "running" }) + const finished = yield* shell.create({ shell: "/bin/sh", command: "finished" }) + env.daemon.exit(2, 0) + yield* shell.wait(finished.id) + const lostFile = yield* shell.create({ shell: "/bin/sh", command: "lost file" }) + env.daemon.exit(3, 0) + yield* shell.wait(lostFile.id) + const old = yield* shell.create({ shell: "/bin/sh", command: "old" }) + env.daemon.exit(4, 0) + yield* shell.wait(old.id) + return { running, finished, lostFile, old } + }), + ) + // A finished process whose file was deleted elsewhere, and one that ended more than 7 days ago. + yield* Effect.promise(() => rm(ids.lostFile.file)) + const ended = env.daemon.processes.find((process) => process.id === 4) + if (ended) ended.exited_at = Date.now() - 8 * 24 * 60 * 60 * 1_000 + const expired = new Date(Date.now() - 8 * 24 * 60 * 60 * 1_000) + yield* Effect.promise(() => utimes(ids.running.file, expired, expired)) + yield* Effect.promise(() => utimes(ids.finished.file, expired, expired)) + const orphan = path.join(path.dirname(ids.running.file), "sh_0123456789abORPHANORPHANOR.out") + yield* Effect.promise(() => writeFile(orphan, "orphan")) + yield* Effect.promise(() => utimes(orphan, expired, expired)) + + yield* Shell.cleanup().pipe( + Effect.provide( + AppNodeBuilder.build(LayerNode.group([FSUtil.node, Global.node, PersistentPty.node]), [ + Global.node.replace(env.global), + PersistentPty.node.replace(Layer.succeed(PersistentPty.Service, env.pty)), + ]), + ), + ) + + expect(existsSync(ids.running.file)).toBe(true) + expect(existsSync(ids.finished.file)).toBe(false) + expect(existsSync(orphan)).toBe(false) + expect(env.daemon.processes.map((process) => process.id)).toEqual([1]) + // The old process's file is recent, so it stays. + expect(existsSync(ids.old.file)).toBe(true) + })) +}) + +// Against the real opencode-pty, like server/test/persistent-pty.test.ts. +const binary = unix ? (process.env.OPENCODE_PTY_BIN ?? binaryPath) : undefined +const real = binary !== undefined && existsSync(binary) ? it.live : it.live.skip + +describe("Shell on a real process daemon", () => { + real( + "starts the daemon on the first command, never on boot or sweep, and runs it with the server's environment", + () => + Effect.gen(function* () { + const data = (yield* tmpdirScoped()).path + const directory = (yield* tmpdirScoped()).path + const state = path.join(data, "state") + const global = Global.layerWith({ data, state }) + const started = () => existsSync(path.join(state, "pty")) && readdirSync(path.join(state, "pty")).length > 0 + process.env.OPENCODE_TEST_REMOVED = "1" + yield* Effect.addFinalizer(() => Effect.sync(() => delete process.env.OPENCODE_TEST_REMOVED)) + yield* Effect.gen(function* () { + const shell = yield* Shell.Service + yield* Shell.cleanup() + expect(started()).toBe(false) + const info = yield* shell.create( + { + shell: "/bin/sh", + command: 'printf "%s|%s|%s\\n" "${OPENCODE_TEST_REMOVED:-unset}" "$TERM" "$(pwd)"; printf tail', + }, + (invocation) => + Effect.sync(() => { + delete invocation.env.OPENCODE_TEST_REMOVED + }), + ) + expect(started()).toBe(true) + expect(yield* shell.result(info)).toMatchObject({ + info: { status: "exited", exit: 0 }, + capture: { + output: `unset|xterm-256color|${realpathSync(directory)}\ntail`, + }, + }) + }).pipe( + Effect.scoped, + Effect.provide( + AppNodeBuilder.build(LayerNode.group([Shell.node, FSUtil.node, Global.node, PersistentPty.node]), [ + Location.node.replace( + Layer.succeed( + Location.Service, + Location.Service.of(location(Location.Ref.make({ directory: AbsolutePath.make(directory) }))), + ), + ), + Global.node.replace(global), + Config.node.replace(Config.testLayer()), + Environment.node.replace(hostEnvironmentLayer), + ]), + ), + ) + }), + { timeout: 20_000 }, + ) +}) diff --git a/packages/core/test/shell-restart.test.ts b/packages/core/test/shell-restart.test.ts new file mode 100644 index 000000000000..604371cfb18b --- /dev/null +++ b/packages/core/test/shell-restart.test.ts @@ -0,0 +1,267 @@ +// A foreground shell call across a graceful restart, through the whole app: server 1 runs the turn, prepares a +// handoff and is torn down mid-call, server 2 recovers the command from the same process daemon and resumes the session. +import { describe, expect } from "bun:test" +import path from "path" +import { Context, Effect, Exit, Fiber, Layer, Scope } from "effect" +import type { ChildProcess } from "effect/process" +import { LanguageModel, LLMClient } from "@opencode/ai/effect" +import { OpenAIChat } from "@opencode/ai/protocols/openai-chat" +import { TestLLM } from "@opencode/ai/testing" +import { Agent } from "@opencode/core/agent" +import { Database } from "@opencode/core/database/database" +import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" +import { LayerNodePlatform } from "@opencode/core/effect/app-node-platform" +import { Watcher } from "@opencode/core/filesystem/watcher" +import { LocationServiceMap } from "@opencode/core/location-service-map" +import { Model } from "@opencode/core/model" +import { Permission } from "@opencode/core/permission" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { Plugin } from "@opencode/core/plugin" +import { Provider } from "@opencode/core/provider" +import { AbsolutePath } from "@opencode/core/schema" +import { Session } from "@opencode/core/session" +import { SessionRestart } from "@opencode/core/session/execution/restart" +import { SessionRunnerModel } from "@opencode/core/session/runner/model" +import { Shell } from "@opencode/core/shell" +import { Environment } from "@opencode/core/environment/index" +import { LayerNode } from "@opencode/util/effect/layer-node" +import { Global } from "@opencode/util/global" +import { fakeDaemon } from "./fixture/persistent-pty-daemon" +import { recordingEnvironmentLayer } from "./fixture/environment" +import { tempGlobalLayer } from "./fixture/global" +import { offlineModels } from "./fixture/models" +import { tmpdirScoped } from "./fixture/tmpdir" +import { testEffect } from "./lib/effect" +import { permissionLayer } from "./lib/permission" + +const it = testEffect(tempGlobalLayer) + +const model = SessionRunnerModel.node.replace( + Layer.succeed(SessionRunnerModel.Service, { + resolve: () => + Effect.succeed( + SessionRunnerModel.resolved(LanguageModel.make({ id: "restart", provider: "test", route: OpenAIChat.route }), { + capabilities: { tools: true, input: ["text"], output: ["text"] }, + cost: [], + limit: { context: 200_000, output: 32_000 }, + }), + ), + }), +) + +// Everything both servers share: the process daemon, the data directory, the database and the model. +const world = Effect.gen(function* () { + const daemon = yield* fakeDaemon() + const root = (yield* tmpdirScoped()).path + const data = path.join(root, "data") + const global = Global.layerWith({ + data, + cache: path.join(root, "cache"), + config: path.join(root, "config"), + state: path.join(root, "state"), + tmp: path.join(root, "tmp"), + bin: path.join(root, "cache", "bin"), + log: path.join(data, "log"), + repos: path.join(data, "repos"), + }) + const database = yield* Layer.build(Database.layer()) + const llm = yield* Layer.build(TestLLM.testLayer({ fallback: TestLLM.text("Done", "done") })) + const project = (yield* tmpdirScoped()).path + // Every command the servers spawn instead of handing to the daemon. + const spawns: ChildProcess.Command[] = [] + // A server: the whole app on the shared pieces, torn down when its scope closes. The daemon comes first, as in the + // server's routes, so it is released after the sessions and jobs. + const server = (scope: Scope.Scope) => + Layer.buildWithScope( + AppNodeBuilder.build( + LayerNode.group([ + PersistentPty.node, + Shell.daemonNode, + Session.node, + LocationServiceMap.node, + SessionRestart.node, + ]), + [ + Global.node.replace(global), + Database.node.replace(Layer.succeedContext(database)), + LayerNodePlatform.llmClient.replace(Layer.succeedContext(Context.pick(LLMClient.Service)(llm))), + PersistentPty.node.replace(PersistentPty.configured(daemon.options)), + Environment.node.replace(recordingEnvironmentLayer(spawns)), + Permission.node.replace(permissionLayer({ assert: () => Effect.void })), + Watcher.node.replace(Watcher.configured({ enabled: false })), + offlineModels, + model, + ], + ), + scope, + ) + const shellSpawns = () => spawns.filter((command) => JSON.stringify(command).includes("sleep 60")) + return { daemon, llm: Context.get(llm, TestLLM.Test), project, server, shellSpawns } +}) + +const shellCall = (sessions: Session.Interface, sessionID: Session.ID) => + sessions + .context(sessionID) + .pipe( + Effect.map((messages) => + messages.flatMap((message) => + message.type === "assistant" ? message.content.filter((part) => part.type === "tool") : [], + ), + ), + ) + +const until = (check: () => boolean) => + Effect.gen(function* () { + while (!check()) yield* Effect.sleep(5) + }) + +// Shell commands only run in the process daemon off Windows (pipe mode is Unix-only). +describe.skipIf(process.platform === "win32")("shell commands across a graceful restart", () => { + for (const finished of [false, true]) + it.live( + `settles a foreground call ${finished ? "whose command finished during the restart" : "whose command still runs"} and resumes its session`, + () => + Effect.gen(function* () { + const { daemon, llm, project, server } = yield* world + // The turn's first request runs the command; anything else (a title, the resumed turn) just answers. + let turns = 0 + yield* llm.serve((request) => + request.tools.some((tool) => tool.name === "shell") && turns++ === 0 + ? TestLLM.toolCall("call-restart", "shell", { command: "sleep 60" }) + : TestLLM.text("Done", "done"), + ) + + // Server 1 starts the turn, which waits on the command, then shuts down. + const first = yield* Scope.make() + const one = yield* server(first) + const sessions = Context.get(one, Session.Service) + const session = yield* sessions.create({ + location: { directory: AbsolutePath.make(project) }, + agent: Agent.ID.make("build"), + model: Model.Ref.make({ id: Model.ID.make("restart"), providerID: Provider.ID.make("test") }), + }) + yield* Plugin.Service.use((plugins) => plugins.awaitActivation).pipe( + Effect.provide(Context.get(one, LocationServiceMap.Service).get(session.location)), + ) + yield* sessions.prompt({ sessionID: session.id, text: "run it" }) + yield* until(() => daemon.processes.length === 1 && daemon.waiters(daemon.processes[0]!.id) > 0) + const command = daemon.processes[0]! + // A graceful restart: the server hands its daemon over to the next one, then shuts down. + expect(yield* Context.get(one, PersistentPty.Service).handoff()).not.toBeNull() + yield* Scope.close(first, Exit.void) + + // The shutdown stopped nothing and settled nothing. + expect(command.lifecycle.status).toBe("running") + expect(daemon.ops).not.toContain("signal") + expect(turns).toBe(1) + if (finished) { + daemon.output(command.id, "finished output\n") + daemon.exit(command.id, 0) + } + + // Server 2 takes the command back, settles the call and resumes the turn. + const second = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(second, Exit.void)) + const two = yield* server(second) + const resumed = Context.get(two, Session.Service) + yield* Context.get(two, SessionRestart.Service).resumeSuspendedSessions + yield* until(() => turns >= 2) + const shellID = String(command.pipe?.metadata.id) + expect(yield* shellCall(resumed, session.id)).toMatchObject([ + { + id: "call-restart", + state: { + status: "completed", + content: [ + { + type: "text", + text: expect.stringContaining(`Command moved to the background (shell ID: ${shellID})`), + }, + { type: "text", text: expect.stringContaining("You will be notified automatically") }, + ], + }, + }, + ]) + if (!finished) { + // The resumed turn ends while the command still runs; its completion wakes the session again. + yield* resumed.wait(session.id) + expect(turns).toBe(2) + daemon.exit(command.id, 0) + yield* until(() => turns >= 3) + } + // Either way the session hears how it ended. + const notices = Effect.map(resumed.context(session.id), (messages) => + messages.filter((message) => message.type === "synthetic" && message.metadata?.source === "shell"), + ) + while ((yield* notices).length === 0) yield* Effect.sleep(5) + expect((yield* notices)[0]).toMatchObject({ metadata: { shellID, state: "completed" } }) + if (finished) expect((yield* notices)[0]).toMatchObject({ text: expect.stringContaining("finished output") }) + yield* resumed.wait(session.id) + }), + ) + + // Without a handoff nothing takes the commands over: the shutdown stops them, while the daemon is still reachable. + for (const starting of [false, true]) + it.live( + `stops a command ${starting ? "still being created" : "a foreground call waits on"} when the server shuts down without a handoff`, + () => + Effect.gen(function* () { + const { daemon, llm, project, server, shellSpawns } = yield* world + let turns = 0 + yield* llm.serve((request) => + request.tools.some((tool) => tool.name === "shell") && turns++ === 0 + ? TestLLM.toolCall("call-shutdown", "shell", { command: "sleep 60" }) + : TestLLM.text("Done", "done"), + ) + let release = () => {} + if (starting) daemon.holdCreate.until = new Promise((resolve) => (release = resolve)) + + const first = yield* Scope.make() + const one = yield* server(first) + const sessions = Context.get(one, Session.Service) + const session = yield* sessions.create({ + location: { directory: AbsolutePath.make(project) }, + agent: Agent.ID.make("build"), + model: Model.Ref.make({ id: Model.ID.make("restart"), providerID: Provider.ID.make("test") }), + }) + yield* Plugin.Service.use((plugins) => plugins.awaitActivation).pipe( + Effect.provide(Context.get(one, LocationServiceMap.Service).get(session.location)), + ) + yield* sessions.prompt({ sessionID: session.id, text: "run it" }) + if (starting) yield* until(() => daemon.ops.includes("create_pipe")) + else yield* until(() => daemon.processes.length === 1 && daemon.waiters(daemon.processes[0]!.id) > 0) + + const closing = yield* Scope.close(first, Exit.void).pipe(Effect.forkChild({ startImmediately: true })) + if (starting) { + yield* Effect.sleep(20) + daemon.holdCreate.until = undefined + release() + } + yield* Fiber.join(closing) + + // The command reached the daemon, and the shutdown stopped it there; nothing was spawned instead. + const command = daemon.processes[0] + expect(command).toBeDefined() + expect(daemon.requests).toContainEqual( + expect.objectContaining({ op: "signal", id: command?.id, signal: "SIGTERM" }), + ) + expect(command?.lifecycle.status).toBe("exited") + expect(shellSpawns()).toEqual([]) + + // The call was settled on the way down, and the next server resumes the session. + const second = yield* Scope.make() + yield* Effect.addFinalizer(() => Scope.close(second, Exit.void)) + const two = yield* server(second) + const resumed = Context.get(two, Session.Service) + expect(yield* shellCall(resumed, session.id)).toMatchObject([ + { + id: "call-shutdown", + state: { status: "error", error: { type: "aborted", message: "Tool execution interrupted" } }, + }, + ]) + yield* Context.get(two, SessionRestart.Service).resumeSuspendedSessions + yield* until(() => turns >= 2) + yield* resumed.wait(session.id) + }), + ) +}) diff --git a/packages/core/test/shell-result.test.ts b/packages/core/test/shell-result.test.ts index 480028db69ab..25cc8133985d 100644 --- a/packages/core/test/shell-result.test.ts +++ b/packages/core/test/shell-result.test.ts @@ -46,3 +46,20 @@ describe("ShellResult.interrupted", () => { }) }) }) + +describe("ShellResult.output", () => { + test("reports a command killed without an exit code or signal of its own", () => { + const lost = ShellResult.output({ + info: info("killed"), + capture: { output: "partial\n", truncated: false, killed: "Command was killed: the process daemon stopped" }, + }) + expect(ShellResult.notice(lost)).toBe("Command was killed: the process daemon stopped") + expect(ShellResult.metadata(lost)).toMatchObject({ killed: true }) + }) + + test("keeps a removed command's result unchanged", () => { + const removed = ShellResult.output({ info: info("killed"), capture: undefined }) + expect(removed).toEqual({ output: ShellResult.unavailable.output, truncated: false }) + expect(ShellResult.notice(removed)).toBeUndefined() + }) +}) diff --git a/packages/core/test/shell-retention.test.ts b/packages/core/test/shell-retention.test.ts index 3f776e4bb275..e650568496d1 100644 --- a/packages/core/test/shell-retention.test.ts +++ b/packages/core/test/shell-retention.test.ts @@ -10,6 +10,8 @@ import { AbsolutePath } from "@opencode/core/schema" import { Shell } from "@opencode/core/shell" import { Workspace } from "@opencode/core/workspace" import { Global } from "@opencode/util/global" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { disabledPersistentPtyLayer } from "./fixture/persistent-pty" import { hostEnvironmentLayer } from "./fixture/environment" import { tempGlobalLayer } from "./fixture/global" import { location, tempLocationLayer } from "./fixture/location" @@ -95,6 +97,7 @@ it.live("eviction makes progress past an already-removed shell", () => Global.node.replace(tempGlobalLayer), Config.node.replace(Config.testLayer()), Environment.node.replace(environment), + PersistentPty.node.replace(disabledPersistentPtyLayer), ]), ), ) diff --git a/packages/core/test/tool-shell.test.ts b/packages/core/test/tool-shell.test.ts index e6b5d38f8198..623823b73445 100644 --- a/packages/core/test/tool-shell.test.ts +++ b/packages/core/test/tool-shell.test.ts @@ -42,6 +42,10 @@ import { ShellTool } from "@opencode/core/tool/plugin/shell" import { ToolOutput } from "@opencode/core/tool-output" import { Tool } from "@opencode/core/tool" import { PluginHooks } from "@opencode/core/plugin/hooks" +import { SessionStep } from "@opencode/core/session/runner/step" +import { PersistentPty } from "@opencode/core/persistent-pty" +import { disabledPersistentPtyLayer } from "./fixture/persistent-pty" +import { fakeDaemon } from "./fixture/persistent-pty-daemon" import { tmpdir, tmpdirScoped, withTempDir } from "./fixture/tmpdir" import { tempGlobalLayer } from "./fixture/global" import { offlineModels } from "./fixture/models" @@ -142,12 +146,15 @@ const shellPluginSupervisor = makeLocationNode({ Session.node, Job.node, Shell.node, + Shell.daemonNode, ShellSelect.node, Tool.node, ], }) const nodes = LayerNode.group([ + PersistentPty.node, + Shell.daemonNode, Database.node, Bus.node, Job.node, @@ -158,21 +165,61 @@ const nodes = LayerNode.group([ FSUtil.node, Global.node, ]) -const replacements = [ +const daemonReplacements = [ SessionExecution.node.replace(executionNode), Permission.node.replace(permission), Global.node.replace(tempGlobalLayer), offlineModels, ] satisfies LayerNode.Replacements +// Most suites spawn commands from the server, so CI starts no daemon for them; `parity` runs a subset on both. +const replacements = [ + ...daemonReplacements, + PersistentPty.node.replace(disabledPersistentPtyLayer), +] satisfies LayerNode.Replacements const productionIt = testEffect(AppNodeBuilder.build(nodes, replacements)) const it = testEffect( AppNodeBuilder.build(nodes, [...replacements, PluginSupervisor.node.replace(shellPluginSupervisor)]), ) +// On the real process daemon (when its binary is compatible; otherwise this falls back to spawning). +const daemonIt = testEffect( + AppNodeBuilder.build(nodes, [...daemonReplacements, PluginSupervisor.node.replace(shellPluginSupervisor)]), +) +const parity = { + live: (name: string, body: Parameters[1], options?: Parameters[2]) => { + it.live(`${name} (spawned)`, body, options) + daemonIt.live(`${name} (daemon)`, body, options) + }, + skip: (name: string, body: Parameters[1], options?: Parameters[2]) => + it.live.skip(name, body, options), +} +// On an in-process fake daemon, so daemon-only behavior is covered without the real binary. +let fake: Effect.Success> | undefined +const fakeDaemonIt = testEffect( + AppNodeBuilder.build(nodes, [ + ...daemonReplacements, + PluginSupervisor.node.replace(shellPluginSupervisor), + PersistentPty.node.replace( + makeGlobalNode({ + service: PersistentPty.Service, + layer: Layer.unwrap( + fakeDaemon().pipe( + Effect.map((daemon) => { + fake = daemon + return PersistentPty.layerWith(daemon.options) + }), + ), + ), + deps: [Bus.node, Global.node], + }), + ), + ]), +) const permissionIt = testEffect( AppNodeBuilder.build(LayerNode.group([nodes, PermissionSaved.node]), [ SessionExecution.node.replace(executionNode), Global.node.replace(tempGlobalLayer), PluginSupervisor.node.replace(shellPluginSupervisor), + PersistentPty.node.replace(disabledPersistentPtyLayer), offlineModels, ]), ) @@ -184,6 +231,21 @@ const call = (input: typeof ShellTool.Input.Type, id = "call-shell") => ({ }) const isWindows = process.platform === "win32" +// Shell commands only run in the process daemon off Windows (pipe mode is Unix-only), so these cases have no daemon there. +const daemonLive = isWindows ? fakeDaemonIt.live.skip : fakeDaemonIt.live + +// The process daemon keeps a finished command's zombie until the command is forgotten. +const running = (pid: number) => { + try { + process.kill(pid, 0) + } catch { + return false + } + return !Bun.spawnSync(["ps", "-o", "stat=", "-p", String(pid)]) + .stdout.toString() + .trim() + .startsWith("Z") +} const cwdCommand = isWindows ? "(Get-Location).Path; Start-Sleep -Milliseconds 100" : "pwd" const helloCommand = isWindows ? "[Console]::Out.Write('hello'); Start-Sleep -Milliseconds 100" : "printf hello" const stderrCommand = isWindows @@ -1199,6 +1261,7 @@ describe("ShellTool", () => { Session.node, Job.node, Shell.node, + Shell.daemonNode, ShellSelect.node, Tool.node, ], @@ -1268,7 +1331,7 @@ describe("ShellTool", () => { ) } - it.live("keeps non-zero exits useful", () => + parity.live("keeps non-zero exits useful", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => @@ -1558,7 +1621,7 @@ describe("ShellTool", () => { { timeout: 15_000 }, ) - it.live("uses configured line limits", () => + parity.live("uses configured line limits", () => withTempDir((tmp) => { reset() return Effect.gen(function* () { @@ -1742,7 +1805,79 @@ describe("ShellTool", () => { }), ) - it.live("returns the shell id and PID for a background command", () => + daemonLive("keeps a background daemon command's job across a Location reload and reports how it ended", () => + withTempDir((tmp) => + Effect.gen(function* () { + reset() + const sessions = yield* Session.Service + const jobs = yield* Job.Service + const bus = yield* Bus.Service + const locations = yield* LocationServiceMap.Service + const location = Location.Ref.make({ directory: AbsolutePath.make(tmp.path) }) + yield* sessions.create({ id: sessionID, title: "shell test", location, model: sessionModel }) + const admitted = yield* bus.subscribe(SessionEvent.InboxEnqueued).pipe( + Stream.filter((event) => event.data.sessionID === sessionID && event.data.item.type === "synthetic"), + Stream.take(2), + Stream.runCollect, + Effect.forkScoped({ startImmediately: true }), + ) + const started = yield* Effect.gen(function* () { + const plugins = yield* Plugin.Service + yield* plugins.awaitActivation + const registry = yield* Tool.Service + const shell = yield* Shell.Service + const run = (id: string) => + Effect.gen(function* () { + const settled = yield* executeTool(registry, call({ command: "work", background: true }, id)) + return yield* shell.get(ID.make(String(settled.metadata?.shellID))) + }) + return { finishing: yield* run("call-finishing"), cancelled: yield* run("call-cancelled") } + }).pipe(Effect.provide(locations.get(location))) + const process = (info: { file: string }) => { + const found = fake?.processes.find((item) => item.pipe?.output_path === info.file) + if (!found) throw new Error("no daemon process for the command") + return found.id + } + + // A reload closes the Location and boots its replacement, which takes the commands back. + yield* locations.invalidate(location) + const replacement = yield* Effect.gen(function* () { + return yield* Shell.Service + }).pipe(Effect.provide(locations.get(location))) + expect((yield* replacement.list()).map((info) => info.id).toSorted()).toEqual( + [started.finishing.id, started.cancelled.id].toSorted(), + ) + expect((yield* jobs.get(started.finishing.id))?.status).toBe("running") + + // Cancelling the session's job still stops its command. + yield* jobs.cancel(started.cancelled.id) + while (!fake?.ops.includes("signal")) yield* Effect.promise(() => Bun.sleep(2)) + expect(fake?.requests).toContainEqual( + expect.objectContaining({ op: "signal", id: process(started.cancelled), signal: "SIGTERM" }), + ) + + fake?.output(process(started.finishing), "done\n") + fake?.exit(process(started.finishing), 0) + const notices = Array.from(yield* Fiber.join(admitted)).map((event) => event.data.item.payload) + expect(notices).toContainEqual( + expect.objectContaining({ + text: expect.stringContaining("done"), + metadata: expect.objectContaining({ shellID: started.finishing.id, state: "completed", exit: 0 }), + }), + ) + expect(notices).toContainEqual( + expect.objectContaining({ + metadata: expect.objectContaining({ shellID: started.cancelled.id, state: "cancelled" }), + }), + ) + expect((yield* jobs.get(started.finishing.id))?.status).toBe("completed") + expect((yield* replacement.get(started.finishing.id)).status).toBe("exited") + yield* locations.invalidate(location) + }), + ), + ) + + parity.live("returns the shell id and PID for a background command", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => @@ -1910,8 +2045,8 @@ describe("ShellTool", () => { { timeout: 15_000 }, ) - const signalTest = isWindows ? it.live.skip : it.live - signalTest( + const signalParity = isWindows ? parity.skip : parity.live + signalParity( "escalates a shell timeout when the ready process ignores SIGTERM", () => Effect.gen(function* () { @@ -1946,7 +2081,7 @@ describe("ShellTool", () => { }) const pid = info.pid if (pid === undefined) throw new Error("Expected shell PID") - expect(() => process.kill(pid, 0)).toThrow() + expect(running(pid)).toBe(false) }), (info) => Effect.try(() => { @@ -2009,7 +2144,7 @@ describe("ShellTool", () => { ) } - it.live("backgrounds a foreground command when the session is signaled", () => + parity.live("backgrounds a foreground command when the session is signaled", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => @@ -2089,7 +2224,7 @@ describe("ShellTool stopped", () => { return { shellID, exit, result: Exit.isSuccess(exit) ? exit.value : undefined } }) - const test = isWindows ? it.live.skip : it.live + const test = isWindows ? parity.skip : parity.live test("stops a command the user stopped, keeps its file and fails the call with the tail, file and shell ID", () => withTempDir((tmp) => { @@ -2181,6 +2316,155 @@ describe("ShellTool stopped", () => { { timeout: 15_000 }, ) + // Starts a call whose daemon command is slow to be created, and returns once the daemon was asked for it. + const startSlowCall = (registry: Tool.Interface, id: string) => + Effect.gen(function* () { + let release = () => {} + if (fake) fake.holdCreate.until = new Promise((resolve) => (release = resolve)) + // The step's own gate: a stop goes through `cancel` only if the tool is awaiting it. + const cancel = SessionStep.makeCancel() + const fiber = yield* executeTool(registry, { + ...call({ command: "sleep 60" }, id), + cancel: cancel.effect, + }).pipe(Effect.forkChild({ startImmediately: true })) + while (!fake?.ops.includes("create_pipe")) yield* Effect.promise(() => Bun.sleep(2)) + return { + fiber, + stop: Effect.sync(() => cancel.trigger()), + release: Effect.sync(() => { + if (fake) fake.holdCreate.until = undefined + release() + }), + } + }) + + daemonLive("stops a command the user stopped while it was being created", () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const jobs = yield* Job.Service + const shell = yield* Shell.Service + const call = yield* startSlowCall(registry, "call-slow-create") + // The tool already listens while its command is being created, so the step would not interrupt it. + expect(yield* call.stop).toBe(true) + yield* call.release + const result = yield* Fiber.join(call.fiber) + expect(result.error?.message).toContain("Command interrupted by the user") + const process = fake?.processes[0] + expect(fake?.requests).toContainEqual( + expect.objectContaining({ op: "signal", id: process?.id, signal: "SIGTERM" }), + ) + expect(process?.lifecycle.status).toBe("exited") + expect(yield* shell.list()).toEqual([]) + expect((yield* jobs.get(String(process?.pipe?.metadata.id)))?.status).not.toBe("running") + }), + ) + }), + ) + + daemonLive("stops a command the user stopped after its Location reloaded", () => + withTempDir((tmp) => + Effect.gen(function* () { + reset() + const sessions = yield* Session.Service + const locations = yield* LocationServiceMap.Service + const location = Location.Ref.make({ directory: AbsolutePath.make(tmp.path) }) + yield* sessions.create({ id: sessionID, title: "shell test", location, model: sessionModel }) + yield* Effect.gen(function* () { + const plugins = yield* Plugin.Service + yield* plugins.awaitActivation + const registry = yield* Tool.Service + const jobs = yield* Job.Service + const cancel = SessionStep.makeCancel() + const started = yield* Deferred.make() + const fiber = yield* executeTool(registry, { + ...call({ command: "sleep 60" }, "call-reloaded"), + progress: (update) => + typeof update.shellID === "string" + ? Deferred.succeed(started, update.shellID).pipe(Effect.asVoid) + : Effect.void, + cancel: cancel.effect, + }).pipe(Effect.forkChild({ startImmediately: true })) + const shellID = yield* Deferred.await(started) + // The call still holds the old Location when a reload closes it. + yield* locations.invalidate(location) + expect(cancel.trigger()).toBe(true) + const result = yield* Fiber.join(fiber) + expect(result.error?.message).toContain(`Command interrupted by the user (shell ID: ${shellID})`) + expect(fake?.processes[0]?.lifecycle.status).toBe("exited") + expect((yield* jobs.get(shellID))?.status).not.toBe("running") + }).pipe(Effect.provide(locations.get(location))) + yield* locations.invalidate(location) + }), + ), + ) + + for (const handoff of [false, true]) + for (const starting of [true, false]) + daemonLive( + `${handoff ? "leaves a command running" : "stops a command"} when an interrupt ${handoff ? "during a handoff " : ""}cuts its call off while it ${starting ? "starts" : "runs"}`, + () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const jobs = yield* Job.Service + const pty = yield* PersistentPty.Service + const call = yield* startSlowCall(registry, "call-interrupted") + if (handoff) expect(yield* pty.handoff()).not.toBeNull() + if (!starting) { + yield* call.release + while (fake!.waiters(fake!.processes[0]?.id ?? -1) === 0) yield* Effect.sleep(2) + } + const interrupting = yield* Fiber.interrupt(call.fiber).pipe( + Effect.forkChild({ startImmediately: true }), + ) + if (starting) yield* call.release + yield* Fiber.join(interrupting) + expect(Exit.hasInterrupts(yield* Fiber.await(call.fiber))).toBe(true) + const process = fake?.processes[0] + const shellID = String(process?.pipe?.metadata.id) + if (handoff) { + // Nothing stopped it: the next server takes it over through its recorded tool call. + expect(process?.lifecycle.status).toBe("running") + expect(process?.pipe?.metadata).toMatchObject({ toolCall: { callID: "call-interrupted" } }) + expect(fake?.ops).not.toContain("signal") + expect((yield* jobs.get(shellID))?.status).toBe("running") + yield* jobs.cancel(shellID) + return + } + // Nothing will take it over, so it is stopped and its job ends. + const shell = yield* Shell.Service + expect((yield* shell.wait(ID.make(shellID))).status).toBe("killed") + expect(fake?.requests).toContainEqual( + expect.objectContaining({ op: "signal", id: process?.id, signal: "SIGTERM" }), + ) + expect(process?.lifecycle.status).toBe("exited") + expect((yield* jobs.get(shellID))?.status).toBe("cancelled") + }), + ) + }), + ) + + daemonLive("records which call waits on a daemon command, and keeps no timeout for a restart", () => + withTempDir((tmp) => { + reset() + return withSession(tmp.path, (registry) => + Effect.gen(function* () { + const { shellID } = yield* stopCall(registry, "sleep 60", "call-recorded", "interrupt", false) + const request = fake?.requests.find((item) => item.op === "create_pipe") + expect(request?.metadata).toMatchObject({ + id: shellID, + toolCall: { sessionID, callID: "call-recorded" }, + }) + // The call's timeout applies to this server only: a restart moves the command to the background. + expect(request?.metadata).not.toHaveProperty("timeoutDeadline") + }), + ) + }), + ) + test("lets an execute.after hook replace a stopped call's failure", () => withTempDir((tmp) => { reset() @@ -2209,18 +2493,17 @@ describe("ShellTool stopped", () => { ) })) - test("leaves a call a shutdown interrupted to the step", () => + test("stops a command whose call an interrupt cut off, with no handoff to take it over", () => withTempDir((tmp) => { reset() return withSession(tmp.path, (registry) => Effect.gen(function* () { const shell = yield* Shell.Service + const jobs = yield* Job.Service const { shellID, exit } = yield* stopCall(registry, "printf ready; sleep 60", "call-teardown", "interrupt") expect(Exit.hasInterrupts(exit)).toBe(true) - // The job is cancelled as before, which stops the command; its file stays. - const info = yield* shell.wait(shellID) - expect(info.status).toBe("killed") - expect(yield* Effect.promise(() => fs.readFile(info.file, "utf8"))).toBe("ready") + expect((yield* shell.wait(shellID)).status).toBe("killed") + expect((yield* jobs.get(shellID))?.status).toBe("cancelled") }), ) })) diff --git a/packages/server/src/routes.ts b/packages/server/src/routes.ts index d987cd479716..5cf2d3ffa6f4 100644 --- a/packages/server/src/routes.ts +++ b/packages/server/src/routes.ts @@ -20,6 +20,7 @@ import { Instance } from "@opencode/core/instance/service" import { SessionTransfer } from "@opencode/core/session/transfer" import { InstructionEntry } from "@opencode/core/session/instruction-entry" import { ShellSelect } from "@opencode/core/shell/select" +import { Shell } from "@opencode/core/shell" import { Job } from "@opencode/core/job" import { KV } from "@opencode/core/kv" import { Mcp } from "@opencode/core/mcp/index" @@ -60,6 +61,11 @@ const applicationServiceNodes = [ Database.node, KV.node, Bus.node, + // Before the jobs and sessions, so they are released after them: a shutdown's interrupted shell calls and jobs + // still reach the daemon to stop their commands, and a command being created never falls back to spawning. + PersistentPty.node, + // Daemon-run shell commands belong to the server, so a Location closing never takes them down. + Shell.daemonNode, EventLogger.node, httpClient, Job.node, @@ -74,7 +80,6 @@ const applicationServiceNodes = [ PluginUpdate.node, PermissionSaved.node, PtyTicket.node, - PersistentPty.node, Credential.node, WellKnown.node, ServerPairing.node,