diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index a8a254f..9a59ef9 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -74,6 +74,18 @@ for the protocol, `internal/cli/reverse_forward.go` for the host end). The daemon pushes set changes down the same channel, so those edits apply to a running guest. vz only — firecracker's vsock is one-way. +Serial ports (`clawk serial add`) ride the same asymmetry and reuse its whole +shape: the guest agent creates a PTY per configured device, and holding that +PTY open is what makes the agent dial the daemon, which validates the device +name against the configured set and opens the physical tty (`internal/ +serialfwd` for the protocol, `internal/cli/serial_proxy.go` for the host end, +`internal/serialport` for the tty itself). Passing the USB device through +instead is not an option on either provider — Virtualization.framework +exposes no physical USB passthrough and firecracker has no USB bus — and it +isn't needed: the tooling wants a tty and a baud rate, not a USB endpoint. +Tying the host-side open to the guest-side open also reproduces the DTR edge +that resets an Arduino at the moment an upload expects it. + Live allow-list edits reach the running daemon over a control socket (`internal/vzdctl`); when the sandbox is down they apply on the next `up`. The same socket carries the VM lifecycle verbs: `clawk pause` / `resume` @@ -105,6 +117,7 @@ because it pins a vendored `gvisor-tap-vsock` fork; everything clawk-specific | `internal/agentembed` | The in-guest binaries (clawk-init, pty-agent, time-sync), cross-compiled and injected into the rootfs. | | `internal/vsockproto` / `internal/vsockclient` | The host↔guest vsock framing and the host-side client. | | `internal/revfwd` | Reverse-forward wire protocol (host loopback services exposed on the guest's loopback), mirrored in the guest agent. | +| `internal/serialfwd` / `internal/serialport` | Serial-forwarding wire protocol (mirrored in the guest agent) and the host-side tty open/termios. | | `internal/netfilter` | Egress allow-list (IPs/CIDRs/domains, DNS-aware) consumed by gvproxy. | | `internal/vzdctl` | Daemon control socket (live policy edits, denial ledger, VM pause/resume/suspend). | | `internal/worktree` / `internal/pr` | Multi-repo branch coordination and PR creation. | diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ae0f22..bfe80c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,224 @@ tagged. ## Unreleased +## v0.4.0 + +### Added + +- **Host-wide defaults: `~/.config/clawk/clawk.mod`.** One file outside any + repo whose anonymous `sandbox ( … )` block supplies settings for every + sandbox on the machine — including repos with no `clawk.mod` at all. It is + the lowest layer of the chain (built-in defaults < host-wide < namespace < + repo `clawk.mod` < `clawk.mod.` < flags): lists union with the + host-wide entries first, scalars apply only where nothing narrower declared + one. Read at create time like every other template, so editing it shapes the + next sandbox rather than retro-modifying existing ones. + ([#14](https://github.com/clawkwork/clawk/issues/14)) + + The point is that a kernel path, a token alias, a personal skill mount and a + house rule about commit messages are properties of the host, not of the repo + they happened to be written in — and committing them puts an absolute + `/Users/you/...` path in a shared file. + + Location resolves `$CLAWK_GLOBAL_MOD` (explicit, must exist) → + `$XDG_CONFIG_HOME/clawk/clawk.mod` → `~/.clawk/clawk.mod` (compatibility). + `~/.config` is the documented home because this is the one file in clawk's + footprint you hand-edit and may symlink out of a dotfiles repo, whereas + `~/.clawk` is disposable machine state — VM disks, an image cache, a live + OAuth token — that people exclude from backups and delete to start clean. + Both present is an error, not a silent pick. + + Scope is enforced: the block must be anonymous (a header name labels one + repo's phases, so a name here would silently relabel every repo), `includes` + and the unwired `on down` / `on enter` are rejected, `policy` blocks are + accepted as a personal policy library, and `namespace` blocks are refused — + the file declares defaults for every sandbox, not named resources, and clawk + owns those records itself. Relative paths resolve against the file's own + directory. `--no-global` drops the layer for a reproducible run. + +- **Serial devices.** `clawk serial add /dev/cu.usbmodem1101` puts a + serial port plugged into your Mac inside the sandbox as `/dev/`, so + `arduino-cli`, `esptool`, `avrdude` and a serial monitor can run in there + against real hardware. Declarable in `clawk.mod` as `serial ( … )`, applied + live like reverse forwards, and vz-only for the same reason. + + The USB device is not passed through, because nothing clawk runs on can do + that: Virtualization.framework's USB controller carries virtual + mass-storage devices only, and firecracker has no USB at all. What crosses + is the byte stream and the line settings — which is all any of those tools + ever wanted from a serial port. The guest side is a PTY; the host side is + the real tty, opened only while a process in the sandbox holds the device, + so the board stays available to the Mac the rest of the time. + + Tying the host's open to the guest's open is also what makes auto-reset + work: opening a serial port asserts DTR, which is the pulse a board's reset + circuit is waiting for. A PTY carries the baud rate too, so the 1200-baud + touch that native-USB boards use to enter their bootloader survives the + trip. What a PTY cannot carry is a modem-control line under direct control + — a plain ESP32 DevKit that needs DTR and RTS driven in sequence still + needs its BOOT button. [docs/serial.md](docs/serial.md) has the full table. + + A device may be named by a glob (`'/dev/cu.usbmodem*:ttyACM0'`), resolved + each time the port is opened rather than when it is configured, so a board + that re-enumerates into its bootloader under a neighbouring name stays + reachable. The guest-side name never changes, so `-p /dev/ttyACM0` keeps + working across the whole flash cycle. + +- **Sandboxes now boot with swap.** Every sandbox gets a swap device — 2 GiB + by default, sized with `vm ( swap )` and disabled with + `vm ( swap off )`. + + The reason is the balloon controller, not the guest's appetite. Under host + memory pressure clawk takes RAM back from a guest against its demand (75% + of the ceiling at WARN, 50% at CRITICAL), and a guest with nowhere to put + cold anonymous pages answers that with direct-reclaim stalls and, at the + limit, its OOM killer. A multi-second stall in the agent process is not + merely slow: a process that stops draining its socket lets the connection + go idle, and on a link whose NAT reaps idle mappings in well under a minute + — a phone hotspot, a hotel network — that ends the streaming API response + outright. Swap turns the stall into paging. + + It rides on its own sparse virtio-blk device rather than a swapfile on the + rootfs, because `swapon(2)` rejects a file with holes: a swapfile would + cost its full size in real host bytes at first boot, per sandbox. The + device costs only the pages actually swapped. Note that nothing punches + those holes back — no virtio-blk in the stack advertises discard yet — so + read the usage as a high-water mark until the sandbox is destroyed. + + The balloon controller learned about swap at the same time, and had to: + paging out cold pages raises `MemAvailable` and lowers PSI, so a guest that + answered a squeeze by swapping reads as *roomy* on both of the signals the + controller uses. Left alone it would have reclaimed another step, and + parked working guests at their baseline, swapping, for good. + +- **`mcp ( … )`: MCP servers ready on first boot.** Declare the servers a + project needs and every sandbox created from that `clawk.mod` comes up with + them configured — no per-sandbox setup, no interactive login inside the VM: + + ```text + mcp ( + linear https://mcp.linear.app/mcp header "Authorization: Bearer ${LINEAR_TOKEN}" + github stdio "npx -y @modelcontextprotocol/server-github" env GITHUB_TOKEN + ) + ``` + + Declaring a server also allows its host in the egress policy, in a derived + `mcp` layer that ranks below anything you wrote yourself — so a remote + server doesn't need a matching `network allow`, and doesn't get to + override your own `deny`. Valid in a repo file, a workspace root and a + `namespace` block, merging by name with the narrowest scope winning, so a + namespace can carry the org-wide set. + + Credential values stay out of clawk entirely: `clawk.mod` and the rendered + guest config hold a `${VAR}` reference, and the value travels from your + shell to the runner's process environment at attach time. Pair it with + `env ( TOKEN = ${TOKEN:?message} )` and a missing PAT fails sandbox + creation with your message instead of surfacing as a 401 mid-task. Static + credentials only — that's what can be in place before the VM boots. See + [docs/mcp.md](docs/mcp.md). + +- **The pi coding agent is a built-in runner.** `clawk run pi` attaches + [pi](https://github.com/earendil-works/pi) the same way `clawk run claude` + and `clawk run codex` do, and the `clawk-dev` image ships it. `pi.dev` (its + model catalog and version check) joins the default network allow-list, and + its `~/.pi/` is persisted per sandbox like the other runners'. + + The image also gains `fd-find`. pi's search tools probe PATH for `rg` and + `fd`/`fdfind` and download GitHub release tarballs when they're missing — + the image had ripgrep but not fd, so every fresh sandbox printed "fd not + found. Downloading..." on pi's first run. Debian ships the binary as + `fdfind`, one of the names pi probes for, so the package alone settles it. + +- **opencode is a fully wired runner.** It was in the registry but nothing + else: not installed in `clawk-dev`, and not persisted. Both are fixed, so + `clawk run opencode` works out of the box and its sessions and login + survive `down`/`up` and `destroy` like every other runner's. + + It launches with `--auto` (approve anything not explicitly denied); a deny + rule in your `opencode.jsonc` still wins, and `--safe` drops the flag. + `opencode.ai` joins the default network allow-list, alongside the + `models.dev` entry it already had. + + opencode is the one runner needing two state mounts: it follows the XDG + split rather than keeping a single home, so `~/.local/share/opencode` + (auth.json, mcp-auth.json, opencode.db, repos/) and `~/.config/opencode` + are mounted separately. Its `~/.local/state/opencode` and + `~/.cache/opencode` deliberately stay on the disposable rootfs — the + former holds only lock files, and a lock outliving the VM that took it is + worse than none. + + Note the image grows by roughly 180 MB: `opencode-ai` resolves a prebuilt + platform binary rather than shipping JS. Drop it from the Dockerfile's npm + line for a slimmer variant. + + pi launches with `--approve`. It has no approval prompts to bypass — it + ships no built-in sandbox at all, by design, on the grounds that real + isolation has to come from a VM — but it does gate project-local + `.pi/settings.json` and `.pi/extensions` behind an interactive trust + prompt, and answering that inside a sandbox whose whole premise is that + the project already has the machine is theatre. `--safe` drops the flag + like it does for the other runners. A sandbox declaring `mcp ( … )` gets + no config flag for pi: pi loads MCP through an extension rather than a + config file. + +### Fixed + +- Workspace-position `env ( … )` and `agent ( … )` blocks were silently + dropped: only a repo's own `clawk.mod` fed the sandbox's required-env set and + agent instructions, so a workspace root declaring `env ( GITHUB_TOKEN )` got + nothing. They now apply, ordered scope-outward (workspace, then namespace, + then repo). + +- **An `env ( … )` declaration now overrides clawk's own variables.** The + vsock handshake emitted clawk's vars — `CLAUDE_CODE_OAUTH_TOKEN`, + `COLORTERM`, the terminal-identification set — *after* the sandbox's + declared env, so they won on the last-write-wins fold in the guest agent + and a `clawk.mod` declaration of the same name was silently discarded. + + The two other delivery paths already disagreed with it: `/etc/profile.d` + sources `99-clawk-env.sh` after `98-clawk-claude-oauth.sh`, so login + shells and the `bash -lc` exec fallback have always let the declaration + win. Only the default path — the one `clawk run claude` uses — did not. + + The case that needs it: a sandbox with `ANTHROPIC_BASE_URL` pointed at a + gateway that needs no credential of its own. Claude Code falls back to + `CLAUDE_CODE_OAUTH_TOKEN` when `ANTHROPIC_AUTH_TOKEN` is unset, so clawk's + `sk-ant-oat-…` was sent to that third-party endpoint as an + `Authorization: Bearer` header — a token leak, and one no clawk.mod could + prevent. `env ( CLAUDE_CODE_OAUTH_TOKEN = "" )` now expresses it per + sandbox; previously the only lever was `clawk auth clear`, which disarms + every sandbox on the host. clawk still supplies all of these when the + sandbox says nothing about them. + +- **Codex no longer loses its history on `clawk down && clawk up`.** Only + `~/.claude` was mounted from the host; `~/.codex` lived on the VM rootfs, + which vz re-clones from the image master on *every* boot. So codex's + sessions, `history.jsonl` and login were discarded by a plain stop/start — + not just by `clawk destroy` — even though the docs promised + `state//codex/` was mounted at `~/.codex/`. Each runner's home + directory is now a per-sandbox host mount under + `~/.clawk/namespaces//state//` — `~/.claude`, `~/.codex`, + `~/.pi`, and opencode's two XDG directories — so `--resume` works across + stops and recreates for all of them. + + Existing sandboxes pick this up on their next boot; history from before + the fix was on the disposable disk and is not recoverable. Note that this + costs four more virtio devices on every sandbox against the 32-device + PCIe ceiling: a large multi-repo workspace that booted on v0.3.0 may now + be refused at start, with a message naming the count and what to cut. + +- **`env ( … )` vars now reach the agent process, not just its shells.** The + pty agent spawns the runner directly, so `/etc/profile.d/99-clawk-env.sh` + was never sourced for it: declared variables were visible to a login shell + the agent started, but absent from the agent's own environment. Anything + reading it saw nothing — including `${VAR}` expansion in MCP config, where + a forwarded token would silently become an empty header. The values now + ride the vsock handshake alongside `CLAUDE_CODE_OAUTH_TOKEN`, resolved from + your shell on every dispatch (so rotating a token needs no VM cycle), and + layered so that clawk's own vars fill in only the names your `clawk.mod` + didn't claim (see the entry above). As a side effect the agent's shell tool + sees `GITHUB_TOKEN` and friends without `bash -lc`. + ## v0.3.0 ### Added diff --git a/DESIGN.md b/DESIGN.md index 0fed5ee..634e8d3 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -174,9 +174,13 @@ Everything lives under `~/.clawk/`, namespace-first: `suspend/` directory holding the saved memory + device state, consumed one-shot by the next boot and discarded by `clawk down`. Wiped by `destroy`. -- `namespaces//state//` — agent state mounted into the guest - (Claude projects/memory, Codex state). **Survives `destroy`** — recreating - a sandbox restores history. +- `namespaces//state//` — agent state mounted into the guest, one + subdirectory per runner home (`claude/` → `~/.claude`, `codex/` → + `~/.codex`, `pi/` → `~/.pi`, and `opencode-data/` + `opencode-config/` → + opencode's two XDG dirs). **Survives `destroy`** — recreating a sandbox restores + history. It also survives `down`/`up`, which the rootfs does not: vz + re-clones `disk.raw` from the image master on every boot, so a runner + whose home isn't mounted here starts fresh each time the VM comes up. - `namespaces//worktrees//` — git worktrees for ticket-mode sandboxes. - `cache/` — built rootfs disks (CoW masters) and kernels, shared across @@ -210,6 +214,12 @@ lexer + recursive-descent parser in the go.mod two-form style - **Egress:** default-deny beyond a built-in allow-list of common registries + the configured domains/IPs, enforced in the userspace stack the guest can't reconfigure. +- **Host devices:** a serial port is reachable only if the user attached it, + and the guest names a *device*, never a host path — the mapping to + `/dev/cu.…` stays on the host, which validates every attach against the + configured set. The same shape as reverse forwarding, and deliberately not + a general "run something on the host" channel: the host picks the resource, + the guest only picks the bytes. - **Host credentials:** the ssh-agent is *forwarded* (keys stay on the host); the Claude OAuth token and any `files ( … )` secrets are pushed in deliberately and are the user's explicit choice. diff --git a/README.md b/README.md index 0795a9a..ce1354d 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ prompt every few seconds), or you run `--dangerously-skip-permissions` and hope nothing important is one `rm -rf` or one leaked token away. clawk is a third option. `cd` into a repo, type `clawk`, and Claude Code (or -Codex, or a shell) is working inside a disposable Linux VM (your code mounted +Codex, or pi, or a shell) is working inside a disposable Linux VM (your code mounted in, root in the guest, no permission prompts) while your files, your keychain, and the rest of your machine stay out of reach. **The agent gets its own machine instead of yours.** @@ -156,7 +156,7 @@ The everyday case, a sandbox for the directory you're in: cd ~/code/my-project clawk # boot a sandbox for this dir + attach claude clawk run shell # drop into a shell in the same sandbox -clawk run codex # or another agent: codex, opencode, shell +clawk run codex # or another agent: codex, pi, opencode, shell clawk down # stop the VM (repo + agent state persist) clawk attach # come back later — boots if stopped, reattaches claude clawk destroy # remove the VM (conversation history is kept) @@ -196,7 +196,7 @@ lives on the host.* | | `clawk down` | `clawk destroy` | | --- | :---: | :---: | | Your repo (mounted worktree; commits, branches) | ✅ | ✅ | -| Agent state (Claude/Codex conversations, memory) | ✅ | ✅ | +| Agent state (Claude/Codex/pi/opencode conversations, memory) | ✅ | ✅ | | The VM disk (apt installs, caches, `$HOME`) | ❌ (rebuilt fresh at every boot*) | ❌ (that's the point) | \* Two exceptions: resuming a `clawk snapshot` restores the disk and @@ -204,16 +204,23 @@ memory exactly as suspended, and the Linux/firecracker provider keeps its disk until destroy. Tools every boot needs belong in the image (`vm ( image … )`); per-boot setup belongs in `on up` hooks. -Agent state is host-mounted per sandbox: the guest's `~/.claude/projects/` -and `~/.claude/memory/` (and codex's `~/.codex/`) live under +Agent state is host-mounted per sandbox: each runner's home directory — +claude's `~/.claude/`, codex's `~/.codex/`, pi's `~/.pi/`, opencode's two XDG +dirs — live under `~/.clawk/namespaces/default/state//` on the host, so a recreated -sandbox picks up its old conversations with `--resume`. +sandbox picks up its old conversations with `--resume`. That mount is what +makes the promise real: the VM disk itself is re-cloned from the image on +every boot, so anything a runner writes outside those directories is gone +at the next `clawk up`. ## Full autonomy by default (and the `--safe` opt-out) Runners launch in their "externally sandboxed" modes: claude gets `--dangerously-skip-permissions`, codex gets -`--dangerously-bypass-approvals-and-sandbox`. On your own machine those flags +`--dangerously-bypass-approvals-and-sandbox`, pi gets `--approve` (it has no +approval prompts to bypass — it ships no sandbox at all — but it does gate +project-local `.pi/` settings and extensions behind a trust prompt), and +opencode gets `--auto`. On your own machine those flags would be reckless; here they are the point: the VM boundary and the network allow-list provide the containment, so the agent works at full speed without per-action prompts. The agent can only affect what you mounted and @@ -261,6 +268,9 @@ sandbox my-project ( forwards ( 3000 ) env ( DATABASE_URL ) # forward a host var; values come from your shell # also: GH=${OTHER_NAME}, LOG=${LOG:-info} defaults, API=${API:?required} + mcp ( # MCP servers, ready on first boot + linear https://mcp.linear.app/mcp header "Authorization: Bearer ${LINEAR_TOKEN}" + ) on create ( "go mod download" ) agent ( instructions "Ask before running destructive commands." @@ -271,8 +281,11 @@ sandbox my-project ( The block is a *template*: snapshotted when the sandbox is created, so a running sandbox never changes unexpectedly. The full reference (shares, secret files, skills, agent memory seeding, multi-repo workspace roots) is -in **[docs/configuration.md](docs/configuration.md)**; images and custom -guest kernels (including the KVM-enabled kernel used for nested +in **[docs/configuration.md](docs/configuration.md)**; MCP servers and how +their credentials stay off disk are in **[docs/mcp.md](docs/mcp.md)**; +putting a USB-serial board from your Mac inside the sandbox for +microcontroller work is in **[docs/serial.md](docs/serial.md)**; images +and custom guest kernels (including the KVM-enabled kernel used for nested virtualization) are in **[docs/images.md](docs/images.md)**. ## Lifecycle @@ -304,7 +317,7 @@ you ──▶ clawk CLI ──▶ per-sandbox daemon (detached; owns the VM) └─ VM: Virtualization.framework (macOS) / firecracker (Linux) ├─ clawk-init, PID 1 (no systemd, no cloud-init) ├─ your repo, live-mounted over virtio-fs - └─ claude / codex / shell on a PTY + └─ claude / codex / pi / shell on a PTY ``` A few deliberate choices, in brief: diff --git a/docs/commands.md b/docs/commands.md index 06e2d90..a7260a6 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -13,6 +13,10 @@ clawk resume [] # continue a paused or snapshotted sandbo clawk down [] # stop; discards any snapshot (next up is a cold boot) clawk destroy [] # remove (host-side state persists) +clawk serial add # a host serial port, inside the guest +clawk serial list [--json] # forwarded ports + whether they're present +clawk serial remove # stop forwarding one + clawk system info [--json] # host prereqs + active components clawk system df [--json] # disk usage by sandbox / cache clawk system prune [--image] # reap unreferenced OCI rootfs disks @@ -41,8 +45,8 @@ field is optional. ## Runners -Built-in runners: `claude`, `codex`, `opencode`, `shell`. The dispatch -shape is the same for all four: +Built-in runners: `claude`, `codex`, `pi`, `opencode`, `shell`. The dispatch +shape is the same for all five: ```sh clawk run [] [-- ] @@ -55,21 +59,36 @@ clawk run claude # cwd-sandbox clawk run claude foo # named sandbox clawk run claude -- --resume # pass-through args clawk run codex foo -- --model o4 +clawk run pi foo -- --resume # pi's own session picker clawk run shell foo # interactive bash ``` +The `clawk-dev` default image ships all four agents, so every name above works +out of the box. A runner an alternative image doesn't have fails with a plain +"command not found". + Each attach starts a fresh agent process in the guest and ends when you -disconnect; claude and codex resume from their own on-disk state next time, so +disconnect; every runner resumes from its own on-disk state next time, so detaching and reattaching is cheap. -State that should outlive the VM is kept on the host: +State that should outlive the VM is kept on the host — one directory per +runner, mounted over the guest's home: + +| Path on host (default namespace) | Mounted as | +|-------------------------------------------------------------|-----------------------------| +| `~/.clawk/namespaces/default/state//claude/` | `~/.claude/` | +| `~/.clawk/namespaces/default/state//codex/` | `~/.codex/` | +| `~/.clawk/namespaces/default/state//pi/` | `~/.pi/` | +| `~/.clawk/namespaces/default/state//opencode-data/` | `~/.local/share/opencode/` | +| `~/.clawk/namespaces/default/state//opencode-config/` | `~/.config/opencode/` | -| Path on host (default namespace) | Mounted as | -|---------------------------------------------------------------|-----------------------| -| `~/.clawk/namespaces/default/state//claude/projects/` | `~/.claude/projects/` | -| `~/.clawk/namespaces/default/state//claude/memory/` | `~/.claude/memory/` | -| `~/.clawk/namespaces/default/state//codex/` | `~/.codex/` | +opencode needs two because it follows the XDG split rather than keeping one +home directory. Its `~/.local/state/opencode` (locks) and `~/.cache/opencode` +are deliberately left on the disposable rootfs. +This is the only thing that persists a runner's sessions: the VM disk is +re-cloned from the image on every boot, so a runner writing anywhere else +loses its history at the next `clawk up`, not just at `clawk destroy`. `clawk destroy` wipes the VM disk but not the state directory, so a recreate returns the same conversation history. diff --git a/docs/configuration.md b/docs/configuration.md index 524a44e..92bf672 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -36,6 +36,7 @@ sandbox my-project ( memory 4GiB memory_max 8GiB disk 64GiB + swap 8GiB nested image golang:1.25 ) @@ -65,6 +66,16 @@ sandbox my-project ( ~/.terraform.d rw ) + serial ( + /dev/cu.usbmodem* ttyACM0 # an Arduino; the glob survives a re-enumeration + ) + + mcp ( + linear https://mcp.linear.app/mcp header "Authorization: Bearer ${LINEAR_TOKEN}" + sentry sse https://mcp.sentry.dev/sse + github stdio "npx -y @modelcontextprotocol/server-github" env GITHUB_TOKEN + ) + skills ( # A manifest of distributed skills for `clawk mod tidy` to pin. # Fetching them into the guest is not implemented yet — provision a @@ -102,7 +113,8 @@ sandbox my-project ( - `sandbox ( … )` — the header names the template (defaults to the directory when omitted: `sandbox ( … )`). - `vm ( … )` — runtime shape: `provider`, `cpu`, `memory`, `memory_max`, - `disk`, `nested`, `idle_timeout`, `image`, `kernel`. Memory and `disk` + `disk`, `swap`, `nested`, `idle_timeout`, `image`, `kernel`. Memory, + `disk` and `swap` sizes require an explicit unit, case-sensitive: IEC (`MiB`/`GiB`/`TiB`, shorthands `M`/`G`/`T`) or SI (`MB`/`GB`/`TB`); SI values convert to MiB rounding down (`1GB` → 953 MiB). `disk` sets the root filesystem ceiling @@ -112,8 +124,16 @@ sandbox my-project ( written up front. Raise it for repos with large dependency trees. Like `cpu` and `memory`, the value is snapshotted when the sandbox is created and baked into the rootfs, so editing it affects the next sandbox (or the - next rootfs rebuild), not a running one. See [Images](images.md) for - `image` and `kernel`, and + next rootfs rebuild), not a running one. `swap` sizes the guest's swap + device (default 2 GiB, minimum 64 MiB, `swap off` to disable). Every + sandbox gets one because clawk reclaims guest RAM under host memory + pressure, and a guest with nowhere to page out answers that with + multi-second stalls — long enough for a streaming API response to lose its + connection. It's a separate sparse file rather than part of the rootfs, so + it costs host bytes only as the guest actually swaps, and resizing it takes + effect on the next boot rather than on a rootfs rebuild. Across a workspace + the largest `swap` wins, but a single `swap off` anywhere disables it for + the whole VM. See [Images](images.md) for `image` and `kernel`, and [Commands & resource usage](commands.md#resource-usage) for `idle_timeout`. - `network ( … )` — egress policy: `allow` / `deny` a domain or `ip `, plus `use …` chains — see @@ -143,6 +163,28 @@ sandbox my-project ( shell-variable shaped (letters, digits, `_`; not starting with a digit) — lowercase names like `http_proxy` are fine. Whitespace around `=` is optional. + + A declaration here **overrides** any variable clawk would otherwise set + for the runner itself, including `CLAUDE_CODE_OAUTH_TOKEN`. That is how a + sandbox opts out of clawk's Anthropic credential when it points claude at + something else: + + ```text + env ( + ANTHROPIC_BASE_URL = https://gateway.example + CLAUDE_CODE_OAUTH_TOKEN = "" # don't send Anthropic's token there + ) + ``` + + This matters when the gateway needs no credential of its own, or + authenticates some other way. Claude Code's credential order is + `ANTHROPIC_AUTH_TOKEN` first, then `CLAUDE_CODE_OAUTH_TOKEN`, and an + empty value counts as unset — so a sandbox that *does* set + `ANTHROPIC_AUTH_TOKEN` already sends the gateway key regardless. Without + one, clawk's `sk-ant-oat-…` goes to that third-party endpoint as a + `Authorization: Bearer` credential, which is a token leak rather than a + broken request. The blunt alternative, `clawk auth clear`, disarms every + sandbox on the host. - `on create ( … )` / `on up ( … )` — shell hooks. `create` runs once after the first boot; `up` runs on every boot. Each command runs inside the guest via `bash -lc` as a login shell: variable expansion, globs, and @@ -156,6 +198,20 @@ sandbox my-project ( configs that rotate rarely). - `shares ( … )` — host directories live-mounted via virtio-fs (good for rotating secrets like AWS STS tokens). +- `serial ( … )` — host serial ports presented as devices in the guest, for + microcontroller work. Each line is ` []`; the name + defaults to the host device's basename, and the host device may be a glob + (resolved when the port is opened, which is how a board that re-enumerates + into its bootloader stays reachable). vz only, and applied live. See + [Serial devices](serial.md). +- `mcp ( … )` — MCP servers made available to the agent, ready on first + boot. See [MCP servers](mcp.md). Each line is + ` ` (http), ` http|sse `, or + ` stdio ""`, followed by any number of + `header "Name: value"` (http/sse) or `env NAME` (stdio) modifiers. + Declaring a server also allows its host in the egress policy, so it needs + no matching `network allow` line. Credentials are referenced, never + stored: write `${VAR}` and declare `VAR` in `env ( … )`. - `skills ( … )` — a manifest of **distributed** Claude skills (`/…` pinned to a version), maintained with `clawk mod tidy`. **Fetching skills into the guest is not implemented yet** — until it @@ -198,6 +254,81 @@ tied to any single repo's directory. A repo listed in `includes` keeps its own per-worktree `on up` / `on create`; the two scopes are independent and both run. +## Host-wide defaults + +Some settings are properties of *you*, not of the repo: a kernel path, a token +alias, a personal skill mount, a house rule about commit messages. Put those in +one file outside any repo and every sandbox on the machine picks them up — +including repos with no `clawk.mod` at all. + +```text +# ~/.config/clawk/clawk.mod — never committed, one per host +sandbox ( + vm ( + memory_max 8GiB + ) + network ( + allow *.internal.myco.com + ) + env ( + GITHUB_TOKEN = ${MYCO_GH_TOKEN} + ) + shares ( + ~/.claude/skills/idiomatic-go + ) + agent ( + instructions ./house-rules.md + ) + on up ( + "scripts/ensure-swap.sh" + ) +) +``` + +It is the lowest layer of the chain, narrowest wins: + +```text +built-in defaults < ~/.config/clawk/clawk.mod < namespace < repo clawk.mod + < clawk.mod. < flags +``` + +Lists union with the host-wide entries first (so a conflict message reads +scope-outward, and `on up` hooks from here run before a repo's). Scalars — +`provider`, `image`, `kernel`, `cpu`, `memory`, `disk`, `swap`, `idle_timeout` — +apply only where nothing narrower declared one. Like every other template it is +read at **sandbox-create time**, so editing it changes the next sandbox you +create, never one that already exists. + +The block must be **anonymous** (`sandbox ( … )`): a header name labels one +repo's phases, which means nothing for defaults, so a name here is an error +rather than a silent relabel of every repo on the host. `includes` is rejected +for the same reason. `policy ( … )` blocks are welcome — that's a +personal policy library. `namespace` blocks are not: this file declares defaults +for every sandbox, not named resources, and clawk owns those records itself. + +Relative paths (`./house-rules.md`, `./kernels/vmlinux`) resolve against the +file's own directory, so they keep working from any repo. + +**Where it lives**, in resolution order: + +| location | notes | +|---|---| +| `$CLAWK_GLOBAL_MOD` | explicit override; must exist, or it's an error | +| `$XDG_CONFIG_HOME/clawk/clawk.mod` | the documented home, default `~/.config/clawk/clawk.mod` | +| `~/.clawk/clawk.mod` | compatibility fallback | + +`~/.config` rather than `~/.clawk` because this is the one file in clawk's +footprint you hand-edit and might symlink out of a dotfiles repo: `~/.clawk` is +disposable machine state (VM disks, an image cache, per-sandbox records, a live +OAuth token) that people exclude from backups and delete to start clean. Having +both locations present is an error, never a silent precedence pick. + +**`--no-global`** ignores the layer entirely — for a CI run or a bug report +that should depend only on what's in the repo. `CLAWK_GLOBAL_MOD=/path/to/file` +does the opposite: pins the layer regardless of what the host has in +`~/.config`. When the layer applies, `clawk work` names the file it came from, +since it explains configuration nobody can find by reading the repo. + ## How clawk finds the file `clawk` in a repo uses that repo's own `clawk.mod` (beside its `.git`), diff --git a/docs/images.md b/docs/images.md index 0de9be2..a84c681 100644 --- a/docs/images.md +++ b/docs/images.md @@ -13,9 +13,9 @@ clawk image gc [--dry-run] [--layers] # reclaim disks no sandbox needs `clawk-dev` (`ghcr.io/clawkwork/clawk-dev`) bundles `go`, `node` + `pnpm`, `python3` + `uv`, `rustc` + `cargo`, `bun`, `zig`, plus `git`, `gh`, `jq`, -`ripgrep`, `claude`, and `codex`. The rootfs is rebuilt from the image each -boot, so bake system dependencies into the image and use `on up` for per-boot -setup. +`ripgrep`, `fd-find`, `claude`, `codex`, `pi`, and `opencode`. The rootfs is +rebuilt from the image each boot, so bake system dependencies into the image +and use `on up` for per-boot setup. A sandbox records the image *reference* it was created with, and the reference is re-resolved against the registry each time the rootfs is diff --git a/docs/mcp.md b/docs/mcp.md new file mode 100644 index 0000000..0f17a3b --- /dev/null +++ b/docs/mcp.md @@ -0,0 +1,127 @@ +# MCP servers + +Declare the MCP servers a project needs and every sandbox created from that +`clawk.mod` comes up with them configured — no per-sandbox setup step, no +interactive login inside the VM. + +```text +sandbox my-project ( + mcp ( + linear https://mcp.linear.app/mcp header "Authorization: Bearer ${LINEAR_TOKEN}" + github stdio "npx -y @modelcontextprotocol/server-github" env GITHUB_TOKEN + ) + + env ( + LINEAR_TOKEN = ${LINEAR_TOKEN:?create a Linear PAT and export it} + GITHUB_TOKEN + ) +) +``` + +That is the whole setup. On `clawk up`, clawk renders the server list into +the guest before the VM boots, allows each server's host in the egress +policy, and hands the runner the config file — so the agent's first tool +call works. + +## Line shapes + +| Written as | Transport | +| --- | --- | +| `name https://host/mcp` | `http` (the default) | +| `name http https://host/mcp` | `http`, explicit | +| `name sse https://host/sse` | `sse` | +| `name stdio "cmd --flag arg"` | `stdio`, a local process | + +Modifiers repeat and may be combined on one line: + +- `header "Name: value"` — an extra HTTP header, for `http` / `sse`. +- `env NAME` — an environment variable handed to a `stdio` server. + +## Credentials: use a token, not a browser + +Only static credentials are supported, and that's deliberate: a personal +access token is a value that can be in place *before* the VM boots, which is +what makes a fresh sandbox usable immediately. Interactive OAuth +(`claude mcp login`) can't be arranged ahead of time — it stores a grant +inside one guest, so every new sandbox would need its own login. + +Most services that offer an MCP endpoint also issue a PAT. Prefer it. + +Credential **values** never enter clawk's config or state: + +- `clawk.mod` holds a `${VAR}` reference; so does the config clawk renders + into the sandbox state dir on the host. +- The value is read from your shell at attach time and delivered straight to + the runner's process environment over the vsock handshake, where the + runner expands `${VAR}` as it connects. + +Declare the variable in `env ( … )` so it's carried. The `${VAR:?message}` +form is worth using: a missing PAT then fails sandbox creation with your +message instead of producing a server that quietly returns 401 mid-task. + +A URL with credentials in it — `https://user:token@host/mcp` — is rejected +for the same reason. The URL is stored verbatim on the sandbox record and in +the rendered guest config, both unencrypted on host disk, so it is the one +spelling that would defeat the guarantee above. Move the credential to a +`header` and it stays in the runner's environment. + +## Servers that need no declaration + +If a service is available as a **claude.ai connector**, use that instead of +declaring the server here. Connectors are authorized once against your +Anthropic account and proxied server-side, so they reach a sandbox with no +local credential, no config, and no egress rule of its own — they ride the +token clawk already forwards. Check with `claude mcp list` inside a sandbox; +anything listed as `claude.ai ` is already working. + +Declare a server in `mcp ( … )` when there is no connector for it, or when +you want to pin a specific endpoint. + +## Scopes + +`mcp ( … )` is valid in a repo `clawk.mod`, a workspace root, and a +`namespace` block. They merge by server name, narrowest scope winning — so a +namespace is the place for the org-wide set, and which namespace a sandbox +belongs to then decides what it can reach: + +```text +namespace acme ( + mcp ( + linear https://mcp.linear.app/mcp header "Authorization: Bearer ${LINEAR_TOKEN}" + ) +) +``` + +Two repos in one workspace declaring the same server identically is fine. +Declaring the same *name* with different targets is rejected, naming both +sources — clawk won't silently pick one. + +## Egress + +Every declared `http` / `sse` host is allowed automatically, in a network +block of origin `mcp`. It sits at the bottom of the precedence chain, just +above the namespace layer: a `deny` you write in `clawk.mod` or via +`clawk network deny` still wins, so the derivation is a convenience and +never a way around your own rules. + +`stdio` servers get no allow — they're a local process. They may still need +egress of their own for whatever they talk to, and one for the package +registry if the command is an `npx`-style fetch (already in the default +allowlist). + +## Notes + +- The rendered config lives at `~/.claude/mcp/clawk.json` in the guest, + passed to the runner with `--mcp-config`. It is not `.mcp.json` in your + repo (clawk won't write into your worktree) and not `~/.claude.json` + (concurrent-write races, and clawk uses it as the onboarding marker). +- Editing `mcp ( … )` takes effect on the next `clawk up` — the file is + rewritten every boot. Removing an entry retires the server. +- clawk does not pass `--strict-mcp-config`, so claude.ai connectors and any + plugin MCP servers your settings enable keep working alongside these. +- Only the `claude` runner is wired today. `codex` and `opencode` use their + own MCP config formats and `pi` loads MCP through an extension rather than + a flag; a sandbox declaring servers simply doesn't get a config flag for + those runners. +- An `npx`-style `stdio` server downloads on first use in each fresh VM. Put + the fetch in `on create ( … )` if you want the first tool call to be fast. diff --git a/docs/networking.md b/docs/networking.md index 40b9f6e..48d57dc 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -66,6 +66,12 @@ clawk policy refresh clawk policy delete ``` +One layer is derived rather than written: each `http`/`sse` server in an +`mcp ( … )` block contributes an allow for its host (see +[MCP servers](mcp.md)). It sits just above the namespace layer, below +everything you wrote yourself — so declaring a server saves you a `network +allow` line, but a `deny` of your own still wins. + `clawk apply -f ` registers `policy` and `namespace` blocks from manifest files (same grammar, no sandbox created). A directory applies every file independently — one broken manifest is reported by name @@ -120,6 +126,11 @@ rather than silently doing nothing. Reverse forwards can also be declared in `clawk.mod` — see [Configuration](configuration.md#reference). +A serial port is the same idea pointed at hardware rather than a socket: +`clawk serial add` puts a USB-serial device from your Mac into the guest as +`/dev/`, over the same vsock transport and with the same live-apply and +vz-only caveats. See [Serial devices](serial.md). + ### Recipe: the Claude Code IDE plugin The JetBrains and VS Code plugins run a websocket server on the host's diff --git a/docs/serial.md b/docs/serial.md new file mode 100644 index 0000000..10ea424 --- /dev/null +++ b/docs/serial.md @@ -0,0 +1,168 @@ +# Serial devices + +Present a serial port plugged into your Mac — an Arduino, an ESP32, a +USB-TTL adapter — as a device inside a sandbox, so `arduino-cli`, `esptool`, +`avrdude` and a serial monitor can run in there against real hardware. + +```sh +clawk serial add /dev/cu.usbmodem1101 # same name in the guest +clawk serial add /dev/cu.usbmodem1101:ttyACM0 # /dev/ttyACM0 in the guest +clawk serial list +clawk serial remove ttyACM0 +``` + +Like reverse forwards, these apply to a running sandbox immediately — no +`down`/`up` cycle — and they are vz (macOS) only, because the guest is the +end that dials and firecracker's vsock is one-way. + +## What actually crosses + +The USB device is **not** passed through. No hypervisor clawk targets can do +that: Virtualization.framework's USB controller carries virtual mass-storage +devices only, with no API for a physical device, and firecracker has no USB +at all — no PCI bus to hang a controller off. + +What crosses is the serial stream and its line settings, over vsock. Inside +the guest that arrives as a PTY symlinked to `/dev/`; on the host, +clawk opens the real port and pumps bytes between the two. This is what every +serial tool actually wants — none of them care that a tty is behind a USB +device rather than a 16550. + +Two consequences worth knowing up front: + +- **The port is only held while the guest is using it.** clawk opens the + physical device when a process in the sandbox opens `/dev/` and + closes it when that process lets go. The Arduino IDE on your Mac can have + the board the rest of the time. +- **Opening the device resets the board.** That is not a clawk behaviour, it + is the auto-reset circuit on the board: opening a serial port asserts DTR, + and DTR is wired to RESET through a capacitor. It's the same reason opening + the Arduino IDE's serial monitor reboots an Uno. Because the host open is + tied to the guest open, this happens at exactly the moment the tooling + expects it. + +## What doesn't cross: DTR and RTS + +A PTY has no modem-control lines. `TIOCMGET` and `TIOCMSET` return `ENOTTY` +on both ends of one, so a guest tool that toggles DTR or RTS explicitly gets +an error, and no amount of protocol work on clawk's side can change that. + +In practice this affects less than it sounds like, because the two things +those lines are used for both have another path: + +| Board style | How it enters the bootloader | Works? | +|---|---|---| +| Native USB (Leonardo, Micro, most ESP32-S3, RP2040) | 1200-baud touch — open at 1200 baud, close | **Yes.** A PTY does carry the baud rate, and clawk forwards the close too | +| Classic auto-reset (Uno, Nano, Mega) | DTR pulse | **Yes, via the open.** Opening the port asserts DTR, which is the pulse | +| ESP32 with the classic auto-program circuit | DTR *and* RTS in sequence, to drive GPIO0 and EN separately | **No.** Two lines in a specific order can't be expressed | + +For that last row — a plain ESP32 DevKit with `esptool` — hold the BOOT +button while the upload starts, or use a board with native USB. `esptool`'s +`--before no_reset` skips the sequence it can't perform. + +You may still see an `ioctl("TIOCMGET")` warning from `avrdude` even on a +board that uploads fine. It is telling the truth about the ioctl and is +harmless: the reset already happened when the port opened. + +## Boards that re-enumerate + +A board entering its bootloader drops off the USB bus and comes back, often +under a *different* device name — `cu.usbmodem1101` becomes +`cu.usbmodem14201` and back again. A literal path breaks on that; a glob +doesn't: + +```sh +clawk serial add '/dev/cu.usbmodem*:ttyACM0' +``` + +The pattern is resolved each time the port is opened, not when you configure +it. Quote it so your shell doesn't expand it first. A pattern matching two +boards is refused rather than guessed at — flashing the wrong device is the +one failure worth being loud about. + +The guest-side name never changes across a re-enumeration, so `arduino-cli +upload -p /dev/ttyACM0` keeps working through the whole cycle. + +## Declaring devices in clawk.mod + +``` +sandbox firmware ( + serial ( + /dev/cu.usbmodem1101 # /dev/cu.usbmodem1101 in the guest + /dev/cu.usbserial-A50285BI ttyUSB0 # /dev/ttyUSB0 in the guest + /dev/cu.usbmodem* ttyACM0 # resolved at open time + ) +) +``` + +Host device first, optional guest name second — space-separated, matching +`files` and `shares` rather than the CLI's colon form, because a colon inside +a path is ambiguous in a way a port number never is. + +Two entries claiming the same guest name, or the same host port, are refused +with an error naming both contributors rather than silently resolved. + +## Working with a board from the sandbox + +`arduino-cli board list` won't find anything: it enumerates USB VID/PID +through libusb, and there is no USB in there. Name the port and the board +explicitly instead — which is what you'd do in CI anyway: + +```sh +arduino-cli compile --fqbn arduino:avr:uno sketch/ +arduino-cli upload --fqbn arduino:avr:uno -p /dev/ttyACM0 sketch/ +arduino-cli monitor -p /dev/ttyACM0 -c baudrate=115200 +``` + +`esptool` and `avrdude` take `-p`/`-P` the same way. Anything that opens a +tty and sets a baud rate works; `screen`, `picocom` and `cat` are all fine. + +One-shot writes work too, but note what they cost: + +```sh +echo 'status?' > /dev/ttyACM0 # opens, writes, closes — and resets +``` + +The host port is open only while a process in the sandbox holds the device, +so a command like that opens and closes it around a single write — and since +opening asserts DTR, each one resets a board wired for auto-reset. Fine for a +one-off; wrong for a loop. Hold the device open instead, and the port stays +open with it: + +```sh +exec 3<>/dev/ttyACM0 # one open, one reset +echo 'status?' >&3 +cat <&3 & +exec 3<&- # release it +``` + +## On macOS: `cu.` not `tty.` + +Use the callout device (`/dev/cu.*`), not the dial-in device +(`/dev/tty.*`). The dial-in side blocks on carrier detect, which shows up as +a port that opens and then does nothing. `clawk serial add` warns if you name +a `tty.` device. + +## Troubleshooting + +**"no device matches"** — the board isn't plugged in, or is mid-reset. clawk +waits about three seconds for it during an attach, which covers a +re-enumeration; past that the guest retries on its own. `clawk serial list` +shows what's present right now. + +**"already in use"** — something else has the port. Inside the guest, only +one process can hold a device at a time; on the Mac, check for an open +Arduino IDE serial monitor. + +**Uploads hang at "not in sync"** — the board didn't reset. See the DTR table +above; for a classic ESP32 DevKit, hold BOOT. + +**Nothing appears at `/dev/`** — the sandbox has to be running vz and +its guest agent has to be current. `clawk down && clawk up` re-injects the +agent; `clawk serial add` says so explicitly when the daemon is too old. + +## See also + +- [Networking](networking.md) — port forwarding in both directions, which + serial forwarding is deliberately shaped like +- [Configuration](configuration.md) — the full `clawk.mod` reference diff --git a/images/clawk-dev/Dockerfile b/images/clawk-dev/Dockerfile index 4967360..151e6eb 100644 --- a/images/clawk-dev/Dockerfile +++ b/images/clawk-dev/Dockerfile @@ -40,8 +40,14 @@ FROM node:22-bookworm-slim COPY --from=go /usr/local/go /usr/local/go ENV PATH=/usr/local/go/bin:$PATH -# Coding agents. claude is what bare `clawk` attaches to; codex is the -# other built-in runner. +# Coding agents. claude is what bare `clawk` attaches to; codex, pi, and +# opencode are the other built-in runners, all installed here so every +# `clawk run ` in the registry works out of the box. +# +# opencode-ai resolves a large prebuilt platform binary (~180 MB for +# linux-arm64) rather than shipping JS, so it dominates this layer's size. +# Drop it from the list for a slimmer image; `clawk run opencode` then +# fails with a plain "command not found". # # The global npm prefix is relocated to /usr/local/npm-global and made # world-writable so claude/codex can self-update at runtime. Why not just @@ -58,6 +64,7 @@ ENV NPM_CONFIG_PREFIX=/usr/local/npm-global ENV PATH=/usr/local/npm-global/bin:$PATH RUN mkdir -p /usr/local/npm-global \ && npm install -g @anthropic-ai/claude-code @openai/codex \ + @earendil-works/pi-coding-agent opencode-ai \ && npm cache clean --force \ && chmod -R a+rwX /usr/local/npm-global @@ -68,10 +75,18 @@ RUN mkdir -p /usr/local/npm-global \ # runs root work through the agent and never needs it. sox is the audio # recorder Claude Code's voice dictation falls back to on Linux; it reads # the virtio-snd mic (CLAWK_AUDIO_INPUT, on by default) via ALSA. +# +# ripgrep and fd-find are here for pi as much as for the agents' own use: +# pi's search tools look for `rg` and `fd`/`fdfind` on PATH and, finding +# neither, download release tarballs from GitHub on first run ("fd not +# found. Downloading..."). Debian ships the fd binary as `fdfind` to avoid +# a name clash, which is precisely one of the names pi probes for, so the +# package satisfies it as-is — no symlink needed. RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ ca-certificates \ curl \ + fd-find \ git \ jq \ less \ diff --git a/internal/agentembed/agent_compile_test.go b/internal/agentembed/agent_compile_test.go index a576fb2..7f728d3 100644 --- a/internal/agentembed/agent_compile_test.go +++ b/internal/agentembed/agent_compile_test.go @@ -73,6 +73,38 @@ func TestAgentSourceCompiles(t *testing.T) { } } +// TestInitSourceCompiles is TestAgentSourceCompiles for clawk-init. The +// init is built by the same guestbuild path and injected into every OCI +// sandbox disk, but it went uncovered while only the agent was compiled +// here — and it is the harder of the two to review by eye: PID 1, raw +// syscalls, and no way to observe a failure except a guest that boots +// wrong. +func TestInitSourceCompiles(t *testing.T) { + if _, err := exec.LookPath("go"); err != nil { + t.Skip("`go` not on PATH") + } + if os.Getenv("AGENT_TEST_NO_BUILD") != "" { + t.Skip("AGENT_TEST_NO_BUILD set") + } + + tmp := t.TempDir() + mustWrite(t, filepath.Join(tmp, "main.go"), InitMainGo) + mustWrite(t, filepath.Join(tmp, "go.mod"), InitGoMod) + + if out, err := runIn(t, tmp, "go", "mod", "tidy"); err != nil { + t.Fatalf("go mod tidy failed: %v\n%s", err, out) + } + bin := filepath.Join(tmp, "clawk-init") + if out, err := runIn(t, tmp, "go", "build", "-o", bin, "."); err != nil { + t.Fatalf("go build failed: %v\n%s", err, out) + } + fi, err := os.Stat(bin) + require.NoError(t, err, "init binary not produced") + if fi.Size() < 100*1024 { + t.Fatalf("init binary suspiciously small (%d bytes); build probably hollow", fi.Size()) + } +} + // mustWrite writes content to path and fatals on error. Tiny helper so // the assertion-heavy tests above stay readable. func mustWrite(t *testing.T, path string, content []byte) { @@ -98,3 +130,45 @@ func runIn(t *testing.T, dir, cmdName string, args ...string) ([]byte, error) { err := cmd.Run() return buf.Bytes(), err } + +// TestSerialForwarderRuns compiles the agent into a throwaway module along +// with its own test file and runs `go test` there. +// +// TestAgentSourceCompiles proves the agent builds; this proves its serial +// half works. That half is a state machine over a PTY with several edges +// that are invisible to a compiler and awkward to reason about: whether a +// hangup really distinguishes "no client" from "idle client", whether a +// baud change with no accompanying event is noticed at all, and whether the +// attachment's lifetime tracks the client's open and close — which is what +// makes a board reset at the right moment. Those are worth executing. +// +// Linux-only and native-arch only: the tests open PTYs and issue termios +// ioctls, so they have to run rather than cross-compile. +func TestSerialForwarderRuns(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("the guest agent's serial tests need a Linux PTY; cross-compiling can't run them") + } + if _, err := exec.LookPath("go"); err != nil { + t.Skip("`go` not on PATH") + } + if os.Getenv("AGENT_TEST_NO_BUILD") != "" { + t.Skip("AGENT_TEST_NO_BUILD set") + } + + tmp := t.TempDir() + mustWrite(t, filepath.Join(tmp, "main.go"), AgentMainGo) + mustWrite(t, filepath.Join(tmp, "go.mod"), AgentGoMod) + mustWrite(t, filepath.Join(tmp, "serial_test.go"), SerialAgentTest) + + if out, err := runIn(t, tmp, "go", "mod", "tidy"); err != nil { + t.Fatalf("go mod tidy failed: %v\n%s", err, out) + } + // Native, not cross-compiled: runIn pins GOOS/GOARCH for the build + // tests, and these have to actually execute. + cmd := exec.Command("go", "test", "-count=1", "-timeout=120s", "./...") + cmd.Dir = tmp + cmd.Env = append(os.Environ(), "CGO_ENABLED=0") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("guest agent serial tests failed: %v\n%s", err, out) + } +} diff --git a/internal/agentembed/embed.go b/internal/agentembed/embed.go index e4948ba..82fdf22 100644 --- a/internal/agentembed/embed.go +++ b/internal/agentembed/embed.go @@ -54,3 +54,12 @@ var InitMainGo []byte // //go:embed init_go.mod.in var InitGoMod []byte + +// SerialAgentTest is the agent's own test suite for the serial forwarder, +// run inside a throwaway module built from AgentMainGo — see +// TestSerialForwarderRuns. Embedded rather than kept as a normal _test.go +// for the same reason main.go.in is: it is package main in the guest's +// module, not in this one. +// +//go:embed serial_agent_test.go.in +var SerialAgentTest []byte diff --git a/internal/agentembed/go.mod.in b/internal/agentembed/go.mod.in index 81d787d..5f2129c 100644 --- a/internal/agentembed/go.mod.in +++ b/internal/agentembed/go.mod.in @@ -5,4 +5,5 @@ go 1.25.0 require ( github.com/creack/pty v1.1.21 github.com/mdlayher/vsock v1.2.1 + golang.org/x/sys v0.47.0 ) diff --git a/internal/agentembed/init_main.go.in b/internal/agentembed/init_main.go.in index 64c360b..fba2134 100644 --- a/internal/agentembed/init_main.go.in +++ b/internal/agentembed/init_main.go.in @@ -25,9 +25,11 @@ package main import ( + "encoding/binary" "encoding/json" "errors" "fmt" + "io" "net" "os" "os/exec" @@ -37,6 +39,7 @@ import ( "strings" "sync" "time" + "unsafe" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" @@ -51,11 +54,21 @@ type manifest struct { Hostname string `json:"hostname,omitempty"` Network *manifestNet `json:"network,omitempty"` User *manifestUser `json:"user,omitempty"` + Swap *swapSpec `json:"swap,omitempty"` Mounts []mountSpec `json:"mounts,omitempty"` Files []fileSpec `json:"files,omitempty"` Services []serviceSpec `json:"services,omitempty"` } +// swapSpec is the swap device to format and enable. Nil for a sandbox with +// swap disabled, and absent from manifests written by a clawk that predates +// swap — an additive field, so an old init ignores it and a new init sees +// nil, both booting without swap. +type swapSpec struct { + Device string `json:"device"` + Swappiness int `json:"swappiness,omitempty"` +} + type manifestNet struct { Interface string `json:"interface"` Address string `json:"address"` // CIDR, e.g. "192.168.127.2/24" @@ -143,6 +156,16 @@ func main() { } } + // Before the mounts and the services, so the guest already has somewhere + // to put cold anonymous pages by the time anything starts allocating. + // Non-fatal: a sandbox without swap works, it just has no cushion when the + // host's balloon controller reclaims its memory. + if m.Swap != nil { + if err := setupSwap(m.Swap); err != nil { + logf("swap: %v", err) + } + } + for _, mt := range m.Mounts { err := mountShare(mt, m.User) if err == nil { @@ -300,12 +323,18 @@ func loadManifest() (*manifest, error) { return &m, nil } -// waitForDevice opens dev, retrying briefly — virtio-blk probing can race -// the first userspace instructions on a fast direct-kernel boot. +// waitForDevice opens dev read-only, retrying briefly — virtio-blk probing +// can race the first userspace instructions on a fast direct-kernel boot. func waitForDevice(dev string, timeout time.Duration) (*os.File, error) { + return waitForDeviceFlags(dev, os.O_RDONLY, timeout) +} + +// waitForDeviceFlags is waitForDevice with an explicit open mode, for the +// swap device, whose header we have to write. +func waitForDeviceFlags(dev string, flags int, timeout time.Duration) (*os.File, error) { deadline := time.Now().Add(timeout) for { - f, err := os.Open(dev) + f, err := os.OpenFile(dev, flags, 0) if err == nil { return f, nil } @@ -316,6 +345,109 @@ func waitForDevice(dev string, timeout time.Duration) (*os.File, error) { } } +// ──────────────────────────────────────────────────────────────────────── +// Swap +// ──────────────────────────────────────────────────────────────────────── + +// setupSwap formats the swap device and enables it. +// +// The header is written here rather than shelled out to mkswap(8) for the +// same reason the user is created by editing /etc/passwd: the rootfs is an +// arbitrary OCI image, and nothing guarantees util-linux is in it. +func setupSwap(s *swapSpec) error { + if s.Device == "" { + return errors.New("manifest declares swap with no device") + } + f, err := waitForDeviceFlags(s.Device, os.O_RDWR, 5*time.Second) + if err != nil { + return err + } + size, err := f.Seek(0, io.SeekEnd) + if err != nil { + f.Close() + return fmt.Errorf("sizing %s: %w", s.Device, err) + } + err = writeSwapHeader(f, size) + f.Close() + if err != nil { + return err + } + if err := swapon(s.Device); err != nil { + return fmt.Errorf("swapon %s: %w", s.Device, err) + } + if s.Swappiness > 0 { + if err := os.WriteFile("/proc/sys/vm/swappiness", + []byte(strconv.Itoa(s.Swappiness)), 0o644); err != nil { + logf("swap: setting swappiness: %v", err) + } + } + logf("swap: enabled %s (%d MiB, swappiness %d)", s.Device, size>>20, s.Swappiness) + return nil +} + +// swapMagic is the signature that identifies a formatted swap area. It sits +// at the very end of the first page, not at its start. +const swapMagic = "SWAPSPACE2" + +// writeSwapHeader lays a version-1 swap header on the device, the layout +// swapon(2) validates (union swap_header in linux/include/linux/swap.h): +// +// [0:1024) bootbits, left zero +// [1024:1028) version = 1 +// [1028:1032) last_page, the highest usable page index +// [1032:1036) nr_badpages = 0 +// [1036:1068) uuid and label, both left zero +// [pagesize-10:pagesize) "SWAPSPACE2" +// +// The u32s are native-endian: the kernel reads this as a struct off the +// device, not as a wire format, and the page size is the running kernel's +// (arm64 may be 4K or 16K) for the same reason. +// +// Rewritten on every boot rather than validated first. It costs one page, +// and it keeps the header true after the host resizes the device — the +// alternative is a stale last_page that hands the kernel pages the device no +// longer has. +func writeSwapHeader(f *os.File, size int64) error { + pageSize := os.Getpagesize() + pages := size / int64(pageSize) + // The kernel refuses anything under 10 pages outright; a device that + // small means the host wrote the wrong file, not that swap is tiny. + if pages < 10 { + return fmt.Errorf("swap device is %d bytes, too small to format", size) + } + page := make([]byte, pageSize) + binary.NativeEndian.PutUint32(page[1024:1028], 1) // version + binary.NativeEndian.PutUint32(page[1028:1032], uint32(pages-1)) // last_page + binary.NativeEndian.PutUint32(page[1032:1036], 0) // nr_badpages + copy(page[pageSize-len(swapMagic):], swapMagic) + if _, err := f.WriteAt(page, 0); err != nil { + return fmt.Errorf("writing swap header: %w", err) + } + return f.Sync() +} + +// swapFlagDiscard is SWAP_FLAG_DISCARD: discard freed swap pages, which +// would let the sparse backing file give space back to the host. Neither +// vz's nor firecracker's virtio-blk advertises discard today and the kernel +// drops the flag when the device can't honor it, so this is currently a +// no-op that costs nothing and starts paying if either ever does. +const swapFlagDiscard = 0x10000 + +// swapon enables dev as swap. x/sys/unix wraps neither swapon(2) nor its +// flags, so this goes through the raw syscall. +func swapon(dev string) error { + p, err := unix.BytePtrFromString(dev) + if err != nil { + return err + } + _, _, errno := unix.Syscall( + unix.SYS_SWAPON, uintptr(unsafe.Pointer(p)), swapFlagDiscard, 0) + if errno != 0 { + return errno + } + return nil +} + func cmdlineValue(key string) string { data, err := os.ReadFile("/proc/cmdline") if err != nil { diff --git a/internal/agentembed/main.go.in b/internal/agentembed/main.go.in index d04a1fe..1cd5a30 100644 --- a/internal/agentembed/main.go.in +++ b/internal/agentembed/main.go.in @@ -31,6 +31,7 @@ import ( "os" "os/exec" "os/user" + "path/filepath" "strconv" "strings" "sync" @@ -39,6 +40,7 @@ import ( "github.com/creack/pty" "github.com/mdlayher/vsock" + "golang.org/x/sys/unix" ) // ──────────────────────────────────────────────────────────────────────── @@ -632,6 +634,14 @@ func main() { // proxy yet). go runReverseForwarder(logger) + // Serial forwarder: holds a control connection to the host, creates a + // PTY at /dev/ for every serial device the host publishes, and + // bridges each one to the physical port while a process in here has it + // open. Idle unless the sandbox has serial devices configured; retries + // forever if the host end isn't there, exactly like the reverse + // forwarder above. + go runSerialForwarder(logger) + var wg sync.WaitGroup for { conn, err := listener.Accept() @@ -997,7 +1007,7 @@ func revfwdReadLine(r *bufio.Reader, v any) error { // exactly memReportBytes of snapshot, then close. The host's idle watchdog // (internal/cli) polls the same snapshot for the activity fields — load and // network I/O — to tell a quiescent guest from one still working detached. -// The wire format is five big-endian uint64s and MUST stay in lock-step +// The wire format is seven big-endian uint64s and MUST stay in lock-step // with machine/vz/memreport.go (memReport / decodeMemReport): // // [0:8] MemTotal (KiB) @@ -1005,6 +1015,8 @@ func revfwdReadLine(r *bufio.Reader, v any) error { // [16:24] memory PSI "some avg10" × 100 (0 if PSI unavailable) // [24:32] /proc/loadavg load1 × 100 // [32:40] cumulative rx+tx bytes across non-lo interfaces +// [40:48] SwapTotal (KiB) +// [48:56] SwapFree (KiB) // // Hosts accept the legacy 24-byte prefix alone, so new fields may only be // appended, never reordered. @@ -1016,8 +1028,8 @@ const ( // (1026). Keep in sync with machine/vz/memreport.go's agentMemPort. memReportVSockPort = 1027 - // memReportBytes is the fixed snapshot size (5 × uint64). - memReportBytes = 40 + // memReportBytes is the fixed snapshot size (7 × uint64). + memReportBytes = 56 ) // runMemReporter listens on memReportVSockPort and writes a fresh memory @@ -1046,15 +1058,17 @@ func runMemReporter(logger *log.Logger) { // files, so this runs inline rather than in a goroutine. func serveMemReport(conn net.Conn, logger *log.Logger) { defer conn.Close() - total, avail := readMemInfo() + mem := readMemInfo() psi := readMemPSICenti() var buf [memReportBytes]byte - binary.BigEndian.PutUint64(buf[0:8], total) - binary.BigEndian.PutUint64(buf[8:16], avail) + binary.BigEndian.PutUint64(buf[0:8], mem.total) + binary.BigEndian.PutUint64(buf[8:16], mem.avail) binary.BigEndian.PutUint64(buf[16:24], psi) binary.BigEndian.PutUint64(buf[24:32], readLoad1Centi()) binary.BigEndian.PutUint64(buf[32:40], readNetIOBytes()) + binary.BigEndian.PutUint64(buf[40:48], mem.swapTotal) + binary.BigEndian.PutUint64(buf[48:56], mem.swapFree) _ = conn.SetWriteDeadline(time.Now().Add(2 * time.Second)) if _, err := conn.Write(buf[:]); err != nil && !isClosedConn(err) { @@ -1062,14 +1076,30 @@ func serveMemReport(conn net.Conn, logger *log.Logger) { } } -// readMemInfo returns MemTotal and MemAvailable from /proc/meminfo in KiB. -// Missing fields read as zero; a zero MemTotal tells the host "no usable -// data", which it treats as absent rather than "guest has no memory". -func readMemInfo() (total, avail uint64) { +// meminfo is the slice of /proc/meminfo the host's balloon controller +// reads. The swap pair matters because MemAvailable is by definition what +// the guest can reclaim *without* swapping: a guest that has pushed its +// cold anonymous pages out looks roomy by that measure alone, and the +// controller would answer by taking the memory back (see guestDesiredTarget +// in machine/vz/balloon.go). +type meminfo struct { + total uint64 + avail uint64 + swapTotal uint64 + swapFree uint64 +} + +// readMemInfo returns the meminfo fields in KiB. Missing fields read as +// zero; a zero total tells the host "no usable data", which it treats as +// absent rather than "guest has no memory". A guest with no swap device +// reports zero for both swap fields, which is also what a pre-swap guest +// agent's shorter report decodes to. +func readMemInfo() meminfo { data, err := os.ReadFile("/proc/meminfo") if err != nil { - return 0, 0 + return meminfo{} } + var m meminfo for _, line := range strings.Split(string(data), "\n") { key, rest, ok := strings.Cut(line, ":") if !ok { @@ -1077,12 +1107,16 @@ func readMemInfo() (total, avail uint64) { } switch key { case "MemTotal": - total = parseMeminfoKiB(rest) + m.total = parseMeminfoKiB(rest) case "MemAvailable": - avail = parseMeminfoKiB(rest) + m.avail = parseMeminfoKiB(rest) + case "SwapTotal": + m.swapTotal = parseMeminfoKiB(rest) + case "SwapFree": + m.swapFree = parseMeminfoKiB(rest) } } - return total, avail + return m } // parseMeminfoKiB parses a " 4033244 kB" meminfo value into its KiB number. @@ -1171,3 +1205,749 @@ func readMemPSICenti() uint64 { } return 0 } + +// ──────────────────────────────────────────────────────────────────────── +// Serial forwarder +// +// Wire protocol kept in lock-step with internal/serialfwd in the parent +// repo; any change there must be mirrored here. Two connection kinds, both +// guest-initiated, both starting with one JSON greeting line: +// +// op=control the host answers with the full device set now and again on +// every change, so `clawk serial add` applies to a running +// sandbox. +// op=attach the host validates the device name, opens the physical +// port, answers ok, and then speaks frames. +// +// For each device the host publishes we create a PTY, symlink it to +// /dev/, and hold the master. The attach connection exists for +// exactly as long as some process in here holds the slave open, which is +// what makes the host open and close the real port in step — see the +// serialfwd package comment for why that matters to an Arduino. +// ──────────────────────────────────────────────────────────────────────── + +const ( + // serialVSockPort is the host-side port serving both kinds. Mirrors + // serialfwd.VSockPort. Distinct from the pty agent (1024), time-sync + // (1025), ssh-agent (1026), mem-report (1027) and reverse-forward + // (1028). + serialVSockPort = 1029 + + // serialProtoVersion mirrors serialfwd.ProtoVersion. + serialProtoVersion = 1 + + // serialMaxLine mirrors serialfwd.MaxLineBytes. + serialMaxLine = 64 * 1024 + + // serialFrameHeaderBytes / serialMaxFrame mirror + // serialfwd.FrameHeaderBytes and serialfwd.MaxFrameBytes. + serialFrameHeaderBytes = 5 + serialMaxFrame = 256 * 1024 + + // Frame types, mirroring serialfwd.FrameData / serialfwd.FrameMode. + serialFrameData byte = 0x01 + serialFrameMode byte = 0x02 + + // serialControlMinBackoff / serialControlMaxBackoff bound the retry + // delay when the host end isn't answering — the first moments of every + // boot, and permanently under firecracker. Same reasoning as the + // reverse forwarder's. + serialControlMinBackoff = 1 * time.Second + serialControlMaxBackoff = 30 * time.Second + + // serialStableSession is how long a control connection must last to + // count as healthy and reset the backoff. + serialStableSession = 30 * time.Second + + // serialPollInterval is how long the per-device loop parks waiting for + // output from its PTY client. + // + // It is a timeout rather than a pure block because a PTY delivers no + // event when its client changes the line configuration: master and + // slave share the termios state, so the change is visible to a + // tcgetattr but wakes nothing. The loop therefore has to look, and this + // is how often. 100ms is far below human perception for "the monitor + // opened at the wrong baud" and cheap enough to run only while a + // process actually holds the device open. + serialPollInterval = 100 * time.Millisecond + + // serialReopenMinInterval / serialReopenMaxInterval bound how often a + // detached device checks whether a client has come back. + // + // It starts fast on purpose. A board being flashed goes through + // close-then-immediately-reopen (the 1200-baud touch, then the + // bootloader's port), and a slow first check would add dead time to + // every upload. It backs off because an idle sandbox with a forwarded + // board that nobody is using shouldn't wake the vCPU 50 times a second + // forever. + serialReopenMinInterval = 20 * time.Millisecond + serialReopenMaxInterval = 500 * time.Millisecond + + // serialAttachRetryInterval is the pause after the host refuses an + // attach — an unplugged board, or one held by something else on the + // Mac. The client in here still has the PTY open, so we keep trying, + // just not in a spin. + serialAttachRetryInterval = 1 * time.Second + + // serialDrainMaxReads caps how many buffered reads the hangup path + // flushes before giving up — see drainToHost. A PTY's input queue is a + // few kilobytes and forwardOutput moves 4 KiB a time, so this is roughly + // an order of magnitude of headroom over anything reachable: a backstop + // against spinning if a read ever reports data it does not deliver, not + // a limit on legitimate writes. + serialDrainMaxReads = 16 +) + +// serialDevDir is where forwarded devices are published, and +// serialDialHost is how the host end is reached. Both are variables solely +// so the agent's own tests can point them at a temp directory and a local +// listener: creating a node in /dev needs root, and AF_VSOCK needs a +// hypervisor. Nothing at runtime reassigns either. +var ( + serialDevDir = "/dev" + + serialDialHost = func() (net.Conn, error) { + return vsock.Dial(vsockHostCID, serialVSockPort, nil) + } +) + +type serialGreeting struct { + Op string `json:"op"` + V int `json:"v"` + Name string `json:"name,omitempty"` + Mode *serialMode `json:"mode,omitempty"` +} + +type serialSnapshot struct { + Devices []serialDeviceMsg `json:"devices"` +} + +type serialDeviceMsg struct { + Name string `json:"name"` +} + +type serialAttachReply struct { + OK bool `json:"ok"` + Error string `json:"error,omitempty"` +} + +type serialMode struct { + Baud int `json:"baud"` + Bits int `json:"bits"` + Parity string `json:"parity"` + Stop int `json:"stop"` +} + +// serialForwarder owns the per-device PTYs, keyed by guest-visible name. +type serialForwarder struct { + log *log.Logger + + mu sync.Mutex + devices map[string]*serialDevice +} + +// runSerialForwarder keeps the control connection to the host alive, +// applying every device set the host publishes. Never returns. +func runSerialForwarder(logger *log.Logger) { + f := &serialForwarder{log: logger, devices: make(map[string]*serialDevice)} + backoff := serialControlMinBackoff + quiet := false + for { + start := time.Now() + err := f.controlSession() + // Every PTY exists to reach a port through the host, so once the + // host is gone they must go too: a device node left in /dev would + // accept an open and then sit mute, which reads as broken hardware + // rather than an absent forward. + f.apply(nil) + + if time.Since(start) >= serialStableSession { + backoff, quiet = serialControlMinBackoff, false + } + if err != nil && !quiet { + logger.Printf("serial: %v (retrying, quietly from here)", err) + quiet = true + } + time.Sleep(backoff) + if backoff *= 2; backoff > serialControlMaxBackoff { + backoff = serialControlMaxBackoff + } + } +} + +// controlSession dials the host and applies snapshots until the connection +// drops. A clean EOF (the host shutting down) returns nil. +func (f *serialForwarder) controlSession() error { + conn, err := serialDialHost() + if err != nil { + return fmt.Errorf("vsock dial host:%d: %w", serialVSockPort, err) + } + defer conn.Close() + + if err := serialWriteLine(conn, serialGreeting{ + Op: "control", V: serialProtoVersion, + }); err != nil { + return fmt.Errorf("sending control greeting: %w", err) + } + r := bufio.NewReaderSize(conn, serialMaxLine) + for { + var snap serialSnapshot + if err := serialReadLine(r, &snap); err != nil { + if errors.Is(err, io.EOF) || isClosedConn(err) { + return nil + } + return fmt.Errorf("reading device set: %w", err) + } + f.apply(snap.Devices) + } +} + +// apply reconciles the live PTYs with want, which is always the complete +// desired set. +func (f *serialForwarder) apply(want []serialDeviceMsg) { + desired := make(map[string]bool, len(want)) + for _, d := range want { + if err := validSerialName(d.Name); err != nil { + f.log.Printf("serial: ignoring device: %v", err) + continue + } + desired[d.Name] = true + } + + f.mu.Lock() + defer f.mu.Unlock() + + for name, dev := range f.devices { + if desired[name] { + continue + } + dev.stop() + delete(f.devices, name) + } + for name := range desired { + if _, ok := f.devices[name]; ok { + continue + } + dev, err := newSerialDevice(f.log, name) + if err != nil { + f.log.Printf("serial: %s: %v", name, err) + continue + } + f.devices[name] = dev + f.log.Printf("serial: /dev/%s -> %s", name, dev.slavePath) + go dev.run() + } +} + +// serialDevice is one forwarded port: a PTY whose master we hold, a symlink +// in /dev pointing at its slave, and — whenever some process in here has +// that slave open — a vsock connection to the physical port on the host. +type serialDevice struct { + log *log.Logger + name string + + master *os.File + slavePath string + linkPath string + + done chan struct{} + once sync.Once + + // conn and mode belong to the current attachment. Only run() touches + // them, so they need no lock; the reader goroutine gets its own handle. + conn net.Conn + mode serialMode +} + +// newSerialDevice creates the PTY and publishes it at /dev/. +func newSerialDevice(logger *log.Logger, name string) (*serialDevice, error) { + master, slave, err := pty.Open() + if err != nil { + return nil, fmt.Errorf("opening pty: %w", err) + } + slavePath := slave.Name() + + // The agent runs as root but the tools that will open this don't, and + // devpts hands out slaves as root:tty 0620. A sandbox is single-user by + // construction, so widening this is not the concession it would be on a + // shared machine. + if err := os.Chmod(slavePath, 0o666); err != nil { + _ = slave.Close() + _ = master.Close() + return nil, fmt.Errorf("chmod %s: %w", slavePath, err) + } + + linkPath := filepath.Join(serialDevDir, name) + // Replace rather than fail: a stale link from a previous boot, or from + // a device that was removed and re-added, is the normal case. + if err := os.Remove(linkPath); err != nil && !errors.Is(err, os.ErrNotExist) { + _ = slave.Close() + _ = master.Close() + return nil, fmt.Errorf("clearing %s: %w", linkPath, err) + } + if err := os.Symlink(slavePath, linkPath); err != nil { + _ = slave.Close() + _ = master.Close() + return nil, fmt.Errorf("linking %s: %w", linkPath, err) + } + + // Put the line in raw mode before anyone can open it. + // + // A fresh PTY comes up cooked — ICANON, ECHO, OPOST — and that default + // is actively wrong for a forwarded serial port. Echo is the harmful + // part: with it on, every byte the board sends is echoed straight back + // out to the board, which turns a chatty device into a loop. Canonical + // mode is merely surprising, holding input back until a newline that a + // binary protocol never sends. + // + // A real tty has the same unhelpful defaults, and every serial tool + // fixes them on open — but the tools are not the only clients here, and + // a device that misbehaves for `cat` is a device people will assume is + // broken. Anything that does configure the port overrides this anyway. + if err := rawSerialPTY(int(master.Fd())); err != nil { + _ = slave.Close() + _ = master.Close() + return nil, fmt.Errorf("raw mode: %w", err) + } + + // Prime the hangup state by closing our own handle on the slave. + // + // This is what makes "is a client attached?" answerable. A PTY that has + // never had its slave opened polls exactly like one that is open and + // idle — no POLLHUP either way — so without this the loop would think a + // client was present from the moment the device appeared and would open + // the physical port for nobody. Opening and closing the slave once + // drives the master into the sticky hangup state, and from then on + // POLLHUP means precisely "nothing in here has it open". + _ = slave.Close() + + return &serialDevice{ + log: logger, + name: name, + master: master, + slavePath: slavePath, + linkPath: linkPath, + done: make(chan struct{}), + }, nil +} + +func (d *serialDevice) stop() { + d.once.Do(func() { close(d.done) }) +} + +// run is the device's whole life: watch the PTY for a client, hold an +// attachment to the host while one is there, and move bytes and mode +// changes between them. +func (d *serialDevice) run() { + defer func() { + d.detach() + _ = d.master.Close() + // Only remove the link if it is still ours. A device removed and + // re-added in quick succession may already have been relinked to a + // new PTY by the time this runs. + if target, err := os.Readlink(d.linkPath); err == nil && target == d.slavePath { + _ = os.Remove(d.linkPath) + } + }() + + reopen := serialReopenMinInterval + for { + select { + case <-d.done: + return + default: + } + + events, err := pollSerialMaster(d.master, serialPollInterval) + if err != nil { + d.log.Printf("serial: %s: poll: %v", d.name, err) + return + } + + if events&unix.POLLHUP != 0 { + // Nothing in here holds the device open — but it may have left + // something behind, so this is a flush-then-let-go, not a + // let-go. POLLIN and POLLHUP are independent: n_tty_poll reports + // the first from input_available_p() and the second from + // TTY_OTHER_CLOSED, so a writer that opened, wrote and closed + // inside one poll interval arrives here with its bytes still + // readable. Honouring the hangup without draining first threw + // them away, which lost `echo cmd > /dev/ttyACM0` — and every + // script that does printf into a device — with no error anywhere. + // + // Attach if that is what it takes to have somewhere to put them. + if events&unix.POLLIN != 0 && d.conn == nil { + d.attach() + } + // Mode first, for the reason the steady-state path below gives, + // and because a 1200-baud touch is a mode change immediately + // followed by a close where the mode is the entire message: + // losing it would mean a native-USB board never enters its + // bootloader. + d.syncMode() + if events&unix.POLLIN != 0 { + d.drainToHost() + } + d.detach() + + select { + case <-d.done: + return + case <-time.After(reopen): + } + if reopen *= 2; reopen > serialReopenMaxInterval { + reopen = serialReopenMaxInterval + } + continue + } + reopen = serialReopenMinInterval + + if d.conn == nil { + if !d.attach() { + select { + case <-d.done: + return + case <-time.After(serialAttachRetryInterval): + } + continue + } + } + + // Mode before data, every iteration. Neither order is right in + // general — the kernel gives no way to place a termios change + // within the byte stream — but this one matches how tools behave: + // they configure the port and then talk on it, so a change seen in + // the same wakeup as some output almost always preceded it. + d.syncMode() + + if events&unix.POLLIN != 0 { + if !d.forwardOutput() { + d.detach() + } + } + } +} + +// drainToHost flushes whatever the PTY still holds through to the physical +// port. Called on the hangup path, where a writer has already gone. +// +// forwardOutput moves at most one buffer per call, so a writer that left more +// than that behind needs several — hence the loop, with a non-blocking poll +// between reads to stop as soon as the queue is empty. serialDrainMaxReads +// bounds it: the PTY's own input queue is finite, so the cap is a backstop +// against spinning rather than a real limit, and it is logged if it is ever +// reached. +// +// Note what the caller's attach means for a board wired for auto-reset: +// opening the host port asserts DTR, so flushing a one-shot write pulses +// RESET. That is the right trade — the user explicitly wrote to the device, +// and dropping the write silently is worse than a reset. Holding the port +// open (`exec 3<>/dev/ttyACM0`) is how you write repeatedly without one. +func (d *serialDevice) drainToHost() { + if d.conn == nil { + return // attach failed; nothing to flush through + } + for i := 0; i < serialDrainMaxReads; i++ { + if !d.forwardOutput() { + return + } + // Timeout 0: ask what is pending right now and never park. The + // hangup is already latched, so a blocking poll here would return + // immediately anyway — but only because of that, and relying on it + // would be a trap for whoever next changes this loop. + events, err := pollSerialMaster(d.master, 0) + if err != nil || events&unix.POLLIN == 0 { + return + } + } + d.log.Printf("serial: %s: stopped draining after %d reads", d.name, serialDrainMaxReads) +} + +// attach opens a connection to the physical port on the host. Reports +// whether the device is ready to carry traffic. +func (d *serialDevice) attach() bool { + mode, err := readSerialMode(d.master) + if err != nil { + d.log.Printf("serial: %s: reading pty mode: %v", d.name, err) + return false + } + + conn, err := serialDialHost() + if err != nil { + d.log.Printf("serial: %s: vsock dial host:%d: %v", d.name, serialVSockPort, err) + return false + } + if err := serialWriteLine(conn, serialGreeting{ + Op: "attach", V: serialProtoVersion, Name: d.name, Mode: &mode, + }); err != nil { + d.log.Printf("serial: %s: sending attach greeting: %v", d.name, err) + _ = conn.Close() + return false + } + + r := bufio.NewReaderSize(conn, serialMaxLine) + var reply serialAttachReply + if err := serialReadLine(r, &reply); err != nil { + d.log.Printf("serial: %s: reading attach reply: %v", d.name, err) + _ = conn.Close() + return false + } + if !reply.OK { + d.log.Printf("serial: %s: host refused: %s", d.name, reply.Error) + _ = conn.Close() + return false + } + + d.conn, d.mode = conn, mode + go d.forwardInput(conn, r) + d.log.Printf("serial: %s attached (%d baud)", d.name, mode.Baud) + return true +} + +// detach tears down the current attachment, if any. The host closes the +// physical port when this connection goes, which is the other half of the +// open/close pairing that gives a board its reset pulse. +func (d *serialDevice) detach() { + if d.conn == nil { + return + } + _ = d.conn.Close() + d.conn = nil + d.log.Printf("serial: %s detached", d.name) +} + +// forwardInput pumps frames from the host into the PTY. It owns conn's read +// side for the lifetime of the attachment and exits when detach closes it. +func (d *serialDevice) forwardInput(conn net.Conn, r *bufio.Reader) { + for { + typ, payload, err := serialReadFrame(r) + if err != nil { + if !errors.Is(err, io.EOF) && !isClosedConn(err) { + d.log.Printf("serial: %s: reading frame: %v", d.name, err) + } + return + } + switch typ { + case serialFrameData: + if _, err := d.master.Write(payload); err != nil { + // EIO here means the client let go between the host + // sending and us writing. Not worth a log line: the poll + // loop is about to notice the hangup anyway. + return + } + default: + // The host only ever sends data today. Ignoring the rest keeps + // a future frame type from being a breaking change. + } + } +} + +// forwardOutput moves one batch of PTY output to the host. Reports whether +// the attachment is still good. +func (d *serialDevice) forwardOutput() bool { + buf := make([]byte, 4096) + n, err := d.master.Read(buf) + if n > 0 { + if werr := serialWriteFrame(d.conn, serialFrameData, buf[:n]); werr != nil { + d.log.Printf("serial: %s: writing to host: %v", d.name, werr) + return false + } + } + if err != nil { + // EIO is the client closing the slave, which the next poll reports + // as POLLHUP; everything else is the PTY going away under us. + return false + } + return true +} + +// syncMode forwards the PTY's line configuration to the host when it has +// changed since the last time we looked. +func (d *serialDevice) syncMode() { + if d.conn == nil { + return + } + mode, err := readSerialMode(d.master) + if err != nil || mode == d.mode { + return + } + if err := serialWriteFrame(d.conn, serialFrameMode, serialMarshalMode(mode)); err != nil { + d.log.Printf("serial: %s: sending mode: %v", d.name, err) + return + } + d.mode = mode + d.log.Printf("serial: %s reconfigured (%d baud)", d.name, mode.Baud) +} + +// pollSerialMaster waits for the PTY master to become readable or hang up. +func pollSerialMaster(f *os.File, timeout time.Duration) (int16, error) { + fds := []unix.PollFd{{Fd: int32(f.Fd()), Events: unix.POLLIN}} + for { + n, err := unix.Poll(fds, int(timeout.Milliseconds())) + if errors.Is(err, unix.EINTR) { + continue + } + if err != nil { + return 0, err + } + if n == 0 { + return 0, nil + } + return fds[0].Revents, nil + } +} + +// readSerialMode reads the line configuration the PTY's client has set. +// +// A PTY master and its slave share one termios, so this reads back exactly +// what the process on the other end asked for — which is the only reason +// forwarding a serial port through a PTY is possible at all. The one thing +// it cannot see is the modem-control lines: a PTY has none, and DTR and RTS +// are simply not expressible here. +func readSerialMode(f *os.File) (serialMode, error) { + t, err := unix.IoctlGetTermios(int(f.Fd()), unix.TCGETS2) + if err != nil { + return serialMode{}, err + } + + mode := serialMode{Baud: int(t.Ospeed), Bits: 8, Parity: "n", Stop: 1} + switch t.Cflag & unix.CSIZE { + case unix.CS5: + mode.Bits = 5 + case unix.CS6: + mode.Bits = 6 + case unix.CS7: + mode.Bits = 7 + } + if t.Cflag&unix.PARENB != 0 { + mode.Parity = "e" + if t.Cflag&unix.PARODD != 0 { + mode.Parity = "o" + } + } + if t.Cflag&unix.CSTOPB != 0 { + mode.Stop = 2 + } + // A rate of zero means the client hung the line up rather than picked a + // speed. Reporting it would fail validation on the host, so keep the + // last real value by reporting the tty default. + if mode.Baud <= 0 { + mode.Baud = 9600 + } + return mode, nil +} + +// rawSerialPTY takes the PTY out of cooked mode. See the call site in +// newSerialDevice for why that is the right default here. +func rawSerialPTY(fd int) error { + t, err := unix.IoctlGetTermios(fd, unix.TCGETS2) + if err != nil { + return err + } + t.Iflag &^= unix.IGNBRK | unix.BRKINT | unix.PARMRK | unix.ISTRIP | + unix.INLCR | unix.IGNCR | unix.ICRNL | unix.IXON + t.Oflag &^= unix.OPOST + t.Lflag &^= unix.ECHO | unix.ECHONL | unix.ICANON | unix.ISIG | unix.IEXTEN + t.Cc[unix.VMIN] = 1 + t.Cc[unix.VTIME] = 0 + return unix.IoctlSetTermios(fd, unix.TCSETS2, t) +} + +// validSerialName mirrors serialfwd.ValidDeviceName. The host validates +// first, but this string becomes a path in /dev on this side, so it is +// checked again here rather than trusted across the wire. +func validSerialName(name string) error { + switch { + case name == "": + return errors.New("device name is empty") + case len(name) > 64: + return fmt.Errorf("device name %q is too long", name) + case strings.ContainsAny(name, "/\\"): + return fmt.Errorf("device name %q must not contain a path separator", name) + case strings.HasPrefix(name, "."): + return fmt.Errorf("device name %q must not start with a dot", name) + } + for _, r := range name { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case r == '.' || r == '-' || r == '_': + default: + return fmt.Errorf("device name %q contains %q", name, r) + } + } + return nil +} + +// serialMarshalMode encodes a mode for a FrameMode payload. Errors are +// impossible for this struct, so the signature doesn't carry one. +func serialMarshalMode(m serialMode) []byte { + b, err := json.Marshal(m) + if err != nil { + return []byte(`{}`) + } + return b +} + +// serialWriteLine mirrors serialfwd.WriteLine. +func serialWriteLine(w io.Writer, v any) error { + b, err := json.Marshal(v) + if err != nil { + return err + } + if len(b)+1 > serialMaxLine { + return errors.New("control line too long") + } + _, err = w.Write(append(b, '\n')) + return err +} + +// serialReadLine mirrors serialfwd.ReadLine. The reader must be reused for +// the frames that follow, or bytes buffered past the newline are lost. +func serialReadLine(r *bufio.Reader, v any) error { + line, err := r.ReadSlice('\n') + if errors.Is(err, bufio.ErrBufferFull) { + return errors.New("control line too long") + } + if err != nil { + return err + } + return json.Unmarshal(line, v) +} + +// serialWriteFrame mirrors serialfwd.WriteFrame. +func serialWriteFrame(w io.Writer, typ byte, payload []byte) error { + if len(payload) > serialMaxFrame { + return errors.New("frame too long") + } + var hdr [serialFrameHeaderBytes]byte + hdr[0] = typ + binary.BigEndian.PutUint32(hdr[1:], uint32(len(payload))) + if _, err := w.Write(hdr[:]); err != nil { + return err + } + if len(payload) == 0 { + return nil + } + _, err := w.Write(payload) + return err +} + +// serialReadFrame mirrors serialfwd.ReadFrame. +func serialReadFrame(r *bufio.Reader) (byte, []byte, error) { + var hdr [serialFrameHeaderBytes]byte + if _, err := io.ReadFull(r, hdr[:]); err != nil { + return 0, nil, err + } + n := binary.BigEndian.Uint32(hdr[1:]) + if n > serialMaxFrame { + return 0, nil, errors.New("frame too long") + } + if n == 0 { + return hdr[0], nil, nil + } + buf := make([]byte, n) + if _, err := io.ReadFull(r, buf); err != nil { + return 0, nil, err + } + return hdr[0], buf, nil +} diff --git a/internal/agentembed/revfwd_lockstep_test.go b/internal/agentembed/revfwd_lockstep_test.go index 27c95a0..296612b 100644 --- a/internal/agentembed/revfwd_lockstep_test.go +++ b/internal/agentembed/revfwd_lockstep_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/clawkwork/clawk/internal/revfwd" + "github.com/clawkwork/clawk/internal/serialfwd" "github.com/stretchr/testify/require" ) @@ -60,6 +61,7 @@ func TestGuestVSockPortsAreDistinct(t *testing.T) { {"sshAgentVSockPort", "1026"}, {"memReportVSockPort", "1027"}, {"reverseForwardVSockPort", fmt.Sprint(revfwd.VSockPort)}, + {"serialVSockPort", fmt.Sprint(serialfwd.VSockPort)}, } { require.Contains(t, src, decl.name+" = "+decl.port, "%s moved; update this test and every mirror of it", decl.name) diff --git a/internal/agentembed/serial_agent_test.go.in b/internal/agentembed/serial_agent_test.go.in new file mode 100644 index 0000000..fc088fc --- /dev/null +++ b/internal/agentembed/serial_agent_test.go.in @@ -0,0 +1,512 @@ +// Tests for the guest agent's serial forwarder, compiled and run inside a +// throwaway module built from main.go.in — see TestSerialForwarderRuns in +// the parent repo for how they get here. +// +// The agent can't be imported (it is package main in its own module), and +// its serial half is a state machine with genuinely tricky edges: the +// primed-hangup trick that makes "is a client attached?" answerable, the +// attachment lifetime that drives the host's open and close of the real +// port, and reading back a line configuration the kernel never announces. +// Compiling it proves none of that. Running it does. + +//go:build linux + +package main + +import ( + "bufio" + "encoding/json" + "io" + "log" + "net" + "os" + "path/filepath" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +// fakeHost is the other end of the vsock the agent normally dials: it +// accepts attachments and records what arrives. +type fakeHost struct { + t *testing.T + ln net.Listener + + greetings chan serialGreeting + frames chan hostFrame + conns chan net.Conn + refuse bool +} + +type hostFrame struct { + typ byte + payload []byte +} + +func newFakeHost(t *testing.T) *fakeHost { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + h := &fakeHost{ + t: t, + ln: ln, + greetings: make(chan serialGreeting, 16), + frames: make(chan hostFrame, 256), + conns: make(chan net.Conn, 16), + } + t.Cleanup(func() { _ = ln.Close() }) + + // Point the agent's dialer at us. + prev := serialDialHost + serialDialHost = func() (net.Conn, error) { return net.Dial("tcp", ln.Addr().String()) } + t.Cleanup(func() { serialDialHost = prev }) + + go h.accept() + return h +} + +func (h *fakeHost) accept() { + for { + conn, err := h.ln.Accept() + if err != nil { + return + } + go h.serve(conn) + } +} + +func (h *fakeHost) serve(conn net.Conn) { + r := bufio.NewReaderSize(conn, serialMaxLine) + var g serialGreeting + if err := serialReadLine(r, &g); err != nil { + return + } + h.greetings <- g + if g.Op != "attach" { + return + } + if h.refuse { + _ = serialWriteLine(conn, serialAttachReply{OK: false, Error: "board unplugged"}) + _ = conn.Close() + return + } + if err := serialWriteLine(conn, serialAttachReply{OK: true}); err != nil { + return + } + h.conns <- conn + for { + typ, payload, err := serialReadFrame(r) + if err != nil { + return + } + h.frames <- hostFrame{typ: typ, payload: payload} + } +} + +func (h *fakeHost) waitGreeting(timeout time.Duration) (serialGreeting, bool) { + select { + case g := <-h.greetings: + return g, true + case <-time.After(timeout): + return serialGreeting{}, false + } +} + +func (h *fakeHost) waitFrame(typ byte, timeout time.Duration) (hostFrame, bool) { + deadline := time.After(timeout) + for { + select { + case f := <-h.frames: + if f.typ == typ { + return f, true + } + case <-deadline: + return hostFrame{}, false + } + } +} + +// startDevice brings up one forwarded device rooted in a temp dir, so the +// symlink lands somewhere writable without root. +func startDevice(t *testing.T, name string) *serialDevice { + t.Helper() + dir := t.TempDir() + prev := serialDevDir + serialDevDir = dir + t.Cleanup(func() { serialDevDir = prev }) + + dev, err := newSerialDevice(log.New(io.Discard, "", 0), name) + if err != nil { + t.Fatalf("newSerialDevice: %v", err) + } + t.Cleanup(dev.stop) + go dev.run() + return dev +} + +// openClient opens the device the way a tool in the sandbox would: through +// the published name, not the PTY path. +func openClient(t *testing.T, dev *serialDevice) *os.File { + t.Helper() + f, err := os.OpenFile(filepath.Join(serialDevDir, dev.name), os.O_RDWR|unix.O_NOCTTY, 0) + if err != nil { + t.Fatalf("opening client end: %v", err) + } + return f +} + +func setClientBaud(t *testing.T, f *os.File, cbaud uint32) { + t.Helper() + tio, err := unix.IoctlGetTermios(int(f.Fd()), unix.TCGETS) + if err != nil { + t.Fatalf("TCGETS: %v", err) + } + tio.Cflag = (tio.Cflag &^ unix.CBAUD) | cbaud + if err := unix.IoctlSetTermios(int(f.Fd()), unix.TCSETS, tio); err != nil { + t.Fatalf("TCSETS: %v", err) + } +} + +// The device node has to exist as soon as the host publishes it, whether or +// not anything ever opens it. +func TestSerialDevicePublishesNode(t *testing.T) { + newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + link := filepath.Join(serialDevDir, "ttyTEST0") + target, err := os.Readlink(link) + if err != nil { + t.Fatalf("device node not published: %v", err) + } + if target != dev.slavePath { + t.Fatalf("node points at %q, want %q", target, dev.slavePath) + } +} + +// Nothing has the device open, so nothing should be holding the physical +// port on the host. This is the primed-hangup trick working: without it the +// loop cannot tell "never opened" from "open and idle" and attaches to a +// board nobody asked for. +func TestSerialNoAttachUntilClientOpens(t *testing.T) { + host := newFakeHost(t) + startDevice(t, "ttyTEST0") + + if g, ok := host.waitGreeting(500 * time.Millisecond); ok { + t.Fatalf("attached with no client holding the device: %+v", g) + } +} + +// Opening the device is what makes the host open the real port — the event +// an Arduino sees as its reset pulse. +func TestSerialAttachOnClientOpen(t *testing.T) { + host := newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + client := openClient(t, dev) + defer client.Close() + + g, ok := host.waitGreeting(5 * time.Second) + if !ok { + t.Fatal("no attach after the client opened the device") + } + if g.Op != "attach" || g.Name != "ttyTEST0" { + t.Fatalf("unexpected greeting: %+v", g) + } + if g.Mode == nil { + t.Fatal("attach greeting carried no mode; the host would open at a guessed baud") + } +} + +// Closing the device must drop the attachment, which is what makes the host +// release the port — both so the reset pulse has a falling edge and so the +// Mac's own tooling can have the board back. +func TestSerialDetachOnClientClose(t *testing.T) { + host := newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + client := openClient(t, dev) + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("no attach") + } + conn := <-host.conns + _ = client.Close() + + // The host sees the attachment end as its connection closing. + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + buf := make([]byte, 1) + if _, err := conn.Read(buf); err == nil { + t.Fatal("attachment outlived the client that opened the device") + } +} + +// Close then reopen is every upload to a native-USB board: the client drops +// the port, the board reboots into its bootloader, the client comes back. +// The host has to see a matching close and reopen of the physical port, or +// the board never gets its reset — so this asserts the detach lands +// *between* the two attaches rather than just that a second one happens. +func TestSerialReattachAfterClientReopens(t *testing.T) { + host := newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + client := openClient(t, dev) + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("no first attach") + } + conn := <-host.conns + _ = client.Close() + + // Wait for the host to observe the detach before reopening. Any real + // close/reopen has a gap — the board is re-enumerating on the USB bus — + // and closing and reopening with no gap at all would be asking the + // agent to notice a state the kernel never held long enough to report. + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + if _, err := conn.Read(make([]byte, 1)); err == nil { + t.Fatal("attachment outlived the client") + } + + client2 := openClient(t, dev) + defer client2.Close() + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("device never reattached after the client reopened it") + } +} + +func TestSerialForwardsBytesToHost(t *testing.T) { + host := newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + client := openClient(t, dev) + defer client.Close() + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("no attach") + } + + if _, err := client.Write([]byte("hello board")); err != nil { + t.Fatalf("client write: %v", err) + } + f, ok := host.waitFrame(serialFrameData, 5*time.Second) + if !ok { + t.Fatal("client output never reached the host") + } + if string(f.payload) != "hello board" { + t.Fatalf("host got %q", f.payload) + } +} + +func TestSerialForwardsBytesToClient(t *testing.T) { + host := newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + client := openClient(t, dev) + defer client.Close() + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("no attach") + } + conn := <-host.conns + + if err := serialWriteFrame(conn, serialFrameData, []byte("hello guest")); err != nil { + t.Fatalf("host write: %v", err) + } + + _ = client.SetReadDeadline(time.Now().Add(5 * time.Second)) + buf := make([]byte, 64) + n, err := client.Read(buf) + if err != nil { + t.Fatalf("client read: %v", err) + } + if string(buf[:n]) != "hello guest" { + t.Fatalf("client got %q", buf[:n]) + } +} + +// A baud change produces no event of any kind on a PTY — the loop has to +// notice it by looking. This is the mechanism the 1200-baud touch rides on, +// so it gets its own test. +func TestSerialReportsBaudChange(t *testing.T) { + host := newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + client := openClient(t, dev) + defer client.Close() + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("no attach") + } + + setClientBaud(t, client, unix.B115200) + + f, ok := host.waitFrame(serialFrameMode, 5*time.Second) + if !ok { + t.Fatal("baud change never reached the host") + } + var mode serialMode + if err := json.Unmarshal(f.payload, &mode); err != nil { + t.Fatalf("bad mode frame: %v", err) + } + if mode.Baud != 115200 { + t.Fatalf("host told %d baud, want 115200", mode.Baud) + } +} + +// The 1200-baud touch in full: set 1200, then close immediately. The mode +// has to reach the host before the detach, or the board never enters its +// bootloader and every upload to a Leonardo or ESP32-S3 fails. +func TestSerial1200BaudTouchReachesHostBeforeDetach(t *testing.T) { + host := newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + client := openClient(t, dev) + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("no attach") + } + + setClientBaud(t, client, unix.B1200) + _ = client.Close() + + f, ok := host.waitFrame(serialFrameMode, 5*time.Second) + if !ok { + t.Fatal("the 1200-baud touch was lost when the client closed") + } + var mode serialMode + if err := json.Unmarshal(f.payload, &mode); err != nil { + t.Fatalf("bad mode frame: %v", err) + } + if mode.Baud != 1200 { + t.Fatalf("host told %d baud, want 1200", mode.Baud) + } +} + +// An unchanged configuration must not produce a frame every time the loop +// looks — that would be 10 pointless wakeups a second on the host for as +// long as a monitor is open. +func TestSerialDoesNotRepeatUnchangedMode(t *testing.T) { + host := newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + client := openClient(t, dev) + defer client.Close() + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("no attach") + } + + if _, ok := host.waitFrame(serialFrameMode, 700*time.Millisecond); ok { + t.Fatal("resent an unchanged mode") + } +} + +// A refused attach — an unplugged board, or one the Mac's own tooling holds +// — must not spin. The client keeps the device open, so the agent keeps +// trying, but at a pace that doesn't burn a core. +func TestSerialRefusedAttachRetriesWithoutSpinning(t *testing.T) { + host := newFakeHost(t) + host.refuse = true + dev := startDevice(t, "ttyTEST0") + + client := openClient(t, dev) + defer client.Close() + + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("never tried to attach") + } + // Second attempt should come after the retry interval, not immediately. + start := time.Now() + if _, ok := host.waitGreeting(5 * time.Second); !ok { + t.Fatal("gave up after one refusal while the client still held the device") + } + if elapsed := time.Since(start); elapsed < serialAttachRetryInterval/2 { + t.Fatalf("retried after %v; that is a spin, not a retry", elapsed) + } +} + +// Removing a device from the published set has to take the node with it. A +// leftover /dev entry would accept an open and then sit mute, which reads +// as broken hardware rather than an absent forward. +func TestSerialStopRemovesNode(t *testing.T) { + newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + link := filepath.Join(serialDevDir, "ttyTEST0") + + if _, err := os.Readlink(link); err != nil { + t.Fatalf("node was never published: %v", err) + } + dev.stop() + + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Readlink(link); err != nil { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("device node outlived the device") +} + +// The name arrives over the wire and becomes a path, so the guest validates +// it again rather than trusting the host to have done so. +func TestSerialApplyRejectsUnsafeNames(t *testing.T) { + newFakeHost(t) + dir := t.TempDir() + prev := serialDevDir + serialDevDir = dir + t.Cleanup(func() { serialDevDir = prev }) + + f := &serialForwarder{log: log.New(io.Discard, "", 0), devices: map[string]*serialDevice{}} + f.apply([]serialDeviceMsg{ + {Name: "../escape"}, + {Name: "sub/dir"}, + {Name: ""}, + {Name: ".hidden"}, + }) + t.Cleanup(func() { f.apply(nil) }) + + if len(f.devices) != 0 { + t.Fatalf("created %d device(s) from unsafe names", len(f.devices)) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("unsafe names left %d entries behind", len(entries)) + } +} + +// A writer can be gone before the loop ever sees it open. `echo cmd > +// /dev/ttyACM0`, and every script that does printf into a device, opens, +// writes and closes in microseconds — far inside serialPollInterval — so the +// loop's first sight of it is POLLIN and POLLHUP together. The two flags are +// independent (input_available_p vs TTY_OTHER_CLOSED), and honouring the +// hangup first discarded the bytes with no error on either side. +func TestSerialShortLivedWriterIsNotLost(t *testing.T) { + host := newFakeHost(t) + dev := startDevice(t, "ttyTEST0") + + // Let the loop settle into its detached backoff first, so the whole + // open-write-close below lands between two polls. Catching the client + // while it still holds the device would exercise the ordinary attached + // path and prove nothing about this one. + time.Sleep(150 * time.Millisecond) + + client := openClient(t, dev) + if _, err := client.Write([]byte("reset\r\n")); err != nil { + t.Fatalf("client write: %v", err) + } + _ = client.Close() + + f, ok := host.waitFrame(serialFrameData, 5*time.Second) + if !ok { + t.Fatal("a write followed immediately by close never reached the host") + } + if string(f.payload) != "reset\r\n" { + t.Fatalf("host got %q, want %q", f.payload, "reset\r\n") + } + + // And the flush must not leave the port held: the host closes the + // physical device when the attachment goes, so a drain that forgot to + // let go would keep the board away from the Mac indefinitely. + if _, ok := host.waitGreeting(time.Second); !ok { + t.Fatal("no attach greeting for the flush") + } +} diff --git a/internal/agentembed/serialfwd_lockstep_test.go b/internal/agentembed/serialfwd_lockstep_test.go new file mode 100644 index 0000000..3e09bf0 --- /dev/null +++ b/internal/agentembed/serialfwd_lockstep_test.go @@ -0,0 +1,83 @@ +package agentembed + +import ( + "fmt" + "reflect" + "testing" + + "github.com/clawkwork/clawk/internal/serialfwd" + "github.com/stretchr/testify/require" +) + +// The guest agent builds standalone inside the guest, so it can't import +// internal/serialfwd — the serial protocol is transcribed into main.go.in by +// hand. Nothing else notices when the two drift: a renamed JSON field or a +// moved frame type compiles fine on both sides and fails only as a sandbox +// where a board is visible but mute. +// +// So: check the parts that have to match byte-for-byte against the +// parent-module definition, deriving them from the real declarations rather +// than restating them, so this test can't drift either. Same rule and same +// shape as TestReverseForwardProtocolMirroredInAgent. +func TestSerialProtocolMirroredInAgent(t *testing.T) { + src := string(AgentMainGo) + + require.Contains(t, src, fmt.Sprintf("serialVSockPort = %d", serialfwd.VSockPort), + "guest dials a different vsock port than the host listens on") + require.Contains(t, src, fmt.Sprintf("serialProtoVersion = %d", serialfwd.ProtoVersion), + "guest announces a protocol version the host will hang up on") + require.Contains(t, src, fmt.Sprintf("serialMaxLine = %d", serialfwd.MaxLineBytes/1024)+" * 1024", + "guest and host disagree on the control-line cap") + + // Framing constants. A mismatch here desynchronises the stream rather + // than failing cleanly, which is the worst way for this to break. + require.Contains(t, src, fmt.Sprintf("serialFrameHeaderBytes = %d", serialfwd.FrameHeaderBytes), + "guest frames a different header length than the host parses") + require.Contains(t, src, fmt.Sprintf("serialFrameData byte = 0x%02x", byte(serialfwd.FrameData)), + "guest tags data frames differently from the host") + require.Contains(t, src, fmt.Sprintf("serialFrameMode byte = 0x%02x", byte(serialfwd.FrameMode)), + "guest tags mode frames differently from the host") + + // Op values are string literals on the wire, sent by the guest and + // switched on by the host. + for _, op := range []string{serialfwd.OpControl, serialfwd.OpAttach} { + require.Contains(t, src, fmt.Sprintf("Op: %q", op), + "guest never sends the %q greeting", op) + } + + // Parity travels as a one-letter string; a guest that spelled it + // differently would be rejected by the host's mode validation and the + // port would silently keep its old settings. + for _, parity := range []string{serialfwd.ParityNone, serialfwd.ParityEven, serialfwd.ParityOdd} { + require.Contains(t, src, fmt.Sprintf("%q", parity), + "guest never produces parity %q", parity) + } + + for _, typ := range []any{ + serialfwd.Greeting{}, serialfwd.Snapshot{}, serialfwd.Device{}, + serialfwd.AttachReply{}, serialfwd.Mode{}, + } { + rt := reflect.TypeOf(typ) + for i := range rt.NumField() { + tag, ok := rt.Field(i).Tag.Lookup("json") + if !ok { + continue + } + want := fmt.Sprintf("`json:%q`", tag) + require.Contains(t, src, want, + "guest is missing the %s.%s field tag %s", + rt.Name(), rt.Field(i).Name, want) + } + } +} + +// The guest turns a published device name into /dev/. The host +// validates it first, but the guest must not rely on that — this checks the +// second line of defence is actually wired up rather than merely present. +func TestSerialDeviceNameValidatedInAgent(t *testing.T) { + src := string(AgentMainGo) + require.Contains(t, src, "func validSerialName(", + "guest has no device-name validation") + require.Contains(t, src, "if err := validSerialName(d.Name); err != nil", + "guest validates device names but never calls the validator on a published set") +} diff --git a/internal/cli/agent_session.go b/internal/cli/agent_session.go index 044ca35..412b397 100644 --- a/internal/cli/agent_session.go +++ b/internal/cli/agent_session.go @@ -228,10 +228,10 @@ func tryVSockAgent(sb *config.Sandbox, provider sandbox.Provider, agent Agent, e cfg := vsockclient.Config{ SocketPath: sockPath, Cmd: cmd, - Args: append(append([]string{}, agent.DefaultArgs...), extra...), + Args: launchArgs(sb, agent, extra), Cwd: agentStartDir(provider, sb), User: sandbox.GuestUser, - Env: buildVSockEnv(), + Env: buildVSockEnv(sb), // Agents are full-screen TUIs: clear so they don't overdraw the // CLI's boot narration. ClearScreen: true, @@ -271,15 +271,77 @@ func tryVSockAgent(sb *config.Sandbox, provider sandbox.Provider, agent Agent, e // vsock path the env must be threaded through the handshake here. // Loading on every dispatch picks up edits to the token file without // requiring a sandbox rebuild. -func buildVSockEnv() []string { +// +// The sandbox's own `env ( … )` entries ride along for the same reason: +// without them the runner process cannot see them at all, only its +// login-shell children can. Anything that reads the environment of the +// runner itself therefore needs this — notably `${VAR}` references in the +// generated MCP config (see sandbox.MCPConfigFile), whose expansion +// happens in the runner's process env, so a PAT delivered only via +// profile.d would silently expand to empty. +// +// Ordering and precedence: clawk.mod entries go first and clawk's own vars +// last, because the guest agent folds the slice into a map in order (last +// write wins) — EXCEPT for a name the sandbox declared itself, which clawk +// then does not emit at all. +// +// That exception is the whole point. clawk's defaults exist so the common +// case works with no configuration, not to overrule a user who wrote the +// variable down. Concretely: a sandbox pointed at a gateway that needs no +// credential of its own (ANTHROPIC_BASE_URL, no ANTHROPIC_AUTH_TOKEN) has +// claude fall back to CLAUDE_CODE_OAUTH_TOKEN, so clawk's injected +// sk-ant-oat-… goes to that third-party endpoint as an Authorization +// header — and before this, a clawk.mod declaration was silently +// overwritten, so no config could stop it. Nobody writes +// `CLAUDE_CODE_OAUTH_TOKEN = …` in a clawk.mod by accident, so treating it +// as intent costs nothing. Same precedence rule as launchArgs, where the +// user's `-- ` land after clawk's defaults. +// +// A declared-but-empty value is emitted as empty rather than dropped, so +// it still shadows whatever the image or a parent process might have set. +// Note that empty is not the same as absent: the guest agent folds these +// into a map (agentembed/main.go.in, buildChildEnv), so `NAME=` reaches the +// child set-but-empty. Whether that reads as "unset" is the consumer's +// call — claude treats an empty credential as unset, but not every program +// does, and clawk has no way to express a true unset today. +// +// Resolution here is best-effort by design: attach is the hot path and +// must not become unusable because a variable left the host shell since +// create (the guest still has the value profile.d captured then). A +// failure warns and the remaining entries still go through — but the +// failing name stays suppressed, because the alternative is clawk quietly +// re-injecting the credential the declaration existed to displace. +func buildVSockEnv(sb *config.Sandbox) []string { env := []string{} + modEnv, err := sandbox.ResolveEnv(sb) + if err != nil { + fmt.Fprintf(os.Stderr, + "warning: some clawk.mod env vars could not be resolved for "+ + "sandbox %q; the agent process will not see them: %v\n", + sb.DisplayName(), err) + } + env = append(env, modEnv...) + + // Names the sandbox set for itself; clawk yields on every one of them. + // + // Taken from the declarations, NOT from modEnv: an entry that failed to + // resolve is still a name the user spoke for, and reading it back out of + // the resolved set would hand it to clawk again — see + // sandbox.DeclaredEnvNames. + declared := sandbox.DeclaredEnvNames(sb) + add := func(k, v string) { + if !declared[k] { + env = append(env, k+"="+v) + } + } + if tok, _ := sandbox.LoadOAuthToken(clawkRoot()); tok != "" { - env = append(env, "CLAUDE_CODE_OAUTH_TOKEN="+tok) + add("CLAUDE_CODE_OAUTH_TOKEN", tok) } if v := os.Getenv("COLORTERM"); v != "" { - env = append(env, "COLORTERM="+v) + add("COLORTERM", v) } else { - env = append(env, "COLORTERM=truecolor") + add("COLORTERM", "truecolor") } // Forward terminal-identification env vars verbatim. Empty values // are dropped — we don't want to claim "I'm iTerm2" when the user @@ -293,14 +355,14 @@ func buildVSockEnv() []string { "ITERM_PROFILE", // ditto } { if v := os.Getenv(k); v != "" { - env = append(env, k+"="+v) + add(k, v) } } if v := os.Getenv("LANG"); v != "" { - env = append(env, "LANG="+v) + add("LANG", v) } if v := os.Getenv("LC_ALL"); v != "" { - env = append(env, "LC_ALL="+v) + add("LC_ALL", v) } return env } @@ -321,7 +383,7 @@ func attachAgentViaExec(sb *config.Sandbox, provider sandbox.Provider, agent Age if !ok { return fmt.Errorf("provider for %q cannot exec the agent", sb.Name) } - defaults := strings.Join(quoteAll(agent.DefaultArgs), " ") + defaults := strings.Join(quoteAll(launchArgs(sb, agent, nil)), " ") cmdline := `: "${TERM:=xterm-256color}"; : "${COLORTERM:=truecolor}"; ` + "export TERM COLORTERM; " + "cd " + agentStartDir(provider, sb) + " 2>/dev/null || true; " + diff --git a/internal/cli/agent_session_test.go b/internal/cli/agent_session_test.go index a2f4679..29d4753 100644 --- a/internal/cli/agent_session_test.go +++ b/internal/cli/agent_session_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/clawkwork/clawk/internal/config" "github.com/clawkwork/clawk/internal/sandbox" "github.com/stretchr/testify/require" ) @@ -38,7 +39,7 @@ func TestBuildVSockEnvForwardsOAuthToken(t *testing.T) { t.Setenv("CLAUDE_CODE_OAUTH_TOKEN", "") require.NoError(t, sandbox.SaveOAuthToken(filepath.Join(home, ".clawk"), "sk-test-vsock")) - env := buildVSockEnv() + env := buildVSockEnv(&config.Sandbox{Name: "sb"}) want := "CLAUDE_CODE_OAUTH_TOKEN=sk-test-vsock" for _, e := range env { if e == want { @@ -56,9 +57,157 @@ func TestBuildVSockEnvOmitsTokenWhenAbsent(t *testing.T) { _, err := os.Stat(filepath.Join(home, ".clawk", "claude-oauth-token")) require.True(t, os.IsNotExist(err), "expected no token file, got err=%v", err) - for _, e := range buildVSockEnv() { + for _, e := range buildVSockEnv(&config.Sandbox{Name: "sb"}) { if strings.HasPrefix(e, "CLAUDE_CODE_OAUTH_TOKEN=") { t.Errorf("unexpected token entry when unconfigured: %q", e) } } } + +// TestBuildVSockEnvForwardsModEnv is the guard for the same class of bug as +// the OAuth token above, for clawk.mod `env ( … )`: the runner process is +// spawned non-login, so a var delivered only through /etc/profile.d/ is +// invisible to it — visible to its shell children, but not to the runner +// itself. Anything reading the runner's own environment therefore breaks, +// notably `${VAR}` expansion inside the generated MCP config, where an +// unforwarded PAT silently becomes an empty Authorization header. +func TestBuildVSockEnvForwardsModEnv(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("CLAUDE_CODE_OAUTH_TOKEN", "") + t.Setenv("CLAWK_TEST_PAT", "pat-abc123") + + sb := &config.Sandbox{Name: "sb", RequiredEnv: []string{ + "LINEAR_TOKEN=${CLAWK_TEST_PAT}", + "LITERAL=plain", + }} + env := buildVSockEnv(sb) + require.Contains(t, env, "LINEAR_TOKEN=pat-abc123", "aliased host var must reach the runner") + require.Contains(t, env, "LITERAL=plain", "literal must reach the runner") +} + +// TestBuildVSockEnvClawkVarsWinOrdering pins the layering for names the +// sandbox did NOT declare: the guest agent folds the handshake env into a +// map in slice order (agentembed/main.go.in, buildChildEnv), so the LAST +// occurrence wins, and clawk's own vars are emitted after the sandbox's +// declared env. +func TestBuildVSockEnvClawkVarsWinOrdering(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("CLAUDE_CODE_OAUTH_TOKEN", "") + t.Setenv("COLORTERM", "") + require.NoError(t, sandbox.SaveOAuthToken(filepath.Join(home, ".clawk"), "sk-real")) + + sb := &config.Sandbox{Name: "sb", RequiredEnv: []string{"UNRELATED=x"}} + env := buildVSockEnv(sb) + + idx := func(prefix string) int { + for i, e := range env { + if strings.HasPrefix(e, prefix) { + return i + } + } + return -1 + } + require.Contains(t, env, "CLAUDE_CODE_OAUTH_TOKEN=sk-real") + require.Greater(t, idx("CLAUDE_CODE_OAUTH_TOKEN="), idx("UNRELATED="), + "undeclared clawk vars still come after the sandbox's own entries") +} + +// TestBuildVSockEnvDeclarationOverridesClawkVar is the counterpart, and the +// reason the rule above carves out declared names: a sandbox pointed at a +// non-Anthropic provider must be able to stop clawk injecting an Anthropic +// credential, or the runner authenticates against the wrong service and +// every request fails with an unrecognized-model error. Before this, the +// declaration was silently overwritten and `clawk auth clear` — which +// disarms EVERY sandbox on the host — was the only lever. +func TestBuildVSockEnvDeclarationOverridesClawkVar(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("CLAUDE_CODE_OAUTH_TOKEN", "") + require.NoError(t, sandbox.SaveOAuthToken(filepath.Join(home, ".clawk"), "sk-real")) + + sb := &config.Sandbox{Name: "sb", RequiredEnv: []string{ + `CLAUDE_CODE_OAUTH_TOKEN=""`, + }} + env := buildVSockEnv(sb) + + var got []string + for _, e := range env { + if strings.HasPrefix(e, "CLAUDE_CODE_OAUTH_TOKEN=") { + got = append(got, e) + } + } + require.Equal(t, []string{"CLAUDE_CODE_OAUTH_TOKEN="}, got, + "the declared value must be the only occurrence — clawk must not re-add its own") +} + +// TestBuildVSockEnvDeclarationOverridesTerminalVar pins that the carve-out +// is a general precedence rule, not a special case for the OAuth token. +func TestBuildVSockEnvDeclarationOverridesTerminalVar(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("CLAUDE_CODE_OAUTH_TOKEN", "") + t.Setenv("COLORTERM", "truecolor") + + sb := &config.Sandbox{Name: "sb", RequiredEnv: []string{"COLORTERM=256"}} + env := buildVSockEnv(sb) + + var got []string + for _, e := range env { + if strings.HasPrefix(e, "COLORTERM=") { + got = append(got, e) + } + } + require.Equal(t, []string{"COLORTERM=256"}, got) +} + +// TestBuildVSockEnvSurvivesUnresolvableEntry covers the best-effort +// contract: attach is the hot path, so a `${HOST:?msg}` that no longer +// resolves in this shell must not make the sandbox unreachable. The +// failing entry drops out; everything else still goes through. +func TestBuildVSockEnvSurvivesUnresolvableEntry(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("CLAUDE_CODE_OAUTH_TOKEN", "") + t.Setenv("CLAWK_TEST_PRESENT", "here") + + sb := &config.Sandbox{Name: "sb", RequiredEnv: []string{ + "GONE=${CLAWK_TEST_ABSENT:?set it on the host}", + "KEPT=${CLAWK_TEST_PRESENT}", + }} + env := buildVSockEnv(sb) + require.Contains(t, env, "KEPT=here", "resolvable entries must survive a failing sibling") + for _, e := range env { + require.False(t, strings.HasPrefix(e, "GONE="), "unresolvable entry must be dropped: %q", e) + } +} + +// TestBuildVSockEnvDeclarationOverridesEvenWhenUnresolvable is the security +// counterpart to TestBuildVSockEnvSurvivesUnresolvableEntry: best-effort +// resolution must not become best-effort PRECEDENCE. +// +// A sandbox pointed at a non-Anthropic gateway disowns clawk's token by +// declaring the name itself. If that declaration reads a host variable that +// has since left the shell, the entry drops out of ResolveEnv's output — and +// deriving the "declared" set from that output handed the name straight back +// to clawk, which injected its own sk-ant-oat-… as an Authorization header to +// the third-party endpoint. Exactly the leak the declaration existed to stop, +// reachable from any shell missing the gateway variable. +func TestBuildVSockEnvDeclarationOverridesEvenWhenUnresolvable(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("CLAUDE_CODE_OAUTH_TOKEN", "") + // "sk-real" like the fixtures above, deliberately NOT the live sk-ant- + // prefix: this repo is public, and a literal shaped like a real Anthropic + // token trips secret scanners and push protection for no benefit. The + // assertion only needs a string distinctive enough to spot in the output. + require.NoError(t, sandbox.SaveOAuthToken(filepath.Join(home, ".clawk"), "sk-real")) + + sb := &config.Sandbox{Name: "sb", RequiredEnv: []string{ + "CLAUDE_CODE_OAUTH_TOKEN=${CLAWK_TEST_GATEWAY:?set it on the host}", + }} + env := buildVSockEnv(sb) + + for _, e := range env { + require.NotContains(t, e, "sk-real", + "a declared-but-unresolvable name must stay suppressed, not fall back to clawk's token") + } +} diff --git a/internal/cli/agents.go b/internal/cli/agents.go index d34e14b..3151181 100644 --- a/internal/cli/agents.go +++ b/internal/cli/agents.go @@ -3,13 +3,16 @@ package cli import ( "fmt" "sort" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/sandbox" ) // Agent describes how a coding-agent runner is launched inside a -// sandbox. Treating each agent (claude, codex, opencode, ...) as a +// sandbox. Treating each agent (claude, codex, pi, opencode, ...) as a // data-driven entry rather than a per-agent code path keeps the CLI // surface symmetric: every agent's verb works the same way, and adding -// a fourth runner is a one-line registry change. +// another runner is a one-line registry change. type Agent struct { // Name is the user-facing runner argument (`clawk run `). // Must be unique. Lowercase ASCII, no whitespace. Also the command @@ -18,9 +21,23 @@ type Agent struct { Name string // DefaultArgs are prepended to whatever the user passes through - // `-- `. Claude and Codex both have explicit "externally - // sandboxed" modes, so clawk enables those by default. + // `-- `. Every runner that can be told "you are already + // externally sandboxed" is told so here: claude and codex have + // explicit modes for it, pi has project trust. DefaultArgs []string + + // MCPConfigFlag is the runner's flag for loading an external MCP + // server config file, or "" if clawk doesn't know how to hand this + // runner one. When set and the sandbox declares servers, the flag and + // sandbox.GuestMCPConfigPath are appended to the launch args. + // + // A per-runner flag rather than a line in DefaultArgs because the + // value is conditional (a sandbox with no `mcp ( … )` block must not + // get a flag pointing at a file that isn't there) and because each + // runner spells this differently — codex and opencode use their own + // config formats and pi loads MCP through an extension, so they stay + // empty until someone renders those too. + MCPConfigFlag string } // agents is the builtin runner registry. `clawk run ` consults @@ -29,16 +46,65 @@ var agents = []Agent{ { Name: "claude", DefaultArgs: []string{"--dangerously-skip-permissions"}, + // Not --strict-mcp-config: that would suppress every other MCP + // source, including the claude.ai account connectors that already + // work inside a sandbox (they're proxied server-side, so they need + // no local credential and no egress of their own) and any plugin + // the user's settings enable. + MCPConfigFlag: "--mcp-config", }, { Name: "codex", DefaultArgs: []string{"--dangerously-bypass-approvals-and-sandbox"}, }, + { + Name: "pi", + // pi has no approval prompts to bypass — it ships no built-in + // sandbox at all, by design (its docs/security.md: "Real isolation + // needs to come from the operating system or a virtualization + // boundary", which is exactly what a clawk VM is). What it does + // gate is project trust: a repo carrying .pi/settings.json, + // .pi/extensions or .agents/skills triggers an interactive + // "trust this project?" prompt before those load, and answering it + // is meaningless inside a VM whose whole premise is that the + // project already has the machine. --approve trusts the project for + // the run, the same posture as claude's and codex's flags above. + DefaultArgs: []string{"--approve"}, + // No MCPConfigFlag: pi speaks MCP through an extension + // (pi-mcp-adapter), not a config-file flag, so a sandbox with an + // `mcp ( … )` block gets nothing here until that's rendered too. + }, { Name: "opencode", + // opencode gates every tool call behind a permission prompt unless + // told otherwise; --auto approves anything not explicitly denied + // (its own help says "dangerous!", which is true on a laptop and + // beside the point inside a disposable VM). A deny rule in the + // user's opencode.jsonc still wins, so this widens the default + // without overriding a considered choice. + DefaultArgs: []string{"--auto"}, + // No MCPConfigFlag: opencode manages MCP servers through its own + // config file and `opencode mcp` subcommand, not a flag taking a + // path, so a sandbox's `mcp ( … )` block doesn't reach it yet. }, } +// launchArgs assembles the runner's argv tail: its permission-mode +// defaults, the MCP config flag when this sandbox declares servers and the +// runner knows how to take one, then the user's `-- ` last so an +// explicit flag always sits after (and therefore overrides) clawk's. +// +// Both attach paths — the vsock handshake and the `bash -lc` exec fallback +// — go through here, so a runner never gets a different command line +// depending on which transport happened to be available. +func launchArgs(sb *config.Sandbox, agent Agent, extra []string) []string { + args := append([]string{}, agent.DefaultArgs...) + if agent.MCPConfigFlag != "" && len(sb.MCP) > 0 { + args = append(args, agent.MCPConfigFlag, sandbox.GuestMCPConfigPath) + } + return append(args, extra...) +} + // agentByName returns the registry entry for a given runner name. // Used by `clawk run ` dispatch. func agentByName(name string) (Agent, error) { diff --git a/internal/cli/agents_test.go b/internal/cli/agents_test.go new file mode 100644 index 0000000..15e026d --- /dev/null +++ b/internal/cli/agents_test.go @@ -0,0 +1,58 @@ +package cli + +import ( + "strings" + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/sandbox" + "github.com/stretchr/testify/require" +) + +// TestPiRunnerRegistered pins the pi harness's registry entry. pi ships no +// built-in sandbox (its security docs say isolation must come from a VM, +// which is what clawk provides), so the only thing to pre-answer is project +// trust: without --approve a repo carrying .pi/settings.json or .pi/extensions +// stops the run on an interactive prompt that means nothing inside a sandbox. +func TestPiRunnerRegistered(t *testing.T) { + pi, err := agentByName("pi") + require.NoError(t, err) + require.Equal(t, []string{"--approve"}, pi.DefaultArgs) + require.Empty(t, pi.MCPConfigFlag, + "pi loads MCP through an extension, not a config-file flag") + + // The user's own args land after clawk's, so an explicit --no-approve + // still wins. + withMCP := &config.Sandbox{MCP: []config.MCPServer{{Name: "linear"}}} + require.Equal(t, []string{"--approve", "--no-approve"}, + launchArgs(withMCP, pi, []string{"--no-approve"}), + "a runner clawk can't render MCP config for gets no flag, and user args go last") +} + +// TestAgentRegistryNames guards the invariants the registry's doc comment +// states: names are unique, and each is usable both as a `clawk run` argument +// and as a bare command on the guest's PATH. +func TestAgentRegistryNames(t *testing.T) { + seen := map[string]bool{} + for _, a := range agents { + require.Falsef(t, seen[a.Name], "duplicate runner name %q", a.Name) + seen[a.Name] = true + require.Equalf(t, strings.ToLower(a.Name), a.Name, "runner %q must be lowercase", a.Name) + require.NotContainsf(t, a.Name, " ", "runner %q must not contain whitespace", a.Name) + } + require.Subset(t, seen, map[string]bool{"claude": true, "codex": true, "pi": true, "opencode": true}) + + // Runner names are reserved as sandbox names — a sandbox called "pi" + // would make `clawk run pi` ambiguous. + require.Contains(t, reservedAgentNames(), "pi") +} + +// TestAgentStateDirsNameRealRunners keeps the persistence list honest: every +// home directory clawk mounts must belong to a runner someone can actually +// launch. A typo there costs a PCIe device per sandbox and persists nothing. +func TestAgentStateDirsNameRealRunners(t *testing.T) { + for _, d := range sandbox.AgentStateDirs { + _, err := agentByName(d.Agent) + require.NoErrorf(t, err, "AgentStateDirs names %q, which is not a registered runner", d.Agent) + } +} diff --git a/internal/cli/apply.go b/internal/cli/apply.go index 5a3cdfd..427f7ed 100644 --- a/internal/cli/apply.go +++ b/internal/cli/apply.go @@ -193,6 +193,14 @@ func namespaceFromDef(def template.NamespaceDef, dryRun bool) (*config.Namespace if err != nil { return nil, err } + var mcpSources []mcpSource + for _, s := range tmpl.MCP { + mcpSources = append(mcpSources, mcpSource{Origin: "manifest", Spec: s}) + } + mcpServers, err := composeMCP(mcpSources) + if err != nil { + return nil, err + } denied := make([]string, 0, len(tmpl.DenyDomains)) for _, d := range tmpl.DenyDomains { @@ -217,5 +225,6 @@ func namespaceFromDef(def template.NamespaceDef, dryRun bool) (*config.Namespace Files: files, Shares: shares, Env: dedupStrings(tmpl.Env), + MCP: mcpServers, }, nil } diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go index c3d5a63..166d002 100644 --- a/internal/cli/cli_test.go +++ b/internal/cli/cli_test.go @@ -60,6 +60,7 @@ func setupTest(t *testing.T) (*config.Store, *sandbox.MockProvider) { statusJSON = false statusBrief = false forwardListJSON = false + serialListJSON = false networkListJSON = false worktreeListJSON = false imageFlag = "" diff --git a/internal/cli/complete.go b/internal/cli/complete.go index 2444e28..275fbca 100644 --- a/internal/cli/complete.go +++ b/internal/cli/complete.go @@ -109,7 +109,7 @@ func runnerNames() []string { // completeRunArgs is the position-aware ValidArgsFunction for `clawk run`: // -// - position 0: runner name (claude / codex / opencode / shell) +// - position 0: runner name (claude / codex / pi / opencode / shell) // - position 1: sandbox name // - position 2+: nothing (runner passthrough args follow `--`) func completeRunArgs(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { diff --git a/internal/cli/compose_mcp.go b/internal/cli/compose_mcp.go new file mode 100644 index 0000000..4eda4c7 --- /dev/null +++ b/internal/cli/compose_mcp.go @@ -0,0 +1,173 @@ +package cli + +import ( + "fmt" + "net/netip" + "net/url" + "slices" + "strings" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/template" +) + +// mergeMCP gathers `mcp (...)` entries from the workspace and every repo +// Clawkfile into the sandbox's server list. +// +// Same conflict rule as mergeFiles/mergeShares: two sources declaring the +// same server name identically collapse to one entry, but a genuine +// disagreement about what that name means is a config bug rather than +// something to silently tie-break — the agent would otherwise get whichever +// definition happened to be composed first. +func mergeMCP(ws *template.Workspace) ([]config.MCPServer, error) { + var sources []mcpSource + for _, s := range ws.File.MCP { + sources = append(sources, mcpSource{Origin: "workspace", Spec: s}) + } + for _, r := range ws.Repos { + if r.Clawkfile == nil { + continue + } + for _, s := range r.Clawkfile.MCP { + sources = append(sources, mcpSource{Origin: r.Name, Spec: s}) + } + } + return composeMCP(sources) +} + +type mcpSource struct { + Origin string + Spec template.MCPSpec +} + +// composeMCP is the pure conflict-detection core, split out so the here-mode +// path (a single Clawkfile) and the workspace path share it. +func composeMCP(sources []mcpSource) ([]config.MCPServer, error) { + byName := make(map[string]struct { + Origin string + Server config.MCPServer + }) + var out []config.MCPServer + for _, s := range sources { + srv := config.MCPServer{ + Name: s.Spec.Name, + Transport: s.Spec.Transport, + URL: s.Spec.URL, + Command: s.Spec.Command, + Headers: s.Spec.Headers, + Env: s.Spec.Env, + } + if prev, dup := byName[srv.Name]; dup { + if sameMCPServer(prev.Server, srv) { + continue + } + return nil, fmt.Errorf( + "mcp server %q declared differently by %s (%s) and %s (%s) — rename one", + srv.Name, prev.Origin, mcpTarget(prev.Server), s.Origin, mcpTarget(srv)) + } + byName[srv.Name] = struct { + Origin string + Server config.MCPServer + }{s.Origin, srv} + out = append(out, srv) + } + return out, nil +} + +// sameMCPServer reports whether two declarations of one server name mean the +// same thing, so repeated identical entries collapse instead of erroring. +func sameMCPServer(a, b config.MCPServer) bool { + return a.Transport == b.Transport && + a.URL == b.URL && + slices.Equal(a.Command, b.Command) && + slices.Equal(a.Headers, b.Headers) && + slices.Equal(a.Env, b.Env) +} + +// mcpTarget renders a server's endpoint for error messages: the URL for a +// remote server, the command for a stdio one. +func mcpTarget(s config.MCPServer) string { + if s.URL != "" { + return s.URL + } + return strings.Join(s.Command, " ") +} + +// appendMissingMCP folds add into have, keeping entries already present by +// name. Used to layer the namespace's servers under a sandbox's own, matching +// how appendMissingShares treats namespace shares: the narrower scope wins a +// name collision. +func appendMissingMCP(have, add []config.MCPServer) []config.MCPServer { + seen := make(map[string]bool, len(have)) + for _, s := range have { + seen[s.Name] = true + } + for _, s := range add { + if seen[s.Name] { + continue + } + seen[s.Name] = true + have = append(have, s) + } + return have +} + +// mcpNetworkBlock derives the egress a sandbox's declared MCP servers need: +// one allow entry per distinct http/sse host. Without it every remote server +// would be refused by the default policy and the user would have to mirror +// each declaration with a `network allow` line — the failure mode being a +// bare "ConnectionRefused" from inside the guest, far from its cause. +// +// Loopback targets are skipped: a server on the guest's own 127.0.0.1 (or a +// host service reached through a reverse forward) never leaves the VM, so +// there is nothing to allow. Literal IPs land in AllowIPs and hostnames in +// AllowDomains, matching how the two are enforced — SYN-time for addresses, +// DNS-time for names. +// +// Returns ok=false when nothing needs allowing, so callers don't append an +// empty block to the policy. +func mcpNetworkBlock(servers []config.MCPServer) (config.NetworkBlock, bool) { + block := config.NetworkBlock{Origin: config.BlockOriginMCP, Name: "mcp servers"} + for _, s := range servers { + if s.URL == "" { + continue // stdio: a local process, no egress of its own + } + u, err := url.Parse(s.URL) + if err != nil { + continue // validated at parse time; nothing to derive if it ever isn't + } + host := u.Hostname() + if host == "" || host == "localhost" { + continue + } + if addr, err := netip.ParseAddr(host); err == nil { + if addr.IsLoopback() { + continue + } + block.AllowIPs = append(block.AllowIPs, host) + continue + } + block.AllowDomains = append(block.AllowDomains, host) + } + block.AllowDomains = dedupStrings(block.AllowDomains) + block.AllowIPs = dedupStrings(block.AllowIPs) + if len(block.AllowDomains)+len(block.AllowIPs) == 0 { + return config.NetworkBlock{}, false + } + return block, true +} + +// applyMCPNetwork installs (or refreshes) the derived allow layer on a +// sandbox's policy. Idempotent: the block is rebuilt from the current server +// list every time, so removing a server from clawk.mod and re-creating drops +// its allow entry rather than leaving it behind. +func applyMCPNetwork(sb *config.Sandbox) { + block, ok := mcpNetworkBlock(sb.MCP) + sb.Network.Blocks = slices.DeleteFunc(sb.Network.Blocks, + func(b config.NetworkBlock) bool { return b.Origin == config.BlockOriginMCP }) + if ok { + sb.Network.Blocks = append(sb.Network.Blocks, block) + } + // Restore the origin-order invariant the store relies on. + sb.Network.Normalize() +} diff --git a/internal/cli/compose_mcp_test.go b/internal/cli/compose_mcp_test.go new file mode 100644 index 0000000..b2b2be7 --- /dev/null +++ b/internal/cli/compose_mcp_test.go @@ -0,0 +1,226 @@ +package cli + +import ( + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/sandbox" + "github.com/clawkwork/clawk/internal/template" + "github.com/stretchr/testify/require" +) + +// TestMCPPipelineEndToEnd walks a realistic clawk.mod through every stage +// that stands between the file and a working server — parse, compose, derive +// egress, render the guest config, build the runner's argv — because each of +// those lives in a different package and a field dropped between two of them +// is exactly the kind of gap unit tests on either side both pass through. +func TestMCPPipelineEndToEnd(t *testing.T) { + tmpl, err := template.ParseString(`sandbox proj ( + mcp ( + linear https://mcp.linear.app/mcp header "Authorization: Bearer ${LINEAR_TOKEN}" + github stdio "npx -y @modelcontextprotocol/server-github" env GITHUB_TOKEN + ) + env ( + LINEAR_TOKEN = ${LINEAR_TOKEN:?create a Linear PAT} + GITHUB_TOKEN + ) +) +`) + require.NoError(t, err) + + var sources []mcpSource + for _, s := range tmpl.MCP { + sources = append(sources, mcpSource{Origin: "clawk.mod", Spec: s}) + } + servers, err := composeMCP(sources) + require.NoError(t, err) + + sb := &config.Sandbox{Name: "proj", MCP: servers, RequiredEnv: tmpl.Env} + applyMCPNetwork(sb) + + // The remote server's host is reachable; the stdio one adds no egress. + var allowed []string + for _, b := range sb.Network.Blocks { + if b.Origin == config.BlockOriginMCP { + allowed = b.AllowDomains + } + } + require.Equal(t, []string{"mcp.linear.app"}, allowed) + + // The rendered guest config names both servers and carries references, + // not values, even with the credential present in this process's env. + t.Setenv("LINEAR_TOKEN", "leaked-if-this-appears") + content, ok, err := sandbox.RenderMCPConfig(sb.MCP) + require.NoError(t, err) + require.True(t, ok) + require.NotContains(t, string(content), "leaked-if-this-appears") + require.Contains(t, string(content), "${LINEAR_TOKEN}") + require.Contains(t, string(content), "${GITHUB_TOKEN}") + + // The runner is told where to find it. + claude, err := agentByName("claude") + require.NoError(t, err) + require.Contains(t, launchArgs(sb, claude, nil), sandbox.GuestMCPConfigPath) + + // And the credential the config references is one the sandbox carries, + // so ${LINEAR_TOKEN} has something to expand to at connect time. + resolved, err := sandbox.ResolveEnv(sb) + require.NoError(t, err) + require.Contains(t, resolved, "LINEAR_TOKEN=leaked-if-this-appears", + "the value travels via the env path, never the config file") +} + +func httpSpec(name, url string, headers ...string) template.MCPSpec { + return template.MCPSpec{ + Name: name, + Transport: config.MCPTransportHTTP, + URL: url, + Headers: headers, + } +} + +// TestComposeMCPCollapsesIdenticalDeclarations: two repos in one workspace +// both needing the same server is normal, not a conflict. +func TestComposeMCPCollapsesIdenticalDeclarations(t *testing.T) { + spec := httpSpec("linear", "https://mcp.linear.app/mcp", "Authorization: Bearer ${T}") + got, err := composeMCP([]mcpSource{ + {Origin: "api", Spec: spec}, + {Origin: "web", Spec: spec}, + }) + require.NoError(t, err) + require.Len(t, got, 1) + require.Equal(t, "linear", got[0].Name) +} + +// TestComposeMCPRejectsConflict: the same name meaning two different things +// is a config bug. Silently picking one would hand the agent whichever +// definition composed first — the kind of thing you only notice when a tool +// call goes to the wrong place. +func TestComposeMCPRejectsConflict(t *testing.T) { + _, err := composeMCP([]mcpSource{ + {Origin: "api", Spec: httpSpec("linear", "https://mcp.linear.app/mcp")}, + {Origin: "web", Spec: httpSpec("linear", "https://staging.linear.app/mcp")}, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "declared differently") + require.Contains(t, err.Error(), "api") + require.Contains(t, err.Error(), "web") +} + +// TestAppendMissingMCPNarrowerScopeWins mirrors appendMissingShares: a +// namespace supplies the org-wide default, a repo's own declaration of the +// same name overrides it. +func TestAppendMissingMCPNarrowerScopeWins(t *testing.T) { + own := []config.MCPServer{{Name: "linear", Transport: config.MCPTransportHTTP, URL: "https://own/mcp"}} + ns := []config.MCPServer{ + {Name: "linear", Transport: config.MCPTransportHTTP, URL: "https://ns/mcp"}, + {Name: "notion", Transport: config.MCPTransportHTTP, URL: "https://mcp.notion.com/mcp"}, + } + got := appendMissingMCP(own, ns) + require.Len(t, got, 2) + require.Equal(t, "https://own/mcp", got[0].URL, "sandbox's own entry wins the name") + require.Equal(t, "notion", got[1].Name, "namespace-only entries still arrive") +} + +// TestMCPNetworkBlockDerivesHosts is the fix for the failure this feature +// exists to prevent: a declared remote server that the egress ACL refuses, +// surfacing inside the guest as a bare ConnectionRefused. +func TestMCPNetworkBlockDerivesHosts(t *testing.T) { + block, ok := mcpNetworkBlock([]config.MCPServer{ + {Name: "linear", Transport: config.MCPTransportHTTP, URL: "https://mcp.linear.app/mcp"}, + {Name: "sentry", Transport: config.MCPTransportSSE, URL: "https://mcp.sentry.dev/sse"}, + {Name: "dup", Transport: config.MCPTransportHTTP, URL: "https://mcp.linear.app/other"}, + {Name: "byip", Transport: config.MCPTransportHTTP, URL: "http://10.20.0.7:8080/mcp"}, + {Name: "local", Transport: config.MCPTransportHTTP, URL: "http://127.0.0.1:9000/mcp"}, + {Name: "named-local", Transport: config.MCPTransportHTTP, URL: "http://localhost:9000/mcp"}, + {Name: "gh", Transport: config.MCPTransportStdio, Command: []string{"npx", "server"}}, + }) + require.True(t, ok) + require.Equal(t, config.BlockOriginMCP, block.Origin) + require.ElementsMatch(t, []string{"mcp.linear.app", "mcp.sentry.dev"}, block.AllowDomains, + "hostnames are matched at DNS time and deduped") + require.Equal(t, []string{"10.20.0.7"}, block.AllowIPs, + "a literal address is enforced at SYN time, so it belongs in AllowIPs") +} + +func TestMCPNetworkBlockEmptyWhenNothingToAllow(t *testing.T) { + _, ok := mcpNetworkBlock(nil) + require.False(t, ok) + _, ok = mcpNetworkBlock([]config.MCPServer{ + {Name: "gh", Transport: config.MCPTransportStdio, Command: []string{"npx", "server"}}, + {Name: "local", Transport: config.MCPTransportHTTP, URL: "http://127.0.0.1:9000/mcp"}, + }) + require.False(t, ok, "a stdio server and a loopback URL need no egress") +} + +// TestApplyMCPNetworkIsIdempotent: the block is rebuilt from the current +// server list on every create, so removing a server retires its allow entry +// instead of leaving it behind forever. +func TestApplyMCPNetworkIsIdempotent(t *testing.T) { + sb := &config.Sandbox{ + MCP: []config.MCPServer{{Name: "linear", Transport: config.MCPTransportHTTP, URL: "https://mcp.linear.app/mcp"}}, + } + applyMCPNetwork(sb) + applyMCPNetwork(sb) + var mcpBlocks int + for _, b := range sb.Network.Blocks { + if b.Origin == config.BlockOriginMCP { + mcpBlocks++ + require.Equal(t, []string{"mcp.linear.app"}, b.AllowDomains) + } + } + require.Equal(t, 1, mcpBlocks, "re-applying must replace, not accumulate") + + sb.MCP = nil + applyMCPNetwork(sb) + for _, b := range sb.Network.Blocks { + require.NotEqual(t, config.BlockOriginMCP, b.Origin, "dropping the server drops its allow") + } +} + +// TestApplyMCPNetworkRanksBelowUserRules pins the precedence choice: the +// derived allow is a convenience, so an explicit deny the user wrote in +// clawk.mod or via the CLI has to outrank it. +func TestApplyMCPNetworkRanksBelowUserRules(t *testing.T) { + sb := &config.Sandbox{ + MCP: []config.MCPServer{{Name: "linear", Transport: config.MCPTransportHTTP, URL: "https://mcp.linear.app/mcp"}}, + Network: config.NetworkPolicy{Blocks: []config.NetworkBlock{ + {Origin: config.BlockOriginCustom, DenyDomains: []string{"mcp.linear.app"}}, + {Origin: config.BlockOriginMod, Name: "clawk.mod"}, + }}, + } + applyMCPNetwork(sb) + + var order []string + for _, b := range sb.Network.Blocks { + order = append(order, b.Origin) + } + require.Equal(t, + []string{config.BlockOriginMCP, config.BlockOriginMod, config.BlockOriginCustom}, + order, "mcp sits below mod and custom in increasing precedence") +} + +// TestLaunchArgsMCPConfig covers both directions: a sandbox with servers gets +// the flag, one without must not (the file wouldn't exist), and the user's +// own args always land last so they can override clawk. +func TestLaunchArgsMCPConfig(t *testing.T) { + claude, err := agentByName("claude") + require.NoError(t, err) + + withMCP := &config.Sandbox{MCP: []config.MCPServer{{Name: "linear"}}} + args := launchArgs(withMCP, claude, []string{"--resume"}) + require.Equal(t, []string{ + "--dangerously-skip-permissions", + "--mcp-config", sandbox.GuestMCPConfigPath, + "--resume", + }, args) + + bare := &config.Sandbox{} + require.Equal(t, []string{"--dangerously-skip-permissions"}, launchArgs(bare, claude, nil), + "no declared servers means no flag pointing at a missing file") + + codex, err := agentByName("codex") + require.NoError(t, err) + require.NotContains(t, launchArgs(withMCP, codex, nil), "--mcp-config", + "a runner clawk can't render config for gets no flag") +} diff --git a/internal/cli/compose_serial.go b/internal/cli/compose_serial.go new file mode 100644 index 0000000..54ae508 --- /dev/null +++ b/internal/cli/compose_serial.go @@ -0,0 +1,110 @@ +package cli + +import ( + "fmt" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/serialfwd" + "github.com/clawkwork/clawk/internal/template" +) + +type serialSource struct { + Origin string + Spec template.SerialSpec +} + +// mergeSerials gathers `serial (...)` entries from the workspace file and +// every repo's Clawkfile and resolves them into the sandbox's device list. +func mergeSerials(ws *template.Workspace) ([]config.SerialDevice, error) { + var sources []serialSource + for _, s := range ws.File.Serials { + sources = append(sources, serialSource{Origin: "workspace", Spec: s}) + } + for _, r := range ws.Repos { + if r.Clawkfile == nil { + continue + } + for _, s := range r.Clawkfile.Serials { + sources = append(sources, serialSource{Origin: r.Name, Spec: s}) + } + } + return composeSerials(sources) +} + +// composeSerials is the pure conflict-detection core split out from +// mergeSerials so the here-mode path (a single Clawkfile) and the workspace +// path share it — same split as composeFiles and composeShares. +// +// Two things can collide, and both are refused with a message naming the +// contributors rather than resolved by precedence: a guest name, because +// the guest creates /dev/ exactly once and a second claim would +// silently lose in there, and a host device, because forwarding one port +// under two names would let the sandbox open it twice. +func composeSerials(sources []serialSource) ([]config.SerialDevice, error) { + type claim struct { + Origin string + Device config.SerialDevice + } + byName := make(map[string]claim) + byHost := make(map[string]claim) + + var out []config.SerialDevice + for _, s := range sources { + dev, err := resolveSerialSpec(s.Spec) + if err != nil { + return nil, fmt.Errorf("%s serial %q: %w", s.Origin, s.Spec.HostPath, err) + } + if prev, dup := byName[dev.GuestName]; dup { + if prev.Device == dev { + // The same device declared twice — by a repo and the + // workspace, say. Harmless; keep the first. + continue + } + return nil, fmt.Errorf( + "serial device %q is claimed by both %s (%s) and %s (%s)", + dev.GuestName, prev.Origin, prev.Device.HostPath, s.Origin, dev.HostPath) + } + if prev, dup := byHost[dev.HostPath]; dup { + return nil, fmt.Errorf( + "serial port %s is forwarded twice: as %q by %s and as %q by %s", + dev.HostPath, prev.Device.GuestName, prev.Origin, dev.GuestName, s.Origin) + } + byName[dev.GuestName] = claim{Origin: s.Origin, Device: dev} + byHost[dev.HostPath] = claim{Origin: s.Origin, Device: dev} + out = append(out, dev) + } + return out, nil +} + +// resolveSerialSpec turns one parsed clawk.mod entry into a device, +// applying the same defaults and validation the CLI's HOST:GUEST spec +// parser does. +func resolveSerialSpec(spec template.SerialSpec) (config.SerialDevice, error) { + hostPath, err := template.ExpandPath(spec.HostPath) + if err != nil { + return config.SerialDevice{}, err + } + if !isAbsSerialPath(hostPath) { + return config.SerialDevice{}, fmt.Errorf( + "must be an absolute path (e.g. /dev/cu.usbmodem1101)") + } + + guestName := spec.GuestName + if guestName == "" { + guestName = config.DefaultSerialGuestName(hostPath) + if guestName == "" { + return config.SerialDevice{}, fmt.Errorf( + "a device pattern needs an explicit guest name (e.g. '%s ttyACM0')", hostPath) + } + } + if err := serialfwd.ValidDeviceName(guestName); err != nil { + return config.SerialDevice{}, err + } + return config.SerialDevice{HostPath: hostPath, GuestName: guestName}, nil +} + +// isAbsSerialPath is filepath.IsAbs pinned to POSIX semantics. Serial +// devices are named the same way on both hosts clawk runs on, and a +// clawk.mod is shared across machines, so this shouldn't vary by where it +// happens to be parsed. +func isAbsSerialPath(p string) bool { return len(p) > 0 && p[0] == '/' } diff --git a/internal/cli/compose_serial_test.go b/internal/cli/compose_serial_test.go new file mode 100644 index 0000000..e715c02 --- /dev/null +++ b/internal/cli/compose_serial_test.go @@ -0,0 +1,93 @@ +package cli + +import ( + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/template" + "github.com/stretchr/testify/require" +) + +func src(origin, host, guest string) serialSource { + return serialSource{ + Origin: origin, + Spec: template.SerialSpec{HostPath: host, GuestName: guest}, + } +} + +func TestComposeSerials(t *testing.T) { + got, err := composeSerials([]serialSource{ + src("workspace", "/dev/cu.usbmodem1101", ""), + src("firmware", "/dev/cu.usbserial-A50285BI", "ttyUSB0"), + }) + require.NoError(t, err) + require.Equal(t, []config.SerialDevice{ + {HostPath: "/dev/cu.usbmodem1101", GuestName: "cu.usbmodem1101"}, + {HostPath: "/dev/cu.usbserial-A50285BI", GuestName: "ttyUSB0"}, + }, got) +} + +// The workspace and a repo both declaring the same board is ordinary, not a +// conflict — it only becomes one when they disagree. +func TestComposeSerialsIdenticalDuplicateIsFine(t *testing.T) { + got, err := composeSerials([]serialSource{ + src("workspace", "/dev/cu.usbmodem1101", "ttyACM0"), + src("firmware", "/dev/cu.usbmodem1101", "ttyACM0"), + }) + require.NoError(t, err) + require.Len(t, got, 1) +} + +// Both collision kinds must name their contributors: with several repos in +// a workspace, "something clashed" is not actionable. +func TestComposeSerialsRejectsNameClash(t *testing.T) { + _, err := composeSerials([]serialSource{ + src("workspace", "/dev/cu.usbmodem1101", "ttyACM0"), + src("firmware", "/dev/cu.usbmodem2201", "ttyACM0"), + }) + require.Error(t, err) + require.Contains(t, err.Error(), "workspace") + require.Contains(t, err.Error(), "firmware") + require.Contains(t, err.Error(), "ttyACM0") +} + +func TestComposeSerialsRejectsSamePortTwice(t *testing.T) { + _, err := composeSerials([]serialSource{ + src("workspace", "/dev/cu.usbmodem1101", "ttyACM0"), + src("firmware", "/dev/cu.usbmodem1101", "ttyACM1"), + }) + require.Error(t, err) + require.Contains(t, err.Error(), "forwarded twice") + require.Contains(t, err.Error(), "/dev/cu.usbmodem1101") +} + +func TestComposeSerialsRejectsRelativePath(t *testing.T) { + _, err := composeSerials([]serialSource{src("workspace", "cu.usbmodem1101", "")}) + require.Error(t, err) + require.Contains(t, err.Error(), "absolute path") +} + +func TestComposeSerialsRejectsGlobWithoutName(t *testing.T) { + _, err := composeSerials([]serialSource{src("workspace", "/dev/cu.usbmodem*", "")}) + require.Error(t, err) + require.Contains(t, err.Error(), "needs an explicit guest name") +} + +// The name becomes /dev/ in the guest, so a clawk.mod is just as much +// an untrusted-ish input as the wire is — it may come from a cloned repo. +func TestComposeSerialsRejectsUnsafeGuestName(t *testing.T) { + for _, name := range []string{"../escape", "sub/dir", ".hidden", "has space"} { + t.Run(name, func(t *testing.T) { + _, err := composeSerials([]serialSource{ + src("workspace", "/dev/cu.usbmodem1101", name), + }) + require.Error(t, err) + }) + } +} + +func TestComposeSerialsEmpty(t *testing.T) { + got, err := composeSerials(nil) + require.NoError(t, err) + require.Empty(t, got) +} diff --git a/internal/cli/daemon.go b/internal/cli/daemon.go index 2f57103..a34f7f4 100644 --- a/internal/cli/daemon.go +++ b/internal/cli/daemon.go @@ -472,12 +472,20 @@ type reverseForwardSink interface { Set([]config.PortForward) } +// serialSink publishes a serial-device set to the in-guest agent. An +// interface for the same reason reverseForwardSink is one: the only +// implementation is the darwin serialProxy, and the firecracker daemon +// passes nil. +type serialSink interface { + Set([]config.SerialDevice) +} + // controlHandlers builds the control-socket callbacks shared by both VM // daemons: the denial ledger, a live network-policy reload from the store, // the VM lifecycle surface (pause/resume/suspend), reverse-forward reloads // when the daemon has a sink for them, and (when the allow list has one) // the interactive gate. -func controlHandlers(sb *config.Sandbox, allow *netfilter.AllowList, lc *vmLifecycle, rev reverseForwardSink, logger *log.Logger) vzdctl.Handlers { +func controlHandlers(sb *config.Sandbox, allow *netfilter.AllowList, lc *vmLifecycle, rev reverseForwardSink, ser serialSink, logger *log.Logger) vzdctl.Handlers { h := vzdctl.Handlers{ Denials: allow.Denials, Lifecycle: lc.lifecycleHandlers(), @@ -508,6 +516,17 @@ func controlHandlers(sb *config.Sandbox, allow *netfilter.AllowList, lc *vmLifec return nil } } + if ser != nil { + h.ReloadSerials = func() error { + cur, err := store.Load(sb.Name) + if err != nil { + return fmt.Errorf("reloading sandbox record: %w", err) + } + ser.Set(cur.Serials) + logger.Printf("serial devices reloaded: %d", len(cur.Serials)) + return nil + } + } return h } diff --git a/internal/cli/fcd.go b/internal/cli/fcd.go index a4fcfc1..a07768d 100644 --- a/internal/cli/fcd.go +++ b/internal/cli/fcd.go @@ -91,7 +91,7 @@ func runFcd(_ *cobra.Command, args []string) (retErr error) { // No reverse-forward sink: firecracker's vsock is one-way (guest listens, // host dials), so there is nothing for the guest agent to dial. The // endpoint 404s and the CLI says so. - ctl, err := vzdctl.Start(vzdctl.SocketPath(vmDir), controlHandlers(sb, allow, lc, nil, logger)) + ctl, err := vzdctl.Start(vzdctl.SocketPath(vmDir), controlHandlers(sb, allow, lc, nil, nil, logger)) if err != nil { logger.Printf("control socket: disabled (%v) — network edits apply on next up", err) } else { diff --git a/internal/cli/global_defaults_test.go b/internal/cli/global_defaults_test.go new file mode 100644 index 0000000..710c8f2 --- /dev/null +++ b/internal/cli/global_defaults_test.go @@ -0,0 +1,207 @@ +package cli + +import ( + "os" + "path/filepath" + "slices" + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/template" + "github.com/stretchr/testify/require" +) + +// withGlobalMod writes src as the host-wide clawk.mod inside a fresh +// XDG_CONFIG_HOME and enables the layer for the duration of the test (the +// package otherwise runs with it off — see TestMain). Returns its directory. +func withGlobalMod(t *testing.T, src string) string { + t.Helper() + home := t.TempDir() + dir := filepath.Join(home, ".config", "clawk") + require.NoError(t, os.MkdirAll(dir, 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, template.RepoFileName), []byte(src), 0o644)) + + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config")) + t.Setenv(template.GlobalModEnvVar, "") + + template.GlobalDisabled = false + noGlobalFlag = false + t.Cleanup(func() { + template.GlobalDisabled = true + noGlobalFlag = true + }) + return dir +} + +// End to end through `clawk work`: a host-wide clawk.mod reaches the sandbox +// record for every directive group, and the repo's own file still wins. +func TestGlobalDefaultsReachTheRecord(t *testing.T) { + setupTest(t) + dir := withGlobalMod(t, `sandbox ( + vm ( + memory_max 8GiB + provider vz + ) + network ( + allow global.example.com + ) + env ( + GLOBAL_TOKEN + ) + files ( + ./house.netrc /home/agent/.netrc + ) + agent ( + instructions "house rule" + ) +) +`) + require.NoError(t, os.WriteFile(filepath.Join(dir, "house.netrc"), []byte("x"), 0o600)) + + repo := filepath.Join(t.TempDir(), "proj") + require.NoError(t, os.MkdirAll(repo, 0o755)) + gitInit(t, repo) + require.NoError(t, os.WriteFile(filepath.Join(repo, template.RepoFileName), []byte(`sandbox ( + vm ( + memory_max 16GiB + ) + network ( + allow repo.example.com + ) + env ( + REPO_TOKEN + ) + agent ( + instructions "repo rule" + ) +) +`), 0o644)) + + _, err := executeCommand("work", filepath.Join(repo, template.RepoFileName), "TICKET-9", "--bare") + require.NoError(t, err) + + sb, err := store.Load("TICKET-9") + require.NoError(t, err) + + require.EqualValues(t, 16384, sb.MemoryMaxMiB, "the repo's own memory_max must win") + require.Equal(t, config.Provider("vz"), sb.Provider, "global provider applies where the repo is silent") + + mod := sb.Network.Block(config.BlockOriginMod) + require.Contains(t, mod.AllowDomains, "global.example.com") + require.Contains(t, mod.AllowDomains, "repo.example.com") + + require.True(t, slices.Contains(sb.RequiredEnv, "GLOBAL_TOKEN"), "env: %v", sb.RequiredEnv) + require.True(t, slices.Contains(sb.RequiredEnv, "REPO_TOKEN"), "env: %v", sb.RequiredEnv) + + // The relative host path resolved against the global file's directory, not + // the repo or the process CWD. + var netrc *config.HostFile + for i := range sb.Files { + if sb.Files[i].GuestPath == "/home/agent/.netrc" { + netrc = &sb.Files[i] + } + } + require.NotNil(t, netrc, "global file entry missing: %v", sb.Files) + require.Equal(t, filepath.Join(dir, "house.netrc"), netrc.HostPath) + + require.Equal(t, []string{"house rule", "repo rule"}, sb.Instructions, + "the broader scope's instructions read first") +} + +// A broken host-wide file must be reported as such. resolveSource walks +// workspace → standalone → bare-git-repo and treats each failure as "not this +// shape", which swallowed the real error and answered "no clawk.mod found, and +// X is not a git repo" — for a directory that had a clawk.mod and was a git +// repo. See template.ErrGlobalMod. +func TestBrokenGlobalIsReportedNotSwallowed(t *testing.T) { + setupTest(t) + // A header name is the scope violation with the most misleading fallback: + // the file parses, so nothing else complains. + withGlobalMod(t, "sandbox house (\n vm (\n cpu 2\n )\n)\n") + + repo := filepath.Join(t.TempDir(), "proj") + require.NoError(t, os.MkdirAll(repo, 0o755)) + gitInit(t, repo) + require.NoError(t, os.WriteFile(filepath.Join(repo, template.RepoFileName), + []byte("sandbox (\n)\n"), 0o644)) + + t.Chdir(repo) + _, err := resolveSource("", "") + require.ErrorIs(t, err, template.ErrGlobalMod) + require.ErrorContains(t, err, "must be anonymous") + require.NotContains(t, err.Error(), "is not a git repo") +} + +func TestNoGlobalFlagDropsTheLayer(t *testing.T) { + setupTest(t) + withGlobalMod(t, `sandbox ( + network ( + allow global.example.com + ) + env ( + GLOBAL_TOKEN + ) +) +`) + repo := filepath.Join(t.TempDir(), "proj") + require.NoError(t, os.MkdirAll(repo, 0o755)) + gitInit(t, repo) + require.NoError(t, os.WriteFile(filepath.Join(repo, template.RepoFileName), + []byte("sandbox (\n network (\n allow repo.example.com\n )\n)\n"), 0o644)) + + _, err := executeCommand("work", filepath.Join(repo, template.RepoFileName), + "TICKET-10", "--bare", "--no-global") + require.NoError(t, err) + + sb, err := store.Load("TICKET-10") + require.NoError(t, err) + mod := sb.Network.Block(config.BlockOriginMod) + require.Contains(t, mod.AllowDomains, "repo.example.com") + require.NotContains(t, mod.AllowDomains, "global.example.com") + require.Empty(t, sb.RequiredEnv) +} + +// The here-mode path (`clawk` bare) must refuse a broken host-wide file for +// the same reason resolveSource does — and it matters more here, because the +// standalone loader fails as a WHOLE: degrading to "no defaults" threw away +// the repo's own clawk.mod too, creating a sandbox with none of its forwards, +// env or instructions and only a line on stderr to say so. +func TestHereModeRefusesBrokenGlobal(t *testing.T) { + setupTest(t) + // A header name parses fine and violates only the global scope, so + // nothing but the ErrGlobalMod check stands between it and a silently + // misconfigured sandbox. + withGlobalMod(t, "sandbox house (\n vm (\n cpu 2\n )\n)\n") + + repo := filepath.Join(t.TempDir(), "proj") + require.NoError(t, os.MkdirAll(repo, 0o755)) + gitInit(t, repo) + require.NoError(t, os.WriteFile(filepath.Join(repo, template.RepoFileName), + []byte("sandbox (\n forwards ( 3000 )\n env ( MY_TOKEN )\n)\n"), 0o644)) + + _, _, _, err := loadHereClawkfile(repo) + require.ErrorIs(t, err, template.ErrGlobalMod) + require.ErrorContains(t, err, "must be anonymous") +} + +// The repo's own clawk.mod must still come back intact when the host-wide +// layer is merely absent — the overwhelmingly common case, and the one the +// error path above must not swallow. +func TestHereModeKeepsRepoConfigWithoutGlobal(t *testing.T) { + setupTest(t) + + repo := filepath.Join(t.TempDir(), "proj") + require.NoError(t, os.MkdirAll(repo, 0o755)) + gitInit(t, repo) + require.NoError(t, os.WriteFile(filepath.Join(repo, template.RepoFileName), + []byte("sandbox (\n forwards ( 3000 )\n env ( MY_TOKEN )\n)\n"), 0o644)) + + tmpl, _, globalPath, err := loadHereClawkfile(repo) + require.NoError(t, err) + require.Empty(t, globalPath) + require.NotNil(t, tmpl) + require.Equal(t, []string{"MY_TOKEN"}, tmpl.Env) + require.Len(t, tmpl.Forwards, 1) +} diff --git a/internal/cli/here.go b/internal/cli/here.go index 1d7eccd..a2e59ce 100644 --- a/internal/cli/here.go +++ b/internal/cli/here.go @@ -157,7 +157,13 @@ func loadOrCreateHereSandbox(name, cwd string) (*config.Sandbox, bool, error) { return sb, false, nil } - clawkfile, policies := loadHereClawkfile(cwd) + clawkfile, policies, globalPath, err := loadHereClawkfile(cwd) + if err != nil { + return nil, false, err + } + if globalPath != "" { + fmt.Printf("Using host-wide defaults from %s\n", globalPath) + } // Register the file's `policy ( ... )` blocks before composing // the network policy — `use` references resolve against the store at @@ -187,6 +193,8 @@ func loadOrCreateHereSandbox(name, cwd string) (*config.Sandbox, bool, error) { var memory string var fileSources []fileSource var shareSources []shareSource + var serialSources []serialSource + var mcpSources []mcpSource if clawkfile != nil { forwardSpecs = append(forwardSpecs, clawkfile.Forwards...) reverseForwardSpecs = append(reverseForwardSpecs, clawkfile.ReverseForwards...) @@ -209,6 +217,12 @@ func loadOrCreateHereSandbox(name, cwd string) (*config.Sandbox, bool, error) { for _, s := range clawkfile.Shares { shareSources = append(shareSources, shareSource{Origin: "clawk.mod", Spec: s}) } + for _, s := range clawkfile.Serials { + serialSources = append(serialSources, serialSource{Origin: "clawk.mod", Spec: s}) + } + for _, s := range clawkfile.MCP { + mcpSources = append(mcpSources, mcpSource{Origin: "clawk.mod", Spec: s}) + } } forwards, err := parseForwardSpecs(forwardSpecs, cwd) @@ -227,18 +241,27 @@ func loadOrCreateHereSandbox(name, cwd string) (*config.Sandbox, bool, error) { if err != nil { return nil, false, err } + serials, err := composeSerials(serialSources) + if err != nil { + return nil, false, err + } + mcpServers, err := composeMCP(mcpSources) + if err != nil { + return nil, false, err + } var nested bool var cpu uint var memoryMiB, memoryMaxMiB, diskMiB uint64 var image, kernel string - var idleTimeoutSec int64 + var idleTimeoutSec, swapMiB int64 if clawkfile != nil { nested = clawkfile.Nested cpu = clawkfile.CPU memoryMiB = clawkfile.MemoryMiB memoryMaxMiB = clawkfile.MemoryMaxMiB diskMiB = clawkfile.DiskMiB + swapMiB = clawkfile.SwapMiB image = clawkfile.Image kernel = clawkfile.Kernel idleTimeoutSec = clawkfile.IdleTimeoutSec @@ -265,6 +288,8 @@ func loadOrCreateHereSandbox(name, cwd string) (*config.Sandbox, bool, error) { ReverseForwards: reverseForwards, Files: files, Shares: shares, + Serials: serials, + MCP: mcpServers, RequiredEnv: requiredEnv, Instructions: instructions, Memory: memory, @@ -273,6 +298,7 @@ func loadOrCreateHereSandbox(name, cwd string) (*config.Sandbox, bool, error) { MemoryMiB: memoryMiB, MemoryMaxMiB: memoryMaxMiB, DiskMiB: diskMiB, + SwapMiB: swapMiB, // IdleTimeoutSec rides the same snapshot-at-create rule as every // other clawk.mod value; it was the one vm(...) field this path // forgot to copy when idle-stop landed, which silently pinned every @@ -295,6 +321,8 @@ func loadOrCreateHereSandbox(name, cwd string) (*config.Sandbox, bool, error) { }}, CreatedAt: time.Now(), } + // Derived MCP egress, before the first Save — see applyMCPNetwork. + applyMCPNetwork(sb) if err := store.Save(sb); err != nil { return nil, false, fmt.Errorf("saving sandbox: %w", err) } @@ -302,21 +330,44 @@ func loadOrCreateHereSandbox(name, cwd string) (*config.Sandbox, bool, error) { } // loadHereClawkfile reads cwd/clawk.mod if present and returns its sandbox -// template plus any policy blocks declared beside it. Missing or unreadable -// clawk.mod is not an error — callers fall back to defaults. -func loadHereClawkfile(cwd string) (*template.Template, []template.PolicyDef) { +// template — with the host-wide clawk.mod folded in underneath — plus any +// policy blocks declared beside either, and the host-wide file's path for the +// create-time note. Missing or unreadable clawk.mod is not an error: callers +// fall back to defaults, which is where the host-wide layer alone lands. +// +// A broken host-wide layer IS an error, and it has to be, for the same reason +// resolveSource surfaces it: the standalone loader fails as a whole, so +// degrading to "no defaults" here would throw away the repo's OWN clawk.mod +// too — its forwards, env and instructions — and create the sandbox anyway +// with nothing but a line on stderr to say so. See template.ErrGlobalMod. +func loadHereClawkfile(cwd string) (*template.Template, []template.PolicyDef, string, error) { ws, err := template.LoadStandaloneClawkfileWithProfile(cwd, "") if err != nil { + // The host-wide layer carries its own context, so it is not prefixed + // with a clawk.mod the user may not even have. + if errors.Is(err, template.ErrGlobalMod) { + return nil, nil, "", err + } // Only surface non-ENOENT errors; absence is expected. if !errors.Is(err, os.ErrNotExist) { fmt.Fprintf(os.Stderr, "warning: reading clawk.mod: %v\n", err) + return nil, nil, "", nil } - return nil, nil + // No clawk.mod here — the host-wide defaults are the whole template. + // The standalone loader never got far enough to fold them in. + g, gerr := template.LoadGlobal() + if gerr != nil { + if !errors.Is(gerr, template.ErrNoGlobalMod) { + return nil, nil, "", gerr + } + return nil, nil, "", nil + } + return g.Template, g.Policies, g.Path, nil } if ws == nil || len(ws.Repos) == 0 { - return nil, nil + return nil, nil, "", nil } - return ws.Repos[0].Clawkfile, ws.Policies + return ws.Repos[0].Clawkfile, ws.Policies, ws.GlobalPath, nil } // parseForwardSpecs turns a list of "PORT" or "HOST:GUEST" strings into diff --git a/internal/cli/main_test.go b/internal/cli/main_test.go new file mode 100644 index 0000000..2e70961 --- /dev/null +++ b/internal/cli/main_test.go @@ -0,0 +1,23 @@ +package cli + +import ( + "os" + "testing" + + "github.com/clawkwork/clawk/internal/template" +) + +// TestMain runs the whole package as if --no-global had been passed: a +// developer's own ~/.config/clawk/clawk.mod must never leak into the sandbox +// records these tests compose. +// +// Both knobs are set. GlobalDisabled covers tests that call the loaders +// directly; noGlobalFlag covers those going through executeCommand, whose +// PersistentPreRunE re-derives GlobalDisabled from the flag on every +// invocation. Tests for the layer itself flip both (see withGlobalMod in +// global_defaults_test.go). +func TestMain(m *testing.M) { + template.GlobalDisabled = true + noGlobalFlag = true + os.Exit(m.Run()) +} diff --git a/internal/cli/namespace.go b/internal/cli/namespace.go index 8d50575..8630825 100644 --- a/internal/cli/namespace.go +++ b/internal/cli/namespace.go @@ -64,6 +64,7 @@ func applyNamespaceDefaults(sb *config.Sandbox) error { sb.RequiredEnv = dedupStrings(append(sb.RequiredEnv, ns.Env...)) sb.Files = appendMissingFiles(sb.Files, ns.Files) sb.Shares = appendMissingShares(sb.Shares, ns.Shares) + sb.MCP = appendMissingMCP(sb.MCP, ns.MCP) // Namespace scope is broader than the sandbox's own (clawk.mod) entries, // so it reads first: namespace instructions, then sandbox-specific. sb.Instructions = append(append([]string{}, ns.Instructions...), sb.Instructions...) diff --git a/internal/cli/resources.go b/internal/cli/resources.go index 1ea0d0a..b1db8b1 100644 --- a/internal/cli/resources.go +++ b/internal/cli/resources.go @@ -95,6 +95,31 @@ func resolveDisk(ws *template.Workspace) uint64 { return disk } +// resolveSwap merges the swap directive across a workspace and its repos. +// An explicit "off" (negative) wins over any size, mirroring how +// resolveIdleTimeout lets "off" win: a repo that declares `swap off` is +// saying its workload must not be swapped — a latency measurement, a +// benchmark — and honoring a different repo's larger size would break that, +// while the reverse only costs the other repo some headroom. Among sizes the +// max wins, the same rule as resolveResources: the VM is shared. +func resolveSwap(ws *template.Workspace) int64 { + swap := ws.File.SwapMiB + for _, r := range ws.Repos { + if r.Clawkfile == nil { + continue + } + v := r.Clawkfile.SwapMiB + if v < 0 || swap < 0 { + swap = -1 + continue + } + if v > swap { + swap = v + } + } + return swap +} + // minDiskMiB is the floor for a per-sandbox `vm ( disk )` override. // Below ~1 GiB there is no room for the base image plus any writes, and // such a value is almost always a unit typo (disk 32M meaning 32G). Zero diff --git a/internal/cli/root.go b/internal/cli/root.go index 40ffb5a..6c66fcb 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -135,8 +135,16 @@ func init() { "OCI image for new sandboxes (registry ref or docker-save tarball path; overrides clawk.mod)") rootCmd.PersistentFlags().StringVar(&kernelFlag, "kernel", "", "guest kernel override for new sandboxes (local vmlinux path or http(s) URL; overrides clawk.mod). Default: the Kata kernel") + rootCmd.PersistentFlags().BoolVar(&noGlobalFlag, "no-global", false, + "ignore the host-wide clawk.mod (~/.config/clawk/clawk.mod) — the repo's own config only, for a reproducible run") } +// noGlobalFlag (--no-global) drops the host-wide defaults layer. It exists +// because that file makes a sandbox's shape depend on host-local config: a CI +// run or a bug report needs a way to say "just what's in the repo". Applied in +// PersistentPreRunE (see setup.go), before any template load. +var noGlobalFlag bool + // testProvider, if non-nil, is returned from providerFor — used by tests to // inject a MockProvider without going through provider selection. var testProvider sandbox.Provider diff --git a/internal/cli/run.go b/internal/cli/run.go index f7be58d..01f51ca 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -53,8 +53,9 @@ func registerSafeFlag(cmd *cobra.Command) { cmd.Flags().BoolVar(&runSafe, "safe", false, "attach the runner without its permission-bypass default args "+ "(claude's --dangerously-skip-permissions, codex's "+ - "--dangerously-bypass-approvals-and-sandbox), so the agent asks "+ - "for confirmation as it would on the host") + "--dangerously-bypass-approvals-and-sandbox, pi's --approve, "+ + "opencode's --auto), so the agent asks for confirmation as it "+ + "would on the host") } // applySafeMode drops a runner's permission-bypass DefaultArgs when --safe is @@ -239,6 +240,12 @@ func resolveSource(source, profile string) (*template.Workspace, error) { } if ws, err := template.LoadStandaloneClawkfileWithProfile(cwd, profile); err == nil { return ws, nil + } else if errors.Is(err, template.ErrGlobalMod) { + // The repo's own file may well be absent — but the host-wide layer + // being broken is not a reason to try the next shape, it's the + // answer. Without this the rung below swallows it and the user gets + // "no clawk.mod found, and X is not a git repo". + return nil, err } // Last resort: cwd is itself a git repo. Treat it as a single-repo // workspace inheriting only defaults — keeps `clawk work ` @@ -248,11 +255,22 @@ func resolveSource(source, profile string) (*template.Workspace, error) { return nil, fmt.Errorf( "--profile requires a clawk.mod (none found in %s)", cwd) } - if ws, err := template.WorkspaceFromGitRepo(cwd); err == nil { - fmt.Fprintf(os.Stderr, - "note: no clawk.mod in %s; using defaults "+ - "(no forwards, no setup). Add clawk.mod to declare them.\n", - ws.Root) + ws, gerr := template.WorkspaceFromGitRepo(cwd) + if errors.Is(gerr, template.ErrGlobalMod) { + return nil, gerr + } + if gerr == nil { + if ws.GlobalPath != "" { + fmt.Fprintf(os.Stderr, + "note: no clawk.mod in %s; using the host-wide defaults in %s. "+ + "Add clawk.mod to declare repo-specific settings.\n", + ws.Root, ws.GlobalPath) + } else { + fmt.Fprintf(os.Stderr, + "note: no clawk.mod in %s; using defaults "+ + "(no forwards, no setup). Add clawk.mod to declare them.\n", + ws.Root) + } return ws, nil } return nil, fmt.Errorf( @@ -354,6 +372,14 @@ func loadOrCreateSandboxFromWorkspace(name string, ws *template.Workspace) (*con if err != nil { return nil, err } + serials, err := mergeSerials(ws) + if err != nil { + return nil, err + } + mcpServers, err := mergeMCP(ws) + if err != nil { + return nil, err + } // Nested virt: workspace-level `nested` directive OR any repo's // Clawkfile requesting it turns it on for the whole sandbox. This @@ -398,11 +424,14 @@ func loadOrCreateSandboxFromWorkspace(name string, ws *template.Workspace) (*con ReverseForwards: reverseForwards, Files: files, Shares: shares, + Serials: serials, + MCP: mcpServers, NestedVirt: nested, CPU: cpu, MemoryMiB: memoryMiB, MemoryMaxMiB: memoryMaxMiB, DiskMiB: disk, + SwapMiB: resolveSwap(ws), IdleTimeoutSec: resolveIdleTimeout(ws), Image: image, Kernel: kernel, @@ -418,6 +447,14 @@ func loadOrCreateSandboxFromWorkspace(name string, ws *template.Workspace) (*con if err := applyNamespaceDefaults(sb); err != nil { return nil, err } + // After the namespace, so its entries stay narrower than the workspace's. + if err := applyWorkspaceLevelDefaults(sb, ws); err != nil { + return nil, err + } + // After the namespace has folded its own servers in, so the derived + // allow layer covers the full set. Before the first Save, so the very + // first boot already has the egress its servers need. + applyMCPNetwork(sb) if err := store.Save(sb); err != nil { return nil, err } @@ -427,9 +464,41 @@ func loadOrCreateSandboxFromWorkspace(name string, ws *template.Workspace) (*con } else { fmt.Printf("Created sandbox %q (provider: %s)\n", sb.DisplayName(), sb.Provider) } + if ws.GlobalPath != "" { + // Named at create because the layer is host-local: it explains config + // nobody can find by reading the repo. --no-global excludes it. + fmt.Printf(" host-wide defaults from %s\n", ws.GlobalPath) + } return sb, nil } +// applyWorkspaceLevelDefaults folds the workspace-position `env ( … )` and +// `agent ( … )` blocks into the record — the workspace file's own, plus the +// host-wide clawk.mod folded underneath it (see template/global.go). +// +// Both were previously dropped on the floor: only repo Clawkfiles fed +// RequiredEnv and the agent docs (see addPhases), so a workspace root +// declaring `env ( GITHUB_TOKEN )` silently got nothing. Ordering is +// scope-outward — workspace, then namespace, then repo — which for env is what +// decides precedence, since both the profile.d exports and exec's environment +// let the LAST occurrence of a name win. +func applyWorkspaceLevelDefaults(sb *config.Sandbox, ws *template.Workspace) error { + if len(ws.File.Env) > 0 { + sb.RequiredEnv = unionStrings(ws.File.Env, sb.RequiredEnv) + } + instr, err := resolveAgentDocs(ws.Root, ws.File.Instructions) + if err != nil { + return fmt.Errorf("workspace agent instructions: %w", err) + } + sb.Instructions = append(instr, sb.Instructions...) + mem, err := resolveAgentDocs(ws.Root, ws.File.Memory) + if err != nil { + return fmt.Errorf("workspace agent memory: %w", err) + } + sb.Memory = joinMemory(joinMemory(mem...), sb.Memory) + return nil +} + // resolveProvider picks the provider for a new sandbox, erroring on // workspace-vs-repo or repo-vs-repo disagreement when the workspace itself // is silent. This matches the design: config should reject ambiguity. @@ -466,7 +535,7 @@ func resolveProvider(ws *template.Workspace) (config.Provider, error) { // defaultImage is the rootfs new sandboxes boot when neither --image // nor clawk.mod chooses one: clawk-dev — our own image bundling a -// development toolchain (Go/Node/Rust/etc. + claude/codex), built +// development toolchain (Go/Node/Rust/etc. + claude/codex/pi/opencode), built // and published by .github/workflows/publish-clawk-dev.yml. Override // per invocation with --image, or per repo/workspace with // `vm ( image ... )`. diff --git a/internal/cli/serial.go b/internal/cli/serial.go new file mode 100644 index 0000000..52a3c88 --- /dev/null +++ b/internal/cli/serial.go @@ -0,0 +1,368 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "runtime" + "strings" + "text/tabwriter" + "time" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/serialfwd" + "github.com/clawkwork/clawk/internal/serialport" + "github.com/clawkwork/clawk/internal/template" + "github.com/clawkwork/clawk/internal/vzdctl" + "github.com/spf13/cobra" +) + +var serialListJSON bool + +func init() { + rootCmd.AddCommand(serialCmd) + serialCmd.AddCommand(serialAddCmd) + serialCmd.AddCommand(serialRemoveCmd) + serialCmd.AddCommand(serialListCmd) + serialListCmd.Flags().BoolVar(&serialListJSON, "json", false, "emit JSON") +} + +var serialCmd = &cobra.Command{ + Use: "serial", + Short: "Expose a host serial port inside a sandbox", + Long: `Present a serial port plugged into this machine — an Arduino, an +ESP32, a USB-TTL adapter — as a device inside the sandbox, so tooling in +there can flash and monitor it. + +The USB device itself is not passed through; nothing clawk runs on can do +that. What crosses is the serial stream and its line settings, which is all +avrdude, esptool and a serial monitor ever wanted. See docs/serial.md for +what that does and doesn't cover.`, +} + +var serialAddCmd = &cobra.Command{ + ValidArgsFunction: completeSandboxNames, + Use: "add [device-spec...]", + Short: "Forward a host serial port into the sandbox", + Long: `Device specs read host-side first, like the forward commands: + + /dev/cu.usbmodem1101 — same path inside the guest + /dev/cu.usbmodem1101:ttyACM0 — /dev/ttyACM0 inside the guest + '/dev/cu.usbmodem*:ttyACM0' — resolved at open time, not now + +The glob form is worth knowing about: a board that reboots into its +bootloader leaves /dev and comes back, often under a neighbouring name, and +a pattern survives that where a literal path doesn't. Quote it so the shell +doesn't expand it first. A pattern that matches two boards is refused rather +than guessed at. + +These apply to a running sandbox immediately — no down/up cycle. The port is +opened on the host only while a process in the guest holds the device open, +so the Arduino IDE on this machine can still have it the rest of the time. + +vz (macOS) only: firecracker's vsock is one-way, so the guest has no channel +to connect back through.`, + Args: cobra.MinimumNArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + sb, err := store.Load(args[0]) + if err != nil { + return err + } + // Every spec is parsed and checked before anything is printed or + // saved. The command is already all-or-nothing on the store — a + // later bad spec returns before Save — so reporting devices as + // "added" mid-loop announced work that then didn't persist. + var added []config.SerialDevice + for _, spec := range args[1:] { + dev, err := parseSerialSpec(spec) + if err != nil { + return err + } + if prev, dup := serialByGuestName(sb.Serials, dev.GuestName); dup { + if prev.HostPath == dev.HostPath { + fmt.Fprintf(cmd.OutOrStdout(), " (already forwarded: %s)\n", describeSerial(dev)) + continue + } + // One name, one device. A second spec claiming it would + // silently lose to the first inside the guest, so reject it + // here where both can be named. + return fmt.Errorf( + "guest device %q is already forwarded from %s — remove that first", + dev.GuestName, prev.HostPath) + } + // The same port under two names would let the guest open it + // twice, and the second open would fail in a way that points at + // the wrong thing. + if prev, dup := serialByHostPath(sb.Serials, dev.HostPath); dup { + return fmt.Errorf( + "%s is already forwarded as %q — remove that first", + dev.HostPath, prev.GuestName) + } + sb.Serials = append(sb.Serials, dev) + added = append(added, dev) + } + if len(added) == 0 { + return nil + } + if err := store.Save(sb); err != nil { + return err + } + for _, dev := range added { + fmt.Fprintf(cmd.OutOrStdout(), "Serial device added: %s\n", describeSerial(dev)) + for _, w := range warnAboutDevice(dev) { + fmt.Fprintf(cmd.OutOrStdout(), " note: %s\n", w) + } + } + applySerials(cmd, sb) + return nil + }, +} + +var serialRemoveCmd = &cobra.Command{ + ValidArgsFunction: completeSandboxNames, + Use: "remove [more...]", + Aliases: []string{"rm"}, + Short: "Stop forwarding a host serial port", + Long: `Accepts whatever identifies the device: the spec as added, the host +path on its own, or the guest name on its own.`, + Args: cobra.MinimumNArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + sb, err := store.Load(args[0]) + if err != nil { + return err + } + var kept, removed []config.SerialDevice + for _, d := range sb.Serials { + if serialMatchesAny(d, args[1:]) { + removed = append(removed, d) + continue + } + kept = append(kept, d) + } + // Nothing matched: say so and leave the record untouched rather than + // rewriting it identically. + if len(removed) == 0 { + fmt.Fprintf(cmd.OutOrStdout(), " (nothing matched: %s)\n", strings.Join(args[1:], ", ")) + return nil + } + sb.Serials = kept + if err := store.Save(sb); err != nil { + return err + } + for _, d := range removed { + fmt.Fprintf(cmd.OutOrStdout(), "Serial device removed: %s\n", describeSerial(d)) + } + applySerials(cmd, sb) + return nil + }, +} + +var serialListCmd = &cobra.Command{ + ValidArgsFunction: completeSandboxNames, + Use: "list ", + Aliases: []string{"ls"}, + Short: "List a sandbox's forwarded serial ports", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + sb, err := store.Load(args[0]) + if err != nil { + return err + } + if serialListJSON { + enc := json.NewEncoder(cmd.OutOrStdout()) + enc.SetIndent("", " ") + // Never null: a caller iterating the result shouldn't have to + // special-case a sandbox with no devices. + devices := sb.Serials + if devices == nil { + devices = []config.SerialDevice{} + } + return enc.Encode(devices) + } + if len(sb.Serials) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No serial devices forwarded.") + return nil + } + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "HOST\tGUEST\tPRESENT") + for _, d := range sb.Serials { + resolved, err := serialport.Resolve(d.HostPath) + present := "yes" + switch { + case err != nil: + present = "no" + case resolved != d.HostPath: + // A glob that currently resolves somewhere is worth showing + // resolved: "which board is this right now" is the question + // the column exists to answer. + present = resolved + } + fmt.Fprintf(w, "%s\t/dev/%s\t%s\n", d.HostPath, d.GuestName, present) + } + return w.Flush() + }, +} + +// parseSerialSpec parses HOST[:GUEST]. +// +// Split on the last colon rather than the first: a host path may contain +// one (rare but legal), while a guest name may not contain anything that +// isn't a bare filename, so the right-hand side is unambiguous. +func parseSerialSpec(spec string) (config.SerialDevice, error) { + hostPath, guestName := spec, "" + if i := strings.LastIndex(spec, ":"); i >= 0 { + hostPath, guestName = spec[:i], spec[i+1:] + } + + expanded, err := template.ExpandPath(hostPath) + if err != nil { + return config.SerialDevice{}, fmt.Errorf("device %q: %w", spec, err) + } + hostPath = expanded + + if hostPath == "" { + return config.SerialDevice{}, fmt.Errorf("device spec %q has no host path", spec) + } + if !filepath.IsAbs(hostPath) { + return config.SerialDevice{}, fmt.Errorf( + "device %q must be an absolute path (e.g. /dev/cu.usbmodem1101)", hostPath) + } + + if guestName == "" { + guestName = config.DefaultSerialGuestName(hostPath) + if guestName == "" { + // Only reachable for a glob, which has no basename to borrow. + return config.SerialDevice{}, fmt.Errorf( + "device pattern %q needs an explicit guest name (e.g. %s:ttyACM0)", + hostPath, hostPath) + } + } + if err := serialfwd.ValidDeviceName(guestName); err != nil { + return config.SerialDevice{}, err + } + return config.SerialDevice{HostPath: hostPath, GuestName: guestName}, nil +} + +// warnAboutDevice returns advisory notes for a device just added. None of +// these are errors: a board that is currently unplugged is a perfectly +// reasonable thing to configure, and the forward starts working when it +// appears. +func warnAboutDevice(dev config.SerialDevice) []string { + var notes []string + if _, err := serialport.Resolve(dev.HostPath); err != nil { + if errors.Is(err, serialport.ErrNoMatch) { + notes = append(notes, fmt.Sprintf( + "%s isn't there right now — it'll be picked up when it appears", dev.HostPath)) + } else { + notes = append(notes, err.Error()) + } + } + // The /dev/tty.* device on macOS blocks on carrier detect and is meant + // for dial-in; /dev/cu.* is the callout side and the one every serial + // tool uses. Getting this wrong looks like a port that opens and then + // does nothing. + if runtime.GOOS == "darwin" && strings.HasPrefix(dev.HostPath, "/dev/tty.") { + notes = append(notes, fmt.Sprintf( + "prefer the callout device /dev/cu.%s over /dev/tty.%[1]s", + strings.TrimPrefix(dev.HostPath, "/dev/tty."))) + } + return notes +} + +// applySerials pushes the just-saved device set into the running daemon, +// which relays it to the in-guest agent. Reports what happened but never +// fails the command: the store is already updated, so the worst case is +// that the edit lands on the next boot. Mirrors applyReverseForwards. +// +// Takes the sandbox rather than its name because ErrSerialUnsupported is +// ambiguous by construction (see vzdctl.ErrSerialUnsupported): it means +// either "this daemon predates serial forwarding" or "this backend has no +// host-side vsock listener". Only the record distinguishes them, and telling +// a firecracker user to restart the daemon sends them after a fix that can +// never work. +func applySerials(cmd *cobra.Command, sb *config.Sandbox) { + ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Second) + defer cancel() + err := vzdctl.NewClient(vzdctl.SocketPath(store.VMDir(sb.Name))).ReloadSerials(ctx) + switch { + case err == nil: + fmt.Fprintln(cmd.OutOrStdout(), "Applied to running sandbox.") + case errors.Is(err, vzdctl.ErrNotRunning): + if !serialSupported(sb) { + fmt.Fprintf(cmd.OutOrStdout(), + "Note: this sandbox uses the %s provider, which cannot forward "+ + "serial devices — the guest has no channel to dial back through. "+ + "The entry is recorded but will not appear in the sandbox.\n", + sb.Provider) + return + } + fmt.Fprintln(cmd.OutOrStdout(), "Sandbox not running — applies on next 'clawk up'.") + case errors.Is(err, vzdctl.ErrSerialUnsupported): + if !serialSupported(sb) { + fmt.Fprintf(cmd.OutOrStdout(), + "Note: this sandbox uses the %s provider, which cannot forward "+ + "serial devices — the guest has no channel to dial back through. "+ + "The entry is recorded but will not appear in the sandbox.\n", + sb.Provider) + return + } + fmt.Fprintln(cmd.OutOrStdout(), + "Sandbox is running an older daemon — restart it ('clawk down && clawk up') to apply.") + default: + fmt.Fprintf(cmd.ErrOrStderr(), + "clawk: live apply failed (%v) — applies on next 'clawk up'\n", err) + } +} + +// serialSupported reports whether sb's provider can forward serial devices +// at all. Only vz can: the guest is the end that dials (see +// internal/serialfwd), and firecracker's vsock carries host→guest only. +func serialSupported(sb *config.Sandbox) bool { + return sb.Provider.Normalize() == config.ProviderVZ +} + +// describeSerial spells out a device in the direction it is used, because +// the HOST:GUEST spec alone doesn't say which name belongs where. +func describeSerial(d config.SerialDevice) string { + return fmt.Sprintf("/dev/%s in the guest → %s", d.GuestName, d.HostPath) +} + +func serialByGuestName(devs []config.SerialDevice, name string) (config.SerialDevice, bool) { + for _, d := range devs { + if d.GuestName == name { + return d, true + } + } + return config.SerialDevice{}, false +} + +func serialByHostPath(devs []config.SerialDevice, path string) (config.SerialDevice, bool) { + for _, d := range devs { + if d.HostPath == path { + return d, true + } + } + return config.SerialDevice{}, false +} + +// serialMatchesAny reports whether d is named by any of the given +// identifiers — the spec it was added as, its host path, or its guest name. +func serialMatchesAny(d config.SerialDevice, identifiers []string) bool { + for _, id := range identifiers { + if id == d.String() || id == d.HostPath || id == d.GuestName { + return true + } + // The guest name is also accepted with the /dev/ prefix people + // naturally type when copying it out of an error message. + if strings.TrimPrefix(id, "/dev/") == d.GuestName && strings.HasPrefix(id, "/dev/") { + return true + } + // A spec whose host path needs expanding (~) won't match literally. + if dev, err := parseSerialSpec(id); err == nil && dev == d { + return true + } + } + return false +} diff --git a/internal/cli/serial_proxy.go b/internal/cli/serial_proxy.go new file mode 100644 index 0000000..66a7ce0 --- /dev/null +++ b/internal/cli/serial_proxy.go @@ -0,0 +1,425 @@ +package cli + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net" + "slices" + "sync" + "time" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/serialfwd" + "github.com/clawkwork/clawk/internal/serialport" + "github.com/clawkwork/clawk/machine" +) + +// serialProxy is the host end of serial forwarding: it publishes the +// sandbox's serial devices to the in-guest agent and, for as long as a +// guest process holds the matching PTY open, bridges bytes to and from the +// physical port. See internal/serialfwd for the wire protocol and why the +// guest is the end that dials. +// +// It is a near-twin of reverseProxy — same lifecycle, same subscribe/ +// snapshot machinery, same reason for existing before the VM does. The +// bridging half is where they part company: a reverse forward hands off two +// sockets and copies until one ends, while an attachment owns a tty whose +// line configuration changes under it mid-stream. +type serialProxy struct { + logger *log.Logger + + ctx context.Context + cancel context.CancelFunc + listener net.Listener + wg sync.WaitGroup + + mu sync.Mutex + devices []config.SerialDevice + // subs are the live control connections' wakeup channels. Each is + // buffered by one and written non-blockingly, so a burst of edits + // collapses into a single resend of the (complete) current set. + subs map[chan struct{}]struct{} + // busy is the set of device names with an attachment in flight. A + // serial port takes exactly one reader: a second attach has to be + // refused rather than queued, or two guest processes would silently + // steal each other's bytes. + busy map[string]bool +} + +func newSerialProxy(logger *log.Logger) *serialProxy { + return &serialProxy{ + logger: logger, + subs: make(map[chan struct{}]struct{}), + busy: make(map[string]bool), + } +} + +// Set replaces the published device set and wakes every connected guest. +func (p *serialProxy) Set(devices []config.SerialDevice) { + p.mu.Lock() + p.devices = slices.Clone(devices) + for sub := range p.subs { + select { + case sub <- struct{}{}: + default: // already pending — it will read the latest set anyway + } + } + p.mu.Unlock() +} + +// Start begins accepting guest connections on serialfwd.VSockPort. It is a +// no-op (with one log line) on a backend that can't accept guest-initiated +// vsock connections, matching reverseProxy.Start. +func (p *serialProxy) Start(ctx context.Context, m machine.Machine) error { + listener, ok := m.(machine.VSockListener) + if !ok { + p.logger.Printf("serial: backend doesn't expose VSockListen; not starting") + return nil + } + pCtx, cancel := context.WithCancel(ctx) + l, err := listener.VSockListen(pCtx, serialfwd.VSockPort) + if err != nil { + cancel() + return fmt.Errorf("vsock listen port=%d: %w", serialfwd.VSockPort, err) + } + p.serve(pCtx, cancel, l) + + p.mu.Lock() + n := len(p.devices) + p.mu.Unlock() + p.logger.Printf("serial: listening on guest vsock port %d (%d device(s))", + serialfwd.VSockPort, n) + return nil +} + +// serve takes ownership of l and starts accepting. Split out from Start so +// the protocol can be exercised over any net.Listener — see reverseProxy. +func (p *serialProxy) serve(ctx context.Context, cancel context.CancelFunc, l net.Listener) { + p.ctx, p.cancel, p.listener = ctx, cancel, l + p.wg.Add(1) + go p.acceptLoop() +} + +// Stop closes the listener and waits for in-flight attachments. Idempotent, +// and safe on a proxy that was never started. +func (p *serialProxy) Stop() { + if p == nil || p.listener == nil { + return + } + p.cancel() + _ = p.listener.Close() + p.wg.Wait() +} + +func (p *serialProxy) acceptLoop() { + defer p.wg.Done() + for { + conn, err := p.listener.Accept() + if err != nil { + if errors.Is(err, net.ErrClosed) || p.ctx.Err() != nil { + return + } + p.logger.Printf("serial: accept: %v", err) + return + } + p.wg.Add(1) + go p.handle(conn) + } +} + +// handle reads the greeting and dispatches to the control stream or a +// device attachment. +func (p *serialProxy) handle(conn net.Conn) { + defer p.wg.Done() + defer conn.Close() + + // Shutdown has to reach an attachment that is simply idle — a serial + // monitor on a quiet board sends nothing for hours, and Stop waits on + // this goroutine. Closing the conn from the context unblocks it. + defer context.AfterFunc(p.ctx, func() { _ = conn.Close() })() + + r := serialfwd.NewReader(conn) + var g serialfwd.Greeting + if err := serialfwd.ReadLine(r, &g); err != nil { + if !errors.Is(err, io.EOF) { + p.logger.Printf("serial: reading greeting: %v", err) + } + return + } + if g.V != serialfwd.ProtoVersion { + // The guest agent is re-injected from the host's sources every cold + // boot, so this only happens across a resumed suspend state — where + // a down/up is exactly the fix. + p.logger.Printf("serial: guest speaks protocol v%d, host speaks v%d — "+ + "'clawk down && clawk up' to refresh the guest agent", g.V, serialfwd.ProtoVersion) + return + } + switch g.Op { + case serialfwd.OpControl: + p.serveControl(conn, r) + case serialfwd.OpAttach: + p.serveAttach(conn, r, g) + default: + p.logger.Printf("serial: unknown op %q", g.Op) + } +} + +// serveControl streams the device set to one guest until it disconnects or +// the daemon shuts down. The guest treats every snapshot as the complete +// desired state, so no delta bookkeeping is needed on either side. +func (p *serialProxy) serveControl(conn net.Conn, r *bufio.Reader) { + updates, unsubscribe := p.subscribe() + defer unsubscribe() + + // The guest never writes again on a control connection, so a read that + // returns is the guest going away. + gone := make(chan struct{}) + go func() { + defer close(gone) + _, _ = io.Copy(io.Discard, r) + }() + + for { + if err := serialfwd.WriteLine(conn, p.snapshot()); err != nil { + if p.ctx.Err() == nil && !errors.Is(err, net.ErrClosed) { + p.logger.Printf("serial: sending snapshot: %v", err) + } + return + } + select { + case <-updates: + case <-gone: + return + case <-p.ctx.Done(): + return + } + } +} + +// openRetryWindow bounds how long an attach waits for a device that isn't +// in /dev yet, and openRetryInterval is how often it looks. +// +// This window is not about patience with an unplugged board — it is the +// re-enumeration gap. A board told to enter its bootloader (the 1200-baud +// touch) drops off the USB bus and comes back a moment later, and the guest +// tool reopens the port straight away. Failing that reopen would break +// every upload to a native-USB board; waiting a couple of seconds makes it +// work. Past the window the attach fails and the guest retries on its own, +// so nothing is lost by keeping it short. +const ( + openRetryWindow = 3 * time.Second + openRetryInterval = 100 * time.Millisecond +) + +// serveAttach bridges one guest PTY to a physical serial port for as long as +// the guest holds it open. +func (p *serialProxy) serveAttach(conn net.Conn, r *bufio.Reader, g serialfwd.Greeting) { + dev, ok := p.lookup(g.Name) + if !ok { + // Reachable when an edit races an attach the guest already started; + // also the backstop if a guest ever names a device nobody + // configured. + p.logger.Printf("serial: refused attach to %q (not configured)", g.Name) + p.refuse(conn, fmt.Sprintf("no serial device named %q is forwarded", g.Name)) + return + } + if !p.claim(dev.GuestName) { + p.logger.Printf("serial: refused attach to %q (already attached)", dev.GuestName) + p.refuse(conn, fmt.Sprintf("serial device %q is already in use", dev.GuestName)) + return + } + defer p.release(dev.GuestName) + + mode := serialfwd.DefaultMode() + if g.Mode != nil { + mode = *g.Mode + } + + port, err := p.openWithRetry(dev.HostPath, mode) + if err != nil { + p.logger.Printf("serial: attach %s (%s): %v", dev.GuestName, dev.HostPath, err) + p.refuse(conn, err.Error()) + return + } + defer port.Close() + + if err := serialfwd.WriteLine(conn, serialfwd.AttachReply{OK: true}); err != nil { + return + } + p.logger.Printf("serial: %s attached to %s at %s", dev.GuestName, port.Path(), mode) + defer p.logger.Printf("serial: %s detached from %s", dev.GuestName, port.Path()) + + p.pump(conn, r, port, dev.GuestName) +} + +// pump moves bytes and mode changes between one guest attachment and one +// open port until either end stops. +// +// The two directions are deliberately asymmetric. Device → guest is only +// ever data, so it gets a plain goroutine that owns the connection's write +// side outright — no lock, because nothing else writes after the handshake. +// Guest → device is a frame loop, because a mode change has to be applied +// in the right place in the byte stream rather than whenever it happens to +// arrive. +func (p *serialProxy) pump(conn net.Conn, r *bufio.Reader, port *serialport.Port, name string) { + done := make(chan struct{}, 2) + + go func() { + defer func() { done <- struct{}{} }() + buf := make([]byte, 4096) + for { + n, err := port.Read(buf) + if n > 0 { + if werr := serialfwd.WriteFrame(conn, serialfwd.FrameData, buf[:n]); werr != nil { + return + } + } + if err != nil { + // Any read error ends the attachment: the board was + // unplugged, or Close raced us during teardown. Both mean + // this port is finished, and the guest reattaches if its + // client is still there. + if p.ctx.Err() == nil && !errors.Is(err, io.EOF) && !errors.Is(err, net.ErrClosed) { + p.logger.Printf("serial: %s: reading %s: %v", name, port.Path(), err) + } + return + } + } + }() + + go func() { + defer func() { done <- struct{}{} }() + for { + typ, payload, err := serialfwd.ReadFrame(r) + if err != nil { + if p.ctx.Err() == nil && !errors.Is(err, io.EOF) && !errors.Is(err, net.ErrClosed) { + p.logger.Printf("serial: %s: reading frame: %v", name, err) + } + return + } + switch typ { + case serialfwd.FrameData: + if _, err := port.Write(payload); err != nil { + p.logger.Printf("serial: %s: writing %s: %v", name, port.Path(), err) + return + } + case serialfwd.FrameMode: + var mode serialfwd.Mode + if err := json.Unmarshal(payload, &mode); err != nil { + p.logger.Printf("serial: %s: bad mode frame: %v", name, err) + continue + } + if err := port.Configure(mode); err != nil { + // Not fatal: a board that won't do 250000 baud is worth + // a log line, but tearing the attachment down would + // lose the session over a setting the tool may not even + // depend on. + p.logger.Printf("serial: %s: %v", name, err) + continue + } + p.logger.Printf("serial: %s reconfigured to %s", name, mode) + default: + p.logger.Printf("serial: %s: ignoring %s frame", name, typ) + } + } + }() + + // The first direction to finish ends the attachment. Closing the port + // unblocks the reader goroutine (its fd is registered with the Go + // poller); closing the conn unblocks the frame loop. Both happen in the + // deferred cleanup of serveAttach and handle respectively, so returning + // here is enough. + <-done +} + +// openWithRetry opens the device, tolerating a short absence — see +// openRetryWindow. +func (p *serialProxy) openWithRetry(pattern string, mode serialfwd.Mode) (*serialport.Port, error) { + deadline := time.Now().Add(openRetryWindow) + for { + port, err := serialport.Open(pattern, mode) + if err == nil { + return port, nil + } + // Only absence is retried. A device that exists but refuses to open + // — held by the Arduino IDE on the Mac, or a permissions problem — + // will refuse again in 100ms, and reporting it at once gives the + // user something to act on. + if !errors.Is(err, serialport.ErrNoMatch) || time.Now().After(deadline) { + return nil, err + } + select { + case <-time.After(openRetryInterval): + case <-p.ctx.Done(): + return nil, err + } + } +} + +// refuse sends a rejected AttachReply. Errors are dropped: the guest is +// about to see the connection close either way. +func (p *serialProxy) refuse(conn net.Conn, reason string) { + _ = serialfwd.WriteLine(conn, serialfwd.AttachReply{OK: false, Error: reason}) +} + +// snapshot renders the current set in wire form. Host paths are left out — +// the guest has no use for them. +func (p *serialProxy) snapshot() serialfwd.Snapshot { + p.mu.Lock() + defer p.mu.Unlock() + snap := serialfwd.Snapshot{Devices: make([]serialfwd.Device, 0, len(p.devices))} + for _, d := range p.devices { + snap.Devices = append(snap.Devices, serialfwd.Device{Name: d.GuestName}) + } + return snap +} + +// lookup resolves a guest-visible name to its configured device. The guest +// names a device and never a path, and an unlisted name is refused — +// otherwise any process in the sandbox could open anything in the Mac's +// /dev. +func (p *serialProxy) lookup(name string) (config.SerialDevice, bool) { + p.mu.Lock() + defer p.mu.Unlock() + for _, d := range p.devices { + if d.GuestName == name { + return d, true + } + } + return config.SerialDevice{}, false +} + +// claim reserves a device for one attachment, reporting false if another +// already holds it. +func (p *serialProxy) claim(name string) bool { + p.mu.Lock() + defer p.mu.Unlock() + if p.busy[name] { + return false + } + p.busy[name] = true + return true +} + +func (p *serialProxy) release(name string) { + p.mu.Lock() + delete(p.busy, name) + p.mu.Unlock() +} + +func (p *serialProxy) subscribe() (<-chan struct{}, func()) { + ch := make(chan struct{}, 1) + p.mu.Lock() + p.subs[ch] = struct{}{} + p.mu.Unlock() + return ch, func() { + p.mu.Lock() + delete(p.subs, ch) + p.mu.Unlock() + } +} diff --git a/internal/cli/serial_proxy_test.go b/internal/cli/serial_proxy_test.go new file mode 100644 index 0000000..2df0b5c --- /dev/null +++ b/internal/cli/serial_proxy_test.go @@ -0,0 +1,379 @@ +//go:build darwin || linux + +package cli + +import ( + "bufio" + "context" + "io" + "log" + "net" + "os" + "path/filepath" + "testing" + "time" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/serialfwd" + "github.com/clawkwork/clawk/internal/serialport/serialporttest" + "github.com/stretchr/testify/require" +) + +// startTestSerialProxy runs a serialProxy over a loopback TCP listener +// instead of vsock, the same trick startTestProxy uses for reverse +// forwarding: the protocol is transport-agnostic, so everything below +// exercises the real accept/handshake/bridge path. +func startTestSerialProxy(t *testing.T, devices ...config.SerialDevice) (*serialProxy, string) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + p := newSerialProxy(log.New(io.Discard, "", 0)) + p.Set(devices) + ctx, cancel := context.WithCancel(context.Background()) + p.serve(ctx, cancel, ln) + t.Cleanup(p.Stop) + return p, ln.Addr().String() +} + +// dialSerial opens a connection and sends one greeting, returning the +// connection plus the reader that must be used for everything after it. +func dialSerial(t *testing.T, addr string, g serialfwd.Greeting) (net.Conn, *bufio.Reader) { + t.Helper() + conn, err := net.Dial("tcp", addr) + require.NoError(t, err) + t.Cleanup(func() { conn.Close() }) + require.NoError(t, conn.SetDeadline(time.Now().Add(10*time.Second))) + require.NoError(t, serialfwd.WriteLine(conn, g)) + return conn, serialfwd.NewReader(conn) +} + +// fakeBoard is a PTY standing in for a plugged-in microcontroller: the +// proxy opens the slave as its serial port, and the test drives the master. +type fakeBoard struct { + master *os.File + path string +} + +func newFakeBoard(t *testing.T) fakeBoard { + t.Helper() + master, slavePath := serialporttest.OpenPTYPair(t) + return fakeBoard{master: master, path: slavePath} +} + +// device is the config entry pointing at this board. +func (b fakeBoard) device(guestName string) config.SerialDevice { + return config.SerialDevice{HostPath: b.path, GuestName: guestName} +} + +func TestSerialControlSendsSnapshotAndUpdates(t *testing.T) { + p, addr := startTestSerialProxy(t, + config.SerialDevice{HostPath: "/dev/cu.usbmodem1101", GuestName: "ttyACM0"}) + + _, r := dialSerial(t, addr, serialfwd.Greeting{Op: serialfwd.OpControl, V: serialfwd.ProtoVersion}) + + var snap serialfwd.Snapshot + require.NoError(t, serialfwd.ReadLine(r, &snap)) + require.Equal(t, []serialfwd.Device{{Name: "ttyACM0"}}, snap.Devices) + + // An edit to a running sandbox pushes a fresh full set — the guest + // reconciles rather than applying deltas. + p.Set([]config.SerialDevice{ + {HostPath: "/dev/cu.usbmodem1101", GuestName: "ttyACM0"}, + {HostPath: "/dev/cu.usbserial-A50285BI", GuestName: "ttyUSB0"}, + }) + require.NoError(t, serialfwd.ReadLine(r, &snap)) + require.Equal(t, []serialfwd.Device{{Name: "ttyACM0"}, {Name: "ttyUSB0"}}, snap.Devices) +} + +// The host path is the guest's business to never learn: it names devices, +// and the mapping to the Mac's /dev stays on the host. +func TestSerialSnapshotOmitsHostPaths(t *testing.T) { + _, addr := startTestSerialProxy(t, + config.SerialDevice{HostPath: "/dev/cu.usbmodem1101", GuestName: "ttyACM0"}) + + conn, r := dialSerial(t, addr, serialfwd.Greeting{Op: serialfwd.OpControl, V: serialfwd.ProtoVersion}) + _ = conn + + line, err := r.ReadString('\n') + require.NoError(t, err) + require.NotContains(t, line, "usbmodem") + require.Contains(t, line, "ttyACM0") +} + +func TestSerialAttachRefusesUnknownDevice(t *testing.T) { + _, addr := startTestSerialProxy(t, + config.SerialDevice{HostPath: "/dev/cu.usbmodem1101", GuestName: "ttyACM0"}) + + _, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM9", + }) + + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + require.False(t, reply.OK) + require.Contains(t, reply.Error, "ttyACM9") +} + +// A guest speaking a version the host doesn't gets hung up on rather than +// half-served — the alternative is a stream neither end can parse. +func TestSerialRejectsProtocolMismatch(t *testing.T) { + _, addr := startTestSerialProxy(t) + + _, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpControl, V: serialfwd.ProtoVersion + 1, + }) + + _, err := r.ReadString('\n') + require.Error(t, err, "host should close the connection") +} + +func TestSerialAttachBridgesBytesBothWays(t *testing.T) { + board := newFakeBoard(t) + _, addr := startTestSerialProxy(t, board.device("ttyACM0")) + + conn, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + Mode: &serialfwd.Mode{Baud: 115200, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1}, + }) + + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + require.True(t, reply.OK, "attach refused: %s", reply.Error) + + // Guest → board. + require.NoError(t, serialfwd.WriteFrame(conn, serialfwd.FrameData, []byte("AT\r\n"))) + buf := make([]byte, 64) + require.NoError(t, board.master.SetDeadline(time.Now().Add(5*time.Second))) + n, err := board.master.Read(buf) + require.NoError(t, err) + require.Equal(t, "AT\r\n", string(buf[:n])) + + // Board → guest. + _, err = board.master.WriteString("OK\r\n") + require.NoError(t, err) + typ, payload, err := serialfwd.ReadFrame(r) + require.NoError(t, err) + require.Equal(t, serialfwd.FrameData, typ) + require.Equal(t, "OK\r\n", string(payload)) +} + +// The greeting's mode has to reach the hardware before the first byte does. +// A tool that opens at 115200 and immediately writes must not have that +// write go out at the port's previous rate. +func TestSerialAttachAppliesGreetingMode(t *testing.T) { + board := newFakeBoard(t) + _, addr := startTestSerialProxy(t, board.device("ttyACM0")) + + _, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + Mode: &serialfwd.Mode{Baud: 115200, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1}, + }) + + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + require.True(t, reply.OK, "attach refused: %s", reply.Error) + + require.Equal(t, 115200, serialporttest.Speed(t, int(board.master.Fd()))) +} + +// The 1200-baud touch: a mid-stream mode change is how a native-USB board is +// told to reboot into its bootloader, so it has to actually reach the tty. +func TestSerialModeFrameReconfiguresPort(t *testing.T) { + board := newFakeBoard(t) + _, addr := startTestSerialProxy(t, board.device("ttyACM0")) + + conn, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + Mode: &serialfwd.Mode{Baud: 115200, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1}, + }) + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + require.True(t, reply.OK, "attach refused: %s", reply.Error) + + require.NoError(t, serialfwd.WriteModeFrame(conn, serialfwd.Mode{ + Baud: 1200, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1, + })) + + require.Eventually(t, func() bool { + return serialporttest.Speed(t, int(board.master.Fd())) == 1200 + }, 5*time.Second, 20*time.Millisecond, "mode frame never reached the port") +} + +// Ordering is the reason frames exist here rather than a raw byte stream +// with mode on the side. Data written before a mode change must be on the +// wire before the port is reconfigured. +func TestSerialDataBeforeModeChangeIsNotReordered(t *testing.T) { + board := newFakeBoard(t) + _, addr := startTestSerialProxy(t, board.device("ttyACM0")) + + conn, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + Mode: &serialfwd.Mode{Baud: 115200, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1}, + }) + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + require.True(t, reply.OK) + + require.NoError(t, serialfwd.WriteFrame(conn, serialfwd.FrameData, []byte("before"))) + require.NoError(t, serialfwd.WriteModeFrame(conn, serialfwd.Mode{ + Baud: 1200, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1, + })) + + buf := make([]byte, 64) + require.NoError(t, board.master.SetDeadline(time.Now().Add(5*time.Second))) + n, err := board.master.Read(buf) + require.NoError(t, err) + require.Equal(t, "before", string(buf[:n])) + + require.Eventually(t, func() bool { + return serialporttest.Speed(t, int(board.master.Fd())) == 1200 + }, 5*time.Second, 20*time.Millisecond) +} + +// One port, one reader. A second attach has to be refused rather than +// queued, or two guest processes silently steal each other's bytes. +func TestSerialSecondAttachIsRefused(t *testing.T) { + board := newFakeBoard(t) + _, addr := startTestSerialProxy(t, board.device("ttyACM0")) + + greeting := serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + } + _, r1 := dialSerial(t, addr, greeting) + var first serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r1, &first)) + require.True(t, first.OK, "first attach refused: %s", first.Error) + + _, r2 := dialSerial(t, addr, greeting) + var second serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r2, &second)) + require.False(t, second.OK) + require.Contains(t, second.Error, "already in use") +} + +// Detaching has to release the claim, or a board is usable exactly once per +// boot. +func TestSerialReattachAfterDetach(t *testing.T) { + board := newFakeBoard(t) + _, addr := startTestSerialProxy(t, board.device("ttyACM0")) + + greeting := serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + } + conn, r := dialSerial(t, addr, greeting) + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + require.True(t, reply.OK) + require.NoError(t, conn.Close()) + + // The release happens as the handler unwinds, so give it a moment. + require.Eventually(t, func() bool { + c, rr := dialSerial(t, addr, greeting) + defer c.Close() + var again serialfwd.AttachReply + if err := serialfwd.ReadLine(rr, &again); err != nil { + return false + } + return again.OK + }, 5*time.Second, 50*time.Millisecond, "device never became reattachable") +} + +// A board that is mid-reset isn't there yet. The attach waits out the +// re-enumeration gap rather than failing, which is what makes an upload to +// a native-USB board work. +func TestSerialAttachWaitsForDeviceToAppear(t *testing.T) { + board := newFakeBoard(t) + + // A symlink under a temp dir stands in for the device node: it can be + // created after the fact, which a PTY path cannot. + dir := t.TempDir() + link := filepath.Join(dir, "cu.usbmodem1101") + + _, addr := startTestSerialProxy(t, + config.SerialDevice{HostPath: link, GuestName: "ttyACM0"}) + + appeared := make(chan struct{}) + go func() { + time.Sleep(300 * time.Millisecond) + _ = os.Symlink(board.path, link) + close(appeared) + }() + + _, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + }) + + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + <-appeared + require.True(t, reply.OK, "attach gave up before the device came back: %s", reply.Error) +} + +// Past the retry window the attach fails with something the user can act +// on, rather than hanging until the guest gives up. +func TestSerialAttachFailsWhenDeviceNeverAppears(t *testing.T) { + dir := t.TempDir() + _, addr := startTestSerialProxy(t, config.SerialDevice{ + HostPath: filepath.Join(dir, "cu.usbmodem1101"), GuestName: "ttyACM0", + }) + + start := time.Now() + _, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + }) + + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + require.False(t, reply.OK) + require.Contains(t, reply.Error, "no device matches") + require.GreaterOrEqual(t, time.Since(start), openRetryWindow, + "should have used the whole retry window before giving up") +} + +// A glob that matches two boards must refuse rather than guess — flashing +// the wrong device is the worst outcome available here. And because it is +// not an absence, it must not burn the retry window first. +func TestSerialAttachRefusesAmbiguousGlob(t *testing.T) { + dir := t.TempDir() + for _, n := range []string{"cu.usbmodem1101", "cu.usbmodem2201"} { + require.NoError(t, os.WriteFile(filepath.Join(dir, n), nil, 0o600)) + } + _, addr := startTestSerialProxy(t, config.SerialDevice{ + HostPath: filepath.Join(dir, "cu.usbmodem*"), GuestName: "ttyACM0", + }) + + start := time.Now() + _, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + }) + + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + require.False(t, reply.OK) + require.Contains(t, reply.Error, "matches 2 devices") + require.Less(t, time.Since(start), openRetryWindow, "should have failed immediately") +} + +// Stop has to reach an attachment parked on a silent board, or the daemon +// hangs on shutdown waiting for a byte that never comes. +func TestSerialStopUnblocksIdleAttachment(t *testing.T) { + board := newFakeBoard(t) + p, addr := startTestSerialProxy(t, board.device("ttyACM0")) + + _, r := dialSerial(t, addr, serialfwd.Greeting{ + Op: serialfwd.OpAttach, V: serialfwd.ProtoVersion, Name: "ttyACM0", + }) + var reply serialfwd.AttachReply + require.NoError(t, serialfwd.ReadLine(r, &reply)) + require.True(t, reply.OK) + + stopped := make(chan struct{}) + go func() { p.Stop(); close(stopped) }() + + select { + case <-stopped: + case <-time.After(5 * time.Second): + t.Fatal("Stop hung on an idle attachment") + } +} diff --git a/internal/cli/serial_test.go b/internal/cli/serial_test.go new file mode 100644 index 0000000..ccb8fd5 --- /dev/null +++ b/internal/cli/serial_test.go @@ -0,0 +1,283 @@ +package cli + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/stretchr/testify/require" +) + +func TestSerialAdd(t *testing.T) { + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + })) + + out, err := executeCommand("serial", "add", "ard", + "/dev/cu.usbmodem1101", "/dev/cu.usbserial-A50285BI:ttyUSB0") + require.NoError(t, err) + // Which name belongs on which side is the confusion the wording exists + // to prevent, so it has to show up in the output. + require.Contains(t, out, "/dev/cu.usbmodem1101 in the guest → /dev/cu.usbmodem1101") + require.Contains(t, out, "/dev/ttyUSB0 in the guest → /dev/cu.usbserial-A50285BI") + require.Contains(t, out, "applies on next 'clawk up'") + + sb, err := s.Load("ard") + require.NoError(t, err) + require.Equal(t, []config.SerialDevice{ + {HostPath: "/dev/cu.usbmodem1101", GuestName: "cu.usbmodem1101"}, + {HostPath: "/dev/cu.usbserial-A50285BI", GuestName: "ttyUSB0"}, + }, sb.Serials) +} + +// A board that isn't plugged in right now is a perfectly reasonable thing +// to configure — it must be a note, not a failure. +func TestSerialAddAbsentDeviceIsANoteNotAnError(t *testing.T) { + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + })) + + out, err := executeCommand("serial", "add", "ard", "/dev/cu.definitelyNotThere") + require.NoError(t, err) + require.Contains(t, out, "isn't there right now") + + sb, err := s.Load("ard") + require.NoError(t, err) + require.Len(t, sb.Serials, 1) +} + +func TestSerialAddIsIdempotent(t *testing.T) { + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + })) + + _, err := executeCommand("serial", "add", "ard", "/dev/cu.usbmodem1101") + require.NoError(t, err) + out, err := executeCommand("serial", "add", "ard", "/dev/cu.usbmodem1101") + require.NoError(t, err) + require.Contains(t, out, "already forwarded") + + sb, err := s.Load("ard") + require.NoError(t, err) + require.Len(t, sb.Serials, 1) +} + +// One guest name can only be one device — the guest creates /dev/ +// once, and a second claim would silently lose in there. +func TestSerialAddRejectsGuestNameClash(t *testing.T) { + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + })) + + _, err := executeCommand("serial", "add", "ard", "/dev/cu.usbmodem1101:ttyACM0") + require.NoError(t, err) + _, err = executeCommand("serial", "add", "ard", "/dev/cu.usbmodem2201:ttyACM0") + require.Error(t, err) + require.Contains(t, err.Error(), "already forwarded from /dev/cu.usbmodem1101") + + sb, err := s.Load("ard") + require.NoError(t, err) + require.Len(t, sb.Serials, 1) +} + +// The same port under two names would let the guest hold it twice, and the +// second open fails pointing at the wrong thing. +func TestSerialAddRejectsDuplicateHostPath(t *testing.T) { + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + })) + + _, err := executeCommand("serial", "add", "ard", "/dev/cu.usbmodem1101:ttyACM0") + require.NoError(t, err) + _, err = executeCommand("serial", "add", "ard", "/dev/cu.usbmodem1101:ttyACM1") + require.Error(t, err) + require.Contains(t, err.Error(), `already forwarded as "ttyACM0"`) +} + +func TestSerialRemove(t *testing.T) { + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + Serials: []config.SerialDevice{ + {HostPath: "/dev/cu.usbmodem1101", GuestName: "ttyACM0"}, + {HostPath: "/dev/cu.usbserial-A50285BI", GuestName: "ttyUSB0"}, + }, + })) + + // Removal by guest name, which is what an error message inside the + // guest would have shown the user. + out, err := executeCommand("serial", "remove", "ard", "ttyACM0") + require.NoError(t, err) + require.Contains(t, out, "Serial device removed") + + sb, err := s.Load("ard") + require.NoError(t, err) + require.Equal(t, []config.SerialDevice{ + {HostPath: "/dev/cu.usbserial-A50285BI", GuestName: "ttyUSB0"}, + }, sb.Serials) +} + +func TestSerialRemoveAcceptsEveryIdentifier(t *testing.T) { + for _, id := range []string{ + "/dev/cu.usbmodem1101", // host path + "ttyACM0", // guest name + "/dev/ttyACM0", // guest name as it appears in the VM + "/dev/cu.usbmodem1101:ttyACM0", // the spec as added + } { + t.Run(id, func(t *testing.T) { + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + Serials: []config.SerialDevice{ + {HostPath: "/dev/cu.usbmodem1101", GuestName: "ttyACM0"}, + }, + })) + + _, err := executeCommand("serial", "remove", "ard", id) + require.NoError(t, err) + + sb, err := s.Load("ard") + require.NoError(t, err) + require.Empty(t, sb.Serials) + }) + } +} + +func TestSerialRemoveUnknownSaysSo(t *testing.T) { + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + Serials: []config.SerialDevice{ + {HostPath: "/dev/cu.usbmodem1101", GuestName: "ttyACM0"}, + }, + })) + + out, err := executeCommand("serial", "remove", "ard", "ttyNOPE") + require.NoError(t, err) + require.Contains(t, out, "nothing matched") + + sb, err := s.Load("ard") + require.NoError(t, err) + require.Len(t, sb.Serials, 1) +} + +func TestSerialListJSONIsNeverNull(t *testing.T) { + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + })) + + out, err := executeCommand("serial", "list", "ard", "--json") + require.NoError(t, err) + var devices []config.SerialDevice + require.NoError(t, json.Unmarshal([]byte(out), &devices)) + require.NotNil(t, devices) + require.Empty(t, devices) +} + +func TestSerialListShowsPresence(t *testing.T) { + dir := t.TempDir() + present := filepath.Join(dir, "cu.present") + require.NoError(t, os.WriteFile(present, nil, 0o600)) + + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + Serials: []config.SerialDevice{ + {HostPath: present, GuestName: "ttyACM0"}, + {HostPath: filepath.Join(dir, "cu.absent"), GuestName: "ttyACM1"}, + }, + })) + + out, err := executeCommand("serial", "list", "ard") + require.NoError(t, err) + require.Regexp(t, `cu\.present.*/dev/ttyACM0\s+yes`, out) + require.Regexp(t, `cu\.absent.*/dev/ttyACM1\s+no`, out) +} + +// A glob resolves to whichever board is on the bus right now, and that is +// the question the PRESENT column exists to answer. +func TestSerialListResolvesGlob(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "cu.usbmodem14201"), nil, 0o600)) + + s, _ := setupTest(t) + require.NoError(t, s.Save(&config.Sandbox{ + Name: "ard", Provider: config.ProviderVZ, VMState: config.VMStateStopped, + Serials: []config.SerialDevice{ + {HostPath: filepath.Join(dir, "cu.usbmodem*"), GuestName: "ttyACM0"}, + }, + })) + + out, err := executeCommand("serial", "list", "ard") + require.NoError(t, err) + require.Contains(t, out, "cu.usbmodem14201") +} + +func TestParseSerialSpec(t *testing.T) { + tests := []struct { + name string + spec string + want config.SerialDevice + }{ + { + name: "bare path keeps its basename in the guest", + spec: "/dev/cu.usbmodem1101", + want: config.SerialDevice{HostPath: "/dev/cu.usbmodem1101", GuestName: "cu.usbmodem1101"}, + }, + { + name: "explicit guest name", + spec: "/dev/cu.usbmodem1101:ttyACM0", + want: config.SerialDevice{HostPath: "/dev/cu.usbmodem1101", GuestName: "ttyACM0"}, + }, + { + name: "glob with explicit name", + spec: "/dev/cu.usbmodem*:ttyACM0", + want: config.SerialDevice{HostPath: "/dev/cu.usbmodem*", GuestName: "ttyACM0"}, + }, + { + name: "trailing colon falls back to the default name", + spec: "/dev/ttyACM0:", + want: config.SerialDevice{HostPath: "/dev/ttyACM0", GuestName: "ttyACM0"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseSerialSpec(tt.spec) + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func TestParseSerialSpecRejects(t *testing.T) { + tests := []struct { + name string + spec string + contains string + }{ + {"relative path", "cu.usbmodem1101", "absolute path"}, + {"empty", "", "no host path"}, + {"glob without a name", "/dev/cu.usbmodem*", "needs an explicit guest name"}, + // The guest name becomes /dev/, so anything with a path in it + // has to be refused here rather than inside the VM. + {"guest name with a slash", "/dev/ttyACM0:sub/dir", "path separator"}, + {"guest name escaping /dev", "/dev/ttyACM0:../../etc/passwd", "path separator"}, + {"guest name starting with a dot", "/dev/ttyACM0:.hidden", "must not start with a dot"}, + {"guest name with a space", "/dev/ttyACM0:tty ACM0", "contains"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := parseSerialSpec(tt.spec) + require.Error(t, err) + require.Contains(t, err.Error(), tt.contains) + }) + } +} diff --git a/internal/cli/sessions.go b/internal/cli/sessions.go index f846efd..7b7efb4 100644 --- a/internal/cli/sessions.go +++ b/internal/cli/sessions.go @@ -31,7 +31,7 @@ func sessionHistoryEnabled(sb *config.Sandbox) bool { } // sessionClaudeDir is the host working tree mounted into the guest as -// ~/.claude — the same path PersistentClaudeShares serves. +// ~/.claude — the same path PersistentAgentShares serves. func sessionClaudeDir(sb *config.Sandbox) string { return filepath.Join(store.StateDir(sb.Name), "claude") } diff --git a/internal/cli/setup.go b/internal/cli/setup.go index 0f4237c..7e8d30c 100644 --- a/internal/cli/setup.go +++ b/internal/cli/setup.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" + "github.com/clawkwork/clawk/internal/template" "github.com/spf13/cobra" ) @@ -133,6 +134,9 @@ func buildPrereqChecks(root string) []prereqCheck { // have been parsed." func init() { rootCmd.PersistentPreRunE = func(cmd *cobra.Command, _ []string) error { + // --no-global has to land before the first template load, and every + // load happens inside a command's RunE. + template.GlobalDisabled = noGlobalFlag // Walk up the command tree to the top-level child of root // (e.g. `clawk image build` → "image"). The skip list is // keyed on the top-level verb because that's the granularity diff --git a/internal/cli/shares.go b/internal/cli/shares.go index 02bdd2f..b58bd3e 100644 --- a/internal/cli/shares.go +++ b/internal/cli/shares.go @@ -69,14 +69,17 @@ func collectSandboxShares(sb *config.Sandbox) []machine.Share { } } - // Per-sandbox persistent state — whole ~/.claude/, pre-seeded with - // settings.json/CLAUDE.md/.credentials.json by SeedClaudeStateDir - // (called from the provider Create path). Host dir lives outside - // VMDir so destroy cannot touch it; the same sandbox name always - // sees the same state dir across recreate cycles. MUST come before - // DefaultHostShares so the agents/commands sub-mounts land on top - // of the parent rather than being shadowed under it. - for _, sh := range sandbox.PersistentClaudeShares(store.StateDir(sb.Name)) { + // Per-sandbox persistent state — one whole home dir per coding-agent + // runner (~/.claude, ~/.codex, ~/.pi). The vz rootfs is re-cloned from + // the image on every boot, so these mounts are the ONLY thing keeping a + // runner's sessions and login across `clawk down && clawk up`. Claude's + // is additionally pre-seeded with settings.json/CLAUDE.md/.credentials.json + // by SeedClaudeStateDir (called from the provider Create path). Host dirs + // live outside VMDir so destroy cannot touch them; the same sandbox name + // always sees the same state dir across recreate cycles. MUST come before + // DefaultHostShares so the agents/commands/skills sub-mounts land on top + // of their parents rather than being shadowed under them. + for _, sh := range sandbox.PersistentAgentShares(store.StateDir(sb.Name)) { out = append(out, machine.Share{ HostPath: sh.HostPath, Tag: sh.Tag, diff --git a/internal/cli/shares_test.go b/internal/cli/shares_test.go index e29b72b..edd7a70 100644 --- a/internal/cli/shares_test.go +++ b/internal/cli/shares_test.go @@ -1,6 +1,8 @@ package cli import ( + "os" + "path/filepath" "testing" "github.com/clawkwork/clawk/internal/config" @@ -77,3 +79,78 @@ func TestCollectSandboxShares_InPlaceOnly(t *testing.T) { "in-place-only sandbox needs no consolidated parent") require.Contains(t, byTag, "here") } + +// TestCollectSandboxShares_AgentStateDevices pins the host-device side of the +// per-runner state mounts. Every entry in sandbox.AgentStateDirs must get a +// virtio-fs device here, or the guest manifest asks clawk-init to mount a tag +// vz never exposed — the runner then writes to the rootfs, which vz re-clones +// from the image on every boot, and its history disappears on `clawk up`. +func TestCollectSandboxShares_AgentStateDevices(t *testing.T) { + withTempStore(t) + + sb := &config.Sandbox{Name: "box"} + byTag := map[string]machine.Share{} + for _, s := range collectSandboxShares(sb) { + byTag[s.Tag] = s + } + + stateRoot := store.StateDir("box") + for _, d := range sandbox.AgentStateDirs { + sh, ok := byTag[d.Tag] + require.Truef(t, ok, "%s state device (tag %s) missing", d.Agent, d.Tag) + require.Equal(t, filepath.Join(stateRoot, d.Sub), sh.HostPath, + "%s state must come from the sandbox's host state dir, which survives destroy", d.Agent) + require.Falsef(t, sh.ReadOnly, "%s state must be writable", d.Agent) + } +} + +// TestCollectSandboxShares_MatchesGuestManifest is the lock-step check the +// comment on collectSandboxShares demands: the vz device list and the guest +// mount manifest are built by two separate functions, and a tag in one but +// not the other is either a mount of a device that doesn't exist or a device +// nothing ever mounts. Both failures are silent at boot. +func TestCollectSandboxShares_MatchesGuestManifest(t *testing.T) { + withTempStore(t) + + home := t.TempDir() + t.Setenv("HOME", home) + require.NoError(t, os.MkdirAll(filepath.Join(home, ".claude", "agents"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(home, ".codex", "skills"), 0o755)) + + sb := &config.Sandbox{ + Name: "box", + Image: "golang:1.25", + Phases: []config.Phase{ + {Worktree: filepath.Join(store.WorktreeDir("box"), "proj"), Repo: "/code/proj"}, + {Worktree: "/code/here", Repo: "/code/here", InPlace: true}, + }, + Shares: []config.HostShare{ + {HostPath: "/Users/u/.aws", GuestPath: sandbox.GuestHome + "/.aws", ReadOnly: true}, + }, + } + + deviceTags := map[string]bool{} + for _, s := range collectSandboxShares(sb) { + deviceTags[s.Tag] = true + } + + m, err := sandbox.OCIGuestManifest(sb, store.StateDir("box"), store.CacheDir(), store.RootDir()) + require.NoError(t, err) + mountTags := map[string]bool{} + for _, mt := range m.Mounts { + if mt.Block != "" { + continue // block devices carry no virtio-fs tag + } + mountTags[mt.Tag] = true + } + + for tag := range mountTags { + require.Containsf(t, deviceTags, tag, + "guest mounts tag %q but no virtio-fs device exposes it", tag) + } + for tag := range deviceTags { + require.Containsf(t, mountTags, tag, + "virtio-fs device %q is exposed but the guest never mounts it", tag) + } + require.Contains(t, mountTags, "codex_home", "test set up no agent state mounts") +} diff --git a/internal/cli/shell.go b/internal/cli/shell.go index 4c8add8..12ffd1b 100644 --- a/internal/cli/shell.go +++ b/internal/cli/shell.go @@ -89,7 +89,7 @@ func tryVSockShell(sb *config.Sandbox, provider sandbox.Provider) error { Args: []string{"-l"}, Cwd: agentStartDir(provider, sb), User: sandbox.GuestUser, - Env: buildVSockEnv(), + Env: buildVSockEnv(sb), // No ClearScreen: a login shell is line-oriented, so keep the // user's scrollback — the shell should read as a continuation of // the terminal it launched from, not a wiped screen. diff --git a/internal/cli/status.go b/internal/cli/status.go index 0f2867f..649e457 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -67,10 +67,16 @@ type statusJSONOutput struct { // v2 additive blocks. Forwards []statusJSONForward `json:"forwards,omitempty"` ReverseForwards []statusJSONForward `json:"reverse_forwards,omitempty"` + Serials []statusJSONSerial `json:"serials,omitempty"` Network *statusJSONNetwork `json:"network,omitempty"` Setup []statusJSONSetup `json:"setup,omitempty"` } +type statusJSONSerial struct { + HostPath string `json:"host_path"` + GuestName string `json:"guest_name"` +} + type statusJSONBranch struct { Index int `json:"index"` Repo string `json:"repo"` @@ -191,6 +197,11 @@ func renderStatusJSON(w io.Writer, sb *config.Sandbox, liveStatus string) error HostPort: f.HostPort, GuestPort: f.GuestPort, }) } + for _, d := range sb.Serials { + out.Serials = append(out.Serials, statusJSONSerial{ + HostPath: d.HostPath, GuestName: d.GuestName, + }) + } out.Network = &statusJSONNetwork{ Use: effectiveUseForLog(sb), Blocks: sb.Network.Blocks, @@ -291,6 +302,15 @@ func renderStatusDashboard(w io.Writer, provider sandbox.Provider, sb *config.Sa } fmt.Fprintf(w, " Reverse %s (guest → host loopback)\n", strings.Join(parts, ", ")) } + // Same direction convention as the rows above: what the guest sees on + // the left, what it reaches on the right. + if len(sb.Serials) > 0 { + parts := make([]string, 0, len(sb.Serials)) + for _, d := range sb.Serials { + parts = append(parts, fmt.Sprintf("/dev/%s → %s", d.GuestName, d.HostPath)) + } + fmt.Fprintf(w, " Serial %s\n", strings.Join(parts, ", ")) + } // One line per policy layer, lowest precedence first: the use chain, // then the sandbox's own blocks with entry counts. Full contents live diff --git a/internal/cli/swap_test.go b/internal/cli/swap_test.go new file mode 100644 index 0000000..d4e4b3f --- /dev/null +++ b/internal/cli/swap_test.go @@ -0,0 +1,37 @@ +package cli + +import ( + "testing" + + "github.com/clawkwork/clawk/internal/template" + "github.com/stretchr/testify/require" +) + +// TestResolveSwap covers the workspace-merge rule: the max size across the +// workspace file and every repo Clawkfile, with zero as "unset" — except +// that an explicit "off" (negative) beats any size, the same asymmetry +// resolveIdleTimeout has and for the same reason. +func TestResolveSwap(t *testing.T) { + ws := func(fileMiB int64, repoMiB ...int64) *template.Workspace { + w := &template.Workspace{File: &template.Template{SwapMiB: fileMiB}} + for _, m := range repoMiB { + w.Repos = append(w.Repos, template.Repo{ + Clawkfile: &template.Template{SwapMiB: m}, + }) + } + return w + } + + require.Equal(t, int64(0), resolveSwap(ws(0)), "unset everywhere stays unset") + require.Equal(t, int64(8192), resolveSwap(ws(8192)), "workspace value") + require.Equal(t, int64(16384), resolveSwap(ws(8192, 16384)), "larger repo wins") + require.Equal(t, int64(8192), resolveSwap(ws(8192, 4096)), "smaller repo ignored") + require.Equal(t, int64(-1), resolveSwap(ws(8192, -1)), "a repo's 'off' wins over a size") + require.Equal(t, int64(-1), resolveSwap(ws(-1, 8192)), "the workspace's 'off' wins too") + require.Equal(t, int64(-1), resolveSwap(ws(0, 4096, -1, 16384)), "off wins wherever it appears") + + // A repo with no Clawkfile must not panic or count. + w := ws(2048) + w.Repos = append(w.Repos, template.Repo{Clawkfile: nil}) + require.Equal(t, int64(2048), resolveSwap(w)) +} diff --git a/internal/cli/up.go b/internal/cli/up.go index 7a7f699..84dd89c 100644 --- a/internal/cli/up.go +++ b/internal/cli/up.go @@ -215,6 +215,21 @@ func ensureRuntimeMounts(provider sandbox.Provider, sb *config.Sandbox) error { } } } + // Per-sandbox agent state (~/.claude, ~/.codex, ~/.pi). Normally the + // boot manifest has these mounted already and every call here is a + // `mountpoint -q` no-op — but clawk-init only LOGS a failed host-share + // mount and carries on booting, and a runner whose home didn't mount + // writes to the rootfs vz re-clones next boot, i.e. loses its history + // with no error anywhere the user looks. Cheap retry, expensive miss. + // + // Mounted BEFORE DefaultHostShares: its capability dirs are sub-mounts + // inside these homes (~/.claude/agents, ~/.codex/skills), and a parent + // mounted afterwards would shadow them. + for _, sh := range sandbox.PersistentAgentShares(store.StateDir(sb.Name)) { + if err := mountIfMissing(sp, sb, sh.Tag, sh.GuestPath); err != nil { + return fmt.Errorf("%s: %w", sh.GuestPath, err) + } + } for _, sh := range sandbox.DefaultHostShares() { if err := mountIfMissing(sp, sb, sh.Tag, sh.GuestPath); err != nil { return fmt.Errorf("%s: %w", sh.GuestPath, err) diff --git a/internal/cli/vshell.go b/internal/cli/vshell.go index 72760c7..4f6675f 100644 --- a/internal/cli/vshell.go +++ b/internal/cli/vshell.go @@ -68,7 +68,7 @@ returns an error rather than falling back.`, Cmd: cmdPath, Args: cmdArgs, User: sandbox.GuestUser, - Env: buildVSockEnv(), + Env: buildVSockEnv(sb), } code, err := vsockclient.Run(context.Background(), cfg) if err != nil { diff --git a/internal/cli/vzd.go b/internal/cli/vzd.go index f1ddc53..e8c5aed 100644 --- a/internal/cli/vzd.go +++ b/internal/cli/vzd.go @@ -105,12 +105,20 @@ func runVzd(_ *cobra.Command, args []string) (retErr error) { rev := newReverseProxy(logger) rev.Set(sb.ReverseForwards) + // Serial proxy: publishes the sandbox's forwarded serial ports to the + // in-guest agent and bridges each PTY the guest opens through to the + // physical device. Built alongside the reverse-forward proxy and for + // the same reason — `clawk serial add` needs somewhere to push while + // the VM is still booting. + ser := newSerialProxy(logger) + ser.Set(sb.Serials) + // Control socket: lets the CLI push network-policy edits into the live // allow list (`clawk network allow` without a down/up cycle), read // the denial ledger (`clawk network denials`), push reverse-forward // edits, and drive the VM lifecycle (`clawk pause/resume/snapshot`). // Best-effort — without it, policy edits apply on the next up, as before. - ctl, err := vzdctl.Start(vzdctl.SocketPath(vmDir), controlHandlers(sb, allow, lc, rev, logger)) + ctl, err := vzdctl.Start(vzdctl.SocketPath(vmDir), controlHandlers(sb, allow, lc, rev, ser, logger)) if err != nil { logger.Printf("control socket: disabled (%v) — network edits apply on next up", err) } else { @@ -215,6 +223,16 @@ func runVzd(_ *cobra.Command, args []string) (retErr error) { defer rev.Stop() } + // Serial devices: same shape as the reverse forwards above — the guest + // agent dials this listener to learn which PTYs to create, and again + // each time a process in the sandbox opens one. Best-effort; a failure + // here means those devices simply don't appear inside the guest. + if err := ser.Start(ctx, m); err != nil { + logger.Printf("serial: disabled (%v)", err) + } else { + defer ser.Stop() + } + // 9p cache servers: one per toolchain cache, each serving its host cache // dir over 9p on the guest vsock port from ToolchainCacheShares. A // 9p-capable clawk-init mounts these instead of the caches' virtio-fs @@ -337,14 +355,25 @@ func buildOCISandboxSpec(sb *config.Sandbox, vmDir string, allow *netfilter.Allo return machine.Spec{}, fmt.Errorf("guest config disk: %w", err) } + swapPath, err := sandbox.EnsureSwapDisk(vmDir, sandbox.SwapDiskMiB(sb)) + if err != nil { + return machine.Spec{}, fmt.Errorf("swap disk: %w", err) + } + spec := buildSandboxSpec(sb, vmDir, allow) spec.Boot = machine.DirectKernel{ Vmlinux: vmlinux, Cmdline: sandbox.OCICmdline, } spec.RootFS = sandbox.OCIRootFS(sb, store.CacheDir(), bins) + // Order is the guest's device order: vdb=guestcfg, vdc=swap + // (sandbox.OCISwapDevice). The manifest names that device by path, so + // anything added here must go after it. spec.Disks = []machine.Disk{ {Path: filepath.Join(vmDir, sandbox.OCIConfigDiskName), ReadOnly: true}, } + if swapPath != "" { + spec.Disks = append(spec.Disks, machine.Disk{Path: swapPath}) + } return spec, nil } diff --git a/internal/cli/workspace_defaults_test.go b/internal/cli/workspace_defaults_test.go new file mode 100644 index 0000000..6185a8f --- /dev/null +++ b/internal/cli/workspace_defaults_test.go @@ -0,0 +1,56 @@ +package cli + +import ( + "os" + "path/filepath" + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/template" + "github.com/stretchr/testify/require" +) + +// Workspace-position `env` and `agent` blocks used to be dropped on the floor: +// only repo Clawkfiles fed RequiredEnv and the agent docs. That position is +// also where the host-wide clawk.mod lands (see template/global.go), so the +// layer would have been silently half-applied. +func TestApplyWorkspaceLevelDefaults(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "house-rules.md"), + []byte("house rule"), 0o644)) + + ws := &template.Workspace{ + Root: root, + File: &template.Template{ + Env: []string{"GLOBAL_TOKEN"}, + Instructions: []template.AgentDoc{{Path: "./house-rules.md"}}, + Memory: []template.AgentDoc{{Text: "global memory"}}, + }, + } + sb := &config.Sandbox{ + Name: "x", + RequiredEnv: []string{"NS_TOKEN"}, + Instructions: []string{"namespace rule"}, + Memory: "namespace memory", + } + + require.NoError(t, applyWorkspaceLevelDefaults(sb, ws)) + + // Scope-outward ordering: workspace (and the global layer under it) first, + // then the namespace, then the repo — which for env is what decides + // precedence, since the last occurrence of a name wins. + require.Equal(t, []string{"GLOBAL_TOKEN", "NS_TOKEN"}, sb.RequiredEnv) + require.Equal(t, []string{"house rule", "namespace rule"}, sb.Instructions) + require.Equal(t, "global memory\n\nnamespace memory", sb.Memory) +} + +func TestApplyWorkspaceLevelDefaultsMissingDoc(t *testing.T) { + ws := &template.Workspace{ + Root: t.TempDir(), + File: &template.Template{ + Instructions: []template.AgentDoc{{Path: "./absent.md"}}, + }, + } + err := applyWorkspaceLevelDefaults(&config.Sandbox{Name: "x"}, ws) + require.ErrorContains(t, err, "absent.md") +} diff --git a/internal/config/namespace.go b/internal/config/namespace.go index 8841b87..af4ecbe 100644 --- a/internal/config/namespace.go +++ b/internal/config/namespace.go @@ -38,6 +38,12 @@ type Namespace struct { Files []HostFile `json:"files,omitempty"` Shares []HostShare `json:"shares,omitempty"` Env []string `json:"env,omitempty"` + // MCP are MCP servers made available to every sandbox in the namespace. + // This is the natural place for the org-wide set: which namespace a + // sandbox lives in then decides what it can reach, with no per-sandbox + // configuration. Merged with a repo's clawk.mod entries by name in + // applyNamespaceDefaults. See MCPServer. + MCP []MCPServer `json:"mcp,omitempty"` // Instructions and Memory seed every sandbox in the namespace: extra // CLAUDE.md guidance and baseline auto-memory respectively. They merge // with a repo's clawk.mod equivalents — namespace first, as the broader diff --git a/internal/config/types.go b/internal/config/types.go index 7c3cfdd..7402a99 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -2,8 +2,11 @@ package config import ( "fmt" + "path" + "path/filepath" "slices" "strconv" + "strings" "time" ) @@ -122,20 +125,32 @@ type NetworkBlock struct { // silently invert which rules win. const ( BlockOriginNamespace = "namespace" - BlockOriginMod = "mod" - BlockOriginCustom = "custom" + // BlockOriginMCP carries the allow entries derived from declared MCP + // servers (see MCPServer). Deliberately the lowest-precedence stored + // origin above "namespace": it is a convenience — clawk inferring what + // a declared server needs to reach — so an explicit `deny` written in + // clawk.mod ("mod") or via the CLI ("custom") must still win over it. + BlockOriginMCP = "mcp" + BlockOriginMod = "mod" + BlockOriginCustom = "custom" ) // blockOriginRank orders stored blocks; unknown origins sort first so a // record written by a newer clawk never outranks the user's custom block. +// +// The numbers are computed, never persisted — only the origin strings above +// are frozen — so inserting a new origin mid-chain is a matter of +// renumbering here. func blockOriginRank(origin string) int { switch origin { case BlockOriginNamespace: return 1 - case BlockOriginMod: + case BlockOriginMCP: return 2 - case BlockOriginCustom: + case BlockOriginMod: return 3 + case BlockOriginCustom: + return 4 default: return 0 } @@ -224,7 +239,9 @@ var DefaultAllowedDomains = []string{ "claude.ai", "gemini.google.com", "generativelanguage.googleapis.com", - "models.dev", // model catalog fetched by the opencode runner + "models.dev", // model catalog fetched by the opencode runner + "opencode.ai", // auth + update check for the opencode runner + "pi.dev", // model catalog + version check fetched by the pi runner "platform.claude.com", "play.googleapis.com", "statsig.anthropic.com", @@ -386,6 +403,62 @@ func (p PortForward) String() string { return fmt.Sprintf("%d:%d", p.HostPort, p.GuestPort) } +// SerialDevice is a host serial port presented as a PTY inside the guest — +// an Arduino or ESP32 on the Mac's USB, reachable from tooling in the +// sandbox. Bytes and line configuration are tunnelled over vsock (see +// internal/serialfwd); the USB device itself is not passed through, because +// no hypervisor clawk targets can do that. +// +// Like ReverseForwards these apply live to a running sandbox, and for the +// same reason they are vz-only: the guest is the end that dials. +type SerialDevice struct { + // HostPath is the device on the host, e.g. /dev/cu.usbmodem1101 on + // macOS or /dev/ttyACM0 on Linux. It may be a glob, which is resolved + // at open time rather than when it is configured — boards that + // re-enumerate into a bootloader often come back under a neighbouring + // name, and `/dev/cu.usbmodem*` survives that where a literal path + // doesn't. A glob that matches more than one device at open time is an + // error, not a guess. + HostPath string `json:"host_path"` + + // GuestName is the basename the PTY is symlinked to inside the guest, + // at /dev/. Never a path: the guest refuses anything with a + // directory part, and so does the host on the way in. + GuestName string `json:"guest_name"` +} + +// String renders the device in the HOST:GUEST spelling the CLI accepts, +// collapsing to the bare host path when the guest name is the default. +func (s SerialDevice) String() string { + if s.GuestName == DefaultSerialGuestName(s.HostPath) { + return s.HostPath + } + return fmt.Sprintf("%s:%s", s.HostPath, s.GuestName) +} + +// IsGlob reports whether HostPath is a pattern to resolve at open time +// rather than a literal device path. +func (s SerialDevice) IsGlob() bool { return strings.ContainsAny(s.HostPath, "*?[") } + +// DefaultSerialGuestName is the guest name used when a spec doesn't give +// one: the host device's basename, unchanged. +// +// Keeping the name identical on both sides is the same choice reverse +// forwarding makes with ports — you refer to the thing by the name you +// already know it by, and there is no second identifier to keep straight. +// A Mac's /dev/cu.usbmodem1101 is an odd name for a Linux device node, but +// no tool cares, and `arduino-cli -p /dev/cu.usbmodem1101` reading the same +// on both sides is worth more than tidiness. +// +// Returns "" for a glob, which has no meaningful basename; callers must +// require an explicit name in that case. +func DefaultSerialGuestName(hostPath string) string { + if strings.ContainsAny(hostPath, "*?[") { + return "" + } + return path.Base(filepath.ToSlash(hostPath)) +} + // HostFile is a snapshot-on-up file pushed from host into guest. Sourced // from clawk.mod `files (...)` and refreshed on every `clawk up` — edits // on the host are NOT live (use HostShare for that). HostPath is tilde- @@ -409,6 +482,55 @@ type HostShare struct { ReadOnly bool `json:"read_only"` } +// MCP transport identifiers. Persisted in sandbox records — frozen (see +// VMState). These are the wire names Claude Code's MCP config uses, so a +// rendered server config can pass them straight through. +const ( + MCPTransportHTTP = "http" + MCPTransportSSE = "sse" + MCPTransportStdio = "stdio" +) + +// MCPServer is one entry of a clawk.mod `mcp ( … )` block: an MCP server +// made available to the agent inside the sandbox. Sourced from the +// namespace and clawk.mod, snapshotted onto the record at create like +// every other template value, and rendered into the guest by +// sandbox.SeedClaudeMCP. +// +// Credentials are deliberately NOT modeled here. A `${VAR}` reference in +// Headers or Env is stored and rendered verbatim; the runner expands it +// against its own process environment at connect time (confirmed for +// headers, env and args). That keeps this record — and the rendered guest +// config, which lives on host disk under the sandbox state dir — free of +// secret values, exactly like RequiredEnv. The value itself travels +// separately: declare the variable in `env ( … )` and it reaches the +// runner's environment via the vsock handshake (see sandbox.ResolveEnv). +// +// Only static-credential transports are supported. Interactive OAuth +// (`claude mcp login`) stores its grant inside the guest and is out of +// scope: it can't be arranged at create time, which is the whole point of +// declaring servers here. +type MCPServer struct { + // Name is the server's identifier, as the agent sees it. Unique + // within a sandbox. + Name string `json:"name"` + // Transport is one of the MCPTransport* constants above. + Transport string `json:"transport"` + // URL is the endpoint for the http and sse transports; empty for stdio. + URL string `json:"url,omitempty"` + // Command is the argv of a stdio server, already split into words. + // Empty for http and sse. + Command []string `json:"command,omitempty"` + // Headers are extra HTTP request headers in "Name: value" form, for + // http and sse. This is where a PAT goes: + // "Authorization: Bearer ${SENTRY_TOKEN}". + Headers []string `json:"headers,omitempty"` + // Env names the environment variables handed to a stdio server. Each + // is rendered as NAME=${NAME}, so the value comes from the runner's + // environment rather than from this record. + Env []string `json:"env,omitempty"` +} + // DefaultNamespace is the grouping a sandbox belongs to when none is set. const DefaultNamespace = "default" @@ -451,6 +573,10 @@ type Sandbox struct { // by the daemon, so edits apply to a running sandbox. See // internal/revfwd. ReverseForwards []PortForward `json:"reverse_forwards,omitempty"` + // Serials are host serial ports presented as PTYs in the guest — see + // SerialDevice and internal/serialfwd. Like ReverseForwards these are + // tunnelled over vsock and apply to a running sandbox. + Serials []SerialDevice `json:"serials,omitempty"` // Files is the list of host->guest file copies refreshed on every // `clawk up`. See HostFile. Empty = no snapshots. Files []HostFile `json:"files,omitempty"` @@ -466,6 +592,13 @@ type Sandbox struct { // block read on every boot — the place for "always ask before X" or // project conventions that must survive a throwaway VM. Instructions []string `json:"instructions,omitempty"` + // MCP is the set of MCP servers made available to the agent inside the + // sandbox, sourced from the namespace and clawk.mod's `mcp ( … )` + // block. Rendered into the guest before first boot, and each http/sse + // entry's host is folded into the network policy's "mcp" block so a + // declared server is reachable without a separate `network allow`. + // See MCPServer. + MCP []MCPServer `json:"mcp,omitempty"` // Memory is seed content for the agent's auto-memory MEMORY.md, sourced // from the namespace and clawk.mod. Written into the memory dir once on // first boot and never afterward (see sandbox.SeedClaudeMemory), so a @@ -534,6 +667,12 @@ type Sandbox struct { // the next rootfs (re)build. DiskMiB uint64 `json:"disk_mib,omitempty"` + // SwapMiB is the sparse swap device's capacity in mebibytes, from + // clawk.mod `vm ( swap )`. Zero = sandbox.DefaultSwapSizeMiB; + // negative = no swap device at all. Unlike DiskMiB this is not baked into + // the rootfs — the device is a separate file, resized on the next boot. + SwapMiB int64 `json:"swap_mib,omitempty"` + // IdleTimeoutSec is how long the sandbox may sit idle (no attached // session, quiescent guest) before its VM daemon stops it to reclaim // host memory. Zero = the built-in default; negative = never stop. diff --git a/internal/config/types_test.go b/internal/config/types_test.go index 71a3e0c..427b17b 100644 --- a/internal/config/types_test.go +++ b/internal/config/types_test.go @@ -40,3 +40,44 @@ func TestSandboxNamespaceName(t *testing.T) { }) } } + +func TestDefaultSerialGuestName(t *testing.T) { + tests := []struct { + name string + hostPath string + want string + }{ + {"macOS callout device", "/dev/cu.usbmodem1101", "cu.usbmodem1101"}, + {"linux acm device", "/dev/ttyACM0", "ttyACM0"}, + {"bare name", "ttyUSB0", "ttyUSB0"}, + // A glob has no basename worth defaulting to — the caller must + // insist on an explicit guest name rather than inventing one. + {"glob has no default", "/dev/cu.usbmodem*", ""}, + {"character class glob", "/dev/ttyUSB[01]", ""}, + {"single-char glob", "/dev/ttyUSB?", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, DefaultSerialGuestName(tt.hostPath)) + }) + } +} + +func TestSerialDeviceString(t *testing.T) { + // The default guest name collapses back to the bare host path, so a + // round-tripped spec reads the way the user typed it. + require.Equal(t, "/dev/cu.usbmodem1101", + SerialDevice{HostPath: "/dev/cu.usbmodem1101", GuestName: "cu.usbmodem1101"}.String()) + require.Equal(t, "/dev/cu.usbmodem1101:ttyACM0", + SerialDevice{HostPath: "/dev/cu.usbmodem1101", GuestName: "ttyACM0"}.String()) + // A glob never has a default name, so it always shows both halves. + require.Equal(t, "/dev/cu.usbmodem*:ttyACM0", + SerialDevice{HostPath: "/dev/cu.usbmodem*", GuestName: "ttyACM0"}.String()) +} + +func TestSerialDeviceIsGlob(t *testing.T) { + require.False(t, SerialDevice{HostPath: "/dev/cu.usbmodem1101"}.IsGlob()) + require.True(t, SerialDevice{HostPath: "/dev/cu.usbmodem*"}.IsGlob()) + require.True(t, SerialDevice{HostPath: "/dev/ttyUSB[01]"}.IsGlob()) + require.True(t, SerialDevice{HostPath: "/dev/ttyUSB?"}.IsGlob()) +} diff --git a/internal/guestcfg/manifest.go b/internal/guestcfg/manifest.go index bc59f27..a5b0875 100644 --- a/internal/guestcfg/manifest.go +++ b/internal/guestcfg/manifest.go @@ -47,11 +47,43 @@ type Manifest struct { Hostname string `json:"hostname,omitempty"` Network *Network `json:"network,omitempty"` User *User `json:"user,omitempty"` + Swap *Swap `json:"swap,omitempty"` Mounts []Mount `json:"mounts,omitempty"` Files []File `json:"files,omitempty"` Services []Service `json:"services,omitempty"` } +// Swap is the guest's swap device: a sparse virtio-blk disk the host +// attaches and clawk-init formats and enables at boot. Nil means the +// sandbox runs without swap. +// +// A dedicated device rather than a swapfile on the rootfs, because +// swapon(2) rejects a file with holes: a 4 GiB swapfile would cost 4 GiB +// of real host bytes at first boot, per sandbox. A block device has no +// such rule, so the backing file stays sparse and only materializes the +// pages actually swapped. +// +// Why swap at all: the balloon controller (machine/vz) reclaims guest RAM +// under host memory pressure, against guest demand — see mergedBalloonTarget. +// With nowhere to put anonymous pages the guest answers that with direct +// reclaim stalls and, at the limit, its OOM killer. Multi-second stalls in +// the agent process are what turn a marginal network link into dropped API +// streams, since a stalled process stops draining its socket. +// +// Additive like Mount.Block — an older clawk-init ignores the field and +// boots without swap, so this needs no Version bump. +type Swap struct { + // Device is the in-guest block device path (e.g. "/dev/vdc"). The + // provider owns the ordering that produces it: disks are attached + // rootfs-first, then Spec.Disks in order. + Device string `json:"device"` + + // Swappiness sets vm.swappiness when non-zero. Zero leaves the kernel + // default (60) alone — it is not a way to say "never swap", which is + // what a literal swappiness of 0 would mean. + Swappiness int `json:"swappiness,omitempty"` +} + // Network is the static interface configuration. gvproxy assigns a fixed // DHCP lease to the sandbox MAC, but arbitrary images have no DHCP // client — clawk-init configures the same values statically instead. diff --git a/internal/sandbox/agentstate_test.go b/internal/sandbox/agentstate_test.go new file mode 100644 index 0000000..30d076f --- /dev/null +++ b/internal/sandbox/agentstate_test.go @@ -0,0 +1,162 @@ +package sandbox + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/clawkwork/clawk/internal/guestcfg" + "github.com/stretchr/testify/require" +) + +// TestPersistentAgentSharesCoverEveryRunnerHome is the regression guard for +// the bug this file exists for: only claude's home was mounted from the host, +// so codex's sessions and login lived on the vz rootfs — which is re-cloned +// from the image on EVERY boot. `clawk down && clawk up` silently wiped codex +// history even though the docs promised it persisted. Each runner clawk +// claims to persist needs its own share here. +func TestPersistentAgentSharesCoverEveryRunnerHome(t *testing.T) { + stateRoot := t.TempDir() + + byGuest := map[string]HostShare{} + for _, sh := range PersistentAgentShares(stateRoot) { + byGuest[sh.GuestPath] = sh + } + + want := map[string]struct{ sub, tag string }{ + GuestHome + "/.claude": {"claude", "claude_home"}, + GuestHome + "/.codex": {"codex", "codex_home"}, + GuestHome + "/.pi": {"pi", "pi_home"}, + // opencode follows the XDG split, so it needs two: the data dir + // (auth.json, mcp-auth.json, opencode.db) and the config dir. + GuestHome + "/.local/share/opencode": {"opencode-data", "opencode_data"}, + GuestHome + "/.config/opencode": {"opencode-config", "opencode_config"}, + } + require.Len(t, byGuest, len(want), "one share per persisted runner home") + for guest, w := range want { + sh, ok := byGuest[guest] + require.Truef(t, ok, "no persistent share mounted at %s", guest) + require.Equal(t, filepath.Join(stateRoot, w.sub), sh.HostPath) + require.Equal(t, w.tag, sh.Tag) + require.Falsef(t, sh.ReadOnly, "%s must be writable — the runner writes its sessions there", guest) + + // virtiofs refuses a missing source path, so the host dir has to + // exist by the time the share is handed to the provider. + info, err := os.Stat(sh.HostPath) + require.NoErrorf(t, err, "stat %s", sh.HostPath) + require.Truef(t, info.IsDir(), "%s exists but is not a directory", sh.HostPath) + } +} + +// TestPersistentAgentSharesExcludeVolatileDirs pins the two opencode XDG +// dirs we deliberately leave on the disposable rootfs. ~/.local/state holds +// only locks/, and a lock that outlives the VM that took it is worse than +// none — a hard stop would strand one for the next boot. ~/.cache is a +// cache. Both look like "state opencode writes", so without this guard the +// obvious-seeming fix is to add them. +func TestPersistentAgentSharesExcludeVolatileDirs(t *testing.T) { + for _, sh := range PersistentAgentShares(t.TempDir()) { + require.NotContains(t, sh.GuestPath, "/.local/state/", + "%s persists a lock directory across boots", sh.Tag) + require.NotContains(t, sh.GuestPath, "/.cache/", + "%s persists a cache directory, costing a PCIe device for no correctness gain", sh.Tag) + } +} + +func TestPersistentAgentSharesEmptyStateRoot(t *testing.T) { + require.Nil(t, PersistentAgentShares(""), "empty state root opts out entirely") +} + +// TestPersistentAgentSharesIdempotent pins that a second call (every boot +// rewrites the manifest) returns the identical list — a churned tag means the +// guest tries to mount a device vz never exposed. +func TestPersistentAgentSharesIdempotent(t *testing.T) { + stateRoot := t.TempDir() + require.Equal(t, PersistentAgentShares(stateRoot), PersistentAgentShares(stateRoot)) +} + +// TestAgentStateDirsUniqueTagsAndPaths guards the two uniqueness constraints +// the list carries: virtio-fs tags identify a device per VM, and two entries +// mounting the same guest path would have the second shadow the first. +func TestAgentStateDirsUniqueTagsAndPaths(t *testing.T) { + tags, guests, subs := map[string]bool{}, map[string]bool{}, map[string]bool{} + for _, d := range AgentStateDirs { + require.NotEmpty(t, d.Agent, "every entry names its runner") + require.Falsef(t, tags[d.Tag], "duplicate virtio-fs tag %q", d.Tag) + require.Falsef(t, guests[d.GuestPath], "duplicate guest path %q", d.GuestPath) + require.Falsef(t, subs[d.Sub], "duplicate state subdir %q", d.Sub) + require.Truef(t, strings.HasPrefix(d.GuestPath, GuestHome+"/"), + "%s must live under the agent's home so clawk-init chowns it", d.GuestPath) + tags[d.Tag], guests[d.GuestPath], subs[d.Sub] = true, true, true + } +} + +// TestPersistentAgentSharesPrecedeCapabilitySubmounts pins the ordering rule +// both share assemblers depend on. DefaultHostShares mounts capability dirs +// (~/.claude/agents, ~/.codex/skills) that live INSIDE the persisted homes; +// clawk-init mounts in list order, so a home mounted afterwards would shadow +// the capability dir already mounted under it. +func TestPersistentAgentSharesPrecedeCapabilitySubmounts(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + for _, sub := range []string{".claude/agents", ".claude/commands", ".codex/skills"} { + require.NoError(t, os.MkdirAll(filepath.Join(home, sub), 0o755)) + } + + // Same assembly order as OCIGuestManifest and collectSandboxShares. + assembled := append([]HostShare{}, PersistentAgentShares(t.TempDir())...) + assembled = append(assembled, DefaultHostShares()...) + + idx := map[string]int{} + for i, sh := range assembled { + idx[sh.GuestPath] = i + } + var checked int + for _, child := range assembled { + for _, parent := range assembled { + if child.GuestPath == parent.GuestPath || + !strings.HasPrefix(child.GuestPath, parent.GuestPath+"/") { + continue + } + checked++ + require.Lessf(t, idx[parent.GuestPath], idx[child.GuestPath], + "parent %s must be mounted before its sub-mount %s", + parent.GuestPath, child.GuestPath) + } + } + require.Positivef(t, checked, "test set up no nested mounts — it would pass vacuously") +} + +// TestOCIGuestManifestMountsEveryAgentHome ties the guest side to the list: +// a runner added to AgentStateDirs but missing from the manifest would write +// to the disposable rootfs instead, which is exactly how codex lost its +// history. +func TestOCIGuestManifestMountsEveryAgentHome(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + require.NoError(t, os.MkdirAll(filepath.Join(home, ".codex", "skills"), 0o755)) + + sb := &config.Sandbox{Name: "box", Image: "golang:1.25"} + m, err := OCIGuestManifest(sb, t.TempDir(), "", t.TempDir()) + require.NoError(t, err) + + idx := map[string]int{} + mounts := map[string]guestcfg.Mount{} + for i, mt := range m.Mounts { + idx[mt.Tag] = i + mounts[mt.Tag] = mt + } + for _, d := range AgentStateDirs { + mt, ok := mounts[d.Tag] + require.Truef(t, ok, "%s state (tag %s) missing from the guest manifest", d.Agent, d.Tag) + require.Equal(t, d.GuestPath, mt.Path) + require.Falsef(t, mt.ReadOnly, "%s state must be writable", d.Agent) + } + // The one nested pair the default shares produce, spelled out so a + // reordering of the assembly in OCIGuestManifest fails loudly here. + require.Contains(t, idx, "codex_skills") + require.Less(t, idx["codex_home"], idx["codex_skills"], + "~/.codex must mount before ~/.codex/skills or the skills mount is shadowed") +} diff --git a/internal/sandbox/firecracker_linux.go b/internal/sandbox/firecracker_linux.go index 3d1770e..8516031 100644 --- a/internal/sandbox/firecracker_linux.go +++ b/internal/sandbox/firecracker_linux.go @@ -63,6 +63,11 @@ const ( // are attached in spec order after the rootfs (machine/firecracker puts // rootfs first, then Spec.Disks), so vda=rootfs, vdb=guestcfg, vdc=this. fcWorktreeDevice = "/dev/vdc" + // fcSwapDevice is the swap disk, attached after the worktree — so + // vda=rootfs, vdb=guestcfg, vdc=worktree, vdd=this. Firecracker's + // virtio-blk advertises no discard, so freed swap pages stay allocated in + // the backing file (see sandbox.DefaultSwapSizeMiB). + fcSwapDevice = "/dev/vdd" // worktreeDiskSlack is the free space added to a worktree disk beyond the // tree itself, for whatever the agent builds in there. Sparse, so unused // capacity costs nothing on the host. @@ -151,6 +156,9 @@ func (f *FirecrackerProvider) Create(sb *config.Sandbox) error { if err := guestcfg.WriteDisk(f.manifest(sb), filepath.Join(vmDir, "guestcfg.img")); err != nil { return fmt.Errorf("guest config disk: %w", err) } + if _, err := EnsureSwapDisk(vmDir, SwapDiskMiB(sb)); err != nil { + return fmt.Errorf("swap disk: %w", err) + } sb.GuestIP = guestIP return nil } @@ -171,7 +179,11 @@ func (f *FirecrackerProvider) hasRestorableState(sb *config.Sandbox, vmDir, root if !machine.SuspendStateExists(filepath.Join(vmDir, "suspend")) { return false } - for _, disk := range []string{rootfs, f.worktreeDiskPath(sb), filepath.Join(vmDir, "guestcfg.img")} { + disks := []string{rootfs, f.worktreeDiskPath(sb), filepath.Join(vmDir, "guestcfg.img")} + if SwapDiskMiB(sb) > 0 { + disks = append(disks, SwapDiskPath(vmDir)) + } + for _, disk := range disks { if _, err := os.Stat(disk); err != nil { return false } @@ -195,6 +207,7 @@ func (f *FirecrackerProvider) manifest(sb *config.Sandbox) guestcfg.Manifest { DNS: []string{gvproxyGateway}, MTU: gvproxyMTU, }, + Swap: fcSwap(sb), Mounts: []guestcfg.Mount{{ Path: f.guestWorktreePath(sb), Block: fcWorktreeDevice, @@ -204,6 +217,16 @@ func (f *FirecrackerProvider) manifest(sb *config.Sandbox) guestcfg.Manifest { } } +// fcSwap is the manifest's swap entry, or nil when the sandbox opted out. +// Keyed off the same SwapDiskMiB as buildSpec's disk list so the manifest +// never names a device firecracker wasn't asked to attach. +func fcSwap(sb *config.Sandbox) *guestcfg.Swap { + if SwapDiskMiB(sb) == 0 { + return nil + } + return &guestcfg.Swap{Device: fcSwapDevice, Swappiness: GuestSwappiness} +} + // guestWorktreePath is where the worktree disk is mounted in the guest — // the same /workspace/ layout the bake produced, so nothing downstream // (sessions, runners, cwd inference) sees a change. @@ -456,6 +479,13 @@ func (f *FirecrackerProvider) DaemonSpec(sb *config.Sandbox, allow *netfilter.Al cleanup() return machine.Spec{}, nil, fmt.Errorf("kernel: %w", err) } + // Ensured on every boot, not just at create, so an edited + // `vm ( swap )` takes effect on the next start — buildSpec attaches + // whatever SwapDiskMiB says, and the device has to exist by then. + if _, err := EnsureSwapDisk(f.vmDir(sb), SwapDiskMiB(sb)); err != nil { + cleanup() + return machine.Spec{}, nil, fmt.Errorf("swap disk: %w", err) + } spec := f.buildSpec(sb, kernelPath) if ns != nil { // Handed over here and nowhere earlier: machine.UserMode documents that @@ -508,6 +538,13 @@ func (f *FirecrackerProvider) buildSpec(sb *config.Sandbox, kernelPath string) m if sb.MemoryMaxMiB > memMaxMiB { memMaxMiB = sb.MemoryMaxMiB } + disks := []machine.Disk{ + {Path: filepath.Join(f.vmDir(sb), "guestcfg.img"), ReadOnly: true}, + {Path: f.worktreeDiskPath(sb)}, + } + if SwapDiskMiB(sb) > 0 { + disks = append(disks, machine.Disk{Path: SwapDiskPath(f.vmDir(sb))}) + } return machine.Spec{ ID: sb.Name, VCPU: vcpu, @@ -523,12 +560,9 @@ func (f *FirecrackerProvider) buildSpec(sb *config.Sandbox, kernelPath string) m }, RootFS: machine.RawDisk{Path: filepath.Join(f.vmDir(sb), "rootfs.raw")}, // Order is the guest's device order: vdb=guestcfg, vdc=worktree - // (fcWorktreeDevice). Appending anything here shifts later devices, - // so keep new disks after these two. - Disks: []machine.Disk{ - {Path: filepath.Join(f.vmDir(sb), "guestcfg.img"), ReadOnly: true}, - {Path: f.worktreeDiskPath(sb)}, - }, + // (fcWorktreeDevice), vdd=swap (fcSwapDevice). Appending anything here + // shifts later devices, so keep new disks after these. + Disks: disks, VSockCID: fcVSockCID, Serial: machine.Serial{LogPath: filepath.Join(f.vmDir(sb), "console.log")}, } diff --git a/internal/sandbox/firecracker_linux_test.go b/internal/sandbox/firecracker_linux_test.go index ada44bc..eeee8fb 100644 --- a/internal/sandbox/firecracker_linux_test.go +++ b/internal/sandbox/firecracker_linux_test.go @@ -39,12 +39,39 @@ func TestHasRestorableState(t *testing.T) { "a state saved before the worktree had its own disk must not be trusted") touch(f.worktreeDiskPath(sb)) + require.False(t, f.hasRestorableState(sb, vmDir, rootfs), + "a state saved before sandboxes had a swap disk must not be trusted either") + + touch(SwapDiskPath(vmDir)) require.True(t, f.hasRestorableState(sb, vmDir, rootfs), "the full disk set is present") require.NoError(t, os.Remove(filepath.Join(vmDir, "guestcfg.img"))) require.False(t, f.hasRestorableState(sb, vmDir, rootfs), "guestcfg.img is attached too") } +// A sandbox with swap disabled attaches no swap disk, so its absence must +// not hold the restore back — the check has to track buildSpec's disk list, +// not a fixed set. +func TestHasRestorableStateSwapOff(t *testing.T) { + root := t.TempDir() + f := &FirecrackerProvider{store: config.NewStoreAt(root)} + sb := &config.Sandbox{Name: "proj", SwapMiB: -1} + vmDir := f.vmDir(sb) + rootfs := filepath.Join(vmDir, "rootfs.raw") + + for _, path := range []string{ + filepath.Join(vmDir, "suspend", "snapshot.state"), + rootfs, + filepath.Join(vmDir, "guestcfg.img"), + f.worktreeDiskPath(sb), + } { + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte("x"), 0o644)) + } + require.True(t, f.hasRestorableState(sb, vmDir, rootfs), + "no swap disk is expected when swap is off") +} + func TestWorktreeDiskSize(t *testing.T) { dir := t.TempDir() require.NoError(t, os.MkdirAll(filepath.Join(dir, "sub"), 0o755)) diff --git a/internal/sandbox/mcp.go b/internal/sandbox/mcp.go new file mode 100644 index 0000000..aa37c68 --- /dev/null +++ b/internal/sandbox/mcp.go @@ -0,0 +1,153 @@ +package sandbox + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/clawkwork/clawk/internal/config" + "github.com/google/renameio/v2" +) + +// GuestMCPConfigPath is where clawk renders the sandbox's declared MCP +// servers inside the guest, and the path passed to the runner's +// --mcp-config flag (see cli.mcpConfigArgs). +// +// It deliberately sits inside ~/.claude/ rather than at either of the +// places the runner would find on its own: +// +// - ~/.claude.json (user scope) is clawk's onboarding marker and carries +// a documented concurrent-write race (anthropics/claude-code#28847). +// - .mcp.json in a project root would land inside a git worktree mounted +// from the host, i.e. clawk would be writing config into the user's +// repo. +// +// Living under ~/.claude/ also means it arrives through the per-sandbox +// PersistentAgentShares mount with the rest of the seeded state, and the +// sessions history repo ignores it by construction — that gitignore denies +// everything (`/*`) and only re-admits transcripts and memory. +const GuestMCPConfigPath = GuestHome + "/.claude/mcp/clawk.json" + +// mcpConfig is the on-disk shape the runner reads. Field names are the +// runner's, not clawk's. +type mcpConfig struct { + MCPServers map[string]mcpServerConfig `json:"mcpServers"` +} + +type mcpServerConfig struct { + // Type is emitted for remote transports only. A stdio server is + // identified by having a command, and omitting the key there matches + // the shape the runner documents for `claude mcp add`. + Type string `json:"type,omitempty"` + URL string `json:"url,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + Command string `json:"command,omitempty"` + Args []string `json:"args,omitempty"` + Env map[string]string `json:"env,omitempty"` +} + +// RenderMCPConfig builds the guest MCP config for a sandbox's declared +// servers. Returns ok=false when there is nothing to declare. +// +// `${VAR}` references in headers and env values are passed through +// verbatim: the runner expands them against its own process environment +// when it connects, so no credential value is ever written here. That +// matters because this file is created on the HOST, inside the sandbox +// state dir — the same reason config.Sandbox.RequiredEnv stores names +// rather than values. The values reach the runner's environment through +// the vsock handshake instead (ResolveEnv → cli.buildVSockEnv). +// +// A stdio server's env is rendered as NAME=${NAME} for the same reason: +// clawk names the variable, the runner supplies the value. +func RenderMCPConfig(servers []config.MCPServer) ([]byte, bool, error) { + if len(servers) == 0 { + return nil, false, nil + } + cfg := mcpConfig{MCPServers: make(map[string]mcpServerConfig, len(servers))} + for _, s := range servers { + entry := mcpServerConfig{} + switch s.Transport { + case config.MCPTransportStdio: + if len(s.Command) == 0 { + return nil, false, fmt.Errorf("mcp server %q: stdio transport with no command", s.Name) + } + entry.Command = s.Command[0] + entry.Args = s.Command[1:] + if len(s.Env) > 0 { + entry.Env = make(map[string]string, len(s.Env)) + for _, name := range s.Env { + entry.Env[name] = "${" + name + "}" + } + } + case config.MCPTransportHTTP, config.MCPTransportSSE: + if s.URL == "" { + return nil, false, fmt.Errorf("mcp server %q: %s transport with no URL", s.Name, s.Transport) + } + entry.Type = s.Transport + entry.URL = s.URL + if len(s.Headers) > 0 { + entry.Headers = make(map[string]string, len(s.Headers)) + for _, h := range s.Headers { + name, val, ok := strings.Cut(h, ":") + if !ok { + return nil, false, fmt.Errorf("mcp server %q: malformed header %q", s.Name, h) + } + entry.Headers[strings.TrimSpace(name)] = strings.TrimSpace(val) + } + } + default: + return nil, false, fmt.Errorf("mcp server %q: unknown transport %q", s.Name, s.Transport) + } + cfg.MCPServers[s.Name] = entry + } + // MarshalIndent sorts map keys, so the bytes are stable for a given + // server list — this file is rewritten on every boot into a virtio-fs + // mount, and a stable rendering keeps that a no-op write. + content, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return nil, false, err + } + return append(content, '\n'), true, nil +} + +// SeedClaudeMCP writes (or clears) the guest MCP config in the per-sandbox +// state dir that PersistentAgentShares mounts at ~/.claude/. Run by the +// provider during sandbox preparation, alongside SeedClaudeStateDir and +// before the VM boots — so the servers are in place for the runner's first +// connection attempt, with no post-boot step and no `on create` hook. +// +// Rewritten on every call, like settings.json: a clawk.mod edit propagates +// on the next `up`. An empty server list removes the file rather than +// leaving a stale one behind, so deleting an `mcp ( … )` entry actually +// retires the server. +// +// Note this only arranges the config. Whether a server then authenticates +// is up to the credential its headers/env reference — see +// config.MCPServer. +func SeedClaudeMCP(stateRoot string, servers []config.MCPServer) error { + if stateRoot == "" { + return nil + } + path := filepath.Join(stateRoot, "claude", "mcp", "clawk.json") + content, ok, err := RenderMCPConfig(servers) + if err != nil { + return err + } + if !ok { + if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("clearing mcp config: %w", err) + } + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return fmt.Errorf("creating mcp config dir: %w", err) + } + if err := renameio.WriteFile(path, content, 0o644); err != nil { + return fmt.Errorf("seeding mcp config: %w", err) + } + return nil +} diff --git a/internal/sandbox/mcp_test.go b/internal/sandbox/mcp_test.go new file mode 100644 index 0000000..8149730 --- /dev/null +++ b/internal/sandbox/mcp_test.go @@ -0,0 +1,152 @@ +package sandbox + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/stretchr/testify/require" +) + +func TestRenderMCPConfigRemote(t *testing.T) { + content, ok, err := RenderMCPConfig([]config.MCPServer{{ + Name: "linear", + Transport: config.MCPTransportHTTP, + URL: "https://mcp.linear.app/mcp", + Headers: []string{"Authorization: Bearer ${LINEAR_TOKEN}", "X-Env: prod"}, + }}) + require.NoError(t, err) + require.True(t, ok) + + var got struct { + MCPServers map[string]struct { + Type string `json:"type"` + URL string `json:"url"` + Headers map[string]string `json:"headers"` + } `json:"mcpServers"` + } + require.NoError(t, json.Unmarshal(content, &got)) + srv := got.MCPServers["linear"] + require.Equal(t, "http", srv.Type) + require.Equal(t, "https://mcp.linear.app/mcp", srv.URL) + require.Equal(t, "Bearer ${LINEAR_TOKEN}", srv.Headers["Authorization"], + "the ${VAR} reference must survive verbatim — the runner expands it in-guest") + require.Equal(t, "prod", srv.Headers["X-Env"]) +} + +func TestRenderMCPConfigStdio(t *testing.T) { + content, ok, err := RenderMCPConfig([]config.MCPServer{{ + Name: "github", + Transport: config.MCPTransportStdio, + Command: []string{"npx", "-y", "@modelcontextprotocol/server-github"}, + Env: []string{"GITHUB_TOKEN"}, + }}) + require.NoError(t, err) + require.True(t, ok) + + var got struct { + MCPServers map[string]struct { + Type string `json:"type"` + Command string `json:"command"` + Args []string `json:"args"` + Env map[string]string `json:"env"` + } `json:"mcpServers"` + } + require.NoError(t, json.Unmarshal(content, &got)) + srv := got.MCPServers["github"] + require.Equal(t, "npx", srv.Command) + require.Equal(t, []string{"-y", "@modelcontextprotocol/server-github"}, srv.Args) + require.Equal(t, map[string]string{"GITHUB_TOKEN": "${GITHUB_TOKEN}"}, srv.Env, + "clawk names the variable; the runner's environment supplies the value") + require.Empty(t, srv.Type, "stdio is implied by having a command") +} + +// TestRenderMCPConfigCarriesNoSecrets is the invariant that lets this file +// live on host disk inside the sandbox state dir: it holds references, never +// values, exactly like config.Sandbox.RequiredEnv. +func TestRenderMCPConfigCarriesNoSecrets(t *testing.T) { + t.Setenv("LINEAR_TOKEN", "super-secret-value") + content, _, err := RenderMCPConfig([]config.MCPServer{{ + Name: "linear", + Transport: config.MCPTransportHTTP, + URL: "https://mcp.linear.app/mcp", + Headers: []string{"Authorization: Bearer ${LINEAR_TOKEN}"}, + }}) + require.NoError(t, err) + require.NotContains(t, string(content), "super-secret-value") + require.Contains(t, string(content), "${LINEAR_TOKEN}") +} + +// TestRenderMCPConfigStable: the file is rewritten on every boot into a +// virtio-fs mount, so identical input must produce identical bytes. +func TestRenderMCPConfigStable(t *testing.T) { + servers := []config.MCPServer{ + {Name: "b", Transport: config.MCPTransportHTTP, URL: "https://b/mcp"}, + {Name: "a", Transport: config.MCPTransportHTTP, URL: "https://a/mcp"}, + } + first, _, err := RenderMCPConfig(servers) + require.NoError(t, err) + second, _, err := RenderMCPConfig(servers) + require.NoError(t, err) + require.Equal(t, string(first), string(second)) + require.Less(t, strings.Index(string(first), `"a"`), strings.Index(string(first), `"b"`), + "keys are sorted, so declaration order can't churn the file") +} + +func TestRenderMCPConfigRejectsIncomplete(t *testing.T) { + _, _, err := RenderMCPConfig([]config.MCPServer{{Name: "x", Transport: config.MCPTransportStdio}}) + require.ErrorContains(t, err, "no command") + + _, _, err = RenderMCPConfig([]config.MCPServer{{Name: "x", Transport: config.MCPTransportHTTP}}) + require.ErrorContains(t, err, "no URL") + + _, _, err = RenderMCPConfig([]config.MCPServer{{Name: "x", Transport: "carrier-pigeon"}}) + require.ErrorContains(t, err, "unknown transport") +} + +func TestRenderMCPConfigEmpty(t *testing.T) { + content, ok, err := RenderMCPConfig(nil) + require.NoError(t, err) + require.False(t, ok) + require.Nil(t, content) +} + +// TestSeedClaudeMCP covers the full lifecycle at the path the guest sees +// through the ~/.claude mount: written before boot, refreshed on the next +// one, and removed when the declaration goes away. +func TestSeedClaudeMCP(t *testing.T) { + stateRoot := t.TempDir() + path := filepath.Join(stateRoot, "claude", "mcp", "clawk.json") + + require.NoError(t, SeedClaudeMCP(stateRoot, []config.MCPServer{{ + Name: "linear", Transport: config.MCPTransportHTTP, URL: "https://mcp.linear.app/mcp", + }})) + first, err := os.ReadFile(path) + require.NoError(t, err, "seeded before the VM boots, so the dir is created on demand") + require.Contains(t, string(first), "mcp.linear.app") + + // A clawk.mod edit propagates on the next up. + require.NoError(t, SeedClaudeMCP(stateRoot, []config.MCPServer{{ + Name: "notion", Transport: config.MCPTransportHTTP, URL: "https://mcp.notion.com/mcp", + }})) + second, err := os.ReadFile(path) + require.NoError(t, err) + require.Contains(t, string(second), "mcp.notion.com") + require.NotContains(t, string(second), "mcp.linear.app", "rewritten, not merged") + + // Dropping the block retires the servers rather than leaving a stale file. + require.NoError(t, SeedClaudeMCP(stateRoot, nil)) + _, err = os.Stat(path) + require.True(t, os.IsNotExist(err), "stale config must not survive an emptied mcp block") + + // Clearing an already-absent file is not an error (every boot calls this). + require.NoError(t, SeedClaudeMCP(stateRoot, nil)) +} + +func TestSeedClaudeMCPNoStateRoot(t *testing.T) { + require.NoError(t, SeedClaudeMCP("", []config.MCPServer{{Name: "x"}}), + "an opted-out state root is a no-op, matching the other seed functions") +} diff --git a/internal/sandbox/oci_sandbox.go b/internal/sandbox/oci_sandbox.go index 1617103..ab5ec9f 100644 --- a/internal/sandbox/oci_sandbox.go +++ b/internal/sandbox/oci_sandbox.go @@ -106,6 +106,16 @@ func OCIGuestManifest(sb *config.Sandbox, stateDir, cacheDir, rootDir string) (g }, } + // Swap rides on its own virtio-blk disk (EnsureSwapDisk), attached right + // after the config disk. Both sides key off SwapDiskMiB, so a sandbox + // with swap off gets neither the device nor the manifest entry. + if SwapDiskMiB(sb) > 0 { + m.Swap = &guestcfg.Swap{ + Device: OCISwapDevice, + Swappiness: GuestSwappiness, + } + } + // Consolidated worktree mount: every managed (non-in-place) worktree for // this sandbox lives under one host parent (store.WorktreeDir), which // collectSandboxShares exposes as a single virtio-fs device. Mount that @@ -150,10 +160,11 @@ func OCIGuestManifest(sb *config.Sandbox, stateDir, cacheDir, rootDir string) (g }) } - // Host shares, in parent-before-child order: the whole ~/.claude - // state mount must precede the skills/agents/commands sub-mounts or - // the latter end up shadowed. - shares := append([]HostShare{}, PersistentClaudeShares(stateDir)...) + // Host shares, in parent-before-child order: the per-runner state + // mounts (~/.claude, ~/.codex, ~/.pi) must precede the + // skills/agents/commands sub-mounts that land inside them, or the + // latter end up shadowed. + shares := append([]HostShare{}, PersistentAgentShares(stateDir)...) shares = append(shares, DefaultHostShares()...) shares = append(shares, ToolchainCacheShares(cacheDir)...) shares = append(shares, UserHostShares(sb.Shares)...) diff --git a/internal/sandbox/oci_sandbox_test.go b/internal/sandbox/oci_sandbox_test.go index 35754e2..85437b5 100644 --- a/internal/sandbox/oci_sandbox_test.go +++ b/internal/sandbox/oci_sandbox_test.go @@ -81,7 +81,7 @@ func TestOCIGuestManifest(t *testing.T) { t.Errorf("workspace parent (idx %d) must be mounted before in-place sub-mount (idx %d)", wsIdx, hereIdx) } if mt, ok := mounts["claude_home"]; !ok || mt.Path != GuestHome+"/.claude" { - // PersistentClaudeShares' tag; if its tag ever changes this test + // PersistentAgentShares' tag; if its tag ever changes this test // must change together with collectSandboxShares. t.Errorf("claude state mount missing/wrong: %+v", mounts) } diff --git a/internal/sandbox/shares.go b/internal/sandbox/shares.go index 3129b37..dd94029 100644 --- a/internal/sandbox/shares.go +++ b/internal/sandbox/shares.go @@ -6,6 +6,7 @@ import ( _ "embed" "encoding/hex" "encoding/json" + "errors" "fmt" "os" "os/exec" @@ -60,61 +61,124 @@ type HostFile struct { Owner string // "user:group" or "" for root } -// PersistentClaudeShares returns the per-sandbox host share that -// carries Claude Code's entire ~/.claude/ across destroy/recreate -// cycles. Call sites already resolved a Store so they pass the state -// root directly rather than re-deriving the path. -// -// The host directory is created idempotently so virtiofs never -// encounters a missing source. The guest-side mount point is -// ~/.claude/ — per-sandbox storage that does NOT suffer the shared- -// .claude.json races documented below; two sandboxes can't touch -// the same path because each sandbox name maps to a distinct host -// dir. -// -// We mount the whole dir rather than a curated subdir list because: -// -// - The "ephemeral" subdirs (cache/, paste-cache/, shell-snapshots/, -// telemetry/) measure in hundreds of KB total — exclusion isn't -// worth the bookkeeping. -// - settings.json and CLAUDE.md, formerly snapshot HostFiles, are -// now seeded via SeedClaudeStateDir straight into the synced -// dir before the mount happens. That removes the snapshot-file -// vs share-mount layering issue. -// - .credentials.json (non-token path) lives at its canonical place -// inside the synced dir, so claude's atomic write-rename refresh -// persists naturally — no more copy-in/copy-out trick via -// auth/credentials.json. -// -// Mount ordering: this share must come BEFORE DefaultHostShares in -// the assembled share list. The agents/commands sub-mounts land on -// top of ~/.claude/ at boot, and Linux would shadow them under a -// later parent mount. -// -// Cross-sandbox races (the ones DefaultHostShares is avoiding) don't -// apply here — each sandbox name maps to its own state dir, so two -// sandboxes never write to the same path. +// AgentStateDir maps one coding-agent runner's home directory onto the +// per-sandbox host storage that backs it. See AgentStateDirs. +type AgentStateDir struct { + // Agent is the runner name in the CLI's agent registry, purely for + // documentation and error messages. + Agent string + // Sub is the subdirectory under the sandbox's state root. + Sub string + // Tag is the virtio-fs tag; unique across every share on one VM. + Tag string + // GuestPath is where the runner looks for its state inside the VM. + GuestPath string +} + +// AgentStateDirs lists the runner home directories clawk persists per +// sandbox. Every entry is one virtio-fs device, so the list is the +// contract both the device side (cli.collectSandboxShares) and the guest +// manifest (OCIGuestManifest) iterate. +// +// Why each runner needs one: the vz rootfs is re-cloned from the image on +// EVERY boot (see suspendBootRootFS — a per-boot-disposable disk is the +// design), so anything a runner writes under $HOME on the rootfs is gone +// after a plain `clawk down && clawk up`, never mind a destroy. Claude +// survived that because its ~/.claude was mounted from the host; codex's +// sessions, history, and login did not, so every restart looked like a +// fresh install. Persisting each runner's home dir is what makes the +// documented "conversation memory persists across destroys" promise true +// for more than one runner. +// +// Guest paths, all whole-home mounts rather than curated subdir lists — +// the ephemeral parts (caches, logs) measure in hundreds of KB and +// excluding them isn't worth the bookkeeping: +// +// claude ~/.claude projects/, memory/, settings.json, .credentials.json +// codex ~/.codex sessions/, history.jsonl, auth.json, config.toml +// pi ~/.pi agent/{sessions,settings.json,auth.json,trust.json} +// +// opencode is the one runner that needs two, because it follows the XDG +// split rather than keeping a single home (verified with `opencode debug +// paths`, which is the authority for its layout): +// +// opencode ~/.local/share/opencode auth.json, mcp-auth.json, opencode.db, repos/ +// opencode ~/.config/opencode opencode.jsonc +// +// Its other two XDG dirs are deliberately left on the disposable rootfs. +// ~/.local/state/opencode holds only locks/, and a lock that outlives the +// VM it was taken in is worse than no lock at all — a hard stop would +// leave one behind for the next boot to trip over. ~/.cache/opencode is a +// cache by name and contract; the only real cost is re-downloading +// cache/bin per sandbox, which is the same trade the toolchain caches make +// (see ToolchainCachesEnabled). +// +// Cost note: every entry here is a PCIe device on every sandbox, against +// the ceiling documented in machine/vz (32, with a field-confirmed failure +// at 34). Five entries plus the workspace, the per-repo aliases, and the +// default capability shares still leaves room for a normal multi-repo +// sandbox, but this list is not free to extend — a sixth runner wanting +// three dirs is where the consolidation trick (one mount plus a +// dir-override env var, e.g. OPENCODE_CONFIG_DIR) starts paying for its +// complexity. +var AgentStateDirs = []AgentStateDir{ + {Agent: "claude", Sub: "claude", Tag: "claude_home", GuestPath: GuestHome + "/.claude"}, + {Agent: "codex", Sub: "codex", Tag: "codex_home", GuestPath: GuestHome + "/.codex"}, + {Agent: "pi", Sub: "pi", Tag: "pi_home", GuestPath: GuestHome + "/.pi"}, + {Agent: "opencode", Sub: "opencode-data", Tag: "opencode_data", + GuestPath: GuestHome + "/.local/share/opencode"}, + {Agent: "opencode", Sub: "opencode-config", Tag: "opencode_config", + GuestPath: GuestHome + "/.config/opencode"}, +} + +// PersistentAgentShares returns the per-sandbox host shares that carry +// each coding agent's home directory across down/up and destroy/recreate +// cycles. Call sites already resolved a Store so they pass the state root +// directly rather than re-deriving the path. +// +// Host directories are created idempotently so virtiofs never encounters +// a missing source; a directory that can't be created drops its share +// rather than failing sandbox creation, matching ToolchainCacheShares. +// The guest mount points are per-sandbox storage, so they do NOT suffer +// the shared-state races documented on DefaultHostShares — two sandboxes +// can't touch the same path because each sandbox name maps to a distinct +// host dir. +// +// For claude specifically, the whole-dir mount is also what lets +// settings.json, CLAUDE.md and .credentials.json be seeded straight into +// the synced dir by SeedClaudeStateDir before the mount happens (no +// snapshot-file vs share-mount layering issue), and lets claude's atomic +// write-rename credential refresh persist naturally. +// +// Mount ordering: these shares must come BEFORE DefaultHostShares in the +// assembled share list. Its sub-mounts land INSIDE these homes +// (~/.claude/agents, ~/.claude/commands, ~/.codex/skills), and Linux +// would shadow them under a later parent mount. // // Opt out by passing an empty stateRoot. -func PersistentClaudeShares(stateRoot string) []HostShare { +func PersistentAgentShares(stateRoot string) []HostShare { if stateRoot == "" { return nil } - hostPath := filepath.Join(stateRoot, "claude") - if err := os.MkdirAll(hostPath, 0o755); err != nil { - return nil + out := make([]HostShare, 0, len(AgentStateDirs)) + for _, d := range AgentStateDirs { + hostPath := filepath.Join(stateRoot, d.Sub) + if err := os.MkdirAll(hostPath, 0o755); err != nil { + continue + } + out = append(out, HostShare{ + HostPath: hostPath, + Tag: d.Tag, + GuestPath: d.GuestPath, + ReadOnly: false, + }) } - return []HostShare{{ - HostPath: hostPath, - Tag: "claude_home", - GuestPath: GuestHome + "/.claude", - ReadOnly: false, - }} + return out } // SeedClaudeStateDir writes the host-snapshot files (settings.json, // CLAUDE.md, .credentials.json) directly into the per-sandbox state dir that -// PersistentClaudeShares mounts at ~/.claude/. Run by the provider +// PersistentAgentShares mounts at ~/.claude/. Run by the provider // during sandbox preparation, BEFORE the VM boots — so by the time // virtio-fs mounts the dir, the files are already in place. // @@ -188,7 +252,7 @@ func SeedClaudeStateDir(stateRoot, clawkRootDir string) error { // claudeCredentialsPath is where claude reads and refreshes its OAuth // credentials, in host terms: inside the per-sandbox state dir that -// PersistentClaudeShares mounts at ~/.claude/. +// PersistentAgentShares mounts at ~/.claude/. func claudeCredentialsPath(stateRoot string) string { return filepath.Join(stateRoot, "claude", ".credentials.json") } @@ -274,7 +338,7 @@ const ToolchainCachesEnabled = false // transparent. Host dirs are created on demand (virtiofs refuses // missing source paths); MkdirAll failures silently drop the offending // share rather than failing sandbox creation, matching the behavior -// of PersistentClaudeShares. +// of PersistentAgentShares. // // Caches included (well-documented concurrent safety, real payoff): // @@ -357,9 +421,15 @@ func ToolchainCacheShares(cacheDir string) []HostShare { // // Sharing Claude agents/commands and Codex skills is safe: user-authored // capability dirs with low write contention. Sharing .claude.json, -// credentials, projects/, file-history/, or the whole ~/.codex state dir -// would risk concurrent writers. Each sandbox authenticates independently — -// one-time cost per sandbox in exchange for no cross-session corruption. +// credentials, projects/, file-history/, or the rest of ~/.codex would risk +// concurrent writers. Each sandbox authenticates independently — one-time +// cost per sandbox in exchange for no cross-session corruption. +// +// "Not shared with the host" is not the same as "not persisted": the +// runners' state dirs still survive down/up and destroy through +// PersistentAgentShares, which gives each sandbox its OWN host directory. +// These sub-mounts land inside those homes, so PersistentAgentShares must +// be assembled first or the parent mount shadows them. // // ~/.claude/skills is deliberately NOT shared. A skill like gstack carries // a large node_modules tree, and virtio-fs caches an inode (and host fd) @@ -444,7 +514,7 @@ func userShareTag(guestPath string) string { // Files that DO live inside ~/.claude/ — settings.json, CLAUDE.md, // .credentials.json — are pre-seeded into the per-sandbox state dir // by SeedClaudeStateDir before the VM boots; they appear at the -// canonical paths through the PersistentClaudeShares mount, no +// canonical paths through the PersistentAgentShares mount, no // snapshot-file involvement needed. // // clawkRootDir resolves the optional long-lived OAuth token @@ -635,52 +705,126 @@ func shellEscapeDoubleQuoted(v string) string { return shellEscapeDoubleQuotedReplacer.Replace(v) } -// EnvFile synthesizes an /etc/profile.d script that exports every -// sandbox-required env var. Entries come from sb.RequiredEnv (declared in -// clawk.mod, in canonical envspec form); values are resolved against the -// host's process env at this call. +// DeclaredEnvNames is the set of guest variable names a sandbox's +// `env ( … )` block declares, independent of whether any of them can be +// resolved right now. +// +// Separate from ResolveEnv because the two answer different questions, and +// conflating them is a security bug: "which names did the user speak for" +// must not depend on the host shell. cli.buildVSockEnv suppresses clawk's +// own injected variables for every declared name, and deriving that set +// from ResolveEnv's output meant an entry that failed to resolve (a +// ${HOST:?msg} whose host var left the shell) silently handed the name back +// to clawk — re-injecting the Anthropic OAuth token into a sandbox whose +// clawk.mod had explicitly disowned it. See config.MCPServer for why that +// token must not reach a third-party endpoint. +// +// Entries that don't parse are skipped: they name nothing usable, and +// ResolveEnv reports them. +func DeclaredEnvNames(sb *config.Sandbox) map[string]bool { + if sb == nil || len(sb.RequiredEnv) == 0 { + return nil + } + names := make(map[string]bool, len(sb.RequiredEnv)) + for _, entry := range sb.RequiredEnv { + spec, err := envspec.Parse(entry) + if err != nil { + continue + } + names[spec.Name] = true + } + return names +} + +// ResolveEnv resolves a sandbox's declared `env ( … )` entries against the +// host process environment, returning them as canonical NAME=value strings +// in declaration order. // // Resolution follows the envspec grammar: a bare passthrough / ${HOST} -// alias exports the host value (empty + a warning when unset); ${HOST:-x} -// / ${HOST-x} fall back to a default; a bare/quoted literal is exported -// verbatim; and ${HOST:?msg} / ${HOST?msg} make a missing variable a hard -// error (returned here, failing sandbox creation with a clear message). -// -// Written into /etc/profile.d/99-clawk-env.sh so every login shell -// (ssh, `claude ...`, interactive bash, phase setup scripts) picks up -// the values without any per-tool configuration. -// -// Returns ok=false if the sandbox has no required env — saves the -// caller from having to filter empty HostFiles. -func EnvFile(sb *config.Sandbox) (HostFile, bool, error) { +// alias takes the host value (empty + a warning when unset); ${HOST:-x} +// / ${HOST-x} fall back to a default; a bare/quoted literal is used +// verbatim; and ${HOST:?msg} / ${HOST?msg} make a missing variable an +// error. +// +// It is the single resolution step behind both delivery paths, so a hard +// failure or an unset-variable warning reads the same either way: +// +// - EnvFile below, which renders /etc/profile.d/99-clawk-env.sh for +// every login shell in the guest. +// - the pty agent's vsock handshake (internal/cli.buildVSockEnv), which +// spawns the runner directly — no login shell, no /etc/profile — and +// so has to carry the values itself. +// +// Entries that fail to resolve are skipped but still reported, so the +// returned slice always holds everything that DID resolve: strict callers +// (sandbox create) treat a non-nil error as fatal, while best-effort +// callers (agent attach, which must not become unusable just because one +// variable left the host shell) can warn and carry on with the rest. +// +// Values are never persisted — they're read from the host env at call +// time, which is why both callers re-resolve on every use. +func ResolveEnv(sb *config.Sandbox) ([]string, error) { if len(sb.RequiredEnv) == 0 { - return HostFile{}, false, nil + return nil, nil } - var b bytes.Buffer - b.WriteString("# Generated by clawk — host env vars requested in clawk.mod\n") + out := make([]string, 0, len(sb.RequiredEnv)) + var errs []error for _, entry := range sb.RequiredEnv { spec, err := envspec.Parse(entry) if err != nil { // The template parser already validated every entry, so this // only fires on a hand-edited sandbox record — still worth a // clear error rather than a malformed export line. - return HostFile{}, false, fmt.Errorf( - "sandbox %q: invalid env entry %q: %w", sb.Name, entry, err) + errs = append(errs, fmt.Errorf( + "sandbox %q: invalid env entry %q: %w", sb.Name, entry, err)) + continue } val, warnUnset, err := spec.Resolve(os.LookupEnv) if err != nil { - return HostFile{}, false, fmt.Errorf("sandbox %q: %w", sb.Name, err) + errs = append(errs, fmt.Errorf("sandbox %q: %w", sb.Name, err)) + continue } if warnUnset { fmt.Fprintf(os.Stderr, "warning: %s required by clawk.mod is unset on host; "+ "exporting empty in sandbox %q\n", spec.Host, sb.Name) } + out = append(out, spec.Name+"="+val) + } + return out, errors.Join(errs...) +} + +// EnvFile synthesizes an /etc/profile.d script that exports every +// sandbox-required env var. Entries come from sb.RequiredEnv (declared in +// clawk.mod, in canonical envspec form); values are resolved by ResolveEnv +// against the host's process env at this call, and a resolution failure +// (e.g. an unset ${HOST:?msg}) is fatal here — it fails sandbox creation +// with a clear message. +// +// Written into /etc/profile.d/99-clawk-env.sh so every login shell +// (ssh, interactive bash, phase setup scripts, the `bash -lc` agent +// fallback) picks up the values without any per-tool configuration. The +// primary agent path does NOT go through a login shell — see ResolveEnv. +// +// Returns ok=false if the sandbox has no required env — saves the +// caller from having to filter empty HostFiles. +func EnvFile(sb *config.Sandbox) (HostFile, bool, error) { + entries, err := ResolveEnv(sb) + if err != nil { + return HostFile{}, false, err + } + if len(entries) == 0 { + return HostFile{}, false, nil + } + var b bytes.Buffer + b.WriteString("# Generated by clawk — host env vars requested in clawk.mod\n") + for _, e := range entries { + name, val, _ := strings.Cut(e, "=") // Double-quoted with $ and " escaped — covers secrets that // contain dollar signs or quotes. Single quotes aren't safe // because bash single-quote strings can't contain single // quotes, and API keys sometimes do. - fmt.Fprintf(&b, "export %s=\"%s\"\n", spec.Name, shellEscapeDoubleQuoted(val)) + fmt.Fprintf(&b, "export %s=\"%s\"\n", name, shellEscapeDoubleQuoted(val)) } return HostFile{ Content: b.Bytes(), diff --git a/internal/sandbox/shares_test.go b/internal/sandbox/shares_test.go index c53a8b1..bae0c39 100644 --- a/internal/sandbox/shares_test.go +++ b/internal/sandbox/shares_test.go @@ -101,7 +101,7 @@ func TestToolchainCacheSharesUniqueTags(t *testing.T) { stateRoot := t.TempDir() all := append([]HostShare{}, DefaultHostShares()...) - all = append(all, PersistentClaudeShares(stateRoot)...) + all = append(all, PersistentAgentShares(stateRoot)...) all = append(all, ToolchainCacheShares(cacheDir)...) seen := make(map[string]string, len(all)) @@ -208,6 +208,62 @@ func TestEnvFileComposeModel(t *testing.T) { } } +// TestResolveEnvSharedByBothDeliveryPaths pins the contract EnvFile and the +// vsock handshake share: canonical NAME=value strings in declaration order, +// values read from the host env at call time (never persisted). +func TestResolveEnvSharedByBothDeliveryPaths(t *testing.T) { + t.Setenv("HOST_GH", "ghp_xyz") + os.Unsetenv("HOST_MISSING") + + got, err := ResolveEnv(&config.Sandbox{ + Name: "sb", + RequiredEnv: []string{ + "GH_TOKEN=${HOST_GH}", + "LOG_LEVEL=${HOST_MISSING:-info}", + "EDITOR=vim", + }, + }) + if err != nil { + t.Fatalf("ResolveEnv: %v", err) + } + want := []string{"GH_TOKEN=ghp_xyz", "LOG_LEVEL=info", "EDITOR=vim"} + if len(got) != len(want) { + t.Fatalf("got %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("entry %d = %q, want %q", i, got[i], want[i]) + } + } + + if entries, err := ResolveEnv(&config.Sandbox{Name: "sb"}); err != nil || entries != nil { + t.Errorf("no declared env: got %v, %v; want nil, nil", entries, err) + } +} + +// TestResolveEnvReturnsPartialResults is what lets attach stay best-effort +// while create stays strict: one unresolvable entry must not take the +// resolvable ones down with it. EnvFile treats the error as fatal; +// cli.buildVSockEnv warns and uses what came back. +func TestResolveEnvReturnsPartialResults(t *testing.T) { + os.Unsetenv("HOST_REQUIRED") + t.Setenv("HOST_OK", "fine") + + got, err := ResolveEnv(&config.Sandbox{ + Name: "sb", + RequiredEnv: []string{ + "API_KEY=${HOST_REQUIRED:?set it in your shell}", + "OK=${HOST_OK}", + }, + }) + if err == nil { + t.Fatal("expected an error for the required-but-missing entry") + } + if len(got) != 1 || got[0] != "OK=fine" { + t.Errorf("got %v, want the one resolvable entry [OK=fine]", got) + } +} + // TestEnvFileRequiredMissingErrors verifies that a ${HOST:?msg} whose // host variable is unset fails EnvFile (and therefore sandbox creation) // with a message that includes the author's note. diff --git a/internal/sandbox/swapdisk.go b/internal/sandbox/swapdisk.go new file mode 100644 index 0000000..cd435fa --- /dev/null +++ b/internal/sandbox/swapdisk.go @@ -0,0 +1,109 @@ +package sandbox + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/clawkwork/clawk/internal/config" +) + +// Swap. Every sandbox gets a swap device unless it opts out with +// `vm ( swap off )`. +// +// The reason is the balloon controller, not the guest's own appetite. Under +// host memory pressure clawk reclaims guest RAM against guest demand +// (machine/vz.mergedBalloonTarget drops the cap to 75% of the ceiling at WARN +// and 50% at CRITICAL), and the guest's only answers without swap are direct +// reclaim stalls and the OOM killer — the balloon's DEFLATE_ON_OOM safety net +// assumes exactly that. A multi-second stall in the agent process is not just +// slow: a process that stops draining its TLS socket lets the connection go +// idle, and on a link whose NAT reaps idle mappings in under a minute that +// ends the streaming response outright. +// +// The cost is bounded by sparseness, not by size — see DefaultSwapSizeMiB. + +// DefaultSwapSizeMiB is the swap device's capacity when the sandbox doesn't +// say otherwise. It is a ceiling on how much the guest may swap, not an +// allocation: the backing file is sparse and materializes host bytes only as +// pages are actually written to it. A sandbox that never swaps carries a +// 2 GiB device that costs a few hundred bytes of directory entry. +// +// It does not shrink again on its own, though. Nothing in the stack punches +// the holes back: swapon(2) is asked for SWAP_FLAG_DISCARD, but neither +// firecracker's virtio-blk nor vz's advertises discard, so the kernel drops +// the flag and freed swap pages stay allocated on the host until the sandbox +// is destroyed. Read the number as a high-water mark — which is the reason +// not to make it larger just because the device is sparse. +const DefaultSwapSizeMiB = 2048 + +// SwapDiskName is the swap device's filename inside a sandbox's VM +// directory. Removed with the rest of the VM dir on destroy. +const SwapDiskName = "swap.img" + +// OCISwapDevice is where the swap disk lands in a vz OCI sandbox. Disks are +// attached in spec order after the rootfs, so vda=rootfs, vdb=guestcfg, and +// swap is the next one. Keep in lock-step with buildOCISandboxSpec's +// Spec.Disks in internal/cli/vzd.go. +const OCISwapDevice = "/dev/vdc" + +// GuestSwappiness is the vm.swappiness clawk-init sets on a swap-enabled +// guest. Above the kernel's default 60 on purpose: the pages we want the +// guest to give up under balloon inflation are cold anonymous ones (an idle +// agent heap), and the page cache we want it to keep is a repo and toolchain +// an active build reads constantly. The default's more even split trades the +// wrong way for this workload. +const GuestSwappiness = 80 + +// SwapDiskMiB is the swap capacity for sb, in MiB, or 0 when the sandbox has +// swap disabled. Mirrors RootDiskSizeMiB's shape: an explicit positive +// override wins, negative means off, and unset takes the default. +func SwapDiskMiB(sb *config.Sandbox) uint64 { + switch { + case sb == nil: + return DefaultSwapSizeMiB + case sb.SwapMiB < 0: + return 0 + case sb.SwapMiB > 0: + return uint64(sb.SwapMiB) + default: + return DefaultSwapSizeMiB + } +} + +// SwapDiskPath is the swap device's host path inside vmDir. +func SwapDiskPath(vmDir string) string { return filepath.Join(vmDir, SwapDiskName) } + +// EnsureSwapDisk makes vmDir hold a sparse swap device of sizeMiB and returns +// its path. A sizeMiB of 0 removes any device a previous configuration left +// behind and returns "" — callers use the empty path as "attach nothing". +// +// Resizing an existing device just truncates it. Swap contents are worthless +// across a boot (the guest re-formats whenever the header doesn't match the +// device), so there is nothing to preserve, and truncation keeps the file +// sparse where a rewrite would not. +func EnsureSwapDisk(vmDir string, sizeMiB uint64) (string, error) { + path := SwapDiskPath(vmDir) + if sizeMiB == 0 { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return "", fmt.Errorf("removing swap disk: %w", err) + } + return "", nil + } + size := int64(sizeMiB) << 20 + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600) + if err != nil { + return "", fmt.Errorf("creating swap disk: %w", err) + } + defer f.Close() + st, err := f.Stat() + if err != nil { + return "", fmt.Errorf("stat swap disk: %w", err) + } + if st.Size() != size { + if err := f.Truncate(size); err != nil { + return "", fmt.Errorf("sizing swap disk to %d MiB: %w", sizeMiB, err) + } + } + return path, nil +} diff --git a/internal/sandbox/swapdisk_test.go b/internal/sandbox/swapdisk_test.go new file mode 100644 index 0000000..49f3e0d --- /dev/null +++ b/internal/sandbox/swapdisk_test.go @@ -0,0 +1,95 @@ +package sandbox + +import ( + "os" + "path/filepath" + "syscall" + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/stretchr/testify/require" +) + +// allocatedBlocks is the file's real footprint on the host in 512-byte +// blocks — st_blocks, which stays at zero across a hole and grows only +// where something was actually written. st_size would report the whole +// ceiling and tell us nothing about what the sandbox costs. +func allocatedBlocks(t *testing.T, path string) int64 { + t.Helper() + fi, err := os.Stat(path) + require.NoError(t, err) + st, ok := fi.Sys().(*syscall.Stat_t) + require.True(t, ok, "stat is not a syscall.Stat_t") + return int64(st.Blocks) +} + +func TestSwapDiskMiB(t *testing.T) { + tests := []struct { + name string + sb *config.Sandbox + want uint64 + }{ + {name: "nil sandbox takes the default", sb: nil, want: DefaultSwapSizeMiB}, + {name: "unset takes the default", sb: &config.Sandbox{}, want: DefaultSwapSizeMiB}, + {name: "explicit size wins", sb: &config.Sandbox{SwapMiB: 8192}, want: 8192}, + // Negative is the "off" sentinel the parser stores for `swap off`, + // distinct from unset — otherwise disabling swap would be + // indistinguishable from never having configured it. + {name: "negative disables", sb: &config.Sandbox{SwapMiB: -1}, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, SwapDiskMiB(tt.sb)) + }) + } +} + +func TestEnsureSwapDisk(t *testing.T) { + vmDir := t.TempDir() + + path, err := EnsureSwapDisk(vmDir, 64) + require.NoError(t, err) + require.Equal(t, filepath.Join(vmDir, SwapDiskName), path) + + fi, err := os.Stat(path) + require.NoError(t, err) + require.Equal(t, int64(64<<20), fi.Size(), "apparent size") + + // The point of the device is that it costs host bytes only as the guest + // swaps into it. A freshly created one must be a hole, not 64 MiB of + // zeros — st_blocks, not st_size, is what the host actually pays. + require.Zero(t, allocatedBlocks(t, path), "a fresh swap disk must be sparse") + + // Re-running is idempotent, and a size change resizes in place. + again, err := EnsureSwapDisk(vmDir, 64) + require.NoError(t, err) + require.Equal(t, path, again) + + _, err = EnsureSwapDisk(vmDir, 128) + require.NoError(t, err) + fi, err = os.Stat(path) + require.NoError(t, err) + require.Equal(t, int64(128<<20), fi.Size(), "resized") + + // Shrinking works too: swap contents don't survive a boot, so there is + // nothing to preserve and truncation keeps the file sparse. + _, err = EnsureSwapDisk(vmDir, 32) + require.NoError(t, err) + fi, err = os.Stat(path) + require.NoError(t, err) + require.Equal(t, int64(32<<20), fi.Size(), "shrunk") + + // Zero means "swap off": the device from a previous configuration has to + // go, or buildSpec would keep finding a file it no longer attaches. + empty, err := EnsureSwapDisk(vmDir, 0) + require.NoError(t, err) + require.Empty(t, empty) + _, err = os.Stat(path) + require.True(t, os.IsNotExist(err), "swap disk removed") + + // Removing an absent device is not an error — the common case is a + // sandbox that never had swap in the first place. + empty, err = EnsureSwapDisk(vmDir, 0) + require.NoError(t, err) + require.Empty(t, empty) +} diff --git a/internal/sandbox/vzprovider_oci_darwin.go b/internal/sandbox/vzprovider_oci_darwin.go index 9d76c12..85324b9 100644 --- a/internal/sandbox/vzprovider_oci_darwin.go +++ b/internal/sandbox/vzprovider_oci_darwin.go @@ -114,6 +114,13 @@ func (v *VZProvider) createOCI(sb *config.Sandbox) error { if err := SeedClaudeStateDir(v.store.StateDir(sb.Name), v.store.RootDir()); err != nil { return fmt.Errorf("seeding claude state dir: %w", err) } + // Declared MCP servers, rendered before boot so the runner's first + // connection attempt already has them. Fatal, unlike the memory seed + // below: a sandbox that silently comes up without the servers its + // clawk.mod asked for is the failure this whole path exists to avoid. + if err := SeedClaudeMCP(v.store.StateDir(sb.Name), sb.MCP); err != nil { + return fmt.Errorf("seeding mcp config: %w", err) + } // Seed baseline auto-memory once (no-op if the sandbox already has memory, // e.g. a re-create whose state dir survived). Best-effort: never block boot. if err := SeedClaudeMemory(v.store.StateDir(sb.Name), sb.Memory); err != nil { diff --git a/internal/serialfwd/serialfwd.go b/internal/serialfwd/serialfwd.go new file mode 100644 index 0000000..6bed51e --- /dev/null +++ b/internal/serialfwd/serialfwd.go @@ -0,0 +1,388 @@ +// Package serialfwd is the wire protocol for serial forwarding: a physical +// serial port on the host presented as a PTY inside the guest. +// +// The motivating case is microcontroller work — an Arduino or ESP32 plugged +// into the Mac, flashed and monitored by tooling running in the sandbox. +// Passing the USB device itself through is not an option: Virtualization. +// framework exposes no physical USB passthrough (its USB controller carries +// virtual mass-storage devices only) and firecracker has no USB at all. But +// none of the tooling actually wants USB — avrdude, esptool and every serial +// monitor want a tty and a baud rate. That is a byte stream plus a little +// out-of-band state, which vsock carries fine. +// +// Shape, per connection (the guest always dials, the host always listens on +// VSockPort — the same asymmetry as internal/revfwd, and the same reason +// this is vz-only): +// +// guest → host one JSON Greeting, newline-terminated +// op=control host replies with a Snapshot line now and another on +// every change to the device set, until the connection is +// closed. This is how the guest learns which PTYs to +// create, so `clawk serial add` applies live. +// op=attach host validates Name against the current set, opens the +// physical port, replies with one AttachReply line, and +// then speaks frames (below) for the rest of the +// connection. +// +// Validation is host-side on purpose: the guest names a *device*, never a +// host path, and a name the user didn't configure is refused. The host +// holds the mapping from name to /dev/cu.usbmodem…, so a process in the +// sandbox can reach exactly the ports the user attached and no others. +// +// # Connection lifetime is the open/close signal +// +// An attach connection exists for exactly as long as some guest process +// holds the PTY open, and the host opens and closes the physical port to +// match. That is not just bookkeeping — it is how auto-reset works. The +// classic Arduino reset circuit pulses RESET from the DTR line, and opening +// a serial port asserts DTR; this is why opening the Arduino IDE's serial +// monitor reboots an Uno. Tying the host-side open to the guest-side open +// reproduces that pulse at the one moment the tooling expects it, without +// the guest ever naming a modem-control line. +// +// It also means the port is free whenever the sandbox isn't using it, so +// the Arduino IDE on the Mac can still have it. +// +// # What a PTY cannot carry +// +// A PTY has no modem-control lines: TIOCMGET/TIOCMSET on either end return +// ENOTTY on Linux. So a guest tool that toggles DTR or RTS explicitly gets +// an error, and no protocol here can fix that — see docs/serial.md for +// which boards that affects and the workarounds. What a PTY *does* carry is +// the termios state (master and slave share it, so the guest agent can read +// back the baud rate its client set) and the open/close edges. Those are +// the two things the FrameMode message and the connection lifetime +// respectively convey, and between them they cover the 1200-baud touch that +// native-USB boards use to enter their bootloader. +// +// # Framing +// +// The handshake is JSON lines like revfwd's, because it's a handful of +// messages and being greppable in a log is worth more than the bytes. What +// follows is *not* a raw byte stream like revfwd's, though: serial data and +// mode changes have to stay in order relative to each other. A tool that +// writes a command, changes baud, then writes another command must have +// those land in that order on the wire — so both travel as frames on the +// one connection rather than data inline and mode on the side. +// +// The guest half is inlined in internal/agentembed/main.go.in (the agent +// builds standalone inside the guest and can't import this package). Any +// change here must be mirrored there — same rule as internal/revfwd, and +// TestSerialProtocolMirroredInAgent enforces it. +package serialfwd + +import ( + "bufio" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "strings" +) + +// VSockPort is the host-side AF_VSOCK port the guest dials for both +// connection kinds. Disjoint from the other fixed ports: 1024 pty-agent, +// 1025 time-sync, 1026 ssh-agent, 1027 mem-report, 1028 reverse-forward, +// 1100+ 9p caches. +const VSockPort uint32 = 1029 + +// ProtoVersion is the current wire version. Bumped only on a breaking +// change; the guest binary is rebuilt from these sources and re-injected on +// every `clawk up`, so host and guest can't drift within a release. +const ProtoVersion = 1 + +// Greeting is the first line of every connection. +type Greeting struct { + // Op is OpControl or OpAttach. + Op string `json:"op"` + + // V is the sender's ProtoVersion. + V int `json:"v"` + + // Name is the guest-visible device name to attach to, without any + // directory part — the same string the host published in a Snapshot. + // Set for OpAttach only. + Name string `json:"name,omitempty"` + + // Mode is the PTY's termios state at the moment the guest attached, so + // the host can configure the port before the first byte moves rather + // than opening at some default and correcting. Set for OpAttach only; + // nil means "whatever the port already had". + Mode *Mode `json:"mode,omitempty"` +} + +// Greeting Op values. +const ( + OpControl = "control" + OpAttach = "attach" +) + +// Snapshot is the host's reply on an OpControl connection: the complete set +// of forwarded serial devices as of now. Every update resends the full set — +// the guest reconciles against it rather than applying deltas, so a +// dropped-and-redialed control connection converges the same way. +type Snapshot struct { + Devices []Device `json:"devices"` +} + +// Device is one host serial port exposed in the guest. +type Device struct { + // Name is the guest-visible name, created at /dev/. The host + // path is deliberately not sent: the guest has no use for it and + // shouldn't learn the shape of the host's /dev. + Name string `json:"name"` +} + +// AttachReply is the host's verdict on an OpAttach greeting. Frames flow +// only after ok=true; on ok=false the host closes the connection. +type AttachReply struct { + OK bool `json:"ok"` + // Error is a short human-readable reason when OK is false — an unknown + // device name, or the port being unplugged or already held by another + // process. The guest logs it; it's the only place an unplugged board + // surfaces. + Error string `json:"error,omitempty"` +} + +// Mode is the line configuration of a serial port. It is the subset of +// termios that describes the wire format, which is all a remote end can +// meaningfully apply — flow control and the special characters belong to +// whichever end is doing the cooking. +type Mode struct { + // Baud is the symbol rate. Both directions are always set to it: + // split-speed serial has no users left and no way to express itself + // through the PTY the guest reads this back from. + Baud int `json:"baud"` + + // Bits is the character size, 5 through 8. + Bits int `json:"bits"` + + // Parity is ParityNone, ParityEven or ParityOdd. + Parity string `json:"parity"` + + // Stop is the number of stop bits, 1 or 2. + Stop int `json:"stop"` +} + +// Parity values. +const ( + ParityNone = "n" + ParityEven = "e" + ParityOdd = "o" +) + +// DefaultMode is what a port is configured to when the guest attaches +// without stating a mode. 9600-8N1 because that is what a tty defaults to +// nearly everywhere, so a guest tool that never calls tcsetattr sees the +// same thing it would on real hardware. +func DefaultMode() Mode { + return Mode{Baud: 9600, Bits: 8, Parity: ParityNone, Stop: 1} +} + +func (m Mode) String() string { + return fmt.Sprintf("%d-%d%s%d", m.Baud, m.Bits, strings.ToUpper(m.Parity), m.Stop) +} + +// Validate reports whether m is a line configuration that can actually be +// applied. It is called on the host before touching the port, because the +// values arrive from the guest: a nonsense character size would otherwise +// become a confusing tcsetattr failure at the far end of the stack. +func (m Mode) Validate() error { + if m.Baud <= 0 { + return fmt.Errorf("serialfwd: baud %d out of range", m.Baud) + } + if m.Bits < 5 || m.Bits > 8 { + return fmt.Errorf("serialfwd: character size %d out of range (5-8)", m.Bits) + } + switch m.Parity { + case ParityNone, ParityEven, ParityOdd: + default: + return fmt.Errorf("serialfwd: unknown parity %q", m.Parity) + } + if m.Stop != 1 && m.Stop != 2 { + return fmt.Errorf("serialfwd: stop bits %d out of range (1-2)", m.Stop) + } + return nil +} + +// ──────────────────────────────────────────────────────────────────────── +// Framing +// ──────────────────────────────────────────────────────────────────────── + +// FrameType tags a post-handshake message. +type FrameType byte + +const ( + // FrameData carries raw serial bytes, in either direction. + FrameData FrameType = 0x01 + + // FrameMode carries a JSON Mode, guest → host, when the guest's PTY + // client changed the line configuration. Ordered with respect to the + // FrameData frames around it, which is the entire reason frames exist + // here — see the package comment. + FrameMode FrameType = 0x02 +) + +func (t FrameType) String() string { + switch t { + case FrameData: + return "data" + case FrameMode: + return "mode" + default: + return fmt.Sprintf("unknown(0x%02x)", byte(t)) + } +} + +// FrameHeaderBytes is the fixed header: one type byte then a big-endian +// uint32 payload length. +const FrameHeaderBytes = 5 + +// MaxFrameBytes caps one frame's payload. Serial is slow — even at 921600 +// baud a port produces ~90 KiB/s — so this is far above any real read, and +// exists only so a peer can't make the reader allocate without bound. +const MaxFrameBytes = 256 * 1024 + +// ErrFrameTooLong reports a frame past MaxFrameBytes. The connection must +// be closed: the reader has no way to resynchronise with the framing. +var ErrFrameTooLong = errors.New("serialfwd: frame exceeds MaxFrameBytes") + +// WriteFrame writes one frame. Callers that interleave frames from several +// goroutines must serialise their own writes — a partially written frame +// desynchronises the stream for good. +func WriteFrame(w io.Writer, t FrameType, payload []byte) error { + if len(payload) > MaxFrameBytes { + return ErrFrameTooLong + } + var hdr [FrameHeaderBytes]byte + hdr[0] = byte(t) + binary.BigEndian.PutUint32(hdr[1:], uint32(len(payload))) + // One Write for the header and one for the body rather than a copy + // into a joined buffer: these go to a vsock conn, and the extra + // allocation per data frame costs more than the extra syscall. + if _, err := w.Write(hdr[:]); err != nil { + return err + } + if len(payload) == 0 { + return nil + } + _, err := w.Write(payload) + return err +} + +// WriteModeFrame encodes m and writes it as a FrameMode. +func WriteModeFrame(w io.Writer, m Mode) error { + b, err := json.Marshal(m) + if err != nil { + return fmt.Errorf("serialfwd: encoding mode: %w", err) + } + return WriteFrame(w, FrameMode, b) +} + +// ReadFrame reads one frame. The returned slice aliases a fresh allocation, +// so the caller may retain it. +// +// It takes a *bufio.Reader for the same reason ReadLine does: the handshake +// and the frames share a connection, and a reader that buffered past the +// greeting's newline holds bytes belonging to the first frame. +func ReadFrame(r *bufio.Reader) (FrameType, []byte, error) { + var hdr [FrameHeaderBytes]byte + if _, err := io.ReadFull(r, hdr[:]); err != nil { + return 0, nil, err + } + n := binary.BigEndian.Uint32(hdr[1:]) + if n > MaxFrameBytes { + return 0, nil, ErrFrameTooLong + } + if n == 0 { + return FrameType(hdr[0]), nil, nil + } + buf := make([]byte, n) + if _, err := io.ReadFull(r, buf); err != nil { + // A truncated payload is a protocol error, not a clean end: report + // it as unexpected EOF so callers don't mistake it for the peer + // hanging up tidily between frames. + if errors.Is(err, io.EOF) { + return 0, nil, io.ErrUnexpectedEOF + } + return 0, nil, err + } + return FrameType(hdr[0]), buf, nil +} + +// ──────────────────────────────────────────────────────────────────────── +// Handshake lines +// ──────────────────────────────────────────────────────────────────────── + +// MaxLineBytes caps one handshake line. Snapshots are a few dozen bytes per +// device; the cap exists so a peer can't make the reader allocate without +// bound. +const MaxLineBytes = 64 * 1024 + +// ErrLineTooLong reports a handshake line past MaxLineBytes. The connection +// must be closed — the reader is out of sync with the framing. +var ErrLineTooLong = errors.New("serialfwd: control line exceeds MaxLineBytes") + +// WriteLine JSON-encodes v and writes it as one newline-terminated line. +func WriteLine(w io.Writer, v any) error { + b, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("serialfwd: encoding %T: %w", v, err) + } + if len(b)+1 > MaxLineBytes { + return ErrLineTooLong + } + _, err = w.Write(append(b, '\n')) + return err +} + +// ReadLine reads one newline-terminated JSON line into v. The reader must +// be the one the caller keeps using — see ReadFrame. +func ReadLine(r *bufio.Reader, v any) error { + line, err := r.ReadSlice('\n') + if errors.Is(err, bufio.ErrBufferFull) { + return ErrLineTooLong + } + if err != nil { + return err + } + if err := json.Unmarshal(line, v); err != nil { + return fmt.Errorf("serialfwd: decoding %T: %w", v, err) + } + return nil +} + +// NewReader wraps r with a reader sized for MaxLineBytes, so ReadLine's +// buffer-full case really does mean "line too long" rather than "buffer too +// small". +func NewReader(r io.Reader) *bufio.Reader { return bufio.NewReaderSize(r, MaxLineBytes) } + +// ValidDeviceName reports whether name is usable as a guest device name. +// +// The guest creates /dev/, so anything with a directory part, a +// relative-path element, or a leading dot is refused — the host validates +// on the way in and the guest validates again on the way out, because this +// is the one field that becomes a filesystem path on the far side. +func ValidDeviceName(name string) error { + switch { + case name == "": + return errors.New("serial: device name is empty") + case len(name) > 64: + return fmt.Errorf("serial: device name %q is longer than 64 characters", name) + case strings.ContainsAny(name, "/\\"): + return fmt.Errorf("serial: device name %q must not contain a path separator", name) + case strings.HasPrefix(name, "."): + return fmt.Errorf("serial: device name %q must not start with a dot", name) + } + for _, r := range name { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case r == '.' || r == '-' || r == '_': + default: + return fmt.Errorf("serial: device name %q contains %q; "+ + "use letters, digits, dot, dash or underscore", name, r) + } + } + return nil +} diff --git a/internal/serialfwd/serialfwd_test.go b/internal/serialfwd/serialfwd_test.go new file mode 100644 index 0000000..0334662 --- /dev/null +++ b/internal/serialfwd/serialfwd_test.go @@ -0,0 +1,182 @@ +package serialfwd + +import ( + "bytes" + "encoding/binary" + "errors" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFrameRoundTrip(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, WriteFrame(&buf, FrameData, []byte("hello"))) + require.NoError(t, WriteModeFrame(&buf, Mode{Baud: 115200, Bits: 8, Parity: ParityNone, Stop: 1})) + require.NoError(t, WriteFrame(&buf, FrameData, nil)) + + r := NewReader(&buf) + + typ, payload, err := ReadFrame(r) + require.NoError(t, err) + require.Equal(t, FrameData, typ) + require.Equal(t, []byte("hello"), payload) + + typ, payload, err = ReadFrame(r) + require.NoError(t, err) + require.Equal(t, FrameMode, typ) + require.JSONEq(t, `{"baud":115200,"bits":8,"parity":"n","stop":1}`, string(payload)) + + // An empty data frame is legal and must not be confused with EOF. + typ, payload, err = ReadFrame(r) + require.NoError(t, err) + require.Equal(t, FrameData, typ) + require.Empty(t, payload) + + _, _, err = ReadFrame(r) + require.ErrorIs(t, err, io.EOF) +} + +// Frames and handshake lines share one connection, and the reader that read +// the greeting has already buffered whatever followed it. Reusing that +// reader is a documented requirement of the protocol, so it gets a test: +// the failure mode if it regresses is a first frame that silently vanishes. +func TestHandshakeReaderCarriesBufferedFrames(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, WriteLine(&buf, Greeting{Op: OpAttach, V: ProtoVersion, Name: "ttyACM0"})) + require.NoError(t, WriteFrame(&buf, FrameData, []byte("pipelined"))) + + r := NewReader(&buf) + var g Greeting + require.NoError(t, ReadLine(r, &g)) + require.Equal(t, OpAttach, g.Op) + require.Equal(t, "ttyACM0", g.Name) + + typ, payload, err := ReadFrame(r) + require.NoError(t, err) + require.Equal(t, FrameData, typ) + require.Equal(t, []byte("pipelined"), payload) +} + +func TestGreetingCarriesMode(t *testing.T) { + var buf bytes.Buffer + mode := Mode{Baud: 1200, Bits: 7, Parity: ParityEven, Stop: 2} + require.NoError(t, WriteLine(&buf, Greeting{ + Op: OpAttach, V: ProtoVersion, Name: "ttyACM0", Mode: &mode, + })) + + var got Greeting + require.NoError(t, ReadLine(NewReader(&buf), &got)) + require.NotNil(t, got.Mode) + require.Equal(t, mode, *got.Mode) + + // Absent rather than zero-valued when unset: a guest that doesn't state + // a mode must not be read as asking for 0 baud. + buf.Reset() + require.NoError(t, WriteLine(&buf, Greeting{Op: OpAttach, V: ProtoVersion, Name: "x"})) + require.NotContains(t, buf.String(), "mode") +} + +func TestReadFrameRejectsOversizedLength(t *testing.T) { + // A header claiming more than MaxFrameBytes must be refused before the + // allocation, not after. + var hdr [FrameHeaderBytes]byte + hdr[0] = byte(FrameData) + binary.BigEndian.PutUint32(hdr[1:], MaxFrameBytes+1) + + _, _, err := ReadFrame(NewReader(bytes.NewReader(hdr[:]))) + require.ErrorIs(t, err, ErrFrameTooLong) +} + +func TestWriteFrameRejectsOversizedPayload(t *testing.T) { + err := WriteFrame(io.Discard, FrameData, make([]byte, MaxFrameBytes+1)) + require.ErrorIs(t, err, ErrFrameTooLong) +} + +// A truncated payload is a broken peer, not a tidy hangup. Callers treat +// io.EOF between frames as "the other end went away cleanly", so a short +// read in the middle of one has to be distinguishable. +func TestReadFrameTruncatedPayload(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, WriteFrame(&buf, FrameData, []byte("0123456789"))) + truncated := buf.Bytes()[:FrameHeaderBytes+4] + + _, _, err := ReadFrame(NewReader(bytes.NewReader(truncated))) + require.ErrorIs(t, err, io.ErrUnexpectedEOF) +} + +func TestReadLineRejectsOversizedLine(t *testing.T) { + line := append(bytes.Repeat([]byte("x"), MaxLineBytes+10), '\n') + var g Greeting + err := ReadLine(NewReader(bytes.NewReader(line)), &g) + require.ErrorIs(t, err, ErrLineTooLong) +} + +func TestModeValidate(t *testing.T) { + require.NoError(t, DefaultMode().Validate()) + require.NoError(t, Mode{Baud: 115200, Bits: 8, Parity: ParityNone, Stop: 1}.Validate()) + require.NoError(t, Mode{Baud: 300, Bits: 5, Parity: ParityOdd, Stop: 2}.Validate()) + + for name, m := range map[string]Mode{ + "zero baud": {Baud: 0, Bits: 8, Parity: ParityNone, Stop: 1}, + "negative baud": {Baud: -1, Bits: 8, Parity: ParityNone, Stop: 1}, + "bits too few": {Baud: 9600, Bits: 4, Parity: ParityNone, Stop: 1}, + "bits too many": {Baud: 9600, Bits: 9, Parity: ParityNone, Stop: 1}, + "bad parity": {Baud: 9600, Bits: 8, Parity: "mark", Stop: 1}, + "empty parity": {Baud: 9600, Bits: 8, Parity: "", Stop: 1}, + "bad stop": {Baud: 9600, Bits: 8, Parity: ParityNone, Stop: 3}, + } { + t.Run(name, func(t *testing.T) { + require.Error(t, m.Validate()) + }) + } +} + +func TestModeString(t *testing.T) { + require.Equal(t, "115200-8N1", Mode{Baud: 115200, Bits: 8, Parity: ParityNone, Stop: 1}.String()) + require.Equal(t, "9600-7E2", Mode{Baud: 9600, Bits: 7, Parity: ParityEven, Stop: 2}.String()) +} + +// The guest turns this name into /dev/. Everything that could escape +// that directory, or land somewhere surprising inside it, has to be refused +// on the host before it is ever published. +func TestValidDeviceName(t *testing.T) { + for _, ok := range []string{"ttyACM0", "ttyUSB0", "cu.usbmodem1101", "arduino-uno", "a_b", "x"} { + require.NoError(t, ValidDeviceName(ok), "%q should be accepted", ok) + } + for _, bad := range []string{ + "", + "..", + ".hidden", + "../../etc/passwd", + "sub/dir", + `back\slash`, + "has space", + "null\x00byte", + "emoji✨", + strings.Repeat("a", 65), + } { + require.Error(t, ValidDeviceName(bad), "%q should be refused", bad) + } +} + +func TestFrameTypeString(t *testing.T) { + require.Equal(t, "data", FrameData.String()) + require.Equal(t, "mode", FrameMode.String()) + require.Equal(t, "unknown(0x7f)", FrameType(0x7f).String()) +} + +// WriteFrame emits the header and body as separate writes. A conn that +// reports a short write on the header must not leave the caller thinking +// the frame landed. +func TestWriteFrameSurfacesWriteErrors(t *testing.T) { + want := errors.New("conn closed") + err := WriteFrame(errWriter{want}, FrameData, []byte("x")) + require.ErrorIs(t, err, want) +} + +type errWriter struct{ err error } + +func (w errWriter) Write([]byte) (int, error) { return 0, w.err } diff --git a/internal/serialport/serialport.go b/internal/serialport/serialport.go new file mode 100644 index 0000000..936c342 --- /dev/null +++ b/internal/serialport/serialport.go @@ -0,0 +1,140 @@ +// Package serialport opens and configures a physical serial port on the +// host. It is the host end of `clawk serial` — internal/serialfwd carries +// the bytes, internal/cli/serial_proxy.go brokers them, and this is what +// actually talks to the tty. +// +// Only what forwarding needs is here: open a port (possibly named by a +// glob), put it in raw mode, apply a line configuration the guest asked +// for, and read and write bytes until someone closes it. No enumeration, +// no modem-line control, no flow control — see the serialfwd package +// comment for why the last of those can't be plumbed through a PTY anyway. +package serialport + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/clawkwork/clawk/internal/serialfwd" +) + +// Port is an open serial port. +// +// It embeds no lock: Read and Write may be called concurrently (they are, +// by the two halves of the proxy's pump), which is safe on an *os.File, but +// two concurrent Writes will interleave and two concurrent Reads will race +// for bytes. The proxy runs exactly one of each. +type Port struct { + f *os.File + // path is the resolved device, which for a glob is not what the user + // configured. Every log line uses this rather than the pattern, so a + // board that came back as usbmodem14201 says so. + path string +} + +// ErrNoMatch reports a device path that matched nothing. It is worth +// distinguishing because it is the ordinary "the board is unplugged, or is +// mid-reset" case, and the caller retries on it rather than giving up. +var ErrNoMatch = errors.New("serialport: no device matches") + +// Open resolves pattern, opens the device, and puts it in raw mode with +// mode's line configuration. +// +// pattern may be a literal path or a glob. A glob is resolved here, at open +// time, rather than when the device was configured: a board that reboots +// into its bootloader disappears from /dev and comes back — often under a +// neighbouring name — and re-globbing on each open is what lets a forward +// survive that. A glob matching several devices is an error rather than a +// guess, because picking the wrong board silently is worse than saying so. +func Open(pattern string, mode serialfwd.Mode) (*Port, error) { + if err := mode.Validate(); err != nil { + return nil, err + } + path, err := Resolve(pattern) + if err != nil { + return nil, err + } + + // O_NOCTTY: this is a daemon, and acquiring a controlling terminal + // would hand the board's line disciplines a say in our signal + // handling. + // + // O_NONBLOCK does two jobs. It stops the open itself from blocking on + // carrier detect (which /dev/tty.* on macOS does, unlike the /dev/cu.* + // callout device users should be naming), and it makes the descriptor + // pollable, so os.NewFile hands back a File registered with the Go + // runtime poller. That is what allows Close to unblock a Read parked + // waiting for a byte that may never come — the whole teardown path + // depends on it, so TestCloseUnblocksBlockedRead guards it. + fd, err := openNonblock(path) + if err != nil { + return nil, fmt.Errorf("serialport: opening %s: %w", path, err) + } + p := &Port{f: os.NewFile(uintptr(fd), path), path: path} + if p.f == nil { + _ = closeFD(fd) + return nil, fmt.Errorf("serialport: %s: invalid descriptor", path) + } + if err := p.Configure(mode); err != nil { + _ = p.f.Close() + return nil, err + } + return p, nil +} + +// Resolve turns a device pattern into a single concrete path. A literal +// path is returned once it is confirmed to exist, so an unplugged board +// fails here with ErrNoMatch rather than at open with a bare ENOENT. +func Resolve(pattern string) (string, error) { + if !isGlob(pattern) { + if _, err := os.Stat(pattern); err != nil { + if errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("%w %s", ErrNoMatch, pattern) + } + return "", fmt.Errorf("serialport: %s: %w", pattern, err) + } + return pattern, nil + } + matches, err := filepath.Glob(pattern) + if err != nil { + return "", fmt.Errorf("serialport: bad device pattern %q: %w", pattern, err) + } + switch len(matches) { + case 0: + return "", fmt.Errorf("%w %s", ErrNoMatch, pattern) + case 1: + return matches[0], nil + default: + sort.Strings(matches) + return "", fmt.Errorf( + "serialport: pattern %s matches %d devices (%s) — narrow it so it names one", + pattern, len(matches), strings.Join(matches, ", ")) + } +} + +func isGlob(s string) bool { return strings.ContainsAny(s, "*?[") } + +// Configure applies mode to the open port, leaving it in raw mode. +func (p *Port) Configure(mode serialfwd.Mode) error { + if err := mode.Validate(); err != nil { + return err + } + if err := applyMode(int(p.f.Fd()), mode); err != nil { + return fmt.Errorf("serialport: configuring %s to %s: %w", p.path, mode, err) + } + return nil +} + +// Path is the resolved device this port is open on. +func (p *Port) Path() string { return p.path } + +func (p *Port) Read(b []byte) (int, error) { return p.f.Read(b) } +func (p *Port) Write(b []byte) (int, error) { return p.f.Write(b) } + +// Close releases the port. On a tty configured by Open this lowers DTR, +// which on a board wired for auto-reset is half of the reset pulse the next +// Open completes — see the serialfwd package comment. +func (p *Port) Close() error { return p.f.Close() } diff --git a/internal/serialport/serialport_other.go b/internal/serialport/serialport_other.go new file mode 100644 index 0000000..6329d60 --- /dev/null +++ b/internal/serialport/serialport_other.go @@ -0,0 +1,20 @@ +//go:build !darwin && !linux + +package serialport + +import ( + "errors" + + "github.com/clawkwork/clawk/internal/serialfwd" +) + +// errUnsupported keeps the package compiling on platforms clawk doesn't +// host a VM on. Nothing reaches these: serial forwarding is driven by the +// vz daemon (darwin) and its tests run on darwin and linux. +var errUnsupported = errors.New("serialport: serial ports are not supported on this platform") + +func openNonblock(string) (int, error) { return -1, errUnsupported } + +func closeFD(int) error { return errUnsupported } + +func applyMode(int, serialfwd.Mode) error { return errUnsupported } diff --git a/internal/serialport/serialport_test.go b/internal/serialport/serialport_test.go new file mode 100644 index 0000000..e58d35d --- /dev/null +++ b/internal/serialport/serialport_test.go @@ -0,0 +1,188 @@ +//go:build darwin || linux + +package serialport + +import ( + "errors" + "io" + "os" + "path/filepath" + "testing" + "time" + + "github.com/clawkwork/clawk/internal/serialfwd" + "github.com/clawkwork/clawk/internal/serialport/serialporttest" + "github.com/stretchr/testify/require" +) + +func TestResolveLiteralPath(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "ttyFAKE0") + require.NoError(t, os.WriteFile(path, nil, 0o600)) + + got, err := Resolve(path) + require.NoError(t, err) + require.Equal(t, path, got) +} + +// An unplugged board is the common case, not an exceptional one — the proxy +// retries on ErrNoMatch instead of failing the attach outright, so the +// sentinel has to survive Resolve. +func TestResolveMissingLiteralIsErrNoMatch(t *testing.T) { + _, err := Resolve(filepath.Join(t.TempDir(), "nope")) + require.ErrorIs(t, err, ErrNoMatch) +} + +func TestResolveGlob(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "cu.usbmodem1101") + require.NoError(t, os.WriteFile(path, nil, 0o600)) + + got, err := Resolve(filepath.Join(dir, "cu.usbmodem*")) + require.NoError(t, err) + require.Equal(t, path, got) +} + +func TestResolveGlobMatchingNothing(t *testing.T) { + _, err := Resolve(filepath.Join(t.TempDir(), "cu.usbmodem*")) + require.ErrorIs(t, err, ErrNoMatch) +} + +// Two boards behind one pattern must be an error naming both. Silently +// picking the first would flash the wrong device, which is the single most +// expensive way this package could be wrong. +func TestResolveAmbiguousGlob(t *testing.T) { + dir := t.TempDir() + for _, n := range []string{"cu.usbmodem1101", "cu.usbmodem2201"} { + require.NoError(t, os.WriteFile(filepath.Join(dir, n), nil, 0o600)) + } + + _, err := Resolve(filepath.Join(dir, "cu.usbmodem*")) + require.Error(t, err) + require.NotErrorIs(t, err, ErrNoMatch) + require.Contains(t, err.Error(), "cu.usbmodem1101") + require.Contains(t, err.Error(), "cu.usbmodem2201") +} + +func TestOpenAppliesModeAndPumpsBytes(t *testing.T) { + master, slavePath := serialporttest.OpenPTYPair(t) + + port, err := Open(slavePath, serialfwd.Mode{ + Baud: 115200, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1, + }) + require.NoError(t, err) + defer port.Close() + require.Equal(t, slavePath, port.Path()) + + // Device → host. Raw mode has to be in force for this to arrive + // untouched: with OPOST still set the tty layer would rewrite the \n. + _, err = master.WriteString("from board\n") + require.NoError(t, err) + + buf := make([]byte, 64) + n, err := readWithin(t, port, buf, 2*time.Second) + require.NoError(t, err) + require.Equal(t, "from board\n", string(buf[:n])) + + // Host → device. ECHO must be off, or this comes straight back at us + // and the next read sees its own bytes. + _, err = port.Write([]byte("to board\n")) + require.NoError(t, err) + + n, err = readWithin(t, master, buf, 2*time.Second) + require.NoError(t, err) + require.Equal(t, "to board\n", string(buf[:n])) +} + +// The proxy tears a port down by closing it, from a different goroutine +// than the one parked in Read. That only works because Open leaves the +// descriptor non-blocking and therefore registered with the Go poller; if +// that ever regresses the read blocks forever and the daemon leaks a +// goroutine per unplugged board. +func TestCloseUnblocksBlockedRead(t *testing.T) { + _, slavePath := serialporttest.OpenPTYPair(t) + + port, err := Open(slavePath, serialfwd.DefaultMode()) + require.NoError(t, err) + + readErr := make(chan error, 1) + go func() { + _, err := port.Read(make([]byte, 16)) + readErr <- err + }() + + // Give the read time to actually park before pulling the rug. + time.Sleep(100 * time.Millisecond) + require.NoError(t, port.Close()) + + select { + case err := <-readErr: + require.Error(t, err, "Read should fail once the port is closed") + case <-time.After(2 * time.Second): + t.Fatal("Read did not return after Close — the fd is not pollable") + } +} + +// Every mode the guest can legally ask for has to survive the round trip to +// the kernel. The baud rates matter most: 250000 and 460800 have no B +// constant on Linux, which is the whole reason applyMode goes through +// termios2/BOTHER there. +func TestConfigureAcceptsEveryLegalMode(t *testing.T) { + _, slavePath := serialporttest.OpenPTYPair(t) + port, err := Open(slavePath, serialfwd.DefaultMode()) + require.NoError(t, err) + defer port.Close() + + for _, mode := range []serialfwd.Mode{ + {Baud: 1200, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1}, + {Baud: 9600, Bits: 7, Parity: serialfwd.ParityEven, Stop: 2}, + {Baud: 57600, Bits: 8, Parity: serialfwd.ParityOdd, Stop: 1}, + {Baud: 115200, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1}, + {Baud: 250000, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1}, + {Baud: 460800, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1}, + {Baud: 921600, Bits: 8, Parity: serialfwd.ParityNone, Stop: 1}, + {Baud: 5, Bits: 5, Parity: serialfwd.ParityNone, Stop: 1}, + } { + t.Run(mode.String(), func(t *testing.T) { + require.NoError(t, port.Configure(mode)) + }) + } +} + +// A mode arrives from the guest, so a bad one is a wire-level input rather +// than a programming error. It must be refused before any ioctl, and before +// the device is opened at all. +func TestOpenRejectsInvalidModeBeforeTouchingTheDevice(t *testing.T) { + _, err := Open("/definitely/not/a/device", serialfwd.Mode{Baud: 0}) + require.Error(t, err) + require.NotErrorIs(t, err, ErrNoMatch, "should fail on the mode, not the path") +} + +func TestConfigureRejectsInvalidMode(t *testing.T) { + _, slavePath := serialporttest.OpenPTYPair(t) + port, err := Open(slavePath, serialfwd.DefaultMode()) + require.NoError(t, err) + defer port.Close() + + require.Error(t, port.Configure(serialfwd.Mode{Baud: 9600, Bits: 99, Parity: "n", Stop: 1})) +} + +// readWithin fails the test rather than hanging when a read doesn't land. +func readWithin(t *testing.T, r io.Reader, buf []byte, d time.Duration) (int, error) { + t.Helper() + type result struct { + n int + err error + } + ch := make(chan result, 1) + go func() { + n, err := r.Read(buf) + ch <- result{n, err} + }() + select { + case res := <-ch: + return res.n, res.err + case <-time.After(d): + return 0, errors.New("timed out waiting for read") + } +} diff --git a/internal/serialport/serialporttest/pty_darwin.go b/internal/serialport/serialporttest/pty_darwin.go new file mode 100644 index 0000000..816b02d --- /dev/null +++ b/internal/serialport/serialporttest/pty_darwin.go @@ -0,0 +1,50 @@ +package serialporttest + +import ( + "bytes" + "os" + "testing" + "unsafe" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +// OpenPTYPair is the darwin half of the Linux function of the same name — +// see there for what it is for. Darwin spells the three steps differently: +// TIOCPTYGRANT and TIOCPTYUNLK for grantpt/unlockpt, and TIOCPTYGNAME to +// read the slave path into a caller-supplied buffer, which x/sys/unix has +// no typed wrapper for. +func OpenPTYPair(t *testing.T) (master *os.File, slavePath string) { + t.Helper() + + // O_NONBLOCK so os.NewFile registers the master with the Go poller, + // which is what makes SetDeadline work on it — tests read from this end + // and need to fail rather than hang when nothing arrives. + fd, err := unix.Open("/dev/ptmx", unix.O_RDWR|unix.O_NOCTTY|unix.O_NONBLOCK, 0) + require.NoError(t, err, "opening /dev/ptmx") + master = os.NewFile(uintptr(fd), "/dev/ptmx") + t.Cleanup(func() { _ = master.Close() }) + + require.NoError(t, unix.IoctlSetInt(fd, unix.TIOCPTYGRANT, 0), "grantpt") + require.NoError(t, unix.IoctlSetInt(fd, unix.TIOCPTYUNLK, 0), "unlockpt") + + // TIOCPTYGNAME writes a NUL-terminated path into a 128-byte buffer. + var buf [128]byte + if _, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), + uintptr(unix.TIOCPTYGNAME), uintptr(unsafe.Pointer(&buf[0]))); errno != 0 { + t.Fatalf("ptsname: %v", errno) + } + name, _, _ := bytes.Cut(buf[:], []byte{0}) + + return master, string(name) +} + +// Speed reads back the line speed currently set on fd. Darwin stores the +// rate numerically in the termios speed fields, so it needs no decoding. +func Speed(t *testing.T, fd int) int { + t.Helper() + tio, err := unix.IoctlGetTermios(fd, unix.TIOCGETA) + require.NoError(t, err) + return int(tio.Ospeed) +} diff --git a/internal/serialport/serialporttest/pty_linux.go b/internal/serialport/serialporttest/pty_linux.go new file mode 100644 index 0000000..7b7c6c1 --- /dev/null +++ b/internal/serialport/serialporttest/pty_linux.go @@ -0,0 +1,49 @@ +package serialporttest + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +// OpenPTYPair returns an open PTY master and the path of its slave. +// +// A PTY is not a UART — it has no modem-control lines and does nothing with +// the baud rate — but it is a real tty, so every ioctl the serial code +// issues goes through the kernel's tty layer rather than a mock, and the +// termios state is genuinely shared between the two ends. That last part is +// what lets a test assert that a mode change actually landed. +func OpenPTYPair(t *testing.T) (master *os.File, slavePath string) { + t.Helper() + + // O_NONBLOCK so os.NewFile registers the master with the Go poller, + // which is what makes SetDeadline work on it — tests read from this end + // and need to fail rather than hang when nothing arrives. + fd, err := unix.Open("/dev/ptmx", unix.O_RDWR|unix.O_NOCTTY|unix.O_NONBLOCK, 0) + require.NoError(t, err, "opening /dev/ptmx") + master = os.NewFile(uintptr(fd), "/dev/ptmx") + t.Cleanup(func() { _ = master.Close() }) + + // unlockpt, then ptsname. + require.NoError(t, unix.IoctlSetPointerInt(fd, unix.TIOCSPTLCK, 0), "unlockpt") + n, err := unix.IoctlGetInt(fd, unix.TIOCGPTN) + require.NoError(t, err, "ptsname") + + return master, fmt.Sprintf("/dev/pts/%d", n) +} + +// Speed reads back the line speed currently set on fd, which for a PTY +// master is the speed its slave was configured to. +// +// Linux keeps the real rate in the termios2 speed fields whenever the CBAUD +// field says BOTHER, which is how internal/serialport sets every rate, so +// that is where this reads it from. +func Speed(t *testing.T, fd int) int { + t.Helper() + tio, err := unix.IoctlGetTermios(fd, unix.TCGETS2) + require.NoError(t, err) + return int(tio.Ospeed) +} diff --git a/internal/serialport/termios_darwin.go b/internal/serialport/termios_darwin.go new file mode 100644 index 0000000..6f8c520 --- /dev/null +++ b/internal/serialport/termios_darwin.go @@ -0,0 +1,46 @@ +package serialport + +import ( + "github.com/clawkwork/clawk/internal/serialfwd" + "golang.org/x/sys/unix" +) + +// applyMode configures the port through TIOCGETA/TIOCSETA. +// +// Darwin needs no equivalent of Linux's BOTHER dance: its B constants +// are the numeric rates themselves (B115200 is 115200), and c_ispeed and +// c_ospeed are plain speed_t fields, so assigning the requested baud +// directly is both correct for the standard rates and the way non-standard +// ones are expressed. A rate the driver can't divide down to is rejected by +// the driver at TIOCSETA, which is the error the caller wants anyway. +func applyMode(fd int, mode serialfwd.Mode) error { + t, err := unix.IoctlGetTermios(fd, unix.TIOCGETA) + if err != nil { + return err + } + + cbits, err := cflagBits(mode) + if err != nil { + return err + } + + t.Iflag &^= uint64(rawIflagClear) + t.Oflag &^= uint64(rawOflagClear) + t.Lflag &^= uint64(rawLflagClear) + t.Cflag &^= uint64(rawCflagClear) + t.Cflag |= uint64(rawCflagSet) | cbits + + t.Ispeed = uint64(mode.Baud) + t.Ospeed = uint64(mode.Baud) + + // See the Linux implementation for why VMIN=1/VTIME=0 is the only safe + // pairing here. + t.Cc[unix.VMIN] = 1 + t.Cc[unix.VTIME] = 0 + + // TIOCSETA applies immediately. The draining variants (TIOCSETAW / + // TIOCSETAF) would wait for output to flush, and a mode change arrives + // here precisely when a tool is mid-handshake with a board that may + // have stopped listening — blocking there would wedge the pump. + return unix.IoctlSetTermios(fd, unix.TIOCSETA, t) +} diff --git a/internal/serialport/termios_linux.go b/internal/serialport/termios_linux.go new file mode 100644 index 0000000..adfa48b --- /dev/null +++ b/internal/serialport/termios_linux.go @@ -0,0 +1,54 @@ +package serialport + +import ( + "github.com/clawkwork/clawk/internal/serialfwd" + "golang.org/x/sys/unix" +) + +// applyMode configures the port through the termios2 interface. +// +// TCGETS2/TCSETS2 rather than the plain TCGETS/TCSETS pair, and BOTHER +// rather than a B constant, so any baud rate works. The classic +// interface can only express the rates that have a constant, which leaves +// out everything from an ESP-PROG at 460800 to the 250000 that DMX and some +// 3D-printer firmwares use. With BOTHER set in the speed field the kernel +// reads the rate from c_ispeed/c_ospeed instead, and the driver rounds to +// whatever its hardware can divide down to. +// +// golang.org/x/sys/unix.Termios is already the termios2 layout on Linux — +// it carries Ispeed and Ospeed after Cc, which the kernel's `struct +// termios` does not — so the same struct serves both ioctls. +func applyMode(fd int, mode serialfwd.Mode) error { + t, err := unix.IoctlGetTermios(fd, unix.TCGETS2) + if err != nil { + return err + } + + cbits, err := cflagBits(mode) + if err != nil { + return err + } + + t.Iflag &^= uint32(rawIflagClear) + t.Oflag &^= uint32(rawOflagClear) + t.Lflag &^= uint32(rawLflagClear) + t.Cflag &^= uint32(rawCflagClear) + t.Cflag |= uint32(rawCflagSet) | uint32(cbits) + + // Speed: clear the CBAUD field, select BOTHER, and put the real rate in + // the dedicated fields. + t.Cflag = (t.Cflag &^ uint32(unix.CBAUD)) | uint32(unix.BOTHER) + t.Ispeed = uint32(mode.Baud) + t.Ospeed = uint32(mode.Baud) + + // VMIN=1/VTIME=0: a read blocks until at least one byte arrives and + // never returns early empty. The alternative (VTIME as a poll timeout) + // would surface as a zero-length read, which os.File reports as io.EOF + // — indistinguishable from the port going away. Blocking is safe here + // only because the descriptor is registered with the Go poller, so + // Close can still interrupt it. + t.Cc[unix.VMIN] = 1 + t.Cc[unix.VTIME] = 0 + + return unix.IoctlSetTermios(fd, unix.TCSETS2, t) +} diff --git a/internal/serialport/termios_unix.go b/internal/serialport/termios_unix.go new file mode 100644 index 0000000..4030e49 --- /dev/null +++ b/internal/serialport/termios_unix.go @@ -0,0 +1,77 @@ +//go:build darwin || linux + +package serialport + +import ( + "fmt" + + "github.com/clawkwork/clawk/internal/serialfwd" + "golang.org/x/sys/unix" +) + +// openNonblock opens a tty without acquiring it as a controlling terminal +// and without blocking on carrier. See the comment in Open for why both +// matter. +func openNonblock(path string) (int, error) { + return unix.Open(path, unix.O_RDWR|unix.O_NOCTTY|unix.O_NONBLOCK, 0) +} + +func closeFD(fd int) error { return unix.Close(fd) } + +// Raw-mode masks, as uint64 so one definition serves both platforms — +// darwin's termios flags are 64-bit and Linux's are 32-bit, and each +// applyMode narrows these to its own width. +// +// This is cfmakeraw plus the two bits a forwarded port needs on top of it: +// CLOCAL, because nothing here should care about a modem carrier that +// USB-serial adapters don't really have, and CREAD, because a port we +// can't read from is useless. +const ( + rawIflagClear = uint64(unix.IGNBRK | unix.BRKINT | unix.PARMRK | unix.ISTRIP | + unix.INLCR | unix.IGNCR | unix.ICRNL | unix.IXON) + rawOflagClear = uint64(unix.OPOST) + rawLflagClear = uint64(unix.ECHO | unix.ECHONL | unix.ICANON | unix.ISIG | unix.IEXTEN) + + // CRTSCTS goes off with the rest: hardware flow control on a forwarded + // port would stall on an RTS line the guest has no way to drive. + rawCflagClear = uint64(unix.CSIZE | unix.PARENB | unix.PARODD | unix.CSTOPB | unix.CRTSCTS) + + // HUPCL lowers DTR when the last descriptor closes. That is deliberate + // and load-bearing rather than incidental: it is what makes the next + // open a rising DTR edge, and so what resets an Arduino at exactly the + // moment a guest process opens the PTY. + rawCflagSet = uint64(unix.CLOCAL | unix.CREAD | unix.HUPCL) +) + +// cflagBits returns the character-size, parity and stop-bit c_cflag bits +// for mode. The caller has already validated mode, so the switches here are +// total; the default arms exist to make that a compile-time-checkable claim +// rather than an assumption. +func cflagBits(mode serialfwd.Mode) (uint64, error) { + var bits uint64 + switch mode.Bits { + case 5: + bits |= unix.CS5 + case 6: + bits |= unix.CS6 + case 7: + bits |= unix.CS7 + case 8: + bits |= unix.CS8 + default: + return 0, fmt.Errorf("serialport: unsupported character size %d", mode.Bits) + } + switch mode.Parity { + case serialfwd.ParityNone: + case serialfwd.ParityEven: + bits |= unix.PARENB + case serialfwd.ParityOdd: + bits |= unix.PARENB | unix.PARODD + default: + return 0, fmt.Errorf("serialport: unsupported parity %q", mode.Parity) + } + if mode.Stop == 2 { + bits |= unix.CSTOPB + } + return bits, nil +} diff --git a/internal/template/global.go b/internal/template/global.go new file mode 100644 index 0000000..8c476c3 --- /dev/null +++ b/internal/template/global.go @@ -0,0 +1,407 @@ +package template + +// The host-wide defaults layer: one clawk.mod outside any repo whose sandbox +// block supplies values for every sandbox created on this machine. It answers +// the "same file over and over" problem (clawkwork/clawk#14) — a kernel path, a +// token alias, a personal skill mount and a house rule are properties of the +// HOST, not of the repo they currently sit in. +// +// It is the lowest layer of the precedence chain: +// +// built-in defaults +// < ~/.config/clawk/clawk.mod (this file) +// < namespace +// < repo clawk.mod +// < clawk.mod. +// < flags +// +// Lists union with the global entries first, so a conflict message reads +// scope-outward; scalars are filled only where nothing narrower declared one. +// It is read at sandbox-create time like every other template, so editing it +// never retro-modifies existing sandboxes. + +import ( + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" +) + +// GlobalModEnvVar names the environment variable that overrides the host-wide +// defaults file outright. Pointing it at a file makes a run reproducible +// regardless of what the host happens to have in ~/.config; pointing it at a +// path that does not exist is an error rather than a silent fall-through, so a +// typo in CI surfaces. +// +// Named for the layer it overrides, so it pairs with --no-global and can't be +// misread as "the repo's clawk.mod", and prefixed like every other variable +// clawk reads (CLAWK_DEBUG, CLAWK_NET_MODE, CLAWK_MAX_VZ_DEVICES) so it +// doesn't collide with an unrelated ROOT_* in someone's shell. +const GlobalModEnvVar = "CLAWK_GLOBAL_MOD" + +// GlobalDisabled skips the host-wide layer entirely — wired to `--no-global`. +// A package var rather than a parameter threaded through nine loader +// signatures: it is written once from PersistentPreRunE before any load, and +// tests set and restore it around a call. +var GlobalDisabled bool + +// ErrNoGlobalMod reports that no host-wide defaults file exists. Not a +// failure — the overwhelmingly common case is a host that never wrote one. +var ErrNoGlobalMod = errors.New("no host-wide clawk.mod") + +// ErrGlobalMod marks every OTHER host-wide-layer failure: unreadable, +// unparseable, out-of-scope directive, two candidate locations. Callers that +// try loaders in sequence (see the cli's resolveSource, which walks workspace → +// standalone → bare-git-repo and treats a failure as "not this shape") must +// test for it and surface it instead of moving on: a broken defaults file is +// not a hint to try somewhere else, and degrading to "no defaults" would hand +// back a sandbox quietly missing half its configuration. +var ErrGlobalMod = errors.New("host-wide clawk.mod") + +// GlobalModPath resolves the host-wide defaults file, in order: +// +// $CLAWK_GLOBAL_MOD explicit override (must exist) +// $XDG_CONFIG_HOME/clawk/clawk.mod default ~/.config/clawk/clawk.mod +// ~/.clawk/clawk.mod compatibility fallback +// +// ~/.config is the primary home because this file is the one thing in clawk's +// footprint a user hand-edits, symlinks out of a dotfiles repo and would be +// annoyed to lose: ~/.clawk is disposable machine state (VM disks, an image +// cache, per-sandbox records, a live OAuth token) that people exclude from +// backups and delete to start clean. Config must not be collateral. +// +// Deliberately NOT os.UserConfigDir(): on darwin that is +// ~/Library/Application Support, which is the wrong place for a +// dotfile-managed text file. Honouring $XDG_CONFIG_HOME with a ~/.config +// fallback on every platform is what gh, git and nvim do on macOS, and what +// anyone writing this file expects. +// +// Both non-env locations present is an error, never a silent precedence pick. +func GlobalModPath() (string, error) { + if p := os.Getenv(GlobalModEnvVar); p != "" { + expanded, err := ExpandPath(p) + if err != nil { + return "", err + } + abs, err := filepath.Abs(expanded) + if err != nil { + return "", err + } + if !fileExists(abs) { + return "", fmt.Errorf("%s=%s: no such file", GlobalModEnvVar, p) + } + return abs, nil + } + + xdg := os.Getenv("XDG_CONFIG_HOME") + if xdg == "" { + home, err := os.UserHomeDir() + if err != nil { + return "", ErrNoGlobalMod + } + xdg = filepath.Join(home, ".config") + } + primary := filepath.Join(xdg, "clawk", RepoFileName) + + var legacy string + if home, err := os.UserHomeDir(); err == nil { + legacy = filepath.Join(home, ".clawk", RepoFileName) + } + + switch { + case fileExists(primary) && legacy != "" && fileExists(legacy): + return "", fmt.Errorf( + "two host-wide clawk.mod files: %s and %s — keep one (%s is the documented location) or set %s", + primary, legacy, primary, GlobalModEnvVar) + case fileExists(primary): + return primary, nil + case legacy != "" && fileExists(legacy): + return legacy, nil + } + return "", ErrNoGlobalMod +} + +// Global is a loaded host-wide defaults layer. +type Global struct { + // Path is the file it came from, for the note printed at create. + Path string + // Template is the file's sandbox block with every host-side path made + // absolute against Path's directory (see absolutiseHostPaths), so it can + // be merged under a repo template that resolves paths against its own + // root. + Template *Template + // Policies are `policy ( … )` blocks declared beside it — a + // personal policy library, registered by the create paths exactly like a + // repo's own. + Policies []PolicyDef + // ProfileMatched reports whether a clawk.mod. overlay beside the + // global file existed and was applied, so a profile satisfied only by the + // host-wide layer is not reported as matching nothing. + ProfileMatched bool +} + +// LoadGlobal is LoadGlobalWithProfile with no profile. +func LoadGlobal() (*Global, error) { return LoadGlobalWithProfile("") } + +// LoadGlobalWithProfile loads and validates the host-wide defaults layer, +// applying a clawk.mod. overlay beside it when profile is non-empty. +// Returns ErrNoGlobalMod when there is no such file (or --no-global was +// passed) — callers treat that as "no defaults", not as a failure. +func LoadGlobalWithProfile(profile string) (*Global, error) { + g, err := loadGlobal(profile) + if err != nil && !errors.Is(err, ErrNoGlobalMod) { + // Tagged so a loader ladder can tell "this layer is broken" from + // "this shape doesn't apply here" — see ErrGlobalMod. + return nil, fmt.Errorf("%w: %w", ErrGlobalMod, err) + } + return g, err +} + +// loadGlobal is the body of LoadGlobalWithProfile, split out so every failure +// path gets the ErrGlobalMod tag from one place. +func loadGlobal(profile string) (*Global, error) { + if GlobalDisabled { + return nil, ErrNoGlobalMod + } + path, err := GlobalModPath() + if err != nil { + return nil, err + } + f, err := loadFile(path) + if err != nil { + return nil, err + } + if err := validateGlobalFile(path, f); err != nil { + return nil, err + } + + g := &Global{Path: path, Template: f.Sandbox, Policies: f.Policies} + if g.Template == nil { + // A file carrying only policy blocks is legitimate: a personal policy + // library with no defaults of its own. + g.Template = &Template{} + } + + if profile != "" { + overlayPath := path + "." + profile + overFile, err := maybeParseOverlay(overlayPath) + if err != nil { + return nil, err + } + if overFile != nil { + if err := validateGlobalFile(overlayPath, overFile); err != nil { + return nil, err + } + g.ProfileMatched = true + g.Template.Merge(overFile.Sandbox) + g.Policies = append(g.Policies, overFile.Policies...) + } + } + + // After the overlay merge, so its entries are resolved too — an overlay + // lives beside the file it extends, so one directory covers both. + absolutiseHostPaths(g.Template, filepath.Dir(path)) + return g, nil +} + +// validateGlobalFile enforces the global scope: the file describes defaults +// for ANY sandbox, so anything that identifies a particular one is rejected by +// name rather than silently ignored. Mirrors how parseNamespaceBlock rejects +// sandbox-level directives and rejectLifecycleAtWorkspace rejects unwired +// hooks — a third scope in the same family. +func validateGlobalFile(path string, f *File) error { + if len(f.Namespaces) > 0 { + return fmt.Errorf( + "%s: 'namespace' blocks are not accepted in the host-wide clawk.mod — "+ + "it declares defaults for every sandbox, not named resources, and clawk "+ + "owns the namespace records itself (a hand-edited copy here would be overwritten)", + path) + } + tmpl := f.Sandbox + if tmpl == nil { + return nil + } + switch { + case tmpl.Name != "": + // The header name is the repo/phase label (it lands in Repo.Name and + // names worktrees), not the sandbox's name — so a name here would + // silently relabel every repo on the host. + return fmt.Errorf( + "%s: the host-wide clawk.mod's sandbox block must be anonymous — "+ + "write `sandbox ( … )`; a header name labels one repo's phases, which is meaningless for defaults", + path) + case len(tmpl.Includes) > 0: + return fmt.Errorf( + "%s: 'includes' declares a workspace root and cannot be host-wide — "+ + "it would pull those repos into every sandbox", + path) + case len(tmpl.OnDown) > 0: + return fmt.Errorf("%s: 'on down' is reserved and wired nowhere yet", path) + case len(tmpl.OnEnter) > 0: + return fmt.Errorf("%s: 'on enter' is reserved and wired nowhere yet", path) + } + return nil +} + +// absolutiseHostPaths rewrites every host-side relative path in tmpl to be +// absolute against dir — the global file's own directory. +// +// This is what lets the layer be merged as a plain base template: once +// resolved, a `files ( ./x )` or `agent ( instructions ./house-rules.md )` +// entry no longer needs to remember which file declared it, so the downstream +// compose steps (which resolve relative paths against the repo root, or the +// process CWD) reach the right file with no provenance plumbing. +// +// Guest-side paths, ~ / $HOME prefixes and URLs are left alone; expanding ~ +// here would hide the user's spelling from error messages for no gain. +func absolutiseHostPaths(tmpl *Template, dir string) { + if tmpl == nil { + return + } + for i := range tmpl.Files { + tmpl.Files[i].HostPath = absHostPath(dir, tmpl.Files[i].HostPath) + } + for i := range tmpl.Shares { + tmpl.Shares[i].HostPath = absHostPath(dir, tmpl.Shares[i].HostPath) + } + for i := range tmpl.Serials { + tmpl.Serials[i].HostPath = absHostPath(dir, tmpl.Serials[i].HostPath) + } + for i := range tmpl.Instructions { + tmpl.Instructions[i].Path = absHostPath(dir, tmpl.Instructions[i].Path) + } + for i := range tmpl.Memory { + tmpl.Memory[i].Path = absHostPath(dir, tmpl.Memory[i].Path) + } + // `vm ( kernel … )` takes a path OR an http(s) URL; only the former moves. + if !isURL(tmpl.Kernel) { + tmpl.Kernel = absHostPath(dir, tmpl.Kernel) + } +} + +// absHostPath makes p absolute against dir, leaving empty values, absolute +// paths and home-relative spellings (~, $HOME) untouched. +func absHostPath(dir, p string) string { + if p == "" || filepath.IsAbs(p) { + return p + } + if strings.HasPrefix(p, "~") || strings.HasPrefix(p, "$HOME") { + return p + } + return filepath.Join(dir, p) +} + +// isURL reports whether s is an http(s) URL (a kernel may be either). +func isURL(s string) bool { + u, err := url.Parse(s) + return err == nil && (u.Scheme == "http" || u.Scheme == "https") +} + +// foldGlobalUnderOnlyRepo layers the global defaults under the sole repo's +// clawk.mod, so the repo's scalars win and its list entries follow the global +// ones. Repo.Clawkfile may be nil — a repo with no clawk.mod is exactly the +// case the host-wide layer exists for — and then the layer becomes its whole +// template. +// +// Only correct for a single-repo workspace: folding into N repos would +// duplicate agent instructions and run every global hook once per phase. See +// foldGlobalIntoWorkspace for the multi-repo shape. +func foldGlobalUnderOnlyRepo(ws *Workspace, g *Global) { + if len(ws.Repos) != 1 { + return + } + base := g.Template.Clone() + base.Merge(ws.Repos[0].Clawkfile) + ws.Repos[0].Clawkfile = base + // A repo whose name came from its own block header keeps it; the global + // file is required to be anonymous, so there is nothing to inherit. +} + +// foldGlobalIntoWorkspace layers the global defaults under a workspace root's +// own sandbox block. +// +// The workspace position is the right one for a multi-repo sandbox: its +// `files`/`shares`/`env` compose once for the VM and its `on up` / `on create` +// run at the guest workspace root, whereas folding the layer into each repo +// would duplicate agent instructions and run every hook once per phase. +// +// The catch is precedence. resolveProvider / resolveImage / resolveKernel and +// the resource resolvers consult ws.File FIRST, by design: a workspace root +// exists to settle disagreements between its repos. A default must not inherit +// that authority, so any scalar that arrived from the global layer is dropped +// again when some repo declares its own — leaving the global value to apply +// only where nothing narrower did. +// +// That decision is made against the FULL repo list, before any later +// FilterRepos (`--only`). So a global scalar yielded to a repo that a subsequent +// --only excludes stays yielded, and the provider default applies instead of +// the host's. Deliberate: re-deriving it per selection would make one repo's +// resolved shape depend on which siblings came along. +func foldGlobalIntoWorkspace(ws *Workspace, g *Global) { + own := ws.File // what the workspace file (plus its profile overlay) declared + base := g.Template.Clone() + base.Merge(own) + ws.File = base + + if own.Provider == "" && anyRepo(ws, func(t *Template) bool { return t.Provider != "" }) { + ws.File.Provider = "" + } + if own.Image == "" && anyRepo(ws, func(t *Template) bool { return t.Image != "" }) { + ws.File.Image = "" + } + if own.Kernel == "" && anyRepo(ws, func(t *Template) bool { return t.Kernel != "" }) { + ws.File.Kernel = "" + } + if own.CPU == 0 && anyRepo(ws, func(t *Template) bool { return t.CPU != 0 }) { + ws.File.CPU = 0 + } + if own.MemoryMiB == 0 && anyRepo(ws, func(t *Template) bool { return t.MemoryMiB != 0 }) { + ws.File.MemoryMiB = 0 + } + if own.MemoryMaxMiB == 0 && anyRepo(ws, func(t *Template) bool { return t.MemoryMaxMiB != 0 }) { + ws.File.MemoryMaxMiB = 0 + } + if own.DiskMiB == 0 && anyRepo(ws, func(t *Template) bool { return t.DiskMiB != 0 }) { + ws.File.DiskMiB = 0 + } + if own.SwapMiB == 0 && anyRepo(ws, func(t *Template) bool { return t.SwapMiB != 0 }) { + ws.File.SwapMiB = 0 + } + if own.IdleTimeoutSec == 0 && anyRepo(ws, func(t *Template) bool { return t.IdleTimeoutSec != 0 }) { + ws.File.IdleTimeoutSec = 0 + } + // Nested is deliberately left as the union: it is opt-in-only everywhere + // (there is no `nested false`), so a host that asks for it gets it. +} + +// anyRepo reports whether some repo's clawk.mod satisfies pred. +func anyRepo(ws *Workspace, pred func(*Template) bool) bool { + for _, r := range ws.Repos { + if r.Clawkfile != nil && pred(r.Clawkfile) { + return true + } + } + return false +} + +// attachGlobal loads the host-wide layer and hands it to fold, recording the +// path on the workspace for the create-time note. A missing file is not an +// error; anything else (unreadable, unparseable, out-of-scope directive) is — +// silently ignoring a broken defaults file would leave the user staring at a +// sandbox that quietly lacks half its configuration. +func attachGlobal(ws *Workspace, profile string, fold func(*Workspace, *Global)) error { + g, err := LoadGlobalWithProfile(profile) + if errors.Is(err, ErrNoGlobalMod) { + return nil + } + if err != nil { + return err + } + fold(ws, g) + ws.GlobalPath = g.Path + ws.GlobalProfileMatched = g.ProfileMatched + ws.Policies = append(g.Policies, ws.Policies...) + return nil +} diff --git a/internal/template/global_test.go b/internal/template/global_test.go new file mode 100644 index 0000000..7defd6f --- /dev/null +++ b/internal/template/global_test.go @@ -0,0 +1,371 @@ +package template + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// emptySandbox is the smallest valid global file: a sandbox block declaring +// nothing. Used by the path-resolution tests, which only care about location. +const emptySandbox = "sandbox (\n)\n" + +// withGlobalMod writes src as the host-wide clawk.mod inside a fresh +// XDG_CONFIG_HOME and enables the layer for the duration of the test. Returns +// the file's path. HOME is redirected too, so the ~/.clawk fallback can never +// resolve to the developer's real one. +func withGlobalMod(t *testing.T, src string) string { + t.Helper() + home := t.TempDir() + xdg := filepath.Join(home, ".config") + require.NoError(t, os.MkdirAll(filepath.Join(xdg, "clawk"), 0o755)) + path := filepath.Join(xdg, "clawk", RepoFileName) + require.NoError(t, os.WriteFile(path, []byte(src), 0o644)) + + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", xdg) + t.Setenv(GlobalModEnvVar, "") + + prev := GlobalDisabled + GlobalDisabled = false + t.Cleanup(func() { GlobalDisabled = prev }) + return path +} + +// newRepo creates an initialised git repo with an optional clawk.mod. +func newRepo(t *testing.T, name, clawkmod string) string { + t.Helper() + repo := filepath.Join(t.TempDir(), name) + require.NoError(t, os.MkdirAll(repo, 0o755)) + initRepo(t, repo) + if clawkmod != "" { + require.NoError(t, os.WriteFile( + filepath.Join(repo, RepoFileName), []byte(clawkmod), 0o644)) + } + return repo +} + +func TestGlobalModPathPrecedence(t *testing.T) { + t.Run("env var wins", func(t *testing.T) { + xdgPath := withGlobalMod(t, emptySandbox) + explicit := filepath.Join(t.TempDir(), "custom.mod") + require.NoError(t, os.WriteFile(explicit, []byte(emptySandbox), 0o644)) + t.Setenv(GlobalModEnvVar, explicit) + + got, err := GlobalModPath() + require.NoError(t, err) + require.Equal(t, explicit, got) + require.NotEqual(t, xdgPath, got) + }) + + t.Run("env var pointing nowhere is an error", func(t *testing.T) { + withGlobalMod(t, emptySandbox) + t.Setenv(GlobalModEnvVar, filepath.Join(t.TempDir(), "absent.mod")) + + _, err := GlobalModPath() + require.ErrorContains(t, err, GlobalModEnvVar) + require.ErrorContains(t, err, "no such file") + }) + + t.Run("xdg location", func(t *testing.T) { + path := withGlobalMod(t, emptySandbox) + got, err := GlobalModPath() + require.NoError(t, err) + require.Equal(t, path, got) + }) + + t.Run("legacy ~/.clawk fallback", func(t *testing.T) { + xdgPath := withGlobalMod(t, emptySandbox) + require.NoError(t, os.Remove(xdgPath)) + + legacy := filepath.Join(os.Getenv("HOME"), ".clawk", RepoFileName) + require.NoError(t, os.MkdirAll(filepath.Dir(legacy), 0o755)) + require.NoError(t, os.WriteFile(legacy, []byte(emptySandbox), 0o644)) + + got, err := GlobalModPath() + require.NoError(t, err) + require.Equal(t, legacy, got) + }) + + t.Run("both locations is an error, not a silent pick", func(t *testing.T) { + withGlobalMod(t, emptySandbox) + legacy := filepath.Join(os.Getenv("HOME"), ".clawk", RepoFileName) + require.NoError(t, os.MkdirAll(filepath.Dir(legacy), 0o755)) + require.NoError(t, os.WriteFile(legacy, []byte(emptySandbox), 0o644)) + + _, err := GlobalModPath() + require.ErrorContains(t, err, "two host-wide clawk.mod files") + }) + + t.Run("no file at all", func(t *testing.T) { + p := withGlobalMod(t, emptySandbox) + require.NoError(t, os.Remove(p)) + + _, err := GlobalModPath() + require.ErrorIs(t, err, ErrNoGlobalMod) + }) +} + +func TestLoadGlobalScopeRejections(t *testing.T) { + cases := []struct { + name, src, want string + }{ + { + name: "named sandbox block", + src: "sandbox house (\n vm (\n cpu 2\n )\n)\n", + want: "must be anonymous", + }, + { + name: "includes", + src: "sandbox (\n includes (\n ~/code/a\n )\n)\n", + want: "cannot be host-wide", + }, + { + name: "namespace block", + src: "namespace work (\n env (\n TOKEN\n )\n)\n", + want: "not accepted in the host-wide clawk.mod", + }, + { + name: "unwired lifecycle hook", + src: "sandbox (\n on down (\n \"echo bye\"\n )\n)\n", + want: "reserved", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + withGlobalMod(t, tc.src) + _, err := LoadGlobal() + require.ErrorContains(t, err, tc.want) + }) + } +} + +func TestLoadGlobalAbsolutisesHostPaths(t *testing.T) { + path := withGlobalMod(t, `sandbox ( + vm ( + kernel ./kernels/vmlinux + ) + files ( + ./house.netrc /home/agent/.netrc + ) + shares ( + ~/.aws + ) + agent ( + instructions ./house-rules.md + ) +) +`) + dir := filepath.Dir(path) + + g, err := LoadGlobal() + require.NoError(t, err) + require.Equal(t, filepath.Join(dir, "kernels", "vmlinux"), g.Template.Kernel) + require.Equal(t, filepath.Join(dir, "house.netrc"), g.Template.Files[0].HostPath) + require.Equal(t, filepath.Join(dir, "house-rules.md"), g.Template.Instructions[0].Path) + // A home-relative spelling is the user's own and stays verbatim, so error + // messages echo what they wrote. + require.Equal(t, "~/.aws", g.Template.Shares[0].HostPath) +} + +func TestLoadGlobalLeavesKernelURLAlone(t *testing.T) { + withGlobalMod(t, "sandbox (\n vm (\n kernel https://example.com/vmlinux\n )\n)\n") + g, err := LoadGlobal() + require.NoError(t, err) + require.Equal(t, "https://example.com/vmlinux", g.Template.Kernel) +} + +func TestLoadGlobalPolicyOnlyFile(t *testing.T) { + withGlobalMod(t, "policy house (\n allow example.com\n)\n") + g, err := LoadGlobal() + require.NoError(t, err) + require.NotNil(t, g.Template) + require.Len(t, g.Policies, 1) + require.Equal(t, "house", g.Policies[0].Name) +} + +func TestGlobalDisabledSkipsTheLayer(t *testing.T) { + withGlobalMod(t, "sandbox (\n vm (\n cpu 8\n )\n)\n") + GlobalDisabled = true + + _, err := LoadGlobal() + require.ErrorIs(t, err, ErrNoGlobalMod) +} + +// The single-repo shape: the layer folds under the repo's own clawk.mod, so the +// repo wins scalars and its list entries follow the global ones. +func TestGlobalUnderStandaloneRepo(t *testing.T) { + globalPath := withGlobalMod(t, `sandbox ( + vm ( + cpu 2 + memory_max 8GiB + provider vz + ) + network ( + allow global.example.com + ) + env ( + GITHUB_TOKEN = ${HOST_TOKEN} + ) + shares ( + ~/.claude/skills/idiomatic-go + ) +) +`) + repo := newRepo(t, "proj", `sandbox ( + vm ( + cpu 4 + ) + network ( + allow repo.example.com + ) +) +`) + + ws, err := LoadStandaloneClawkfile(repo) + require.NoError(t, err) + require.Equal(t, globalPath, ws.GlobalPath) + + tmpl := ws.Repos[0].Clawkfile + require.EqualValues(t, 4, tmpl.CPU, "repo cpu must beat the global default") + require.EqualValues(t, 8192, tmpl.MemoryMaxMiB, "global memory applies where the repo is silent") + require.Equal(t, "vz", tmpl.Provider) + require.Equal(t, []string{"global.example.com", "repo.example.com"}, tmpl.Domains, + "lists union with the global entries first") + require.Equal(t, []string{"GITHUB_TOKEN=${HOST_TOKEN}"}, tmpl.Env) + require.Len(t, tmpl.Shares, 1) + // The repo's own name is untouched by the (necessarily anonymous) layer. + require.Equal(t, "proj", ws.Repos[0].Name) +} + +func TestGlobalIsWholeTemplateForRepoWithoutClawkMod(t *testing.T) { + withGlobalMod(t, `sandbox ( + vm ( + cpu 3 + ) + network ( + allow global.example.com + ) +) +`) + repo := newRepo(t, "bare", "") + + ws, err := WorkspaceFromGitRepo(repo) + require.NoError(t, err) + require.NotNil(t, ws.Repos[0].Clawkfile) + require.EqualValues(t, 3, ws.Repos[0].Clawkfile.CPU) + require.Equal(t, []string{"global.example.com"}, ws.Repos[0].Clawkfile.Domains) +} + +// In a multi-repo workspace the layer sits at the workspace position, but must +// not inherit the workspace file's authority to settle repo disagreements. +func TestGlobalScalarsDemotedBehindRepos(t *testing.T) { + withGlobalMod(t, `sandbox ( + vm ( + image golang:1.25 + cpu 2 + memory_max 8GiB + ) +) +`) + + root := t.TempDir() + repo := filepath.Join(root, "svc") + require.NoError(t, os.MkdirAll(repo, 0o755)) + initRepo(t, repo) + require.NoError(t, os.WriteFile(filepath.Join(repo, RepoFileName), []byte(`sandbox ( + vm ( + image node:22 + cpu 6 + ) +) +`), 0o644)) + + wsPath := filepath.Join(root, RepoFileName) + require.NoError(t, os.WriteFile(wsPath, + []byte("sandbox (\n includes (\n ./svc\n )\n)\n"), 0o644)) + + ws, err := LoadWorkspace(wsPath) + require.NoError(t, err) + require.Empty(t, ws.File.Image, "a global image must not outrank the repo's own") + require.Zero(t, ws.File.CPU, "a global cpu must not outrank the repo's own") + require.EqualValues(t, 8192, ws.File.MemoryMaxMiB, + "but it still applies where no repo declared one") +} + +func TestGlobalAtWorkspacePositionUnderTheWorkspaceFile(t *testing.T) { + withGlobalMod(t, `sandbox ( + vm ( + memory_max 8GiB + ) + network ( + allow global.example.com + ) + env ( + GLOBAL_TOKEN + ) + on up ( + "global-hook" + ) +) +`) + root := t.TempDir() + repo := filepath.Join(root, "svc") + require.NoError(t, os.MkdirAll(repo, 0o755)) + initRepo(t, repo) + + wsPath := filepath.Join(root, RepoFileName) + require.NoError(t, os.WriteFile(wsPath, []byte(`sandbox ( + includes ( + ./svc + ) + vm ( + memory_max 16GiB + ) + on up ( + "workspace-hook" + ) +) +`), 0o644)) + + ws, err := LoadWorkspace(wsPath) + require.NoError(t, err) + require.EqualValues(t, 16384, ws.File.MemoryMaxMiB, "the workspace file wins") + require.Equal(t, []string{"global.example.com"}, ws.File.Domains) + require.Equal(t, []string{"GLOBAL_TOKEN"}, ws.File.Env) + require.Equal(t, []string{"global-hook", "workspace-hook"}, ws.File.OnUp, + "the broader scope's hooks run first") +} + +func TestGlobalProfileOverlay(t *testing.T) { + path := withGlobalMod(t, "sandbox (\n network (\n allow base.example.com\n )\n)\n") + require.NoError(t, os.WriteFile(path+".investigation", + []byte("sandbox (\n network (\n allow deep.example.com\n )\n)\n"), 0o644)) + + repo := newRepo(t, "proj", emptySandbox) + + // The repo has no overlay of its own: the profile is satisfied entirely by + // the host-wide layer, which must count as a match rather than erroring. + ws, err := LoadStandaloneClawkfileWithProfile(repo, "investigation") + require.NoError(t, err) + require.Equal(t, []string{"base.example.com", "deep.example.com"}, + ws.Repos[0].Clawkfile.Domains) + + // An unknown profile still fails loudly. + _, err = LoadStandaloneClawkfileWithProfile(repo, "nope") + require.ErrorContains(t, err, "nope") +} + +func TestGlobalParseErrorSurfaces(t *testing.T) { + withGlobalMod(t, "sandbox (\n vm (\n cpu\n )\n)\n") + repo := newRepo(t, "proj", emptySandbox) + + // A broken defaults file must not degrade to "no defaults" — that would + // leave the user with a sandbox quietly missing half its configuration. + _, err := LoadStandaloneClawkfile(repo) + require.Error(t, err) + require.True(t, strings.Contains(err.Error(), "cpu"), "got %v", err) +} diff --git a/internal/template/main_test.go b/internal/template/main_test.go new file mode 100644 index 0000000..75ee7ce --- /dev/null +++ b/internal/template/main_test.go @@ -0,0 +1,16 @@ +package template + +import ( + "os" + "testing" +) + +// TestMain disables the host-wide clawk.mod for the whole package. Without +// this, every loader test would silently pick up whatever the developer (or +// CI runner) has in ~/.config/clawk/clawk.mod and assert against a moving +// target. Tests that mean to exercise the layer re-enable it through +// withGlobalMod (see global_test.go). +func TestMain(m *testing.M) { + GlobalDisabled = true + os.Exit(m.Run()) +} diff --git a/internal/template/mcp_test.go b/internal/template/mcp_test.go new file mode 100644 index 0000000..bd94f60 --- /dev/null +++ b/internal/template/mcp_test.go @@ -0,0 +1,155 @@ +package template + +import ( + "testing" + + "github.com/clawkwork/clawk/internal/config" + "github.com/stretchr/testify/require" +) + +// TestParseMCPBlock covers the three line shapes an `mcp ( … )` entry can +// take, plus the repeatable modifiers. The bare-URL form is the one users +// write most, so its http default is pinned explicitly. +func TestParseMCPBlock(t *testing.T) { + src := `mcp ( + linear https://mcp.linear.app/mcp header "Authorization: Bearer ${LINEAR_TOKEN}" + sentry sse https://mcp.sentry.dev/sse + corridor http https://app.corridor.dev/api/mcp header "X-Api-Key: ${CORRIDOR_KEY}" header "X-Env: prod" + github stdio "npx -y @modelcontextprotocol/server-github" env GITHUB_TOKEN +) +` + tmpl, err := parseBody(src) + require.NoError(t, err) + require.Len(t, tmpl.MCP, 4) + + linear := tmpl.MCP[0] + require.Equal(t, "linear", linear.Name) + require.Equal(t, config.MCPTransportHTTP, linear.Transport, "a bare URL defaults to http") + require.Equal(t, "https://mcp.linear.app/mcp", linear.URL) + require.Equal(t, []string{"Authorization: Bearer ${LINEAR_TOKEN}"}, linear.Headers) + + require.Equal(t, config.MCPTransportSSE, tmpl.MCP[1].Transport) + require.Equal(t, "https://mcp.sentry.dev/sse", tmpl.MCP[1].URL) + + corridor := tmpl.MCP[2] + require.Equal(t, config.MCPTransportHTTP, corridor.Transport) + require.Equal(t, []string{"X-Api-Key: ${CORRIDOR_KEY}", "X-Env: prod"}, corridor.Headers, + "header is repeatable and order-preserving") + + gh := tmpl.MCP[3] + require.Equal(t, config.MCPTransportStdio, gh.Transport) + require.Equal(t, []string{"npx", "-y", "@modelcontextprotocol/server-github"}, gh.Command) + require.Equal(t, []string{"GITHUB_TOKEN"}, gh.Env) + require.Empty(t, gh.URL) +} + +// TestParseMCPKeepsCredentialReferencesLiteral is the invariant that keeps +// secrets off host disk: a ${VAR} in a header is stored as written, never +// resolved at parse time. The runner expands it inside the guest. +func TestParseMCPKeepsCredentialReferencesLiteral(t *testing.T) { + t.Setenv("LINEAR_TOKEN", "should-not-be-read") + tmpl, err := parseBody(`mcp ( + linear https://mcp.linear.app/mcp header "Authorization: Bearer ${LINEAR_TOKEN}" +) +`) + require.NoError(t, err) + require.Equal(t, []string{"Authorization: Bearer ${LINEAR_TOKEN}"}, tmpl.MCP[0].Headers) +} + +func TestParseMCPErrors(t *testing.T) { + cases := []struct { + name string + src string + want string + }{ + { + name: "no target", + src: "mcp (\n linear\n)\n", + want: "expected a URL or a transport", + }, + { + name: "non-http scheme", + src: "mcp (\n linear ftp://mcp.linear.app/mcp\n)\n", + want: "must use http or https", + }, + { + name: "url without host", + src: "mcp (\n linear https:///mcp\n)\n", + want: "has no host", + }, + { + name: "stdio without quoted command", + src: "mcp (\n github stdio npx\n)\n", + want: "expected a quoted command", + }, + { + name: "header without colon", + src: "mcp (\n linear https://mcp.linear.app/mcp header \"Authorization Bearer x\"\n)\n", + want: "want \"Name: value\"", + }, + { + name: "header on stdio server", + src: "mcp (\n github stdio \"npx server\" header \"X: y\"\n)\n", + want: "'header' applies to http/sse servers", + }, + { + name: "env on remote server", + src: "mcp (\n linear https://mcp.linear.app/mcp env LINEAR_TOKEN\n)\n", + want: "'env' applies to stdio servers", + }, + { + name: "invalid env name", + src: "mcp (\n github stdio \"npx server\" env 9BAD\n)\n", + want: "9BAD", + }, + { + name: "invalid server name", + src: "mcp (\n my:server https://mcp.linear.app/mcp\n)\n", + want: "invalid mcp server name", + }, + { + name: "trailing junk", + src: "mcp (\n linear https://mcp.linear.app/mcp wat\n)\n", + want: "unexpected token after mcp entry", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := parseBody(tc.src) + require.Error(t, err) + require.Contains(t, err.Error(), tc.want) + }) + } +} + +// TestParseMCPInNamespaceBlock guards the namespace scope: the org-wide +// server set is the whole reason per-sandbox MCP setup can be zero-touch, +// so `mcp ( … )` has to be accepted there too. +func TestParseMCPInNamespaceBlock(t *testing.T) { + f, err := ParseFileString(`namespace acme ( + mcp ( + linear https://mcp.linear.app/mcp header "Authorization: Bearer ${LINEAR_TOKEN}" + ) +) +`) + require.NoError(t, err) + require.Len(t, f.Namespaces, 1) + require.Len(t, f.Namespaces[0].Template.MCP, 1) + require.Equal(t, "linear", f.Namespaces[0].Template.MCP[0].Name) +} + +// A URL with userinfo is the one way a secret VALUE could reach the sandbox +// record and the rendered guest config, both of which live unencrypted on +// host disk — the invariant the whole `${VAR}`-reference design exists to +// hold. Refuse it and name the supported spelling. +func TestMCPURLRejectsEmbeddedCredentials(t *testing.T) { + _, err := parseBody(`mcp ( + sentry https://user:s3cr3t@mcp.sentry.dev/mcp +) +`) + require.Error(t, err) + require.ErrorContains(t, err, "embeds credentials") + require.ErrorContains(t, err, "Authorization: Bearer") + require.NotContains(t, err.Error(), "s3cr3t", + "the error must not echo the secret it is refusing") +} diff --git a/internal/template/parse.go b/internal/template/parse.go index 4e2b83c..eb17dcc 100644 --- a/internal/template/parse.go +++ b/internal/template/parse.go @@ -5,11 +5,13 @@ import ( "fmt" "io/fs" "net/netip" + "net/url" "slices" "strconv" "strings" "time" + "github.com/clawkwork/clawk/internal/config" "github.com/clawkwork/clawk/internal/envspec" ) @@ -35,6 +37,23 @@ type FileSpec struct { Line, Col int } +// SerialSpec is one `serial (...)` entry: a host serial port presented as a +// device inside the guest. +// +// - HostPath: the device on this machine, e.g. /dev/cu.usbmodem1101. May +// be a glob, resolved when the port is opened rather than now. +// - GuestName: the bare name the device appears under in the guest's +// /dev. Empty defaults to the host device's basename — except for a +// glob, which has no basename and must name one explicitly. +type SerialSpec struct { + HostPath string + GuestName string + + // Line/Col record where the entry appeared so duplicate-name conflicts + // can be reported back to the right line. + Line, Col int +} + // ShareSpec is one entry in a `shares (...)` block: a host directory // live-mounted into the guest via virtio-fs. Edits on the host are // visible inside the guest without clawk involvement — the host owns @@ -50,6 +69,32 @@ type ShareSpec struct { Line, Col int } +// MCPSpec is one entry in an `mcp (...)` block: an MCP server the project +// wants available to the agent inside the sandbox. Written as +// +// http transport (the default) +// http|sse explicit remote transport +// stdio " " a local server process +// +// followed by any number of `header "Name: value"` and `env NAME` +// modifiers on the same line. +// +// Credential values never appear here — a `${VAR}` inside a header (or the +// implied `NAME=${NAME}` of an env entry) is kept verbatim and expanded by +// the runner inside the guest. See config.MCPServer. +type MCPSpec struct { + Name string + Transport string // config.MCPTransport* value + URL string + Command []string + Headers []string + Env []string + + // Line/Col record where the entry appeared so a conflict between two + // repos declaring the same server name is reported at the right line. + Line, Col int +} + // AgentDoc is one entry in an `agent (...)` block — a unit of persistent // instructions or memory seed. Exactly one field is set: Text is inline // markdown from a quoted string (one line); Path is a markdown file, relative @@ -129,6 +174,14 @@ type Template struct { // See ShareSpec for semantics. Shares []ShareSpec + // Serials is the forwarded serial-port list (`serial (...)`). + // See SerialSpec for semantics. + Serials []SerialSpec + + // MCP is the declared MCP server list (`mcp (...)`). + // See MCPSpec for semantics. + MCP []MCPSpec + // Instructions and Memory come from the `agent ( ... )` block: extra // persistent CLAUDE.md guidance and a baseline auto-memory seed. Each is // an ordered list of AgentDocs (inline text or a markdown file) resolved @@ -165,6 +218,13 @@ type Template struct { // on the rootfs. DiskMiB uint64 + // SwapMiB is the guest swap device's capacity in mebibytes, declared as + // `swap ` inside the `vm ( ... )` block. Zero = unset (the + // built-in sandbox.DefaultSwapSizeMiB applies); negative = "off", no swap + // device. The device is sparse, so the size bounds how much the guest may + // swap rather than reserving anything up front. + SwapMiB int64 + // IdleTimeoutSec is the sandbox's idle-stop timeout in seconds: how // long the VM may sit with no attached session and a quiescent guest // before the daemon stops it to reclaim host memory. Declared as @@ -224,6 +284,8 @@ func (t *Template) Merge(over *Template) { t.Skills = mergeSkills(t.Skills, over.Skills) t.Files = append(t.Files, over.Files...) t.Shares = append(t.Shares, over.Shares...) + t.Serials = append(t.Serials, over.Serials...) + t.MCP = append(t.MCP, over.MCP...) t.Instructions = append(t.Instructions, over.Instructions...) t.Memory = append(t.Memory, over.Memory...) // Bool fields: OR so a profile can only enable, never disable. @@ -245,11 +307,47 @@ func (t *Template) Merge(over *Template) { if over.DiskMiB != 0 { t.DiskMiB = over.DiskMiB } + if over.SwapMiB != 0 { + t.SwapMiB = over.SwapMiB + } if over.IdleTimeoutSec != 0 { t.IdleTimeoutSec = over.IdleTimeoutSec } } +// Clone returns a deep-enough copy of t for Merge to write into without +// touching the original's backing arrays. Every field is either a scalar or a +// slice of values, so copying the slices is sufficient — used by the host-wide +// defaults layer, which is loaded once and merged under several templates. +func (t *Template) Clone() *Template { + if t == nil { + return &Template{} + } + c := *t + c.Includes = slices.Clone(t.Includes) + c.Domains = slices.Clone(t.Domains) + c.IPs = slices.Clone(t.IPs) + c.Use = slices.Clone(t.Use) + c.DenyDomains = slices.Clone(t.DenyDomains) + c.DenyIPs = slices.Clone(t.DenyIPs) + c.DenySources = slices.Clone(t.DenySources) + c.Forwards = slices.Clone(t.Forwards) + c.ReverseForwards = slices.Clone(t.ReverseForwards) + c.Env = slices.Clone(t.Env) + c.OnCreate = slices.Clone(t.OnCreate) + c.OnUp = slices.Clone(t.OnUp) + c.OnDown = slices.Clone(t.OnDown) + c.OnEnter = slices.Clone(t.OnEnter) + c.Skills = slices.Clone(t.Skills) + c.Files = slices.Clone(t.Files) + c.Shares = slices.Clone(t.Shares) + c.Serials = slices.Clone(t.Serials) + c.MCP = slices.Clone(t.MCP) + c.Instructions = slices.Clone(t.Instructions) + c.Memory = slices.Clone(t.Memory) + return &c +} + type parser struct { toks []Token i int @@ -309,6 +407,10 @@ func (p *parser) parseTemplateDirective(tmpl *Template, t Token) error { return p.parseFilesBlock(tmpl) case "shares": return p.parseSharesBlock(tmpl) + case "serial": + return p.parseSerialBlock(tmpl) + case "mcp": + return p.parseMCPBlock(tmpl) case "agent": return p.parseAgentBlock(tmpl) case "env": @@ -336,8 +438,8 @@ func (p *parser) parseTemplateDirective(tmpl *Template, t Token) error { return p.errorAt(t, "unknown directive %q (want %s)", t.Val, describeFirst([]string{ - "vm", "network", "forwards", "files", "shares", - "agent", "skills", "on", "includes", "env", + "vm", "network", "forwards", "files", "shares", "serial", + "mcp", "agent", "skills", "on", "includes", "env", })) } } @@ -699,6 +801,44 @@ func (p *parser) parseIdleTimeout(tmpl *Template) error { return p.expectNewlineOrEOF() } +// parseSwap handles `swap ` inside the vm block — the capacity of +// the guest's swap device. Sizes use the same units as `memory` and `disk`; +// "off" and "0" disable swap entirely (stored as -1, so "explicitly off" +// stays distinguishable from "unset", exactly as idle_timeout does). +// +// A floor of 64 MiB rejects the unit typo `swap 512M` meant as 512 GiB less +// harshly than it rejects `swap 4` — anything smaller is too little to +// absorb a balloon inflation and is almost certainly a mistake. +func (p *parser) parseSwap(tmpl *Template) error { + p.advance() // consume "swap" + val := p.peek() + if val.Kind != TokIdent { + return p.errorAt(val, "expected size or 'off' after 'swap', got %s", val) + } + if tmpl.SwapMiB != 0 { + return p.errorAt(val, "duplicate 'swap' directive") + } + switch val.Val { + case "off", "0": + tmpl.SwapMiB = -1 + default: + mib, err := parseMiB(val.Val) + if err != nil { + return p.errorAt(val, "%q: %v", "swap", err) + } + if mib < minSwapMiB { + return p.errorAt(val, + "swap %q too small: minimum %d MiB (use 'off' to disable)", val.Val, minSwapMiB) + } + tmpl.SwapMiB = int64(mib) + } + p.advance() + return p.expectNewlineOrEOF() +} + +// minSwapMiB is the floor for an explicit `vm ( swap )`. +const minSwapMiB = 64 + // parseNested handles the bare `nested` directive. It takes no value — // setting Template.Nested true and requiring the line to end immediately // keeps the syntax unambiguous and matches how Go's `go 1.x` line works @@ -751,6 +891,10 @@ func (p *parser) parseVMBlock(tmpl *Template) error { if err := p.parseMemory(&tmpl.DiskMiB, "disk"); err != nil { return err } + case "swap": + if err := p.parseSwap(tmpl); err != nil { + return err + } case "nested": if err := p.parseNested(tmpl); err != nil { return err @@ -771,7 +915,7 @@ func (p *parser) parseVMBlock(tmpl *Template) error { return p.errorAt(t, "unknown 'vm' directive %q (want %s)", t.Val, describeFirst([]string{ - "provider", "cpu", "memory", "memory_max", "disk", "nested", "idle_timeout", "image", "kernel", + "provider", "cpu", "memory", "memory_max", "disk", "swap", "nested", "idle_timeout", "image", "kernel", })) } } @@ -1064,6 +1208,53 @@ func (p *parser) parseFilesBlock(tmpl *Template) error { } } +// parseSerialBlock handles `serial ( [] ... )`. +// Each entry is one or two whitespace-separated tokens on a line: +// +// serial ( +// /dev/cu.usbmodem1101 # same name inside the guest +// /dev/cu.usbserial-A50285BI ttyUSB0 +// /dev/cu.usbmodem* ttyACM0 # resolved when the port is opened +// ) +// +// Space-separated rather than the CLI's HOST:GUEST spelling, matching +// `files` and `shares`: these are paths, and a colon inside one would be +// ambiguous in a way a port number never is. +// +// Nothing is validated here beyond the shape — internal/cli turns these +// into config.SerialDevice and checks the names, the same division of +// labour as the other resource blocks. +func (p *parser) parseSerialBlock(tmpl *Template) error { + p.advance() // consume "serial" + t := p.peek() + if t.Kind != TokLParen { + return p.errorAt(t, "expected '(' after 'serial', got %s", t) + } + p.advance() + for { + p.skipNewlines() + t := p.peek() + if t.Kind == TokRParen { + p.advance() + return p.expectNewlineOrEOF() + } + if t.Kind != TokIdent { + return p.errorAt(t, "expected serial device path or ')', got %s", t) + } + spec := SerialSpec{HostPath: t.Val, Line: t.Line, Col: t.Col} + p.advance() + if nx := p.peek(); nx.Kind == TokIdent { + spec.GuestName = nx.Val + p.advance() + } + if next := p.peek(); next.Kind != TokNewline && next.Kind != TokRParen && next.Kind != TokEOF { + return p.errorAt(next, + "unexpected token after serial entry; one host device and optional guest name per line") + } + tmpl.Serials = append(tmpl.Serials, spec) + } +} + // parseSharesBlock handles `shares ( [] [ro|rw] ... )`. // Mount points default to ReadOnly = true: the host owns rotation for // the use cases we built this for (`~/.aws`), and an accidental in-VM @@ -1125,6 +1316,212 @@ func (p *parser) parseSharesBlock(tmpl *Template) error { } } +// parseMCPBlock handles `mcp ( [http|sse|stdio] [modifiers] ... )`. +// See MCPSpec for the accepted line shapes. +// +// The transport keyword is optional in the common case: a target that looks +// like an http(s) URL implies the http transport, which keeps the frequent +// one-line remote declaration short. stdio always needs its keyword, since a +// bare command word would otherwise be ambiguous with a server name. +func (p *parser) parseMCPBlock(tmpl *Template) error { + p.advance() // consume "mcp" + t := p.peek() + if t.Kind != TokLParen { + return p.errorAt(t, "expected '(' after 'mcp', got %s", t) + } + p.advance() + for { + p.skipNewlines() + t := p.peek() + if t.Kind == TokRParen { + p.advance() + return p.expectNewlineOrEOF() + } + if t.Kind != TokIdent { + return p.errorAt(t, "expected MCP server name or ')', got %s", t) + } + spec := MCPSpec{Name: t.Val, Line: t.Line, Col: t.Col} + if err := validateMCPName(spec.Name); err != nil { + return p.errorAt(t, "%v", err) + } + p.advance() + + if err := p.parseMCPTarget(&spec); err != nil { + return err + } + if err := p.parseMCPModifiers(&spec); err != nil { + return err + } + if next := p.peek(); next.Kind != TokNewline && next.Kind != TokRParen && next.Kind != TokEOF { + return p.errorAt(next, "unexpected token after mcp entry %q", spec.Name) + } + tmpl.MCP = append(tmpl.MCP, spec) + } +} + +// parseMCPTarget reads the transport and endpoint that follow a server name: +// an explicit `http|sse ` / `stdio ""`, or a bare URL taking +// the http default. +func (p *parser) parseMCPTarget(spec *MCPSpec) error { + t := p.peek() + if t.Kind != TokIdent { + return p.errorAt(t, + "expected a URL or a transport (http, sse, stdio) after mcp server %q, got %s", + spec.Name, t) + } + switch t.Val { + case config.MCPTransportHTTP, config.MCPTransportSSE: + spec.Transport = t.Val + p.advance() + return p.parseMCPURL(spec) + case config.MCPTransportStdio: + spec.Transport = config.MCPTransportStdio + p.advance() + cmd := p.peek() + if cmd.Kind != TokString { + return p.errorAt(cmd, + "expected a quoted command after 'stdio' for mcp server %q, got %s", + spec.Name, cmd) + } + argv := strings.Fields(cmd.Val) + if len(argv) == 0 { + return p.errorAt(cmd, "empty stdio command for mcp server %q", spec.Name) + } + spec.Command = argv + p.advance() + return nil + default: + // No transport keyword: the token must be the URL itself. + spec.Transport = config.MCPTransportHTTP + return p.parseMCPURL(spec) + } +} + +// parseMCPURL consumes and validates the endpoint of an http/sse server. +func (p *parser) parseMCPURL(spec *MCPSpec) error { + t := p.peek() + if t.Kind != TokIdent { + return p.errorAt(t, "expected a URL for mcp server %q, got %s", spec.Name, t) + } + if err := validateMCPURL(t.Val); err != nil { + return p.errorAt(t, "mcp server %q: %v", spec.Name, err) + } + spec.URL = t.Val + p.advance() + return nil +} + +// parseMCPModifiers reads the repeatable `header "..."` / `env NAME` trailers +// of one mcp entry, stopping at the end of the line. +func (p *parser) parseMCPModifiers(spec *MCPSpec) error { + for { + t := p.peek() + if t.Kind != TokIdent { + return nil + } + switch t.Val { + case "header": + p.advance() + v := p.peek() + if v.Kind != TokString { + return p.errorAt(v, + "expected a quoted \"Name: value\" after 'header' for mcp server %q, got %s", + spec.Name, v) + } + if err := validateMCPHeader(v.Val); err != nil { + return p.errorAt(v, "mcp server %q: %v", spec.Name, err) + } + if spec.Transport == config.MCPTransportStdio { + return p.errorAt(t, "mcp server %q: 'header' applies to http/sse servers, not stdio", spec.Name) + } + spec.Headers = append(spec.Headers, v.Val) + p.advance() + case "env": + p.advance() + v := p.peek() + if v.Kind != TokIdent { + return p.errorAt(v, + "expected a variable name after 'env' for mcp server %q, got %s", spec.Name, v) + } + if err := envspec.ValidateName(v.Val); err != nil { + return p.errorAt(v, "mcp server %q: %v", spec.Name, err) + } + if spec.Transport != config.MCPTransportStdio { + return p.errorAt(t, + "mcp server %q: 'env' applies to stdio servers; pass a credential to an "+ + "http/sse server with header \"Authorization: Bearer ${VAR}\"", spec.Name) + } + spec.Env = append(spec.Env, v.Val) + p.advance() + default: + // Not a modifier — leave it for the caller's end-of-entry check, + // which reports it against the entry as a whole. + return nil + } + } +} + +// validateMCPName rejects server names that would not survive the round trip +// into the guest's MCP config or the agent's tool namespace. Claude Code +// derives tool names from the server name, so keep it to the conservative +// set every runner tolerates. +func validateMCPName(name string) error { + if name == "" { + return errors.New("empty mcp server name") + } + for _, r := range name { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case r == '-', r == '_', r == '.': + default: + return fmt.Errorf("invalid mcp server name %q: use letters, digits, '-', '_' or '.'", name) + } + } + return nil +} + +// validateMCPURL requires an absolute http(s) URL with a host and no +// credentials in it. Catching this at parse time means the +// network-derivation step downstream can assume it has a host to allow. +// +// Userinfo is refused rather than accepted-and-ignored because it is the one +// way to smuggle a secret VALUE past the design invariant that clawk stores +// only references: the URL is persisted verbatim onto the sandbox record and +// rendered into the guest MCP config, both of which sit unencrypted on host +// disk. `header "Authorization: Bearer ${VAR}"` is the supported spelling, +// and it keeps the value in the runner's process environment. See +// config.MCPServer. +func validateMCPURL(raw string) error { + u, err := url.Parse(raw) + if err != nil { + return fmt.Errorf("invalid URL %q: %w", raw, err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("URL %q must use http or https", raw) + } + if u.Hostname() == "" { + return fmt.Errorf("URL %q has no host", raw) + } + if u.User != nil { + return fmt.Errorf( + "URL %q embeds credentials — clawk would store them on disk; "+ + "drop the user:password@ and pass the credential with "+ + "header \"Authorization: Bearer ${VAR}\" instead", + u.Redacted()) + } + return nil +} + +// validateMCPHeader requires the "Name: value" spelling, so the rendered +// config never carries a header the runner will silently drop. +func validateMCPHeader(h string) error { + name, _, ok := strings.Cut(h, ":") + if !ok || strings.TrimSpace(name) == "" { + return fmt.Errorf("invalid header %q: want \"Name: value\"", h) + } + return nil +} + // parseAgentBlock handles `agent ( instructions ... | memory "..." )` — the // persistent agent context seeded into a sandbox. `instructions` accumulates // markdown guidance blocks (inline string or a parenthesised list); `memory` diff --git a/internal/template/parse_test.go b/internal/template/parse_test.go index 25ff50a..ea1c747 100644 --- a/internal/template/parse_test.go +++ b/internal/template/parse_test.go @@ -299,6 +299,12 @@ func TestParseResourceErrors(t *testing.T) { {"duplicate cpu", "cpu 2\ncpu 4\n", "duplicate"}, {"duplicate memory", "memory 1G\nmemory 2G\n", "duplicate"}, {"duplicate memory_max", "memory_max 1G\nmemory_max 2G\n", "duplicate"}, + {"duplicate swap", "swap 1G\nswap 2G\n", "duplicate"}, + {"bare swap number", "swap 4096", "unit suffix"}, + {"empty swap", "swap\n", "expected size or 'off'"}, + // The unit typo this floor exists for: `swap 512M` written meaning + // half a gigabyte is fine, but a bare small number is not. + {"swap below the floor", "swap 32M", "too small"}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -314,6 +320,33 @@ func TestParseResourceErrors(t *testing.T) { } } +func TestParseSwap(t *testing.T) { + cases := []struct { + src string + wantMiB int64 + }{ + {"swap 8GiB", 8192}, + {"swap 512M", 512}, + // "off" and "0" store -1, not 0: the zero value has to keep meaning + // "unset" so the built-in default still applies to everyone else. + {"swap off", -1}, + {"swap 0", -1}, + } + for _, c := range cases { + t.Run(c.src, func(t *testing.T) { + tmpl, err := parseBody("vm (\n" + c.src + "\n)\n") + require.NoError(t, err) + require.Equal(t, c.wantMiB, tmpl.SwapMiB) + }) + } + + t.Run("unset stays zero", func(t *testing.T) { + tmpl, err := parseBody("vm (\n cpu 2\n)\n") + require.NoError(t, err) + require.Zero(t, tmpl.SwapMiB) + }) +} + func TestParseIdleTimeout(t *testing.T) { cases := []struct { src string diff --git a/internal/template/resources.go b/internal/template/resources.go index 3ed1420..16addfb 100644 --- a/internal/template/resources.go +++ b/internal/template/resources.go @@ -57,7 +57,8 @@ type PolicyDef struct { } // NamespaceDef is one `namespace ( ... )` block: a named overlay of -// the per-namespace template subset (network / files / shares / env / agent). +// the per-namespace template subset (network / files / shares / mcp / env / +// agent). // VM shape, includes and lifecycle hooks are sandbox-level concerns and are // rejected inside a namespace body. type NamespaceDef struct { @@ -373,6 +374,8 @@ func (p *parser) parseNamespaceBlock() (NamespaceDef, error) { err = p.parseFilesBlock(def.Template) case "shares": err = p.parseSharesBlock(def.Template) + case "mcp": + err = p.parseMCPBlock(def.Template) case "env": err = p.parseEnvBlock(&def.Template.Env) case "agent": @@ -380,11 +383,11 @@ func (p *parser) parseNamespaceBlock() (NamespaceDef, error) { case "vm", "includes", "on": err = p.errorAt(t, "%q is a sandbox-level directive, not allowed in a namespace (want %s)", - t.Val, describeFirst([]string{"network", "files", "shares", "env", "agent"})) + t.Val, describeFirst([]string{"network", "files", "shares", "mcp", "env", "agent"})) default: err = p.errorAt(t, "unknown 'namespace' directive %q (want %s)", t.Val, - describeFirst([]string{"network", "files", "shares", "env", "agent"})) + describeFirst([]string{"network", "files", "shares", "mcp", "env", "agent"})) } if err != nil { return def, err diff --git a/internal/template/serial_test.go b/internal/template/serial_test.go new file mode 100644 index 0000000..ca2d4e7 --- /dev/null +++ b/internal/template/serial_test.go @@ -0,0 +1,51 @@ +package template + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseSerialBlock(t *testing.T) { + tmpl, err := parseBody(`serial ( + /dev/cu.usbmodem1101 + /dev/cu.usbserial-A50285BI ttyUSB0 + /dev/cu.usbmodem* ttyACM0 +) +`) + require.NoError(t, err) + require.Len(t, tmpl.Serials, 3) + + require.Equal(t, "/dev/cu.usbmodem1101", tmpl.Serials[0].HostPath) + require.Empty(t, tmpl.Serials[0].GuestName, "a bare entry defaults its name downstream") + + require.Equal(t, "/dev/cu.usbserial-A50285BI", tmpl.Serials[1].HostPath) + require.Equal(t, "ttyUSB0", tmpl.Serials[1].GuestName) + + require.Equal(t, "/dev/cu.usbmodem*", tmpl.Serials[2].HostPath) + require.Equal(t, "ttyACM0", tmpl.Serials[2].GuestName) + + // Positions are recorded so a downstream name clash can point back at + // the line that caused it. + require.NotZero(t, tmpl.Serials[0].Line) +} + +func TestParseSerialBlockRejectsThirdToken(t *testing.T) { + _, err := parseBody("serial (\n /dev/cu.usbmodem1101 ttyACM0 extra\n)\n") + require.Error(t, err) + require.Contains(t, err.Error(), "one host device and optional guest name per line") +} + +func TestParseSerialBlockRequiresParen(t *testing.T) { + _, err := parseBody("serial /dev/cu.usbmodem1101\n") + require.Error(t, err) + require.Contains(t, err.Error(), "expected '(' after 'serial'") +} + +// The directive has to be listed in the unknown-directive hint, or a typo +// sends the user looking for a feature the error says doesn't exist. +func TestUnknownDirectiveMentionsSerial(t *testing.T) { + _, err := parseBody("serialz (\n /dev/cu.usbmodem1101\n)\n") + require.Error(t, err) + require.Contains(t, err.Error(), `"serial"`) +} diff --git a/internal/template/workspace.go b/internal/template/workspace.go index 1bebd45..451b184 100644 --- a/internal/template/workspace.go +++ b/internal/template/workspace.go @@ -18,11 +18,22 @@ type Workspace struct { Repos []Repo // every repo included by the workspace, in declaration order // Policies collects every `policy ( ... )` block declared across - // the loaded files — workspace file and its overlay first (broader - // scope), then each repo's clawk.mod and overlay. The create paths - // register them into the host policy store, so later (nearer) blocks - // win a name collision. + // the loaded files — the host-wide clawk.mod first, then the workspace + // file and its overlay (broader scope), then each repo's clawk.mod and + // overlay. The create paths register them into the host policy store, so + // later (nearer) blocks win a name collision. Policies []PolicyDef + + // GlobalPath is the host-wide clawk.mod folded in as the lowest layer, + // or "" when there is none. Reporting only — its directives have already + // been merged into File or a Repo's Clawkfile by the time a caller sees + // this. See global.go. + GlobalPath string + + // GlobalProfileMatched records that a clawk.mod. overlay beside + // the host-wide file was applied, so a profile satisfied only by the + // host-wide layer isn't reported as matching nothing. + GlobalProfileMatched bool } // Repo is one git repository brought into a sandbox by a workspace. The @@ -193,6 +204,14 @@ func loadWorkspaceParsed(abs string, f *File, profile string) (*Workspace, error if err := checkNameCollisions(ws.Repos); err != nil { return nil, err } + // The host-wide layer lands last, once the repos are known: fold needs + // them to decide which of its scalars a repo has already spoken for. + if err := attachGlobal(ws, profile, foldGlobalIntoWorkspace); err != nil { + return nil, err + } + if ws.GlobalProfileMatched { + profileMatched = true + } if profile != "" && !profileMatched { return nil, fmt.Errorf( "profile %q matched no overlay file (looked for %s.%s beside the workspace file and in each repo)", @@ -257,16 +276,23 @@ func LoadStandaloneClawkfileWithProfile(dir, profile string) (*Workspace, error) if err != nil { return nil, err } - if profile != "" && !matched { - return nil, fmt.Errorf("profile %q has no matching overlay in %s", - profile, abs) - } - return &Workspace{ + ws := &Workspace{ Root: repo.RepoPath, File: &Template{}, Repos: []Repo{repo}, Policies: policies, - }, nil + } + // One repo, so the host-wide layer folds straight under its clawk.mod: + // the repo's scalars win, its list entries follow the global ones, and its + // `on` hooks keep the per-phase position they have today. + if err := attachGlobal(ws, profile, foldGlobalUnderOnlyRepo); err != nil { + return nil, err + } + if profile != "" && !matched && !ws.GlobalProfileMatched { + return nil, fmt.Errorf("profile %q has no matching overlay in %s", + profile, abs) + } + return ws, nil } // LoadClawkfilePathWithProfile loads an explicitly-given clawk.mod path, @@ -302,7 +328,7 @@ func WorkspaceFromGitRepo(dir string) (*Workspace, error) { if err != nil { return nil, err } - return &Workspace{ + ws := &Workspace{ Root: repoRoot, File: &Template{}, Repos: []Repo{{ @@ -310,7 +336,13 @@ func WorkspaceFromGitRepo(dir string) (*Workspace, error) { Path: repoRoot, RepoPath: repoRoot, }}, - }, nil + } + // The repo has no clawk.mod at all — the case the host-wide layer exists + // for. It becomes the repo's entire template. + if err := attachGlobal(ws, "", foldGlobalUnderOnlyRepo); err != nil { + return nil, err + } + return ws, nil } // resolveRepoWithProfile resolves a workspace repo entry, optionally @@ -466,7 +498,16 @@ func (w *Workspace) FilterRepos(only []string) (*Workspace, error) { return nil, fmt.Errorf("unknown repo(s): %s (known: %s)", strings.Join(missing, ", "), strings.Join(known, ", ")) } - return &Workspace{Root: w.Root, File: w.File, Repos: kept, Policies: w.Policies}, nil + return &Workspace{ + Root: w.Root, + File: w.File, + Repos: kept, + Policies: w.Policies, + // Carried so the create-time note still names the host-wide file. Its + // directives already live in File, which is shared with the original. + GlobalPath: w.GlobalPath, + GlobalProfileMatched: w.GlobalProfileMatched, + }, nil } // rejectLifecycleAtWorkspace flags the `on ` lists that stay repo-local diff --git a/internal/vsockclient/client.go b/internal/vsockclient/client.go index 1afec7b..2eed7c1 100644 --- a/internal/vsockclient/client.go +++ b/internal/vsockclient/client.go @@ -65,7 +65,7 @@ type Config struct { User string // ClearScreen asks Run to clear the terminal before relaying, so a - // full-screen TUI child (claude, codex, opencode) starts on a clean + // full-screen TUI child (claude, codex, pi, opencode) starts on a clean // canvas instead of overdrawing whatever the CLI printed first (boot // progress, hints): such TUIs position with absolute cursor moves and // don't erase the cells they skip, so stale text shows through. diff --git a/internal/vzdctl/vzdctl.go b/internal/vzdctl/vzdctl.go index 1df7868..a17af93 100644 --- a/internal/vzdctl/vzdctl.go +++ b/internal/vzdctl/vzdctl.go @@ -45,6 +45,12 @@ type Handlers struct { // ErrReverseForwardsUnsupported. ReloadForwards func() error + // ReloadSerials, if non-nil, re-reads the sandbox's serial devices from + // the store and pushes them to the in-guest agent. Nil on backends with + // no vsock listener (firecracker), where the endpoint reports 404 and + // the client maps it to ErrSerialUnsupported. + ReloadSerials func() error + // Gate, if non-nil, powers the interactive allow/deny endpoints // (/v1/events, /v1/decide, /v1/pending). When nil those endpoints // report 404 and the daemon serves only the denial ledger + reload. @@ -149,6 +155,21 @@ func Start(path string, h Handlers) (*Server, error) { } w.WriteHeader(http.StatusNoContent) }) + mux.HandleFunc("POST /v1/reload-serials", func(w http.ResponseWriter, _ *http.Request) { + // Its own endpoint for the same reason reload-forwards is: a daemon + // that predates serial forwarding would answer a shared endpoint + // happily and the CLI would report a live apply that never + // happened. A 404 here is the honest answer. + if h.ReloadSerials == nil { + http.Error(w, "serial forwarding not supported", http.StatusNotFound) + return + } + if err := h.ReloadSerials(); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusNoContent) + }) mux.HandleFunc("GET /v1/events", func(w http.ResponseWriter, r *http.Request) { serveEvents(w, r, h.Gate) }) @@ -410,6 +431,11 @@ var ErrLifecycleUnsupported = errors.New("daemon does not support lifecycle cont // errors.Is. var ErrReverseForwardsUnsupported = errors.New("daemon does not support reverse port forwarding") +// ErrSerialUnsupported reports that the daemon answered but has no +// serial endpoint — either it predates the feature or its backend has no +// host-side vsock listener (firecracker). Callers check with errors.Is. +var ErrSerialUnsupported = errors.New("daemon does not support serial forwarding") + // Lifecycle fetches the VM's live lifecycle snapshot. func (c *Client) Lifecycle(ctx context.Context) (LifecycleState, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://vzd/v1/lifecycle", nil) @@ -528,6 +554,28 @@ func (c *Client) ReloadForwards(ctx context.Context) error { } } +// ReloadSerials asks the daemon to re-read the sandbox's serial devices +// from the store and push them to the in-guest agent. +func (c *Client) ReloadSerials(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://vzd/v1/reload-serials", nil) + if err != nil { + return fmt.Errorf("building reload-serials request: %w", err) + } + resp, err := c.do(req) + if err != nil { + return err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusNoContent: + return nil + case http.StatusNotFound: + return fmt.Errorf("%w: %s", ErrSerialUnsupported, responseError(resp)) + default: + return fmt.Errorf("reload-serials: %s", responseError(resp)) + } +} + // Pending fetches the daemon's outstanding interactive holds. func (c *Client) Pending(ctx context.Context) ([]netfilter.Pending, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://vzd/v1/pending", nil) diff --git a/internal/vzdctl/vzdctl_test.go b/internal/vzdctl/vzdctl_test.go index 853e77c..aedb1f4 100644 --- a/internal/vzdctl/vzdctl_test.go +++ b/internal/vzdctl/vzdctl_test.go @@ -329,3 +329,67 @@ func TestReloadForwardsErrorSurfaces(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "record vanished") } + +func TestReloadSerialsRoundTrip(t *testing.T) { + sock := testSocket(t) + var reloaded int + srv, err := Start(sock, Handlers{ + Denials: func() []netfilter.Denial { return nil }, + Reload: func() error { return nil }, + ReloadSerials: func() error { reloaded++; return nil }, + }) + require.NoError(t, err) + t.Cleanup(func() { srv.Close() }) + + require.NoError(t, NewClient(sock).ReloadSerials(context.Background())) + require.Equal(t, 1, reloaded) +} + +// A daemon with no serial sink (firecracker, or one predating the feature) +// must say so rather than let the CLI report a live apply that never +// happened. +func TestReloadSerialsUnsupported(t *testing.T) { + sock := testSocket(t) + srv, err := Start(sock, Handlers{ + Denials: func() []netfilter.Denial { return nil }, + Reload: func() error { return nil }, + }) + require.NoError(t, err) + t.Cleanup(func() { srv.Close() }) + + err = NewClient(sock).ReloadSerials(context.Background()) + require.ErrorIs(t, err, ErrSerialUnsupported) +} + +// The two reload endpoints must stay independent: a daemon that supports +// reverse forwards but predates serial forwarding has to report exactly +// that, not blanket success. +func TestReloadSerialsIndependentOfReloadForwards(t *testing.T) { + sock := testSocket(t) + srv, err := Start(sock, Handlers{ + Denials: func() []netfilter.Denial { return nil }, + Reload: func() error { return nil }, + ReloadForwards: func() error { return nil }, + }) + require.NoError(t, err) + t.Cleanup(func() { srv.Close() }) + + c := NewClient(sock) + require.NoError(t, c.ReloadForwards(context.Background())) + require.ErrorIs(t, c.ReloadSerials(context.Background()), ErrSerialUnsupported) +} + +func TestReloadSerialsErrorSurfaces(t *testing.T) { + sock := testSocket(t) + srv, err := Start(sock, Handlers{ + Denials: func() []netfilter.Denial { return nil }, + Reload: func() error { return nil }, + ReloadSerials: func() error { return errors.New("record vanished") }, + }) + require.NoError(t, err) + t.Cleanup(func() { srv.Close() }) + + err = NewClient(sock).ReloadSerials(context.Background()) + require.Error(t, err) + require.Contains(t, err.Error(), "record vanished") +} diff --git a/machine/oci/integration_test.go b/machine/oci/integration_test.go index 1dbfb36..c4b2004 100644 --- a/machine/oci/integration_test.go +++ b/machine/oci/integration_test.go @@ -48,8 +48,26 @@ func TestBuildAlpine_Integration(t *testing.T) { assertValidRootFS(t, res.DiskPath) + // A cache hit is asserted by observing that the disk was not rebuilt, not + // by timing the call. + // + // Wall-clock is the wrong instrument here for two reasons. It is a proxy + // for "didn't redo the work" that depends entirely on the hardware the + // test happens to run on — and it isn't even measuring the network, since + // resolveSource calls crane.Digest to turn the tag into a digest BEFORE + // the cache lookup, so every Build makes a registry round-trip whose + // latency the test cannot bound. A one-second budget therefore fails on a + // slow or emulated host, and on a slow link, while the cache is working + // perfectly. + // + // The cache-hit path stats the done marker and returns; a miss re-pulls + // and rewrites disk.ext4. So an unchanged mtime, size and identity prove + // the artifact was reused, which is the property the timing bound was + // standing in for — and prove it deterministically. t.Run("cache hit", func(t *testing.T) { - start := time.Now() + before, err := os.Stat(res.DiskPath) + require.NoError(t, err, "stat before cached Build") + res2, err := Build(ctx, Options{ Ref: ref, CacheDir: cache, @@ -57,8 +75,14 @@ func TestBuildAlpine_Integration(t *testing.T) { }) require.NoError(t, err, "cached Build") require.Equal(t, res.DiskPath, res2.DiskPath, "cache miss") - d := time.Since(start) - require.LessOrEqual(t, d, time.Second, "cache hit too slow: %s (expected <1s)", d) + + after, err := os.Stat(res2.DiskPath) + require.NoError(t, err, "stat after cached Build") + require.Equal(t, before.ModTime(), after.ModTime(), + "disk.ext4 was rewritten — the second Build rebuilt instead of hitting the cache") + require.Equal(t, before.Size(), after.Size(), "disk.ext4 changed size") + require.True(t, os.SameFile(before, after), + "disk.ext4 was replaced by a different file — the second Build rebuilt and renamed") }) t.Run("materialize produces independent disks", func(t *testing.T) { diff --git a/machine/vz/balloon.go b/machine/vz/balloon.go index 3090ca9..022d63f 100644 --- a/machine/vz/balloon.go +++ b/machine/vz/balloon.go @@ -92,8 +92,81 @@ const ( // "roomy" and idle memory can be reclaimed: 2/5 = 40% available. The gap // between low and high is the hysteresis band in which we hold steady. highSlackNum, highSlackDen = 2, 5 + + // swapUsedFloorKiB is how much swap a guest must be holding before we + // stop believing that its free memory means it has memory to spare. + // + // Sandboxes ship with a swap device, and that changes what the two + // signals above mean. Swapping out cold anonymous pages frees them — + // MemAvailable rises — and removes the reclaim stalls PSI measures, so a + // guest that answered a squeeze by paging out reads as roomy on both + // counts. Reclaiming another step there is how a guest ends up living at + // its baseline, swapping, instead of growing to the ceiling it was given. + // + // 64 MiB is above the incidental few megabytes a long-lived guest pages + // out and never touches again, and far below anything that indicates real + // pressure. + // + // Occupancy alone is not enough to act on, though — see swapTrend. + swapUsedFloorKiB = 64 * 1024 + + // swapQuietReportsBeforeReclaim is how many consecutive guest reports may + // show swap at or above the floor WITHOUT growing before the controller + // stops reading it as pressure. + // + // Occupancy latches: a swap slot is released only when its page is faulted + // back in or its owner exits, so a guest that paged out cold anonymous + // memory once reports the same number forever. With GuestSwappiness at 80 + // that is an ordinary thing for a long-lived sandbox to do — one big link + // step early on — and holding on it indefinitely would retire the reclaim + // path for the rest of that sandbox's life on a signal that never decays. + // + // Growth is the part that means "being squeezed right now". 24 reports at + // memPollInterval (5s) is two minutes of quiet: long enough that a guest + // still working through a squeeze keeps the hold, short enough that a spike + // from an hour ago stops speaking for a sandbox that has been idle since. + // + // Guests that are actively thrashing are not this signal's job. Paging in + // and out at the same rate holds occupancy flat, so the trend reads quiet — + // but that case is exactly what elevated PSI describes, and it grows the + // guest to the ceiling two branches earlier in guestDesiredTarget. + swapQuietReportsBeforeReclaim = 24 ) +// swapTrend tracks swap occupancy across guest reports, so the controller can +// tell a guest paging out right now from one that has old cold pages parked in +// swap. One per VM, owned by runBalloonController's goroutine — no lock. +type swapTrend struct { + prevUsedKiB uint64 + quiet int +} + +// observe folds one guest report in and reports whether swap use should still +// count as evidence of memory pressure. +// +// Call it once per REPORT, not once per balloon re-evaluation. Growth is only +// observable between two distinct reports, so re-running it on the same report +// (as the reeval ticker would) counts as quiet and would decay the hold in +// seconds instead of minutes. The controller keeps the verdict between reports, +// exactly as it already keeps the report itself. +func (s *swapTrend) observe(r memReport) bool { + used := r.SwapUsedKiB() + // Below the floor is the same "no signal" as a guest with no swap device, + // or one whose agent is too old to report any — reset, so a later rise + // starts a fresh hold rather than inheriting a stale quiet count. + if used < swapUsedFloorKiB { + s.prevUsedKiB, s.quiet = used, 0 + return false + } + if used > s.prevUsedKiB { + s.quiet = 0 // still paging out + } else { + s.quiet++ + } + s.prevUsedKiB = used + return s.quiet < swapQuietReportsBeforeReclaim +} + // reclaimStepBytes is how much we inflate per reclaim step when the guest is // idle. Gentle by design — a step at a time avoids reclaiming a large chunk // the instant a guest goes briefly quiet, only to deflate it again moments @@ -109,7 +182,11 @@ const reclaimStepBytes = 128 * 1024 * 1024 // A zero report (TotalKiB == 0) means "no fresh guest data" — the caller is // responsible for not constraining the guest in that case; here we simply hold // cur clamped into range. -func guestDesiredTarget(cur, baseline, ceiling uint64, r memReport) uint64 { +// swapPressure is swapTrend.observe's latest verdict for this guest: swap is +// in use AND recently growing. It is a parameter rather than something derived +// from r because the judgement needs history across reports, which this +// deliberately-pure function has none of. +func guestDesiredTarget(cur, baseline, ceiling uint64, r memReport, swapPressure bool) uint64 { if ceiling <= baseline || r.TotalKiB == 0 { return clampRange(cur, baseline, ceiling) } @@ -119,6 +196,14 @@ func guestDesiredTarget(cur, baseline, ceiling uint64, r memReport) uint64 { r.AvailableKiB*uint64(lowSlackDen) < r.TotalKiB*uint64(lowSlackNum) { return ceiling } + // A guest that is paging out is not idle, whatever its slack says: it is + // being squeezed hard enough to evict, and its headroom is the product of + // that, not evidence it was never needed. Hold instead of reclaiming — + // growth stays available through the branch above, which fires as soon as + // it stalls or runs genuinely tight. + if swapPressure { + return clampRange(cur, baseline, ceiling) + } // Reclaim a step toward the baseline when the guest is roomy: high slack // means the guest genuinely isn't using the memory, so handing it back to // the host is exactly right. @@ -138,8 +223,12 @@ func guestDesiredTarget(cur, baseline, ceiling uint64, r memReport) uint64 { // CRITICAL the cap drops to balloonTarget's fractions, reclaiming RAM for the // host even against guest demand (the guest's DEFLATE_ON_OOM is its safety // net). The result is always min(guest-desired, host-allowed). -func mergedBalloonTarget(level pressureLevel, cur, baseline, ceiling uint64, r memReport) uint64 { - desired := guestDesiredTarget(cur, baseline, ceiling, r) +// Note that swapPressure only ever suppresses the guest's own voluntary +// reclaim; it is not a veto over the host. A guest holding swap still gets +// clipped to 3/4 (WARN) or 1/2 (CRITICAL) of its ceiling, because the host +// needing RAM outranks any read of what the guest would prefer. +func mergedBalloonTarget(level pressureLevel, cur, baseline, ceiling uint64, r memReport, swapPressure bool) uint64 { + desired := guestDesiredTarget(cur, baseline, ceiling, r, swapPressure) allowed := balloonTarget(level, ceiling) if desired > allowed { return allowed diff --git a/machine/vz/balloon_test.go b/machine/vz/balloon_test.go index 9906f8d..9974acf 100644 --- a/machine/vz/balloon_test.go +++ b/machine/vz/balloon_test.go @@ -85,7 +85,11 @@ func TestGuestDesiredTarget(t *testing.T) { name string cur uint64 r memReport - want uint64 + // swapPressure is swapTrend.observe's verdict — swap in use and + // recently growing. Its own logic is covered by TestSwapTrend; here it + // is an input, so these cases pin what the target does with it. + swapPressure bool + want uint64 }{ { name: "no report holds current", @@ -129,6 +133,55 @@ func TestGuestDesiredTarget(t *testing.T) { r: memReport{TotalKiB: baseline / 1024, AvailableKiB: baseline / 1024 / 2}, want: baseline, // ceiling == baseline here }, + // The slack a paging guest shows is manufactured: it evicted to produce + // it. Reclaiming on the strength of it is the feedback loop that parks + // a working guest at its baseline, swapping, forever. + { + name: "high slack with swap pressure holds instead of reclaiming", + cur: ceiling, + r: memReport{ + TotalKiB: totalKiB, AvailableKiB: totalKiB / 2, // 50% available > 40% + SwapTotalKiB: 4 << 20, SwapFreeKiB: 4<<20 - swapUsedFloorKiB, + }, + swapPressure: true, + want: ceiling, + }, + // Occupancy without pressure is history, not demand: once the trend has + // gone quiet the headroom is real and the guest gives memory back. This + // is what stops one early spike retiring reclaim for the sandbox's life. + { + name: "swap held but no longer growing reclaims again", + cur: ceiling, + r: memReport{ + TotalKiB: totalKiB, AvailableKiB: totalKiB / 2, + SwapTotalKiB: 4 << 20, SwapFreeKiB: 4<<20 - 2*swapUsedFloorKiB, + }, + swapPressure: false, + want: ceiling - reclaimStepBytes, + }, + // Swap pressure suppresses reclaim, never growth: a guest that is both + // paging and stalling still needs the ceiling. + { + name: "swap pressure does not block growth on demand", + cur: baseline, + r: memReport{ + TotalKiB: totalKiB, AvailableKiB: totalKiB / 2, PSIMemSomeCenti: psiDemandCenti, + SwapTotalKiB: 4 << 20, SwapFreeKiB: 0, + }, + swapPressure: true, + want: ceiling, + }, + // An untouched swap device says nothing at all — a guest that never + // swapped is as reclaimable as one with no swap device. + { + name: "swap present but unused reclaims normally", + cur: ceiling, + r: memReport{ + TotalKiB: totalKiB, AvailableKiB: totalKiB / 2, + SwapTotalKiB: 4 << 20, SwapFreeKiB: 4 << 20, + }, + want: ceiling - reclaimStepBytes, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -136,12 +189,90 @@ func TestGuestDesiredTarget(t *testing.T) { if tt.name == "no burst headroom collapses to baseline" { c = baseline } - got := guestDesiredTarget(tt.cur, baseline, c, tt.r) + got := guestDesiredTarget(tt.cur, baseline, c, tt.r, tt.swapPressure) require.Equal(t, tt.want, got) }) } } +// TestSwapTrend covers the signal that decides swapPressure. The property that +// matters is that it DECAYS: swap occupancy latches (a slot is freed only when +// its page is faulted back in or its owner exits), so reading the raw level as +// pressure pinned a guest that spiked once and then idled, and retired the +// reclaim path for the rest of its life. +func TestSwapTrend(t *testing.T) { + const total = 4 << 20 // 4 GiB of swap, in KiB + + used := func(kib uint64) memReport { + return memReport{TotalKiB: 4 << 20, AvailableKiB: 2 << 20, + SwapTotalKiB: total, SwapFreeKiB: total - kib} + } + + t.Run("below the floor is no signal", func(t *testing.T) { + var s swapTrend + for i := 0; i < 3; i++ { + require.False(t, s.observe(used(swapUsedFloorKiB-1))) + } + }) + + t.Run("no swap device at all is no signal", func(t *testing.T) { + var s swapTrend + require.False(t, s.observe(memReport{TotalKiB: 4 << 20, AvailableKiB: 2 << 20})) + }) + + t.Run("crossing the floor starts a hold", func(t *testing.T) { + var s swapTrend + require.True(t, s.observe(used(swapUsedFloorKiB))) + }) + + t.Run("hold decays once growth stops", func(t *testing.T) { + var s swapTrend + require.True(t, s.observe(used(2*swapUsedFloorKiB)), "first sighting holds") + // Same occupancy, report after report: the guest evicted and moved on. + for i := 1; i < swapQuietReportsBeforeReclaim; i++ { + require.True(t, s.observe(used(2*swapUsedFloorKiB)), + "quiet report %d should still hold", i) + } + require.False(t, s.observe(used(2*swapUsedFloorKiB)), + "past the quiet window the occupancy is history, not pressure") + require.False(t, s.observe(used(2*swapUsedFloorKiB)), "and stays decayed") + }) + + t.Run("renewed growth restarts the hold", func(t *testing.T) { + var s swapTrend + s.observe(used(2 * swapUsedFloorKiB)) + for i := 0; i < swapQuietReportsBeforeReclaim+5; i++ { + s.observe(used(2 * swapUsedFloorKiB)) + } + require.False(t, s.observe(used(2*swapUsedFloorKiB)), "decayed first") + require.True(t, s.observe(used(3*swapUsedFloorKiB)), + "growing again means the guest is being squeezed now") + }) + + t.Run("dropping below the floor resets the quiet count", func(t *testing.T) { + var s swapTrend + s.observe(used(2 * swapUsedFloorKiB)) + for i := 0; i < swapQuietReportsBeforeReclaim+5; i++ { + s.observe(used(2 * swapUsedFloorKiB)) + } + require.False(t, s.observe(used(2*swapUsedFloorKiB)), "decayed") + // Pages came back in and the guest fell under the floor; a later rise + // must not inherit the stale quiet count. + require.False(t, s.observe(used(0))) + require.True(t, s.observe(used(2*swapUsedFloorKiB))) + }) + + t.Run("shrinking swap counts as quiet", func(t *testing.T) { + var s swapTrend + require.True(t, s.observe(used(10*swapUsedFloorKiB))) + // Faulting pages back in is not eviction pressure. + for i := 1; i < swapQuietReportsBeforeReclaim; i++ { + require.True(t, s.observe(used(uint64(10-i/4)*swapUsedFloorKiB))) + } + require.False(t, s.observe(used(5*swapUsedFloorKiB))) + }) +} + func TestGuestDesiredTargetStaysInRange(t *testing.T) { const ( gib = 1024 * 1024 * 1024 @@ -156,7 +287,7 @@ func TestGuestDesiredTargetStaysInRange(t *testing.T) { } for _, cur := range []uint64{baseline, 2 * gib, ceiling} { for _, r := range reports { - got := guestDesiredTarget(cur, baseline, ceiling, r) + got := guestDesiredTarget(cur, baseline, ceiling, r, false) require.GreaterOrEqual(t, got, uint64(baseline)) require.LessOrEqual(t, got, uint64(ceiling)) } @@ -174,18 +305,18 @@ func TestMergedBalloonTargetHostPressureWins(t *testing.T) { // Under normal pressure the guest gets what it asks for. require.Equal(t, uint64(ceiling), - mergedBalloonTarget(pressureNormal, baseline, baseline, ceiling, starved)) + mergedBalloonTarget(pressureNormal, baseline, baseline, ceiling, starved, false)) // Under WARN the host caps growth at balloonTarget(warn, ceiling) even // though the guest is starving. wantWarn := balloonTarget(pressureWarn, ceiling) require.Equal(t, wantWarn, - mergedBalloonTarget(pressureWarn, baseline, baseline, ceiling, starved)) + mergedBalloonTarget(pressureWarn, baseline, baseline, ceiling, starved, false)) require.Less(t, wantWarn, uint64(ceiling)) // Under CRITICAL it caps even lower. wantCrit := balloonTarget(pressureCritical, ceiling) require.Equal(t, wantCrit, - mergedBalloonTarget(pressureCritical, baseline, baseline, ceiling, starved)) + mergedBalloonTarget(pressureCritical, baseline, baseline, ceiling, starved, false)) require.Less(t, wantCrit, wantWarn) } diff --git a/machine/vz/memreport.go b/machine/vz/memreport.go index af24d8a..629bde0 100644 --- a/machine/vz/memreport.go +++ b/machine/vz/memreport.go @@ -21,12 +21,14 @@ const agentMemPort uint32 = 1027 // memReport wire sizes. The report grew from three to five big-endian // uint64s when the guest agent started sampling activity (load, net I/O) -// alongside memory. Decoding accepts the legacy 24-byte prefix alone so a -// new host reads an old guest's report (and vice versa: an old host's -// fixed 24-byte read simply ignores the tail a new guest appends). +// alongside memory, and to seven when sandboxes gained a swap device. +// Decoding accepts any of the three lengths, so a new host reads an old +// guest's report (and vice versa: an old host's fixed read simply ignores +// the tail a new guest appends). const ( - memReportLegacySize = 24 - memReportSize = 40 + memReportLegacySize = 24 + memReportActivitySize = 40 + memReportSize = 56 ) // memReport is a guest snapshot. The balloon controller polls the memory @@ -63,12 +65,33 @@ type memReport struct { // samples: moving counters mean traffic is flowing. NetIOBytes uint64 + // SwapTotalKiB and SwapFreeKiB mirror /proc/meminfo's SwapTotal and + // SwapFree. Both zero on a guest with no swap device — and equally on a + // guest agent too old to report them, which is why the controller only + // ever reads them as "swap is in use", never as "swap is not in use". + // + // They exist because AvailableKiB alone stops describing demand once a + // guest can swap: pushing cold anonymous pages out raises MemAvailable + // and lowers PSI, so a guest under real pressure can look idle. See + // guestDesiredTarget. + SwapTotalKiB uint64 + SwapFreeKiB uint64 + // HasActivity reports whether the guest agent is new enough to include // the activity fields (Load1Centi, NetIOBytes). When false those fields // are zero because they're absent, not because the guest is quiet. HasActivity bool } +// SwapUsedKiB is the swap the guest has actually written. Zero when the +// guest has no swap, reports none, or has swapped nothing. +func (r memReport) SwapUsedKiB() uint64 { + if r.SwapTotalKiB == 0 || r.SwapFreeKiB > r.SwapTotalKiB { + return 0 + } + return r.SwapTotalKiB - r.SwapFreeKiB +} + // encodeMemReport serializes r into exactly memReportSize bytes. func encodeMemReport(r memReport) []byte { b := make([]byte, memReportSize) @@ -77,6 +100,8 @@ func encodeMemReport(r memReport) []byte { binary.BigEndian.PutUint64(b[16:24], r.PSIMemSomeCenti) binary.BigEndian.PutUint64(b[24:32], r.Load1Centi) binary.BigEndian.PutUint64(b[32:40], r.NetIOBytes) + binary.BigEndian.PutUint64(b[40:48], r.SwapTotalKiB) + binary.BigEndian.PutUint64(b[48:56], r.SwapFreeKiB) return b } @@ -94,11 +119,15 @@ func decodeMemReport(b []byte) (memReport, error) { AvailableKiB: binary.BigEndian.Uint64(b[8:16]), PSIMemSomeCenti: binary.BigEndian.Uint64(b[16:24]), } - if len(b) >= memReportSize { + if len(b) >= memReportActivitySize { r.Load1Centi = binary.BigEndian.Uint64(b[24:32]) r.NetIOBytes = binary.BigEndian.Uint64(b[32:40]) r.HasActivity = true } + if len(b) >= memReportSize { + r.SwapTotalKiB = binary.BigEndian.Uint64(b[40:48]) + r.SwapFreeKiB = binary.BigEndian.Uint64(b[48:56]) + } return r, nil } @@ -111,6 +140,8 @@ type GuestStats struct { PSIMemSomeCenti uint64 Load1Centi uint64 NetIOBytes uint64 + SwapTotalKiB uint64 + SwapFreeKiB uint64 // HasActivity is false when the guest runs a legacy agent without the // activity fields; Load1Centi and NetIOBytes are then meaningless. @@ -151,6 +182,8 @@ func FetchGuestStats(ctx context.Context, m machine.Machine) (GuestStats, error) PSIMemSomeCenti: r.PSIMemSomeCenti, Load1Centi: r.Load1Centi, NetIOBytes: r.NetIOBytes, + SwapTotalKiB: r.SwapTotalKiB, + SwapFreeKiB: r.SwapFreeKiB, HasActivity: r.HasActivity, }, nil } diff --git a/machine/vz/memreport_test.go b/machine/vz/memreport_test.go index 36cc308..4324f50 100644 --- a/machine/vz/memreport_test.go +++ b/machine/vz/memreport_test.go @@ -59,6 +59,48 @@ func TestDecodeMemReportLegacy(t *testing.T) { require.False(t, got.HasActivity) } +// TestDecodeMemReportActivityOnly: a 40-byte report from a guest agent +// that predates the swap fields keeps everything up to NetIOBytes and +// reports no swap. Zero swap is indistinguishable from "no swap device", +// which is why the controller only ever reads these fields as evidence +// that swap IS in use. +func TestDecodeMemReportActivityOnly(t *testing.T) { + full := encodeMemReport(memReport{ + TotalKiB: 4 * 1024 * 1024, AvailableKiB: 2_000_000, PSIMemSomeCenti: 17, + Load1Centi: 250, NetIOBytes: 4242, + SwapTotalKiB: 999, SwapFreeKiB: 999, // must NOT survive the shorter decode + }) + got, err := decodeMemReport(full[:memReportActivitySize]) + require.NoError(t, err) + require.Equal(t, memReport{ + TotalKiB: 4 * 1024 * 1024, AvailableKiB: 2_000_000, PSIMemSomeCenti: 17, + Load1Centi: 250, NetIOBytes: 4242, HasActivity: true, + }, got) + require.Zero(t, got.SwapUsedKiB()) +} + +func TestSwapUsedKiB(t *testing.T) { + tests := []struct { + name string + total uint64 + free uint64 + want uint64 + }{ + {name: "no swap device", total: 0, free: 0, want: 0}, + {name: "swap present but untouched", total: 4 << 20, free: 4 << 20, want: 0}, + {name: "half used", total: 4 << 20, free: 2 << 20, want: 2 << 20}, + // A report where free exceeds total is nonsense from a torn or + // legacy read; it must not underflow into a huge "used". + {name: "free above total is not underflow", total: 1024, free: 4096, want: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := memReport{SwapTotalKiB: tt.total, SwapFreeKiB: tt.free} + require.Equal(t, tt.want, r.SwapUsedKiB()) + }) + } +} + func TestDecodeMemReportShort(t *testing.T) { _, err := decodeMemReport(make([]byte, memReportLegacySize-1)) require.Error(t, err, "short buffer must error, not zero-pad") diff --git a/machine/vz/pressure_darwin.go b/machine/vz/pressure_darwin.go index a4a3a46..e25b3cb 100644 --- a/machine/vz/pressure_darwin.go +++ b/machine/vz/pressure_darwin.go @@ -199,6 +199,12 @@ func (v *vm) runBalloonController(ctx context.Context, dev *codevz.VirtioTraditi level := pressureNormal var rep memReport var lastReport time.Time + // Swap occupancy needs history to mean anything, so the trend is folded + // once per arriving report and its verdict held until the next one — the + // same lifetime as rep itself. Re-observing on a tick would see the same + // numbers, count them as quiet, and decay the hold in seconds. + var swap swapTrend + var swapPressure bool debug.Log("vz", "balloon controller started", "id", v.spec.ID, "baseline_mib", baseline/(1024*1024), "ceiling_mib", ceiling/(1024*1024), @@ -213,7 +219,7 @@ func (v *vm) runBalloonController(ctx context.Context, dev *codevz.VirtioTraditi if fresh { // Guest reported, so its balloon driver is up: manage between // baseline and ceiling on demand. - target = mergedBalloonTarget(level, cur, baseline, ceiling, rep) + target = mergedBalloonTarget(level, cur, baseline, ceiling, rep, swapPressure) } else { // No fresh report — guest still booting, image has no reporter, or // the agent died. Don't try to inflate: a target set before the @@ -235,7 +241,11 @@ func (v *vm) runBalloonController(ctx context.Context, dev *codevz.VirtioTraditi debug.Log("vz", "balloon apply", "id", v.spec.ID, "trigger", trigger, "level", level.String(), "target_mib", target/(1024*1024), "fresh_report", fresh, "changed", changed, - "guest_avail_mib", rep.AvailableKiB/1024, "guest_psi_centi", rep.PSIMemSomeCenti) + "guest_avail_mib", rep.AvailableKiB/1024, "guest_psi_centi", rep.PSIMemSomeCenti, + // Logged because "why is this guest not shrinking?" is otherwise + // unanswerable from the outside: the hold looks identical to the + // hysteresis band. + "guest_swap_used_mib", rep.SwapUsedKiB()/1024, "swap_pressure", swapPressure) } apply("init") @@ -248,6 +258,7 @@ func (v *vm) runBalloonController(ctx context.Context, dev *codevz.VirtioTraditi apply("pressure") case rep = <-reports: lastReport = time.Now() + swapPressure = swap.observe(rep) apply("report") case <-t.C: apply("tick")