diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a3fe28..732f075 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,6 +143,8 @@ jobs: config-file: ".markdown-link-check.ci.json" use-quiet-mode: "yes" folder-path: "." + check-modified-files-only: ${{ github.event_name == 'pull_request' && 'yes' || 'no' }} + base-branch: "main" # Validation job (pattern-specific) validate: diff --git a/.markdown-link-check.json b/.markdown-link-check.json index eee1753..0e32da8 100644 --- a/.markdown-link-check.json +++ b/.markdown-link-check.json @@ -47,6 +47,10 @@ { "_comment": "nvd.nist.gov (NIST National Vulnerability Database) is bot-hostile to the checker's unauthenticated HEAD probe (returns 0) but is a stable live gov site — referenced from the dependency-analysis skill.", "pattern": "^https://nvd\\.nist\\.gov" + }, + { + "_comment": "playbook.cio.gov is a stable federal reference, but CI runners cannot reliably connect to it (status 0 from markdown-link-check).", + "pattern": "^https://playbook\\.cio\\.gov" } ], "replacementPatterns": [], diff --git a/Makefile b/Makefile index 1990689..587cd32 100644 --- a/Makefile +++ b/Makefile @@ -36,12 +36,14 @@ validate: ## Run all validators (frontmatter + sensitive terms) validate-kits: ## Validate neutral acq-kits specs against the hybrid/v1 schema (strict: warnings fail) python integrations/isolation/acq-kits/validate-kits.py --strict -test-kits: ## Run the acq-kits node test suites (usai-provider generator + merge) + providers/usai emitters +test-kits: ## Run the acq-kits node test suites (usai-provider + paseo) + providers/usai emitters @if command -v node >/dev/null 2>&1; then \ - echo "==> usai-provider kit tests"; \ - cd integrations/isolation/acq-kits/usai-provider && node --test 'tests/**/*.test.mjs'; \ - echo "==> providers/usai catalog + emitter tests"; \ - cd "$(CURDIR)/integrations/providers/usai" && node --test 'tests/**/*.test.mjs'; \ + echo "==> usai-provider kit tests" && \ + (cd integrations/isolation/acq-kits/usai-provider && node --test 'tests/**/*.test.mjs') && \ + echo "==> paseo kit tests" && \ + (cd "$(CURDIR)/integrations/isolation/acq-kits/paseo" && node --test 'tests/**/*.test.mjs') && \ + echo "==> providers/usai catalog + emitter tests" && \ + (cd "$(CURDIR)/integrations/providers/usai" && node --test 'tests/**/*.test.mjs'); \ else \ echo "⚠️ node not installed — skipping acq-kits node tests"; \ fi diff --git a/integrations/isolation/acq-kits/paseo/README.md b/integrations/isolation/acq-kits/paseo/README.md index deec9fb..8c9a2cb 100644 --- a/integrations/isolation/acq-kits/paseo/README.md +++ b/integrations/isolation/acq-kits/paseo/README.md @@ -92,13 +92,19 @@ acq ports --publish 6868:6767 # another → host 6868 ## Projects are pre-populated from your mounts -Every **read-write** host directory you mount into the sandbox is registered with -the Paseo daemon at startup, so the web UI already lists your repos when you -connect — no manual **Add project** per directory. This runs on every start -(idempotent) and covers all mounts, not just the primary one the `acq run` -entrypoint opens. +Every **read-write** host directory you mount into the sandbox is inspected at +startup, so the web UI already lists your repos when you connect — no manual +**Add project** per directory. This runs on every start (idempotent) and covers +all mounts, not just the primary one the `acq run` entrypoint opens. - **Read-only mounts are skipped.** +- A mount that is itself a Git repo is registered as that project. +- A plain parent-directory mount with direct child Git repos registers those + child repos instead of the parent. This scan is intentionally shallow (direct + children only), follows direct-child symlinks that resolve to directories, and + `.git` may be either a directory or a file. +- A mount with no direct child Git repos falls back to registering the mount + itself, preserving support for non-git working directories. - The backend runtime dir (`/.msb`) and system mounts (`/etc`, `/run`, …) are excluded. - Discovery reads `/proc/mounts` and keys only on portable mount properties diff --git a/integrations/isolation/acq-kits/paseo/docs/decisions/prepopulate-projects-from-mounts.md b/integrations/isolation/acq-kits/paseo/docs/decisions/prepopulate-projects-from-mounts.md index cb9aa43..a0bdb3f 100644 --- a/integrations/isolation/acq-kits/paseo/docs/decisions/prepopulate-projects-from-mounts.md +++ b/integrations/isolation/acq-kits/paseo/docs/decisions/prepopulate-projects-from-mounts.md @@ -29,8 +29,8 @@ Two facts shape the solution: ## Decision Ship `paseo-register-mounts.mjs`, run from the startup supervisor after the -daemon answers `/api/health`, which registers each qualifying host directory with -the daemon. +daemon answers `/api/health`, which inspects each qualifying host directory and +registers the resulting project directories with the daemon. ### How it registers a project @@ -67,6 +67,26 @@ This keys entirely on portable mount properties, so it captures the three repos on both sbx and msb and needs no per-backend tokens. A future backend that bind-mounts host dirs as read-write `virtiofs` is covered automatically. +### How qualifying mounts become projects + +After mount discovery, each qualifying mount is expanded to the project +directories to register: + +1. If the mount itself has a `.git` entry, register the mount itself. This + preserves the existing behavior for the normal case where the mount is one + repository, and avoids unexpectedly registering submodules or nested repos. +2. Otherwise, inspect only the mount's direct child directories. If any child has + a `.git` entry, register each such child as its own project and do not register + the parent directory. +3. If no direct child Git repositories are found, register the mount itself. This + preserves support for intentionally-mounted non-git working directories. + +The `.git` entry may be either a directory or a file, covering normal clones, +worktrees, and submodules. The child scan is intentionally shallow, not +recursive. It follows direct-child symlinks that resolve to directories, which +keeps symlinked worktrees/submodules usable under an operator-selected mount +without traversing deeper than one child level. + ### Timing and cadence Runs on **every** sandbox start, in the background, after a bounded wait for @@ -99,11 +119,18 @@ means the UI shows fewer projects until the next start. - **Requiring a project marker (`.git`, `package.json`, …) in the dir.** Rejected as unnecessary: it would skip intentionally-mounted non-standard working dirs, and Paseo already records non-git dirs cleanly as `kind: "non_git"`. +- **Recursive scanning under parent mounts.** Rejected as surprising and + potentially expensive: it could register vendored repos, caches, test fixtures, + or deeply nested submodules the user did not intend to expose as top-level + Paseo projects. Direct children cover the common "one parent directory with + many sibling repos" workflow. ## Consequences - Every read-write host project mount is listed in the Paseo UI after any start - (detached `acq create` or interactive `acq run`), with no manual "Add project". + (detached `acq create` or interactive `acq run`), unless it is a plain parent + directory containing direct child Git repos. In that case, each direct child + repo is listed instead, with no manual "Add project". - Read-only mounts and the backend runtime dir are excluded. - The behavior is orthogonal to the worktrees-root pin (which still targets only the primary/first mount via the shim); this helper touches projects only, never diff --git a/integrations/isolation/acq-kits/paseo/files/home/paseo-register-mounts.mjs b/integrations/isolation/acq-kits/paseo/files/home/paseo-register-mounts.mjs index d776457..a97a627 100755 --- a/integrations/isolation/acq-kits/paseo/files/home/paseo-register-mounts.mjs +++ b/integrations/isolation/acq-kits/paseo/files/home/paseo-register-mounts.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node // paseo-register-mounts.mjs — pre-populate Paseo PROJECTS from the host -// directories mounted into this sandbox, so the web UI already lists them the -// moment you connect (no manual "Add project" per repo). +// directories mounted into this sandbox, so the web UI already lists mounted +// repos the moment you connect (no manual "Add project" per repo). // // WHY THIS EXISTS: acq bind-mounts one or more host project directories into the // guest. Paseo, however, only learns about a project when something opens it @@ -16,9 +16,13 @@ // * works for non-git dirs (registered as kind "non_git"), // * returns { project: null, errorCode: "directory_not_found" } for a bad path // instead of throwing. -// (Alternatives were rejected: `workspace create` mints a NEW workspace record -// every call — workspace spam across restarts — and `terminal create` leaves a -// stray terminal behind. See docs/decisions/prepopulate-projects-from-mounts.md.) +// Before registering, a qualifying mount is expanded: if the mount is itself a +// Git repo, register it; otherwise register direct child dirs that have a `.git` +// entry; if none exist, fall back to the mount itself. The child scan is shallow +// by design. (Alternatives were rejected: `workspace create` mints a NEW +// workspace record every call — workspace spam across restarts — and `terminal +// create` leaves a stray terminal behind. See +// docs/decisions/prepopulate-projects-from-mounts.md.) // // We import the CLI's OWN connector (dist/utils/client.js -> connectToDaemon) so // we reuse its socket/localhost resolution and need no host/port here. The CLI @@ -49,9 +53,15 @@ // connection is always closed, and the process ALWAYS exits 0 — even if the // daemon is unreachable or some adds fail. Errors are logged for diagnosis. -import { readFileSync, realpathSync, statSync } from "node:fs"; +import { + existsSync, + readdirSync, + readFileSync, + realpathSync, + statSync, +} from "node:fs"; import { execFileSync } from "node:child_process"; -import { dirname, basename, resolve } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { pathToFileURL } from "node:url"; // Target-path prefixes we never treat as a host project (system / pseudo mounts). @@ -100,9 +110,55 @@ function unescapeMountField(field) { ); } +export function hasGitEntry(dir) { + return existsSync(join(dir, ".git")); +} + +export function expandMountToProjectDirs(mount) { + // If the mount itself is a repo, preserve the existing one-mount-one-project + // behavior rather than expanding nested repos or submodules beneath it. + if (hasGitEntry(mount)) return [mount]; + + let entries; + try { + entries = readdirSync(mount, { withFileTypes: true }); + } catch { + return [mount]; + } + + const childRepos = []; + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + const child = join(mount, entry.name); + let isDir = entry.isDirectory(); + if (!isDir && entry.isSymbolicLink()) { + try { + isDir = statSync(child).isDirectory(); + } catch { + isDir = false; + } + } + if (isDir && hasGitEntry(child)) childRepos.push(child); + } + + return childRepos.length > 0 ? childRepos : [mount]; +} + +export function projectDirsFromMounts(mounts) { + const seen = new Set(); + const projects = []; + for (const mount of mounts) { + for (const project of expandMountToProjectDirs(mount)) { + if (seen.has(project)) continue; + seen.add(project); + projects.push(project); + } + } + return projects; +} + // Parse /proc/mounts and return the set of host-project target directories, // applying the backend-agnostic rule documented in the header. -function discoverProjectMounts() { +export function discoverProjectMounts() { let raw; try { raw = readFileSync("/proc/mounts", "utf8"); @@ -153,7 +209,11 @@ async function main() { log("no read-write host project mounts found; nothing to register"); return; } - log(`found ${mounts.length} host project mount(s): ${mounts.join(", ")}`); + log(`found ${mounts.length} qualifying host mount(s): ${mounts.join(", ")}`); + const projects = projectDirsFromMounts(mounts); + log( + `registering ${projects.length} project director${projects.length === 1 ? "y" : "ies"}: ${projects.join(", ")}`, + ); let connectToDaemon; try { @@ -172,7 +232,7 @@ async function main() { } try { - for (const dir of mounts) { + for (const dir of projects) { try { const res = await client.addProject(dir); if (res && res.project && res.project.projectId) { @@ -192,10 +252,15 @@ async function main() { } // Always exit 0 — never let project pre-population fail the sandbox. -main() - .catch((err) => { - log(`unexpected error: ${err && err.message ? err.message : String(err)}`); - }) - .finally(() => { - process.exit(0); - }); +if ( + process.argv[1] && + import.meta.url === pathToFileURL(resolve(process.argv[1])).href +) { + main() + .catch((err) => { + log(`unexpected error: ${err && err.message ? err.message : String(err)}`); + }) + .finally(() => { + process.exit(0); + }); +} diff --git a/integrations/isolation/acq-kits/paseo/scripts/verify b/integrations/isolation/acq-kits/paseo/scripts/verify old mode 100755 new mode 100644 index 97abcad..d85f500 --- a/integrations/isolation/acq-kits/paseo/scripts/verify +++ b/integrations/isolation/acq-kits/paseo/scripts/verify @@ -60,9 +60,12 @@ WORK="$(mktemp -d "$KIT_DIR/verify-workspace.XXXXXX")" CREATE_LOG="$WORK/create.log" # Extra workspaces mounted alongside the primary to exercise the registrar's # read-write vs read-only distinction in step 10: RO_FIXTURE is mounted ":ro" -# (must be SKIPPED), RW_FIXTURE is mounted read-write (must be REGISTERED). +# (must be SKIPPED), RW_FIXTURE is mounted read-write and shaped like a parent +# directory containing repos (its child repos must be REGISTERED). RW_FIXTURE="$(mktemp -d "$KIT_DIR/verify-rw-mount.XXXXXX")" RO_FIXTURE="$(mktemp -d "$KIT_DIR/verify-ro-mount.XXXXXX")" +RW_CHILD_REPO_A="$RW_FIXTURE/repo-a" +RW_CHILD_REPO_B="$RW_FIXTURE/repo-b" KEEP="${KEEP:-0}" RUN_ACQ="${RUN_ACQ:-0}" RUN_SBX="${RUN_SBX:-0}" @@ -77,6 +80,9 @@ POLL_INTERVAL="${POLL_INTERVAL:-2}" # seconds after install completes so a broken daemon fails quickly. POST_INSTALL_GRACE="${POST_INSTALL_GRACE:-25}" +mkdir -p "$RW_CHILD_REPO_A/.git" "$RW_CHILD_REPO_B" "$RW_FIXTURE/scratch-not-a-repo" +printf 'gitdir: /tmp/paseo-verify-repo-b.git\n' > "$RW_CHILD_REPO_B/.git" + pass=0 fail=0 ok() { printf ' \033[32mPASS\033[0m %s\n' "$1"; pass=$((pass+1)); } @@ -667,12 +673,15 @@ fi info "10. Projects pre-populated from the mounted host directories" # The startup registrar (paseo-register-mounts.mjs) registers each read-write -# host project directory mounted into the sandbox with the daemon. Assert that -# EVERY such mount shows up as a project in the daemon's registry, and — using -# the rw/ro fixtures we mounted at create (step 2) — that a READ-ONLY mount is -# SKIPPED and a READ-WRITE one is REGISTERED. We derive the expected set the same -# backend-agnostic way the helper does — from /proc/mounts inside the guest — so -# the test does not hard-code msb/sbx mount source names. +# host project directory mounted into the sandbox with the daemon. If a mount is +# a plain parent directory containing direct child Git repos, those child repos +# are registered instead of the parent; mounts without child repos still fall +# back to the mount itself. Assert that EVERY expected project shows up in the +# daemon's registry, and — using the rw/ro fixtures we mounted at create (step 2) +# — that a READ-ONLY mount is SKIPPED while child repos under a READ-WRITE parent +# are REGISTERED. We derive the expected set the same backend-agnostic way the +# helper does — from /proc/mounts inside the guest — so the test does not +# hard-code msb/sbx mount source names. # # We read the registered projects from the daemon's own API (the same # `client.listProjects()` the registrar's client library speaks), NOT from the @@ -704,18 +713,33 @@ else } rw_base="$(basename "$RW_FIXTURE")" + rw_child_a_base="$(basename "$RW_CHILD_REPO_A")" + rw_child_b_base="$(basename "$RW_CHILD_REPO_B")" ro_base="$(basename "$RO_FIXTURE")" - # Expected set: fstype virtiofs, rw, target is a dir, not under a system prefix, - # basename not starting with "." (matches the registrar's rule). - expected_mounts="$(in_sbx ' - awk "\$3==\"virtiofs\" && \$4 ~ /^rw/ {print \$2}" /proc/mounts \ - | while read -r t; do - case "$t" in /etc|/etc/*|/run|/run/*|/proc|/proc/*|/sys|/sys/*|/dev|/dev/*|/) continue ;; esac - case "$(basename "$t")" in .*) continue ;; esac - [ -d "$t" ] && echo "$t" - done | sort -u')" - if [ -z "$expected_mounts" ]; then + # Expected set: qualifying rw virtiofs mounts, expanded by the actual registrar + # helper so this live check cannot drift from paseo-register-mounts.mjs. + expected_projects="$(in_sbx ' + node --input-type=module -e " + import { readFileSync, statSync } from \"node:fs\"; + import { basename } from \"node:path\"; + import { projectDirsFromMounts } from \"file://$HOME/paseo-register-mounts.mjs\"; + const systemPrefixes = [\"/etc\", \"/run\", \"/proc\", \"/sys\", \"/dev\"]; + const unescapeMountField = (field) => field.replace(/\\\\(\\d{3})/g, (_, oct) => String.fromCharCode(parseInt(oct, 8))); + const underSystemPrefix = (target) => target === \"/\" || systemPrefixes.some((prefix) => target === prefix || target.startsWith(prefix + \"/\")); + const mounts = []; + for (const line of readFileSync(\"/proc/mounts\", \"utf8\").split(\"\\n\")) { + if (!line) continue; + const parts = line.split(/\\s+/); + if (parts.length < 4 || parts[2] !== \"virtiofs\" || !parts[3].startsWith(\"rw\")) continue; + const target = unescapeMountField(parts[1]); + if (underSystemPrefix(target) || basename(target).startsWith(\".\")) continue; + try { if (!statSync(target).isDirectory()) continue; } catch { continue; } + mounts.push(target); + } + for (const project of projectDirsFromMounts(mounts).sort()) process.stdout.write(project + \"\\n\"); + "')" + if [ -z "$expected_projects" ]; then bad "no read-write host project mounts found in the guest — the primary + rw fixture should be present" in_sbx 'grep virtiofs /proc/mounts' | sed 's/^/ | /' else @@ -727,7 +751,7 @@ else while :; do project_roots="$(list_project_roots)" missing="" - for m in $expected_mounts; do + for m in $expected_projects; do printf '%s\n' "$project_roots" | grep -qx "$m" || missing="$missing $m" done [ -z "$missing" ] && break @@ -735,18 +759,27 @@ else sleep "$POLL_INTERVAL" done if [ -z "$missing" ]; then - _n="$(printf '%s\n' "$expected_mounts" | grep -c .)" - ok "all $_n read-write host project mount(s) registered as projects (via daemon API)" + _n="$(printf '%s\n' "$expected_projects" | grep -c .)" + ok "all $_n expected project path(s) registered as projects (via daemon API)" else bad "some host project mounts were not registered:$missing" in_sbx 'tail -n 20 ~/.local/state/paseo/paseo-register.log 2>/dev/null' | sed 's/^/ | /' fi - # POSITIVE: the read-write fixture must be registered. + # POSITIVE: child repos under the read-write parent fixture must be registered. + if printf '%s\n' "$project_roots" | grep -q "${rw_base}/${rw_child_a_base}\$" && \ + printf '%s\n' "$project_roots" | grep -q "${rw_base}/${rw_child_b_base}\$"; then + ok "child repos under read-write parent fixture ($rw_base) registered as projects" + else + bad "child repos under read-write parent fixture ($rw_base) were NOT registered" + fi + + # NEGATIVE: the parent fixture should not be registered when direct child repos + # were found and registered instead. if printf '%s\n' "$project_roots" | grep -q "${rw_base}\$"; then - ok "read-write fixture ($rw_base) registered as a project" + bad "read-write parent fixture ($rw_base) was registered instead of only its child repos" else - bad "read-write fixture ($rw_base) was NOT registered" + ok "read-write parent fixture ($rw_base) correctly expanded to child repos" fi # NEGATIVE: the read-only fixture must be SKIPPED. Only assert this if the diff --git a/integrations/isolation/acq-kits/paseo/tests/paseo-register-mounts.test.mjs b/integrations/isolation/acq-kits/paseo/tests/paseo-register-mounts.test.mjs new file mode 100644 index 0000000..726f1ac --- /dev/null +++ b/integrations/isolation/acq-kits/paseo/tests/paseo-register-mounts.test.mjs @@ -0,0 +1,82 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, mkdir, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { + expandMountToProjectDirs, + hasGitEntry, + projectDirsFromMounts, +} from "../files/home/paseo-register-mounts.mjs"; + +async function tempDir() { + return mkdtemp(path.join(os.tmpdir(), "paseo-register-mounts-")); +} + +test("hasGitEntry accepts .git directories", async () => { + const dir = await tempDir(); + await mkdir(path.join(dir, ".git")); + + assert.equal(hasGitEntry(dir), true); +}); + +test("hasGitEntry accepts .git files", async () => { + const dir = await tempDir(); + await writeFile(path.join(dir, ".git"), "gitdir: /tmp/example\n"); + + assert.equal(hasGitEntry(dir), true); +}); + +test("mount with no direct child repos falls back to mount itself", async () => { + const mount = await tempDir(); + await mkdir(path.join(mount, "scratch")); + + assert.deepEqual(expandMountToProjectDirs(mount), [mount]); +}); + +test("mount with direct child repos expands to those repos only", async () => { + const mount = await tempDir(); + const repoA = path.join(mount, "repo-a"); + const repoB = path.join(mount, "repo-b"); + const scratch = path.join(mount, "scratch-not-a-repo"); + await mkdir(path.join(repoA, ".git"), { recursive: true }); + await mkdir(repoB); + await writeFile(path.join(repoB, ".git"), "gitdir: /tmp/example\n"); + await mkdir(scratch); + + assert.deepEqual(expandMountToProjectDirs(mount).sort(), [repoA, repoB].sort()); +}); + +test("mount that is itself a repo is not expanded into nested repos", async () => { + const mount = await tempDir(); + await mkdir(path.join(mount, ".git")); + await mkdir(path.join(mount, "nested", ".git"), { recursive: true }); + + assert.deepEqual(expandMountToProjectDirs(mount), [mount]); +}); + +test("direct child symlink to repo directory is accepted", async () => { + const mount = await tempDir(); + const externalRepo = await tempDir(); + await mkdir(path.join(externalRepo, ".git")); + const linkedRepo = path.join(mount, "linked-repo"); + await symlink(externalRepo, linkedRepo); + + assert.deepEqual(expandMountToProjectDirs(mount), [linkedRepo]); +}); + +test("missing mount falls back without throwing", async () => { + const dir = await tempDir(); + const missing = path.join(dir, "missing"); + + assert.deepEqual(expandMountToProjectDirs(missing), [missing]); +}); + +test("projectDirsFromMounts de-duplicates expanded project paths", async () => { + const mount = await tempDir(); + const repo = path.join(mount, "repo"); + await mkdir(path.join(repo, ".git"), { recursive: true }); + + assert.deepEqual(projectDirsFromMounts([mount, repo]), [repo]); +});