From d73bac39b1fce9a1f327a4797b21cdc808affcd3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:34:21 +0000 Subject: [PATCH 1/2] fix(codeapi): pin proxy-addr to 2.0.8 (#4534) Source: ClickHouse/ai@4215c459945b0de73db0f99d6d86d8b470b96572 --- api/bun.lock | 5 ++++- api/package-lock.json | 19 ++++++++++++------- api/package.json | 5 ++++- api/src/api/session-inputs.routes.test.ts | 1 + service/bun.lock | 3 ++- service/package.json | 3 ++- 6 files changed, 25 insertions(+), 11 deletions(-) diff --git a/api/bun.lock b/api/bun.lock index b2003e30..6e8b5dda 100644 --- a/api/bun.lock +++ b/api/bun.lock @@ -24,6 +24,9 @@ }, }, }, + "overrides": { + "proxy-addr": "2.0.8", + }, "packages": { "@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="], @@ -185,7 +188,7 @@ "prom-client": ["prom-client@15.1.3", "", { "dependencies": { "@opentelemetry/api": "^1.4.0", "tdigest": "^0.1.1" } }, "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g=="], - "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], + "proxy-addr": ["proxy-addr@2.0.8", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ=="], "qs": ["qs@6.16.0", "", { "dependencies": { "es-define-property": "^1.0.1", "side-channel": "^1.1.1" } }, "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA=="], diff --git a/api/package-lock.json b/api/package-lock.json index 8cf48ae4..a2dbad66 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -1008,15 +1008,20 @@ } }, "node_modules/proxy-addr": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", - "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", + "integrity": "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==", + "license": "MIT", "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" }, "engines": { "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/qs": { @@ -1756,7 +1761,7 @@ "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", - "proxy-addr": "^2.0.7", + "proxy-addr": "2.0.8", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", @@ -2009,9 +2014,9 @@ } }, "proxy-addr": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", - "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", + "integrity": "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==", "requires": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" diff --git a/api/package.json b/api/package.json index a27fd780..4b455865 100644 --- a/api/package.json +++ b/api/package.json @@ -27,5 +27,8 @@ "@types/semver": "^7.5.8", "typescript": "^5.7.3" }, - "license": "Apache-2.0" + "license": "Apache-2.0", + "overrides": { + "proxy-addr": "2.0.8" + } } diff --git a/api/src/api/session-inputs.routes.test.ts b/api/src/api/session-inputs.routes.test.ts index 48fe7d53..0ba649e4 100644 --- a/api/src/api/session-inputs.routes.test.ts +++ b/api/src/api/session-inputs.routes.test.ts @@ -26,6 +26,7 @@ let baseUrl: string; const savedSessionWorkspaceEnabled = config.session_workspace_enabled; beforeAll(async () => { + await fsp.rm(SESSION_INPUT_CACHE_DIR, { recursive: true, force: true }); config.session_workspace_enabled = true; const app = express(); /* Mirror index.ts: no global JSON parser, router mounted under /api/v2. */ diff --git a/service/bun.lock b/service/bun.lock index 9aacef7e..ce6518ee 100644 --- a/service/bun.lock +++ b/service/bun.lock @@ -52,6 +52,7 @@ }, "overrides": { "decode-uri-component": "0.5.0", + "proxy-addr": "2.0.8", "qs": "6.16.0", "uuid": "11.1.1", }, @@ -852,7 +853,7 @@ "prom-client": ["prom-client@15.1.3", "", { "dependencies": { "@opentelemetry/api": "^1.4.0", "tdigest": "^0.1.1" } }, "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g=="], - "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], + "proxy-addr": ["proxy-addr@2.0.8", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ=="], "proxy-from-env": ["proxy-from-env@2.1.0", "", {}, "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA=="], diff --git a/service/package.json b/service/package.json index a3679c8f..caa64e1b 100644 --- a/service/package.json +++ b/service/package.json @@ -76,6 +76,7 @@ "overrides": { "decode-uri-component": "0.5.0", "uuid": "11.1.1", - "qs": "6.16.0" + "qs": "6.16.0", + "proxy-addr": "2.0.8" } } From e1cc755445b0945fd511757eb187e88552e984b0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:58:46 +0000 Subject: [PATCH 2/2] ci(codeapi): mirror upstream node runtime versions Source: ClickHouse/ai@5f963906ee74d3247802b52ac2cdb8f85a9fcfda --- .github/workflows/ci.yml | 8 +- .github/workflows/extraction.yml | 25 + api/Dockerfile | 6 +- api/src/job.ts | 7 +- api/src/package-init.test.ts | 198 +++++ api/src/pipe-process.test.ts | 12 +- api/src/runtime-module-links.test.ts | 207 ++++++ api/src/workspace-command.test.ts | 79 +- api/src/workspace-command.ts | 6 +- build-packages.sh | 32 +- docker-compose.yaml | 1 + docker/Dockerfile.worker-sandbox | 4 +- docker/package-init.sh | 80 +- docs/adr/0014-workspace-file-import.md | 65 ++ docs/remote-bridge/README.md | 5 + docs/remote-bridge/worker-runbook.md | 16 + helm/codeapi/values.yaml | 2 +- launcher/Dockerfile | 4 +- packages/code/Dockerfile | 4 +- packages/code/README.md | 21 + packages/code/package.json | 4 + .../code/src/browser-enrollment-protocol.ts | 253 +++++++ packages/code/src/browser-protection.test.ts | 31 + packages/code/src/browser-setup-cli.ts | 103 +++ packages/code/src/browser-setup.test.ts | 279 +++++++ packages/code/src/browser-setup.ts | 352 +++++++++ packages/code/src/cli.ts | 88 ++- packages/code/src/command-diagnostics.test.ts | 165 +++++ packages/code/src/command-diagnostics.ts | 155 ++++ packages/code/src/credential-recovery.test.ts | 701 ++++++++++++++++++ packages/code/src/credential-recovery.ts | 223 ++++++ packages/code/src/index.ts | 1 + packages/code/src/linked-worktrees.ts | 4 +- packages/code/src/native-pool.test.ts | 31 + packages/code/src/native-pool.ts | 29 +- packages/code/src/native-process-child.ts | 9 + packages/code/src/native-process.test.ts | 90 +++ packages/code/src/native-process.ts | 39 +- packages/code/src/native-programmatic.test.ts | 113 ++- packages/code/src/native-programmatic.ts | 58 +- packages/code/src/native-sandbox.test.ts | 29 +- packages/code/src/native-sandbox.ts | 22 +- packages/code/src/pairing.test.ts | 58 ++ packages/code/src/pairing.ts | 41 +- packages/code/src/protocol.ts | 215 +++++- packages/code/src/relay.ts | 15 + packages/code/src/setup-cli.test.ts | 266 +++++++ packages/code/src/setup-cli.ts | 284 +++++++ packages/code/src/setup.test.ts | 334 +++++++++ packages/code/src/setup.ts | 254 +++++++ packages/code/src/storage.ts | 46 +- packages/code/src/worker-profile.test.ts | 529 +++++++++++++ packages/code/src/worker-profile.ts | 200 +++++ packages/code/src/worker.test.ts | 20 +- packages/code/src/worker.ts | 112 ++- packages/code/src/workspace-import.test.ts | 565 ++++++++++++++ packages/code/src/workspace-import.ts | 188 +++++ packages/code/src/workspace-instances.ts | 4 +- packages/code/src/workspace-worker.test.ts | 68 ++ packages/code/src/workspace.test.ts | 4 +- packages/code/src/workspace.ts | 432 ++++++++++- packages/extraction/.gitattributes | 2 + packages/extraction/.gitignore | 1 + packages/extraction/Dockerfile | 32 + packages/extraction/compose.yaml | 22 + packages/extraction/native/guard.c | 156 ++++ packages/extraction/package-lock.json | 64 ++ packages/extraction/package.json | 22 + packages/extraction/src/backend.ts | 166 +++++ packages/extraction/src/boundary.test.ts | 22 + packages/extraction/src/health.ts | 15 + packages/extraction/src/main.ts | 64 ++ packages/extraction/src/protocol.test.ts | 68 ++ packages/extraction/src/protocol.ts | 107 +++ packages/extraction/src/server.test.ts | 267 +++++++ packages/extraction/src/server.ts | 260 +++++++ packages/extraction/tests/adversarial.py | 488 ++++++++++++ packages/extraction/tests/block-landlock.c | 21 + .../extraction/tests/container-client.mjs | 303 ++++++++ packages/extraction/tests/container.sh | 61 ++ .../extraction/tests/fixtures/aggregate.pdf | Bin 0 -> 37605 bytes packages/extraction/tests/fixtures/body.docx | Bin 0 -> 36732 bytes .../extraction/tests/fixtures/compressed.pdf | Bin 0 -> 707 bytes packages/extraction/tests/fixtures/empty.docx | Bin 0 -> 36563 bytes packages/extraction/tests/fixtures/empty.pdf | Bin 0 -> 660 bytes .../extraction/tests/fixtures/generate.py | 37 + .../extraction/tests/fixtures/inflate.pdf | Bin 0 -> 8852 bytes packages/extraction/tests/fixtures/pages.pdf | Bin 0 -> 950 bytes packages/extraction/tests/isolation.mjs | 64 ++ packages/extraction/tests/isolation.py | 73 ++ packages/extraction/tsconfig.json | 15 + packages/extraction/worker/extract.py | 503 +++++++++++++ packages/extraction/worker/requirements.txt | 4 + service/Dockerfile.node | 4 +- service/openapi.yml | 7 +- service/rollup.config.js | 1 + .../bridge/browser-enrollment-router.test.ts | 184 +++++ service/src/bridge/browser-enrollment.test.ts | 447 +++++++++++ service/src/bridge/enabled.ts | 5 + service/src/bridge/index.ts | 3 +- service/src/bridge/pairing.ts | 337 ++++++++- service/src/bridge/router.test.ts | 47 ++ service/src/bridge/router.ts | 76 ++ service/src/bridge/store.ts | 3 +- service/src/config.spec.ts | 17 +- service/src/config.ts | 11 +- service/src/session-ownership.ts | 2 +- service/src/workspace-tools/import.test.ts | 295 ++++++++ service/src/workspace-tools/import.ts | 182 +++++ service/src/workspace-tools/index.ts | 13 +- service/src/workspace-tools/requests.ts | 29 +- service/src/workspace-tools/router.ts | 87 ++- tests/compose-bridge-config.cjs | 6 + tests/package_init_runtime_rollout.sh | 5 + tests/test_tool_call_pipes.py | 36 + tests/tool_call_pipes.py | 30 +- 116 files changed, 11688 insertions(+), 172 deletions(-) create mode 100644 .github/workflows/extraction.yml create mode 100644 api/src/package-init.test.ts create mode 100644 api/src/runtime-module-links.test.ts create mode 100644 docs/adr/0014-workspace-file-import.md create mode 100644 packages/code/src/browser-enrollment-protocol.ts create mode 100644 packages/code/src/browser-protection.test.ts create mode 100644 packages/code/src/browser-setup-cli.ts create mode 100644 packages/code/src/browser-setup.test.ts create mode 100644 packages/code/src/browser-setup.ts create mode 100644 packages/code/src/command-diagnostics.test.ts create mode 100644 packages/code/src/command-diagnostics.ts create mode 100644 packages/code/src/credential-recovery.test.ts create mode 100644 packages/code/src/credential-recovery.ts create mode 100644 packages/code/src/setup-cli.test.ts create mode 100644 packages/code/src/setup-cli.ts create mode 100644 packages/code/src/setup.test.ts create mode 100644 packages/code/src/setup.ts create mode 100644 packages/code/src/worker-profile.test.ts create mode 100644 packages/code/src/worker-profile.ts create mode 100644 packages/code/src/workspace-import.test.ts create mode 100644 packages/code/src/workspace-import.ts create mode 100644 packages/extraction/.gitattributes create mode 100644 packages/extraction/.gitignore create mode 100644 packages/extraction/Dockerfile create mode 100644 packages/extraction/compose.yaml create mode 100644 packages/extraction/native/guard.c create mode 100644 packages/extraction/package-lock.json create mode 100644 packages/extraction/package.json create mode 100644 packages/extraction/src/backend.ts create mode 100644 packages/extraction/src/boundary.test.ts create mode 100644 packages/extraction/src/health.ts create mode 100644 packages/extraction/src/main.ts create mode 100644 packages/extraction/src/protocol.test.ts create mode 100644 packages/extraction/src/protocol.ts create mode 100644 packages/extraction/src/server.test.ts create mode 100644 packages/extraction/src/server.ts create mode 100644 packages/extraction/tests/adversarial.py create mode 100644 packages/extraction/tests/block-landlock.c create mode 100644 packages/extraction/tests/container-client.mjs create mode 100755 packages/extraction/tests/container.sh create mode 100644 packages/extraction/tests/fixtures/aggregate.pdf create mode 100644 packages/extraction/tests/fixtures/body.docx create mode 100644 packages/extraction/tests/fixtures/compressed.pdf create mode 100644 packages/extraction/tests/fixtures/empty.docx create mode 100644 packages/extraction/tests/fixtures/empty.pdf create mode 100644 packages/extraction/tests/fixtures/generate.py create mode 100644 packages/extraction/tests/fixtures/inflate.pdf create mode 100644 packages/extraction/tests/fixtures/pages.pdf create mode 100644 packages/extraction/tests/isolation.mjs create mode 100644 packages/extraction/tests/isolation.py create mode 100644 packages/extraction/tsconfig.json create mode 100644 packages/extraction/worker/extract.py create mode 100644 packages/extraction/worker/requirements.txt create mode 100644 service/src/bridge/browser-enrollment-router.test.ts create mode 100644 service/src/bridge/browser-enrollment.test.ts create mode 100644 service/src/workspace-tools/import.test.ts create mode 100644 service/src/workspace-tools/import.ts create mode 100644 tests/test_tool_call_pipes.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0191f0a..28148752 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,7 +108,7 @@ jobs: - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 with: - node-version: 22 + node-version: 22.23.4 - name: Compile anonymous-pipe addon run: | @@ -306,7 +306,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: ['22.21.0', '24.16.0'] + node-version: ['22.23.4', '24.22.1'] defaults: run: working-directory: packages/code @@ -343,7 +343,7 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 with: - node-version: 24.16.0 + node-version: 24.22.1 - run: npm ci - run: npm run build - name: Selected project root containment tests @@ -370,7 +370,7 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 with: - node-version: 24.16.0 + node-version: 24.22.1 - name: Install bubblewrap run: | sudo apt-get update -qq diff --git a/.github/workflows/extraction.yml b/.github/workflows/extraction.yml new file mode 100644 index 00000000..0d9a529d --- /dev/null +++ b/.github/workflows/extraction.yml @@ -0,0 +1,25 @@ +name: Extraction Service +on: + pull_request: + paths: ['packages/extraction/**', '.github/workflows/extraction.yml'] + push: + branches: [main] + paths: ['packages/extraction/**', '.github/workflows/extraction.yml'] +permissions: + contents: read +jobs: + extraction: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 + with: + node-version: 24.22.1 + cache: npm + cache-dependency-path: packages/extraction/package-lock.json + - run: npm ci --ignore-scripts + working-directory: packages/extraction + - run: npm run format:check && npm run typecheck && npm test + working-directory: packages/extraction + - run: npm run test:container + working-directory: packages/extraction diff --git a/api/Dockerfile b/api/Dockerfile index d630c547..40429b72 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -35,7 +35,7 @@ COPY api/src/spec-guard.c /tmp/spec-guard.c COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c COPY api/src/anonymous-pipes.c /tmp/anonymous-pipes.c COPY api/src/supervisor-policy.c /tmp/supervisor-policy.c -COPY --from=node:22-slim /usr/local/include/node /usr/local/include/node +COPY --from=node:22.23.4-slim /usr/local/include/node /usr/local/include/node COPY docker/rootfs-setup.c /tmp/rootfs-setup.c RUN gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I/usr/local/include/node -o /usr/local/lib/anonymous-pipes.node /tmp/anonymous-pipes.c \ && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/sandbox-supervisor-policy /tmp/supervisor-policy.c \ @@ -49,7 +49,7 @@ FROM nsjail-builder AS nsjail-ipc-test ENV SANDBOX_PIPE_ADDON=/usr/local/lib/anonymous-pipes.node COPY docker/python-runtime /python-runtime RUN python3 /python-runtime/install.py -COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node +COPY --from=node:22.23.4-slim /usr/local/bin/node /usr/local/bin/node COPY --from=oven/bun:1.4.2-debian /usr/local/bin/bun /usr/local/bin/bun # ============================================================================ @@ -138,7 +138,7 @@ RUN chmod +x /usr/sbin/nsjail # The per-invocation broker starts Node after restore, so its OpenSSL process # state is not captured in the MicroVM snapshot. Pulling the Node binary # is ~30 MB and avoids inheriting npm/node_modules from a full apt install. -COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node +COPY --from=node:22.23.4-slim /usr/local/bin/node /usr/local/bin/node WORKDIR /sandbox_api diff --git a/api/src/job.ts b/api/src/job.ts index bfa99b63..88d3e5b4 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -169,8 +169,11 @@ export function ensureNodeModulesSymlink( if (!nodeModulesPath) return; const linkPath = path.join(submissionDir, 'node_modules'); try { - fs.lstatSync(linkPath); - return; + if (!fs.lstatSync(linkPath).isSymbolicLink()) return; + const target = path.resolve(submissionDir, fs.readlinkSync(linkPath)); + const packageRoot = `${path.resolve(config.packages_directory)}${path.sep}`; + if (!target.startsWith(packageRoot) || target === path.resolve(nodeModulesPath)) return; + fs.unlinkSync(linkPath); } catch (err) { if (errorCode(err) !== 'ENOENT') throw err; } diff --git a/api/src/package-init.test.ts b/api/src/package-init.test.ts new file mode 100644 index 00000000..a9431bd2 --- /dev/null +++ b/api/src/package-init.test.ts @@ -0,0 +1,198 @@ +import { expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +const initializer = fs.readFileSync( + path.resolve(__dirname, '../../docker/package-init.sh'), + 'utf8' +); + +for (const oldRuntimeExists of [false, true]) { + test(`cached PVC refreshes Node and Bun launchers with ${ + oldRuntimeExists ? 'retained' : 'removed' + } old runtimes`, () => { + const root = fs.mkdtempSync( + path.join(os.tmpdir(), 'cached-package-init-') + ); + try { + const packages = path.join(root, 'pkgs'); + const workspace = path.join(root, 'workspace'); + const manifest = path.join(root, 'javascript-packages.txt'); + fs.mkdirSync(workspace); + fs.writeFileSync(manifest, 'runtime-fixture@1.0.0\n'); + const runtimes = [ + { + directory: 'node/24.22.1', + binary: 'bin/node', + args: ['hello world', '--version'], + }, + { + directory: 'bun/1.4.2', + binary: 'bun', + args: ['run', 'hello world', '--version'], + }, + ]; + for (const runtime of runtimes) { + const destination = path.join(packages, runtime.directory); + const fixture = path.join( + destination, + 'node_modules/runtime-fixture' + ); + fs.mkdirSync(fixture, { recursive: true }); + fs.mkdirSync( + path.dirname(path.join(destination, runtime.binary)), + { recursive: true } + ); + fs.writeFileSync( + path.join(fixture, 'package.json'), + '{"version": "1.0.0"}' + ); + fs.writeFileSync( + path.join(destination, '.package-installed'), + 'installed' + ); + fs.writeFileSync( + path.join(destination, '.env'), + 'preserve runtime environment' + ); + fs.writeFileSync( + path.join(destination, runtime.binary), + '#!/bin/bash\nprintf "<%s>\\n" "$@"\n', + { mode: 0o755 } + ); + fs.writeFileSync( + path.join(destination, 'run'), + '#!/bin/bash\nexit 99\n', + { mode: 0o755 } + ); + } + const python = path.join(packages, 'python/3.14.4'); + for (const name of [ + 'PIL', + 'markitdown', + 'chdb', + 'statsmodels', + 'rasterio', + ]) { + fs.mkdirSync( + path.join(python, 'lib/python3.14/site-packages', name), + { recursive: true } + ); + } + fs.writeFileSync( + path.join(python, 'lib/python3.14/.sandbox-pipe-runtime-v1'), + 'installed' + ); + fs.writeFileSync( + path.join(python, '.package-installed'), + 'installed' + ); + fs.mkdirSync(path.join(packages, 'bash/5.2.0'), { + recursive: true, + }); + fs.writeFileSync( + path.join(packages, 'bash/5.2.0/.package-installed'), + 'installed' + ); + fs.writeFileSync( + path.join(packages, '.initialized'), + 'original initialization' + ); + + const forbidden = path.join(root, 'forbidden'); + fs.mkdirSync(forbidden); + for (const binary of [ + 'curl', + 'wget', + 'make', + 'npm', + 'tar', + 'unzip', + ]) { + fs.writeFileSync( + path.join(forbidden, binary), + '#!/bin/bash\necho "unexpected reinstall" >&2\nexit 98\n', + { mode: 0o755 } + ); + } + const script = path.join(root, 'package-init.sh'); + fs.writeFileSync( + script, + initializer + .split('/pkgs') + .join(packages) + .split('/mnt/data/node_modules') + .join(path.join(workspace, 'node_modules')) + ); + const env = { + ...process.env, + PATH: `${forbidden}:${process.env.PATH}`, + FORCE_REBUILD: 'false', + PYTHON_VERSION: '3.14.4', + NODE_VERSION: '24.22.1', + BUN_VERSION: '1.4.2', + BASH_PACKAGE_VERSION: '5.2.0', + JS_PACKAGE_MANIFEST: manifest, + }; + for (let pass = 0; pass < 2; pass += 1) { + const initialized = Bun.spawnSync({ + cmd: ['bash', script], + env, + }); + expect(initialized.stderr.toString()).toBe(''); + expect(initialized.exitCode).toBe(0); + expect(initialized.stdout.toString()).toContain( + 'Packages already initialized' + ); + expect( + fs.readFileSync(path.join(packages, '.initialized'), 'utf8') + ).toBe('original initialization'); + for (const runtime of runtimes) { + const destination = path.join(packages, runtime.directory); + const oldModules = path.join( + packages, + runtime.directory.split('/')[0], + '0.0.1/node_modules' + ); + if (oldRuntimeExists) + fs.mkdirSync(oldModules, { recursive: true }); + const link = path.join(workspace, 'node_modules'); + fs.rmSync(link, { force: true }); + fs.symlinkSync(oldModules, link); + const launched = Bun.spawnSync({ + cmd: [ + path.join(destination, 'run'), + 'hello world', + '--version', + ], + }); + expect(launched.stderr.toString()).toBe(''); + expect(launched.exitCode).toBe(0); + expect(launched.stdout.toString()).toBe( + runtime.args.map(arg => `<${arg}>\n`).join('') + ); + expect(fs.readlinkSync(link)).toBe( + path.join(destination, 'node_modules') + ); + expect( + fs.readFileSync( + path.join(destination, '.package-installed'), + 'utf8' + ) + ).toBe('installed'); + expect( + fs.readFileSync(path.join(destination, '.env'), 'utf8') + ).toBe('preserve runtime environment'); + expect( + fs + .readdirSync(destination) + .some(name => name.startsWith('.run.')) + ).toBe(false); + } + } + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +} diff --git a/api/src/pipe-process.test.ts b/api/src/pipe-process.test.ts index 608002c8..52d904ea 100644 --- a/api/src/pipe-process.test.ts +++ b/api/src/pipe-process.test.ts @@ -79,11 +79,19 @@ describe('anonymous process pipes', () => { if (process.platform === 'linux') { // Runtime watcher/pid descriptors can appear asynchronously. Assert the // exact pipe inodes we allocated, including duplicates held by FileSink. - const survivors = fs.readdirSync('/proc/self/fd').flatMap(fd => { + const survivors = () => fs.readdirSync('/proc/self/fd').flatMap(fd => { try { const inode = fs.readlinkSync(`/proc/self/fd/${fd}`); return ownedPipes.has(inode) ? [inode] : []; } catch { return []; } }); - expect(survivors).toEqual([]); + // Bun may finish releasing internal FD duplicates after child close. + // Allow bounded asynchronous cleanup, but still reject persistent leaks. + const deadline = performance.now() + 1000; + let remaining = survivors(); + while (remaining.length && performance.now() < deadline) { + await new Promise(resolve => setTimeout(resolve, 10)); + remaining = survivors(); + } + expect(remaining).toEqual([]); } }, 10000); }); diff --git a/api/src/runtime-module-links.test.ts b/api/src/runtime-module-links.test.ts new file mode 100644 index 00000000..6ef754ab --- /dev/null +++ b/api/src/runtime-module-links.test.ts @@ -0,0 +1,207 @@ +import { expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { config } from './config'; +import { ensureNodeModulesSymlink } from './job'; + +function fixture( + run: (workspace: string, packages: string, modules: string) => void +): void { + const root = fs.mkdtempSync( + path.join(os.tmpdir(), 'runtime-module-links-') + ); + const originalPackages = config.packages_directory; + const workspace = path.join(root, 'workspace'); + const packages = path.join(root, 'pkgs'); + const modules = path.join(packages, 'node', '24.22.1', 'node_modules'); + fs.mkdirSync(workspace); + fs.mkdirSync(modules, { recursive: true }); + config.packages_directory = packages; + try { + run(workspace, packages, modules); + } finally { + config.packages_directory = originalPackages; + fs.rmSync(root, { recursive: true, force: true }); + } +} + +const managers = [ + { + name: 'bash job preparation', + prepare: ensureNodeModulesSymlink, + }, +]; +for (const file of ['build-packages.sh', 'docker/package-init.sh']) { + const source = fs + .readFileSync(path.resolve(__dirname, '../..', file), 'utf8') + .replace(/\\(["$])/g, '$1'); + const launchers = + source.match(/MODULE_DIR=.*\nif \[ -d [\s\S]*?\nfi/g) ?? []; + expect(launchers).toHaveLength(2); + for (const [index, launcher] of launchers.entries()) { + managers.push({ + name: `${file} ${index === 0 ? 'Node' : 'Bun'} launcher`, + prepare(workspace: string, modules?: string): void { + const link = path.join(workspace, 'node_modules'); + const script = `SCRIPT_DIR=$1\n${launcher + .split('/mnt/data/node_modules') + .join('"$2"')}`; + const result = Bun.spawnSync({ + cmd: [ + 'bash', + '-c', + script, + 'runtime-launcher', + path.dirname(modules!), + link, + ], + }); + expect(result.exitCode).toBe(0); + }, + }); + } +} + +for (const manager of managers) { + test(`${manager.name}: links bundled modules into a fresh workspace`, () => { + fixture((workspace, _packages, modules) => { + manager.prepare(workspace, modules); + expect(fs.readlinkSync(path.join(workspace, 'node_modules'))).toBe( + modules + ); + }); + }); + + for (const oldRuntimeExists of [false, true]) { + test(`${manager.name}: refreshes a ${ + oldRuntimeExists ? 'retained' : 'removed' + } runtime link`, () => { + fixture((workspace, packages, modules) => { + const oldModules = path.join( + packages, + 'node', + '24.15.0', + 'node_modules' + ); + if (oldRuntimeExists) + fs.mkdirSync(oldModules, { recursive: true }); + fs.symlinkSync( + oldModules, + path.join(workspace, 'node_modules') + ); + manager.prepare(workspace, modules); + expect( + fs.readlinkSync(path.join(workspace, 'node_modules')) + ).toBe(modules); + }); + }); + } + + test(`${manager.name}: leaves the current runtime link unchanged`, () => { + fixture((workspace, _packages, modules) => { + const link = path.join(workspace, 'node_modules'); + fs.symlinkSync(modules, link); + const inode = fs.lstatSync(link).ino; + manager.prepare(workspace, modules); + expect(fs.lstatSync(link).ino).toBe(inode); + }); + }); + + test(`${manager.name}: preserves a user-managed directory`, () => { + fixture((workspace, _packages, modules) => { + const link = path.join(workspace, 'node_modules'); + fs.mkdirSync(link); + fs.writeFileSync(path.join(link, 'user-package'), 'keep'); + manager.prepare(workspace, modules); + expect( + fs.readFileSync(path.join(link, 'user-package'), 'utf8') + ).toBe('keep'); + }); + }); + + test(`${manager.name}: preserves a valid user link sharing the package-root prefix`, () => { + fixture((workspace, packages, modules) => { + const target = `${packages}-other/node_modules`; + const link = path.join(workspace, 'node_modules'); + fs.mkdirSync(target, { recursive: true }); + fs.symlinkSync(target, link); + manager.prepare(workspace, modules); + expect(fs.readlinkSync(link)).toBe(target); + }); + }); + + test(`${manager.name}: preserves a user link with a package-root traversal`, () => { + fixture((workspace, packages, modules) => { + const target = `${packages}/../user/node_modules`; + const link = path.join(workspace, 'node_modules'); + fs.mkdirSync(path.resolve(target), { recursive: true }); + fs.symlinkSync(target, link); + manager.prepare(workspace, modules); + expect(fs.readlinkSync(link)).toBe(target); + }); + }); + + for (const target of [ + '../user/node_modules', + '../pkgs-other/node_modules', + ]) { + test(`${manager.name}: preserves user link ${target}`, () => { + fixture((workspace, _packages, modules) => { + const link = path.join(workspace, 'node_modules'); + fs.symlinkSync(target, link); + manager.prepare(workspace, modules); + expect(fs.readlinkSync(link)).toBe(target); + }); + }); + } +} + +test('restored Node sessions resolve ESM packages after the old runtime is removed', () => { + fixture((workspace, packages, modules) => { + const oldModules = path.join( + packages, + 'node', + '24.15.0', + 'node_modules' + ); + fs.symlinkSync(oldModules, path.join(workspace, 'node_modules')); + const archive = path.join(path.dirname(workspace), 'checkpoint.tar'); + expect( + Bun.spawnSync({ + cmd: ['tar', '-cf', archive, '-C', workspace, '.'], + }).exitCode + ).toBe(0); + fs.rmSync(workspace, { recursive: true }); + fs.mkdirSync(workspace); + expect( + Bun.spawnSync({ cmd: ['tar', '-xf', archive, '-C', workspace] }) + .exitCode + ).toBe(0); + const pkg = path.join(modules, 'runtime-fixture'); + fs.mkdirSync(pkg); + fs.writeFileSync( + path.join(pkg, 'package.json'), + JSON.stringify({ type: 'module', exports: './index.js' }) + ); + fs.writeFileSync( + path.join(pkg, 'index.js'), + 'export default "upgraded";' + ); + + ensureNodeModulesSymlink(workspace, modules); + + const result = Bun.spawnSync({ + cmd: [ + process.env.NODE_RUNTIME_TEST_BINARY ?? 'node', + '--input-type=module', + '-e', + 'import value from "runtime-fixture"; console.log(value);', + ], + cwd: workspace, + }); + expect(result.stderr.toString()).toBe(''); + expect(result.exitCode).toBe(0); + expect(result.stdout.toString().trim()).toBe('upgraded'); + }); +}); diff --git a/api/src/workspace-command.test.ts b/api/src/workspace-command.test.ts index 38936a3c..e2a9eecb 100644 --- a/api/src/workspace-command.test.ts +++ b/api/src/workspace-command.test.ts @@ -1,10 +1,11 @@ -import { mkdtemp, mkdir, realpath, rm, symlink } from 'node:fs/promises'; +import { mkdtemp, mkdir, readFile, readlink, realpath, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, test } from 'bun:test'; import semver from 'semver'; -import type { Runtime } from './runtime'; +import { config } from './config'; +import { getRuntimes, type Runtime } from './runtime'; import { executeWorkspaceCommand, hasWorkspaceCommandToken, @@ -79,6 +80,80 @@ describe('sandboxed workspace commands', () => { } }); + for (const existingModules of ['absent', 'removed', 'retained', 'directory', 'external']) { + test(`prepares ${existingModules} modules before workspace-command ESM execution`, async () => { + const parent = await mkdtemp(join(tmpdir(), 'workspace-command-modules-')); + const savedPackages = config.packages_directory; + const registered = getRuntimes(); + const savedRuntimes = registered.splice(0); + try { + const packages = join(parent, 'pkgs'); + const root = join(parent, 'workspace'); + const modules = join(packages, 'node/24.22.1/node_modules'); + const link = join(root, 'node_modules'); + await mkdir(join(root, 'src'), { recursive: true }); + await mkdir(join(modules, 'runtime-fixture'), { recursive: true }); + await writeFile(join(modules, 'runtime-fixture/package.json'), JSON.stringify({ type: 'module', exports: './index.js' })); + await writeFile(join(modules, 'runtime-fixture/index.js'), 'export default "upgraded";'); + config.packages_directory = packages; + registered.push({ + ...runtime, + language: 'node', + version: new semver.SemVer('24.22.1'), + pkgdir: join(packages, 'node/24.22.1'), + env_vars: { NODE_PATH: modules }, + }); + let userTarget: string | undefined; + if (existingModules === 'directory') { + await mkdir(link); + await writeFile(join(link, 'keep'), 'user module'); + } else if (existingModules === 'external') { + userTarget = join(packages, '../user/node_modules'); + await mkdir(userTarget, { recursive: true }); + await symlink(userTarget, link); + } else if (existingModules !== 'absent') { + const oldModules = join(packages, 'node/24.15.0/node_modules'); + if (existingModules === 'retained') await mkdir(oldModules, { recursive: true }); + await symlink(oldModules, link); + } + let executed = false; + await executeWorkspaceCommand({ command: 'node app.mjs', cwd: 'src' }, { + workspaceRoot: root, + runtime, + executeNsJail: async options => { + executed = true; + expect(options.extraPkgdirs).toContain(join(packages, 'node/24.22.1')); + expect(options.envVars.NODE_PATH).toBe(modules); + if (existingModules === 'directory') { + expect(await readFile(join(link, 'keep'), 'utf8')).toBe('user module'); + } else if (existingModules === 'external') { + expect(await readlink(link)).toBe(userTarget!); + } else { + expect(await readlink(link)).toBe(modules); + const result = Bun.spawnSync({ + cmd: [process.env.NODE_RUNTIME_TEST_BINARY ?? 'node', '--input-type=module', '-e', + 'import value from "runtime-fixture"; console.log(value);'], + cwd: join(root, 'src'), + }); + expect(result.stderr.toString()).toBe(''); + expect(result.exitCode).toBe(0); + expect(result.stdout.toString().trim()).toBe('upgraded'); + } + return { + stdout: '', stderr: '', code: 0, signal: null, output: '', + memory: null, message: null, status: null, cpu_time: null, wall_time: 1, + }; + }, + }); + expect(executed).toBe(true); + } finally { + registered.splice(0, registered.length, ...savedRuntimes); + config.packages_directory = savedPackages; + await rm(parent, { recursive: true, force: true }); + } + }); + } + test('rejects a cwd symlink that escapes the mounted workspace', async () => { const parent = await mkdtemp(join(tmpdir(), 'workspace-command-')); try { diff --git a/api/src/workspace-command.ts b/api/src/workspace-command.ts index a2cd5339..34180da9 100644 --- a/api/src/workspace-command.ts +++ b/api/src/workspace-command.ts @@ -2,7 +2,7 @@ import * as path from 'node:path'; import * as fsp from 'node:fs/promises'; import { timingSafeEqual } from 'node:crypto'; -import { aggregateBashExtras } from './job'; +import { aggregateBashExtras, ensureNodeModulesSymlink } from './job'; import { execute, type NsJailResult } from './nsjail'; import { getLatestRuntimeMatchingLanguageVersion } from './runtime'; import type { Runtime } from './runtime'; @@ -169,7 +169,9 @@ export async function executeWorkspaceCommand( HOME: '/mnt/data', SANDBOX_LANGUAGE: 'bash', }; - const extraPkgdirs = aggregateBashExtras(runtime.pkgdir, envVars); + const linkTarget: { nodeModulesPath?: string } = {}; + const extraPkgdirs = aggregateBashExtras(runtime.pkgdir, envVars, undefined, linkTarget); + ensureNodeModulesSymlink(workspaceRoot, linkTarget.nodeModulesPath); const relativeCwd = path.relative(workspaceRoot, cwd).split(path.sep).join('/'); const result = await (options.executeNsJail ?? execute)({ command: [ diff --git a/build-packages.sh b/build-packages.sh index 1d635297..59d54750 100755 --- a/build-packages.sh +++ b/build-packages.sh @@ -9,7 +9,7 @@ # # Environment Variables: # PYTHON_VERSION=3.14.4 # Python version to install -# NODE_VERSION=24.15.0 # Node.js version to install +# NODE_VERSION=24.22.1 # Node.js version to install # BUN_VERSION=1.4.2 # Bun version to install # BASH_PACKAGE_VERSION=5.2.0 # Bash package registration version (semver-like x.y.z) # SKIP_PYTHON_PACKAGES=1 # Skip Python pip packages @@ -27,7 +27,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$SCRIPT_DIR" PYTHON_VERSION="${PYTHON_VERSION:-3.14.4}" -NODE_VERSION="${NODE_VERSION:-24.15.0}" +NODE_VERSION="${NODE_VERSION:-24.22.1}" BUN_VERSION="${BUN_VERSION:-1.4.2}" PACKAGES_DIR="./data/pkgs" JS_PACKAGE_MANIFEST="${JS_PACKAGE_MANIFEST:-${SCRIPT_DIR}/javascript-packages.txt}" @@ -271,8 +271,18 @@ EOF" #!/bin/bash SCRIPT_DIR=\"\$(cd \"\$(dirname \"\${BASH_SOURCE[0]}\")\" && pwd)\" MODULE_DIR=\"\${SCRIPT_DIR}/node_modules\" -if [ -d \"\${MODULE_DIR}\" ] && [ ! -e /mnt/data/node_modules ]; then - ln -s \"\${MODULE_DIR}\" /mnt/data/node_modules 2>/dev/null || true +if [ -d \"\${MODULE_DIR}\" ]; then + if [ -L /mnt/data/node_modules ]; then + LINK_TARGET=\"\$(readlink -m /mnt/data/node_modules)\" + if [ \"\${LINK_TARGET}\" != \"\${MODULE_DIR}\" ]; then + case \"\${LINK_TARGET}\" in + \"\${SCRIPT_DIR%/*/*}/\"*/node_modules) + ln -sfnT \"\${MODULE_DIR}\" /mnt/data/node_modules ;; + esac + fi + elif [ ! -e /mnt/data/node_modules ]; then + ln -s \"\${MODULE_DIR}\" /mnt/data/node_modules 2>/dev/null || true + fi fi \"\${SCRIPT_DIR}/bin/node\" \"\$@\" EOF @@ -373,8 +383,18 @@ EOF" #!/bin/bash SCRIPT_DIR=\"\$(cd \"\$(dirname \"\${BASH_SOURCE[0]}\")\" && pwd)\" MODULE_DIR=\"\${SCRIPT_DIR}/node_modules\" -if [ -d \"\${MODULE_DIR}\" ] && [ ! -e /mnt/data/node_modules ]; then - ln -s \"\${MODULE_DIR}\" /mnt/data/node_modules 2>/dev/null || true +if [ -d \"\${MODULE_DIR}\" ]; then + if [ -L /mnt/data/node_modules ]; then + LINK_TARGET=\"\$(readlink -m /mnt/data/node_modules)\" + if [ \"\${LINK_TARGET}\" != \"\${MODULE_DIR}\" ]; then + case \"\${LINK_TARGET}\" in + \"\${SCRIPT_DIR%/*/*}/\"*/node_modules) + ln -sfnT \"\${MODULE_DIR}\" /mnt/data/node_modules ;; + esac + fi + elif [ ! -e /mnt/data/node_modules ]; then + ln -s \"\${MODULE_DIR}\" /mnt/data/node_modules 2>/dev/null || true + fi fi \"\${SCRIPT_DIR}/bun\" run \"\$@\" EOF diff --git a/docker-compose.yaml b/docker-compose.yaml index db517b4c..a4a43cdd 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -15,6 +15,7 @@ services: - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1} - CODEAPI_BRIDGE_LANE_GIT=${CODEAPI_BRIDGE_LANE_GIT:-false} + - CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES=${CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES:-104857600} - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} - CODEAPI_BRIDGE_RECOVERY_SERVER_ID=${CODEAPI_BRIDGE_RECOVERY_SERVER_ID:-} - CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=${CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS:-0} diff --git a/docker/Dockerfile.worker-sandbox b/docker/Dockerfile.worker-sandbox index 673dd42f..7f94409c 100644 --- a/docker/Dockerfile.worker-sandbox +++ b/docker/Dockerfile.worker-sandbox @@ -48,7 +48,7 @@ COPY api/src/spec-guard.c /tmp/spec-guard.c COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c COPY api/src/anonymous-pipes.c /tmp/anonymous-pipes.c COPY api/src/supervisor-policy.c /tmp/supervisor-policy.c -COPY --from=node:22-slim /usr/local/include/node /usr/local/include/node +COPY --from=node:22.23.4-slim /usr/local/include/node /usr/local/include/node COPY docker/rootfs-setup.c /tmp/rootfs-setup.c RUN gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I/usr/local/include/node -o /usr/local/lib/anonymous-pipes.node /tmp/anonymous-pipes.c \ && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/sandbox-supervisor-policy /tmp/supervisor-policy.c \ @@ -160,7 +160,7 @@ COPY --from=nsjail-builder /usr/local/bin/sandbox-supervisor-policy /usr/local/b COPY --from=nsjail-builder /usr/local/lib/anonymous-pipes.node /usr/local/lib/anonymous-pipes.node ENV SANDBOX_PIPE_ADDON=/usr/local/lib/anonymous-pipes.node RUN chmod +x /usr/sbin/nsjail -COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node +COPY --from=node:22.23.4-slim /usr/local/bin/node /usr/local/bin/node WORKDIR /sandbox_api diff --git a/docker/package-init.sh b/docker/package-init.sh index 4b96305f..2273721c 100644 --- a/docker/package-init.sh +++ b/docker/package-init.sh @@ -20,7 +20,7 @@ PYTHON_ALIAS="python${PYTHON_SITE_VERSION}" # below — that's inherent to installing unpinned packages, not specific to uv. PYTHON_PACKAGE_INSTALLER="${PYTHON_PACKAGE_INSTALLER:-uv}" UV_VERSION="${UV_VERSION:-0.11.26}" -NODE_VERSION="${NODE_VERSION:-24.15.0}" +NODE_VERSION="${NODE_VERSION:-24.22.1}" BUN_VERSION="${BUN_VERSION:-1.4.2}" BASH_PACKAGE_VERSION="${BASH_PACKAGE_VERSION:-5.2.0}" INSTALL_FAILED=false @@ -101,8 +101,64 @@ packages_ready() { [ -f "/pkgs/bash/${BASH_PACKAGE_VERSION}/.package-installed" ] } +write_node_launcher() { + local destination="$1" + local launcher + launcher="$(mktemp "${destination}/.run.XXXXXX")" + cat > "$launcher" << 'EOF' +#!/bin/bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MODULE_DIR="${SCRIPT_DIR}/node_modules" +if [ -d "$MODULE_DIR" ]; then + if [ -L /mnt/data/node_modules ]; then + LINK_TARGET="$(readlink -m /mnt/data/node_modules)" + if [ "$LINK_TARGET" != "$MODULE_DIR" ]; then + case "$LINK_TARGET" in + "${SCRIPT_DIR%/*/*}/"*/node_modules) + ln -sfnT "$MODULE_DIR" /mnt/data/node_modules ;; + esac + fi + elif [ ! -e /mnt/data/node_modules ]; then + ln -s "$MODULE_DIR" /mnt/data/node_modules 2>/dev/null || true + fi +fi +"${SCRIPT_DIR}/bin/node" "$@" +EOF + chmod 0755 "$launcher" + mv -f "$launcher" "${destination}/run" +} + +write_bun_launcher() { + local destination="$1" + local launcher + launcher="$(mktemp "${destination}/.run.XXXXXX")" + cat > "$launcher" << 'EOF' +#!/bin/bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MODULE_DIR="${SCRIPT_DIR}/node_modules" +if [ -d "$MODULE_DIR" ]; then + if [ -L /mnt/data/node_modules ]; then + LINK_TARGET="$(readlink -m /mnt/data/node_modules)" + if [ "$LINK_TARGET" != "$MODULE_DIR" ]; then + case "$LINK_TARGET" in + "${SCRIPT_DIR%/*/*}/"*/node_modules) + ln -sfnT "$MODULE_DIR" /mnt/data/node_modules ;; + esac + fi + elif [ ! -e /mnt/data/node_modules ]; then + ln -s "$MODULE_DIR" /mnt/data/node_modules 2>/dev/null || true + fi +fi +"${SCRIPT_DIR}/bun" run "$@" +EOF + chmod 0755 "$launcher" + mv -f "$launcher" "${destination}/run" +} + if [ -f "$MARKER_FILE" ] && [ "$FORCE_REBUILD" != "true" ]; then if packages_ready; then + write_node_launcher "/pkgs/node/${NODE_VERSION}" + write_bun_launcher "/pkgs/bun/${BUN_VERSION}" echo "Packages already initialized (marker file exists)" echo "Set FORCE_REBUILD=true to force reinstall" echo "" @@ -301,16 +357,7 @@ if [ -n "$NODE_ARCH" ]; then } EOF - cat > "$NODE_DEST/run" << 'EOF' -#!/bin/bash -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -MODULE_DIR="${SCRIPT_DIR}/node_modules" -if [ -d "$MODULE_DIR" ] && [ ! -e /mnt/data/node_modules ]; then - ln -s "$MODULE_DIR" /mnt/data/node_modules 2>/dev/null || true -fi -"${SCRIPT_DIR}/bin/node" "$@" -EOF - chmod +x "$NODE_DEST/run" + write_node_launcher "$NODE_DEST" { echo "PATH=${NODE_DEST}/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:." @@ -413,16 +460,7 @@ if [ -n "$BUN_ARCH" ]; then } EOF - cat > "$BUN_DEST/run" << 'EOF' -#!/bin/bash -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -MODULE_DIR="${SCRIPT_DIR}/node_modules" -if [ -d "$MODULE_DIR" ] && [ ! -e /mnt/data/node_modules ]; then - ln -s "$MODULE_DIR" /mnt/data/node_modules 2>/dev/null || true -fi -"${SCRIPT_DIR}/bun" run "$@" -EOF - chmod +x "$BUN_DEST/run" + write_bun_launcher "$BUN_DEST" { echo "PATH=${BUN_DEST}:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:." diff --git a/docs/adr/0014-workspace-file-import.md b/docs/adr/0014-workspace-file-import.md new file mode 100644 index 00000000..e1aac4a7 --- /dev/null +++ b/docs/adr/0014-workspace-file-import.md @@ -0,0 +1,65 @@ +# 0014: Import chat uploads into attached workspaces + +Status: Accepted + +## Context + +A LibreChat chat can run on a personal machine through the remote bridge. Files +the user uploads in that chat stay in Code API storage: only programmatic Bash +stages them, per run, in a private scratch directory. `write_file` accepts +1 MiB of UTF-8 text, so binaries such as archives, images, and PDFs cannot reach +the workspace at all. + +## Decision + +Add one additive workspace operation, `import_file`, under protocol version 1. + +- **Request.** `{ path, source: { storage_session_id, id }, bytes, sha256, + overwrite? }` plus the usual workspace selectors. `sha256` is lowercase hex. +- **Ownership.** Code API, not the caller, decides whether `source` may be read. + It applies the `GET /download/:session_id/:fileId` rule for a user-kind + object: the live `session:` key must name the caller's storage key, and an + upload record must exist for that key, session, and object. +- **Grant.** Code API asks the egress gateway for a grant whose only input is + that object, with no upload capacity, two reads, and an expiry of the + request's admission plus execution budget plus 30 seconds. The relayed + request carries the grant and opaque handles in place of the raw ids. A + caller-supplied `egressGrant` is rejected. Synchronous requests revoke the + grant when they finish; durable requests let it expire. +- **Worker.** The worker streams the object from + `LIBRECHAT_CODE_FILE_RELAY_UPSTREAM`, the route programmatic inputs already + use, and holds no storage credential. It applies `write_file`'s path rules, + refuses `.git` segments and their filesystem aliases, refuses symlinked + parents, creates missing parents as 0755, enforces `bytes` while streaming + into a same-directory temporary file, verifies size and SHA-256, syncs, and + installs the file atomically with mode 0644. The size and hash are checked on + the synced temporary file itself. A create links exclusively; an overwrite + rechecks the inspected inode immediately before its rename, as `write_file` + does. +- **Idempotency.** Identical existing bytes return `unchanged: true` and + `bytesWritten: 0` without a transfer. Different bytes return `EDIT_CONFLICT` + unless `overwrite: true`. +- **Limits.** Code API (`CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES`) and the worker + (`LIBRECHAT_CODE_IMPORT_FILE_MAX_BYTES`) each enforce a per-file ceiling, + 100 MiB by default, as `WRITE_LIMIT_EXCEEDED`. The worker also requires the + file plus 64 MiB to fit in the target filesystem's free space. +- **Policy.** `import_file` is a file write. It shares the mutation guard with + `write_file`, is never advertised for a read-only root, and a read-only root + refuses it with `WRITE_DISABLED`. +- **Negotiation.** Code API lists `import_file` in + `supportedWorkspaceToolOperations` only when `EGRESS_GATEWAY_URL` is set. A + worker advertises it by default whenever writes are enabled, a relay + upstream is configured, and the Code API listed it; operators opt out with + `--no-workspace-imports` or `LIBRECHAT_CODE_ALLOW_WORKSPACE_IMPORTS=false`. + LibreChat sends it only when advertised. LibreChat releases that predate + LibreChat-AI/LibreChat#17376 reject a worker status listing an unknown + operation, so LibreChat is updated first and the opt-out is the escape + hatch for workers serving an older one. + +## Consequences + +Uploads land in the workspace without a model round trip, and older Code APIs, +workers, and LibreChat releases keep their current behavior. Deployments +without an egress gateway do not offer the operation. Parent directories are +created only beneath a descriptor-held root, as for `write_file`; a legacy +pathname-only root reports a missing parent instead. diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index dc7cb98b..40465e00 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -172,6 +172,11 @@ whitespace-tolerant matching and multi-location replacement (see text, require an existing in-workspace parent directory, reject symlinks, and commit atomically. The worker capability is an enforcement boundary; LibreChat should still route every mutation through its configurable tool-approval hooks. +Writable workers with a file relay upstream also offer `import_file` by +default (opt out with `LIBRECHAT_CODE_ALLOW_WORKSPACE_IMPORTS=false`), which +copies a chat upload the caller owns into the workspace through a one-object +egress grant (ADR 0014). It is a file write: a worker without writes enabled +never advertises it and refuses it with `WRITE_DISABLED`. The workspace root can be an existing project, a Git repository, or an empty directory; Git is not required. This boundary keeps that directory local to the diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 99f19992..859104c6 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -561,6 +561,22 @@ To keep every worktree, add `--no-worktree-retirement` (or plist. - **Worker online but not ready:** check native sandbox preparation, definition validation, setup, quarantine, and readiness logs. +- **Files work but commands return `503 COMMAND_UNAVAILABLE`:** inspect the + worker's stderr/service journal for `workspace_command_unavailable`, and + correlate its `assignmentId` with the bridge request. Updated workers emit + one JSON record for a rejected command or programmatic assignment, with + `stage`, `reason`, workspace/worker/incarnation IDs, mutation uncertainty, + quarantine requirement, and an allowlisted `osCode` when available. Stages + distinguish allocation, sandbox startup, programmatic prerequisites, + credential resolution, command wrapping, dispatch, executor transport, and + cleanup. For example, `sandbox_startup` with `osCode: ENOSPC` points to + storage; `credential_resolution` identifies credential setup rather than + command execution. Missing errno does not rule out either cause. + Commands, arguments, paths, credentials, output, and free-form exception + messages are excluded from these records. Diagnostics stay worker-local; + the bridge response and quarantine/retry behavior are unchanged. A worker + package update is required; a Code API restart alone does not enable this. + These records cover assignment failures, not failures before worker startup. - **Setup repeats on restart:** setup is intentionally per-start; make it idempotent, remove it, or opt in to the bounded `setup.reuse` readiness contract in the [worker package guide](../../packages/code/README.md#reusing-a-prepared-checkout). diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index 4858fd71..beca089d 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -310,7 +310,7 @@ workerSandbox: tag: latest pullPolicy: Always pythonVersion: "3.14.4" - nodeVersion: "24.15.0" + nodeVersion: "24.22.1" bunVersion: "1.4.2" bashPackageVersion: "5.2.0" backoffLimit: 2 diff --git a/launcher/Dockerfile b/launcher/Dockerfile index bad5cd2f..f7c305fb 100644 --- a/launcher/Dockerfile +++ b/launcher/Dockerfile @@ -40,7 +40,7 @@ COPY api/src/spec-guard.c /tmp/spec-guard.c COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c COPY api/src/anonymous-pipes.c /tmp/anonymous-pipes.c COPY api/src/supervisor-policy.c /tmp/supervisor-policy.c -COPY --from=node:22-slim /usr/local/include/node /usr/local/include/node +COPY --from=node:22.23.4-slim /usr/local/include/node /usr/local/include/node COPY docker/rootfs-setup.c /tmp/rootfs-setup.c RUN gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I/usr/local/include/node -o /usr/local/lib/anonymous-pipes.node /tmp/anonymous-pipes.c \ && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/sandbox-supervisor-policy /tmp/supervisor-policy.c \ @@ -79,7 +79,7 @@ COPY --from=nsjail-builder /usr/local/bin/tool-call-pipe-bridge /usr/local/bin/t COPY --from=nsjail-builder /usr/local/bin/sandbox-supervisor-policy /usr/local/bin/sandbox-supervisor-policy COPY --from=nsjail-builder /usr/local/lib/anonymous-pipes.node /usr/local/lib/anonymous-pipes.node ENV SANDBOX_PIPE_ADDON=/usr/local/lib/anonymous-pipes.node -COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node +COPY --from=node:22.23.4-slim /usr/local/bin/node /usr/local/bin/node RUN chmod +x /usr/sbin/nsjail WORKDIR /sandbox_api diff --git a/packages/code/Dockerfile b/packages/code/Dockerfile index 93ed9c57..34c54899 100644 --- a/packages/code/Dockerfile +++ b/packages/code/Dockerfile @@ -1,11 +1,11 @@ -FROM node:24-alpine AS build +FROM node:24.22.1-alpine AS build WORKDIR /app COPY package.json package-lock.json tsconfig.json ./ RUN npm ci COPY src ./src RUN npm run build -FROM node:24-alpine +FROM node:24.22.1-alpine ENV NODE_ENV=production RUN apk add --no-cache bash bubblewrap ripgrep socat \ && addgroup -S librechat-code \ diff --git a/packages/code/README.md b/packages/code/README.md index 0437ecfb..8f911996 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -630,6 +630,27 @@ with `--allow-workspace-writes` or `overwrite: false` to require an atomic create that returns `EDIT_CONFLICT` if the target already exists. Code API dispatches that mode only after the worker and server negotiate `create` in `writeFileModes`. + +`import_file` copies one stored chat upload into a writable workspace as a +binary-safe file. It is on by default whenever the worker can perform it: +writes are enabled and `LIBRECHAT_CODE_FILE_RELAY_UPSTREAM` names the +deployment's egress gateway. It is advertised only to a Code API that lists it +in `supportedWorkspaceToolOperations` (one that can mint egress grants). Opt out +with `--no-workspace-imports` or `LIBRECHAT_CODE_ALLOW_WORKSPACE_IMPORTS=false`. +Update LibreChat before workers: LibreChat releases that predate +LibreChat-AI/LibreChat#17376 reject a worker status that lists an operation +they do not know, so a worker serving one of those must opt out. Code API checks that the caller owns the object +and relays a short-lived grant scoped to that object alone; the worker holds no +storage credential. The worker applies `write_file`'s path rules, refuses any +`.git` segment (including case, Unicode-ignorable, trailing-dot and `GIT~1` +aliases) and symlinked parents, creates missing parents as 0755 under a held +root, streams into a same-directory temporary file with `bytes` as a hard cap, +verifies size and SHA-256, syncs, and installs the file atomically with mode +0644. Identical existing bytes return `unchanged: true` without a transfer; +different bytes return `EDIT_CONFLICT` unless `overwrite: true`. A failure +removes the temporary file and any parents it created. The per-file ceiling is +`LIBRECHAT_CODE_IMPORT_FILE_MAX_BYTES` (`--import-file-max-bytes`, default +100 MiB), and the filesystem must keep 64 MiB free after the file lands. `edit_file` accepts either the legacy `oldText`/`newText` pair or an ordered `edits` array; every exact replacement is validated before the updated file is installed as one atomic mutation. Code API dispatches the batch form only after diff --git a/packages/code/package.json b/packages/code/package.json index 95eea841..d3e6395b 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -19,6 +19,10 @@ "types": "./dist/identity.d.ts", "import": "./dist/identity.js" }, + "./browser-enrollment": { + "types": "./dist/browser-enrollment-protocol.d.ts", + "import": "./dist/browser-enrollment-protocol.js" + }, "./worker": { "types": "./dist/worker.d.ts", "import": "./dist/worker.js" diff --git a/packages/code/src/browser-enrollment-protocol.ts b/packages/code/src/browser-enrollment-protocol.ts new file mode 100644 index 00000000..ef1c712b --- /dev/null +++ b/packages/code/src/browser-enrollment-protocol.ts @@ -0,0 +1,253 @@ +import { createHash, createPublicKey, sign, verify } from 'node:crypto'; + +export type BrowserSetupRequest = BrowserEnrollmentRequest; +export type EnrollmentRedemption = BrowserEnrollmentRedemptionContract; +export type EnrollmentApproval = BrowserEnrollmentApproval; + +export const BROWSER_ENROLLMENT_PROTOCOL_VERSION = 1 as const; +export const BROWSER_ENROLLMENT_MAX_TTL_MS = 10 * 60_000; +export const BROWSER_SETUP_PROOF_DOMAIN = 'librechat-code:browser-setup:v1'; +export const BROWSER_ENROLLMENT_REDEMPTION_DOMAIN = 'librechat-code:browser-enrollment-redeem:v1'; + +export interface BrowserEnrollmentRequest { + version: 1; + instance: string; + attemptId: string; + createdAt: string; + publicKey: string; + machineName: string; + workspace: { type: 'private' | 'directory' }; + allowWorkspaceWrites: boolean; + allowWorkspaceCommands: boolean; +} + +export interface BrowserSetupProof { + operation: 'setup.start' | 'setup.poll' | 'setup.cancel' | 'setup.complete'; + instance: string; + attemptId: string; + requestHash: string; + timestamp: string; + nonce: string; +} + +export interface BrowserEnrollmentRedemptionContract { + operation: 'enrollment.redeem'; + serverId: string; + instance: string; + attemptId: string; + requestHash: string; + workerId: string; + expiresAt: string; +} + +export interface BrowserEnrollmentStatusRequest extends BrowserEnrollmentRedemptionContract { + protocolVersion: 1; +} + +export interface BrowserEnrollmentRedemption extends BrowserEnrollmentRedemptionContract { + protocolVersion: 1; + signature: string; +} + +export interface BrowserEnrollmentBinding { + tenantId: string; + principal: { type: 'deployment' | 'tenant' | 'user' | 'role' | 'group'; id: string }; +} + +export interface BrowserEnrollmentApproval extends BrowserEnrollmentRedemptionContract { + protocolVersion: 1; + publicKey: string; + binding: BrowserEnrollmentBinding; +} + +export interface BrowserEnrollmentCapabilities { + browserEnrollmentVersion: 1; + serverId: string; +} + +function record(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function keys(value: Record, names: string[]): boolean { + return Object.keys(value).length === names.length && names.every(name => Object.prototype.hasOwnProperty.call(value, name)); +} + +export function isBrowserEnrollmentTimestamp(value: unknown): value is string { + if (typeof value !== 'string' || value.length !== 24) return false; + const time = Date.parse(value); + return Number.isFinite(time) && new Date(time).toISOString() === value; +} + +export function isBrowserEnrollmentToken(value: unknown): value is string { + return typeof value === 'string' && /^[A-Za-z0-9_-]{43}$/.test(value) && + Buffer.from(value, 'base64url').toString('base64url') === value; +} + +export function isBrowserEnrollmentSignature(value: unknown): value is string { + return typeof value === 'string' && /^[A-Za-z0-9_-]{86}$/.test(value) && + Buffer.from(value, 'base64url').toString('base64url') === value; +} + +export function isBrowserEnrollmentPublicKey(value: unknown): value is string { + if (typeof value !== 'string' || value.length > 4096 || !value.startsWith('-----BEGIN PUBLIC KEY-----\n')) return false; + try { + const key = createPublicKey(value); + return key.asymmetricKeyType === 'ed25519' && key.export({ type: 'spki', format: 'pem' }) === value; + } catch { + return false; + } +} + +export function normalizeBrowserEnrollmentInstance(value: string): string { + if (value.length > 2048 || value !== value.trim()) throw new TypeError('Invalid LibreChat instance URL'); + const url = new URL(value); + if ( + (url.protocol !== 'https:' && !(url.protocol === 'http:' && ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname))) || + url.username !== '' || url.password !== '' || url.search !== '' || url.hash !== '' + ) { + throw new TypeError('LibreChat instance requires HTTPS (or loopback HTTP), without credentials, query, or fragment'); + } + return `${url.origin}${url.pathname.replace(/\/+$/, '')}`; +} + +function instance(value: unknown): value is string { + try { + return typeof value === 'string' && normalizeBrowserEnrollmentInstance(value) === value; + } catch { + return false; + } +} + +function serverOrigin(value: unknown): value is string { + if (typeof value !== 'string' || value.length > 256) return false; + try { + const url = new URL(value); + return url.protocol === 'https:' && url.origin === value && instance(value); + } catch { + return false; + } +} + +function requestHash(value: unknown): value is string { + return typeof value === 'string' && /^[a-f0-9]{64}$/.test(value); +} + +export function isBrowserEnrollmentRequest(value: unknown): value is BrowserEnrollmentRequest { + return record(value) && keys(value, [ + 'version', 'instance', 'attemptId', 'createdAt', 'publicKey', 'machineName', + 'workspace', 'allowWorkspaceWrites', 'allowWorkspaceCommands', + ]) && value.version === 1 && instance(value.instance) && isBrowserEnrollmentToken(value.attemptId) && + isBrowserEnrollmentTimestamp(value.createdAt) && isBrowserEnrollmentPublicKey(value.publicKey) && + typeof value.machineName === 'string' && /^[A-Za-z0-9][A-Za-z0-9 ._()-]{0,79}$/.test(value.machineName) && + value.machineName === value.machineName.trim() && record(value.workspace) && keys(value.workspace, ['type']) && + (value.workspace.type === 'private' || value.workspace.type === 'directory') && + typeof value.allowWorkspaceWrites === 'boolean' && typeof value.allowWorkspaceCommands === 'boolean'; +} + +export function canonicalBrowserEnrollmentRequest(input: BrowserEnrollmentRequest): string { + if (!isBrowserEnrollmentRequest(input)) throw new TypeError('Invalid browser enrollment request'); + return JSON.stringify({ + version: input.version, + instance: input.instance, + attemptId: input.attemptId, + createdAt: input.createdAt, + publicKey: input.publicKey, + machineName: input.machineName, + workspace: { type: input.workspace.type }, + allowWorkspaceWrites: input.allowWorkspaceWrites, + allowWorkspaceCommands: input.allowWorkspaceCommands, + }); +} + +export function hashBrowserEnrollmentRequest(input: BrowserEnrollmentRequest): string { + return createHash('sha256').update(canonicalBrowserEnrollmentRequest(input)).digest('hex'); +} + +export function isBrowserSetupProof(value: unknown): value is BrowserSetupProof { + return record(value) && keys(value, ['operation', 'instance', 'attemptId', 'requestHash', 'timestamp', 'nonce']) && + typeof value.operation === 'string' && ['setup.start', 'setup.poll', 'setup.cancel', 'setup.complete'].includes(value.operation) && + instance(value.instance) && isBrowserEnrollmentToken(value.attemptId) && requestHash(value.requestHash) && + isBrowserEnrollmentTimestamp(value.timestamp) && isBrowserEnrollmentToken(value.nonce); +} + +export function canonicalBrowserSetupProof(input: BrowserSetupProof): string { + if (!isBrowserSetupProof(input)) throw new TypeError('Invalid browser setup proof'); + return [BROWSER_SETUP_PROOF_DOMAIN, input.operation, input.instance, input.attemptId, + input.requestHash, input.timestamp, input.nonce].join('\n'); +} + +export function isBrowserEnrollmentRedemptionContract(value: unknown): value is BrowserEnrollmentRedemptionContract { + return record(value) && value.operation === 'enrollment.redeem' && serverOrigin(value.serverId) && + instance(value.instance) && isBrowserEnrollmentToken(value.attemptId) && requestHash(value.requestHash) && + typeof value.workerId === 'string' && /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(value.workerId) && + isBrowserEnrollmentTimestamp(value.expiresAt); +} + +export function isBrowserEnrollmentBinding(value: unknown): value is BrowserEnrollmentBinding { + return record(value) && keys(value, ['tenantId', 'principal']) && + typeof value.tenantId === 'string' && /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(value.tenantId) && + record(value.principal) && keys(value.principal, ['type', 'id']) && + typeof value.principal.type === 'string' && ['deployment', 'tenant', 'user', 'role', 'group'].includes(value.principal.type) && + typeof value.principal.id === 'string' && /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(value.principal.id); +} + +const CONTRACT_KEYS = ['operation', 'serverId', 'instance', 'attemptId', 'requestHash', 'workerId', 'expiresAt']; + +export function isBrowserEnrollmentApproval(value: unknown): value is BrowserEnrollmentApproval { + return isBrowserEnrollmentRedemptionContract(value) && record(value) && + keys(value, [...CONTRACT_KEYS, 'protocolVersion', 'publicKey', 'binding']) && + value.protocolVersion === 1 && isBrowserEnrollmentPublicKey(value.publicKey) && isBrowserEnrollmentBinding(value.binding); +} + +export function isBrowserEnrollmentStatusRequest(value: unknown): value is BrowserEnrollmentStatusRequest { + return isBrowserEnrollmentRedemptionContract(value) && record(value) && + keys(value, [...CONTRACT_KEYS, 'protocolVersion']) && value.protocolVersion === 1; +} + +export function isBrowserEnrollmentRedemption(value: unknown): value is BrowserEnrollmentRedemption { + return isBrowserEnrollmentRedemptionContract(value) && record(value) && + keys(value, [...CONTRACT_KEYS, 'protocolVersion', 'signature']) && + value.protocolVersion === 1 && isBrowserEnrollmentSignature(value.signature); +} + +export function canonicalBrowserEnrollmentRedemption(input: BrowserEnrollmentRedemptionContract): string { + if (!isBrowserEnrollmentRedemptionContract(input)) throw new TypeError('Invalid browser enrollment redemption'); + return [BROWSER_ENROLLMENT_REDEMPTION_DOMAIN, input.operation, input.serverId, input.instance, + input.attemptId, input.requestHash, input.workerId, input.expiresAt].join('\n'); +} + +export function signBrowserSetupProof(privateKey: string, input: BrowserSetupProof): string { + return sign(null, Buffer.from(canonicalBrowserSetupProof(input)), privateKey).toString('base64url'); +} + +export function verifyBrowserSetupProof(publicKey: string, input: BrowserSetupProof, signature: string): boolean { + try { + return isBrowserEnrollmentPublicKey(publicKey) && isBrowserEnrollmentSignature(signature) && + verify(null, Buffer.from(canonicalBrowserSetupProof(input)), publicKey, Buffer.from(signature, 'base64url')); + } catch { + return false; + } +} + +export function signBrowserEnrollmentRedemption(privateKey: string, input: BrowserEnrollmentRedemptionContract): string { + return sign(null, Buffer.from(canonicalBrowserEnrollmentRedemption(input)), privateKey).toString('base64url'); +} + +export function verifyBrowserEnrollmentRedemption(publicKey: string, input: BrowserEnrollmentRedemptionContract, signature: string): boolean { + try { + return isBrowserEnrollmentPublicKey(publicKey) && isBrowserEnrollmentSignature(signature) && + verify(null, Buffer.from(canonicalBrowserEnrollmentRedemption(input)), publicKey, Buffer.from(signature, 'base64url')); + } catch { + return false; + } +} + +export const browserSetupRequestHash = hashBrowserEnrollmentRequest; +export const canonicalBrowserSetupRequest = canonicalBrowserEnrollmentRequest; +export const canonicalEnrollmentRedemption = canonicalBrowserEnrollmentRedemption; +export const signEnrollmentRedemption = signBrowserEnrollmentRedemption; +export const verifyEnrollmentRedemption = verifyBrowserEnrollmentRedemption; +export const validateBrowserSetupRequest = isBrowserEnrollmentRequest; +export const validateBrowserSetupProof = isBrowserSetupProof; +export const validateEnrollmentRedemption = isBrowserEnrollmentRedemptionContract; diff --git a/packages/code/src/browser-protection.test.ts b/packages/code/src/browser-protection.test.ts new file mode 100644 index 00000000..2da4e27d --- /dev/null +++ b/packages/code/src/browser-protection.test.ts @@ -0,0 +1,31 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { relative, sep } from 'node:path'; +import { SandboxManager } from '@anthropic-ai/sandbox-runtime'; + +import { browserSetupAttemptPath, browserSetupProtectedPaths } from './browser-setup.js'; + +test('browser setup namespace denies journal, lock and crash-left temporaries outside home', async t => { + const attempt = browserSetupAttemptPath('/srv/code-state/profile.json'); + const paths = browserSetupProtectedPaths(attempt); + assert.deepEqual(paths, ['/srv/code-state/profile.json.setup']); + const previous = SandboxManager.getConfig(); + t.after(async () => { + await SandboxManager.reset(); + if (previous) SandboxManager.updateConfig(previous); + }); + SandboxManager.updateConfig({ + network: { allowedDomains: [], deniedDomains: [] }, + filesystem: { denyRead: [], allowRead: [], allowWrite: [], denyWrite: paths }, + credentials: { files: paths.map(path => ({ path, mode: 'deny' as const })) }, + }); + const effective = SandboxManager.getFsReadConfig(); + for (const file of [attempt, `${attempt}.lock`, `${attempt}.0123456789abcdef.tmp`, `${attempt}.future.tmp`]) { + assert.ok(effective.denyOnly.some(directory => { + const child = relative(directory, file); + return child !== '..' && !child.startsWith(`..${sep}`); + }), 'every journal generation must remain beneath an effective directory deny'); + } + assert.deepEqual(browserSetupProtectedPaths(undefined), []); + assert.throws(() => browserSetupProtectedPaths('/srv/code-state/profile.json.setup.json'), /Unsupported setup journal/); +}); diff --git a/packages/code/src/browser-setup-cli.ts b/packages/code/src/browser-setup-cli.ts new file mode 100644 index 00000000..250fcdb7 --- /dev/null +++ b/packages/code/src/browser-setup-cli.ts @@ -0,0 +1,103 @@ +import { spawn } from 'node:child_process'; +import { createHash, createPublicKey } from 'node:crypto'; +import { hostname } from 'node:os'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { cancelBrowserSetup, loadBrowserSetupAttempt, setupBrowserWorker, setupInstanceUrl } from './browser-setup.js'; +import { defaultWorkerProfilePath } from './worker-profile.js'; +import { setupWorkspaceDirectory } from './setup.js'; + +/** No shell, token, or inbound callback listener is involved. */ +export async function openSetupBrowser(url: string): Promise { + if (process.env.SSH_CONNECTION || process.env.SSH_TTY || + (process.platform === 'linux' && !process.env.DISPLAY && !process.env.WAYLAND_DISPLAY)) return false; + const command = process.platform === 'darwin' ? 'open' : process.platform === 'linux' ? 'xdg-open' : undefined; + if (!command) return false; + return new Promise(resolve => { + const child = spawn(command, [url], { stdio: 'ignore', shell: false }); + const timer = setTimeout(() => { child.kill(); resolve(false); }, 5000); + child.once('error', () => { clearTimeout(timer); resolve(false); }); + child.once('exit', code => { clearTimeout(timer); resolve(code === 0); }); + }); +} + +export async function runBrowserSetupCommand( + options: Map, + prompts: { + interactive: boolean; + ask(message: string): Promise; + yesNo(message: string): Promise; + signal: AbortSignal; + }, +): Promise { + const profilePath = resolve(String(options.get('--profile') ?? defaultWorkerProfilePath())); + const instance = setupInstanceUrl(String(options.get('--instance'))); + const saved = await loadBrowserSetupAttempt(profilePath); + if (saved && saved.request.instance !== instance) throw new Error('This profile has a setup attempt for another LibreChat instance. Resume the original instance or choose another profile'); + if (options.has('--cancel')) { + await cancelBrowserSetup(profilePath, prompts.signal); + process.stdout.write('Setup cancellation recorded. The local machine key and attempt were preserved.\n'); + return; + } + let workspaceDirectory = options.get('--worker-dir') as string | undefined; + if (saved) { + if (workspaceDirectory && (saved.profile.workspace.type !== 'directory' || await setupWorkspaceDirectory(workspaceDirectory) !== saved.profile.workspace.path)) + throw new Error('The workspace differs from the saved setup attempt'); + if (options.has('--default-workspace') && saved.profile.workspace.type !== 'private') throw new Error('The workspace differs from the saved setup attempt'); + if ((options.has('--allow-workspace-writes') && !saved.profile.allowWorkspaceWrites) || + (options.has('--allow-workspace-commands') && !saved.profile.allowWorkspaceCommands)) throw new Error('Permissions differ from the saved setup attempt'); + workspaceDirectory = saved.profile.workspace.type === 'directory' ? saved.profile.workspace.path : undefined; + process.stdout.write('Resuming the saved setup attempt with its original machine key and access choices.\n'); + } else if (prompts.interactive && !workspaceDirectory && !options.has('--default-workspace')) { + workspaceDirectory = (await prompts.ask('Project directory (Enter for a new private workspace): ')) || undefined; + } + if (workspaceDirectory !== undefined) workspaceDirectory = await setupWorkspaceDirectory(workspaceDirectory); + const machineName = String(options.get('--machine-name') ?? saved?.request.machineName ?? hostname().replace(/[^A-Za-z0-9 ._()-]/g, '-').slice(0, 80)); + const allowWorkspaceWrites = saved?.profile.allowWorkspaceWrites ?? (options.has('--allow-workspace-writes') || + (prompts.interactive && await prompts.yesNo('Allow file writes within this workspace?'))); + const allowWorkspaceCommands = saved?.profile.allowWorkspaceCommands ?? (options.has('--allow-workspace-commands') || + (prompts.interactive && await prompts.yesNo('Allow sandboxed commands? Commands can modify workspace files independently of file-write tooling'))); + process.stdout.write(`\nLibreChat: ${instance}\nMachine: ${machineName}\nWorkspace: ${workspaceDirectory ?? 'new private workspace'}\nFile writes: ${allowWorkspaceWrites ? 'enabled' : 'disabled'}\nCommands: ${allowWorkspaceCommands ? 'enabled, restricted sandbox policy' : 'disabled'}\nProfile: ${profilePath}\nPairing does not attest this host or replace tool approvals.\n`); + if (!saved && prompts.interactive && !await prompts.yesNo('Request browser approval with these choices?')) { + process.stdout.write('Setup cancelled before enrollment.\n'); + return; + } + await setupBrowserWorker({ + instance, machineName, profilePath, + identityPath: options.get('--identity') as string | undefined ?? saved?.profile.identityPath, + workspaceDirectory, allowWorkspaceWrites, allowWorkspaceCommands, + signal: prompts.signal, + }, { + onVerification: async status => { + const attempt = (await loadBrowserSetupAttempt(profilePath))!; + const fingerprint = createHash('sha256').update(createPublicKey(attempt.request.publicKey).export({ type: 'spki', format: 'der' })).digest('hex'); + process.stdout.write(`Machine key: SHA256:${fingerprint}\n`); + // Always show both, even when opening a browser succeeds. + process.stdout.write(`\nApprove this machine in LibreChat:\n ${status.verificationUrl}\nVerification code: ${status.verificationCode}\nExpires: ${status.expiresAt}\nCheck the matching code and key identity before approving. Waiting for your explicit decision.\n`); + if (!options.has('--no-browser') && prompts.interactive && !await openSetupBrowser(status.verificationUrl)) + process.stdout.write('Could not open a browser here. Open the URL on another device and compare the code.\n'); + }, + }); + process.stdout.write('\nMachine enrolled. Profile saved. It is not connected or ready until the worker starts and readiness is confirmed.\n'); + prompts.signal.throwIfAborted(); + const start = options.has('--start') || (prompts.interactive && await prompts.yesNo('Start the worker in the foreground now?')); + if (start) { + prompts.signal.throwIfAborted(); + process.stdout.write('Starting the saved profile in the foreground. Stop with Ctrl-C.\n'); + await new Promise((resolve, reject) => { + const child = spawn(process.execPath, [fileURLToPath(new URL('./cli.js', import.meta.url)), 'run', '--profile', profilePath], { stdio: 'inherit' }); + const abort = () => child.kill('SIGINT'); + prompts.signal.addEventListener('abort', abort, { once: true }); + child.once('error', reject); + child.once('exit', code => { + prompts.signal.removeEventListener('abort', abort); + if (code === 0 || prompts.signal.aborted) resolve(); + else reject(new Error('Worker stopped before a successful foreground session; enrollment and profile remain saved')); + }); + }); + } else { + const quoted = `'${profilePath.replace(/'/g, "'\\''")}'`; + process.stdout.write(`Start it with:\n librechat-code run${profilePath === resolve(defaultWorkerProfilePath()) ? '' : ` --profile ${quoted}`}\n`); + } +} diff --git a/packages/code/src/browser-setup.test.ts b/packages/code/src/browser-setup.test.ts new file mode 100644 index 00000000..077db673 --- /dev/null +++ b/packages/code/src/browser-setup.test.ts @@ -0,0 +1,279 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import test from 'node:test'; + +import { createBridgeIdentity } from './identity.js'; +import { browserSetupRequestHash, verifyBrowserSetupProof, verifyEnrollmentRedemption } from './browser-enrollment-protocol.js'; +import type { BrowserSetupRequest, BrowserSetupProof } from './browser-enrollment-protocol.js'; +import { browserSetupAttemptPath, browserSetupProtectedPaths, loadBrowserSetupAttempt, setupBrowserWorker, setupInstanceUrl, cancelBrowserSetup } from './browser-setup.js'; +import type { BrowserSetupOptions } from './browser-setup.js'; +import { loadBridgeIdentity, saveBridgeIdentity, savePrivateJson } from './storage.js'; +import { loadWorkerProfile } from './worker-profile.js'; +import { parseSetupArguments } from './setup-cli.js'; +import { runBrowserSetupCommand } from './browser-setup-cli.js'; + +async function fixture(t: test.TestContext) { + const root = await mkdtemp(join(tmpdir(), 'browser-setup-')); + t.after(() => rm(root, { recursive: true, force: true })); + const options: BrowserSetupOptions = { + instance: 'https://chat.example', machineName: 'headless-machine', + profilePath: join(root, 'profile.json'), + allowWorkspaceWrites: false, allowWorkspaceCommands: false, + }; + let completeCalls = 0; + let startCalls = 0; + let initialRequest: BrowserSetupRequest | undefined; + let lostStart = false; + let lostComplete = false; + let status: 'approved' | 'denied' | 'expired' | 'cancelled' = 'approved'; + const credential = { protocolVersion: 1 as const, workerId: 'approved-machine', credential: 'approved-credential-that-is-long-enough', expiresAt: '2099-01-01T00:00:00.000Z' }; + const fetchImpl = (async (url: string | URL | Request, init?: RequestInit) => { + if (String(url).endsWith('/capabilities')) return Response.json({ browserEnrollmentVersion: 1 }); + const body = JSON.parse(String(init?.body)) as { request: BrowserSetupRequest; proof: BrowserSetupProof; signature: string; redemptionSignature?: string }; + assert.equal((body.request as unknown as Record).privateKey, undefined); + assert.equal(verifyBrowserSetupProof(body.request.publicKey, body.proof, body.signature), true); + assert.equal(body.proof.requestHash, browserSetupRequestHash(body.request)); + const saved = await loadBrowserSetupAttempt(options.profilePath!); + assert.equal(saved?.request.publicKey, body.request.publicKey, 'key saved before remote effect'); + assert.equal(saved?.request.attemptId, body.request.attemptId); + initialRequest ??= body.request; + assert.deepEqual(body.request, initialRequest, 'retries retain immutable request and key'); + const expiresAt = new Date(Date.parse(body.request.createdAt) + 600_000).toISOString(); + const approval = { operation: 'enrollment.redeem' as const, serverId: 'https://code.example', instance: body.request.instance, + attemptId: body.request.attemptId, requestHash: browserSetupRequestHash(body.request), workerId: credential.workerId, + expiresAt, codeApiUrl: 'https://code.example/v1' }; + const base = { version: 1, expiresAt, verificationCode: 'ABCD-EFGH', verificationUrl: `https://chat.example/code/setup/${body.request.attemptId}` }; + if (String(url).endsWith('/start')) { + startCalls++; + if (lostStart) { lostStart = false; throw new Error('lost issuance response'); } + return Response.json({ ...base, status: 'pending' }); + } + if (String(url).endsWith('/cancel')) return Response.json({ ...base, status: 'cancelled' }); + if (String(url).endsWith('/complete')) { + completeCalls++; + assert.equal(body.proof.operation, 'setup.complete'); + assert.equal(verifyEnrollmentRedemption(body.request.publicKey, approval, body.redemptionSignature!), true); + if (lostComplete) { lostComplete = false; throw new Error('lost completion response'); } + return Response.json({ ...base, status: 'completed', approval, credential }); + } + return Response.json({ ...base, status, ...(status === 'approved' ? { approval } : {}) }); + }) as typeof fetch; + return { root, options, fetchImpl, credential, counts: () => ({ completeCalls, startCalls }), + loseStart: () => { lostStart = true; }, loseComplete: () => { lostComplete = true; }, setStatus: (value: typeof status) => { status = value; } }; +} + +test('headless setup displays URL and code and saves the existing least-privilege profile', async t => { + const f = await fixture(t); + let displayed = false; + const saved = await setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0, onVerification: async value => { + assert.match(value.verificationUrl, /^https:\/\/chat.example\/code\/setup\//); + assert.equal(value.verificationCode, 'ABCD-EFGH'); + assert.ok(await loadBrowserSetupAttempt(f.options.profilePath!)); + displayed = true; + } }); + assert.equal(displayed, true); + assert.deepEqual(await loadWorkerProfile(saved.profilePath), saved.profile); + assert.equal(saved.profile.allowWorkspaceCommands, false); + assert.equal(saved.profile.allowWorkspaceWrites, false); + assert.deepEqual(saved.profile.workspace, { type: 'private' }); + const identity = await loadBridgeIdentity(saved.profile.identityPath); + assert.equal(identity.credential, f.credential.credential); + assert.equal((await stat(browserSetupAttemptPath(saved.profilePath))).mode & 0o777, 0o600); + assert.ok(!(await readFile(saved.profilePath, 'utf8')).includes('PRIVATE KEY')); + const receipt = JSON.parse(await readFile(browserSetupAttemptPath(saved.profilePath), 'utf8')); + assert.equal(receipt.privateKey, undefined); + assert.equal(receipt.identity, undefined); + assert.equal(receipt.completed, true); + assert.equal(identity.setupAttemptPath, browserSetupAttemptPath(saved.profilePath)); +}); + +for (const boundary of ['start', 'complete'] as const) test(`lost ${boundary} response resumes the identical key and attempt`, async t => { + const f = await fixture(t); + if (boundary === 'start') f.loseStart(); else f.loseComplete(); + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0 }), /lost/); + const before = await loadBrowserSetupAttempt(f.options.profilePath!); + assert.ok(before); + await setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0 }); + const after = await loadBrowserSetupAttempt(f.options.profilePath!); + assert.deepEqual(after?.request, before.request); + assert.equal(after?.privateKey, before.privateKey); + assert.equal(f.counts().startCalls, 2); +}); + +test('SIGKILL during journal publish preserves the attempt and contains the secret temporary after completion', { timeout: 10_000 }, async t => { + const f = await fixture(t); + f.loseComplete(); + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0 }), /lost/); + const before = (await loadBrowserSetupAttempt(f.options.profilePath!))!; + const attempt = browserSetupAttemptPath(f.options.profilePath!); + const child = spawn(process.execPath, ['--input-type=module', '--eval', ` + import fs from 'node:fs/promises'; + import { syncBuiltinESMExports } from 'node:module'; + const rename = fs.rename; + fs.rename = async (from, to) => { + if (to === ${JSON.stringify(attempt)}) { + process.send('journal-written'); + await new Promise(resolve => setTimeout(resolve, 60_000)); + } + return rename(from, to); + }; + syncBuiltinESMExports(); + const { savePrivateJson } = await import(${JSON.stringify(new URL('./storage.js', import.meta.url).href)}); + await savePrivateJson(${JSON.stringify(attempt)}, JSON.parse(await fs.readFile(${JSON.stringify(attempt)}, 'utf8'))); + `], { stdio: ['ignore', 'ignore', 'pipe', 'ipc'] }); + let stderr = ''; + child.stderr!.on('data', chunk => { stderr = (stderr + String(chunk)).slice(-4096); }); + const closed = once(child, 'close'); + t.after(async () => { child.kill('SIGKILL'); await closed; }); + await Promise.race([ + once(child, 'message', { signal: t.signal }).then(([message]) => assert.equal(message, 'journal-written')), + closed.then(() => { throw new Error(`Journal writer exited before the crash boundary: ${stderr}`); }), + ]); + child.kill('SIGKILL'); + assert.deepEqual(await closed, [null, 'SIGKILL']); + const directory = dirname(attempt); + const orphanName = (await readdir(directory)).find(name => /^attempt\.json\.[a-f0-9]{16}\.tmp$/.test(name)); + assert.ok(orphanName, 'real atomic writer left its secret generation when killed before rename'); + const orphan = join(directory, orphanName); + const crashCopy = JSON.parse(await readFile(orphan, 'utf8')); + assert.equal(crashCopy.privateKey, before.privateKey); + assert.deepEqual(browserSetupProtectedPaths(attempt), [directory]); + const saved = await setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0 }); + const identity = await loadBridgeIdentity(saved.profile.identityPath); + assert.equal(identity.privateKey, before.privateKey); + assert.equal(identity.setupAttemptPath, attempt); + assert.deepEqual((await loadBrowserSetupAttempt(saved.profilePath))?.request, before.request); + assert.equal(JSON.parse(await readFile(attempt, 'utf8')).privateKey, undefined); + assert.ok((await stat(orphan)).isFile(), 'crash leftovers stay inside the denied namespace after final-receipt scrubbing'); +}); + +test('an older draft journal is never silently replaced by a different key or attempt', async t => { + const f = await fixture(t); + const oldPath = `${f.options.profilePath}.setup.json`; + await writeFile(oldPath, 'preserve the existing draft key', { mode: 0o600 }); + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: async () => { assert.fail('no remote allocation'); } }), /older draft setup journal/); + assert.equal(await readFile(oldPath, 'utf8'), 'preserve the existing draft key'); +}); + +test('local profile failure after enrollment resumes without remote enrollment effects', async t => { + const f = await fixture(t); + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0, saveProfile: async () => { throw new Error('disk full'); } }), /enrolled.*profile persistence failed/); + assert.ok((await loadBrowserSetupAttempt(f.options.profilePath!))?.identity); + assert.equal(f.counts().completeCalls, 1); + const saved = await setupBrowserWorker(f.options, { fetchImpl: async () => { throw new Error('must not enroll again'); } }); + assert.ok(await loadWorkerProfile(saved.profilePath)); + assert.equal(f.counts().completeCalls, 1); +}); + +test('rerunning completed setup preserves a newer startup credential without network or redundant writes', async t => { + const f = await fixture(t); + const saved = await setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0 }); + const identity = { ...await loadBridgeIdentity(saved.profile.identityPath), credential: 'newer-credential-recovered-after-startup' }; + await saveBridgeIdentity(saved.profile.identityPath, identity); + await setupBrowserWorker(f.options, { + fetchImpl: async () => { assert.fail('completed setup does not enroll again'); }, + saveIdentity: async () => { assert.fail('do not replace a newer credential'); }, + saveProfile: async () => { assert.fail('do not rewrite an unchanged profile'); }, + }); + assert.deepEqual(await loadBridgeIdentity(saved.profile.identityPath), identity); +}); + +test('identity persistence failure after enrollment preserves the completed local receipt', async t => { + const f = await fixture(t); + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0, saveIdentity: async () => { throw new Error('identity disk full'); } }), /disk full/); + assert.ok((await loadBrowserSetupAttempt(f.options.profilePath!))?.identity); + await setupBrowserWorker(f.options, { fetchImpl: async () => { throw new Error('must not enroll again'); } }); + assert.equal(f.counts().completeCalls, 1); +}); + +test('expired setup with a lost completion response recovers only existing key-bound enrollment', async t => { + const f = await fixture(t); + f.loseComplete(); + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0 }), /lost/); + const attempt = (await loadBrowserSetupAttempt(f.options.profilePath!))!; + const now = Date.now(); + t.mock.method(Date, 'now', () => now + 11 * 60_000); + const saved = await setupBrowserWorker(f.options, { + fetchImpl: async () => { assert.fail('an expired attempt must not allocate another session'); }, + recoverCredential: async options => { + assert.equal(options.privateKey, attempt.privateKey); + assert.equal(options.workerId, attempt.approval?.workerId); + assert.equal(options.codeApiUrl, attempt.approval?.codeApiUrl); + return f.credential; + }, + }); + assert.equal((await loadBridgeIdentity(saved.profile.identityPath)).publicKey, attempt.request.publicKey); + assert.equal(f.counts().completeCalls, 1); +}); + +test('unsupported servers fail closed and never invoke manual redemption', async t => { + const f = await fixture(t); + let calls = 0; + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: async url => { + calls++; + assert.ok(String(url).endsWith('/capabilities')); + return Response.json({ protocolVersion: 1 }); + } }), /will not downgrade/); + assert.equal(calls, 1); + assert.ok(await loadBrowserSetupAttempt(f.options.profilePath!)); +}); + +for (const status of ['denied', 'expired', 'cancelled'] as const) test(`${status} setup preserves the saved identity and does not complete`, async t => { + const f = await fixture(t); + f.setStatus(status); + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0 }), new RegExp(status)); + assert.equal(f.counts().completeCalls, 0); + assert.ok(await loadBrowserSetupAttempt(f.options.profilePath!)); +}); + +test('retry cannot widen permissions, rename a machine, switch instances, or replace a key', async t => { + const f = await fixture(t); + f.loseStart(); + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl }), /lost/); + for (const change of [{ allowWorkspaceWrites: true }, { machineName: 'other-machine' }, { instance: 'https://other.example' }]) { + await assert.rejects(setupBrowserWorker({ ...f.options, ...change }, { fetchImpl: async () => { assert.fail('no remote effects'); } }), /choices differ/); + } + const attempt = (await loadBrowserSetupAttempt(f.options.profilePath!))!; + await savePrivateJson(browserSetupAttemptPath(f.options.profilePath!), { ...attempt, privateKey: createBridgeIdentity().privateKey }); + await assert.rejects(loadBrowserSetupAttempt(f.options.profilePath!), /key does not match/); +}); + +test('cancellation is signed by the persisted machine key', async t => { + const f = await fixture(t); + f.loseStart(); + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl }), /lost/); + await cancelBrowserSetup(f.options.profilePath!, undefined, f.fetchImpl); +}); + +test('cancellation after enrollment saves the profile but prevents requested foreground startup', { timeout: 5000 }, async t => { + const f = await fixture(t); + const controller = new AbortController(); + t.mock.method(globalThis, 'fetch', async (input: Parameters[0], init?: RequestInit) => { + const response = await f.fetchImpl(input, init); + if (String(input).endsWith('/complete')) controller.abort(new Error('owner cancelled startup')); + return response; + }); + await assert.rejects(runBrowserSetupCommand(new Map([ + ['--instance', f.options.instance], ['--machine-name', f.options.machineName], + ['--profile', f.options.profilePath!], ['--default-workspace', true], ['--start', true], + ]), { + interactive: false, signal: controller.signal, + ask: async () => { assert.fail('no prompt expected'); }, + yesNo: async () => { assert.fail('no prompt expected'); }, + }), /owner cancelled startup/); + assert.ok(await loadWorkerProfile(f.options.profilePath!)); +}); + +test('browser flags preserve manual automation without accepting mixed credentials', () => { + const flags = parseSetupArguments(['--instance', 'https://chat.example', '--non-interactive', '--default-workspace', '--no-browser']); + assert.equal(flags.get('--instance'), 'https://chat.example'); + for (const manual of ['--url=https://code.example/v1', '--worker-id=worker', '--code-stdin']) + assert.throws(() => parseSetupArguments(['--instance=https://chat.example', manual]), /manual pairing/); + assert.throws(() => parseSetupArguments(['--no-browser']), /require --instance/); + assert.equal(setupInstanceUrl('https://chat.example/'), 'https://chat.example'); + assert.throws(() => setupInstanceUrl('https://user:secret@chat.example'), /credentials/); +}); diff --git a/packages/code/src/browser-setup.ts b/packages/code/src/browser-setup.ts new file mode 100644 index 00000000..7bbbe493 --- /dev/null +++ b/packages/code/src/browser-setup.ts @@ -0,0 +1,352 @@ +import { createPrivateKey, createPublicKey, randomBytes } from 'node:crypto'; +import { lstat } from 'node:fs/promises'; +import { basename, dirname, join, resolve } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { createBridgeIdentity } from './identity.js'; +import { recoverWorkerCredential } from './credential-recovery.js'; +import { + browserSetupRequestHash, + signBrowserSetupProof, + signEnrollmentRedemption, + isBrowserEnrollmentRedemptionContract, +} from './browser-enrollment-protocol.js'; +import type { + BrowserSetupRequest, + BrowserSetupProof, + EnrollmentRedemption, +} from './browser-enrollment-protocol.js'; +import { assertPrivateStorageSupported } from './private-storage.js'; +import { withProcessLock } from './process-lock.js'; +import { + assertIdentityPathIsPrivate, + loadBridgeIdentity, + ensurePrivateWorkspaceDirectory, + readPrivateJson, + saveBridgeIdentity, + savePrivateJson, +} from './storage.js'; +import { checkSetupCommands, setupCodeApiUrl, setupWorkspaceDirectory } from './setup.js'; +import { + assertWorkerProfileIsolation, + defaultWorkerProfilePath, + loadWorkerProfile, + saveWorkerProfile, + workerProfileArguments, +} from './worker-profile.js'; +import type { WorkerProfile } from './worker-profile.js'; +import type { PairedBridgeWorkerIdentity } from './pairing.js'; + +export interface BrowserSetupAttempt { + version: 1; + request: BrowserSetupRequest; + privateKey: string; + profile: WorkerProfile; + approval?: BrowserSetupApproval; + identity?: PairedBridgeWorkerIdentity; + completed?: true; +} + +export type BrowserSetupApproval = EnrollmentRedemption & { codeApiUrl: string }; + +export interface BrowserSetupOptions { + instance: string; + machineName: string; + profilePath?: string; + identityPath?: string; + workspaceDirectory?: string; + allowWorkspaceWrites: boolean; + allowWorkspaceCommands: boolean; + signal?: AbortSignal; +} + +interface SetupStatus { + version: 1; + status: 'pending' | 'approved' | 'completed' | 'denied' | 'cancelled' | 'expired'; + expiresAt: string; + verificationUrl: string; + verificationCode: string; + approval?: BrowserSetupApproval; + credential?: { protocolVersion: 1; workerId: string; credential: string; expiresAt: string }; +} + +interface BrowserSetupDependencies { + fetchImpl?: typeof fetch; + checkCommands?: typeof checkSetupCommands; + onVerification?: (status: Pick) => void | Promise; + pollIntervalMs?: number; + saveProfile?: typeof saveWorkerProfile; + saveIdentity?: typeof saveBridgeIdentity; + recoverCredential?: typeof recoverWorkerCredential; +} + +function record(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function setupInstanceUrl(value: string): string { + return setupCodeApiUrl(value); +} + +export function browserSetupAttemptPath(profilePath: string): string { + return join(`${resolve(profilePath)}.setup`, 'attempt.json'); +} + +export function browserSetupProtectedPaths(attemptPath: string | undefined): string[] { + if (attemptPath === undefined) return []; + const directory = dirname(attemptPath); + if (basename(attemptPath) !== 'attempt.json' || !directory.endsWith('.setup')) + throw new Error('Unsupported setup journal location; preserve the original identity and journal'); + // Directory protection includes atomic-write temporaries, even after SIGKILL. + return [directory]; +} + +export async function loadBrowserSetupAttempt(profilePath: string): Promise { + // The earlier, unreleased draft stored journals beside profiles. Never rekey them silently. + const previous = await lstat(`${resolve(profilePath)}.setup.json`).catch((error: NodeJS.ErrnoException) => { + if (error.code === 'ENOENT') return undefined; + throw error; + }); + if (previous) throw new Error('An older draft setup journal exists; preserve its key and attempt instead of starting a new enrollment'); + let value: unknown; + try { value = await readPrivateJson(browserSetupAttemptPath(profilePath), 32 * 1024); } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; + } + if (!record(value) || value.version !== 1 || !record(value.request) || !record(value.profile)) + throw new Error('Invalid saved setup attempt; preserve it and choose another profile, do not replace its machine key'); + const attempt = value as unknown as BrowserSetupAttempt; + workerProfileArguments(attempt.profile); + if (attempt.completed === true) { + attempt.identity = await loadBridgeIdentity(attempt.profile.identityPath); + attempt.privateKey = attempt.identity.privateKey; + } + if (typeof attempt.privateKey !== 'string') throw new Error('Saved setup key is unavailable; preserve the original identity'); + if (setupInstanceUrl(attempt.request.instance) !== attempt.request.instance || + createPublicKey(createPrivateKey(attempt.privateKey)).export({ type: 'spki', format: 'pem' }) !== attempt.request.publicKey) + throw new Error('Saved setup key does not match its request'); + // The shared canonicalizer validates the immutable request before it is signed. + browserSetupRequestHash(attempt.request); + if (attempt.approval && (!isBrowserEnrollmentRedemptionContract(attempt.approval) || + attempt.approval.instance !== attempt.request.instance || attempt.approval.attemptId !== attempt.request.attemptId || + attempt.approval.requestHash !== browserSetupRequestHash(attempt.request) || + setupCodeApiUrl(attempt.approval.codeApiUrl) !== attempt.approval.codeApiUrl || + new URL(attempt.approval.codeApiUrl).origin !== attempt.approval.serverId)) + throw new Error('Saved approval does not match its setup request'); + if (attempt.identity && (!attempt.approval || attempt.identity.protocolVersion !== 1 || + attempt.identity.publicKey !== attempt.request.publicKey || attempt.identity.privateKey !== attempt.privateKey || + attempt.identity.workerId !== attempt.approval.workerId || attempt.identity.codeApiUrl !== attempt.approval.codeApiUrl || + typeof attempt.identity.credential !== 'string' || attempt.identity.credential.length < 32 || + attempt.identity.credential.length > 4096 || !Number.isFinite(Date.parse(attempt.identity.expiresAt)))) + throw new Error('Saved completion does not match its approved machine key'); + return attempt; +} + +async function boundedJson(response: Response): Promise { + const reader = response.body?.getReader(); + if (!reader) throw new Error('LibreChat returned an empty setup response'); + let size = 0; + const chunks: Uint8Array[] = []; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > 32 * 1024) throw new Error('LibreChat setup response exceeds 32 KiB'); + chunks.push(value); + } + try { return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks))) as unknown; } + catch { throw new Error('LibreChat returned an invalid setup response'); } + } finally { await reader.cancel().catch(() => undefined); reader.releaseLock(); } +} + +async function requestJson(instance: string, path: string, body: unknown, signal: AbortSignal | undefined, fetchImpl: typeof fetch): Promise { + const response = await fetchImpl(`${instance}/api/code-environments/setup/${path}`, { + method: body === undefined ? 'GET' : 'POST', + redirect: 'error', + headers: { 'Content-Type': 'application/json' }, + body: body === undefined ? undefined : JSON.stringify(body), + signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(30_000)]) : AbortSignal.timeout(30_000), + }); + if (!response.ok) { + await response.body?.cancel(); + if ([404, 405, 501].includes(response.status)) + throw new Error('This LibreChat instance does not support browser-approved machine setup. Update compatible servers first, or explicitly use the existing manual pairing flow'); + throw new Error(`LibreChat setup request failed (HTTP ${response.status}). Your saved key and attempt are unchanged; rerun the same setup command to resume`); + } + return boundedJson(response); +} + +function machineBody(attempt: BrowserSetupAttempt, operation: BrowserSetupProof['operation']): object { + const proof: BrowserSetupProof = { + operation, + instance: attempt.request.instance, + attemptId: attempt.request.attemptId, + requestHash: browserSetupRequestHash(attempt.request), + timestamp: new Date().toISOString(), + nonce: randomBytes(32).toString('base64url'), + }; + return { + request: attempt.request, + proof, + signature: signBrowserSetupProof(attempt.privateKey, proof), + ...(operation === 'setup.complete' && attempt.approval ? { + redemptionSignature: signEnrollmentRedemption(attempt.privateKey, attempt.approval), + } : {}), + }; +} + +function parseStatus(value: unknown, attempt: BrowserSetupAttempt): SetupStatus { + if (!record(value) || value.version !== 1 || + !['pending', 'approved', 'completed', 'denied', 'cancelled', 'expired'].includes(String(value.status)) || + typeof value.expiresAt !== 'string' || !Number.isFinite(Date.parse(value.expiresAt)) || + Date.parse(value.expiresAt) > Date.parse(attempt.request.createdAt) + 10 * 60_000 || + typeof value.verificationUrl !== 'string' || typeof value.verificationCode !== 'string' || + !/^[A-Z0-9-]{6,20}$/.test(value.verificationCode)) + throw new Error('LibreChat returned an invalid setup session'); + let url: URL; + try { url = new URL(value.verificationUrl); } + catch { throw new Error('LibreChat returned an invalid approval destination'); } + if (url.href !== `${attempt.request.instance}/code/setup/${attempt.request.attemptId}` || + url.username || url.password || url.search || url.hash) + throw new Error('LibreChat returned an unexpected approval destination'); + if (value.approval !== undefined) { + const approval = value.approval as BrowserSetupApproval; + if (!isBrowserEnrollmentRedemptionContract(approval) || approval.operation !== 'enrollment.redeem' || + approval.instance !== attempt.request.instance || approval.attemptId !== attempt.request.attemptId || + approval.requestHash !== browserSetupRequestHash(attempt.request) || + approval.expiresAt !== value.expiresAt || typeof approval.workerId !== 'string' || + typeof approval.codeApiUrl !== 'string' || setupCodeApiUrl(approval.codeApiUrl) !== approval.codeApiUrl || + new URL(approval.codeApiUrl).origin !== approval.serverId) + throw new Error('LibreChat approval does not match this setup attempt'); + if (attempt.approval && JSON.stringify(attempt.approval) !== JSON.stringify(approval)) + throw new Error('LibreChat changed the saved approval; refusing to replace the machine binding'); + } + return value as unknown as SetupStatus; +} + +async function assertEmptyOrMatching(path: string, expected: unknown, load: (path: string) => Promise, same = (left: unknown, right: unknown) => JSON.stringify(left) === JSON.stringify(right)): Promise { + const metadata = await lstat(path); + if (metadata.isFile() && metadata.size === 0) return false; + if (!same(await load(path), expected)) + throw new Error('Existing profile or identity differs from the saved setup attempt; refusing to replace it'); + return true; +} + +function sameEnrolledKey(left: unknown, right: unknown): boolean { + return record(left) && record(right) && ['protocolVersion', 'workerId', 'codeApiUrl', 'publicKey', 'privateKey', 'setupAttemptPath'] + .every(key => left[key] === right[key]); +} + +/** Every remote mutation follows a durable, private key and immutable local request. */ +export async function setupBrowserWorker(options: BrowserSetupOptions, dependencies: BrowserSetupDependencies = {}): Promise<{ profilePath: string; profile: WorkerProfile }> { + options.signal?.throwIfAborted(); + assertPrivateStorageSupported(); + const instance = setupInstanceUrl(options.instance); + const profilePath = resolve(options.profilePath ?? defaultWorkerProfilePath()); + const attemptPath = browserSetupAttemptPath(profilePath); + await ensurePrivateWorkspaceDirectory(dirname(attemptPath)); + // This stable inode is deliberately retained. Removing an advisory lock can split its holders. + await assertIdentityPathIsPrivate(`${attemptPath}.lock`); + return withProcessLock(`${attemptPath}.lock`, async () => { + let attempt = await loadBrowserSetupAttempt(profilePath); + const workspaceDirectory = options.workspaceDirectory === undefined ? undefined : await setupWorkspaceDirectory(options.workspaceDirectory); + const profile: WorkerProfile = { + version: 1, + identityPath: resolve(options.identityPath ?? `${profilePath}.identity.json`), + workspace: workspaceDirectory ? { type: 'directory', path: workspaceDirectory } : { type: 'private' }, + allowWorkspaceWrites: options.allowWorkspaceWrites, + allowWorkspaceCommands: options.allowWorkspaceCommands, + }; + workerProfileArguments(profile); + if (attempt && (attempt.request.instance !== instance || attempt.request.machineName !== options.machineName || JSON.stringify(attempt.profile) !== JSON.stringify(profile))) + throw new Error('Setup choices differ from the saved attempt. Resume its original instance and permissions, or explicitly select another profile'); + if (profile.allowWorkspaceCommands) await (dependencies.checkCommands ?? checkSetupCommands)(); + const reservations = []; + try { + reservations.push(await assertIdentityPathIsPrivate(profilePath, !attempt)); + reservations.push(await assertIdentityPathIsPrivate(profile.identityPath, !attempt)); + reservations.push(await assertIdentityPathIsPrivate(attemptPath, !attempt)); + await assertWorkerProfileIsolation(profilePath, profile.identityPath, workspaceDirectory); + await assertWorkerProfileIsolation(attemptPath, profilePath, workspaceDirectory); + await assertWorkerProfileIsolation(attemptPath, profile.identityPath, workspaceDirectory); + if (!attempt) { + const key = createBridgeIdentity(); + attempt = { + version: 1, + request: { + version: 1, instance, attemptId: randomBytes(32).toString('base64url'), + createdAt: new Date().toISOString(), publicKey: key.publicKey, + machineName: options.machineName, workspace: { type: profile.workspace.type }, + allowWorkspaceWrites: profile.allowWorkspaceWrites, allowWorkspaceCommands: profile.allowWorkspaceCommands, + }, + privateKey: key.privateKey, profile, + }; + browserSetupRequestHash(attempt.request); + await savePrivateJson(attemptPath, attempt); + } + const persist = () => savePrivateJson(attemptPath, attempt); + const publish = async () => { + if (!await assertEmptyOrMatching(profile.identityPath, attempt!.identity, loadBridgeIdentity, sameEnrolledKey)) + await (dependencies.saveIdentity ?? saveBridgeIdentity)(profile.identityPath, attempt!.identity!); + if (!attempt!.completed) { + await savePrivateJson(attemptPath, { version: 1, request: attempt!.request, profile, approval: attempt!.approval, completed: true }); + attempt!.completed = true; + } + const profileSaved = await assertEmptyOrMatching(profilePath, profile, loadWorkerProfile); + try { if (!profileSaved) await (dependencies.saveProfile ?? saveWorkerProfile)(profilePath, profile); } + catch { throw new Error(`Machine enrolled and identity saved, but profile persistence failed. Rerun setup with this same profile to resume: ${profilePath}`); } + return { profilePath, profile }; + }; + if (attempt.identity) return await publish(); + const fetchImpl = dependencies.fetchImpl ?? fetch; + if (attempt.approval && Date.now() >= Date.parse(attempt.approval.expiresAt)) { + // Session expiry cannot undo enrollment. Recovery proves only existing trust. + const credential = await (dependencies.recoverCredential ?? recoverWorkerCredential)({ + codeApiUrl: attempt.approval.codeApiUrl, workerId: attempt.approval.workerId, + privateKey: attempt.privateKey, fetchImpl, validThroughMs: Date.now() + 60_000, + signal: options.signal ? AbortSignal.any([options.signal, AbortSignal.timeout(30_000)]) : AbortSignal.timeout(30_000), + }); + attempt.identity = { ...credential, codeApiUrl: attempt.approval.codeApiUrl, + publicKey: attempt.request.publicKey, privateKey: attempt.privateKey, setupAttemptPath: attemptPath }; + await persist(); + return await publish(); + } + const capabilities = await requestJson(instance, 'capabilities', undefined, options.signal, fetchImpl); + if (!record(capabilities) || capabilities.browserEnrollmentVersion !== 1) + throw new Error('This instance lacks compatible key-bound browser enrollment. Update compatible servers first; setup will not downgrade to manual pairing'); + let status = parseStatus(await requestJson(instance, 'start', machineBody(attempt, 'setup.start'), options.signal, fetchImpl), attempt); + await dependencies.onVerification?.(status); + for (;;) { + options.signal?.throwIfAborted(); + if (['denied', 'cancelled', 'expired'].includes(status.status)) throw new Error(`Machine setup ${status.status}. The original key and attempt were preserved; choose another profile only for an intentional new enrollment`); + if (status.approval) { attempt.approval = status.approval; await persist(); } + if (status.status === 'approved' || status.status === 'completed') { + if (!attempt.approval) throw new Error('LibreChat did not supply the key-bound approval contract'); + const completed = parseStatus(await requestJson(instance, 'complete', machineBody(attempt, 'setup.complete'), options.signal, fetchImpl), attempt); + const credential = completed.credential; + if (completed.status !== 'completed' || !credential || credential.protocolVersion !== 1 || + credential.workerId !== attempt.approval.workerId || typeof credential.credential !== 'string' || + credential.credential.length < 32 || credential.credential.length > 4096 || + !Number.isFinite(Date.parse(credential.expiresAt))) throw new Error('LibreChat returned an invalid enrollment credential'); + attempt.identity = { protocolVersion: 1, workerId: credential.workerId, credential: credential.credential, expiresAt: credential.expiresAt, + codeApiUrl: attempt.approval.codeApiUrl, publicKey: attempt.request.publicKey, privateKey: attempt.privateKey, setupAttemptPath: attemptPath }; + // Keep replayable local state even if either final file cannot be published. + await persist(); + return await publish(); + } + if (Date.now() >= Date.parse(status.expiresAt)) throw new Error('Machine setup expired; the saved key and attempt were preserved'); + await delay(dependencies.pollIntervalMs ?? 2000, undefined, { signal: options.signal }); + status = parseStatus(await requestJson(instance, 'poll', machineBody(attempt, 'setup.poll'), options.signal, fetchImpl), attempt); + } + } finally { for (const reservation of reservations.reverse()) await reservation.release(); } + }, options.signal, false); +} + +export async function cancelBrowserSetup(profilePath: string, signal?: AbortSignal, fetchImpl: typeof fetch = fetch): Promise { + const attempt = await loadBrowserSetupAttempt(profilePath); + if (!attempt) throw new Error('No saved setup attempt exists for this profile'); + if (attempt.identity) throw new Error('This machine is already enrolled; revoke it through LibreChat instead'); + await requestJson(attempt.request.instance, 'cancel', machineBody(attempt, 'setup.cancel'), signal, fetchImpl); +} diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 85ce947e..a0352d34 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -1,10 +1,17 @@ #!/usr/bin/env node import { createHash, createHmac, randomBytes } from 'node:crypto'; import { readFileSync } from 'node:fs'; -import { readdir, realpath, stat } from 'node:fs/promises'; +import { lstat, readdir, realpath, stat } from 'node:fs/promises'; import { basename, dirname, join, resolve, relative, isAbsolute, sep } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; +import { commandFailureLogLine } from './command-diagnostics.js'; +import { runSetupCommand } from './setup-cli.js'; +import { browserSetupProtectedPaths } from './browser-setup.js'; +import { assertPrivateStorageAncestors } from './private-storage.js'; +import { profileRunConfiguration } from './setup.js'; +import { assertWorkerProfileIsolation } from './worker-profile.js'; +import type { WorkerProfile } from './worker-profile.js'; import { discoverProjects } from './projects.js'; import { loadProjectRoots, projectRootArguments } from './project-roots.js'; import { @@ -33,6 +40,7 @@ import { } from './storage.js'; import { BridgeWorker } from './worker.js'; import { LocalWorkspaceTools, SandboxWorkspaceTools } from './workspace.js'; +import { createWorkspaceFileImporter } from './workspace-import.js'; import { DockerRuntimeSupervisor, EndpointRuntimeSupervisor, @@ -72,6 +80,7 @@ import type { RuntimeSupervisor } from './runtime.js'; import type { GitHubCredentialProvider } from './github.js'; import type { WorkspaceToolExecutor } from './workspace.js'; import { + BRIDGE_WORKSPACE_IMPORT_FILE_DEFAULT_MAX_BYTES, BRIDGE_WORKSPACE_NAME_MAX_LENGTH, BridgeProtocolError, isValidBridgeWorkerCapabilities, @@ -358,6 +367,7 @@ async function relay(): Promise { async function run( runtimeSessionId?: string, args: string[] = [], + savedProfile?: { path: string; config: WorkerProfile }, ): Promise { const projectArgs = projectRootArguments(args); if ( @@ -422,7 +432,7 @@ async function run( } const configuredWorkerId = process.env.LIBRECHAT_CODE_WORKER_ID?.trim(); const configuredIdentityPath = - process.env.LIBRECHAT_CODE_IDENTITY_FILE?.trim(); + savedProfile?.config.identityPath ?? process.env.LIBRECHAT_CODE_IDENTITY_FILE?.trim(); const configuredToken = process.env.LIBRECHAT_CODE_WORKER_TOKEN?.trim(); const identityPath = configuredIdentityPath ?? @@ -432,6 +442,7 @@ async function run( const pairedIdentity = identityPath ? await loadBridgeIdentity(identityPath) : undefined; + const setupProtectedPaths = browserSetupProtectedPaths(pairedIdentity?.setupAttemptPath); const workerId = required( 'LIBRECHAT_CODE_WORKER_ID', configuredWorkerId ?? pairedIdentity?.workerId, @@ -645,6 +656,9 @@ async function run( throw new Error('Invalid workspace registration'); } } + if (savedProfile) { + await assertWorkerProfileIsolation(savedProfile.path, identityPath!, canonicalWorkerDirectory); + } const mutationQuarantinePath = (allowWorkspaceWrites || allowWorkspaceCommands) && canonicalWorkerDirectory @@ -842,6 +856,19 @@ async function run( }), ]), ); + for (const path of setupProtectedPaths) { + const namespace = await lstat(path).catch((error: NodeJS.ErrnoException) => { + if (error.code === 'ENOENT') return undefined; + throw error; + }); + if (!namespace) continue; + if (!namespace.isDirectory() || namespace.isSymbolicLink()) + throw new Error('Setup journal namespace must be a private directory, not a symlink'); + const sourceParents = await assertPrivateStorageAncestors(path); + await assertEnvironmentDefinitionsOutsideRoots([{ + path, sourceParents, definition: { name: 'setup-state', root: path }, fingerprint: '', + }], roots); + } const preparationDirectory = join(dirname(identityPath ?? defaultBridgeIdentityPath(workerId)), 'environment-preparation'); if (environments.some(environment => environment.definition.setup?.reuse)) { const sourceParents = await prepareEnvironmentPreparationDirectory(preparationDirectory); @@ -857,13 +884,13 @@ async function run( definition: { name: 'shared-resource', root: resource.path }, fingerprint: '', }))); await assertEnvironmentResourceIsolation(environments.flatMap(environment => environment.resources ?? []), roots.map(root => root.root), [ - identityPath, preparationDirectory, github.privateKeyPath, + identityPath, ...setupProtectedPaths, preparationDirectory, github.privateKeyPath, ...environments.map(environment => environment.path), ...rootQuarantinePaths.values(), ].filter((path): path is string => path != null)); await assertEnvironmentDefinitionsOutsideRoots(resourceRoots, roots); const snapshotStores = environments.flatMap(environment => environment.snapshotStore ? [environment.snapshotStore] : []); await assertEnvironmentResourceIsolation(snapshotStores.map(store => ({ kind: 'npm-cache' as const, path: store.store, access: 'read-only' as const })), roots.map(root => root.root), [ - identityPath, preparationDirectory, github.privateKeyPath, ...environments.map(environment => environment.path), + identityPath, ...setupProtectedPaths, preparationDirectory, github.privateKeyPath, ...environments.map(environment => environment.path), ...rootQuarantinePaths.values(), ...environments.flatMap(environment => (environment.resources ?? []).map(resource => resource.path)), ].filter((path): path is string => path != null)); await assertEnvironmentDefinitionsOutsideRoots(snapshotStores.map(store => ({ path: store.store, sourceParents: store.controlPaths, @@ -890,10 +917,25 @@ async function run( ]), ) : undefined; + // A chat upload reaches the workspace only through the same relay upstream + // programmatic inputs use; without one the worker never advertises import_file. + const workspaceImporter = createWorkspaceFileImporter({ + optOut: args.includes('--no-workspace-imports'), + enabled: process.env.LIBRECHAT_CODE_ALLOW_WORKSPACE_IMPORTS, + writable: allowWorkspaceWrites, + upstreamUrl: fileRelayUpstream, + maxBytes: positiveInteger( + 'LIBRECHAT_CODE_IMPORT_FILE_MAX_BYTES', + option(args, '--import-file-max-bytes') ?? + process.env.LIBRECHAT_CODE_IMPORT_FILE_MAX_BYTES, + BRIDGE_WORKSPACE_IMPORT_FILE_DEFAULT_MAX_BYTES, + ), + }); const localWorkspaceTools = workerDirectory ? await LocalWorkspaceTools.create({ workspaces: roots, repositoryInstructions: args.includes('--repository-instructions'), + importer: workspaceImporter, }) : undefined; let workspaceTools: WorkspaceToolExecutor | undefined = localWorkspaceTools; @@ -1092,6 +1134,8 @@ async function run( commandPolicy, protectedPaths: [ identityPath, + savedProfile?.path, + ...setupProtectedPaths, ...(environments.some(environment => environment.definition.setup?.reuse) ? [preparationDirectory] : []), ...snapshotStores.map(store => store.store), ...environments.map(environment => environment.path), @@ -1272,6 +1316,7 @@ async function run( }, repositoryInstructions: args.includes('--repository-instructions'), writable: root.writable ?? false, + importer: workspaceImporter, }, ]), ), @@ -1307,6 +1352,7 @@ async function run( command: nativeOptionsForWorkspace(root.id), repositoryInstructions: args.includes('--repository-instructions'), writable: root.writable ?? false, + importer: workspaceImporter, }, ]), ), @@ -1482,8 +1528,17 @@ async function run( throw error; } worktreeRetirementScheduler?.start(); + let readinessReported = false; try { const worker = new BridgeWorker({ + onCommandFailure: event => { + process.stderr.write(commandFailureLogLine(event)); + }, + onReady: () => { + if (readinessReported) return; + readinessReported = true; + process.stdout.write('librechat-code: connected and ready\n'); + }, instructionDescriptors: () => localWorkspaceTools?.instructionDescriptors() ?? Promise.resolve(undefined), codeApiUrl, token: configuredToken, @@ -1766,6 +1821,26 @@ async function clearMutationQuarantine(args: string[]): Promise { async function main(): Promise { const args = process.argv.slice(2); + if (args.length === 1 && (args[0] === '--help' || args[0] === 'help')) { + process.stdout.write(`Usage: librechat-code + + setup --instance Browser-approved enrollment and private profile setup + run [--profile ] Start a saved profile or existing worker configuration + pair --worker-id Existing one-time pairing flow + projects --root Inspect local Git projects + reset-workspace Existing runtime maintenance + clear-workspace-quarantine Existing workspace maintenance + +Run librechat-code setup --help for setup options. +Legacy LIBRECHAT_CODE_* configuration takes precedence over the default profile. +Explicit profiles cannot be mixed with legacy settings or run options. +`); + return; + } + if (args[0] === 'setup') { + await runSetupCommand(args.slice(1)); + return; + } if (args[0] === 'prune-environment-storage') { const paths: string[] = []; for (let i = 1; i < args.length; i++) { @@ -1823,7 +1898,10 @@ async function main(): Promise { if (args[0] && args[0] !== 'run') { throw new Error(`Unknown command: ${args[0]}`); } - await run(undefined, args.slice(1)); + const configuration = await profileRunConfiguration(args.slice(1)); + await run(undefined, configuration.args, configuration.profile && configuration.profilePath + ? { path: configuration.profilePath, config: configuration.profile } + : undefined); } main().catch((error: Error) => { process.stderr.write(`librechat-code: ${error.message}\n`); diff --git a/packages/code/src/command-diagnostics.test.ts b/packages/code/src/command-diagnostics.test.ts new file mode 100644 index 00000000..76e48f30 --- /dev/null +++ b/packages/code/src/command-diagnostics.test.ts @@ -0,0 +1,165 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + commandFailureDiagnostic, + commandFailureEvent, + commandFailureLogLine, + readCommandFailureDiagnostic, +} from './command-diagnostics.js'; + +test('command diagnostics retain only fixed classifications and allowlisted errno', () => { + const error = Object.assign( + new Error('token secret /private/path echo password'), + { + code: 'ENOSPC', + command: 'echo password', + token: 'secret', + }, + ); + assert.deepEqual( + commandFailureDiagnostic('sandbox_startup', 'startup_failed', error), + { + stage: 'sandbox_startup', + reason: 'startup_failed', + osCode: 'ENOSPC', + }, + ); + assert.deepEqual( + commandFailureDiagnostic('credential_resolution', 'credential_failed', { + code: 'private-provider-value', + }), + { stage: 'credential_resolution', reason: 'credential_failed' }, + ); +}); + +test('IPC diagnostics are reconstructed rather than forwarding arbitrary metadata', () => { + const diagnostic = { + stage: 'sandbox_startup', + reason: 'startup_failed', + osCode: 'EPERM', + message: 'secret', + command: 'echo password', + path: '/private/path', + }; + assert.deepEqual(readCommandFailureDiagnostic(diagnostic), { + stage: 'sandbox_startup', + reason: 'startup_failed', + osCode: 'EPERM', + }); + for (const value of [ + null, + 'secret', + {}, + { stage: 'secret', reason: 'startup_failed' }, + { stage: 'sandbox_startup', reason: 'secret' }, + ]) { + assert.equal(readCommandFailureDiagnostic(value), undefined); + } + assert.deepEqual( + readCommandFailureDiagnostic({ ...diagnostic, osCode: 'secret' }), + { + stage: 'sandbox_startup', + reason: 'startup_failed', + }, + ); +}); + +test('failure events bound correlation IDs and omit free-form error data', () => { + const event = commandFailureEvent(undefined, { + assignmentId: 'assignment-1', + workerId: 'worker-1', + incarnationId: 'incarnation-1', + workspaceId: '/private/path\nsecret', + operation: 'execute_command', + mutationMayHaveCommitted: false, + requiresQuarantine: false, + }); + assert.deepEqual(JSON.parse(JSON.stringify(event)), { + event: 'workspace_command_unavailable', + stage: 'dispatch', + reason: 'executor_rejected', + assignmentId: 'assignment-1', + workerId: 'worker-1', + incarnationId: 'incarnation-1', + operation: 'execute_command', + mutationMayHaveCommitted: false, + requiresQuarantine: false, + }); + assert.equal( + commandFailureEvent(undefined, { + ...event, + assignmentId: 'a'.repeat(129), + }).assignmentId, + undefined, + ); +}); + +test('diagnostic correlation retains every valid worker/workspace identifier character', () => { + const event = commandFailureEvent(undefined, { + assignmentId: '_assignment-1', + workerId: 'vm.example', + workspaceId: 'repo:api', + incarnationId: 'worker_v1-2', + operation: 'execute_command', + mutationMayHaveCommitted: false, + requiresQuarantine: false, + }); + assert.equal(event.workerId, 'vm.example'); + assert.equal(event.workspaceId, 'repo:api'); + assert.equal(event.incarnationId, 'worker_v1-2'); + assert.equal(event.assignmentId, '_assignment-1'); + assert.equal( + commandFailureEvent(undefined, { ...event, workerId: 'vm\n' }).workerId, + undefined, + ); +}); + +test('diagnostic extraction cannot invoke exception getters or mask a failure', () => { + const error = { + get code() { + throw new Error('private getter'); + }, + }; + assert.deepEqual( + commandFailureDiagnostic( + 'credential_resolution', + 'credential_failed', + error, + ), + { + stage: 'credential_resolution', + reason: 'credential_failed', + }, + ); + assert.deepEqual( + commandFailureDiagnostic( + 'credential_resolution', + 'credential_failed', + new Proxy( + {}, + { + getOwnPropertyDescriptor() { + throw new Error('private proxy'); + }, + }, + ), + ), + { + stage: 'credential_resolution', + reason: 'credential_failed', + }, + ); +}); + +test('CLI diagnostic lines are complete JSON records without a text prefix', () => { + const event = commandFailureEvent(undefined, { + assignmentId: 'assignment-1', + operation: 'execute_command', + mutationMayHaveCommitted: false, + requiresQuarantine: false, + }); + const line = commandFailureLogLine(event); + assert.equal(line.endsWith('\n'), true); + assert.equal(line.split('\n').length, 2); + assert.deepEqual(JSON.parse(line), JSON.parse(JSON.stringify(event))); +}); diff --git a/packages/code/src/command-diagnostics.ts b/packages/code/src/command-diagnostics.ts new file mode 100644 index 00000000..86b42c5b --- /dev/null +++ b/packages/code/src/command-diagnostics.ts @@ -0,0 +1,155 @@ +import { BRIDGE_WORKER_ID_PATTERN } from './protocol.js'; + +/** Worker-local diagnostics: never add these fields to bridge settlements. */ +export type CommandFailureStage = + | 'allocation' + | 'sandbox_startup' + | 'programmatic_preflight' + | 'credential_resolution' + | 'command_wrapping' + | 'dispatch' + | 'executor_transport' + | 'cleanup'; + +const reasons = [ + 'workspace_busy', + 'capacity_exhausted', + 'allocation_failed', + 'startup_failed', + 'prerequisite_failed', + 'credential_failed', + 'wrapping_failed', + 'executor_rejected', + 'executor_unavailable', + 'executor_lost', + 'send_failed', + 'rpc_timeout', + 'cleanup_failed', +] as const; +export type CommandFailureReason = (typeof reasons)[number]; + +const setupReasons = { + sandbox_startup: 'startup_failed', + programmatic_preflight: 'prerequisite_failed', + credential_resolution: 'credential_failed', + command_wrapping: 'wrapping_failed', +} as const; +export type CommandSetupStage = keyof typeof setupReasons; + +export function commandSetupFailureDiagnostic( + stage: CommandSetupStage, + error: unknown, +): CommandFailureDiagnostic { + return commandFailureDiagnostic(stage, setupReasons[stage], error); +} +export interface CommandFailureDiagnostic { + stage: CommandFailureStage; + reason: CommandFailureReason; + osCode?: string; +} +const stages = new Set([ + 'allocation', + 'sandbox_startup', + 'programmatic_preflight', + 'credential_resolution', + 'command_wrapping', + 'dispatch', + 'executor_transport', + 'cleanup', +]); +const osCodes = new Set([ + 'ENOSPC', + 'EDQUOT', + 'EACCES', + 'EPERM', + 'ENOMEM', + 'EMFILE', + 'ENFILE', + 'EIO', + 'ENOENT', + 'EPIPE', + 'ECONNRESET', + 'ETIMEDOUT', +]); + +export function commandFailureDiagnostic( + stage: CommandFailureStage, + reason: CommandFailureReason, + error?: unknown, +): CommandFailureDiagnostic { + let code: unknown; + try { + // Diagnostic extraction must not execute an exception object's getters. + if (error != null && typeof error === 'object') { + code = Object.getOwnPropertyDescriptor(error, 'code')?.value; + } + } catch { + // Even an unusual provider error/proxy cannot replace the original failure. + } + return { + stage, + reason, + ...(typeof code === 'string' && osCodes.has(code) + ? { osCode: code } + : {}), + }; +} + +/** Reconstruct only allowlisted fields, even across the trusted executor IPC. */ +export function readCommandFailureDiagnostic( + value: unknown, +): CommandFailureDiagnostic | undefined { + if (value == null || typeof value !== 'object') return; + const v = value as Partial; + if (!stages.has(v.stage ?? '') || !reasons.includes(v.reason!)) return; + return commandFailureDiagnostic(v.stage!, v.reason!, { code: v.osCode }); +} + +export interface CommandFailureEvent extends CommandFailureDiagnostic { + event: 'workspace_command_unavailable'; + assignmentId?: string; + workspaceId?: string; + workerId?: string; + incarnationId?: string; + operation: 'execute_command' | 'workspace_programmatic'; + mutationMayHaveCommitted: boolean; + requiresQuarantine: boolean; +} + +/** One complete JSON record, suitable for a JSON-lines log collector. */ +export function commandFailureLogLine(event: CommandFailureEvent): string { + return `${JSON.stringify(event)}\n`; +} + +export function commandFailureEvent( + diagnostic: CommandFailureDiagnostic | undefined, + context: Omit< + CommandFailureEvent, + keyof CommandFailureDiagnostic | 'event' + >, +): CommandFailureEvent { + const id = ( + value: string | undefined, + pattern: RegExp, + ): string | undefined => + value && + value.length <= 128 && + !/[\r\n]/.test(value) && + pattern.test(value) + ? value + : undefined; + // Assignment/incarnation IDs are opaque and may begin with '-' or '_'. + const opaqueIdPattern = /^[A-Za-z0-9._:-]{1,128}$/; + return { + event: 'workspace_command_unavailable', + ...(readCommandFailureDiagnostic(diagnostic) ?? + commandFailureDiagnostic('dispatch', 'executor_rejected')), + assignmentId: id(context.assignmentId, opaqueIdPattern), + workspaceId: id(context.workspaceId, BRIDGE_WORKER_ID_PATTERN), + workerId: id(context.workerId, BRIDGE_WORKER_ID_PATTERN), + incarnationId: id(context.incarnationId, opaqueIdPattern), + operation: context.operation, + mutationMayHaveCommitted: context.mutationMayHaveCommitted, + requiresQuarantine: context.requiresQuarantine, + }; +} diff --git a/packages/code/src/credential-recovery.test.ts b/packages/code/src/credential-recovery.test.ts new file mode 100644 index 00000000..1cce473e --- /dev/null +++ b/packages/code/src/credential-recovery.test.ts @@ -0,0 +1,701 @@ +import assert from 'node:assert/strict'; +import { randomBytes } from 'node:crypto'; +import test from 'node:test'; + +import { recoverWorkerCredential } from './credential-recovery.js'; +import { + createBridgeIdentity, + verifyBridgeRecovery, + verifyBridgeRecoveryStart, + verifyBridgeRequest, +} from './identity.js'; +import { BridgeProtocolError } from './protocol.js'; +import { BridgeWorker } from './worker.js'; +import type { BridgeWorkerOptions } from './worker.js'; + +const workerId = 'recovery:vm'; +const codeApiUrl = 'https://code.example/proxy/v1'; +const incarnationId = 'worker-recovery-incarnation'; +const oldCredential = 'expired-short-lived-credential-value'; +const newCredential = 'recovered-short-lived-credential-value'; +const capabilities = { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], +}; + +function challenge() { + return { + protocolVersion: 1 as const, + operation: 'credential.recover' as const, + serverId: 'https://code.example', + workerId, + enrollmentGeneration: randomBytes(18).toString('base64url'), + challenge: randomBytes(32).toString('base64url'), + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }; +} + +function credential() { + return { + protocolVersion: 1 as const, + workerId, + credential: newCredential, + expiresAt: new Date(Date.now() + 900_000).toISOString(), + }; +} + +function identity() { + return { + ...createBridgeIdentity(), + credential: oldCredential, + expiresAt: new Date(Date.now() - 60_000).toISOString(), + }; +} + +function workerOptions( + overrides: Partial = {}, +): BridgeWorkerOptions { + return { + codeApiUrl, + workerId, + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity: identity(), + capabilities, + ...overrides, + }; +} + +function expired() { + return Response.json( + { + code: 'CREDENTIAL_INVALID', + error: 'Worker credential is invalid or expired', + }, + { status: 401 }, + ); +} + +test('startup recovers the enrolled key and persists before registering with the new credential', async () => { + const key = identity(); + const recovered = credential(); + const issued = challenge(); + const requests: string[] = []; + let persisted = false; + const controller = new AbortController(); + const worker = new BridgeWorker( + workerOptions({ + identity: key, + onIdentityChange: next => { + assert.equal(key.credential, oldCredential); + assert.equal(next.privateKey, key.privateKey); + assert.equal(next.credential, newCredential); + persisted = true; + }, + fetchImpl: async (input, init) => { + const url = String(input); + const body = JSON.parse(String(init?.body)); + requests.push(url); + if (url.endsWith('/refresh')) return expired(); + if (url.endsWith('/challenge') || url.endsWith('/recover')) { + assert.equal( + (init?.headers as Record).Authorization, + undefined, + ); + assert.equal(init?.redirect, 'error'); + assert.ok(!String(init?.body).includes(key.privateKey)); + assert.ok(!String(init?.body).includes(oldCredential)); + assert.match( + url, + /\/proxy\/v1\/bridge\/workers\/recovery%3Avm\/credentials\//, + ); + } + if (url.endsWith('/challenge')) { + assert.equal(body.serverId, 'https://code.example'); + assert.equal(body.operation, 'credential.challenge'); + assert.equal(body.workerId, workerId); + assert.match(body.nonce, /^[A-Za-z0-9_-]{43}$/); + assert.equal( + verifyBridgeRecoveryStart(key.publicKey, body, body.signature), + true, + ); + return Response.json(issued); + } + if (url.endsWith('/recover')) { + assert.equal( + verifyBridgeRecovery(key.publicKey, body, body.signature), + true, + ); + assert.deepEqual( + { ...body, signature: undefined }, + { ...issued, signature: undefined }, + ); + return Response.json(recovered); + } + assert.equal(persisted, true); + const headers = init!.headers as Record; + assert.equal(headers.Authorization, `Bridge ${newCredential}`); + assert.equal( + verifyBridgeRequest( + key.publicKey, + { + credential: newCredential, + method: 'POST', + path: new URL(url).pathname, + timestamp: headers['X-LibreChat-Code-Timestamp'], + nonce: headers['X-LibreChat-Code-Nonce'], + body: String(init!.body), + }, + headers['X-LibreChat-Code-Signature'], + ), + true, + ); + if (url.endsWith('/register')) + return Response.json({ + protocolVersion: 1, + workerId, + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }); + controller.abort(); + return Response.json({ protocolVersion: 1, serverElapsedMs: 0 }); + }, + }), + ); + await worker.run(controller.signal); + assert.equal(key.credential, newCredential); + assert.equal(requests.filter(url => url.endsWith('/challenge')).length, 1); + assert.equal(requests.filter(url => url.endsWith('/recover')).length, 1); + assert.ok( + !requests.some( + url => + url.includes('pairing') || + url.includes('reset') || + url.includes('revoke'), + ), + ); +}); + +test('ordinary successful refresh does not probe recovery and static workers do not recover', async () => { + let calls = 0; + const worker = new BridgeWorker( + workerOptions({ + fetchImpl: async input => { + assert.ok(String(input).endsWith('/refresh')); + calls++; + return Response.json(credential()); + }, + }), + ); + await worker.refreshCredential(); + assert.equal(calls, 1); + const staticWorker = new BridgeWorker( + workerOptions({ + identity: undefined, + token: 'legacy', + fetchImpl: async () => { + assert.fail('static worker has no credential refresh'); + }, + }), + ); + await staticWorker.refreshCredential(); +}); + +for (const [status, code] of [ + [401, undefined], + [401, 'PROOF_INVALID'], + [401, 'PROOF_REPLAYED'], + [403, 'CREDENTIAL_INVALID'], + [409, 'WORKER_FENCED'], + [429, 'RECOVERY_RATE_LIMITED'], + [503, undefined], +] as const) { + test(`refresh failure ${status}/${code} never triggers recovery`, async () => { + let calls = 0; + const worker = new BridgeWorker( + workerOptions({ + fetchImpl: async input => { + calls++; + assert.ok(String(input).endsWith('/refresh')); + return Response.json({ code }, { status }); + }, + }), + ); + await assert.rejects( + worker.refreshCredential(), + error => error instanceof BridgeProtocolError && error.status === status, + ); + assert.equal(calls, 1); + }); +} + +test('active workers never bootstrap recovery after registration', async () => { + const key = identity(); + let recoveries = 0; + const worker = new BridgeWorker( + workerOptions({ + identity: key, + fetchImpl: async input => { + const url = String(input); + if (url.endsWith('/register')) + return Response.json({ + protocolVersion: 1, + workerId, + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }); + if (!url.endsWith('/refresh')) recoveries++; + return expired(); + }, + }), + ); + await worker.register(); + await assert.rejects(worker.refreshCredential(), { + status: 401, + code: 'CREDENTIAL_INVALID', + }); + assert.equal(recoveries, 0); +}); + +for (const status of [404, 405, 401, 403, 429, 503]) { + test(`recovery failure ${status} preserves enrollment, key and old local credential`, async () => { + const key = identity(); + const original = { ...key }; + let persisted = false; + const worker = new BridgeWorker( + workerOptions({ + identity: key, + onIdentityChange: () => { + persisted = true; + }, + fetchImpl: async input => { + if (String(input).endsWith('/refresh')) return expired(); + assert.ok(String(input).endsWith('/challenge')); + return status === 404 + ? new Response('upstream secret', { status }) + : Response.json( + { code: status === 401 ? 'ENROLLMENT_INVALID' : undefined }, + { status }, + ); + }, + }), + ); + await assert.rejects(worker.refreshCredential(), error => { + assert.ok(error instanceof BridgeProtocolError); + assert.equal( + error.status, + status === 404 || status === 405 ? 401 : status, + ); + assert.ok(!error.message.includes('upstream secret')); + if (status === 401) + assert.match(error.message, /revoked.*pairing|pairing.*revoked/i); + if (status === 404) + assert.match( + error.message, + /enable durable enrollment or approve a new pairing/, + ); + return true; + }); + assert.deepEqual(key, original); + assert.equal(persisted, false); + }); +} + +for (const codeApiUrl of [ + 'http://code.example/v1', + 'https://user:password@code.example/v1', + 'https://code.example/v1?token=secret', + 'https://code.example/v1#fragment', +]) { + test('recovery refuses unsafe server context without disclosing identity', async () => { + await assert.rejects( + recoverWorkerCredential({ + codeApiUrl, + workerId, + privateKey: identity().privateKey, + validThroughMs: Date.now(), + signal: AbortSignal.timeout(1000), + fetchImpl: async () => { + assert.fail('must not fetch'); + }, + }), + { code: 'RECOVERY_UNAVAILABLE', status: 401 }, + ); + }); +} + +const badChallenges: Array< + [string, (value: ReturnType) => unknown] +> = [ + ['null', () => null], + ['array', () => []], + ['wrong version', value => ({ ...value, protocolVersion: 2 })], + [ + 'wrong operation', + value => ({ ...value, operation: 'credential.challenge' }), + ], + ['other server', value => ({ ...value, serverId: 'https://other.example' })], + ['other machine', value => ({ ...value, workerId: 'other-worker' })], + ['bad generation', value => ({ ...value, enrollmentGeneration: 'short' })], + ['bad challenge', value => ({ ...value, challenge: 'short' })], + [ + 'expired', + value => ({ ...value, expiresAt: new Date(Date.now() - 1).toISOString() }), + ], + [ + 'unbounded lifetime', + value => ({ ...value, expiresAt: '2099-01-01T00:00:00Z' }), + ], + ['invalid date', value => ({ ...value, expiresAt: 'not a date' })], +]; +for (const [name, alter] of badChallenges) { + test(`recovery refuses ${name} challenge before signing completion`, async () => { + let calls = 0; + await assert.rejects( + recoverWorkerCredential({ + codeApiUrl, + workerId, + privateKey: identity().privateKey, + validThroughMs: Date.now(), + signal: AbortSignal.timeout(1000), + fetchImpl: async () => { + calls++; + return Response.json(alter(challenge())); + }, + }), + /invalid machine recovery challenge/, + ); + assert.equal(calls, 1); + }); +} + +for (const value of [ + null, + [], + { ...credential(), workerId: 'another-worker' }, + { ...credential(), credential: 'short' }, + { ...credential(), credential: 'x'.repeat(4097) }, + { ...credential(), expiresAt: 'bad' }, + { ...credential(), expiresAt: new Date(0).toISOString() }, +]) { + test('invalid recovered credentials are never adopted or persisted', async () => { + const key = identity(); + const worker = new BridgeWorker( + workerOptions({ + identity: key, + onIdentityChange: () => { + assert.fail('must not persist'); + }, + fetchImpl: async input => { + if (String(input).endsWith('/refresh')) return expired(); + if (String(input).endsWith('/challenge')) + return Response.json(challenge()); + return Response.json(value); + }, + }), + ); + await assert.rejects( + worker.refreshCredential(), + /invalid recovered worker credential/, + ); + assert.equal(key.credential, oldCredential); + }); +} + +test('oversized and non-JSON recovery responses have bounded, sanitized failures', async () => { + for (const source of ['sensitive upstream contents', 'x'.repeat(8193)]) { + await assert.rejects( + recoverWorkerCredential({ + codeApiUrl, + workerId, + privateKey: identity().privateKey, + validThroughMs: Date.now(), + signal: AbortSignal.timeout(1000), + fetchImpl: async () => new Response(source), + }), + error => { + assert.ok(!String(error).includes('sensitive upstream contents')); + assert.match( + String(error), + /invalid machine recovery response|exceeds 8 KiB/, + ); + return true; + }, + ); + } +}); + +test('failed persistence never adopts a recovered credential; retry proves the same key with a new nonce', async () => { + const key = identity(); + const nonces = new Set(); + let attempts = 0; + const worker = new BridgeWorker( + workerOptions({ + identity: key, + onIdentityChange: () => { + if (++attempts === 1) throw new Error('disk unavailable'); + }, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/refresh')) return expired(); + if (String(input).endsWith('/challenge')) { + const body = JSON.parse(String(init?.body)); + assert.equal(nonces.has(body.nonce), false); + nonces.add(body.nonce); + assert.equal( + verifyBridgeRecoveryStart(key.publicKey, body, body.signature), + true, + ); + return Response.json(challenge()); + } + return Response.json(credential()); + }, + }), + ); + await assert.rejects(worker.refreshCredential(), /disk unavailable/); + assert.equal(key.credential, oldCredential); + await worker.refreshCredential(); + assert.equal(key.credential, newCredential); + assert.equal(attempts, 2); + assert.equal(nonces.size, 2); +}); + +test('concurrent callers share recovery and one cancellation cannot abort another waiter', async () => { + let release!: () => void; + let notify!: () => void; + const started = new Promise(resolve => { + notify = resolve; + }); + const ready = new Promise(resolve => { + release = resolve; + }); + const first = new AbortController(); + let challenges = 0; + let aborted = false; + const worker = new BridgeWorker( + workerOptions({ + fetchImpl: async (input, init) => { + if (String(input).endsWith('/refresh')) return expired(); + if (String(input).endsWith('/challenge')) { + challenges++; + return Response.json(challenge()); + } + init!.signal!.addEventListener('abort', () => { + aborted = true; + }); + notify(); + await ready; + return Response.json(credential()); + }, + }), + ); + const firstRefresh = worker.refreshCredential(first.signal); + await started; + const secondRefresh = worker.refreshCredential(); + first.abort(); + await assert.rejects(firstRefresh, { name: 'AbortError' }); + assert.equal(aborted, false); + release(); + await secondRefresh; + assert.equal(challenges, 1); + assert.equal(aborted, false); +}); + +test('last waiter cancellation aborts recovery without persisting or registering', async () => { + const controller = new AbortController(); + let notify!: () => void; + const started = new Promise(resolve => { + notify = resolve; + }); + let transportAborted = false; + const worker = new BridgeWorker( + workerOptions({ + onIdentityChange: () => { + assert.fail('must not persist'); + }, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/refresh')) return expired(); + assert.ok(String(input).endsWith('/challenge')); + notify(); + return new Promise((_resolve, reject) => + init!.signal!.addEventListener( + 'abort', + () => { + transportAborted = true; + reject(new DOMException('aborted', 'AbortError')); + }, + { once: true }, + ), + ); + }, + }), + ); + const pending = worker.refreshCredential(controller.signal); + await started; + controller.abort(); + await assert.rejects(pending, { name: 'AbortError' }); + assert.equal(transportAborted, true); +}); + +test('challenge and recovery share the remaining credential-refresh deadline', async () => { + let challengeSignal: AbortSignal | null | undefined; + const worker = new BridgeWorker( + workerOptions({ + credentialRefreshTransportTimeoutMs: 100, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/refresh')) return expired(); + if (String(input).endsWith('/challenge')) { + challengeSignal = init?.signal; + return Response.json(challenge()); + } + assert.equal(init?.signal, challengeSignal); + return new Promise((_resolve, reject) => + init!.signal!.addEventListener( + 'abort', + () => reject(new DOMException('aborted', 'AbortError')), + { once: true }, + ), + ); + }, + }), + ); + // Keep the event loop alive because AbortSignal.timeout timers are unref'ed. + const timer = setTimeout(() => {}, 2000); + try { + await assert.rejects(worker.refreshCredential(), { name: 'AbortError' }); + } finally { + clearTimeout(timer); + } + assert.equal(challengeSignal?.aborted, true); +}); + +test('revoked enrollment terminates startup instead of retrying or issuing a pairing', async () => { + let calls = 0; + const worker = new BridgeWorker( + workerOptions({ + onError: () => { + assert.fail('terminal auth failure must not reconnect'); + }, + fetchImpl: async input => { + calls++; + if (String(input).endsWith('/refresh')) return expired(); + assert.ok(String(input).endsWith('/challenge')); + return Response.json({ code: 'ENROLLMENT_INVALID' }, { status: 401 }); + }, + }), + ); + await assert.rejects(worker.run(), { + status: 401, + code: 'ENROLLMENT_INVALID', + }); + assert.equal(calls, 2); +}); + +test('recovery retries transient startup failure without replacing the enrolled key', async () => { + let challenges = 0; + let failures = 0; + const key = identity(); + const controller = new AbortController(); + const worker = new BridgeWorker( + workerOptions({ + identity: key, + reconnectDelayMs: 1, + reconnectMaxDelayMs: 1, + onError: error => { + assert.ok(error instanceof BridgeProtocolError); + assert.equal(error.status, 503); + failures++; + }, + fetchImpl: async (input, init) => { + const url = String(input); + if (url.endsWith('/refresh')) return expired(); + if (url.endsWith('/challenge')) { + const body = JSON.parse(String(init?.body)); + assert.equal( + verifyBridgeRecoveryStart(key.publicKey, body, body.signature), + true, + ); + return ++challenges === 1 + ? Response.json({}, { status: 503 }) + : Response.json(challenge()); + } + if (url.endsWith('/recover')) return Response.json(credential()); + if (url.endsWith('/register')) + return Response.json({ + protocolVersion: 1, + workerId, + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }); + controller.abort(); + return Response.json({ protocolVersion: 1, serverElapsedMs: 0 }); + }, + }), + ); + await worker.run(controller.signal); + assert.equal(challenges, 2); + assert.equal(failures, 1); + assert.equal(key.credential, newCredential); +}); + +test('revocation between challenge and completion cannot restore access', async () => { + let persisted = false; + const worker = new BridgeWorker( + workerOptions({ + onIdentityChange: () => { + persisted = true; + }, + fetchImpl: async input => { + if (String(input).endsWith('/refresh')) return expired(); + if (String(input).endsWith('/challenge')) + return Response.json(challenge()); + assert.ok(String(input).endsWith('/recover')); + return Response.json({ code: 'ENROLLMENT_INVALID' }, { status: 401 }); + }, + }), + ); + await assert.rejects(worker.run(), { + code: 'ENROLLMENT_INVALID', + status: 401, + }); + assert.equal(persisted, false); +}); + +test('successful credential recovery never clears an existing workspace quarantine', async () => { + let checks = 0; + const worker = new BridgeWorker( + workerOptions({ + workspaceMutationQuarantine: { + assertAvailable: async () => { + checks++; + throw new BridgeProtocolError( + 'Workspace requires inspection', + 409, + 'WORKER_QUARANTINED', + ); + }, + arm: async () => { + assert.fail('must not arm'); + }, + clear: async () => { + assert.fail('must not clear'); + }, + quarantine: async () => { + assert.fail('must not change quarantine'); + }, + }, + fetchImpl: async input => { + if (String(input).endsWith('/refresh')) return expired(); + if (String(input).endsWith('/challenge')) + return Response.json(challenge()); + assert.ok(String(input).endsWith('/recover')); + return Response.json(credential()); + }, + }), + ); + await assert.rejects(worker.run(), { code: 'WORKER_QUARANTINED' }); + assert.equal(checks, 1); +}); diff --git a/packages/code/src/credential-recovery.ts b/packages/code/src/credential-recovery.ts new file mode 100644 index 00000000..01aae3fd --- /dev/null +++ b/packages/code/src/credential-recovery.ts @@ -0,0 +1,223 @@ +import { randomBytes } from 'node:crypto'; + +import { signBridgeRecovery, signBridgeRecoveryStart } from './identity.js'; +import { + BRIDGE_PROTOCOL_VERSION, + BridgeProtocolError, + bridgeWorkerPath, +} from './protocol.js'; +import type { + BridgeRecoveryChallengeResponse, + BridgeWorkerCredentialResponse, +} from './protocol.js'; + +interface RecoverWorkerCredentialOptions { + codeApiUrl: string; + workerId: string; + privateKey: string; + validThroughMs: number; + fetchImpl: typeof fetch; + signal: AbortSignal; +} + +const MAX_RESPONSE_BYTES = 8192; +const MAX_CHALLENGE_LIFETIME_MS = 6 * 60_000; + +async function responsePayload(response: Response): Promise { + const reader = response.body?.getReader(); + if (!reader) + throw new BridgeProtocolError( + 'Code API returned an empty machine recovery response', + 502, + ); + const chunks: Uint8Array[] = []; + let size = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > MAX_RESPONSE_BYTES) { + throw new BridgeProtocolError( + 'Code API machine recovery response exceeds 8 KiB', + 502, + ); + } + chunks.push(value); + } + try { + return JSON.parse( + new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)), + ) as unknown; + } catch { + throw new BridgeProtocolError( + 'Code API returned an invalid machine recovery response', + 502, + ); + } + } finally { + await reader.cancel().catch(() => undefined); + reader.releaseLock(); + } +} + +function record(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +async function recoveryRequest( + url: string, + body: object, + options: RecoverWorkerCredentialOptions, +): Promise { + options.signal.throwIfAborted(); + const response = await options.fetchImpl(url, { + method: 'POST', + redirect: 'error', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + signal: options.signal, + }); + if (!response.ok) { + // Older/disabled servers may return HTML. Do not echo untrusted response text. + let payload: unknown; + try { + payload = await responsePayload(response); + } catch { + /* Status is sufficient. */ + } + options.signal.throwIfAborted(); + if (response.status === 404 || response.status === 405) { + throw new BridgeProtocolError( + 'Machine credential expired or is unavailable. This Code API does not support enabled machine recovery; enable durable enrollment or approve a new pairing in LibreChat.', + 401, + 'RECOVERY_UNAVAILABLE', + ); + } + const code = + record(payload) && + typeof payload.code === 'string' && + [ + 'ENROLLMENT_INVALID', + 'PROOF_INVALID', + 'PROOF_REPLAYED', + 'CHALLENGE_INVALID', + 'RECOVERY_RATE_LIMITED', + ].includes(payload.code) + ? payload.code + : undefined; + const retryable = + response.status === 408 || + response.status === 429 || + response.status >= 500; + throw new BridgeProtocolError( + code === 'ENROLLMENT_INVALID' + ? 'Machine enrollment is unavailable or revoked. Approve a new pairing in LibreChat; recovery cannot restore revoked access.' + : `Machine credential recovery failed with HTTP ${response.status}${retryable ? '; reconnect will retry' : '; check enrollment and approve a new pairing in LibreChat if needed'}`, + retryable ? response.status : response.status === 403 ? 403 : 401, + code, + ); + } + const payload = await responsePayload(response); + options.signal.throwIfAborted(); + return payload; +} + +/** Recover only existing enrollment. Never pair, replace a key, or reset workspace state. */ +export async function recoverWorkerCredential( + options: RecoverWorkerCredentialOptions, +): Promise { + options.signal.throwIfAborted(); + const server = new URL(options.codeApiUrl); + if ( + server.protocol !== 'https:' || + server.username || + server.password || + server.search || + server.hash || + server.origin.length > 256 + ) { + throw new BridgeProtocolError( + 'Automatic machine recovery requires the enrolled Code API HTTPS origin; use the existing pairing flow for this endpoint.', + 401, + 'RECOVERY_UNAVAILABLE', + ); + } + const url = `${options.codeApiUrl.replace(/\/+$/, '')}${bridgeWorkerPath(options.workerId)}/credentials`; + const start = { + operation: 'credential.challenge' as const, + serverId: server.origin, + workerId: options.workerId, + timestamp: new Date().toISOString(), + nonce: randomBytes(32).toString('base64url'), + }; + const payload = await recoveryRequest( + `${url}/challenge`, + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + ...start, + signature: signBridgeRecoveryStart(options.privateKey, start), + }, + options, + ); + if ( + !record(payload) || + payload.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + payload.operation !== 'credential.recover' || + payload.serverId !== server.origin || + payload.workerId !== options.workerId || + typeof payload.enrollmentGeneration !== 'string' || + !/^[A-Za-z0-9_-]{24}$/.test(payload.enrollmentGeneration) || + typeof payload.challenge !== 'string' || + !/^[A-Za-z0-9_-]{43}$/.test(payload.challenge) || + typeof payload.expiresAt !== 'string' || + !Number.isFinite(Date.parse(payload.expiresAt)) || + Date.parse(payload.expiresAt) <= Date.now() || + Date.parse(payload.expiresAt) > Date.now() + MAX_CHALLENGE_LIFETIME_MS + ) { + throw new BridgeProtocolError( + 'Code API returned an invalid machine recovery challenge', + 502, + ); + } + // Sign only the validated recovery operation, never arbitrary server-supplied fields. + const challenge: BridgeRecoveryChallengeResponse = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'credential.recover', + serverId: server.origin, + workerId: options.workerId, + enrollmentGeneration: payload.enrollmentGeneration, + challenge: payload.challenge, + expiresAt: payload.expiresAt, + }; + const credential = await recoveryRequest( + `${url}/recover`, + { + ...challenge, + signature: signBridgeRecovery(options.privateKey, challenge), + }, + options, + ); + if ( + !record(credential) || + credential.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + credential.workerId !== options.workerId || + typeof credential.credential !== 'string' || + credential.credential.length < 32 || + credential.credential.length > 4096 || + typeof credential.expiresAt !== 'string' || + !Number.isFinite(Date.parse(credential.expiresAt)) || + Date.parse(credential.expiresAt) <= options.validThroughMs + ) { + throw new BridgeProtocolError( + 'Code API returned an invalid recovered worker credential', + 502, + ); + } + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: options.workerId, + credential: credential.credential, + expiresAt: credential.expiresAt, + }; +} diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index 6f94b190..d8c53f41 100644 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -1,5 +1,6 @@ export * from './protocol.js'; export * from './identity.js'; +export * from './browser-enrollment-protocol.js'; export * from './pairing.js'; export * from './storage.js'; export * from './runtime.js'; diff --git a/packages/code/src/linked-worktrees.ts b/packages/code/src/linked-worktrees.ts index 225abebd..609bfbd8 100644 --- a/packages/code/src/linked-worktrees.ts +++ b/packages/code/src/linked-worktrees.ts @@ -18,7 +18,7 @@ import type { WorkspaceToolRequest, WorkspaceToolResult, } from './protocol.js'; -import type { WorkspaceToolExecutor } from './workspace.js'; +import type { WorkspaceFileImporter, WorkspaceToolExecutor } from './workspace.js'; /** Linked worktrees are only admitted from this directory beneath a checkout. */ export const LINKED_WORKTREE_DIRECTORY = '.worktrees'; @@ -60,6 +60,7 @@ export interface LinkedWorktreeSource { command?: NativeProcessSandboxOptions; repositoryInstructions: boolean; writable: boolean; + importer?: WorkspaceFileImporter; } export interface VerifiedLinkedWorktree { @@ -324,6 +325,7 @@ export class LinkedWorktreeWorkspaceTools implements WorkspaceToolExecutor, Link fingerprint, value: LocalWorkspaceTools.create({ repositoryInstructions: source.repositoryInstructions, + importer: source.importer, workspaces: [ { id: internalId, identity: lane.identity, root: lane.root, writable: source.writable }, ], diff --git a/packages/code/src/native-pool.test.ts b/packages/code/src/native-pool.test.ts index ce25caa4..2a339d59 100644 --- a/packages/code/src/native-pool.test.ts +++ b/packages/code/src/native-pool.test.ts @@ -14,6 +14,37 @@ const request = (workspaceId: string): WorkspaceExecuteCommandRequest => ({ command: 'fixture', }); +test('pool distinguishes same-root contention from exhausted machine capacity', async () => { + let release!: () => void; + let entered!: () => void; + const running = new Promise(resolve => { entered = resolve; }); + const finish = new Promise(resolve => { release = resolve; }); + const pool = new NativeWorkspaceCommandPool(roots, 1, () => ({ + async prepare() {}, async close() {}, + async execute(req) { + entered(); + await finish; + return { ...req, stdout: '', stderr: '', exitCode: 0, timedOut: false, truncated: false }; + }, + })); + const first = pool.execute(request('a')); + await running; + try { + for (const [root, reason] of [['a', 'workspace_busy'], ['b', 'capacity_exhausted']]) { + await assert.rejects(pool.execute(request(root!)), error => { + assert.ok(error instanceof WorkspaceToolError); + assert.deepEqual(error.commandDiagnostic, { stage: 'allocation', reason }); + assert.equal(error.mutationMayHaveCommitted, false); + return true; + }); + } + } finally { + release(); + await first; + await pool.close(); + } +}); + test('failed provisioning roots are removed only after confirmed executor cleanup', async () => { let failClose = true; const pool = new NativeWorkspaceCommandPool(roots, 2, () => ({ diff --git a/packages/code/src/native-pool.ts b/packages/code/src/native-pool.ts index 59db3e0a..60ba4b43 100644 --- a/packages/code/src/native-pool.ts +++ b/packages/code/src/native-pool.ts @@ -1,5 +1,6 @@ import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { WorkspaceToolError } from './workspace.js'; +import { commandFailureDiagnostic } from './command-diagnostics.js'; import type { NativeProcessSandboxOptions } from './native-process.js'; import type { BridgeWorkspaceProgrammaticRequest, @@ -122,7 +123,8 @@ export class NativeWorkspaceCommandPool { if (entry?.busy) throw new WorkspaceToolError( 'Native workspace already executing', - 'COMMAND_UNAVAILABLE' + 'COMMAND_UNAVAILABLE', false, false, + commandFailureDiagnostic('allocation', 'workspace_busy'), ); if (!entry) { if (this.entries.size >= this.capacity) { @@ -132,9 +134,18 @@ export class NativeWorkspaceCommandPool { if (!idle) throw new WorkspaceToolError( 'Native executor capacity reached', - 'COMMAND_UNAVAILABLE' + 'COMMAND_UNAVAILABLE', false, false, + commandFailureDiagnostic('allocation', 'capacity_exhausted'), ); - await idle[1].sandbox.close(); + try { + await idle[1].sandbox.close(); + } catch (error) { + const diagnostic = commandFailureDiagnostic('cleanup', 'cleanup_failed', error); + throw error instanceof WorkspaceToolError + ? new WorkspaceToolError(error.message, error.code, error.mutationMayHaveCommitted, + error.requiresQuarantine, error.commandDiagnostic ?? diagnostic) + : new WorkspaceToolError('Native executor allocation failed', 'COMMAND_UNAVAILABLE', false, false, diagnostic); + } this.entries.delete(idle[0]); } entry = { @@ -151,10 +162,15 @@ export class NativeWorkspaceCommandPool { const checked = pending.catch((error) => { // Allocation/idle eviction precedes dispatch into the requested root. // Do not turn a pool resource failure into an uncertain mutation there. - if (error instanceof WorkspaceToolError) throw error; + if (error instanceof WorkspaceToolError) { + if (error.commandDiagnostic) throw error; + throw new WorkspaceToolError(error.message, error.code, error.mutationMayHaveCommitted, + error.requiresQuarantine, commandFailureDiagnostic('allocation', 'allocation_failed', error)); + } throw new WorkspaceToolError( 'Native executor allocation failed', - 'COMMAND_UNAVAILABLE' + 'COMMAND_UNAVAILABLE', false, false, + commandFailureDiagnostic('allocation', 'allocation_failed', error), ); }); this.allocation = checked.catch(() => undefined); @@ -236,7 +252,8 @@ export class NativeWorkspaceCommandPool { if (!entry.sandbox.executeProgrammatic) { throw new WorkspaceToolError( 'Native programmatic executor is unavailable', - 'COMMAND_UNAVAILABLE' + 'COMMAND_UNAVAILABLE', false, false, + commandFailureDiagnostic('programmatic_preflight', 'prerequisite_failed'), ); } return await entry.sandbox.executeProgrammatic( diff --git a/packages/code/src/native-process-child.ts b/packages/code/src/native-process-child.ts index c3b9614b..e0e0d322 100644 --- a/packages/code/src/native-process-child.ts +++ b/packages/code/src/native-process-child.ts @@ -1,6 +1,7 @@ import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; import { NativeWorkspaceProgrammaticExecutor } from './native-programmatic.js'; import { WorkspaceToolError } from './workspace.js'; +import { commandFailureDiagnostic } from './command-diagnostics.js'; import type { NativeSrtWorkspaceCommandSandboxOptions } from './native-sandbox.js'; import type { BridgeWorkspaceProgrammaticRequest, @@ -189,6 +190,14 @@ process.on('message', async (raw: unknown) => { reply({ id: message.id, ok: false, + diagnostic: + error instanceof WorkspaceToolError && error.commandDiagnostic + ? error.commandDiagnostic + : commandFailureDiagnostic( + message.type === 'prepare' ? 'sandbox_startup' : message.type === 'close' ? 'cleanup' : 'dispatch', + message.type === 'prepare' ? 'startup_failed' : message.type === 'close' ? 'cleanup_failed' : 'executor_rejected', + error, + ), code: error instanceof WorkspaceToolError ? error.code diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index 8abf263c..afdad6f8 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -484,6 +484,7 @@ test('programmatic staging watchdog expires without claiming a workspace mutatio execution, (error: unknown) => error instanceof WorkspaceToolError && + error.commandDiagnostic?.reason === 'rpc_timeout' && !error.mutationMayHaveCommitted && !error.requiresQuarantine, ); @@ -876,3 +877,92 @@ test('executor classifies every pre-dispatch setup failure as mutation-atomic', await sandbox.close(); } }); + +test('executor diagnoses setup stages without dispatching or retaining private exception values', async () => { + for (const failure of ['fork', 'credential', 'wrapper'] as const) { + const fake = fixture(); + const privateError = Object.assign(new Error('secret /private/path command'), { code: 'ENOSPC' }); + const sandbox = new NativeProcessWorkspaceCommandSandbox({ + workspaceRoot: '/workspace', + maskedEnvironment: { + variables: [], + async resolve() { + if (failure === 'credential') throw privateError; + return {}; + }, + wrapCommand(command) { + if (failure === 'wrapper') throw privateError; + return command; + }, + }, + }, failure === 'fork' ? () => { throw privateError; } : fake.fork); + await assert.rejects(sandbox.execute(request), error => { + assert.ok(error instanceof WorkspaceToolError); + assert.equal(error.code, 'COMMAND_UNAVAILABLE'); + assert.equal(error.mutationMayHaveCommitted, false); + assert.equal(error.requiresQuarantine, false); + assert.deepEqual(error.commandDiagnostic, { + stage: failure === 'fork' ? 'sandbox_startup' : failure === 'credential' ? 'credential_resolution' : 'command_wrapping', + reason: failure === 'fork' ? 'startup_failed' : failure === 'credential' ? 'credential_failed' : 'wrapping_failed', + osCode: 'ENOSPC', + }); + assert.equal(JSON.stringify(error.commandDiagnostic).includes('secret'), false); + return true; + }); + assert.equal(fake.messages.some(m => m.type === 'execute'), false); + await sandbox.close(); + } +}); + +test('executor preserves sanitized child diagnostics and mutation uncertainty', async () => { + const fake = fixture((child, message) => child.emit('message', { + id: message.id, ok: false, mutation: true, + diagnostic: { stage: 'dispatch', reason: 'executor_rejected', osCode: 'EIO', secret: 'token' }, + })); + const sandbox = new NativeProcessWorkspaceCommandSandbox({ workspaceRoot: '/workspace' }, fake.fork); + await assert.rejects(sandbox.execute(request), error => { + assert.ok(error instanceof WorkspaceToolError); + assert.deepEqual(error.commandDiagnostic, { stage: 'dispatch', reason: 'executor_rejected', osCode: 'EIO' }); + assert.equal(error.mutationMayHaveCommitted, true); + assert.equal(error.requiresQuarantine, true); + return true; + }); + await sandbox.close(); +}); + +test('programmatic startup diagnoses child preparation failures before mutation', async () => { + const fake = fixture(undefined, (child, message) => child.emit('message', { + id: message.id, ok: false, + diagnostic: { stage: 'sandbox_startup', reason: 'startup_failed', osCode: 'ENOSPC' }, + })); + const sandbox = new NativeProcessWorkspaceCommandSandbox({ + workspaceRoot: '/workspace', programmaticFileUpstream: 'http://127.0.0.1:3190', + }, fake.fork); + await assert.rejects(sandbox.executeProgrammatic('primary', { + body: { language: 'bash', version: '5.2', session_id: 'session-1', files: [{ name: 'main.sh', content: 'echo ok' }] }, headers: {}, + }), error => { + assert.ok(error instanceof WorkspaceToolError); + assert.deepEqual(error.commandDiagnostic, { stage: 'sandbox_startup', reason: 'startup_failed', osCode: 'ENOSPC' }); + assert.equal(error.mutationMayHaveCommitted, false); + return true; + }); + assert.equal(fake.messages.some(m => m.type === 'programmatic'), false); + await sandbox.close(); +}); + +test('real executor IPC carries a startup errno without including the workspace path in diagnostics', async t => { + const directory = await mkdtemp(join(tmpdir(), 'command-diagnostic-')); + const sandbox = new NativeProcessWorkspaceCommandSandbox({ workspaceRoot: join(directory, 'missing-private-root') }); + t.after(async () => { + await sandbox.close(); + await rm(directory, { recursive: true, force: true }); + }); + await assert.rejects(sandbox.execute(request), error => { + assert.ok(error instanceof WorkspaceToolError); + assert.equal(error.code, 'COMMAND_UNAVAILABLE'); + assert.equal(error.mutationMayHaveCommitted, false); + assert.deepEqual(error.commandDiagnostic, { stage: 'sandbox_startup', reason: 'startup_failed', osCode: 'ENOENT' }); + assert.equal(JSON.stringify(error.commandDiagnostic).includes(directory), false); + return true; + }); +}); diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index 588f4c33..acb1c043 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -5,6 +5,12 @@ import { access, realpath } from 'node:fs/promises'; import { isAbsolute, join, relative, resolve, sep } from 'node:path'; import { promisify } from 'node:util'; import { WorkspaceToolError } from './workspace.js'; +import { + commandFailureDiagnostic, + commandSetupFailureDiagnostic, + readCommandFailureDiagnostic, +} from './command-diagnostics.js'; +import type { CommandFailureReason, CommandSetupStage } from './command-diagnostics.js'; import { NATIVE_PROGRAMMATIC_COMMAND } from './native-programmatic.js'; import { BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, @@ -171,11 +177,13 @@ export function nativeExecutorEnvironment( /** The executor process was lost, refused a send, or stalled past its * deadline, as opposed to a failure the executor reported explicitly. */ class NativeExecutorUnavailableError extends WorkspaceToolError { - constructor(mutation: boolean) { + constructor(mutation: boolean, reason: CommandFailureReason = 'executor_unavailable') { super( 'Native executor is unavailable', 'COMMAND_UNAVAILABLE', mutation, + mutation, + commandFailureDiagnostic('executor_transport', reason), ); this.name = 'NativeExecutorUnavailableError'; } @@ -227,8 +235,8 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan return this.ready; } - private unavailable(mutation: boolean): WorkspaceToolError { - return new NativeExecutorUnavailableError(mutation); + private unavailable(mutation: boolean, reason?: CommandFailureReason): WorkspaceToolError { + return new NativeExecutorUnavailableError(mutation, reason); } private async resolveProgrammaticExecutables(): Promise<{ @@ -271,6 +279,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan errorMessage?: unknown; fatal?: unknown; phase?: unknown; + diagnostic?: unknown; }; if ( !message || @@ -323,13 +332,14 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan code, mutationMayHaveCommitted, mutationMayHaveCommitted && !processTerminationConfirmed, + readCommandFailureDiagnostic(message.diagnostic) ?? commandFailureDiagnostic('dispatch', 'executor_rejected'), ), ); } }); const lost = () => { this.failed = true; - this.pending?.reject(this.unavailable(this.pending.mutation)); + this.pending?.reject(this.unavailable(this.pending.mutation, 'executor_lost')); }; child.on('error', lost); child.on('exit', lost); @@ -407,6 +417,8 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan throw new WorkspaceToolError( 'Native programmatic file transport is unavailable', 'COMMAND_UNAVAILABLE', + false, false, + commandFailureDiagnostic('programmatic_preflight', 'prerequisite_failed'), ); } const active = this.executeProgrammaticOnce( @@ -435,16 +447,20 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan let credentials: Record | undefined; let wrappedCommand: string | undefined; let programmaticExecutables: { shellPath: string; jqPath: string }; + let setupStage: CommandSetupStage = 'sandbox_startup'; try { await this.prepare(); if (signal?.aborted) throw new Error('aborted'); + setupStage = 'programmatic_preflight'; programmaticExecutables = await this.resolveProgrammaticExecutables(); if (signal?.aborted) throw new Error('aborted'); + setupStage = 'credential_resolution'; credentials = await this.options.maskedEnvironment?.resolve( signal, this.options.workspaceRoot, ); if (signal?.aborted) throw new Error('aborted'); + setupStage = 'command_wrapping'; wrappedCommand = this.options.maskedEnvironment?.wrapCommand?.( NATIVE_PROGRAMMATIC_COMMAND, process.platform, @@ -458,11 +474,13 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan 'EXECUTION_ABORTED', ); } + const diagnostic = commandSetupFailureDiagnostic(setupStage, error); throw error instanceof WorkspaceToolError - ? new WorkspaceToolError(error.message, error.code, false) + ? new WorkspaceToolError(error.message, error.code, false, false, error.commandDiagnostic ?? diagnostic) : new WorkspaceToolError( 'Native programmatic executor setup failed before dispatch', 'COMMAND_UNAVAILABLE', + false, false, diagnostic, ); } const result = await this.rpc( @@ -534,12 +552,15 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan ); let credentials: Record | undefined; let wrappedCommand: string | undefined; + let setupStage: CommandSetupStage = 'sandbox_startup'; try { await this.prepare(); if (signal?.aborted) throw new Error('aborted'); const cwd = resolve(this.options.workspaceRoot, request.cwd ?? '.'); + setupStage = 'credential_resolution'; credentials = await this.options.maskedEnvironment?.resolve(signal, cwd); if (signal?.aborted) throw new Error('aborted'); + setupStage = 'command_wrapping'; wrappedCommand = this.options.maskedEnvironment?.wrapCommand?.( request.command, process.platform, @@ -554,11 +575,13 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan 'Command aborted', 'EXECUTION_ABORTED', ); + const diagnostic = commandSetupFailureDiagnostic(setupStage, error); throw error instanceof WorkspaceToolError - ? new WorkspaceToolError(error.message, error.code, false) + ? new WorkspaceToolError(error.message, error.code, false, false, error.commandDiagnostic ?? diagnostic) : new WorkspaceToolError( 'Native executor setup failed before dispatch', 'COMMAND_UNAVAILABLE', + false, false, diagnostic, ); } const result = await this.rpc( @@ -610,8 +633,8 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan if (timer) clearTimeout(timer); timer = this.scheduleRpcTimeout(() => { this.failed = true; + reject(this.unavailable(this.pending?.mutation ?? mutation, 'rpc_timeout')); this.terminate(); - reject(this.unavailable(this.pending?.mutation ?? mutation)); }, timeoutMs); }; this.pending = { @@ -627,8 +650,8 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan signal?.addEventListener('abort', abort, { once: true }); const sendFailed = () => { this.failed = true; + reject(this.unavailable(mutation, 'send_failed')); this.terminate(); - reject(this.unavailable(mutation)); }; try { child.send({ type, id, ...payload }, error => { diff --git a/packages/code/src/native-programmatic.test.ts b/packages/code/src/native-programmatic.test.ts index 902c707a..076f141c 100644 --- a/packages/code/src/native-programmatic.test.ts +++ b/packages/code/src/native-programmatic.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { createServer } from 'node:http'; -import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { chmod, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import test from 'node:test'; @@ -389,6 +389,112 @@ test('preflights copy-on-write isolation and removes its private snapshot', asyn } }); +test('programmatic prerequisite failures retain their stage before executor dispatch', async t => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-diagnostic-')); + t.after(() => rm(scratch, { recursive: true, force: true })); + for (const failure of ['missing_probe', 'private_storage', 'isolation_probe'] as const) { + let executions = 0; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { + if (failure === 'private_storage') throw Object.assign(new Error('secret path'), { code: 'ENOSPC' }); + return await mkdtemp(join(scratch, 'execution-')); + }, + ...(failure === 'missing_probe' ? {} : { + async createProgrammaticProbeWorkspace() { + throw new WorkspaceToolError('Probe isolation unavailable', 'COMMAND_UNAVAILABLE'); + }, + }), + async executeProgrammatic() { executions += 1; throw new Error('must not run'); }, + }, + }); + await assert.rejects(executor.prepare(), error => { + assert.ok(error instanceof WorkspaceToolError); + assert.equal(error.code, 'COMMAND_UNAVAILABLE'); + assert.equal(error.mutationMayHaveCommitted, false); + assert.equal(error.requiresQuarantine, false); + assert.deepEqual(error.commandDiagnostic, { + stage: 'programmatic_preflight', reason: 'prerequisite_failed', + ...(failure === 'private_storage' ? { osCode: 'ENOSPC' } : {}), + }); + assert.equal(JSON.stringify(error.commandDiagnostic).includes('secret'), false); + return true; + }); + assert.equal(executions, 0); + assert.deepEqual(await readdir(scratch), []); + } +}); + +for (const phase of ['preflight', 'probe', 'commit'] as const) { + test(`programmatic ${phase} cleanup failures retain cleanup diagnostics and mutation flags`, { + skip: process.platform === 'win32' || process.getuid?.() === 0, + }, async t => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-cleanup-diagnostic-')); + t.after(async () => { + await chmod(scratch, 0o700); + await rm(scratch, { recursive: true, force: true }); + }); + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { return await mkdtemp(join(scratch, 'execution-')); }, + async createProgrammaticProbeWorkspace(directory) { + const workspace = join(directory, 'workspace'); + await mkdir(workspace); + if (phase === 'preflight') await chmod(scratch, 0o500); + return workspace; + }, + async executeProgrammatic(request) { + await chmod(scratch, 0o500); + if (phase === 'probe') throw new WorkspaceToolError('Probe failed', 'COMMAND_UNAVAILABLE'); + return { protocolVersion: 1, operation: 'execute_command', workspaceId: request.workspaceId, + stdout: '', stderr: '', exitCode: 0, timedOut: false, truncated: false }; + }, + }, + }); + const execution = phase === 'preflight' ? executor.prepare() : executor.execute({ + headers: {}, body: { language: 'bash', version: '5.2.0', session_id: 'session', + replay_tool_count: phase === 'probe' ? 1 : 0, files: [{ name: 'main.sh', content: 'true' }] }, + }, 'primary'); + await assert.rejects(execution, error => { + assert.ok(error instanceof WorkspaceToolError); + assert.equal(error.code, 'COMMAND_UNAVAILABLE'); + assert.equal(error.commandDiagnostic?.stage, 'cleanup'); + assert.equal(error.commandDiagnostic?.reason, 'cleanup_failed'); + assert.ok(['EACCES', 'EPERM'].includes(error.commandDiagnostic?.osCode ?? '')); + assert.equal(error.mutationMayHaveCommitted, phase === 'commit'); + assert.equal(error.requiresQuarantine, phase === 'commit'); + return true; + }); + }); +} + +test('programmatic commit normalization preserves the executor diagnostic', async t => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-commit-diagnostic-')); + t.after(() => rm(scratch, { recursive: true, force: true })); + const diagnostic = { stage: 'executor_transport' as const, reason: 'executor_lost' as const }; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { return await mkdtemp(join(scratch, 'execution-')); }, + async executeProgrammatic() { + throw new WorkspaceToolError('Executor unavailable', 'COMMAND_UNAVAILABLE', false, false, diagnostic); + }, + }, + }); + await assert.rejects(executor.execute({ + headers: {}, body: { language: 'bash', version: '5.2.0', session_id: 'session', + files: [{ name: 'main.sh', content: 'true' }] }, + }, 'primary'), error => { + assert.ok(error instanceof WorkspaceToolError); + assert.equal(error.mutationMayHaveCommitted, true); + assert.equal(error.requiresQuarantine, true); + assert.deepEqual(error.commandDiagnostic, diagnostic); + return true; + }); +}); + test('keeps replay probes read-only and commits the script exactly once', async () => { const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-probe-test-')); const phases: boolean[] = []; @@ -622,7 +728,8 @@ for (const failure of ['truncated', 'process-error']) test(`a failed speculative async createProgrammaticProbeWorkspace(directory) { const root = join(directory, 'workspace'); await mkdir(root); return root; }, async executeProgrammatic(request, _directory, _signal, options) { assert.equal(options?.probe, true); - if (failure === 'process-error') throw new WorkspaceToolError('probe output exceeded its limit', 'COMMAND_UNAVAILABLE', true, true); + if (failure === 'process-error') throw new WorkspaceToolError('probe output exceeded its limit', 'COMMAND_UNAVAILABLE', true, true, + { stage: 'executor_transport', reason: 'executor_lost' }); return { protocolVersion: 1, operation: 'execute_command', workspaceId: request.workspaceId, exitCode: 0, stdout: '', stderr: '', truncated: true, timedOut: false }; }, @@ -635,6 +742,8 @@ for (const failure of ['truncated', 'process-error']) test(`a failed speculative assert.match(String(error), /probe output exceeded/); assert.equal((error as { requiresQuarantine: boolean }).requiresQuarantine, false); assert.equal((error as { mutationMayHaveCommitted: boolean }).mutationMayHaveCommitted, false); + if (failure === 'process-error') assert.deepEqual((error as WorkspaceToolError).commandDiagnostic, + { stage: 'executor_transport', reason: 'executor_lost' }); return true; }); }); diff --git a/packages/code/src/native-programmatic.ts b/packages/code/src/native-programmatic.ts index 9b6eea9e..eca7316d 100644 --- a/packages/code/src/native-programmatic.ts +++ b/packages/code/src/native-programmatic.ts @@ -15,8 +15,13 @@ import { isSafePortableRelativePath, isSupportedBridgeArtifactName, } from './protocol.js'; -import { validateFileRelayUpstream } from './relay.js'; +import { + FILE_RELAY_GRANT_HEADER, + fileRelayObjectUrl, + validateFileRelayUpstream, +} from './relay.js'; import { WorkspaceToolError } from './workspace.js'; +import { commandFailureDiagnostic, commandSetupFailureDiagnostic } from './command-diagnostics.js'; import type { BridgeProgrammaticPayloadFile, @@ -25,7 +30,6 @@ import type { } from './protocol.js'; import type { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; -const EGRESS_GRANT_HEADER = 'X-CodeAPI-Egress-Grant'; const EXECUTION_MAIN_FILE = 'main.sh'; const EXECUTION_HISTORY_FILE = '_ptc_history.json'; const EXECUTION_CONTROL_FILE = '_ptc_pending_result.json'; @@ -248,6 +252,25 @@ export class NativeWorkspaceProgrammaticExecutor { * real replay will use, then removes the snapshot before registration. */ async prepare(signal?: AbortSignal): Promise { + try { + await this.prepareOnce(signal); + } catch (error) { + const diagnostic = commandSetupFailureDiagnostic( + 'programmatic_preflight', error, + ); + throw error instanceof WorkspaceToolError + ? new WorkspaceToolError( + error.message, error.code, error.mutationMayHaveCommitted, + error.requiresQuarantine, error.commandDiagnostic ?? diagnostic, + ) + : new WorkspaceToolError( + 'Native executor request failed', 'COMMAND_UNAVAILABLE', + false, false, diagnostic, + ); + } + } + + private async prepareOnce(signal?: AbortSignal): Promise { const createProbeWorkspace = this.options.sandbox.createProgrammaticProbeWorkspace; if (createProbeWorkspace == null) { @@ -265,7 +288,14 @@ export class NativeWorkspaceProgrammaticExecutor { signal, ); } finally { - await rm(executionDirectory, { recursive: true, force: true }); + try { + await rm(executionDirectory, { recursive: true, force: true }); + } catch (error) { + throw new WorkspaceToolError( + 'Native executor request failed', 'COMMAND_UNAVAILABLE', false, false, + commandFailureDiagnostic('cleanup', 'cleanup_failed', error), + ); + } } } @@ -327,12 +357,9 @@ export class NativeWorkspaceProgrammaticExecutor { const timer = setTimeout(() => controller.abort(), transferTimeoutMs); try { const response = await this.fetchImpl( - new URL( - `sessions/${encodeURIComponent(file.storage_session_id)}/objects/${encodeURIComponent(file.id)}`, - `${this.upstream.toString().replace(/\/+$/, '')}/`, - ), + fileRelayObjectUrl(this.upstream, file.storage_session_id, file.id), { - headers: { [EGRESS_GRANT_HEADER]: grant }, + headers: { [FILE_RELAY_GRANT_HEADER]: grant }, redirect: 'error', signal: controller.signal, }, @@ -704,14 +731,11 @@ export class NativeWorkspaceProgrammaticExecutor { let response: Response; try { response = await this.fetchImpl( - new URL( - `sessions/${encodeURIComponent(outputSessionId!)}/objects/${id}`, - `${this.upstream.toString().replace(/\/+$/, '')}/`, - ), + fileRelayObjectUrl(this.upstream, outputSessionId!, id), { method: 'PUT', headers: { - [EGRESS_GRANT_HEADER]: grant!, + [FILE_RELAY_GRANT_HEADER]: grant!, 'Content-Type': bridgeArtifactMediaType(name), 'Content-Length': String(bytes.byteLength), 'X-Original-Filename': encodeURIComponent(name), @@ -773,6 +797,9 @@ export class NativeWorkspaceProgrammaticExecutor { error instanceof WorkspaceToolError ? error.code : 'COMMAND_UNAVAILABLE', false, false, + error instanceof WorkspaceToolError && error.commandDiagnostic + ? error.commandDiagnostic + : commandSetupFailureDiagnostic('programmatic_preflight', error), ); } if (error instanceof WorkspaceToolError) { @@ -787,6 +814,7 @@ export class NativeWorkspaceProgrammaticExecutor { error.code, true, true, + error.commandDiagnostic, ); } throw new WorkspaceToolError( @@ -794,16 +822,18 @@ export class NativeWorkspaceProgrammaticExecutor { 'COMMAND_UNAVAILABLE', true, true, + commandFailureDiagnostic('dispatch', 'executor_rejected', error), ); } finally { try { await rm(executionDirectory, { recursive: true, force: true }); - } catch { + } catch (error) { throw new WorkspaceToolError( 'Native programmatic execution cleanup failed', 'COMMAND_UNAVAILABLE', commandDispatched, commandDispatched, + commandFailureDiagnostic('cleanup', 'cleanup_failed', error), ); } } diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index 90215156..4b31658d 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -32,6 +32,7 @@ import { NativeSrtWorkspaceCommandSandbox, } from './native-sandbox.js'; import { restoreScratchTraversal } from './native-scratch.js'; +import { browserSetupAttemptPath, browserSetupProtectedPaths } from './browser-setup.js'; import { WorkspaceToolError } from './workspace.js'; import { isWorkspaceToolResult } from './protocol.js'; @@ -1309,6 +1310,30 @@ test('fails closed when SRT dependencies are unavailable', async t => { ); }); +test('protects retained browser setup keys outside home from command reads and writes', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-enrollment-protection-')); + t.after(() => rm(root, { recursive: true, force: true })); + const workspace = join(root, 'workspace'); + const state = join(root, 'state'); + await mkdir(workspace); + await mkdir(state); + const attempt = browserSetupAttemptPath(join(state, 'profile.json')); + const paths = browserSetupProtectedPaths(attempt); + await mkdir(paths[0], { mode: 0o700 }); + for (const path of [attempt, `${attempt}.lock`, `${attempt}.0123456789abcdef.tmp`]) + await writeFile(path, 'private setup state', { mode: 0o600 }); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ workspaceRoot: workspace, protectedPaths: paths, manager: fake.manager }); + t.after(() => sandbox.close()); + await sandbox.prepare(); + for (const path of paths) { + const canonical = await realpath(path); + assert.ok(fake.config?.credentials?.files?.some(file => file.path === canonical && file.mode === 'deny'), + 'setup keys must be denied through the sandbox credential-file policy'); + assert.ok(fake.config?.filesystem.denyWrite.includes(canonical), 'commands must not replace setup state'); + } +}); + test('refuses workspace roots that expose worker home or control files', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); const controlDirectory = join(root, '.control'); @@ -1714,7 +1739,9 @@ test('cleans allocated command state exactly once on every execution exit', asyn (outcome === 'abort-after-spawn') && error.requiresQuarantine === (outcome === 'abort-after-spawn' && - process.platform === 'win32'), + process.platform === 'win32') && + (outcome.startsWith('abort') || (error.commandDiagnostic?.stage === + (outcome === 'wrap-throw' ? 'command_wrapping' : 'dispatch'))), ); } assert.equal( diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 2701fe44..d994b1f9 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -15,6 +15,8 @@ import type { Dirent, Stats } from 'node:fs'; import { access, mkdtemp, open, readdir, realpath, rm, stat } from 'node:fs/promises'; import type { FileHandle } from 'node:fs/promises'; import { matchesWorkspaceRoot } from './root-identity.js'; +import { commandFailureDiagnostic, commandSetupFailureDiagnostic } from './command-diagnostics.js'; +import type { CommandSetupStage } from './command-diagnostics.js'; import { assertEnvironmentResourcesStable, environmentResourceVariables } from './environment-resources.js'; import type { WorkspaceRootIdentity } from './root-identity.js'; import { withWorkspaceRoot, WorkspaceRootAccessError, spawnWithinWorkspace, realpath as rootedRealpath, stat as rootedStat } from './root-access.js'; @@ -1071,9 +1073,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox // permissive proxy session through a live updateConfig. await this.manager.reset(); await this.manager.initialize(config, config.network.strictAllowlist ? undefined : async () => true); - } catch { + } catch (error) { this.resetFailed = true; - throw new WorkspaceToolError('Native probe network cleanup failed', 'COMMAND_UNAVAILABLE'); + throw new WorkspaceToolError('Native probe network cleanup failed', 'COMMAND_UNAVAILABLE', false, false, + commandFailureDiagnostic('cleanup', 'cleanup_failed', error)); } } } @@ -1175,10 +1178,13 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox let wrapped: Awaited< ReturnType >; + let setupStage: CommandSetupStage = 'command_wrapping'; try { const commandConfig = customConfig ?? (await this.rootCommandConfig()); + setupStage = 'credential_resolution'; const credentialEnvironment = await this.options.maskedEnvironment?.resolve(signal, cwd); + setupStage = 'command_wrapping'; const sandboxedCommand = this.options.maskedEnvironment?.wrapCommand ? this.options.maskedEnvironment.wrapCommand( request.command, @@ -1215,6 +1221,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox throw new WorkspaceToolError( 'Native sandbox command could not start', 'COMMAND_UNAVAILABLE', + false, false, + error instanceof WorkspaceToolError && error.commandDiagnostic + ? error.commandDiagnostic + : commandSetupFailureDiagnostic(setupStage, error), ); } try { @@ -1314,11 +1324,13 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox }, ); child.stdin.end(); - } catch { + } catch (error) { reject( new WorkspaceToolError( 'Native sandbox command could not start', 'COMMAND_UNAVAILABLE', + false, false, + commandFailureDiagnostic('dispatch', 'executor_rejected', error), ), ); return; @@ -1344,7 +1356,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox clearTimeout(timer); signal?.removeEventListener('abort', abort); }; - child.once('error', () => { + child.once('error', error => { if (settled) return; settled = true; const mayHaveStarted = child.pid != null; @@ -1355,6 +1367,8 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox 'Native sandbox command could not start', 'COMMAND_UNAVAILABLE', mayHaveStarted, + mayHaveStarted, + commandFailureDiagnostic('dispatch', 'executor_rejected', error), ), ); }); diff --git a/packages/code/src/pairing.test.ts b/packages/code/src/pairing.test.ts index 2b564394..a7dd6bd0 100644 --- a/packages/code/src/pairing.test.ts +++ b/packages/code/src/pairing.test.ts @@ -34,3 +34,61 @@ test('pairing binds a generated worker key to a single-use code', async () => { assert.match(paired.publicKey, /BEGIN PUBLIC KEY/); assert.match(paired.privateKey, /BEGIN PRIVATE KEY/); }); + +test('pairing propagates cancellation and never follows a code-bearing redirect', async () => { + const controller = new AbortController(); + const paired = await pairBridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + code: 'one-time-code', + signal: controller.signal, + fetchImpl: async (_input, init) => { + assert.equal(init?.redirect, 'error'); + assert.equal(init?.signal, controller.signal); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + credential: 'issued-short-lived-credential-value', + expiresAt: new Date(Date.now() + 300_000).toISOString(), + }); + }, + }); + assert.equal(paired.workerId, 'vm-1'); +}); + +for (const payload of [ + null, + [], + 'not a response', + { protocolVersion: 2 }, + { protocolVersion: 1, workerId: 'another-worker' }, +]) { + test(`pairing rejects malformed response ${JSON.stringify(payload)}`, async () => { + await assert.rejects( + pairBridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + code: 'one-time-code', + fetchImpl: async () => Response.json(payload), + }), + /invalid/, + ); + }); +} + +test('non-JSON pairing failures have a bounded diagnosis without echoing response contents', async () => { + await assert.rejects( + pairBridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + code: 'one-time-code', + fetchImpl: async () => + new Response('private upstream response', { status: 502 }), + }), + error => { + assert.match(String(error), /invalid pairing response/); + assert.ok(!String(error).includes('private upstream response')); + return true; + }, + ); +}); diff --git a/packages/code/src/pairing.ts b/packages/code/src/pairing.ts index ed33fb18..cef571eb 100644 --- a/packages/code/src/pairing.ts +++ b/packages/code/src/pairing.ts @@ -1,8 +1,5 @@ import { createBridgeIdentity } from './identity.js'; -import { - BRIDGE_PROTOCOL_VERSION, - BridgeProtocolError, -} from './protocol.js'; +import { BRIDGE_PROTOCOL_VERSION, BridgeProtocolError } from './protocol.js'; import type { BridgeWorkerCredentialResponse } from './protocol.js'; @@ -11,6 +8,7 @@ export interface PairBridgeWorkerOptions { workerId: string; code: string; fetchImpl?: typeof fetch; + signal?: AbortSignal; } export interface PairedBridgeWorkerIdentity @@ -18,14 +16,16 @@ export interface PairedBridgeWorkerIdentity codeApiUrl: string; publicKey: string; privateKey: string; + /** Private setup journal to protect until completion removes its secret copies. */ + setupAttemptPath?: string; } function normalizedBaseUrl(value: string): string { return value.replace(/\/+$/, ''); } -function errorMessage(value: object): string | undefined { - if ('error' in value && typeof value.error === 'string') return value.error; +function errorMessage(value: Record): string | undefined { + if (typeof value.error === 'string') return value.error; return undefined; } @@ -38,6 +38,8 @@ export async function pairBridgeWorker( `${codeApiUrl}/bridge/pairings/redeem`, { method: 'POST', + redirect: 'error', + signal: options.signal, headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ protocolVersion: BRIDGE_PROTOCOL_VERSION, @@ -47,10 +49,29 @@ export async function pairBridgeWorker( }), }, ); - const payload = (await response.json()) as object; + let payload: unknown; + try { + payload = await response.json(); + } catch { + throw new BridgeProtocolError( + 'Code API returned an invalid pairing response', + response.status, + ); + } + if ( + typeof payload !== 'object' || + payload == null || + Array.isArray(payload) + ) { + throw new BridgeProtocolError( + 'Code API returned an invalid pairing response', + response.status, + ); + } if (!response.ok) { throw new BridgeProtocolError( - errorMessage(payload) ?? `Bridge pairing failed with HTTP ${response.status}`, + errorMessage(payload as Record) ?? + `Bridge pairing failed with HTTP ${response.status}`, response.status, ); } @@ -62,7 +83,9 @@ export async function pairBridgeWorker( credential.credential.length < 32 || !Number.isFinite(Date.parse(credential.expiresAt)) ) { - throw new BridgeProtocolError('Code API returned an invalid worker credential'); + throw new BridgeProtocolError( + 'Code API returned an invalid worker credential', + ); } return { ...credential, diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 1451986d..0fcb6090 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -35,6 +35,12 @@ export function programmaticTransferReserveMs(jobTimeoutMs: number): number { export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES = 10 * 1024 * 1024; export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_HISTORY_BYTES = 40_000_000; export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_TOTAL_BYTES = 100 * 1024 * 1024; +/** Default per-file `import_file` ceiling; Code API and each worker may configure their own. */ +export const BRIDGE_WORKSPACE_IMPORT_FILE_DEFAULT_MAX_BYTES = 100 * 1024 * 1024; +/** Longest execution budget Code API grants one `import_file` transfer. */ +export const BRIDGE_WORKSPACE_IMPORT_FILE_TIMEOUT_MS = 5 * 60_000; +const BRIDGE_WORKSPACE_IMPORT_FILE_HANDLE_MAX_LENGTH = 32_768; +const BRIDGE_WORKSPACE_IMPORT_FILE_GRANT_MAX_LENGTH = 256 * 1024; /** How long Code API drains a clean rejection after Stop cancels a workspace mutation. */ export const BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS = 5_000; @@ -313,7 +319,20 @@ export type BridgeWorkspaceToolOperation = | 'write_file' | 'preview_edit' | 'edit_file' - | 'execute_command'; + | 'execute_command' + | 'import_file'; + +/** Operations that may change workspace contents and therefore arm the durable mutation guard. */ +export function isWorkspaceMutationOperation( + operation: BridgeWorkspaceToolOperation, +): boolean { + return ( + operation === 'write_file' || + operation === 'edit_file' || + operation === 'execute_command' || + operation === 'import_file' + ); +} export type WorkspaceWriteFileMode = 'replace' | 'create'; export type WorkspaceEditFileMode = 'single' | 'batch'; @@ -674,6 +693,54 @@ export interface WorkspaceExecuteCommandResult { laneGit?: WorkspaceLaneGit; } +/** A stored object the caller owns, named exactly as Code API file storage names it. */ +export interface WorkspaceImportFileSource { + storage_session_id: string; + id: string; +} + +/** + * Copies one stored object into the workspace as a binary-safe file. The + * worker streams it through its file relay upstream, refuses `.git` paths and + * symlinked parents, verifies `bytes` and `sha256`, and installs it atomically + * with mode 0644. + */ +export interface WorkspaceImportFileRequest { + protocolVersion: BridgeProtocolVersion; + operation: 'import_file'; + workspaceId: string; + workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; + path: string; + source: WorkspaceImportFileSource; + /** Exact object size, enforced as a hard cap while streaming. */ + bytes: number; + /** Lowercase hex SHA-256 of the object's bytes. */ + sha256: string; + /** False or omitted refuses to replace a file whose content differs. */ + overwrite?: boolean; + /** + * Set only by Code API when it relays the request to a worker: a short-lived + * read grant scoped to `source`, whose ids are then opaque grant handles. + * Callers never send it. + */ + egressGrant?: string; +} + +export interface WorkspaceImportFileResult { + protocolVersion: BridgeProtocolVersion; + operation: 'import_file'; + workspaceId: string; + path: string; + created: boolean; + /** True when the target already held these exact bytes; nothing was written. */ + unchanged?: boolean; + /** Bytes installed, or 0 when `unchanged`. */ + bytesWritten: number; + sha256: string; +} + export type WorkspaceToolRequest = | WorkspaceReadFileRequest | WorkspaceSearchTextRequest @@ -681,7 +748,8 @@ export type WorkspaceToolRequest = | WorkspaceWriteFileRequest | WorkspacePreviewEditRequest | WorkspaceEditFileRequest - | WorkspaceExecuteCommandRequest; + | WorkspaceExecuteCommandRequest + | WorkspaceImportFileRequest; export type WorkspaceToolResult = | WorkspaceReadFileResult | WorkspaceSearchTextResult @@ -689,7 +757,8 @@ export type WorkspaceToolResult = | WorkspaceWriteFileResult | WorkspacePreviewEditResult | WorkspaceEditFileResult - | WorkspaceExecuteCommandResult; + | WorkspaceExecuteCommandResult + | WorkspaceImportFileResult; const WORKSPACE_READ_REQUEST_KEYS = new Set([ 'instructionSha256', @@ -771,6 +840,30 @@ const WORKSPACE_COMMAND_REQUEST_KEYS = new Set([ 'timeoutMs', 'maxOutputBytes', ]); +const WORKSPACE_IMPORT_REQUEST_KEYS = new Set([ + 'protocolVersion', + 'operation', + 'workspaceId', + 'workspaceInstanceId', + 'worktree', + 'path', + 'source', + 'bytes', + 'sha256', + 'overwrite', + 'egressGrant', +]); +const WORKSPACE_IMPORT_SOURCE_KEYS = new Set(['storage_session_id', 'id']); +const WORKSPACE_IMPORT_RESULT_KEYS = new Set([ + 'protocolVersion', + 'operation', + 'workspaceId', + 'path', + 'created', + 'unchanged', + 'bytesWritten', + 'sha256', +]); const WORKSPACE_READ_RESULT_KEYS = new Set([ 'protocolVersion', 'operation', @@ -1275,6 +1368,94 @@ export function isSafePortableRelativePath(value: unknown): value is string { return value.split('/').every(segment => segment !== '..'); } +/** + * Whether a path segment names Git metadata on any host filesystem: case + * folding (macOS, Windows), ignorable code points (HFS+), trailing dots and + * spaces (Windows), NTFS stream syntax such as `.git::$INDEX_ALLOCATION`, and + * the NTFS short name all alias `.git`. + */ +function isDotGitPathSegment(segment: string): boolean { + const folded = segment + .split(':', 1)[0]! + .toLowerCase() + .replace(/\p{Default_Ignorable_Code_Point}/gu, '') + .replace(/[. ]+$/, ''); + return folded === '.git' || folded === 'git~1'; +} + +/** `write_file` path rules, plus no `.git` segment and a file-like final segment. */ +export function isWorkspaceImportFilePath(value: unknown): value is string { + if (!isSafePortableRelativePath(value)) return false; + const segments = value.split('/'); + const last = segments[segments.length - 1]; + return ( + last !== undefined && + last !== '' && + last !== '.' && + !segments.some(isDotGitPathSegment) + ); +} + +function isWorkspaceImportFileHandle(value: unknown): value is string { + return ( + typeof value === 'string' && + value.length > 0 && + value.length <= BRIDGE_WORKSPACE_IMPORT_FILE_HANDLE_MAX_LENGTH && + !/[\0\r\n]/.test(value) + ); +} + +function isWorkspaceImportFileRequestShape( + request: Record, +): boolean { + const source = request.source; + return ( + hasOnlyKeys(request, WORKSPACE_IMPORT_REQUEST_KEYS) && + isWorkspaceImportFilePath(request.path) && + typeof source === 'object' && + source !== null && + !Array.isArray(source) && + hasOnlyKeys(source as Record, WORKSPACE_IMPORT_SOURCE_KEYS) && + isWorkspaceImportFileHandle((source as Record).storage_session_id) && + isWorkspaceImportFileHandle((source as Record).id) && + Number.isSafeInteger(request.bytes) && + Number(request.bytes) >= 0 && + typeof request.sha256 === 'string' && + /^[a-f0-9]{64}$/.test(request.sha256) && + (request.overwrite === undefined || typeof request.overwrite === 'boolean') && + (request.egressGrant === undefined || + (typeof request.egressGrant === 'string' && + request.egressGrant.length > 0 && + request.egressGrant.length <= BRIDGE_WORKSPACE_IMPORT_FILE_GRANT_MAX_LENGTH && + !/[\0\r\n]/.test(request.egressGrant))) + ); +} + +/** + * The caller-facing `import_file` request LibreChat sends to Code API. A + * caller may never supply `egressGrant`; Code API mints it when relaying. + */ +export function isBridgeWorkspaceImportFileRequest( + value: unknown, +): value is WorkspaceImportFileRequest & { egressGrant?: never } { + return ( + isWorkspaceToolRequest(value) && + value.operation === 'import_file' && + value.egressGrant === undefined + ); +} + +/** The relayed `import_file` request a worker executes: it must carry Code API's grant. */ +export function isRelayedWorkspaceImportFileRequest( + value: unknown, +): value is WorkspaceImportFileRequest & { egressGrant: string } { + return ( + isWorkspaceToolRequest(value) && + value.operation === 'import_file' && + value.egressGrant !== undefined + ); +} + function normalizePortableRelativePath(value: string): string { return ( value @@ -1573,6 +1754,9 @@ export function isWorkspaceToolRequest( BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES)) ); } + if (request.operation === 'import_file') { + return isWorkspaceImportFileRequestShape(request); + } return false; } @@ -1706,6 +1890,26 @@ export function isWorkspaceToolResult( ); } + if (request.operation === 'import_file') { + if ( + !hasOnlyKeys(result, WORKSPACE_IMPORT_RESULT_KEYS) || + result.path !== request.path || + result.sha256 !== request.sha256 || + typeof result.created !== 'boolean' || + (result.unchanged !== undefined && typeof result.unchanged !== 'boolean') || + !Number.isSafeInteger(result.bytesWritten) + ) { + return false; + } + if (result.unchanged === true) { + return result.created === false && result.bytesWritten === 0; + } + return ( + result.bytesWritten === request.bytes && + (request.overwrite === true || result.created === true) + ); + } + if (request.operation === 'edit_file') { const replacements = request.edits?.length ?? 1; return ( @@ -1808,7 +2012,7 @@ export function isValidBridgeWorkspaceToolCapabilities( capabilities.protocolVersion !== BRIDGE_PROTOCOL_VERSION || !Array.isArray(capabilities.operations) || capabilities.operations.length < 1 || - capabilities.operations.length > 7 || + capabilities.operations.length > 8 || !capabilities.operations.every( operation => operation === 'read_file' || @@ -1817,7 +2021,8 @@ export function isValidBridgeWorkspaceToolCapabilities( operation === 'write_file' || operation === 'preview_edit' || operation === 'edit_file' || - operation === 'execute_command', + operation === 'execute_command' || + operation === 'import_file', ) || new Set(capabilities.operations).size !== capabilities.operations.length || diff --git a/packages/code/src/relay.ts b/packages/code/src/relay.ts index 07aa8714..1cab6e65 100644 --- a/packages/code/src/relay.ts +++ b/packages/code/src/relay.ts @@ -34,6 +34,21 @@ const LOCAL_HTTP_HOSTS = new Set([ class RelayPayloadTooLargeError extends Error {} class UpstreamPayloadTooLargeError extends Error {} +/** Header carrying Code API's egress grant on every file relay object request. */ +export const FILE_RELAY_GRANT_HEADER = 'X-CodeAPI-Egress-Grant'; + +/** `/sessions//objects/`, the egress gateway's object route. */ +export function fileRelayObjectUrl( + upstream: URL, + storageSessionId: string, + objectId: string, +): URL { + return new URL( + `sessions/${encodeURIComponent(storageSessionId)}/objects/${encodeURIComponent(objectId)}`, + `${upstream.toString().replace(/\/+$/, '')}/`, + ); +} + export function validateFileRelayUpstream(value: string): URL { const upstream = new URL(value); if ( diff --git a/packages/code/src/setup-cli.test.ts b/packages/code/src/setup-cli.test.ts new file mode 100644 index 00000000..1bef6e14 --- /dev/null +++ b/packages/code/src/setup-cli.test.ts @@ -0,0 +1,266 @@ +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; +import type { TestContext } from 'node:test'; + +const cli = fileURLToPath(new URL('./cli.js', import.meta.url)); +const code = 'secret-one-time-code-123456789'; + +async function fixture(t: TestContext) { + const home = await mkdtemp(join(tmpdir(), 'setup-cli-home-')); + t.after(() => rm(home, { recursive: true, force: true })); + const env = { + ...Object.fromEntries( + Object.entries(process.env).filter( + ([key]) => !key.startsWith('LIBRECHAT_CODE_'), + ), + ), + HOME: home, + }; + const profilePath = join( + home, + '.config', + 'librechat', + 'code', + 'profiles', + 'default.json', + ); + return { home, env, profilePath }; +} + +async function execute(args: string[], env: NodeJS.ProcessEnv, stdin = '') { + const child = spawn(process.execPath, [cli, ...args], { env, stdio: 'pipe' }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', chunk => { + stdout += String(chunk); + }); + child.stderr.on('data', chunk => { + stderr += String(chunk); + }); + child.stdin.end(stdin); + const [status] = await once(child, 'exit', { + signal: AbortSignal.timeout(10_000), + }); + return { status, stdout, stderr }; +} + +test('CLI help explains the pairing prerequisite and setup does not silently read piped prompts', async t => { + const { env } = await fixture(t); + const help = spawnSync(process.execPath, [cli, 'setup', '--help'], { + env, + encoding: 'utf8', + }); + assert.equal(help.status, 0, help.stderr); + assert.match(help.stdout, /Settings > Code Environments/); + assert.match(help.stdout, /does not\nsign in to LibreChat/); + const guided = await execute(['setup'], env, code); + assert.notEqual(guided.status, 0); + assert.match(guided.stderr, /requires a terminal/); + assert.ok(!guided.stdout.includes(code)); + assert.ok(!guided.stderr.includes(code)); +}); + +test( + 'CLI setup followed by plain run registers the saved identity with read-only access', + { timeout: 15_000 }, + async t => { + const { env, profilePath } = await fixture(t); + let paired = false; + let registered!: (body: Record) => void; + const registration = new Promise>(resolve => { + registered = resolve; + }); + const server = createServer(async (request, response) => { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + const body = JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}'); + response.setHeader('Content-Type', 'application/json'); + if (request.url === '/v1/bridge/pairings/redeem') { + assert.equal(body.workerId, 'guided-vm'); + assert.equal(body.code, code); + assert.match(body.publicKey, /BEGIN PUBLIC KEY/); + assert.equal(body.privateKey, undefined); + paired = true; + response.end( + JSON.stringify({ + protocolVersion: 1, + workerId: body.workerId, + credential: 'short-lived-worker-credential-value', + expiresAt: '2099-01-01T00:00:00Z', + }), + ); + return; + } + if (request.url === '/v1/bridge/workers/register') { + assert.equal(paired, true); + assert.match(request.headers.authorization ?? '', /^Bridge /); + assert.ok(request.headers['x-librechat-code-signature']); + registered(body); + response.end( + JSON.stringify({ + protocolVersion: 1, + workerId: body.workerId, + incarnationId: body.incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }), + ); + return; + } + response.end(JSON.stringify({ protocolVersion: 1, serverElapsedMs: 0 })); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => { + server.closeAllConnections(); + server.close(); + }); + const address = server.address(); + assert.ok(address && typeof address !== 'string'); + const result = await execute( + [ + 'setup', + '--non-interactive', + '--code-stdin', + '--url', + `http://127.0.0.1:${address.port}/v1`, + '--worker-id', + 'guided-vm', + '--default-workspace', + ], + env, + `${code}\n`, + ); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /librechat-code run\n/); + assert.match(result.stdout, /still needs to start/); + assert.ok(!result.stdout.includes(code)); + assert.ok(!result.stderr.includes(code)); + const profile = JSON.parse(await readFile(profilePath, 'utf8')); + assert.equal(profile.allowWorkspaceWrites, false); + assert.equal(profile.allowWorkspaceCommands, false); + const worker = spawn(process.execPath, [cli, 'run'], { + env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stderr = ''; + worker.stderr.on('data', chunk => { + stderr += String(chunk); + }); + const exited = once(worker, 'exit'); + t.after(() => worker.kill('SIGKILL')); + const body = await Promise.race([ + registration, + exited.then(() => { + throw new Error(`Worker exited before registration: ${stderr}`); + }), + ]); + worker.kill('SIGTERM'); + await exited; + assert.equal(body.workerId, 'guided-vm'); + const operations = body.capabilities.workspaceTools.operations as string[]; + assert.ok(operations.includes('read_file')); + assert.ok(!operations.includes('write_file')); + assert.ok(!operations.includes('execute_command')); + assert.ok(!JSON.stringify(body).includes(profile.identityPath)); + assert.ok(!JSON.stringify(body).includes(profilePath)); + }, +); + +test('CLI rejects missing automation inputs without consuming stdin or pairing', async t => { + const { env } = await fixture(t); + const result = await execute(['setup', '--non-interactive'], env, code); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /requires --url, --worker-id, --code-stdin/); + assert.ok(!result.stderr.includes(code)); +}); + +test('CLI refuses unsafe URLs before disclosing the code', async t => { + const { env } = await fixture(t); + const result = await execute( + [ + 'setup', + '--non-interactive', + '--code-stdin', + '--url', + 'http://external.example/v1', + '--worker-id', + 'vm', + '--default-workspace', + ], + env, + code, + ); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /requires HTTPS/); + assert.ok(!result.stdout.includes(code)); + assert.ok(!result.stderr.includes(code)); +}); + +for (const [name, secretEditing] of [ + ['history', ''], + ['kill beginning', '\x15\x19'], + ['kill end', '\x01\x0b\x19'], + ['kill word', '\x17\x19'], +]) + test( + `terminal ${name} never recalls a hidden pairing code at confirmation`, + { timeout: 10_000 }, + async t => { + const { env } = await fixture(t); + const secret = 'SyntheticPairingSecret0123456789'; + // Real readline terminal editing over pipes avoids an OS-specific PTY dependency. + const bootstrap = ` + Object.defineProperty(process.stdin, 'isTTY', { value: true }); + Object.defineProperty(process.stdout, 'isTTY', { value: true }); + process.argv = [process.execPath, ${JSON.stringify(cli)}, 'setup', '--url', + 'https://code.example/v1', '--worker-id', 'history-vm', '--default-workspace']; + await import(${JSON.stringify(new URL('./cli.js', import.meta.url).href)}); + `; + const child = spawn( + process.execPath, + ['--input-type=module', '--eval', bootstrap], + { env, stdio: 'pipe' }, + ); + t.after(() => child.kill('SIGKILL')); + let transcript = ''; + let stderr = ''; + let step = 0; + const inputs = [ + { prompt: 'Allow file writes', input: '\n' }, + { prompt: 'Allow sandboxed commands', input: '\n' }, + { + prompt: 'One-time pairing code (hidden): ', + input: `${secret}${secretEditing}\n`, + }, + // History and yank must not reveal secret editing state before redemption. + { + prompt: 'Pair this machine and save these choices?', + input: '\x1b[A\x10\x19\x15n\n', + }, + ]; + child.stderr.on('data', chunk => { + stderr += String(chunk); + }); + child.stdout.on('data', chunk => { + transcript += String(chunk); + if (step < inputs.length && transcript.includes(inputs[step].prompt)) { + child.stdin.write(inputs[step++].input); + } + }); + const [status] = await once(child, 'exit'); + assert.equal(step, inputs.length, stderr); + assert.equal(status, 0, stderr); + assert.ok( + !transcript.includes(secret), + 'hidden pairing code was recalled visibly from history', + ); + assert.match(transcript, /No pairing was redeemed/); + }, + ); diff --git a/packages/code/src/setup-cli.ts b/packages/code/src/setup-cli.ts new file mode 100644 index 00000000..08262153 --- /dev/null +++ b/packages/code/src/setup-cli.ts @@ -0,0 +1,284 @@ +import { createInterface } from 'node:readline/promises'; +import { Writable } from 'node:stream'; +import { resolve } from 'node:path'; + +import { + setupCodeApiUrl, + setupWorker, + setupWorkspaceDirectory, +} from './setup.js'; +import { defaultWorkerProfilePath } from './worker-profile.js'; +import { isValidBridgeWorkerId } from './protocol.js'; +import { runBrowserSetupCommand } from './browser-setup-cli.js'; + +export const SETUP_HELP = `Usage: librechat-code setup [options] + +Enroll with browser approval: + librechat-code setup --instance https://chat.example.com + +The browser uses LibreChat's normal login, including configured OAuth/OIDC. +SSH/headless machines display a URL and matching verification code. No inbound +listener or CLI identity-provider integration is needed. Interrupted setup +resumes the same saved key and attempt. Never delete it to retry an enrollment. + +Existing manual pairing: +Create a private worker profile using a one-time pairing from LibreChat. +Open Settings > Code Environments in LibreChat to add the machine first. +Use its Code API URL, worker ID, and pairing code. This command does not +sign in to LibreChat or bypass its enrollment policy. + + --instance LibreChat instance URL for browser approval + --machine-name Display name (default: host name) + --no-browser Display approval instructions without opening a browser + --start Start the saved worker in the foreground after setup + --cancel Cancel this profile's pending browser setup + --url Manual Code API URL, including its /v1 prefix + --worker-id Worker ID supplied by LibreChat + --worker-dir Expose one existing project directory + --default-workspace Create a private workspace when the worker starts + --allow-workspace-writes Explicitly enable file writes (default: off) + --allow-workspace-commands Explicitly enable sandboxed commands (default: off) + --profile Profile destination (default: profiles/default.json) + --identity Separate private identity destination + --non-interactive Require explicit inputs instead of prompting + --code-stdin Read the pairing code from stdin, never an argument + --help Show this help + +Existing profiles and identities are never replaced. Choose another profile +and identity for a second machine. Setup does not install OS dependencies, +start a background service, or change the restricted command policy. +`; + +const VALUE_FLAGS = new Set([ + '--instance', + '--machine-name', + '--url', + '--worker-id', + '--worker-dir', + '--profile', + '--identity', +]); +const BOOLEAN_FLAGS = new Set([ + '--no-browser', + '--start', + '--cancel', + '--default-workspace', + '--allow-workspace-writes', + '--allow-workspace-commands', + '--non-interactive', + '--code-stdin', +]); + +export function parseSetupArguments( + args: string[], +): Map { + const options = new Map(); + for (let i = 0; i < args.length; i++) { + const [flag, ...inline] = args[i].split('='); + if ( + (!VALUE_FLAGS.has(flag) && !BOOLEAN_FLAGS.has(flag)) || + options.has(flag) + ) { + throw new Error( + 'Unknown or repeated setup option; use librechat-code setup --help', + ); + } + if (BOOLEAN_FLAGS.has(flag)) { + if (inline.length) + throw new Error( + 'Setup switches do not accept a value; use setup --help', + ); + options.set(flag, true); + } else { + const value = inline.length ? inline.join('=') : args[++i]; + if ( + !value?.trim() || + value.startsWith('--') || + /[\x00-\x1f\x7f]/.test(value) + ) { + throw new Error( + 'A setup option is missing its value; use setup --help', + ); + } + options.set(flag, value); + } + } + if (options.has('--worker-dir') && options.has('--default-workspace')) { + throw new Error('Choose --worker-dir or --default-workspace, not both'); + } + if (options.has('--instance') && ['--url', '--worker-id', '--code-stdin'].some(flag => options.has(flag))) + throw new Error('Browser approval cannot be combined with manual pairing options'); + if (!options.has('--instance') && ['--machine-name', '--no-browser', '--start', '--cancel'].some(flag => options.has(flag))) + throw new Error('Browser setup options require --instance'); + if (options.has('--non-interactive') && options.has('--instance') && !options.has('--cancel') && + !options.has('--worker-dir') && !options.has('--default-workspace')) + throw new Error('Non-interactive browser setup requires --worker-dir or --default-workspace'); + if ( + options.has('--non-interactive') && !options.has('--instance') && + (!options.has('--url') || + !options.has('--worker-id') || + !options.has('--code-stdin') || + (!options.has('--worker-dir') && !options.has('--default-workspace'))) + ) { + throw new Error( + 'Non-interactive setup requires --url, --worker-id, --code-stdin, and --worker-dir or --default-workspace', + ); + } + if (options.has('--code-stdin') && !options.has('--non-interactive')) { + throw new Error('--code-stdin requires --non-interactive'); + } + return options; +} + +async function pairingCodeFromStdin(signal: AbortSignal): Promise { + let code = ''; + const abort = () => process.stdin.destroy(new Error('Setup cancelled')); + signal.addEventListener('abort', abort, { once: true }); + try { + signal.throwIfAborted(); + for await (const chunk of process.stdin) { + code += String(chunk); + if (Buffer.byteLength(code) > 1024) + throw new Error('Pairing code input is too large'); + signal.throwIfAborted(); + } + return code.trim(); + } finally { + signal.removeEventListener('abort', abort); + } +} + +export async function runSetupCommand(args: string[]): Promise { + if (args.length === 1 && args[0] === '--help') { + process.stdout.write(SETUP_HELP); + return; + } + const options = parseSetupArguments(args); + const interactive = !options.has('--non-interactive'); + if (interactive && (!process.stdin.isTTY || !process.stdout.isTTY)) { + throw new Error( + 'Guided setup requires a terminal. For automation use --non-interactive and --code-stdin; see setup --help', + ); + } + const controller = new AbortController(); + const abort = () => controller.abort(new Error('Setup cancelled')); + process.once('SIGINT', abort); + let muted = false; + const output = new Writable({ + write(chunk, _encoding, callback) { + if (!muted) process.stdout.write(chunk); + callback(); + }, + }); + const ask = async (message: string, secret = false): Promise => { + controller.signal.throwIfAborted(); + muted = secret; + // Readline retains more than history (including a kill ring). Never reuse prompt state. + const terminal = createInterface({ + input: process.stdin, + output, + terminal: true, + historySize: 0, + }); + terminal.on('SIGINT', abort); + terminal.once('close', abort); + process.stdout.write(message); + try { + return ( + await terminal.question('', { signal: controller.signal }) + ).trim(); + } finally { + terminal.off('close', abort); + terminal.off('SIGINT', abort); + terminal.close(); + muted = false; + if (secret) process.stdout.write('\n'); + } + }; + const yesNo = async (message: string): Promise => { + for (;;) { + const value = (await ask(`${message} [y/N]: `)).toLowerCase(); + if (value === '' || value === 'n' || value === 'no') return false; + if (value === 'y' || value === 'yes') return true; + process.stdout.write('Enter yes or no.\n'); + } + }; + try { + if (options.has('--instance')) { + await runBrowserSetupCommand(options, { interactive, ask, yesNo, signal: controller.signal }); + return; + } + if (interactive) + process.stdout.write( + 'Connect a machine using the pairing supplied by LibreChat.\nOnly selected folders and permissions will be exposed.\n\n', + ); + const codeApiUrl = setupCodeApiUrl( + String( + options.get('--url') ?? (await ask('Code API URL (including /v1): ')), + ), + ); + const workerId = String( + options.get('--worker-id') ?? (await ask('Worker ID from LibreChat: ')), + ); + if (!isValidBridgeWorkerId(workerId)) + throw new Error('Enter the worker ID supplied by LibreChat'); + let workspaceDirectory = options.get('--worker-dir') as string | undefined; + if ( + interactive && + !workspaceDirectory && + !options.has('--default-workspace') + ) { + workspaceDirectory = + (await ask( + 'Project directory (Enter for a new private workspace): ', + )) || undefined; + } + if (workspaceDirectory != null) + workspaceDirectory = await setupWorkspaceDirectory(workspaceDirectory); + const allowWorkspaceWrites = + options.has('--allow-workspace-writes') || + (interactive && + (await yesNo('Allow file writes within this workspace?'))); + const allowWorkspaceCommands = + options.has('--allow-workspace-commands') || + (interactive && + (await yesNo( + 'Allow sandboxed commands within this workspace? Commands can modify workspace files', + ))); + const profilePath = resolve( + String(options.get('--profile') ?? defaultWorkerProfilePath()), + ); + const code = interactive + ? await ask('One-time pairing code (hidden): ', true) + : await pairingCodeFromStdin(controller.signal); + process.stdout.write( + `\nCode API: ${codeApiUrl}\nWorker: ${workerId}\nWorkspace: ${workspaceDirectory ?? 'new private workspace'}\nFile writes: ${allowWorkspaceWrites ? 'enabled' : 'disabled'}\nCommands: ${allowWorkspaceCommands ? 'enabled, restricted sandbox policy' : 'disabled'}\nProfile: ${profilePath}\n`, + ); + if ( + interactive && + !(await yesNo('Pair this machine and save these choices?')) + ) { + process.stdout.write('Setup cancelled. No pairing was redeemed.\n'); + return; + } + await setupWorker({ + codeApiUrl, + workerId, + code, + profilePath, + identityPath: options.get('--identity') as string | undefined, + workspaceDirectory, + allowWorkspaceWrites, + allowWorkspaceCommands, + signal: controller.signal, + }); + const defaultProfile = profilePath === resolve(defaultWorkerProfilePath()); + const quotedProfile = `'${profilePath.replace(/'/g, "'\\''")}'`; + process.stdout.write( + `\nSetup saved. Start the worker with:\n librechat-code run${defaultProfile ? '' : ` --profile ${quotedProfile}`}\n\nPairing is complete. The worker still needs to start, prepare its sandbox,\nand register before LibreChat can report it ready.\n`, + ); + } finally { + process.off('SIGINT', abort); + output.destroy(); + } +} diff --git a/packages/code/src/setup.test.ts b/packages/code/src/setup.test.ts new file mode 100644 index 00000000..0241f083 --- /dev/null +++ b/packages/code/src/setup.test.ts @@ -0,0 +1,334 @@ +import assert from 'node:assert/strict'; +import { + mkdir, + mkdtemp, + readFile, + readdir, + rm, + writeFile, +} from 'node:fs/promises'; +import { homedir, tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { createBridgeIdentity } from './identity.js'; +import { loadBridgeIdentity } from './storage.js'; +import { loadWorkerProfile, saveWorkerProfile } from './worker-profile.js'; +import { + profileRunConfiguration, + setupCodeApiUrl, + setupWorker, +} from './setup.js'; +import { parseSetupArguments } from './setup-cli.js'; +import type { SetupWorkerOptions } from './setup.js'; +import type { PairBridgeWorkerOptions } from './pairing.js'; + +const code = 'one-time-code-value-123456789'; + +async function fixture(t: test.TestContext) { + const root = await mkdtemp(join(tmpdir(), 'worker-setup-')); + t.after(() => rm(root, { recursive: true, force: true })); + const options: SetupWorkerOptions = { + codeApiUrl: 'https://code.example/v1/', + workerId: 'setup-worker', + code, + profilePath: join(root, 'profiles', 'default.json'), + identityPath: join(root, 'identity.json'), + allowWorkspaceWrites: false, + allowWorkspaceCommands: false, + }; + const pair = async (input: PairBridgeWorkerOptions) => ({ + protocolVersion: 1 as const, + workerId: input.workerId, + codeApiUrl: input.codeApiUrl, + ...createBridgeIdentity(), + credential: 'issued-short-lived-credential-value', + expiresAt: '2099-01-01T00:00:00Z', + }); + return { root, options, pair }; +} + +test('setup saves separate identity and least-privilege profile after pairing', async t => { + const { options, pair } = await fixture(t); + let calls = 0; + const saved = await setupWorker(options, { + pair: async input => { + calls++; + assert.equal(input.codeApiUrl, 'https://code.example/v1'); + assert.equal(input.code, code); + assert.ok(input.signal); + return pair(input); + }, + }); + assert.equal(calls, 1); + assert.deepEqual(await loadWorkerProfile(saved.profilePath), saved.profile); + assert.deepEqual(saved.profile.workspace, { type: 'private' }); + assert.equal(saved.profile.allowWorkspaceCommands, false); + assert.equal(saved.profile.allowWorkspaceWrites, false); + assert.equal( + (await loadBridgeIdentity(saved.profile.identityPath)).workerId, + 'setup-worker', + ); + const profileSource = await readFile(saved.profilePath, 'utf8'); + assert.ok(!profileSource.includes(code)); + assert.ok(!profileSource.includes('PRIVATE KEY')); + assert.ok(!profileSource.includes('issued-short-lived')); +}); + +test('setup canonicalizes an explicitly selected project and preflights command dependencies', async t => { + const { root, options, pair } = await fixture(t); + const project = join(root, 'project with spaces'); + await mkdir(project); + let checked = false; + const saved = await setupWorker( + { ...options, workspaceDirectory: project, allowWorkspaceCommands: true }, + { + checkCommands: async () => { + checked = true; + }, + pair: async input => { + assert.equal(checked, true); + return pair(input); + }, + }, + ); + assert.deepEqual(saved.profile.workspace, { + type: 'directory', + path: project, + }); + assert.equal(saved.profile.allowWorkspaceCommands, true); + assert.equal(saved.profile.allowWorkspaceWrites, false); +}); + +test('missing command prerequisites do not spend pairing code or create storage', async t => { + const { root, options, pair } = await fixture(t); + await assert.rejects( + setupWorker( + { ...options, allowWorkspaceCommands: true }, + { + checkCommands: async () => { + throw new Error('missing sandbox dependency'); + }, + pair: async input => { + assert.fail('must not pair'); + return pair(input); + }, + }, + ), + /missing sandbox dependency/, + ); + assert.deepEqual(await readdir(root), []); +}); + +test('denied or expired pairing releases only the new empty reservations', async t => { + const { root, options } = await fixture(t); + await assert.rejects( + setupWorker(options, { + pair: async () => { + throw new Error('Pairing code is invalid or expired'); + }, + }), + /invalid or expired/, + ); + assert.deepEqual(await readdir(root), ['profiles']); + assert.deepEqual(await readdir(join(root, 'profiles')), []); +}); + +test('existing identity is never replaced and code is not redeemed', async t => { + const { options } = await fixture(t); + await writeFile(options.identityPath!, 'keep this identity', { mode: 0o600 }); + await assert.rejects( + setupWorker(options, { + pair: async () => { + assert.fail('must not pair'); + }, + }), + /already exists/, + ); + assert.equal( + await readFile(options.identityPath!, 'utf8'), + 'keep this identity', + ); + assert.equal(await loadWorkerProfile(options.profilePath!), undefined); +}); + +test('profile write failure preserves the paired identity for manual recovery', async t => { + const { options, pair } = await fixture(t); + await assert.rejects( + setupWorker(options, { + pair: async input => { + await rm(options.profilePath!); + await mkdir(options.profilePath!); + return pair(input); + }, + }), + /Machine paired and identity saved.*profile could not be saved/, + ); + assert.equal( + (await loadBridgeIdentity(options.identityPath!)).workerId, + options.workerId, + ); +}); + +test('setup rejects exposed control files even with read-only permissions', async t => { + const { root, options } = await fixture(t); + await assert.rejects( + setupWorker( + { ...options, workspaceDirectory: root }, + { + pair: async () => { + assert.fail('must not pair'); + }, + }, + ), + /outside the workspace/, + ); +}); + +test('setup never admits home or a parent as a project', async t => { + const { options } = await fixture(t); + for (const workspaceDirectory of [homedir(), '/']) { + await assert.rejects( + setupWorker({ ...options, workspaceDirectory }), + /Do not expose your home directory/, + ); + } +}); + +test('setup stops before storage or pairing when cancelled', async t => { + const { root, options } = await fixture(t); + const controller = new AbortController(); + controller.abort(new Error('cancelled by owner')); + await assert.rejects( + setupWorker({ ...options, signal: controller.signal }), + /cancelled by owner/, + ); + assert.deepEqual(await readdir(root), []); +}); + +test('setup accepts HTTPS prefixes and only permits insecure HTTP on loopback', () => { + assert.equal( + setupCodeApiUrl('https://code.example/proxy/v1/'), + 'https://code.example/proxy/v1', + ); + for (const host of ['localhost', '127.0.0.1', '[::1]']) { + assert.equal( + setupCodeApiUrl(`http://${host}:3000/v1`), + `http://${host}:3000/v1`, + ); + } + for (const value of [ + 'http://code.example/v1', + 'ftp://code.example', + 'https://user:secret@code.example/v1', + 'https://code.example/v1?token=secret', + 'https://code.example/v1#fragment', + 'https://code.example/\n', + ]) { + assert.throws(() => setupCodeApiUrl(value), /HTTPS|credentials/); + } +}); + +test('saved profile startup never merges legacy identity or permission settings', async t => { + const { options, pair } = await fixture(t); + const saved = await setupWorker(options, { pair }); + const selected = await profileRunConfiguration( + ['--profile', saved.profilePath], + {}, + ); + assert.equal(selected.profilePath, saved.profilePath); + assert.deepEqual(selected.args, ['--default-workspace']); + assert.equal(selected.profile?.identityPath, options.identityPath); + for (const key of [ + 'LIBRECHAT_CODE_WORKER_TOKEN', + 'LIBRECHAT_CODE_IDENTITY_FILE', + 'LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS', + ]) { + await assert.rejects( + profileRunConfiguration(['--profile', saved.profilePath], { + [key]: 'configured', + }), + /cannot be combined/, + ); + assert.deepEqual( + await profileRunConfiguration(['--worker-dir', '/legacy'], { + [key]: 'configured', + }), + { args: ['--worker-dir', '/legacy'] }, + ); + } + await assert.rejects( + profileRunConfiguration( + ['--profile', saved.profilePath, '--allow-workspace-commands'], + {}, + ), + /cannot be combined with run options/, + ); +}); + +test('explicit profile parse rejects missing, repeated, absent, or corrupt profiles', async t => { + const { options } = await fixture(t); + await assert.rejects( + profileRunConfiguration(['--profile'], {}), + /requires a file/, + ); + await assert.rejects( + profileRunConfiguration(['--profile=a', '--profile=b'], {}), + /only once/, + ); + await assert.rejects( + profileRunConfiguration(['--profile', options.profilePath!], {}), + /not found/, + ); + await mkdir(join(options.profilePath!, '..'), { recursive: true }); + await writeFile(options.profilePath!, '{}', { mode: 0o600 }); + await assert.rejects( + profileRunConfiguration(['--profile', options.profilePath!], {}), + /Invalid worker profile/, + ); +}); + +test('profile file environment variable selects the same profile without changing process environment', async t => { + const { options } = await fixture(t); + const profile = { + version: 1 as const, + identityPath: options.identityPath!, + workspace: { type: 'private' as const }, + allowWorkspaceWrites: true, + allowWorkspaceCommands: false, + }; + await saveWorkerProfile(options.profilePath!, profile); + const environment = { LIBRECHAT_CODE_PROFILE_FILE: options.profilePath! }; + const result = await profileRunConfiguration([], environment); + assert.deepEqual(result.args, [ + '--default-workspace', + '--allow-workspace-writes', + ]); + assert.deepEqual(environment, { + LIBRECHAT_CODE_PROFILE_FILE: options.profilePath!, + }); +}); + +test('setup CLI parsing rejects ambiguity and requires explicit non-interactive access', () => { + for (const args of [ + ['--worker-id'], + ['--url', '--worker-id'], + ['--unknown'], + ['--url=a', '--url=b'], + ['--default-workspace=true'], + ['--worker-dir', '/project', '--default-workspace'], + ['--non-interactive'], + ['--code-stdin'], + ]) { + assert.throws(() => parseSetupArguments(args)); + } + const parsed = parseSetupArguments([ + '--non-interactive', + '--code-stdin', + '--url=https://code.example/v1', + '--worker-id=vm1', + '--default-workspace', + ]); + assert.equal(parsed.get('--allow-workspace-commands'), undefined); + assert.equal(parsed.get('--allow-workspace-writes'), undefined); +}); diff --git a/packages/code/src/setup.ts b/packages/code/src/setup.ts new file mode 100644 index 00000000..46879207 --- /dev/null +++ b/packages/code/src/setup.ts @@ -0,0 +1,254 @@ +import { access, realpath, stat } from 'node:fs/promises'; +import { constants } from 'node:fs'; +import { homedir } from 'node:os'; +import { isAbsolute, relative, resolve, sep } from 'node:path'; + +import { pairBridgeWorker } from './pairing.js'; +import { isValidBridgeWorkerId } from './protocol.js'; +import { assertPrivateStorageSupported } from './private-storage.js'; +import { + assertIdentityPathIsPrivate, + defaultBridgeIdentityPath, + saveBridgeIdentity, +} from './storage.js'; +import { + assertWorkerProfileIsolation, + defaultWorkerProfilePath, + loadWorkerProfile, + saveWorkerProfile, + workerProfileArguments, +} from './worker-profile.js'; +import type { WorkerProfile } from './worker-profile.js'; + +export interface SetupWorkerOptions { + codeApiUrl: string; + workerId: string; + code: string; + profilePath?: string; + identityPath?: string; + workspaceDirectory?: string; + allowWorkspaceWrites: boolean; + allowWorkspaceCommands: boolean; + signal?: AbortSignal; +} + +export function setupCodeApiUrl(value: string): string { + if (value.length > 2048 || /[\s\x00-\x1f\x7f]/.test(value)) { + throw new Error( + 'Enter a Code API HTTPS URL without credentials, query, or fragment', + ); + } + let url: URL; + try { + url = new URL(value); + } catch { + throw new Error( + 'Enter a valid Code API HTTPS URL, including its /v1 prefix', + ); + } + const loopback = ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname); + if ( + (url.protocol !== 'https:' && !(url.protocol === 'http:' && loopback)) || + url.username || + url.password || + url.search || + url.hash + ) { + throw new Error( + 'Code API requires HTTPS; HTTP is allowed only on loopback, without credentials, query, or fragment', + ); + } + return url.href.replace(/\/+$/, ''); +} + +export async function checkSetupCommands(): Promise { + const { SandboxManager } = await import('@anthropic-ai/sandbox-runtime'); + if (!SandboxManager.isSupportedPlatform()) { + throw new Error( + 'Native sandbox commands require a supported macOS or Linux host', + ); + } + const dependencies = await SandboxManager.checkDependenciesAsync(); + if (dependencies.errors.length) { + throw new Error( + `Install the native sandbox prerequisites before pairing: ${dependencies.errors.join('; ')}`, + ); + } + await access('/bin/bash', constants.X_OK); +} + +export async function setupWorkspaceDirectory(value: string): Promise { + if (!value.trim() || /[\x00-\x1f\x7f]/.test(value)) { + throw new Error('Choose an existing project directory'); + } + const path = await realpath(resolve(value)); + if (!(await stat(path)).isDirectory()) + throw new Error('Workspace must be a directory'); + const home = await realpath(homedir()); + const homeRelative = relative(path, home); + if ( + homeRelative === '' || + (!isAbsolute(homeRelative) && + homeRelative !== '..' && + !homeRelative.startsWith(`..${sep}`)) + ) { + throw new Error( + 'Do not expose your home directory or a directory containing it; choose a project or a private workspace', + ); + } + return path; +} + +export async function setupWorker( + options: SetupWorkerOptions, + dependencies: { + pair?: typeof pairBridgeWorker; + checkCommands?: typeof checkSetupCommands; + } = {}, +): Promise<{ profilePath: string; profile: WorkerProfile }> { + options.signal?.throwIfAborted(); + assertPrivateStorageSupported(); + const codeApiUrl = setupCodeApiUrl(options.codeApiUrl); + if (!isValidBridgeWorkerId(options.workerId)) + throw new Error('Enter the worker ID supplied by LibreChat'); + if (!/^[A-Za-z0-9_-]{16,256}$/.test(options.code)) + throw new Error('Enter a valid one-time pairing code from LibreChat'); + const profilePath = resolve( + options.profilePath ?? defaultWorkerProfilePath(), + ); + const identityPath = resolve( + options.identityPath ?? defaultBridgeIdentityPath(options.workerId), + ); + if ( + [profilePath, identityPath].some( + path => Buffer.byteLength(path) > 4096 || /[\x00-\x1f\x7f]/.test(path), + ) + ) { + throw new Error( + 'Profile and identity paths must contain no control characters and be at most 4096 bytes', + ); + } + if ( + typeof options.allowWorkspaceWrites !== 'boolean' || + typeof options.allowWorkspaceCommands !== 'boolean' + ) { + throw new Error( + 'Workspace permissions must be explicitly enabled or disabled', + ); + } + const workspaceDirectory = + options.workspaceDirectory == null + ? undefined + : await setupWorkspaceDirectory(options.workspaceDirectory); + const profile: WorkerProfile = { + version: 1, + identityPath, + workspace: + workspaceDirectory == null + ? { type: 'private' } + : { type: 'directory', path: workspaceDirectory }, + allowWorkspaceWrites: options.allowWorkspaceWrites, + allowWorkspaceCommands: options.allowWorkspaceCommands, + }; + if (profile.allowWorkspaceCommands) + await (dependencies.checkCommands ?? checkSetupCommands)(); + options.signal?.throwIfAborted(); + const profileReservation = await assertIdentityPathIsPrivate( + profilePath, + true, + ); + try { + const identityReservation = await assertIdentityPathIsPrivate( + identityPath, + true, + ); + try { + await assertWorkerProfileIsolation( + profilePath, + identityPath, + workspaceDirectory, + ); + const identity = await (dependencies.pair ?? pairBridgeWorker)({ + codeApiUrl, + workerId: options.workerId, + code: options.code, + signal: + options.signal == null + ? AbortSignal.timeout(30_000) + : AbortSignal.any([options.signal, AbortSignal.timeout(30_000)]), + }); + await saveBridgeIdentity(identityPath, identity); + try { + await saveWorkerProfile(profilePath, profile); + } catch { + throw new Error( + `Machine paired and identity saved to ${identityPath}, but the profile could not be saved. Keep this identity and use LIBRECHAT_CODE_IDENTITY_FILE with the existing run command; do not pair again automatically.`, + ); + } + return { profilePath, profile }; + } finally { + await identityReservation.release(); + } + } finally { + await profileReservation.release(); + } +} + +export interface ProfileRunConfiguration { + args: string[]; + profilePath?: string; + profile?: WorkerProfile; +} + +export async function profileRunConfiguration( + args: string[], + environment: NodeJS.ProcessEnv = process.env, +): Promise { + const profileArgs = args.filter( + arg => arg === '--profile' || arg.startsWith('--profile='), + ); + if (profileArgs.length > 1) throw new Error('Supply --profile only once'); + let profilePath = environment.LIBRECHAT_CODE_PROFILE_FILE?.trim(); + if (profileArgs.length) { + const flag = profileArgs[0]; + const value = + flag === '--profile' + ? args[args.indexOf(flag) + 1] + : flag.slice('--profile='.length); + if (!value || value.startsWith('--')) + throw new Error('--profile requires a file'); + profilePath = value; + args = args.filter( + (_, index) => + index !== args.indexOf(flag) && + (flag !== '--profile' || index !== args.indexOf(flag) + 1), + ); + } + const legacyConfiguration = Object.entries(environment).some( + ([key, value]) => + key.startsWith('LIBRECHAT_CODE_') && + key !== 'LIBRECHAT_CODE_PROFILE_FILE' && + value?.trim(), + ); + if (legacyConfiguration) { + if (profilePath) + throw new Error( + 'Saved profiles cannot be combined with LIBRECHAT_CODE_* settings; unset them or use the existing run configuration', + ); + return { args }; + } + const path = resolve(profilePath ?? defaultWorkerProfilePath()); + const profile = await loadWorkerProfile(path); + if (!profile) { + if (profilePath) + throw new Error( + 'Saved profile not found; run librechat-code setup first', + ); + return { args }; + } + if (args.length) + throw new Error( + 'Saved profiles cannot be combined with run options; use a separate profile or the existing environment-variable configuration', + ); + return { profilePath: path, profile, args: workerProfileArguments(profile) }; +} diff --git a/packages/code/src/storage.ts b/packages/code/src/storage.ts index 904a8562..578f9fb6 100644 --- a/packages/code/src/storage.ts +++ b/packages/code/src/storage.ts @@ -11,7 +11,7 @@ import { } from 'node:fs/promises'; import type { FileHandle } from 'node:fs/promises'; import { homedir } from 'node:os'; -import { dirname, join, resolve } from 'node:path'; +import { dirname, isAbsolute, join, resolve } from 'node:path'; import { BRIDGE_PROTOCOL_VERSION, BridgeProtocolError } from './protocol.js'; import { assertPrivateStorageAcl, removePrivateStorageAcl, assertPrivateStorageAncestors, assertPrivateStorageSupported } from './private-storage.js'; @@ -34,7 +34,10 @@ function isPairedIdentity(value: unknown): value is PairedBridgeWorkerIdentity { typeof value.expiresAt === 'string' && Number.isFinite(Date.parse(value.expiresAt)) && typeof value.publicKey === 'string' && - typeof value.privateKey === 'string' + typeof value.privateKey === 'string' && + (value.setupAttemptPath === undefined || + (typeof value.setupAttemptPath === 'string' && isAbsolute(value.setupAttemptPath) && + Buffer.byteLength(value.setupAttemptPath) <= 4096 && !/[\x00-\x1f\x7f]/.test(value.setupAttemptPath))) ); } @@ -223,7 +226,18 @@ async function readGuardedFile( if ((mode & 0o077) !== 0) throw new BridgeProtocolError(exposed(mode.toString(8))); } await assertPrivateStorageAcl(handle, path); - return await handle.readFile('utf8'); + if (!Number.isFinite(maxBytes)) return await handle.readFile('utf8'); + const buffer = Buffer.alloc(maxBytes + 1); + let length = 0; + while (length < buffer.length) { + const { bytesRead } = await handle.read(buffer, length, buffer.length - length, null); + if (bytesRead === 0) break; + length += bytesRead; + } + if (length > maxBytes) { + throw new BridgeProtocolError('Invalid private storage file size or type'); + } + return buffer.toString('utf8', 0, length); } finally { await handle.close(); } @@ -360,6 +374,7 @@ async function assertSiblingPublishable(path: string): Promise { export async function assertIdentityPathIsPrivate( path: string, + createOnly = false, ): Promise { assertPrivateStorageSupported(); await assertWriteContainerPrivate(path); @@ -389,6 +404,11 @@ export async function assertIdentityPathIsPrivate( throw error; } if (!isRecord(error) || error.code !== 'EEXIST') throw error; + if (createOnly) { + throw new BridgeProtocolError( + `${path} already exists; choose a different identity/profile path.`, + ); + } /* Already present: judge what is there instead of the placeholder, and * prove the publish itself is possible - an unwritable parent would * otherwise surface as EACCES only after the code was spent. */ @@ -465,7 +485,25 @@ export async function saveEnvironmentPreparationKey(path: string, key: string): await savePrivateJson(path, { version: 1, key }); } -async function savePrivateJson(path: string, value: unknown): Promise { +export async function readPrivateJson(path: string, maxBytes: number): Promise { + assertPrivateStorageSupported(); + if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) { + throw new BridgeProtocolError('Private JSON file limit must be a positive integer'); + } + const content = await readGuardedFile( + path, + (mode) => `Private JSON file ${path} must be owner-only (mode ${mode})`, + maxBytes, + ); + await assertReadPathPrivate(path); + try { + return JSON.parse(content) as unknown; + } catch { + throw new BridgeProtocolError(`Invalid private JSON file: ${path}`); + } +} + +export async function savePrivateJson(path: string, value: unknown): Promise { assertPrivateStorageSupported(); await assertWriteContainerPrivate(path); await mkdir(dirname(path), { recursive: true, mode: 0o700 }); diff --git a/packages/code/src/worker-profile.test.ts b/packages/code/src/worker-profile.test.ts new file mode 100644 index 00000000..585144c0 --- /dev/null +++ b/packages/code/src/worker-profile.test.ts @@ -0,0 +1,529 @@ +import assert from 'node:assert/strict'; +import { + chmod, + link, + mkdir, + mkdtemp, + open, + readFile, + readdir, + rm, + stat, + symlink, + writeFile, +} from 'node:fs/promises'; +import type { FileHandle } from 'node:fs/promises'; +import { homedir, tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { assertIdentityPathIsPrivate } from './storage.js'; +import { + assertWorkerProfileIsolation, + defaultWorkerProfilePath, + loadWorkerProfile, + saveWorkerProfile, + workerProfileArguments, +} from './worker-profile.js'; +import type { WorkerProfile } from './worker-profile.js'; + +function profile(directory: string): WorkerProfile { + return { + version: 1, + identityPath: join(directory, 'identity.json'), + workspace: { type: 'private' }, + allowWorkspaceWrites: false, + allowWorkspaceCommands: false, + }; +} + +test('the default worker profile is separate from identity storage', () => { + assert.equal( + defaultWorkerProfilePath(), + join(homedir(), '.config', 'librechat', 'code', 'profiles', 'default.json'), + ); +}); + +test('worker profiles roundtrip atomically with owner-only file and directory permissions', async t => { + const directory = await mkdtemp(join(tmpdir(), 'worker-profile-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profiles', 'default.json'); + const value = profile(directory); + await saveWorkerProfile(path, value); + assert.deepEqual(await loadWorkerProfile(path), value); + assert.deepEqual(JSON.parse(await readFile(path, 'utf8')), value); + assert.equal((await stat(path)).mode & 0o777, 0o600); + assert.equal((await stat(join(directory, 'profiles'))).mode & 0o777, 0o700); + + const updated: WorkerProfile = { + ...value, + workspace: { + type: 'directory', + path: join(directory, 'project with spaces'), + }, + allowWorkspaceWrites: true, + allowWorkspaceCommands: true, + }; + await saveWorkerProfile(path, updated); + assert.deepEqual(await loadWorkerProfile(path), updated); + assert.equal((await stat(path)).mode & 0o777, 0o600); + assert.deepEqual(await readdir(join(directory, 'profiles')), [ + 'default.json', + ]); +}); + +test('worker profile arguments enable only explicitly granted permissions', () => { + for (const workspace of [ + { type: 'private' as const }, + { type: 'directory' as const, path: '/srv/project with spaces' }, + ]) { + for (const allowWorkspaceWrites of [false, true]) { + for (const allowWorkspaceCommands of [false, true]) { + const value = { + ...profile('/srv'), + workspace, + allowWorkspaceWrites, + allowWorkspaceCommands, + }; + assert.deepEqual(workerProfileArguments(value), [ + ...(workspace.type === 'private' + ? ['--default-workspace'] + : ['--worker-dir', workspace.path]), + ...(allowWorkspaceWrites ? ['--allow-workspace-writes'] : []), + ...(allowWorkspaceCommands ? ['--allow-workspace-commands'] : []), + ]); + } + } + } +}); + +test('only an absent worker profile returns undefined', async t => { + const directory = await mkdtemp(join(tmpdir(), 'worker-profile-missing-')); + t.after(() => rm(directory, { recursive: true, force: true })); + assert.equal( + await loadWorkerProfile(join(directory, 'absent.json')), + undefined, + ); + assert.equal( + await loadWorkerProfile(join(directory, 'missing', 'absent.json')), + undefined, + ); + await assert.rejects(loadWorkerProfile(directory), /size or type/); + + const dangling = join(directory, 'dangling.json'); + await symlink(join(directory, 'absent.json'), dangling); + await assert.rejects(loadWorkerProfile(dangling), { code: 'ENOENT' }); + const loop = join(directory, 'loop.json'); + await symlink(loop, loop); + await assert.rejects(loadWorkerProfile(loop), /Too many storage symlinks/); +}); + +test('malformed, empty, and oversized profiles fail without disclosing their contents', async t => { + const directory = await mkdtemp( + join(tmpdir(), 'worker-profile-invalid-json-'), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + for (const content of ['', ' ', '{"credential":"do-not-disclose",']) { + await writeFile(path, content, { mode: 0o600 }); + await assert.rejects(loadWorkerProfile(path), (error: Error) => { + assert.match(error.message, /Invalid private JSON file/); + assert.doesNotMatch(error.message, /do-not-disclose/); + return true; + }); + } + await writeFile( + path, + `${JSON.stringify(profile(directory))}${' '.repeat(32 * 1024)}`, + { mode: 0o600 }, + ); + await assert.rejects(loadWorkerProfile(path), /size or type/); +}); + +test('profiles reject unknown fields, unsupported versions, invalid paths, and implicit grants', async t => { + const directory = await mkdtemp(join(tmpdir(), 'worker-profile-fields-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + const value = profile(directory); + const cases: [string, unknown][] = [ + ['null', null], + ['array', []], + ['unsupported version', { ...value, version: 2 }], + ['unknown root field', { ...value, credential: 'not-allowed' }], + [ + 'unknown private workspace field', + { ...value, workspace: { type: 'private', path: '/srv' } }, + ], + [ + 'unknown directory workspace field', + { ...value, workspace: { type: 'directory', path: '/srv', extra: true } }, + ], + ['unknown workspace type', { ...value, workspace: { type: 'shared' } }], + ['missing workspace path', { ...value, workspace: { type: 'directory' } }], + ['relative identity path', { ...value, identityPath: 'identity.json' }], + [ + 'relative workspace path', + { ...value, workspace: { type: 'directory', path: './project' } }, + ], + ['empty identity path', { ...value, identityPath: '' }], + ['NUL identity path', { ...value, identityPath: '/srv/identity\0.json' }], + [ + 'NUL workspace path', + { ...value, workspace: { type: 'directory', path: '/srv/project\0' } }, + ], + [ + 'oversized identity path', + { ...value, identityPath: `/${'x'.repeat(4096)}` }, + ], + [ + 'oversized workspace path', + { + ...value, + workspace: { type: 'directory', path: `/${'é'.repeat(2048)}` }, + }, + ], + [ + 'missing write permission', + { + version: 1, + identityPath: value.identityPath, + workspace: value.workspace, + allowWorkspaceCommands: false, + }, + ], + ['string write permission', { ...value, allowWorkspaceWrites: 'true' }], + ['numeric command permission', { ...value, allowWorkspaceCommands: 1 }], + ]; + for (const [name, invalid] of cases) { + await t.test(name, async () => { + const serialized = JSON.stringify(invalid); + await writeFile(path, serialized, { mode: 0o600 }); + await assert.rejects(loadWorkerProfile(path), /worker profile/i); + await assert.rejects( + saveWorkerProfile(path, invalid as WorkerProfile), + /worker profile/i, + ); + assert.equal(await readFile(path, 'utf8'), serialized); + assert.throws( + () => workerProfileArguments(invalid as WorkerProfile), + /worker profile/i, + ); + }); + } + await assert.rejects( + loadWorkerProfile('relative.json'), + /paths must be absolute/, + ); + await assert.rejects( + saveWorkerProfile('relative.json', value), + /paths must be absolute/, + ); +}); + +test('save rejects non-JSON fields and bounds the serialized profile size', async t => { + const directory = await mkdtemp( + join(tmpdir(), 'worker-profile-serialization-'), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + const value = profile(directory); + await assert.rejects( + saveWorkerProfile(path, { ...value, ignored: undefined } as WorkerProfile), + /fields/, + ); + await assert.rejects( + saveWorkerProfile(path, { ...value, [Symbol('ignored')]: true }), + /fields/, + ); + await assert.rejects( + saveWorkerProfile(path, { ...value, toJSON: () => value } as WorkerProfile), + /fields/, + ); + await assert.rejects( + saveWorkerProfile(path, { + ...value, + identityPath: `/${'"'.repeat(4095)}`, + workspace: { type: 'directory', path: `/${'"'.repeat(4095)}` }, + }), + /16 KiB limit/, + ); + await assert.rejects(stat(path), { code: 'ENOENT' }); +}); + +test('private profile reads remain bounded if the file grows after its size check', async t => { + const directory = await mkdtemp(join(tmpdir(), 'worker-profile-growth-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + await saveWorkerProfile(path, profile(directory)); + const probe = await open(path, 'r'); + const prototype = Object.getPrototypeOf(probe) as FileHandle; + await probe.close(); + const originalRead = prototype.read; + let grown = false; + t.mock.method( + prototype, + 'read', + async function (this: FileHandle, ...args: unknown[]) { + if (!grown) { + grown = true; + await writeFile( + path, + `${JSON.stringify(profile(directory))}${' '.repeat(32 * 1024)}`, + ); + } + return Reflect.apply(originalRead, this, args); + }, + ); + await assert.rejects(loadWorkerProfile(path), /size or type/); + assert.equal(grown, true); +}); + +test('permission errors are not mistaken for an absent profile', async t => { + const directory = await mkdtemp(join(tmpdir(), 'worker-profile-permission-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + await saveWorkerProfile(path, profile(directory)); + const probe = await open(path, 'r'); + const prototype = Object.getPrototypeOf(probe) as FileHandle; + await probe.close(); + t.mock.method(prototype, 'read', async () => { + throw Object.assign(new Error('Permission denied'), { code: 'EACCES' }); + }); + await assert.rejects(loadWorkerProfile(path), { code: 'EACCES' }); +}); + +test('exposed profiles and unsafe storage ancestry are rejected', async t => { + const directory = await mkdtemp(join(tmpdir(), 'worker-profile-modes-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + const value = profile(directory); + await saveWorkerProfile(path, value); + await chmod(path, 0o644); + await assert.rejects(loadWorkerProfile(path), /owner-only.*644/); + await chmod(path, 0o600); + await chmod(directory, 0o777); + await assert.rejects(loadWorkerProfile(path), /writable by other accounts/); + await assert.rejects( + loadWorkerProfile(join(directory, 'missing.json')), + /writable by other accounts/, + ); + await assert.rejects( + saveWorkerProfile(path, value), + /writable by other accounts/, + ); +}); + +test('worker profile storage fails closed on unsupported platforms', async t => { + const descriptor = Object.getOwnPropertyDescriptor(process, 'platform')!; + t.after(() => Object.defineProperty(process, 'platform', descriptor)); + Object.defineProperty(process, 'platform', { + value: 'freebsd', + configurable: true, + }); + const value = profile('/srv'); + await assert.rejects( + loadWorkerProfile('/srv/missing.json'), + /unavailable on this platform/, + ); + await assert.rejects( + saveWorkerProfile('/srv/profile.json', value), + /unavailable on this platform/, + ); + await assert.rejects( + assertWorkerProfileIsolation('/srv/profile.json', value.identityPath), + /unavailable on this platform/, + ); +}); + +test('isolation accepts separate reserved files outside the workspace', async t => { + const directory = await mkdtemp(join(tmpdir(), 'worker-profile-isolation-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + const identity = join(directory, 'identity.json'); + const workspace = join(directory, 'workspace'); + await mkdir(workspace); + const reservedProfile = await assertIdentityPathIsPrivate(path, true); + const reservedIdentity = await assertIdentityPathIsPrivate(identity, true); + await assertWorkerProfileIsolation(path, identity); + await assertWorkerProfileIsolation(path, identity, workspace); + await reservedIdentity.release(); + await reservedProfile.release(); + assert.deepEqual(await readdir(directory), ['workspace']); +}); + +test('isolation rejects lexical, symlink, and hardlink aliases of the identity', async t => { + const directory = await mkdtemp(join(tmpdir(), 'worker-profile-alias-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const identity = join(directory, 'identity.json'); + await writeFile(identity, '', { mode: 0o600 }); + const symlinkPath = join(directory, 'symlink.json'); + const hardlinkPath = join(directory, 'hardlink.json'); + await symlink(identity, symlinkPath); + await link(identity, hardlinkPath); + for (const path of [ + identity, + `${directory}/./identity.json`, + symlinkPath, + hardlinkPath, + ]) { + await assert.rejects( + assertWorkerProfileIsolation(path, identity), + /separate files, not aliases/, + ); + } +}); + +test('isolation rejects either private file inside a read-only workspace', async t => { + const directory = await mkdtemp( + join(tmpdir(), 'worker-profile-containment-'), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const workspace = join(directory, 'workspace'); + await mkdir(workspace); + const profilePath = join(directory, 'profile.json'); + const identity = join(directory, 'identity.json'); + const inside = join(workspace, 'storage.json'); + for (const path of [profilePath, identity, inside]) + await writeFile(path, '', { mode: 0o600 }); + await assert.rejects( + assertWorkerProfileIsolation(inside, identity, workspace), + /outside the workspace/, + ); + await assert.rejects( + assertWorkerProfileIsolation(profilePath, inside, workspace), + /outside the workspace/, + ); + const workspaceAlias = join(directory, 'workspace-alias'); + await symlink(workspace, workspaceAlias); + await assert.rejects( + assertWorkerProfileIsolation(profilePath, inside, workspaceAlias), + /outside the workspace/, + ); +}); + +test('isolation checks every intermediate symlink and traversed storage ancestor', async t => { + const directory = await mkdtemp( + join(tmpdir(), 'worker-profile-link-ancestry-'), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const workspace = join(directory, 'workspace'); + const outside = join(directory, 'workspace-private'); + await mkdir(workspace); + await mkdir(outside); + const profilePath = join(outside, 'profile.json'); + const identity = join(outside, 'identity.json'); + await writeFile(profilePath, '', { mode: 0o600 }); + await writeFile(identity, '', { mode: 0o600 }); + await assertWorkerProfileIsolation(profilePath, identity, workspace); + + const linkInWorkspace = join(workspace, 'storage'); + const indirect = join(directory, 'indirect'); + await symlink(outside, linkInWorkspace); + await symlink(linkInWorkspace, indirect); + const cases = [ + join(linkInWorkspace, 'profile.json'), + join(indirect, 'profile.json'), + `${workspace}/../workspace-private/profile.json`, + `${indirect}/../workspace-private/profile.json`, + ]; + for (const path of cases) { + await assert.rejects( + assertWorkerProfileIsolation(path, identity, workspace), + /outside the workspace/, + ); + } + await assert.rejects( + assertWorkerProfileIsolation( + profilePath, + join(indirect, 'identity.json'), + workspace, + ), + /outside the workspace/, + ); + const fileLink = join(workspace, 'identity-link.json'); + await symlink(identity, fileLink); + await assert.rejects( + assertWorkerProfileIsolation(profilePath, fileLink, workspace), + /outside the workspace/, + ); +}); + +test('isolation fails on missing or non-file storage and missing or non-directory workspaces', async t => { + const directory = await mkdtemp( + join(tmpdir(), 'worker-profile-isolation-invalid-'), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + const identity = join(directory, 'identity.json'); + await writeFile(path, '', { mode: 0o600 }); + await writeFile(identity, '', { mode: 0o600 }); + await assert.rejects( + assertWorkerProfileIsolation(join(directory, 'missing.json'), identity), + { code: 'ENOENT' }, + ); + await assert.rejects( + assertWorkerProfileIsolation(directory, identity), + /regular files/, + ); + await assert.rejects( + assertWorkerProfileIsolation(path, identity, join(directory, 'missing')), + { code: 'ENOENT' }, + ); + await assert.rejects( + assertWorkerProfileIsolation(path, identity, path), + /must be a directory/, + ); +}); + +test('create-only reservations never overwrite existing profiles or identities', async t => { + const directory = await mkdtemp( + join(tmpdir(), 'worker-profile-create-only-'), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + const value = profile(directory); + await saveWorkerProfile(path, value); + const before = await stat(path, { bigint: true }); + await assert.rejects( + assertIdentityPathIsPrivate(path, true), + /already exists; choose a different identity\/profile path/, + ); + assert.equal((await stat(path, { bigint: true })).ino, before.ino); + assert.deepEqual(await loadWorkerProfile(path), value); + const existing = await assertIdentityPathIsPrivate(path); + await existing.release(); + assert.deepEqual(await loadWorkerProfile(path), value); +}); + +test('only one concurrent create-only reservation can claim a path', async t => { + const directory = await mkdtemp( + join(tmpdir(), 'worker-profile-reservations-'), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + const results = await Promise.allSettled([ + assertIdentityPathIsPrivate(path, true), + assertIdentityPathIsPrivate(path, true), + ]); + const successes = results.filter(result => result.status === 'fulfilled'); + const failures = results.filter(result => result.status === 'rejected'); + assert.equal(successes.length, 1); + assert.equal(failures.length, 1); + assert.match(String(failures[0].reason), /already exists/); + assert.equal((await stat(path)).size, 0); + await successes[0].value.release(); + await assert.rejects(stat(path), { code: 'ENOENT' }); +}); + +test('reservation cleanup preserves a profile published over its placeholder', async t => { + const directory = await mkdtemp( + join(tmpdir(), 'worker-profile-reservation-release-'), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + const reservation = await assertIdentityPathIsPrivate(path, true); + const value = profile(directory); + await saveWorkerProfile(path, value); + await reservation.release(); + assert.deepEqual(await loadWorkerProfile(path), value); +}); diff --git a/packages/code/src/worker-profile.ts b/packages/code/src/worker-profile.ts new file mode 100644 index 00000000..a466a5d1 --- /dev/null +++ b/packages/code/src/worker-profile.ts @@ -0,0 +1,200 @@ +import { lstat, realpath, stat } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { isAbsolute, join, relative, sep } from 'node:path'; + +import { assertPrivateStorageAncestors } from './private-storage.js'; +import { BridgeProtocolError } from './protocol.js'; +import { readPrivateJson, savePrivateJson } from './storage.js'; + +const MAX_PROFILE_BYTES = 16 * 1024; +const MAX_PATH_BYTES = 4096; + +export interface WorkerProfile { + version: 1; + identityPath: string; + workspace: { type: 'private' } | { type: 'directory'; path: string }; + allowWorkspaceWrites: boolean; + allowWorkspaceCommands: boolean; +} + +function hasExactKeys( + value: unknown, + keys: string[], +): value is Record { + return ( + typeof value === 'object' && + value !== null && + !Array.isArray(value) && + Reflect.ownKeys(value).length === keys.length && + keys.every(key => Object.hasOwn(value, key)) + ); +} + +function assertProfilePath(value: unknown): asserts value is string { + if ( + typeof value !== 'string' || + !isAbsolute(value) || + /[\x00-\x1f\x7f]/.test(value) || + Buffer.byteLength(value, 'utf8') > MAX_PATH_BYTES + ) { + throw new BridgeProtocolError( + 'Worker profile paths must be absolute, contain no control characters, and be at most 4096 bytes', + ); + } +} + +function parseWorkerProfile(value: unknown): WorkerProfile { + if ( + !hasExactKeys(value, [ + 'version', + 'identityPath', + 'workspace', + 'allowWorkspaceWrites', + 'allowWorkspaceCommands', + ]) || + value.version !== 1 || + typeof value.allowWorkspaceWrites !== 'boolean' || + typeof value.allowWorkspaceCommands !== 'boolean' + ) { + throw new BridgeProtocolError( + 'Invalid worker profile fields or unsupported version', + ); + } + assertProfilePath(value.identityPath); + let workspace: WorkerProfile['workspace']; + if ( + hasExactKeys(value.workspace, ['type']) && + value.workspace.type === 'private' + ) { + workspace = { type: 'private' }; + } else if ( + hasExactKeys(value.workspace, ['type', 'path']) && + value.workspace.type === 'directory' + ) { + assertProfilePath(value.workspace.path); + workspace = { type: 'directory', path: value.workspace.path }; + } else { + throw new BridgeProtocolError('Invalid worker profile workspace'); + } + return { + version: 1, + identityPath: value.identityPath, + workspace, + allowWorkspaceWrites: value.allowWorkspaceWrites, + allowWorkspaceCommands: value.allowWorkspaceCommands, + }; +} + +export function defaultWorkerProfilePath(): string { + return join( + homedir(), + '.config', + 'librechat', + 'code', + 'profiles', + 'default.json', + ); +} + +export async function loadWorkerProfile( + path: string, +): Promise { + assertProfilePath(path); + let value: unknown; + try { + value = await readPrivateJson(path, MAX_PROFILE_BYTES); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + // A dangling leaf symlink is not an absent profile. + const entry = await lstat(path).catch((failure: NodeJS.ErrnoException) => { + if (failure.code === 'ENOENT') return undefined; + throw failure; + }); + if (entry !== undefined) throw error; + return undefined; + } + return parseWorkerProfile(value); +} + +export async function saveWorkerProfile( + path: string, + profile: WorkerProfile, +): Promise { + assertProfilePath(path); + const value = parseWorkerProfile(profile); + if ( + Buffer.byteLength(`${JSON.stringify(value, null, 2)}\n`, 'utf8') > + MAX_PROFILE_BYTES + ) { + throw new BridgeProtocolError('Worker profile exceeds the 16 KiB limit'); + } + await savePrivateJson(path, value); +} + +export function workerProfileArguments(profile: WorkerProfile): string[] { + const value = parseWorkerProfile(profile); + const args = + value.workspace.type === 'private' + ? ['--default-workspace'] + : ['--worker-dir', value.workspace.path]; + if (value.allowWorkspaceWrites) args.push('--allow-workspace-writes'); + if (value.allowWorkspaceCommands) args.push('--allow-workspace-commands'); + return args; +} + +function isWithin(root: string, path: string): boolean { + const suffix = relative(root, path); + return ( + suffix === '' || + (suffix !== '..' && !suffix.startsWith(`..${sep}`) && !isAbsolute(suffix)) + ); +} + +export async function assertWorkerProfileIsolation( + profilePath: string, + identityPath: string, + workspaceRoot?: string, +): Promise { + assertProfilePath(profilePath); + assertProfilePath(identityPath); + if (workspaceRoot !== undefined) assertProfilePath(workspaceRoot); + const [profileAncestors, identityAncestors] = await Promise.all([ + assertPrivateStorageAncestors(profilePath), + assertPrivateStorageAncestors(identityPath), + ]); + const [profile, identity, profileMetadata, identityMetadata] = + await Promise.all([ + realpath(profilePath), + realpath(identityPath), + stat(profilePath, { bigint: true }), + stat(identityPath, { bigint: true }), + ]); + if (!profileMetadata.isFile() || !identityMetadata.isFile()) { + throw new BridgeProtocolError( + 'Worker profile and identity paths must be regular files', + ); + } + if ( + profile === identity || + (profileMetadata.dev === identityMetadata.dev && + profileMetadata.ino === identityMetadata.ino) + ) { + throw new BridgeProtocolError( + 'Worker profile and identity must use separate files, not aliases', + ); + } + if (workspaceRoot === undefined) return; + const root = await realpath(workspaceRoot); + if (!(await stat(root)).isDirectory()) { + throw new BridgeProtocolError('Worker workspace root must be a directory'); + } + if ( + [profile, identity, ...profileAncestors, ...identityAncestors].some(path => + isWithin(root, path), + ) + ) { + throw new BridgeProtocolError( + 'Worker profile, identity, and their storage ancestry must stay outside the workspace', + ); + } +} diff --git a/packages/code/src/worker.test.ts b/packages/code/src/worker.test.ts index e26f212a..b555c1cd 100644 --- a/packages/code/src/worker.test.ts +++ b/packages/code/src/worker.test.ts @@ -75,11 +75,12 @@ test('worker confirms readiness only after local registration activation succeed onRegistered: async () => { events.push('activated'); }, + onReady: () => { events.push('observed-ready'); }, }); await worker.register(); - assert.deepEqual(events, ['registered', 'activated', 'ready']); + assert.deepEqual(events, ['registered', 'activated', 'ready', 'observed-ready']); }); test('worker does not confirm readiness when local activation fails', async () => { @@ -110,12 +111,29 @@ test('worker does not confirm readiness when local activation fails', async () = onRegistered: async () => { throw new Error('relay activation failed'); }, + onReady: () => { assert.fail('failed activation is not readiness'); }, }); await assert.rejects(worker.register(), /relay activation failed/); assert.equal(readinessRequests, 0); }); +test('worker never reports ready for an absent or negative readiness acknowledgement', async () => { + for (const ready of [undefined, false]) { + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'nsjail', runtimes: ['bash'], requiresReadyConfirmation: true }, + fetchImpl: async input => Response.json(String(input).endsWith('/ready') + ? { protocolVersion: 1, ready } + : { protocolVersion: 1, workerId: 'vm-1', incarnationId, registrationGeneration: 1, + registeredAt: new Date().toISOString(), leaseTtlMs: 60_000 }), + onReady: () => { assert.fail('readiness was not confirmed'); }, + }); + await assert.rejects(worker.register(), /did not confirm worker readiness/); + } +}); + test('worker forwards a fenced assignment to the sandbox and settles the result', async () => { const requests: Array<{ url: string; init?: RequestInit }> = []; const fetchImpl: typeof fetch = async (input, init) => { diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 49b96eb1..c2ae9924 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -7,6 +7,7 @@ import { bridgeWorkerPath, isBridgeWorkspaceProgrammaticRequest, isValidLinkedWorktreeName, + isWorkspaceMutationOperation, isWorkspaceToolResult, workspaceIsolationKey, workspaceIsolationKeysConflict, @@ -15,7 +16,10 @@ import { } from './protocol.js'; import { EndpointRuntimeSupervisor } from './runtime.js'; import { signBridgeRequest } from './identity.js'; +import { recoverWorkerCredential } from './credential-recovery.js'; import { isWorkspaceToolRequest, WorkspaceToolError } from './workspace.js'; +import { commandFailureEvent } from './command-diagnostics.js'; +import type { CommandFailureEvent } from './command-diagnostics.js'; import type { WorkspaceCommandResultFeature, @@ -93,10 +97,14 @@ export interface BridgeWorkerOptions { credentialRefreshTransportTimeoutMs?: number; fetchImpl?: typeof fetch; onError?: (error: unknown) => void; + /** Local-only, allowlisted command failure metadata; never contains request data. */ + onCommandFailure?: (event: CommandFailureEvent) => void | Promise; onIdentityChange?: (identity: BridgeWorkerIdentity) => void | Promise; onRegistered?: ( registration: BridgeWorkerRegistrationResponse, ) => void | Promise; + /** Called only after registration and required readiness confirmation succeed. */ + onReady?: () => void | Promise; incarnationId?: string; } @@ -561,10 +569,7 @@ export class BridgeWorker { } if ( options.capabilities.workspaceTools?.operations.some( - (operation) => - operation === 'write_file' || - operation === 'edit_file' || - operation === 'execute_command', + isWorkspaceMutationOperation, ) === true && options.workspaceMutationQuarantine == null && options.workspaceQuarantines == null && @@ -839,6 +844,7 @@ export class BridgeWorker { await this.confirmReady(registration, signal); } this.lastRegisteredAtMs = registrationStartedAtMs; + if (!this.maintenanceOnly) await this.options.onReady?.(); return registration; } @@ -855,7 +861,7 @@ export class BridgeWorker { 'Code API does not support explicit worker readiness confirmation', ); } - await this.timedRequest( + const ready = await this.timedRequest<{ ready?: boolean }>( `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}/ready`, { protocolVersion: BRIDGE_PROTOCOL_VERSION, @@ -869,6 +875,9 @@ export class BridgeWorker { ), signal, ); + if (ready.ready !== true) { + throw new BridgeProtocolError('Code API did not confirm worker readiness'); + } } async resetWorkspace( @@ -1317,20 +1326,49 @@ export class BridgeWorker { if (Date.parse(identity.expiresAt) > validThroughMs) { return; } - const credential = await this.timedRequest( - `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}` + - '/credentials/refresh', - { protocolVersion: BRIDGE_PROTOCOL_VERSION }, - Math.max( - 1, - Math.min( - transportTimeoutMs, - this.options.credentialRefreshTransportTimeoutMs ?? - DEFAULT_CONTROL_TRANSPORT_TIMEOUT_MS, - ), + const timeoutMs = Math.max( + 1, + Math.min( + transportTimeoutMs, + this.options.credentialRefreshTransportTimeoutMs ?? + DEFAULT_CONTROL_TRANSPORT_TIMEOUT_MS, ), - signal, ); + const deadline = performance.now() + timeoutMs; + let credential: BridgeWorkerCredentialResponse; + try { + credential = await this.timedRequest( + `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}/credentials/refresh`, + { protocolVersion: BRIDGE_PROTOCOL_VERSION }, + timeoutMs, + signal, + ); + } catch (error) { + // Only bootstrap recovery. Auth failures during active work retain existing fencing/settlement rules. + if ( + this.lastRegisteredAtMs !== 0 || + !(error instanceof BridgeProtocolError) || + error.status !== 401 || + error.code !== 'CREDENTIAL_INVALID' + ) throw error; + signal?.throwIfAborted(); + const remainingMs = Math.floor(deadline - performance.now()); + if (remainingMs <= 0) { + throw new DOMException('Credential recovery timed out', 'TimeoutError'); + } + const recoverySignal = AbortSignal.timeout(remainingMs); + credential = await recoverWorkerCredential({ + codeApiUrl: this.codeApiUrl, + workerId: this.options.workerId, + privateKey: identity.privateKey, + validThroughMs, + fetchImpl: this.fetchImpl, + signal: signal == null + ? recoverySignal + : AbortSignal.any([signal, recoverySignal]), + }); + } + signal?.throwIfAborted(); if ( credential.protocolVersion !== BRIDGE_PROTOCOL_VERSION || credential.workerId !== this.options.workerId || @@ -1849,10 +1887,9 @@ export class BridgeWorker { 'Workspace listing feature is not advertised', ); } - const isMutation = - workspaceRequest.operation === 'write_file' || - workspaceRequest.operation === 'edit_file' || - workspaceRequest.operation === 'execute_command'; + const isMutation = isWorkspaceMutationOperation( + workspaceRequest.operation, + ); if (isMutation) { this.mutationGuardArmed = true; try { @@ -2081,6 +2118,35 @@ export class BridgeWorker { result: payload, }; } catch (error) { + if ( + error instanceof WorkspaceToolError && + error.code === 'COMMAND_UNAVAILABLE' && + (assignment.executionKind === 'workspace_programmatic' || + (assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) && + assignment.request.operation === 'execute_command')) + ) { + try { + const event = commandFailureEvent(error.commandDiagnostic, { + assignmentId: assignment.assignmentId, + workerId: this.options.workerId, + incarnationId: this.incarnationId, + workspaceId: assignment.executionKind === 'workspace_programmatic' + ? assignment.workspaceId + : (isWorkspaceToolRequest(assignment.request) ? assignment.request.workspaceId : undefined), + operation: assignment.executionKind === 'workspace_programmatic' + ? 'workspace_programmatic' + : 'execute_command', + mutationMayHaveCommitted: error.mutationMayHaveCommitted, + requiresQuarantine: error.requiresQuarantine, + }); + void Promise.resolve(this.options.onCommandFailure?.(event)).catch(() => { + // A rejected asynchronous log sink must not interrupt settlement. + }); + } catch { + // Observability must never change settlement or quarantine decisions. + } + } if ( error instanceof BridgeWorkspaceQuarantinedError && !workspaceMutationArmed @@ -2530,9 +2596,7 @@ export class BridgeWorker { (assignment.executionKind === 'workspace_programmatic' || (assignment.executionKind === 'workspace_tool' && isWorkspaceToolRequest(assignment.request) && - (assignment.request.operation === 'write_file' || - assignment.request.operation === 'edit_file' || - assignment.request.operation === 'execute_command'))); + isWorkspaceMutationOperation(assignment.request.operation))); if (signal?.aborted === true) { if (assignment.runtimeSessionId != null || fulfilledWorkspaceMutation) { throw await this.quarantineWorkspace( diff --git a/packages/code/src/workspace-import.test.ts b/packages/code/src/workspace-import.test.ts new file mode 100644 index 00000000..18d37b86 --- /dev/null +++ b/packages/code/src/workspace-import.test.ts @@ -0,0 +1,565 @@ +import assert from 'node:assert/strict'; +import { createHash, randomBytes } from 'node:crypto'; +import { createServer } from 'node:http'; +import { chmod, lstat, mkdir, mkdtemp, readdir, readFile, realpath, rename, rm, stat, symlink, writeFile } from 'node:fs/promises'; +import { setTimeout as delay } from 'node:timers/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import type { IncomingMessage, ServerResponse } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import type { TestContext } from 'node:test'; + +import { + isBridgeWorkspaceImportFileRequest, + isRelayedWorkspaceImportFileRequest, + isValidBridgeWorkspaceToolCapabilities, + isWorkspaceImportFilePath, + isWorkspaceToolRequest, + isWorkspaceToolResult, +} from './protocol.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import { BridgeWorker } from './worker.js'; +import { createWorkspaceFileImporter, HttpWorkspaceFileImporter } from './workspace-import.js'; +import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; + +import type { WorkspaceImportFileRequest, WorkspaceToolRequest } from './protocol.js'; +import type { HttpWorkspaceFileImporterOptions } from './workspace-import.js'; + +const GRANT = 'ceg1.grant-for-one-object'; +const SESSION = 'session-handle'; +const OBJECT = 'object-handle'; +const incarnationId = 'incarnation-00000001'; + +const sha256 = (bytes: Uint8Array): string => createHash('sha256').update(bytes).digest('hex'); + +function importRequest( + bytes: Buffer, + overrides: Partial = {}, +): WorkspaceImportFileRequest { + return { + protocolVersion: 1, + operation: 'import_file', + workspaceId: 'primary', + path: 'uploads/archive.zip', + source: { storage_session_id: SESSION, id: OBJECT }, + bytes: bytes.byteLength, + sha256: sha256(bytes), + egressGrant: GRANT, + ...overrides, + }; +} + +interface EgressStub { + url: string; + hits: number; + body: Buffer; + /** Replace the default response (exact body, declared length, valid grant). */ + respond?: (req: IncomingMessage, res: ServerResponse) => void; +} + +/** A local egress gateway: serves one object to one grant, like the real object route. */ +async function egressStub(t: TestContext, body: Buffer): Promise { + const stub: EgressStub = { url: '', hits: 0, body }; + const server = createServer((req, res) => { + stub.hits += 1; + if (stub.respond) return stub.respond(req, res); + if ( + req.method !== 'GET' || + req.url !== `/v1/egress/sessions/${SESSION}/objects/${OBJECT}` || + req.headers['x-codeapi-egress-grant'] !== GRANT + ) { + res.writeHead(403).end(); + return; + } + res.writeHead(200, { 'content-length': String(stub.body.byteLength) }).end(stub.body); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + })); + stub.url = `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1/egress`; + return stub; +} + +async function workspace( + t: TestContext, + stub: EgressStub, + options: { writable?: boolean; held?: boolean; importer?: Partial } = {}, +) { + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-import-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await LocalWorkspaceTools.create({ + importer: new HttpWorkspaceFileImporter({ upstreamUrl: stub.url, ...options.importer }), + workspaces: [{ + id: 'primary', + root, + writable: options.writable ?? true, + ...(options.held === false ? {} : { identity: await captureWorkspaceRootIdentity(root) }), + }], + }); + return { root, tools }; +} + +async function rejection(promise: Promise): Promise { + try { + await promise; + } catch (error) { + assert.ok(error instanceof WorkspaceToolError, String(error)); + return error; + } + assert.fail('expected a WorkspaceToolError'); +} + +async function temporaryFiles(directory: string): Promise { + return (await readdir(directory).catch(() => [] as string[])).filter((name) => name.endsWith('.tmp')); +} + +test('import requests follow write_file path rules and refuse Git metadata', () => { + const bytes = Buffer.from('payload'); + assert.equal(isWorkspaceToolRequest(importRequest(bytes)), true); + for (const path of [ + '../escape.bin', '/etc/passwd', 'C:/x.bin', 'a\\b.bin', 'dir/', 'dir/.', '.git', '.git/config', + 'src/.git/hooks/pre-commit', '.GIT/config', '.Git./x', 'a/.git /x', 'GIT~1/config', '.g\u200cit/config', + '.git::$INDEX_ALLOCATION/config', '.GIT:stream', '.git. ::$INDEX_ALLOCATION/hooks/x', 'git~1::$INDEX_ALLOCATION/x', + ]) { + assert.equal(isWorkspaceImportFilePath(path), false, path); + assert.equal(isWorkspaceToolRequest(importRequest(bytes, { path })), false, path); + } + for (const path of ['archive.zip', 'uploads/photo.png', '.gitignore', '.github/workflow.yml', 'docs/.gitkeep', 'notes/10:30 standup.md']) { + assert.equal(isWorkspaceImportFilePath(path), true, path); + } + for (const invalid of [ + { sha256: 'A'.repeat(64) }, + { sha256: 'a'.repeat(63) }, + { bytes: -1 }, + { bytes: 1.5 }, + { source: { storage_session_id: SESSION } }, + { source: { storage_session_id: SESSION, id: OBJECT, name: 'x' } }, + { source: { storage_session_id: 'a\nb', id: OBJECT } }, + { overwrite: 'yes' }, + { egressGrant: '' }, + ] as Array>) { + assert.equal(isWorkspaceToolRequest({ ...importRequest(bytes), ...invalid }), false, JSON.stringify(invalid)); + } + assert.equal(isWorkspaceToolRequest({ ...importRequest(bytes), content: 'x' }), false); + + const { egressGrant: _grant, ...callerRequest } = importRequest(bytes); + assert.equal(isBridgeWorkspaceImportFileRequest(callerRequest), true); + assert.equal(isBridgeWorkspaceImportFileRequest(importRequest(bytes)), false, 'callers never supply the grant'); + assert.equal(isRelayedWorkspaceImportFileRequest(importRequest(bytes)), true); + assert.equal(isRelayedWorkspaceImportFileRequest(callerRequest), false); +}); + +test('import results must echo the request and account for every byte', () => { + const bytes = Buffer.from('payload'); + const request = importRequest(bytes); + const result = { + protocolVersion: 1, + operation: 'import_file', + workspaceId: 'primary', + path: request.path, + created: true, + bytesWritten: bytes.byteLength, + sha256: request.sha256, + }; + assert.equal(isWorkspaceToolResult(request, result), true); + assert.equal(isWorkspaceToolResult(request, { ...result, created: false, unchanged: true, bytesWritten: 0 }), true); + assert.equal(isWorkspaceToolResult(request, { ...result, created: false, unchanged: true }), false); + assert.equal(isWorkspaceToolResult(request, { ...result, unchanged: true, bytesWritten: 0 }), false); + assert.equal(isWorkspaceToolResult(request, { ...result, created: false }), false, 'a create cannot replace'); + assert.equal(isWorkspaceToolResult({ ...request, overwrite: true }, { ...result, created: false }), true); + assert.equal(isWorkspaceToolResult(request, { ...result, bytesWritten: 3 }), false); + assert.equal(isWorkspaceToolResult(request, { ...result, sha256: 'b'.repeat(64) }), false); + assert.equal(isWorkspaceToolResult(request, { ...result, path: 'other.zip' }), false); + assert.equal(isWorkspaceToolResult(request, { ...result, content: 'x' }), false); +}); + +test('a worker imports by default whenever it can, and an explicit opt-out removes it', async (t) => { + const upstreamUrl = 'https://code.example.test/egress'; + const enabled = createWorkspaceFileImporter({ optOut: false, writable: true, upstreamUrl }); + assert.ok(enabled instanceof HttpWorkspaceFileImporter, 'on by default'); + for (const value of ['true', ' TRUE ', '']) { + assert.ok(createWorkspaceFileImporter({ optOut: false, enabled: value, writable: true, upstreamUrl })); + } + assert.equal(createWorkspaceFileImporter({ optOut: true, writable: true, upstreamUrl }), undefined, '--no-workspace-imports'); + assert.equal(createWorkspaceFileImporter({ optOut: false, enabled: 'false', writable: true, upstreamUrl }), undefined); + assert.equal(createWorkspaceFileImporter({ optOut: false, enabled: 'False', writable: true, upstreamUrl }), undefined); + assert.equal(createWorkspaceFileImporter({ optOut: false, writable: true }), undefined, 'no relay upstream'); + assert.equal(createWorkspaceFileImporter({ optOut: false, writable: true, upstreamUrl: ' ' }), undefined); + assert.equal(createWorkspaceFileImporter({ optOut: false, writable: false, upstreamUrl }), undefined, 'writes disabled'); + assert.throws(() => createWorkspaceFileImporter({ optOut: false, enabled: 'yes', writable: true, upstreamUrl }), /must be true or false/); + assert.equal(createWorkspaceFileImporter({ optOut: false, writable: true, upstreamUrl, maxBytes: 4096 })?.maxBytes, 4096); + + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-import-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const advertised = await LocalWorkspaceTools.create({ importer: enabled, workspaces: [{ id: 'primary', root, writable: true }] }); + assert.ok(advertised.capabilities.operations.includes('import_file')); + const optedOut = await LocalWorkspaceTools.create({ + importer: createWorkspaceFileImporter({ optOut: true, writable: true, upstreamUrl }), + workspaces: [{ id: 'primary', root, writable: true }], + }); + assert.equal(optedOut.capabilities.operations.includes('import_file'), false); +}); + +test('import_file is advertised only beside writes, on writable roots, with an importer', async (t) => { + const stub = await egressStub(t, Buffer.from('x')); + const writable = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-import-'))); + const readOnly = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-import-'))); + t.after(() => Promise.all([rm(writable, { recursive: true }), rm(readOnly, { recursive: true })])); + const importer = new HttpWorkspaceFileImporter({ upstreamUrl: stub.url }); + + const mixed = await LocalWorkspaceTools.create({ + importer, + workspaces: [{ id: 'primary', root: writable, writable: true }, { id: 'docs', root: readOnly }], + }); + assert.ok(mixed.capabilities.operations.includes('import_file')); + assert.ok(mixed.capabilities.workspaces[0]?.operations?.includes('import_file')); + assert.equal(mixed.capabilities.workspaces[1]?.operations?.includes('import_file'), false); + assert.equal(isValidBridgeWorkspaceToolCapabilities(mixed.capabilities), true); + const denied = await rejection(mixed.execute({ ...importRequest(Buffer.from('x')), workspaceId: 'docs' })); + assert.equal(denied.code, 'WRITE_DISABLED'); + assert.equal(stub.hits, 0); + + const allReadOnly = await LocalWorkspaceTools.create({ importer, workspaces: [{ id: 'docs', root: readOnly }] }); + assert.equal(allReadOnly.capabilities.operations.includes('import_file'), false); + const withoutRelay = await LocalWorkspaceTools.create({ workspaces: [{ id: 'primary', root: writable, writable: true }] }); + assert.equal(withoutRelay.capabilities.operations.includes('import_file'), false); + assert.equal((await rejection(withoutRelay.execute(importRequest(Buffer.from('x'))))).code, 'WRITE_UNAVAILABLE'); +}); + +test('imports binary bytes atomically into new 0755 parents as a 0644 file, whatever the umask', async (t) => { + const bytes = Buffer.concat([Buffer.from([0, 0xff, 0x50, 0x4b, 3, 4, 0]), randomBytes(256 * 1024)]); + const stub = await egressStub(t, bytes); + const { root, tools } = await workspace(t, stub); + const request = importRequest(bytes, { path: 'uploads/2026/archive.zip' }); + const previousUmask = process.umask(0o077); + t.after(() => process.umask(previousUmask)); + + const result = await tools.execute(request); + assert.deepEqual(result, { + protocolVersion: 1, + operation: 'import_file', + workspaceId: 'primary', + path: request.path, + created: true, + bytesWritten: bytes.byteLength, + sha256: request.sha256, + }); + assert.equal(isWorkspaceToolResult(request, result), true); + const target = join(root, 'uploads', '2026', 'archive.zip'); + assert.deepEqual(await readFile(target), bytes); + assert.equal((await stat(target)).mode & 0o777, 0o644); + assert.equal((await stat(join(root, 'uploads'))).mode & 0o777, 0o755); + assert.equal((await stat(join(root, 'uploads', '2026'))).mode & 0o777, 0o755); + assert.deepEqual(await temporaryFiles(join(root, 'uploads', '2026')), []); + assert.equal(stub.hits, 1); +}); + +test('re-importing identical bytes is idempotent and never downloads again', async (t) => { + const bytes = randomBytes(4096); + const stub = await egressStub(t, bytes); + const { root, tools } = await workspace(t, stub); + const request = importRequest(bytes, { path: 'photo.png' }); + await tools.execute(request); + const installed = await stat(join(root, 'photo.png')); + + const again = await tools.execute(request); + assert.deepEqual(again, { + protocolVersion: 1, + operation: 'import_file', + workspaceId: 'primary', + path: 'photo.png', + created: false, + unchanged: true, + bytesWritten: 0, + sha256: request.sha256, + }); + assert.equal(isWorkspaceToolResult(request, again), true); + assert.equal(stub.hits, 1); + assert.equal((await stat(join(root, 'photo.png'))).ino, installed.ino); +}); + +test('different existing content conflicts unless overwrite is explicit', async (t) => { + const bytes = Buffer.from('new report'); + const stub = await egressStub(t, bytes); + const { root, tools } = await workspace(t, stub); + const target = join(root, 'report.pdf'); + await writeFile(target, 'old report'); + await chmod(target, 0o755); + + const conflict = await rejection(tools.execute(importRequest(bytes, { path: 'report.pdf' }))); + assert.equal(conflict.code, 'EDIT_CONFLICT'); + assert.equal(conflict.mutationMayHaveCommitted, false); + assert.equal(await readFile(target, 'utf8'), 'old report'); + assert.equal(stub.hits, 0, 'a conflict is decided before any transfer'); + + const replaced = await tools.execute(importRequest(bytes, { path: 'report.pdf', overwrite: true })); + assert.equal((replaced as { created: boolean }).created, false); + assert.equal(await readFile(target, 'utf8'), 'new report'); + assert.equal((await stat(target)).mode & 0o777, 0o644, 'an import is never executable'); +}); + +test('a hash or size mismatch installs nothing and removes the parents it created', async (t) => { + const bytes = Buffer.from('expected bytes'); + const stub = await egressStub(t, Buffer.from('tampered bytes')); + const { root, tools } = await workspace(t, stub); + + const mismatch = await rejection(tools.execute(importRequest(bytes, { path: 'fresh/dir/file.bin' }))); + assert.equal(mismatch.code, 'INVALID_REQUEST'); + assert.equal(mismatch.mutationMayHaveCommitted, false); + assert.match(mismatch.message, /SHA-256/); + assert.deepEqual(await readdir(root), [], 'created parents are rolled back'); + + stub.respond = (_req, res) => { + res.writeHead(200); + res.write(bytes); + res.end(Buffer.from('and then some')); + }; + const oversize = await rejection(tools.execute(importRequest(bytes, { path: 'file.bin' }))); + assert.equal(oversize.code, 'INVALID_REQUEST'); + assert.match(oversize.message, /larger than its declared size/); + + stub.respond = (_req, res) => res.writeHead(200, { 'content-length': '3' }).end('abc'); + const declared = await rejection(tools.execute(importRequest(bytes, { path: 'file.bin' }))); + assert.equal(declared.code, 'INVALID_REQUEST'); + + stub.respond = (_req, res) => res.writeHead(200).end(bytes.subarray(0, 4)); + const short = await rejection(tools.execute(importRequest(bytes, { path: 'file.bin' }))); + assert.match(short.message, /smaller than its declared size/); + assert.deepEqual(await readdir(root), []); +}); + +test('the staged file is verified after it is synced, not only the bytes that arrived', async (t) => { + const bytes = randomBytes(64 * 1024); + const stub = await egressStub(t, bytes); + const { root } = await workspace(t, stub); + const relay = new HttpWorkspaceFileImporter({ upstreamUrl: stub.url }); + const tools = await LocalWorkspaceTools.create({ + importer: { + maxBytes: relay.maxBytes, + availableBytes: (directory) => relay.availableBytes(directory), + async open(request, signal) { + const chunks = await relay.open(request, signal); + return (async function* tamper() { + let first = true; + for await (const chunk of chunks) { + yield chunk; + if (!first) continue; + first = false; + // Another local writer rewrites already-staged bytes before the transfer finishes. + const [staged] = await temporaryFiles(root); + assert.ok(staged); + await writeFile(join(root, staged), Buffer.alloc(chunk.byteLength, 0x41), { flag: 'r+' }); + } + })(); + }, + }, + workspaces: [{ id: 'primary', root, writable: true, identity: await captureWorkspaceRootIdentity(root) }], + }); + const tampered = await rejection(tools.execute(importRequest(bytes, { path: 'payload.bin' }))); + assert.equal(tampered.code, 'INVALID_REQUEST'); + assert.match(tampered.message, /SHA-256/); + assert.deepEqual(await readdir(root), []); +}); + +test('rollback never removes a directory that replaced one the import created', async (t) => { + const bytes = randomBytes(32 * 1024); + const stub = await egressStub(t, Buffer.concat([bytes.subarray(0, 1024), Buffer.alloc(bytes.byteLength - 1024)])); + const { root } = await workspace(t, stub); + const relay = new HttpWorkspaceFileImporter({ upstreamUrl: stub.url }); + const tools = await LocalWorkspaceTools.create({ + importer: { + maxBytes: relay.maxBytes, + availableBytes: (directory) => relay.availableBytes(directory), + async open(request, signal) { + const chunks = await relay.open(request, signal); + return (async function* swap() { + for await (const chunk of chunks) yield chunk; + // Another local process moves the new parent aside and makes its own at that path. + await rename(join(root, 'incoming'), join(root, 'moved')); + await mkdir(join(root, 'incoming')); + })(); + }, + }, + workspaces: [{ id: 'primary', root, writable: true, identity: await captureWorkspaceRootIdentity(root) }], + }); + const failed = await rejection(tools.execute(importRequest(bytes, { path: 'incoming/file.bin' }))); + assert.equal(failed.code, 'INVALID_REQUEST'); + assert.equal(failed.mutationMayHaveCommitted, true, 'an unremovable created parent is reported, not hidden'); + assert.equal((await stat(join(root, 'incoming'))).isDirectory(), true); +}); + +test('the stall timeout covers waiting on the source, not slow consumer writes', async (t) => { + const bytes = randomBytes(8 * 1024); + const stub = await egressStub(t, bytes); + stub.respond = (_req, res) => { + res.writeHead(200, { 'content-length': String(bytes.byteLength) }); + res.write(bytes.subarray(0, 4096)); + setTimeout(() => res.end(bytes.subarray(4096)), 10); + }; + const importer = new HttpWorkspaceFileImporter({ upstreamUrl: stub.url, stallTimeoutMs: 50 }); + const received: Buffer[] = []; + for await (const chunk of await importer.open(importRequest(bytes) as WorkspaceImportFileRequest & { egressGrant: string })) { + received.push(Buffer.from(chunk)); + await delay(150); + } + assert.deepEqual(Buffer.concat(received), bytes); +}); + +test('file and free-space limits fail before anything is written', async (t) => { + const bytes = randomBytes(2048); + const stub = await egressStub(t, bytes); + const small = await workspace(t, stub, { importer: { maxBytes: 1024 } }); + const tooLarge = await rejection(small.tools.execute(importRequest(bytes, { path: 'big.bin' }))); + assert.equal(tooLarge.code, 'WRITE_LIMIT_EXCEEDED'); + assert.equal(stub.hits, 0); + + const full = await workspace(t, stub, { importer: { availableBytes: async () => 4096 } }); + const noSpace = await rejection(full.tools.execute(importRequest(bytes, { path: 'big.bin' }))); + assert.equal(noSpace.code, 'WRITE_LIMIT_EXCEEDED'); + assert.match(noSpace.message, /free space/); + assert.equal(stub.hits, 0); + assert.deepEqual(await readdir(full.root), []); +}); + +test('imports refuse symlinked parents, Git metadata, and requests without a grant', async (t) => { + const bytes = Buffer.from('payload'); + const stub = await egressStub(t, bytes); + const { root, tools } = await workspace(t, stub); + const outside = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-import-outside-'))); + t.after(() => rm(outside, { recursive: true, force: true })); + await mkdir(join(root, 'real')); + await symlink(join(root, 'real'), join(root, 'alias')); + await symlink(outside, join(root, 'escape')); + await mkdir(join(root, '.git')); + + for (const path of ['alias/file.bin', 'escape/file.bin']) { + const refused = await rejection(tools.execute(importRequest(bytes, { path }))); + assert.equal(refused.code, 'INVALID_PATH', path); + } + assert.deepEqual(await readdir(join(root, 'real')), []); + assert.deepEqual(await readdir(outside), []); + + const git = await rejection(tools.execute(importRequest(bytes, { path: '.git/hooks/post-checkout' }))); + assert.equal(git.code, 'INVALID_REQUEST'); + assert.deepEqual(await readdir(join(root, '.git')), []); + + await symlink(join(root, 'real', 'target.bin'), join(root, 'link.bin')); + const linkTarget = await rejection(tools.execute(importRequest(bytes, { path: 'link.bin', overwrite: true }))); + assert.equal(linkTarget.code, 'INVALID_PATH'); + assert.equal((await lstat(join(root, 'link.bin'))).isSymbolicLink(), true); + + const { egressGrant: _grant, ...ungranted } = importRequest(bytes, { path: 'file.bin' }); + assert.equal((await rejection(tools.execute(ungranted))).code, 'INVALID_REQUEST'); + assert.equal(stub.hits, 0); +}); + +test('an unavailable or refusing relay fails cleanly as WRITE_UNAVAILABLE', async (t) => { + const bytes = Buffer.from('payload'); + const stub = await egressStub(t, bytes); + const { root, tools } = await workspace(t, stub); + const refused = await rejection(tools.execute(importRequest(bytes, { egressGrant: 'ceg1.another-object' }))); + assert.equal(refused.code, 'WRITE_UNAVAILABLE'); + assert.equal(refused.mutationMayHaveCommitted, false); + assert.deepEqual(await readdir(root), []); + + stub.respond = (_req, res) => { + res.writeHead(200, { 'content-length': String(bytes.byteLength) }); + res.write(bytes.subarray(0, 2)); + }; + const stalled = await workspace(t, stub, { importer: { stallTimeoutMs: 50 } }); + const timedOut = await rejection(stalled.tools.execute(importRequest(bytes))); + assert.equal(timedOut.code, 'WRITE_UNAVAILABLE'); + assert.match(timedOut.message, /stalled/); + assert.deepEqual(await readdir(stalled.root), []); +}); + +test('a worker negotiates import_file and executes it as a guarded mutation', async (t) => { + const bytes = randomBytes(8192); + const stub = await egressStub(t, bytes); + const { root, tools } = await workspace(t, stub); + const lifecycle: string[] = []; + const registrations: string[][] = []; + const settlements: Array> = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools: tools.capabilities }, + workspaceTools: tools, + workspaceMutationQuarantine: { + async assertAvailable() {}, + async arm() { lifecycle.push('arm'); }, + async clear() { lifecycle.push('clear'); }, + async quarantine() { lifecycle.push('quarantine'); }, + }, + fetchImpl: async (input, init) => { + const url = String(input); + const body = (init?.body == null ? {} : JSON.parse(String(init.body))) as Record; + if (url.endsWith('/register')) { + registrations.push((body.capabilities as { workspaceTools: { operations: string[] } }).workspaceTools.operations); + return Response.json({ + protocolVersion: 1, workerId: 'vm-1', incarnationId, registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'search_text', 'list_files', 'write_file', 'preview_edit', 'edit_file', 'import_file'], + supportedWorkspaceWriteFileModes: ['replace', 'create'], + }); + } + if (!url.endsWith('/settle')) return Response.json({ protocolVersion: 1, cancelled: false }); + lifecycle.push('settle'); + settlements.push(body); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + await worker.register(); + assert.ok(registrations.at(-1)?.includes('import_file')); + + const request: WorkspaceToolRequest = importRequest(bytes, { path: 'chat/upload.bin' }); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-import', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 10_000).toISOString(), + remainingMs: 10_000, + executionKind: 'workspace_tool', + request, + }); + assert.deepEqual(lifecycle, ['arm', 'settle', 'clear']); + assert.equal(settlements[0]?.status, 'fulfilled'); + assert.deepEqual(await readFile(join(root, 'chat', 'upload.bin')), bytes); +}); + +test('a worker never advertises import_file to a Code API that cannot relay it', async (t) => { + const stub = await egressStub(t, Buffer.from('x')); + const { tools } = await workspace(t, stub); + const registrations: string[][] = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools: tools.capabilities }, + workspaceTools: tools, + workspaceMutationQuarantine: { async assertAvailable() {}, async arm() {}, async clear() {}, async quarantine() {} }, + fetchImpl: async (_input, init) => { + const body = JSON.parse(String(init?.body)) as { capabilities: { workspaceTools: { operations: string[] } } }; + registrations.push(body.capabilities.workspaceTools.operations); + return Response.json({ + protocolVersion: 1, workerId: 'vm-1', incarnationId, registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'search_text', 'list_files', 'write_file', 'preview_edit', 'edit_file'], + supportedWorkspaceWriteFileModes: ['replace', 'create'], + }); + }, + }); + await worker.register(); + assert.ok(registrations.length > 0); + for (const operations of registrations) assert.equal(operations.includes('import_file'), false); +}); diff --git a/packages/code/src/workspace-import.ts b/packages/code/src/workspace-import.ts new file mode 100644 index 00000000..e4c927d3 --- /dev/null +++ b/packages/code/src/workspace-import.ts @@ -0,0 +1,188 @@ +import { statfs } from 'node:fs/promises'; + +import { + BRIDGE_WORKSPACE_IMPORT_FILE_DEFAULT_MAX_BYTES, + BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS, +} from './protocol.js'; +import { + FILE_RELAY_GRANT_HEADER, + fileRelayObjectUrl, + validateFileRelayUpstream, +} from './relay.js'; +import { WorkspaceToolError } from './workspace.js'; + +import type { WorkspaceImportFileRequest } from './protocol.js'; +import type { WorkspaceFileImporter } from './workspace.js'; + +export interface HttpWorkspaceFileImporterOptions { + /** The egress gateway this worker already uses for programmatic inputs. */ + upstreamUrl: string; + maxBytes?: number; + /** Longest wait for response headers or for the next body chunk. */ + stallTimeoutMs?: number; + fetchImpl?: typeof fetch; + availableBytes?: (directory: string) => Promise; +} + +/** + * The importer a worker runs with, if any. `import_file` is on by default + * whenever the worker can perform it: writes enabled and a relay upstream to + * reach stored objects. `--no-workspace-imports` or + * `LIBRECHAT_CODE_ALLOW_WORKSPACE_IMPORTS=false` opts out; any other value of + * that variable fails startup rather than silently choosing either way. + */ +export function createWorkspaceFileImporter(input: { + optOut: boolean; + enabled?: string; + writable: boolean; + upstreamUrl?: string; + maxBytes?: number; +}): HttpWorkspaceFileImporter | undefined { + const enabled = input.enabled?.trim().toLowerCase() ?? ''; + if (enabled !== '' && enabled !== 'true' && enabled !== 'false') { + throw new Error('LIBRECHAT_CODE_ALLOW_WORKSPACE_IMPORTS must be true or false'); + } + const upstreamUrl = input.upstreamUrl?.trim() ?? ''; + if (input.optOut || enabled === 'false' || !input.writable || upstreamUrl === '') { + return undefined; + } + return new HttpWorkspaceFileImporter({ upstreamUrl, maxBytes: input.maxBytes }); +} + +async function filesystemAvailableBytes(directory: string): Promise { + const stats = await statfs(directory); + return Number(stats.bavail) * Number(stats.bsize); +} + +function importAborted(): WorkspaceToolError { + return new WorkspaceToolError('Workspace import aborted', 'EXECUTION_ABORTED'); +} + +function importUnavailable(message: string): WorkspaceToolError { + return new WorkspaceToolError(message, 'WRITE_UNAVAILABLE'); +} + +/** + * Streams one granted object from the file relay upstream. The worker holds no + * storage credential: Code API's grant authorizes exactly this object, briefly. + */ +export class HttpWorkspaceFileImporter implements WorkspaceFileImporter { + readonly maxBytes: number; + private readonly upstream: URL; + private readonly stallTimeoutMs: number; + private readonly fetchImpl: typeof fetch; + private readonly freeBytes: (directory: string) => Promise; + + constructor(options: HttpWorkspaceFileImporterOptions) { + this.upstream = validateFileRelayUpstream(options.upstreamUrl); + this.maxBytes = options.maxBytes ?? BRIDGE_WORKSPACE_IMPORT_FILE_DEFAULT_MAX_BYTES; + this.stallTimeoutMs = options.stallTimeoutMs ?? BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS; + if (!Number.isSafeInteger(this.maxBytes) || this.maxBytes < 1) { + throw new RangeError('Workspace import file limit must be a positive safe integer'); + } + if (!Number.isSafeInteger(this.stallTimeoutMs) || this.stallTimeoutMs < 1) { + throw new RangeError('Workspace import stall timeout must be a positive safe integer'); + } + this.fetchImpl = options.fetchImpl ?? fetch; + this.freeBytes = options.availableBytes ?? filesystemAvailableBytes; + } + + availableBytes(directory: string): Promise { + return this.freeBytes(directory); + } + + async open( + request: WorkspaceImportFileRequest & { egressGrant: string }, + signal?: AbortSignal, + ): Promise> { + if (signal?.aborted) throw importAborted(); + const controller = new AbortController(); + const abort = (): void => controller.abort(signal?.reason); + signal?.addEventListener('abort', abort, { once: true }); + let stalled = false; + let timer: ReturnType | undefined; + const arm = (): void => { + clearTimeout(timer); + timer = setTimeout(() => { + stalled = true; + controller.abort(); + }, this.stallTimeoutMs); + }; + const release = (): void => { + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + }; + const failure = (): WorkspaceToolError => + signal?.aborted + ? importAborted() + : importUnavailable(stalled ? 'Import source transfer stalled' : 'Import source download failed'); + + arm(); + let response: Response; + try { + response = await this.fetchImpl( + fileRelayObjectUrl(this.upstream, request.source.storage_session_id, request.source.id), + { + headers: { + [FILE_RELAY_GRANT_HEADER]: request.egressGrant, + 'Accept-Encoding': 'identity', + }, + redirect: 'error', + signal: controller.signal, + }, + ); + } catch { + release(); + throw failure(); + } + // A compressed transfer declares its encoded length; only the decoded bytes are checked then. + const encoding = response.headers.get('content-encoding')?.trim().toLowerCase(); + const declaredLength = encoding == null || encoding === 'identity' + ? response.headers.get('content-length') + : null; + let rejection: WorkspaceToolError | undefined; + if (!response.ok) { + rejection = importUnavailable(`Import source download failed with HTTP ${response.status}`); + } else if (declaredLength != null && declaredLength !== String(request.bytes)) { + rejection = new WorkspaceToolError( + 'Import source size does not match the declared size', + 'INVALID_REQUEST', + ); + } else if (response.body == null && request.bytes > 0) { + rejection = importUnavailable('Import source returned no content'); + } + if (rejection != null) { + release(); + await response.body?.cancel().catch(() => undefined); + throw rejection; + } + const body = response.body; + async function* chunks(): AsyncGenerator { + if (body == null) { + release(); + return; + } + const reader = body.getReader(); + try { + for (;;) { + let next: Awaited>; + // The stall timer covers only waiting on the source, never the consumer's writes. + arm(); + try { + next = await reader.read(); + } catch { + throw failure(); + } finally { + clearTimeout(timer); + } + if (next.done) return; + yield next.value; + } + } finally { + release(); + await reader.cancel().catch(() => undefined); + } + } + return chunks(); + } +} diff --git a/packages/code/src/workspace-instances.ts b/packages/code/src/workspace-instances.ts index 68e76a10..7927152b 100644 --- a/packages/code/src/workspace-instances.ts +++ b/packages/code/src/workspace-instances.ts @@ -12,12 +12,13 @@ import type { WorkspaceToolRequest, WorkspaceToolResult, } from './protocol.js'; -import type { WorkspaceToolExecutor } from './workspace.js'; +import type { WorkspaceFileImporter, WorkspaceToolExecutor } from './workspace.js'; interface WorkspaceInstanceSource { command?: NativeProcessSandboxOptions; repositoryInstructions: boolean; writable: boolean; + importer?: WorkspaceFileImporter; } export interface GitWorktreeWorkspaceToolsOptions { @@ -119,6 +120,7 @@ export class GitWorktreeWorkspaceTools implements WorkspaceToolExecutor { identity: instance.identity, value: LocalWorkspaceTools.create({ repositoryInstructions: source.repositoryInstructions, + importer: source.importer, workspaces: [ { id: internalId, diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index 99fcdd02..46d117bd 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -8,6 +8,61 @@ import { SandboxWorkspaceTools, WorkspaceToolError } from './workspace.js'; const incarnationId = 'incarnation-00000001'; +test('worker logs command failures locally without changing bridge settlements', async () => { + for (const sink of ['sync', 'async', 'pending'] as const) { + for (const operation of ['execute_command', 'list_files'] as const) { + const events: unknown[] = []; + let settlement: Record | undefined; + const capabilities = { + protocolVersion: 1 as const, + operations: ['execute_command' as const, 'list_files' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools: capabilities }, + workspaceTools: { + capabilities, mutationFailuresAreAtomic: true, + async execute() { + throw new WorkspaceToolError('private error', 'COMMAND_UNAVAILABLE', false, false, + { stage: 'credential_resolution', reason: 'credential_failed' }); + }, + }, + workspaceQuarantines: new Map([['primary', mutationQuarantine()]]), + onCommandFailure(event) { + events.push(event); + if (sink === 'sync') throw new Error('private log sink error'); + if (sink === 'async') return Promise.reject(new Error('private log sink error')); + return new Promise(() => {}); + }, + fetchImpl: async (_input, init) => { + settlement = JSON.parse(String(init?.body)); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + await worker.executeAndSettle({ + protocolVersion: 1, assignmentId: 'assignment-1', workerId: 'vm-1', incarnationId, generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', expiresAt: new Date(Date.now() + 5000).toISOString(), + executionKind: 'workspace_tool', + request: operation === 'execute_command' + ? { protocolVersion: 1, operation, workspaceId: 'primary', command: 'echo secret' } + : { protocolVersion: 1, operation, workspaceId: 'primary', path: '.' }, + }); + assert.equal(events.length, operation === 'execute_command' ? 1 : 0); + if (events.length) assert.deepEqual(events[0], { + event: 'workspace_command_unavailable', stage: 'credential_resolution', reason: 'credential_failed', + assignmentId: 'assignment-1', workerId: 'vm-1', incarnationId, workspaceId: 'primary', + operation: 'execute_command', mutationMayHaveCommitted: false, requiresQuarantine: false, + }); + assert.equal(settlement?.status, 'rejected'); + assert.equal(settlement?.errorCode, 'COMMAND_UNAVAILABLE'); + assert.equal('diagnostic' in settlement!, false); + assert.equal(JSON.stringify(events).includes('secret'), false); + } + } +}); + test('instance advertisement requires a guard resolver even for reads and with base guards', () => { for (const operation of ['read_file', 'write_file'] as const) { const workspaceTools = { protocolVersion: 1 as const, operations: [operation], workspaces: [{ id: 'primary', workspaceInstances: ['git_worktree'] as ['git_worktree'] }] }; @@ -1308,6 +1363,7 @@ test('worker executes programmatic Bash in the selected workspace and preserves test('worker keeps a selected workspace usable after an atomic programmatic setup failure', async () => { const lifecycle: string[] = []; + const diagnostics: unknown[] = []; let settlement: Record | undefined; const workspaceCapabilities = { protocolVersion: 1 as const, @@ -1343,6 +1399,10 @@ test('worker keeps a selected workspace usable after an atomic programmatic setu ); }, }, + onCommandFailure(event) { + diagnostics.push(event); + throw new Error('logging sink failed'); + }, workspaceQuarantines: new Map([ [ 'primary', @@ -1383,6 +1443,14 @@ test('worker keeps a selected workspace usable after an atomic programmatic setu assert.deepEqual(lifecycle, ['arm', 'clear']); assert.equal(settlement?.status, 'rejected'); assert.equal(settlement?.errorCode, 'COMMAND_UNAVAILABLE'); + assert.deepEqual(diagnostics, [{ + event: 'workspace_command_unavailable', stage: 'dispatch', reason: 'executor_rejected', + assignmentId: 'assignment-programmatic-setup-failure', workerId: 'vm-1', incarnationId, + workspaceId: 'primary', operation: 'workspace_programmatic', + mutationMayHaveCommitted: false, requiresQuarantine: false, + }]); + assert.equal('commandDiagnostic' in settlement!, false); + assert.equal('diagnostic' in settlement!, false); }); test('worker keeps a selected workspace usable after confirmed programmatic cancellation cleanup', async () => { diff --git a/packages/code/src/workspace.test.ts b/packages/code/src/workspace.test.ts index 787fbe7d..16e2ad13 100644 --- a/packages/code/src/workspace.test.ts +++ b/packages/code/src/workspace.test.ts @@ -2603,7 +2603,8 @@ test('fails closed on invalid or failed sandbox command results', async (t) => { for (const execute of [ async () => ({ ...request, exitCode: 0, stdout: 'ok', stderr: '' }), async () => { throw new Error('container details'); }, - async () => { throw new WorkspaceToolError('untrusted clean claim', 'COMMAND_UNAVAILABLE'); }, + async () => { throw new WorkspaceToolError('untrusted clean claim', 'COMMAND_UNAVAILABLE', false, false, + { stage: 'command_wrapping', reason: 'wrapping_failed' }); }, ]) { const tools = new SandboxWorkspaceTools({ workspaceTools: local, @@ -2616,6 +2617,7 @@ test('fails closed on invalid or failed sandbox command results', async (t) => { error instanceof WorkspaceToolError && error.code === 'COMMAND_UNAVAILABLE' && error.mutationMayHaveCommitted === true && + (error.message !== 'untrusted clean claim' || error.commandDiagnostic?.stage === 'command_wrapping') && !error.message.includes('container details'), ); assert.equal(tools.mutationFailuresAreAtomic, undefined); diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index d740d48a..5aecbfd9 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -1,9 +1,10 @@ import { createHash, randomBytes } from 'node:crypto'; import { constants } from 'node:fs'; -import { deferWorkspaceRootCleanup, holdsWorkspaceRoot, link, lstat, mkdir, open, realpath, rename, stat, unlink, spawn, withWorkspaceRoot, WorkspaceRootAccessError } from './root-access.js'; +import { deferWorkspaceRootCleanup, holdsWorkspaceRoot, link, lstat, mkdir, open, realpath, rename, rmdir, stat, unlink, spawn, withWorkspaceRoot, WorkspaceRootAccessError } from './root-access.js'; import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path'; import type { FileHandle } from 'node:fs/promises'; +import type { CommandFailureDiagnostic } from './command-diagnostics.js'; import { matchesWorkspaceRoot } from './root-identity.js'; import type { WorkspaceRootIdentity } from './root-identity.js'; @@ -17,6 +18,7 @@ import { BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH, comparePortableRelativePaths, isSafePortableRelativePath, + isWorkspaceImportFilePath, isValidBridgeWorkspaceToolCapabilities, isWorkspaceToolRequest, isWorkspaceToolResult, @@ -42,6 +44,8 @@ import type { WorkspacePreviewEditResult, WorkspaceExecuteCommandRequest, WorkspaceExecuteCommandResult, + WorkspaceImportFileRequest, + WorkspaceImportFileResult, WorkspaceListFilesRequest, WorkspaceListFilesResult, WorkspaceSearchMatch, @@ -65,6 +69,8 @@ export type { WorkspacePreviewEditResult, WorkspaceExecuteCommandRequest, WorkspaceExecuteCommandResult, + WorkspaceImportFileRequest, + WorkspaceImportFileResult, WorkspaceListFilesRequest, WorkspaceListFilesResult, WorkspaceSearchMatch, @@ -85,9 +91,27 @@ export interface LocalWorkspaceConfig { writable?: boolean; } +/** + * Fetches a stored object for `import_file`. Only a worker with a configured + * file relay upstream has one, and `import_file` is advertised only then. + */ +export interface WorkspaceFileImporter { + /** Largest file this worker imports; larger requests fail before any transfer. */ + readonly maxBytes: number; + /** Streams the granted object. Failures are WorkspaceToolErrors. */ + open( + request: WorkspaceImportFileRequest & { egressGrant: string }, + signal?: AbortSignal, + ): Promise>; + /** Bytes a new file in `directory` may still use on its filesystem. */ + availableBytes(directory: string): Promise; +} + export interface LocalWorkspaceToolsOptions { workspaces: LocalWorkspaceConfig[]; repositoryInstructions?: boolean; + /** Enables `import_file` on writable workspaces. */ + importer?: WorkspaceFileImporter; } export interface WorkspaceToolExecutor { @@ -144,6 +168,12 @@ const READ_OPERATIONS = [ 'list_files', ] as const; const WRITE_OPERATIONS = ['write_file', 'preview_edit', 'edit_file'] as const; +const IMPORT_OPERATIONS = ['import_file'] as const; +const IMPORT_DIRECTORY_MODE = 0o755; +const IMPORT_FILE_MODE = 0o644; +const IMPORT_HASH_CHUNK_BYTES = 1024 * 1024; +/** Headroom left on the target filesystem after an import lands. */ +export const WORKSPACE_IMPORT_FREE_SPACE_RESERVE_BYTES = 64 * 1024 * 1024; function isUtf8ScalarString(value: string): boolean { return Buffer.from(value).toString('utf8') === value; @@ -197,6 +227,8 @@ export class WorkspaceToolError extends Error { public readonly mutationMayHaveCommitted = false, /** Retain the durable mutation guard when process or write settlement is uncertain. */ public readonly requiresQuarantine = mutationMayHaveCommitted, + /** Worker-local diagnostic metadata; never serialized into bridge settlements. */ + public readonly commandDiagnostic?: CommandFailureDiagnostic, ) { super(message); this.name = 'WorkspaceToolError'; @@ -682,6 +714,7 @@ async function classifyMissingWorkspacePath( async function createMissingParentDirectories( root: string, candidate: string, + mode = 0o777, ): Promise { const created: string[] = []; if (!holdsWorkspaceRoot()) return created; @@ -699,7 +732,7 @@ async function createMissingParentDirectories( let createdHere = false; if (entry == null) { try { - await mkdir(current, 0o777); + await mkdir(current, mode); created.push(current); createdHere = true; } catch (error) { @@ -1070,6 +1103,373 @@ async function writeWorkspaceFile( }; } +interface ImportTargetIdentity { + dev: bigint | number; + ino: bigint | number; +} + +type ImportTarget = + | { state: 'missing' } + | { state: 'unchanged' } + | ({ state: 'different' } & ImportTargetIdentity); + +function importIntegrityError(message: string): WorkspaceToolError { + return new WorkspaceToolError(message, 'INVALID_REQUEST'); +} + +/** Size and SHA-256 of everything an open file holds now, read through its descriptor. */ +async function hashOpenFile( + handle: FileHandle, + signal?: AbortSignal, +): Promise<{ bytes: number; sha256: string }> { + const hash = createHash('sha256'); + const chunk = Buffer.allocUnsafe(IMPORT_HASH_CHUNK_BYTES); + let position = 0; + for (;;) { + throwIfAborted(signal); + const { bytesRead } = await handle.read(chunk, 0, chunk.byteLength, position); + if (bytesRead === 0) break; + hash.update(chunk.subarray(0, bytesRead)); + position += bytesRead; + } + return { bytes: position, sha256: hash.digest('hex') }; +} + +/** Hash an existing target only when its size already matches the import. */ +async function inspectImportTarget( + root: string, + request: WorkspaceImportFileRequest, + signal?: AbortSignal, +): Promise { + try { + return await withConfinedFile(root, request.path, async (handle, size) => { + const { dev, ino } = await handle.stat(); + if (size !== request.bytes) return { state: 'different', dev, ino }; + const existing = await hashOpenFile(handle, signal); + return existing.bytes === request.bytes && existing.sha256 === request.sha256 + ? { state: 'unchanged' } + : { state: 'different', dev, ino }; + }); + } catch (error) { + if (error instanceof WorkspaceToolError && error.code === 'NOT_FOUND') { + return { state: 'missing' }; + } + throw error; + } +} + +/** + * Undo parent directories this import created; true when every one is gone + * again. A directory is removed only while it is still the inode this import + * created, so a replacement made by another local process is never deleted. + */ +async function removeCreatedDirectories( + created: readonly string[], + identities: readonly ImportTargetIdentity[], +): Promise { + if (identities.length !== created.length) return false; + for (let index = created.length - 1; index >= 0; index -= 1) { + const directory = created[index]!; + try { + const current = await lstat(directory); + if ( + current.isSymbolicLink() || + !current.isDirectory() || + current.dev !== identities[index]!.dev || + current.ino !== identities[index]!.ino + ) { + return false; + } + await rmdir(directory); + await syncWorkspaceDirectory(dirname(directory)); + } catch { + return false; + } + } + return true; +} + +/** + * Record each directory this import created, then give it the documented mode + * regardless of the worker's umask. `identities` fills as each is opened. + */ +async function setImportDirectoryModes( + created: readonly string[], + identities: ImportTargetIdentity[], +): Promise { + for (const directory of created) { + const handle = await open( + directory, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, + ); + try { + const { dev, ino } = await handle.stat(); + identities.push({ dev, ino }); + await handle.chmod(IMPORT_DIRECTORY_MODE); + } finally { + await handle.close(); + } + } +} + +async function writeImportChunk(handle: FileHandle, chunk: Uint8Array, position: number): Promise { + let offset = 0; + while (offset < chunk.byteLength) { + const { bytesWritten } = await handle.write(chunk, offset, chunk.byteLength - offset, position + offset); + if (bytesWritten < 1) throw new Error('short workspace write'); + offset += bytesWritten; + } +} + +/** Confirm the installed entry is the staged inode, inside the root, after syncing its directory. */ +async function confirmInstalledImport( + root: string, + installTarget: string, + staged: ImportTargetIdentity, +): Promise { + let handle: FileHandle | undefined; + try { + await syncWorkspaceDirectory(dirname(installTarget)); + handle = await open( + installTarget, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + const [installed, canonicalPath] = await Promise.all([handle.stat(), realpath(installTarget)]); + if ( + !installed.isFile() || + !isWithinRoot(root, canonicalPath) || + installed.dev !== staged.dev || + installed.ino !== staged.ino + ) { + throw new Error('installed workspace import changed'); + } + } catch { + throw new WorkspaceToolError( + 'Workspace import durability could not be confirmed', + 'WRITE_UNAVAILABLE', + true, + ); + } finally { + await handle?.close().catch(() => undefined); + } +} + +/** + * Stream one object into a same-directory temporary file, verify the synced + * file's size and SHA-256 through its own descriptor, then install it + * atomically. A create links exclusively so a racing file is never replaced. + * An overwrite rechecks the inspected inode immediately before its rename, the + * same check-then-rename `write_file` uses; bridge mutations of one root are + * serialized, so only an out-of-band local writer can race that last step. + */ +async function installImportedFile( + root: string, + candidate: string, + request: WorkspaceImportFileRequest & { egressGrant: string }, + importer: WorkspaceFileImporter, + target: Exclude, + signal?: AbortSignal, +): Promise<{ created: boolean }> { + const parent = dirname(candidate); + let canonicalParent: string; + let parentIdentity: Awaited>; + try { + canonicalParent = await realpath(parent); + parentIdentity = await verifyDirectoryPathHasNoSymlinks(root, parent); + if (!isWithinRoot(root, canonicalParent) || !parentIdentity.isDirectory()) { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + } catch (error) { + if (isMissingEntry(error)) throw await classifyMissingWorkspacePath(root, parent); + throw classifyWritePathValidationError(error); + } + let available: number; + try { + available = await importer.availableBytes(canonicalParent); + } catch { + throw new WorkspaceToolError('Workspace free space could not be determined', 'WRITE_UNAVAILABLE'); + } + if (available < request.bytes + WORKSPACE_IMPORT_FREE_SPACE_RESERVE_BYTES) { + throw new WorkspaceToolError( + 'Workspace filesystem does not have enough free space for this file', + 'WRITE_LIMIT_EXCEEDED', + ); + } + + const temporary = resolve(canonicalParent, `.librechat-code-${randomBytes(18).toString('hex')}.tmp`); + const installTarget = resolve(canonicalParent, basename(candidate)); + let handle: FileHandle | undefined; + let temporaryNeedsCleanup = false; + try { + handle = await open( + temporary, + constants.O_RDWR | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, + 0o600, + ); + temporaryNeedsCleanup = true; + let written = 0; + for await (const chunk of await importer.open(request, signal)) { + throwIfAborted(signal); + if (written + chunk.byteLength > request.bytes) { + throw importIntegrityError('Imported file is larger than its declared size'); + } + await writeImportChunk(handle, chunk, written); + written += chunk.byteLength; + } + throwIfAborted(signal); + if (written !== request.bytes) { + throw importIntegrityError('Imported file is smaller than its declared size'); + } + await handle.chmod(IMPORT_FILE_MODE); + await handle.sync(); + // Verify what will be installed, not what arrived: the staged file is a workspace entry. + const staged = await hashOpenFile(handle, signal); + if (staged.bytes !== request.bytes) { + throw importIntegrityError('Imported file does not match its declared size'); + } + if (staged.sha256 !== request.sha256) { + throw importIntegrityError('Imported file does not match its declared SHA-256'); + } + const stagedStat = await handle.stat(); + const stagedIdentity = { dev: stagedStat.dev, ino: stagedStat.ino }; + if (process.platform === 'win32') { + await handle.close(); + handle = undefined; + } + + const [currentParent, currentParentIdentity] = await Promise.all([ + realpath(parent), + verifyDirectoryPathHasNoSymlinks(root, parent), + ]); + if ( + currentParent !== canonicalParent || + currentParentIdentity.dev !== parentIdentity.dev || + currentParentIdentity.ino !== parentIdentity.ino + ) { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + let current: Awaited> | undefined; + try { + current = await lstat(candidate); + } catch (error) { + if (!isMissingEntry(error)) throw error; + } + if ( + target.state === 'missing' + ? current != null + : current == null || current.isSymbolicLink() || current.dev !== target.dev || current.ino !== target.ino + ) { + throw new WorkspaceToolError( + 'Workspace file changed before import could be committed', + 'EDIT_CONFLICT', + ); + } + throwIfAborted(signal); + if (target.state === 'different') { + await rename(temporary, installTarget); + temporaryNeedsCleanup = false; + } else { + try { + await link(temporary, installTarget); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'EEXIST') { + throw new WorkspaceToolError('Workspace file already exists', 'EDIT_CONFLICT'); + } + throw error; + } + try { + await unlink(temporary); + temporaryNeedsCleanup = false; + } catch { + throw new WorkspaceToolError( + 'Workspace import cleanup could not be confirmed', + 'WRITE_UNAVAILABLE', + true, + ); + } + } + await confirmInstalledImport(root, installTarget, stagedIdentity); + return { created: target.state === 'missing' }; + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + throw new WorkspaceToolError('Workspace storage is unavailable', 'WRITE_UNAVAILABLE'); + } finally { + await handle?.close().catch(() => undefined); + if (temporaryNeedsCleanup) await removeImportStaging(temporary); + } +} + +/** A staging file that cannot be removed is upload bytes left in the workspace: never a clean failure. */ +async function removeImportStaging(temporary: string): Promise { + try { + await unlink(temporary); + } catch (error) { + if (isMissingEntry(error)) return; + throw new WorkspaceToolError( + 'Workspace import staging file could not be removed', + 'WRITE_UNAVAILABLE', + true, + ); + } +} + +async function importWorkspaceFile( + root: string, + request: WorkspaceImportFileRequest, + importer: WorkspaceFileImporter, + signal?: AbortSignal, +): Promise { + throwIfAborted(signal); + if (!isWorkspaceImportFilePath(request.path)) { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + const { egressGrant } = request; + if (egressGrant === undefined) { + throw new WorkspaceToolError('Import source grant is unavailable', 'INVALID_REQUEST'); + } + if (request.bytes > importer.maxBytes) { + throw new WorkspaceToolError('Imported file exceeds the worker file limit', 'WRITE_LIMIT_EXCEEDED'); + } + const result = (created: boolean, bytesWritten: number, unchanged = false): WorkspaceImportFileResult => ({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'import_file', + workspaceId: request.workspaceId, + path: request.path, + created, + ...(unchanged ? { unchanged: true } : {}), + bytesWritten, + sha256: request.sha256, + }); + const candidate = resolveWorkspacePath(root, request.path); + const target = await inspectImportTarget(root, request, signal); + if (target.state === 'unchanged') return result(false, 0, true); + if (target.state === 'different' && request.overwrite !== true) { + throw new WorkspaceToolError( + 'Workspace file already exists with different content', + 'EDIT_CONFLICT', + ); + } + const directories = await createMissingParentDirectories(root, candidate, IMPORT_DIRECTORY_MODE); + const identities: ImportTargetIdentity[] = []; + try { + await setImportDirectoryModes(directories, identities); + const { created } = await installImportedFile( + root, + candidate, + { ...request, egressGrant }, + importer, + target, + signal, + ); + return result(created, request.bytes); + } catch (error) { + const failure = classifyWritePathValidationError(error); + if (directories.length === 0 || failure.mutationMayHaveCommitted) throw failure; + if (await removeCreatedDirectories(directories, identities)) throw failure; + throw afterCreatedDirectories(directories, failure); + } +} + async function editWorkspaceFile( root: string, request: WorkspaceEditFileRequest, @@ -1878,6 +2278,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { editFileModes?: BridgeWorkspaceToolCapabilities['editFileModes'], editFileFeatures?: BridgeWorkspaceToolCapabilities['editFileFeatures'], listFileFeatures?: BridgeWorkspaceToolCapabilities['listFileFeatures'], + private readonly importer?: WorkspaceFileImporter, ) { this.capabilities = { protocolVersion: BRIDGE_PROTOCOL_VERSION, @@ -1897,9 +2298,13 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { const anyWritable = options.workspaces.some( (workspace) => workspace.writable === true, ); + const mutations = [ + ...WRITE_OPERATIONS, + ...(options.importer != null ? IMPORT_OPERATIONS : []), + ]; const operations: BridgeWorkspaceToolCapabilities['operations'] = [ ...READ_OPERATIONS, - ...(anyWritable ? WRITE_OPERATIONS : []), + ...(anyWritable ? mutations : []), ]; const workspaces: BridgeWorkspaceDescriptor[] = options.workspaces.map( (workspace) => ({ @@ -1909,7 +2314,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { ? { operations: [ ...READ_OPERATIONS, - ...(workspace.writable === true ? WRITE_OPERATIONS : []), + ...(workspace.writable === true ? mutations : []), ], } : {}), @@ -1959,6 +2364,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { capabilities.editFileModes, capabilities.editFileFeatures, capabilities.listFileFeatures, + anyWritable ? options.importer : undefined, ); tools.repositoryInstructions = options.repositoryInstructions === true; return tools; @@ -2000,6 +2406,22 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { } const { root } = workspace; + if (request.operation === 'import_file') { + if (!workspace.writable) { + throw new WorkspaceToolError( + 'Workspace mutations are disabled by the worker', + 'WRITE_DISABLED', + ); + } + if (this.importer == null) { + throw new WorkspaceToolError( + 'Workspace file import is not configured on this worker', + 'WRITE_UNAVAILABLE', + ); + } + return importWorkspaceFile(root, request, this.importer, signal); + } + if ( request.operation === 'write_file' || request.operation === 'preview_edit' || @@ -2163,7 +2585,7 @@ export class SandboxWorkspaceTools implements WorkspaceToolExecutor { ) { throw error; } - throw new WorkspaceToolError(error.message, error.code, true); + throw new WorkspaceToolError(error.message, error.code, true, true, error.commandDiagnostic); } throw new WorkspaceToolError( 'Sandboxed command execution unavailable', diff --git a/packages/extraction/.gitattributes b/packages/extraction/.gitattributes new file mode 100644 index 00000000..67da8786 --- /dev/null +++ b/packages/extraction/.gitattributes @@ -0,0 +1,2 @@ +tests/fixtures/*.pdf binary +tests/fixtures/*.docx binary diff --git a/packages/extraction/.gitignore b/packages/extraction/.gitignore new file mode 100644 index 00000000..2e3a563c --- /dev/null +++ b/packages/extraction/.gitignore @@ -0,0 +1 @@ +native/guard diff --git a/packages/extraction/Dockerfile b/packages/extraction/Dockerfile new file mode 100644 index 00000000..cfc48c14 --- /dev/null +++ b/packages/extraction/Dockerfile @@ -0,0 +1,32 @@ +FROM node:24.22.1-bookworm-slim AS build +WORKDIR /build +COPY packages/extraction/package*.json ./ +RUN npm ci --ignore-scripts --no-audit --no-fund +COPY packages/extraction/tsconfig.json ./ +COPY packages/extraction/src ./src +RUN npm run build +RUN apt-get update && apt-get install -y --no-install-recommends gcc libc6-dev linux-libc-dev && rm -rf /var/lib/apt/lists/* +COPY packages/extraction/native/guard.c ./guard.c +RUN gcc -O2 -Wall -Wextra -Werror -o guard guard.c +COPY packages/extraction/tests/block-landlock.c ./block-landlock.c +RUN gcc -O2 -Wall -Wextra -Werror -o block-landlock block-landlock.c + +FROM python:3.13-slim-bookworm@sha256:5024f48ba9441d4b13a95d3945abc6365538e3a31109833367a1923523c6efed AS runtime +RUN apt-get update && apt-get install -y --no-install-recommends libstdc++6 libatomic1 && rm -rf /var/lib/apt/lists/* +COPY packages/extraction/worker/requirements.txt /opt/extraction/requirements.txt +RUN pip install --no-cache-dir -r /opt/extraction/requirements.txt +COPY --from=build /usr/local/bin/node /usr/local/bin/node +COPY --from=build --chmod=0555 /build/guard /opt/extraction/guard +COPY --from=build --chmod=0555 /build/dist /opt/extraction/dist +COPY --chmod=0444 packages/extraction/package.json /opt/extraction/package.json +COPY --chmod=0444 packages/extraction/worker/extract.py /opt/extraction/extract.py +RUN groupadd -g 10001 extraction && useradd -u 10001 -g extraction -M extraction && \ + mkdir /socket /jobs && chown extraction:extraction /socket /jobs && chmod 0700 /jobs && chmod 0770 /socket +USER 10001:10001 +WORKDIR /opt/extraction +HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --retries=3 CMD ["node", "dist/health.js"] +CMD ["node", "dist/main.js"] + +FROM runtime AS qualification +COPY --chown=10001:10001 packages/extraction/tests /tests +COPY --from=build --chmod=0555 /build/block-landlock /tests/block-landlock diff --git a/packages/extraction/compose.yaml b/packages/extraction/compose.yaml new file mode 100644 index 00000000..f6bbf9f4 --- /dev/null +++ b/packages/extraction/compose.yaml @@ -0,0 +1,22 @@ +services: + extraction: + image: ${EXTRACTION_IMAGE:-librechat-extraction:local} + build: + context: ../.. + dockerfile: packages/extraction/Dockerfile + target: runtime + init: true + user: '10001:10001' + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + network_mode: none + mem_limit: 768m + cpus: 2 + pids_limit: 32 + tmpfs: + - /jobs:rw,noexec,nosuid,nodev,size=96m,nr_inodes=1024,uid=10001,gid=10001,mode=0700 + volumes: + - extraction-socket:/socket +volumes: + extraction-socket: diff --git a/packages/extraction/native/guard.c b/packages/extraction/native/guard.c new file mode 100644 index 00000000..ee1d2cbc --- /dev/null +++ b/packages/extraction/native/guard.c @@ -0,0 +1,156 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Stable UAPI values missing from Debian 12's older build headers. */ +#ifndef LANDLOCK_ACCESS_FS_TRUNCATE +#define LANDLOCK_ACCESS_FS_TRUNCATE (1ULL << 14) +#endif +#ifndef __NR_fchmodat2 +#define __NR_fchmodat2 452 +#endif + +#if defined(__x86_64__) +#define ARCH AUDIT_ARCH_X86_64 +#define LOADER "/lib64/ld-linux-x86-64.so.2" +#elif defined(__aarch64__) +#define ARCH AUDIT_ARCH_AARCH64 +#define LOADER "/lib/ld-linux-aarch64.so.1" +#else +#error Unsupported architecture +#endif + +static void fail_at(int line) { fprintf(stderr, "ISOLATION_UNAVAILABLE:%d:%d\n", line, errno); exit(125); } +#define fail() fail_at(__LINE__) +static void limit(int resource, rlim_t value) { + struct rlimit r = {value, value}; + if (setrlimit(resource, &r)) fail(); +} +static void allow(int rules, const char *path, __u64 access) { + int fd = open(path, O_PATH | O_CLOEXEC); + if (fd < 0) fail(); + struct landlock_path_beneath_attr rule = {.allowed_access = access, .parent_fd = fd}; + if (syscall(SYS_landlock_add_rule, rules, LANDLOCK_RULE_PATH_BENEATH, &rule, 0)) fail(); + close(fd); +} +#define DENY(nr) BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, nr, 0, 1), BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM) +static void restrict_syscalls(void) { + struct sock_filter filter[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, arch)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, ARCH, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), +#ifdef __x86_64__ + BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 0x40000000, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), +#endif + DENY(__NR_socket), DENY(__NR_socketpair), DENY(__NR_connect), + DENY(__NR_bind), DENY(__NR_listen), DENY(__NR_accept), DENY(__NR_accept4), + DENY(__NR_clone), DENY(__NR_clone3), +#ifdef __NR_fork + DENY(__NR_fork), DENY(__NR_vfork), +#endif + DENY(__NR_ptrace), DENY(__NR_process_vm_readv), DENY(__NR_process_vm_writev), + DENY(__NR_kill), DENY(__NR_tkill), DENY(__NR_tgkill), DENY(__NR_pidfd_send_signal), + DENY(__NR_rt_sigqueueinfo), DENY(__NR_rt_tgsigqueueinfo), + /* CPython marks opened files close-on-exec through FIOCLEX. */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_ioctl, 0, 3), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[1])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, FIOCLEX, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + /* Async descriptor ownership can cause kernel-delivered signals. */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_fcntl, 0, 7), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[1])), + DENY(F_SETOWN), DENY(F_SETOWN_EX), DENY(F_SETSIG), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + DENY(__NR_mq_open), DENY(__NR_mq_unlink), DENY(__NR_mq_timedsend), + DENY(__NR_mq_timedreceive), DENY(__NR_mq_notify), DENY(__NR_mq_getsetattr), + DENY(__NR_mount), DENY(__NR_umount2), DENY(__NR_pivot_root), DENY(__NR_chroot), + DENY(__NR_unshare), DENY(__NR_setns), DENY(__NR_bpf), DENY(__NR_userfaultfd), + DENY(__NR_io_uring_setup), DENY(__NR_perf_event_open), + DENY(__NR_setpriority), DENY(__NR_sched_setparam), DENY(__NR_sched_setscheduler), + DENY(__NR_sched_setaffinity), DENY(__NR_sched_setattr), + DENY(__NR_pidfd_open), DENY(__NR_pidfd_getfd), + DENY(__NR_keyctl), DENY(__NR_add_key), DENY(__NR_request_key), + DENY(__NR_shmget), DENY(__NR_shmat), DENY(__NR_shmctl), + DENY(__NR_semget), DENY(__NR_semop), DENY(__NR_semctl), + DENY(__NR_msgget), DENY(__NR_msgsnd), DENY(__NR_msgrcv), DENY(__NR_msgctl), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_prlimit64, 0, 3), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + DENY(__NR_utimensat), +#ifdef __NR_utime + DENY(__NR_utime), DENY(__NR_utimes), DENY(__NR_futimesat), +#endif +#ifdef __NR_fchmodat2 + DENY(__NR_fchmodat2), +#endif + DENY(__NR_open_by_handle_at), DENY(__NR_name_to_handle_at), + DENY(__NR_fchmod), DENY(__NR_fchmodat), DENY(__NR_fchown), DENY(__NR_fchownat), +#ifdef __NR_chmod + DENY(__NR_chmod), DENY(__NR_chown), DENY(__NR_lchown), +#endif + DENY(__NR_mknodat), DENY(__NR_setuid), DENY(__NR_setgid), + DENY(__NR_setreuid), DENY(__NR_setregid), DENY(__NR_setresuid), DENY(__NR_setresgid), + DENY(__NR_setgroups), DENY(__NR_setxattr), DENY(__NR_lsetxattr), DENY(__NR_fsetxattr), + DENY(__NR_removexattr), DENY(__NR_lremovexattr), DENY(__NR_fremovexattr), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW) + }; + struct sock_fprog program = {.len = sizeof(filter) / sizeof(filter[0]), .filter = filter}; + if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program)) fail(); +} +int main(int argc, char **argv) { + if (argc != 5 || geteuid() == 0) fail(); + /* Paths are supervisor-owned, never supplied in the HTTP protocol. */ + if (chdir(argv[1])) fail(); + pid_t parent = getppid(); + if (parent == 1 || prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() != parent) fail(); + limit(RLIMIT_AS, 512ULL * 1024 * 1024); + limit(RLIMIT_CPU, 8); + limit(RLIMIT_FSIZE, 4ULL * 1024 * 1024); + limit(RLIMIT_NOFILE, 64); + limit(RLIMIT_CORE, 0); + if (prctl(PR_SET_DUMPABLE, 0) || prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) fail(); + if (syscall(SYS_close_range, 3, ~0U, 0)) fail(); + int abi = syscall(SYS_landlock_create_ruleset, NULL, 0, LANDLOCK_CREATE_RULESET_VERSION); + if (abi < 3) fail(); + __u64 read = LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_READ_DIR; + __u64 write = LANDLOCK_ACCESS_FS_WRITE_FILE | LANDLOCK_ACCESS_FS_TRUNCATE | LANDLOCK_ACCESS_FS_MAKE_REG; + struct landlock_ruleset_attr attr = { + .handled_access_fs = (1ULL << 15) - 1 /* ABI 3, including REFER and TRUNCATE */ + }; + int rules = syscall(SYS_landlock_create_ruleset, &attr, sizeof(attr), 0); + if (rules < 0) fail(); + allow(rules, "/usr", read); + allow(rules, "/lib", read); + if (access("/lib64", F_OK) == 0) allow(rules, "/lib64", read); + allow(rules, argv[2], LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_EXECUTE); + allow(rules, LOADER, LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_EXECUTE); + allow(rules, argv[3], LANDLOCK_ACCESS_FS_READ_FILE); + allow(rules, argv[1], read | write); + allow(rules, "/dev/null", LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_WRITE_FILE); + allow(rules, "/dev/urandom", LANDLOCK_ACCESS_FS_READ_FILE); + if (syscall(SYS_landlock_restrict_self, rules, 0)) fail(); + close(rules); + restrict_syscalls(); + if (clearenv() || setenv("LANG", "C.UTF-8", 1)) fail(); + execl(argv[2], argv[2], "-I", "-B", argv[3], argv[4], (char *)NULL); + fail(); +} diff --git a/packages/extraction/package-lock.json b/packages/extraction/package-lock.json new file mode 100644 index 00000000..86e21c9f --- /dev/null +++ b/packages/extraction/package-lock.json @@ -0,0 +1,64 @@ +{ + "name": "@librechat/extraction-service", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@librechat/extraction-service", + "version": "0.1.0", + "devDependencies": { + "@types/node": "24.10.1", + "prettier": "2.4.1", + "typescript": "5.9.3" + }, + "engines": { + "node": ">=24 <25" + } + }, + "node_modules/@types/node": { + "version": "24.10.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", + "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/prettier": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz", + "integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin-prettier.js" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/packages/extraction/package.json b/packages/extraction/package.json new file mode 100644 index 00000000..5934c126 --- /dev/null +++ b/packages/extraction/package.json @@ -0,0 +1,22 @@ +{ + "name": "@librechat/extraction-service", + "version": "0.1.0", + "private": true, + "type": "module", + "engines": { + "node": ">=24 <25" + }, + "scripts": { + "build": "tsc", + "typecheck": "tsc --noEmit", + "test": "npm run build && node --test dist/*.test.js", + "start": "node dist/main.js", + "test:container": "bash tests/container.sh", + "format:check": "prettier --check src tests/*.mjs *.json compose.yaml ../../.github/workflows/extraction.yml" + }, + "devDependencies": { + "@types/node": "24.10.1", + "prettier": "2.4.1", + "typescript": "5.9.3" + } +} diff --git a/packages/extraction/src/backend.ts b/packages/extraction/src/backend.ts new file mode 100644 index 00000000..a3831c65 --- /dev/null +++ b/packages/extraction/src/backend.ts @@ -0,0 +1,166 @@ +import { spawn } from 'node:child_process'; +import { constants } from 'node:fs'; +import { mkdtemp, open, rm, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { ExtractionError, policy, validateResult } from './protocol.js'; +import type { Format } from './protocol.js'; + +export interface Backend { + healthy: boolean; + create(): Promise; + run( + job: string, + format: Format, + digest: string, + signal: AbortSignal + ): Promise; + cleanup(job: string): Promise; +} +export interface Runtime { + jobs: string; + guard: string; + python: string; + worker: string; +} +const parserErrors = new Set([ + 'INVALID_DOCUMENT', + 'EMPTY_OUTPUT', + 'OUTPUT_LIMIT', + 'DECOMPRESSION_LIMIT', + 'STRUCTURE_LIMIT', + 'ENCRYPTED_DOCUMENT', + 'UNSUPPORTED_ENCODING', + 'UNSUPPORTED_CONTENT', + 'RESOURCE_LIMIT', +]); +export class RestrictedBackend implements Backend { + healthy = true; + constructor(private readonly runtime: Runtime) {} + async create(): Promise { + if (!this.healthy) throw new ExtractionError('UNAVAILABLE'); + return mkdtemp(join(this.runtime.jobs, 'job-')); + } + private async execute( + job: string, + mode: string, + signal: AbortSignal + ): Promise { + signal.throwIfAborted(); + const child = spawn( + this.runtime.guard, + [job, this.runtime.python, this.runtime.worker, mode], + { + detached: true, + stdio: ['ignore', 'pipe', 'pipe'], + env: {}, + } + ); + let output = ''; + let count = 0; + let overflow = false; + const kill = () => { + if ( + child.pid && + child.exitCode === null && + child.signalCode === null + ) { + try { + process.kill(-child.pid, 'SIGKILL'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') + this.healthy = false; + } + } + }; + signal.addEventListener('abort', kill, { once: true }); + if (signal.aborted) kill(); + let spawnError = false; + child.on('error', () => { + spawnError = true; + }); + child.stdout.on('data', (chunk: Buffer) => { + count += chunk.length; + if (count > 1024) { + overflow = true; + kill(); + } else output += chunk.toString('ascii'); + }); + child.stderr.on('data', (chunk: Buffer) => { + count += chunk.length; + if (count > 2048) { + overflow = true; + kill(); + } + }); + const code = await new Promise(resolve => + child.once('close', resolve) + ); + signal.removeEventListener('abort', kill); + signal.throwIfAborted(); + if (spawnError || overflow) throw new ExtractionError('UNAVAILABLE'); + if (code !== 0) { + const value = output.trim(); + if (parserErrors.has(value)) + throw new ExtractionError(value as 'INVALID_DOCUMENT'); + throw new ExtractionError( + code === null ? 'RESOURCE_LIMIT' : 'UNAVAILABLE' + ); + } + return output; + } + async probe(): Promise { + const job = await this.create(); + try { + const output = await this.execute( + job, + 'probe', + AbortSignal.timeout(policy.deadlineMs) + ); + if (output.trim() !== 'READY') + throw new ExtractionError('UNAVAILABLE'); + } finally { + await this.cleanup(job); + } + } + async run( + job: string, + format: Format, + digest: string, + signal: AbortSignal + ): Promise { + await this.execute(job, format, signal); + const file = await open( + join(job, 'result'), + constants.O_RDONLY | constants.O_NOFOLLOW + ); + try { + const stat = await file.stat(); + if ( + !stat.isFile() || + stat.nlink !== 1 || + stat.size > policy.resultBytes + ) { + throw new ExtractionError('OUTPUT_LIMIT'); + } + const bytes = await file.readFile(); + signal.throwIfAborted(); + validateResult(bytes, format, digest); + return bytes; + } finally { + await file.close(); + } + } + async cleanup(job: string): Promise { + try { + await rm(job, { recursive: true, force: true, maxRetries: 0 }); + } catch { + this.healthy = false; + throw new ExtractionError('UNAVAILABLE'); + } + } +} + +/** Used by focused worker tests, not by HTTP callers. */ +export async function writeInput(job: string, input: Buffer): Promise { + await writeFile(join(job, 'input'), input, { flag: 'wx', mode: 0o600 }); +} diff --git a/packages/extraction/src/boundary.test.ts b/packages/extraction/src/boundary.test.ts new file mode 100644 index 00000000..da54749e --- /dev/null +++ b/packages/extraction/src/boundary.test.ts @@ -0,0 +1,22 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { test } from 'node:test'; + +test('supervisor has no parser or application stack dependencies', async () => { + const manifest = JSON.parse( + await readFile(new URL('../package.json', import.meta.url), 'utf8') + ); + assert.equal(manifest.dependencies, undefined); + for (const name of ['server.ts', 'backend.ts', 'protocol.ts', 'main.ts']) { + const source = await readFile( + new URL(`../src/${name}`, import.meta.url), + 'utf8' + ); + for (const match of source.matchAll(/from ['"]([^'"]+)['"]/g)) { + assert.ok( + match[1]?.startsWith('node:') || match[1]?.startsWith('./'), + match[1] + ); + } + } +}); diff --git a/packages/extraction/src/health.ts b/packages/extraction/src/health.ts new file mode 100644 index 00000000..641575c4 --- /dev/null +++ b/packages/extraction/src/health.ts @@ -0,0 +1,15 @@ +import { request } from 'node:http'; +const req = request( + { + socketPath: '/socket/extraction.sock', + path: '/v1/capabilities', + timeout: 1000, + }, + res => { + res.resume(); + res.once('end', () => process.exit(res.statusCode === 200 ? 0 : 1)); + } +); +req.on('timeout', () => req.destroy()); +req.on('error', () => process.exit(1)); +req.end(); diff --git a/packages/extraction/src/main.ts b/packages/extraction/src/main.ts new file mode 100644 index 00000000..424a6ed6 --- /dev/null +++ b/packages/extraction/src/main.ts @@ -0,0 +1,64 @@ +import { chmod, lstat, readdir, unlink } from 'node:fs/promises'; +import { connect } from 'node:net'; +import { RestrictedBackend } from './backend.js'; +import { extractionServer } from './server.js'; + +// Deployment-owned locations, not caller-selected paths or runtimes. +const backend = new RestrictedBackend({ + jobs: '/jobs', + guard: '/opt/extraction/guard', + python: '/usr/local/bin/python3.13', + worker: '/opt/extraction/extract.py', +}); +try { + if ((await readdir('/jobs')).length) throw new Error('DIRTY_SCRATCH'); + try { + const stat = await lstat('/socket/extraction.sock'); + if (!stat.isSocket()) throw new Error('INVALID_SOCKET'); + await new Promise((resolve, reject) => { + const client = connect('/socket/extraction.sock'); + client.setTimeout(1000, () => { + client.destroy(); + reject(new Error('SOCKET_EXISTS')); + }); + client.once('connect', () => { + client.destroy(); + reject(new Error('SOCKET_EXISTS')); + }); + client.once('error', error => { + if ((error as NodeJS.ErrnoException).code === 'ECONNREFUSED') + resolve(); + else reject(error); + }); + }); + await unlink('/socket/extraction.sock'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + await backend.probe(); + const server = extractionServer(backend); + server.on('error', () => { + console.error('EXTRACTION_UNAVAILABLE'); + process.exit(1); + }); + process.umask(0o007); + server.listen('/socket/extraction.sock', async () => { + try { + await chmod('/socket/extraction.sock', 0o660); + } catch { + console.error('EXTRACTION_UNAVAILABLE'); + process.exit(1); + } + console.log('Extraction service ready'); + }); + const shutdown = () => { + server.close(); + server.closeAllConnections(); + setTimeout(() => process.exit(0), 12_000).unref(); + }; + process.once('SIGTERM', shutdown); + process.once('SIGINT', shutdown); +} catch { + console.error('EXTRACTION_UNAVAILABLE'); + process.exit(1); +} diff --git a/packages/extraction/src/protocol.test.ts b/packages/extraction/src/protocol.test.ts new file mode 100644 index 00000000..6688fa49 --- /dev/null +++ b/packages/extraction/src/protocol.test.ts @@ -0,0 +1,68 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { policy, validateResult } from './protocol.js'; +const digest = 'a'.repeat(64); +const value = (text = 'hello') => ({ + version: 1, + operation: 'document.extract-text', + format: 'pdf', + inputSha256: digest, + textBytes: Buffer.byteLength(text), + segments: [{ kind: 'page', index: 1, text }], +}); +const encode = (result: unknown) => Buffer.from(JSON.stringify(result)); +test('preserves ordered page segments and validates digest and byte count', () => { + const result = value(); + assert.deepEqual(validateResult(encode(result), 'pdf', digest), result); + for (const changed of [ + { ...result, inputSha256: 'b'.repeat(64) }, + { ...result, textBytes: 4 }, + { ...result, extra: true }, + { ...result, format: 'docx' }, + { ...result, segments: [{ kind: 'page', index: 0, text: 'hello' }] }, + { ...result, segments: [{ kind: 'sheet', index: 1, text: 'hello' }] }, + ]) + assert.throws(() => validateResult(encode(changed), 'pdf', digest)); +}); +test('rejects malformed UTF-8, lone surrogates, emptiness and invalid JSON', () => { + for (const bytes of [ + Buffer.from([0xff]), + Buffer.from('{'), + encode(value('\ud800')), + encode(value(' ')), + encode(null), + ]) { + assert.throws(() => validateResult(bytes, 'pdf', digest)); + } +}); +test('accepts exact text limit and rejects one byte more or oversized wire result', () => { + assert.equal( + validateResult( + encode(value('x'.repeat(policy.textBytes))), + 'pdf', + digest + ).textBytes, + policy.textBytes + ); + assert.throws(() => + validateResult( + encode(value('x'.repeat(policy.textBytes + 1))), + 'pdf', + digest + ) + ); + assert.throws(() => + validateResult(Buffer.alloc(policy.resultBytes + 1), 'pdf', digest) + ); +}); +test('rejects excess pages and does not accept PDF pages as DOCX structure', () => { + const result = value(); + result.segments = Array.from( + { length: policy.maxSegments + 1 }, + (_, index) => ({ kind: 'page', index: index + 1, text: 'hello' }) + ); + assert.throws(() => validateResult(encode(result), 'pdf', digest)); + assert.throws(() => + validateResult(encode({ ...value(), format: 'docx' }), 'docx', digest) + ); +}); diff --git a/packages/extraction/src/protocol.ts b/packages/extraction/src/protocol.ts new file mode 100644 index 00000000..935c902c --- /dev/null +++ b/packages/extraction/src/protocol.ts @@ -0,0 +1,107 @@ +import { createHash } from 'node:crypto'; + +export const policy = Object.freeze({ + inputBytes: 10 * 1024 * 1024, + textBytes: 1024 * 1024, + resultBytes: 3 * 1024 * 1024, + deadlineMs: 10_000, + maxSegments: 128, +}); +export type Format = 'pdf' | 'docx'; +export type ErrorCode = + | 'INVALID_REQUEST' + | 'INPUT_LIMIT' + | 'BUSY' + | 'DEADLINE' + | 'CANCELLED' + | 'UNAVAILABLE' + | 'INVALID_DOCUMENT' + | 'EMPTY_OUTPUT' + | 'OUTPUT_LIMIT' + | 'DECOMPRESSION_LIMIT' + | 'STRUCTURE_LIMIT' + | 'ENCRYPTED_DOCUMENT' + | 'UNSUPPORTED_ENCODING' + | 'UNSUPPORTED_CONTENT' + | 'RESOURCE_LIMIT'; +export class ExtractionError extends Error { + constructor(readonly code: ErrorCode) { + super(code); + } +} +export interface Segment { + kind: 'page' | 'document'; + index: number; + text: string; +} +export interface Result { + version: 1; + operation: 'document.extract-text'; + format: Format; + inputSha256: string; + textBytes: number; + segments: Segment[]; +} +const keys = (value: object, expected: string[]) => + Object.keys(value).length === expected.length && + expected.every(key => Object.hasOwn(value, key)); +export function validateResult( + bytes: Buffer, + format: Format, + digest: string +): Result { + if (bytes.length > policy.resultBytes) + throw new ExtractionError('OUTPUT_LIMIT'); + let result: Result; + try { + result = JSON.parse( + new TextDecoder('utf-8', { fatal: true }).decode(bytes) + ) as Result; + } catch { + throw new ExtractionError('INVALID_DOCUMENT'); + } + if ( + !result || + typeof result !== 'object' || + !keys(result, [ + 'version', + 'operation', + 'format', + 'inputSha256', + 'textBytes', + 'segments', + ]) || + result.version !== 1 || + result.operation !== 'document.extract-text' || + result.format !== format || + result.inputSha256 !== digest || + !Array.isArray(result.segments) || + !result.segments.length || + result.segments.length > (format === 'pdf' ? policy.maxSegments : 1) + ) { + throw new ExtractionError('INVALID_DOCUMENT'); + } + let size = 0; + let nonempty = false; + for (const [index, segment] of result.segments.entries()) { + if ( + !segment || + typeof segment !== 'object' || + !keys(segment, ['kind', 'index', 'text']) || + segment.kind !== (format === 'pdf' ? 'page' : 'document') || + segment.index !== index + 1 || + typeof segment.text !== 'string' || + !segment.text.isWellFormed() + ) + throw new ExtractionError('INVALID_DOCUMENT'); + size += Buffer.byteLength(segment.text); + nonempty ||= Boolean(segment.text.trim()); + } + if (size > policy.textBytes) throw new ExtractionError('OUTPUT_LIMIT'); + if (size !== result.textBytes || !nonempty) + throw new ExtractionError('INVALID_DOCUMENT'); + return result; +} +export function sha256(bytes: Buffer): string { + return createHash('sha256').update(bytes).digest('hex'); +} diff --git a/packages/extraction/src/server.test.ts b/packages/extraction/src/server.test.ts new file mode 100644 index 00000000..60f9bc63 --- /dev/null +++ b/packages/extraction/src/server.test.ts @@ -0,0 +1,267 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, rm, readdir } from 'node:fs/promises'; +import { request } from 'node:http'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { once } from 'node:events'; +import { test } from 'node:test'; +import { extractionServer } from './server.js'; +import { ExtractionError, policy } from './protocol.js'; +import type { Backend } from './backend.js'; + +async function fixture(deadline = 500) { + const root = await mkdtemp(join(tmpdir(), 'extraction-test-')); + const events: string[] = []; + const backend: Backend = { + healthy: true, + create: async () => { + events.push('create'); + return mkdtemp(join(root, 'job-')); + }, + run: async () => { + events.push('run'); + return Buffer.from('{"segments":[]}'); + }, + cleanup: async job => { + events.push('cleanup'); + await rm(job, { recursive: true }); + }, + }; + const server = extractionServer(backend, deadline); + const socketPath = join(root, 'service.sock'); + server.listen(socketPath); + await once(server, 'listening'); + return { + backend, + events, + root, + socketPath, + server, + close: async () => { + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + await rm(root, { recursive: true, force: true }); + }, + }; +} +const headers = { + 'Content-Type': 'application/octet-stream', + 'Content-Length': '4', + 'X-Extraction-Version': '1', + 'X-Extraction-Format': 'pdf', +}; +async function send( + socketPath: string, + overrides = {}, + path = '/v1/extract-text' +) { + return new Promise<{ status: number; body: string }>((resolve, reject) => { + const req = request( + { + socketPath, + path, + method: 'POST', + headers: { ...headers, ...overrides }, + }, + res => { + let body = ''; + res.on('data', bytes => { + body += bytes; + }); + res.on('end', () => resolve({ status: res.statusCode!, body })); + } + ); + req.on('error', reject); + req.end('test'); + }); +} +test('binary upload executes once and cleanup completes before success', async () => { + const f = await fixture(); + try { + assert.equal((await send(f.socketPath)).status, 200); + assert.deepEqual(f.events, ['create', 'run', 'cleanup']); + assert.deepEqual(await readdir(f.root), ['service.sock']); + } finally { + await f.close(); + } +}); +test('closed parameters and declared input limit are rejected before admission', async () => { + const f = await fixture(); + try { + for (const changes of [ + { 'X-Extraction-Format': 'xls' }, + { 'X-Extraction-Version': '2' }, + { 'X-Extraction-Url': 'https://example.com' }, + { 'Content-Type': 'text/plain' }, + ]) { + assert.equal((await send(f.socketPath, changes)).status, 400); + } + assert.equal( + (await send(f.socketPath, {}, '/v1/extract-text?path=x')).status, + 404 + ); + assert.equal( + ( + await send(f.socketPath, { + 'Content-Length': String(policy.inputBytes + 1), + }) + ).status, + 413 + ); + assert.equal(f.events.length, 0); + } finally { + await f.close(); + } +}); +test('cleanup failure suppresses success and quarantines backend', async () => { + const f = await fixture(); + f.backend.cleanup = async () => { + f.backend.healthy = false; + throw new ExtractionError('UNAVAILABLE'); + }; + try { + assert.equal((await send(f.socketPath)).status, 503); + assert.equal((await send(f.socketPath)).status, 503); + assert.equal(f.events.filter(e => e === 'run').length, 1); + } finally { + await f.close(); + } +}); +test('deadline aborts execution and waits for its close before cleanup', async () => { + const f = await fixture(50); + f.backend.run = async (_job, _format, _digest, signal) => { + await new Promise(resolve => + signal.addEventListener( + 'abort', + () => { + setTimeout(() => { + f.events.push('closed'); + resolve(); + }, 15); + }, + { once: true } + ) + ); + signal.throwIfAborted(); + return Buffer.alloc(0); + }; + try { + assert.equal((await send(f.socketPath)).status, 504); + assert.deepEqual(f.events, ['create', 'closed', 'cleanup']); + } finally { + await f.close(); + } +}); +test('upload disconnect and stalled upload both release scratch and admission', async () => { + const f = await fixture(60); + try { + for (const disconnect of [true, false]) { + const req = request({ + socketPath: f.socketPath, + path: '/v1/extract-text', + method: 'POST', + headers, + }); + req.on('error', () => {}); + req.flushHeaders(); + req.write('t'); + await new Promise(resolve => setTimeout(resolve, 20)); + if (disconnect) req.destroy(); + await new Promise(resolve => setTimeout(resolve, 80)); + assert.deepEqual(await readdir(f.root), ['service.sock']); + } + assert.equal(f.events.includes('run'), false); + assert.equal((await send(f.socketPath)).status, 200); + } finally { + await f.close(); + } +}); +test('two active jobs bound concurrency without a queue', async () => { + const f = await fixture(200); + let active = 0; + f.backend.run = async (_job, _format, _digest, signal) => { + active++; + await new Promise(resolve => + signal.addEventListener('abort', () => resolve(), { once: true }) + ); + signal.throwIfAborted(); + return Buffer.alloc(0); + }; + try { + const a = send(f.socketPath), + b = send(f.socketPath); + while (active !== 2) + await new Promise(resolve => setTimeout(resolve, 5)); + assert.equal((await send(f.socketPath)).status, 429); + assert.equal((await a).status, 504); + assert.equal((await b).status, 504); + } finally { + await f.close(); + } +}); +test('deadline closes stalled output readers while keeping their slots admitted', async () => { + const f = await fixture(250); + f.backend.run = async () => Buffer.alloc(policy.resultBytes, 'a'); + const clients = [0, 1].map(() => { + const req = request( + { + socketPath: f.socketPath, + path: '/v1/extract-text', + method: 'POST', + headers, + }, + res => { + res.pause(); + res.on('error', () => {}); + } + ); + req.on('error', () => {}); + req.end('test'); + return req; + }); + try { + while (f.events.filter(e => e === 'cleanup').length !== 2) + await new Promise(resolve => setTimeout(resolve, 5)); + assert.equal((await send(f.socketPath)).status, 429); + await new Promise(resolve => setTimeout(resolve, 300)); + f.backend.run = async () => Buffer.from('{}'); + assert.equal((await send(f.socketPath)).status, 200); + } finally { + clients.forEach(client => client.destroy()); + await f.close(); + } +}); +test('duplicate extraction control headers are rejected before worker creation', async () => { + const f = await fixture(); + try { + assert.equal( + ( + await send(f.socketPath, { + 'X-Extraction-Format': ['pdf', 'pdf'], + }) + ).status, + 400 + ); + assert.equal(f.events.length, 0); + } finally { + await f.close(); + } +}); + +test('unsupported document content returns a safe failure after cleanup', async () => { + const f = await fixture(); + f.backend.run = async () => { + throw new ExtractionError('UNSUPPORTED_CONTENT'); + }; + try { + const response = await send(f.socketPath); + assert.equal(response.status, 422); + assert.deepEqual(JSON.parse(response.body), { + error: { code: 'UNSUPPORTED_CONTENT' }, + }); + assert.deepEqual(f.events, ['create', 'cleanup']); + assert.deepEqual(await readdir(f.root), ['service.sock']); + } finally { + await f.close(); + } +}); diff --git a/packages/extraction/src/server.ts b/packages/extraction/src/server.ts new file mode 100644 index 00000000..b3bd37c8 --- /dev/null +++ b/packages/extraction/src/server.ts @@ -0,0 +1,260 @@ +import { createHash } from 'node:crypto'; +import { createServer } from 'node:http'; +import { open } from 'node:fs/promises'; +import { join } from 'node:path'; +import { ExtractionError, policy } from './protocol.js'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import type { Backend } from './backend.js'; +import type { Format } from './protocol.js'; + +export const capabilities = Object.freeze({ + version: 1, + operation: 'document.extract-text', + formats: ['pdf', 'docx'], + structure: { + pdf: 'ordered pages, one-based index', + docx: 'document body, paragraphs and tables', + }, + limitations: { + pdf: 'Text only. Simple fonts need interpretable standard/Unicode mappings. Type3/composite fonts need complete Unicode maps. Unmapped glyphs, codec recovery and graphics-state font selection are rejected.', + docx: 'Body paragraphs/tables and direct runs/hyperlinks only. Other content structures, fields, images and wrappers are rejected; formatting is ignored.', + }, + limits: { + ...policy, + concurrency: 2, + expandedBytes: 32 * 1024 * 1024, + archiveEntries: 512, + tableColumns: 512, + }, +}); +const status = (code: string): number => { + if (code === 'BUSY') return 429; + if (code === 'UNAVAILABLE') return 503; + if (code === 'DEADLINE') return 504; + if ( + code === 'INPUT_LIMIT' || + code === 'OUTPUT_LIMIT' || + code === 'DECOMPRESSION_LIMIT' || + code === 'STRUCTURE_LIMIT' + ) + return 413; + if (code === 'INVALID_REQUEST') return 400; + return 422; +}; +function reply(res: ServerResponse, code: number, bytes: Buffer): void { + res.writeHead(code, { + 'Content-Type': 'application/json; charset=utf-8', + 'Content-Length': bytes.length, + Connection: 'close', + 'Cache-Control': 'no-store', + 'X-Content-Type-Options': 'nosniff', + }); + res.end(bytes); +} +function requestFormat(req: IncomingMessage): Format { + const seen = new Set(); + for (let i = 0; i < req.rawHeaders.length; i += 2) { + const name = req.rawHeaders[i]!.toLowerCase(); + if ( + name.startsWith('x-extraction-') && + !['x-extraction-version', 'x-extraction-format'].includes(name) + ) { + throw new ExtractionError('INVALID_REQUEST'); + } + if ( + [ + 'content-length', + 'content-type', + 'x-extraction-version', + 'x-extraction-format', + ].includes(name) + ) { + if (seen.has(name)) throw new ExtractionError('INVALID_REQUEST'); + seen.add(name); + } + } + if ( + req.headers['content-type'] !== 'application/octet-stream' || + req.headers['x-extraction-version'] !== '1' || + req.headers['transfer-encoding'] + ) { + throw new ExtractionError('INVALID_REQUEST'); + } + const format = req.headers['x-extraction-format']; + if (format !== 'pdf' && format !== 'docx') + throw new ExtractionError('INVALID_REQUEST'); + const length = req.headers['content-length']; + if (typeof length !== 'string' || !/^[1-9]\d*$/.test(length)) + throw new ExtractionError('INVALID_REQUEST'); + if ( + !Number.isSafeInteger(Number(length)) || + Number(length) > policy.inputBytes + ) + throw new ExtractionError('INPUT_LIMIT'); + return format; +} +export function extractionServer( + backend: Backend, + deadlineMs: number = policy.deadlineMs +) { + if ( + !Number.isInteger(deadlineMs) || + deadlineMs <= 0 || + deadlineMs > policy.deadlineMs + ) + throw new Error('INVALID_DEADLINE'); + let admitted = 0; + const server = createServer( + { + maxHeaderSize: 4096, + requireHostHeader: true, + connectionsCheckingInterval: 1000, + }, + (req, res) => { + void handle(req, res); + } + ); + server.maxConnections = 16; + server.maxRequestsPerSocket = 1; + server.headersTimeout = 5000; + server.requestTimeout = policy.deadlineMs; + server.setTimeout(policy.deadlineMs, socket => socket.destroy()); + server.on('checkContinue', (req, res) => { + reply(res, 400, Buffer.from('{"error":{"code":"INVALID_REQUEST"}}')); + }); + async function handle( + req: IncomingMessage, + res: ServerResponse + ): Promise { + if (req.method === 'GET' && req.url === '/v1/capabilities') { + reply( + res, + backend.healthy ? 200 : 503, + Buffer.from(JSON.stringify(capabilities)) + ); + return; + } + if (req.method !== 'POST' || req.url !== '/v1/extract-text') { + reply( + res, + 404, + Buffer.from('{"error":{"code":"INVALID_REQUEST"}}') + ); + return; + } + let format: Format; + try { + format = requestFormat(req); + } catch (error) { + const code = + error instanceof ExtractionError + ? error.code + : 'INVALID_REQUEST'; + reply( + res, + status(code), + Buffer.from(JSON.stringify({ error: { code } })) + ); + return; + } + if (!backend.healthy || admitted >= 2) { + const code = backend.healthy ? 'BUSY' : 'UNAVAILABLE'; + reply( + res, + status(code), + Buffer.from(JSON.stringify({ error: { code } })) + ); + return; + } + admitted++; + const controller = new AbortController(); + const timeout = setTimeout( + () => controller.abort(new ExtractionError('DEADLINE')), + deadlineMs + ); + const aborted = () => + controller.abort(new ExtractionError('CANCELLED')); + controller.signal.addEventListener( + 'abort', + () => { + if (!req.complete) req.destroy(); + if (res.headersSent) res.destroy(); + }, + { once: true } + ); + req.once('aborted', aborted); + res.once('close', () => { + if (!res.writableFinished) aborted(); + }); + const delivered = new Promise(resolve => { + res.once('finish', resolve); + res.once('close', resolve); + }); + let job: string | undefined; + let bytes: Buffer | undefined; + let failure: unknown; + try { + controller.signal.throwIfAborted(); + job = await backend.create(); + const file = await open(join(job, 'input'), 'wx', 0o600); + const digest = createHash('sha256'); + let size = 0; + try { + for await (const chunk of req) { + controller.signal.throwIfAborted(); + size += chunk.length; + if (size > policy.inputBytes) + throw new ExtractionError('INPUT_LIMIT'); + digest.update(chunk); + await file.writeFile(chunk); + } + } finally { + await file.close(); + } + if (size !== Number(req.headers['content-length'])) + throw new ExtractionError('INVALID_REQUEST'); + controller.signal.throwIfAborted(); + bytes = await backend.run( + job, + format, + digest.digest('hex'), + controller.signal + ); + } catch (error) { + failure = error; + } finally { + if (job) { + try { + await backend.cleanup(job); + } catch (error) { + failure = error; + } + } + } + try { + if (controller.signal.aborted && !failure) + failure = controller.signal.reason; + if (controller.signal.reason instanceof ExtractionError) + failure = controller.signal.reason; + if (!res.destroyed) { + if (failure || !bytes) { + const code = + failure instanceof ExtractionError + ? failure.code + : 'UNAVAILABLE'; + reply( + res, + status(code), + Buffer.from(JSON.stringify({ error: { code } })) + ); + } else reply(res, 200, bytes); + } + await delivered; + } finally { + clearTimeout(timeout); + req.removeListener('aborted', aborted); + admitted--; + } + } + return server; +} diff --git a/packages/extraction/tests/adversarial.py b/packages/extraction/tests/adversarial.py new file mode 100644 index 00000000..710c3a11 --- /dev/null +++ b/packages/extraction/tests/adversarial.py @@ -0,0 +1,488 @@ +"""Create bounded adversarial test documents inside the disposable test container.""" +from pathlib import Path +import struct +import zipfile +from docx import Document +from pypdf import PdfReader, PdfWriter + +root = Path('/tests/fixtures') +(root / 'bad.docx').write_bytes(b'PK\x03\x04not a zip') +with zipfile.ZipFile(root / 'entries.docx', 'w') as file: + for index in range(513): + file.writestr(str(index), 'x') +with zipfile.ZipFile(root / 'duplicate.docx', 'w') as file: + file.writestr('same', 'a') + file.writestr('same', 'b') +with zipfile.ZipFile(root / 'bomb.docx', 'w', compression=zipfile.ZIP_DEFLATED) as file: + file.writestr('content.xml', b'x' * (8 * 1024 * 1024 + 1)) +with zipfile.ZipFile(root / 'aggregate.docx', 'w', compression=zipfile.ZIP_DEFLATED) as file: + for index in range(5): + file.writestr(str(index), b'x' * (7 * 1024 * 1024)) +for name, value in [('utf8.docx', b'\xff'), ('entities.docx', b']>')]: + with zipfile.ZipFile(root / name, 'w') as file: + file.writestr('word/document.xml', value) +with zipfile.ZipFile(root / 'forged.docx', 'w', compression=zipfile.ZIP_DEFLATED) as file: + file.writestr('content.xml', 'forged-content' * 20) +value = bytearray((root / 'forged.docx').read_bytes()) +central = value.index(b'PK\x01\x02') +struct.pack_into('') + 2 + value = value[:declaration_end] + b']>' + value[declaration_end:] + value = value.replace(b'First paragraph', b'&controlled;') + elif payload == 'utf8': + value = value.replace(b'First paragraph', b'\xff') + elif member == 'word/_rels/document.xml.rels': + member = 'word/_rels/document.bin.rels' + target.writestr(member, value) +print('DOCX completeness and content-type fixtures generated') + +# The supported structure is closed, not an expanding list of forbidden tags. +for tag in ['dir', 'bdo', 'smartTag', 'unknownWrapper']: + for placement in ['paragraph', 'cell', 'hyperlink']: + document = Document() + document.add_paragraph('Ordinary text must not hide wrapped content') + paragraph = (document.add_table(rows=1, cols=1).cell(0, 0).add_paragraph() + if placement == 'cell' else document.add_paragraph()) + run = paragraph.add_run('Wrapped run text')._r + parent = run.getparent() + if placement == 'hyperlink': + link = OxmlElement('w:hyperlink') + link.set(qn('w:anchor'), 'test') + parent.append(link) + parent = link + wrapper = OxmlElement('w:' + tag) + if tag in ('dir', 'bdo'): + wrapper.set(qn('w:val'), 'rtl') + wrapper.append(run) + parent.append(wrapper) + document.save(root / f'wrapper-{tag}-{placement}.docx') + +# A malformed unknown block and run child cannot bypass the same invariant. +document = Document() +document.add_paragraph('Ordinary text') +paragraph = document.add_paragraph('Unknown body wrapper text') +parent = paragraph._p.getparent() +wrapper = OxmlElement('w:unknownWrapper') +wrapper.append(paragraph._p) +parent.insert(0, wrapper) +document.save(root / 'wrapper-block.docx') +document = Document() +document.add_paragraph('Ordinary text') +run = document.add_paragraph().add_run('Run wrapper text')._r +text = run[0] +wrapper = OxmlElement('w:unknownWrapper') +wrapper.append(text) +run.append(wrapper) +document.save(root / 'wrapper-run.docx') + +# Accepted formatting, hyperlinks, bookmarks and run text primitives retain order. +document = Document() +paragraph = document.add_paragraph() +bookmark = OxmlElement('w:bookmarkStart') +bookmark.set(qn('w:id'), '1') +bookmark.set(qn('w:name'), 'test') +paragraph._p.append(bookmark) +paragraph.add_run('Bold ').bold = True +link = OxmlElement('w:hyperlink') +link.set(qn('w:anchor'), 'test') +run = OxmlElement('w:r') +text = OxmlElement('w:t') +text.text = 'Link' +run.append(text) +link.append(run) +paragraph._p.append(link) +paragraph.add_run('\tTabbed\nNext') +run = paragraph.add_run()._r +run.append(OxmlElement('w:noBreakHyphen')) +run.append(OxmlElement('w:ptab')) +run.append(OxmlElement('w:cr')) +text = OxmlElement('w:t') +text.text = 'End' +run.append(text) +run.append(OxmlElement('w:lastRenderedPageBreak')) +bookmark = OxmlElement('w:bookmarkEnd') +bookmark.set(qn('w:id'), '1') +paragraph._p.append(bookmark) +document.save(root / 'supported-runs.docx') +print('DOCX closed-structure fixtures generated') + +# Physical cells are unique. A merged origin's text must never be repeated. +for name, end in [('horizontal.docx', (0, 1)), ('vertical.docx', (1, 0)), + ('rectangle.docx', (1, 1))]: + document = Document() + table = document.add_table(rows=2, cols=3) + for row in range(2): + for column in range(3): + table.cell(row, column).text = f'R{row}C{column}' + table.cell(0, 0).merge(table.cell(*end)).text = 'Merged text' + document.save(root / name) + +document = Document() +table = document.add_table(rows=2, cols=2) +origin = table.cell(0, 0).merge(table.cell(1, 1)) +origin.text = 'Before merged nested table' +origin.add_table(rows=1, cols=1).cell(0, 0).text = 'Unique nested content' +origin.add_paragraph('After merged nested table') +document.save(root / 'merged-nested.docx') + +for name, end in [('merged-large.docx', (0, 1)), ('merged-large-vertical.docx', (1, 0))]: + document = Document() + table = document.add_table(rows=2, cols=2) + table.cell(0, 0).merge(table.cell(*end)).text = 'x' * (600 * 1024) + document.save(root / name) + +document = Document() +table = document.add_table(rows=1, cols=3) +table.cell(0, 1).text = 'Middle cell' +row = table.rows[0]._tr +row.remove(row.tc_lst[-1]) +row.remove(row.tc_lst[0]) +properties = row.get_or_add_trPr() +for tag in ('gridBefore', 'gridAfter'): + value = OxmlElement('w:' + tag) + value.set(qn('w:val'), '1') + properties.append(value) +document.save(root / 'omitted-cells.docx') + +for name, mode in [('merge-content.docx', 'content'), ('merge-orphan.docx', 'orphan'), + ('merge-span.docx', 'span'), ('merge-legacy.docx', 'legacy'), + ('wide-table.docx', 'width')]: + document = Document() + table = document.add_table(rows=2, cols=2) + if mode == 'legacy': + value = OxmlElement('w:hMerge') + value.set(qn('w:val'), 'restart') + table.cell(0, 0)._tc.get_or_add_tcPr().append(value) + elif mode == 'width': + for _ in range(511): + table._tbl.tblGrid.append(OxmlElement('w:gridCol')) + else: + table.cell(0, 0).merge(table.cell(1, 0)).text = 'Origin text' + continuation = table.rows[1]._tr.tc_lst[0] + if mode == 'content': + continuation.p_lst[0].append(OxmlElement('w:r')) + text = OxmlElement('w:t') + text.text = 'Unexpected continuation text' + continuation.p_lst[0][-1].append(text) + elif mode == 'orphan': + table.rows[0]._tr.tc_lst[0].get_or_add_tcPr().remove( + table.rows[0]._tr.tc_lst[0].tcPr.vMerge) + elif mode == 'span': + continuation.get_or_add_tcPr().get_or_add_gridSpan().val = 2 + document.save(root / name) +print('DOCX merged and omitted-cell fixtures generated') + +from pypdf.generic import FloatObject, TextStringObject + +def font_pdf(name, subtype, raw=b'A', mapping=None, glyph='/unmappedGlyph', encoding=None, form=False): + writer = PdfWriter() + writer.append(root / 'pages.pdf') + page = writer.pages[0] + font = DictionaryObject({NameObject('/Type'): NameObject('/Font'), + NameObject('/Subtype'): NameObject('/' + subtype), + NameObject('/BaseFont'): NameObject('/Helvetica')}) + if subtype == 'Type3': + proc = DecodedStreamObject() + proc.set_data(b'600 0 0 0 500 700 d1 0 0 500 700 re f') + font[NameObject('/CharProcs')] = DictionaryObject({NameObject(glyph): writer._add_object(proc)}) + font[NameObject('/FontBBox')] = ArrayObject([NumberObject(v) for v in [0, 0, 500, 700]]) + font[NameObject('/FontMatrix')] = ArrayObject([FloatObject(v) for v in [0.001, 0, 0, 0.001, 0, 0]]) + font[NameObject('/Resources')] = DictionaryObject() + font[NameObject('/FirstChar')] = NumberObject(65) + font[NameObject('/LastChar')] = NumberObject(66) + font[NameObject('/Widths')] = ArrayObject([NumberObject(600), NumberObject(600)]) + if subtype == 'Type0': + descendant = DictionaryObject({ + NameObject('/Type'): NameObject('/Font'), NameObject('/Subtype'): NameObject('/CIDFontType2'), + NameObject('/BaseFont'): NameObject('/TestComposite'), + NameObject('/CIDSystemInfo'): DictionaryObject({ + NameObject('/Registry'): TextStringObject('Adobe'), + NameObject('/Ordering'): TextStringObject('Identity'), NameObject('/Supplement'): NumberObject(0), + }), + NameObject('/FontDescriptor'): DictionaryObject({ + NameObject('/Type'): NameObject('/FontDescriptor'), NameObject('/FontName'): NameObject('/TestComposite'), + NameObject('/Flags'): NumberObject(4), NameObject('/Ascent'): NumberObject(800), + NameObject('/Descent'): NumberObject(-200), NameObject('/CapHeight'): NumberObject(700), + NameObject('/ItalicAngle'): NumberObject(0), NameObject('/StemV'): NumberObject(80), + NameObject('/FontBBox'): ArrayObject([NumberObject(v) for v in [0, -200, 1000, 800]]), + }), NameObject('/CIDToGIDMap'): NameObject('/Identity'), + }) + font[NameObject('/DescendantFonts')] = ArrayObject([writer._add_object(descendant)]) + font[NameObject('/Encoding')] = NameObject('/Identity-H') + else: + font[NameObject('/Encoding')] = DictionaryObject({ + NameObject('/BaseEncoding'): NameObject('/WinAnsiEncoding'), + NameObject('/Differences'): ArrayObject([NumberObject(65), NameObject(glyph)]), + }) + if encoding: + font[NameObject('/Encoding')] = NameObject(encoding) + if mapping is not None: + cmap = DecodedStreamObject() + size = 2 if subtype == 'Type0' else 1 + start, end = ('0000', 'FFFF') if size == 2 else ('00', 'FF') + pairs = '\n'.join(f'<{code:0{size*2}X}> <{value.encode("utf-16-be", errors="surrogatepass").hex()}>' + for code, value in mapping.items()) + cmap.set_data((f'/CIDInit /ProcSet findresource begin\n12 dict begin\nbegincmap\n' + f'/CMapName /TestUnicode def\n/CMapType 2 def\n' + f'1 begincodespacerange\n<{start}> <{end}>\nendcodespacerange\n' + f'{len(mapping)} beginbfchar\n{pairs}\nendbfchar\nendcmap\n' + f'CMapName currentdict /CMap defineresource pop\nend\nend').encode('ascii')) + font[NameObject('/ToUnicode')] = writer._add_object(cmap.flate_encode()) + resources = DictionaryObject({NameObject('/Font'): DictionaryObject({NameObject('/F2'): writer._add_object(font)})}) + stream = DecodedStreamObject() + stream.set_data(b'BT /F2 12 Tf 72 700 Td <' + raw.hex().encode() + b'> Tj ET') + if form: + stream[NameObject('/Type')] = NameObject('/XObject') + stream[NameObject('/Subtype')] = NameObject('/Form') + stream[NameObject('/BBox')] = ArrayObject([NumberObject(v) for v in [0, 0, 612, 792]]) + stream[NameObject('/Resources')] = resources + page['/Resources'][NameObject('/XObject')] = DictionaryObject({NameObject('/FontForm'): writer._add_object(stream)}) + content = page['/Contents'].get_object() + content.set_data(content.get_data() + b'\n/FontForm Do') + else: + page[NameObject('/Resources')] = resources + page[NameObject('/Contents')] = writer._add_object(stream) + writer.write(root / name) + +font_pdf('font-type3-unmapped.pdf', 'Type3') +font_pdf('font-simple-unmapped.pdf', 'Type1') +font_pdf('font-form-unmapped.pdf', 'Type3', form=True) +font_pdf('font-partial-map.pdf', 'Type3', raw=b'AB', mapping={65: 'A'}) +font_pdf('font-empty-map.pdf', 'Type3', mapping={}) +font_pdf('font-bad-unicode.pdf', 'Type3', mapping={65: '\ud800'}) +font_pdf('font-composite-unmapped.pdf', 'Type0', raw=b'\x00A') +font_pdf('font-codec-fallback.pdf', 'Type0', raw=b'\x01', mapping={1: 'A'}) +font_pdf('font-composite-partial.pdf', 'Type0', raw=b'\x00\x01\x00\x02', mapping={1: 'A'}) +font_pdf('font-type3-unicode.pdf', 'Type3', mapping={65: '\u03a9'}) +font_pdf('font-simple-unicode.pdf', 'Type1', mapping={65: '\u03a9'}) +font_pdf('font-known-glyph.pdf', 'Type1', glyph='/Aacute') +font_pdf('font-ligature.pdf', 'Type3', mapping={65: 'fi'}) +font_pdf('font-emoji.pdf', 'Type3', mapping={65: '\U0001f600'}) +font_pdf('font-composite-unicode.pdf', 'Type0', raw=b'\x00\x01\x00\x02', mapping={1: 'A', 2: 'fi'}) +font_pdf('font-form-unicode.pdf', 'Type3', mapping={65: '\u03a9'}, form=True) +writer = PdfWriter() +writer.append(root / 'pages.pdf') +content = writer.pages[0]['/Contents'].get_object() +content.set_data(content.get_data().replace(b'/F1', b'/MissingFont')) +writer.write(root / 'font-missing.pdf') +print('PDF font mapping fixtures generated') + +font_pdf('font-replacement.pdf', 'Type3', mapping={65: '\ufffd'}) +font_pdf('font-partial-simple.pdf', 'Type1', raw=b'AB', mapping={65: 'A'}) +writer = PdfWriter() +writer.append(root / 'pages.pdf') +del writer.pages[0]['/Resources'] +writer.write(root / 'font-no-resources.pdf') +writer = PdfWriter() +writer.append(root / 'pages.pdf') +del writer.pages[0]['/Resources'] +writer.pages[0]['/Contents'].get_object().set_data(b'q 0 0 10 10 re f Q') +writer.write(root / 'no-resources-graphics.pdf') +writer = PdfWriter() +writer.append(root / 'pages.pdf') +page = writer.pages[0] +page['/Resources'][NameObject('/ExtGState')] = DictionaryObject({NameObject('/GS1'): DictionaryObject({ + NameObject('/Font'): ArrayObject([page['/Resources']['/Font'].raw_get('/F1'), NumberObject(12)]), +})}) +stream = page['/Contents'].get_object() +stream.set_data(b'/GS1 gs\n' + stream.get_data()) +writer.write(root / 'font-graphics-state.pdf') + +from pypdf.generic import NullObject + +# Null optional dictionary entries mean absent, including indirect nulls. +for subtype in ['Type1', 'TrueType']: + for mode in ['absent', 'null', 'indirect-null', 'empty-dict', 'null-base', 'invalid-number']: + writer = PdfWriter() + writer.append(root / 'pages.pdf') + font = writer.pages[0]['/Resources']['/Font']['/F1'] + font[NameObject('/Subtype')] = NameObject('/' + subtype) + font[NameObject('/BaseFont')] = NameObject('/UnknownCustomFont') + if mode == 'null': + font[NameObject('/Encoding')] = NullObject() + elif mode == 'indirect-null': + font[NameObject('/Encoding')] = writer._add_object(NullObject()) + elif mode == 'empty-dict': + font[NameObject('/Encoding')] = DictionaryObject() + elif mode == 'null-base': + font[NameObject('/Encoding')] = DictionaryObject({NameObject('/BaseEncoding'): NullObject()}) + elif mode == 'invalid-number': + font[NameObject('/Encoding')] = NumberObject(1) + writer.write(root / f'font-{subtype.lower()}-{mode}.pdf') + +for mode in ['null', 'indirect-null', 'null-unicode', 'indirect-null-unicode', 'null-base']: + writer = PdfWriter() + writer.append(root / 'pages.pdf') + font = writer.pages[0]['/Resources']['/Font']['/F1'] + if mode == 'null-base': + font[NameObject('/Encoding')] = DictionaryObject({NameObject('/BaseEncoding'): NullObject()}) + else: + key = '/ToUnicode' if 'unicode' in mode else '/Encoding' + font[NameObject(key)] = writer._add_object(NullObject()) if mode.startswith('indirect') else NullObject() + writer.write(root / f'font-core-{mode}.pdf') + +writer = PdfWriter() +writer.append(root / 'font-simple-unicode.pdf') +font = writer.pages[0]['/Resources']['/Font']['/F2'] +font[NameObject('/BaseFont')] = NameObject('/UnknownCustomFont') +font[NameObject('/Encoding')] = NullObject() +writer.write(root / 'font-null-with-unicode.pdf') +writer = PdfWriter() +writer.append(root / 'font-known-glyph.pdf') +writer.pages[0]['/Resources']['/Font']['/F2'][NameObject('/BaseFont')] = NameObject('/UnknownCustomFont') +writer.write(root / 'font-custom-standard.pdf') +print('PDF null/absent encoding fixtures generated') diff --git a/packages/extraction/tests/block-landlock.c b/packages/extraction/tests/block-landlock.c new file mode 100644 index 00000000..1185ec95 --- /dev/null +++ b/packages/extraction/tests/block-landlock.c @@ -0,0 +1,21 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +/* Negative startup probe: add a restriction to Docker's existing filter. */ +int main(void) { + struct sock_filter filter[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_landlock_create_ruleset, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | 38), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW) + }; + struct sock_fprog program = {.len = sizeof(filter) / sizeof(filter[0]), .filter = filter}; + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) || prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program)) return 125; + execl("/usr/local/bin/node", "node", "/opt/extraction/dist/main.js", (char *)NULL); + return 125; +} diff --git a/packages/extraction/tests/container-client.mjs b/packages/extraction/tests/container-client.mjs new file mode 100644 index 00000000..14e70f74 --- /dev/null +++ b/packages/extraction/tests/container-client.mjs @@ -0,0 +1,303 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { request } from 'node:http'; +import { createHash } from 'node:crypto'; + +const socketPath = '/socket/extraction.sock'; +function send(format, bytes, path = '/v1/extract-text', extra = {}) { + return new Promise((resolve, reject) => { + const req = request( + { + socketPath, + path, + method: 'POST', + headers: { + 'Content-Type': 'application/octet-stream', + 'Content-Length': bytes.length, + 'X-Extraction-Version': '1', + 'X-Extraction-Format': format, + ...extra, + }, + }, + res => { + const chunks = []; + res.on('data', chunk => chunks.push(chunk)); + res.on('end', () => + resolve({ + status: res.statusCode, + value: JSON.parse( + Buffer.concat(chunks).toString('utf8') + ), + }) + ); + res.on('error', reject); + } + ); + req.on('error', reject); + req.end(bytes); + }); +} +const fixtures = '/tests/fixtures'; +const pages = readFileSync(`${fixtures}/pages.pdf`); +const result = await send('pdf', pages); +assert.equal(result.status, 200); +assert.deepEqual( + result.value.segments.map(s => [s.kind, s.index, s.text]), + [ + ['page', 1, 'First page text'], + ['page', 2, 'Second page text'], + ] +); +assert.equal( + result.value.inputSha256, + createHash('sha256').update(pages).digest('hex') +); +assert.equal( + (await send('pdf', readFileSync(`${fixtures}/compressed.pdf`))).value + .segments[0].text, + 'Compressed page text' +); +const docx = await send('docx', readFileSync(`${fixtures}/body.docx`)); +assert.equal(docx.status, 200); +assert.deepEqual(docx.value.segments, [ + { + kind: 'document', + index: 1, + text: 'First paragraph\nLeft cell\tRight cell\nLast paragraph\n', + }, +]); +const form = await send('pdf', readFileSync(`${fixtures}/form-valid.pdf`)); +assert.equal(form.status, 200); +assert.match(form.value.segments[0].text, /Form content/); +const nested = await send('docx', readFileSync(`${fixtures}/nested.docx`)); +assert.equal(nested.status, 200); +assert.match( + nested.value.segments[0].text, + /Before nested table\nNested left\tNested right[\n]+After nested table/ +); +const nestedOnly = await send( + 'docx', + readFileSync(`${fixtures}/nested-only.docx`) +); +assert.equal(nestedOnly.status, 200); +assert.match(nestedOnly.value.segments[0].text, /Only nested text/); +for (const name of [ + 'renamed-valid.docx', + 'renamed-default.docx', + 'renamed-case.docx', +]) { + const response = await send('docx', readFileSync(`${fixtures}/${name}`)); + assert.equal(response.status, 200, name); + assert.deepEqual(response.value.segments, docx.value.segments, name); +} +const primitives = await send( + 'docx', + readFileSync(`${fixtures}/supported-runs.docx`) +); +assert.equal(primitives.status, 200); +assert.equal( + primitives.value.segments[0].text, + 'Bold Link\tTabbed\nNext-\t\nEnd\n' +); +for (const [name, expected] of [ + ['horizontal.docx', 'Merged text\t\tR0C2\nR1C0\tR1C1\tR1C2\n'], + ['vertical.docx', 'Merged text\tR0C1\tR0C2\n\tR1C1\tR1C2\n'], + ['rectangle.docx', 'Merged text\t\tR0C2\n\t\tR1C2\n'], + ['omitted-cells.docx', '\tMiddle cell\t\n'], +]) { + const response = await send('docx', readFileSync(`${fixtures}/${name}`)); + assert.equal(response.status, 200, name); + assert.equal(response.value.segments[0].text, expected, name); +} +const mergedNested = await send( + 'docx', + readFileSync(`${fixtures}/merged-nested.docx`) +); +assert.equal(mergedNested.status, 200); +assert.equal( + mergedNested.value.segments[0].text.match(/Unique nested content/g).length, + 1 +); +assert.match( + mergedNested.value.segments[0].text, + /Before merged nested table[\n]+Unique nested content[\n]+After merged nested table/ +); +for (const name of ['merged-large.docx', 'merged-large-vertical.docx']) { + const response = await send('docx', readFileSync(`${fixtures}/${name}`)); + assert.equal(response.status, 200, name); + assert.equal( + response.value.segments[0].text.match(/x/g).length, + 600 * 1024, + name + ); +} +for (const [name, expected] of [ + ['font-type3-unicode.pdf', '\u03a9'], + ['font-simple-unicode.pdf', '\u03a9'], + ['font-known-glyph.pdf', '\u00c1'], + ['font-ligature.pdf', 'fi'], + ['font-emoji.pdf', '\ud83d\ude00'], + ['font-composite-unicode.pdf', 'Afi'], +]) { + const response = await send('pdf', readFileSync(`${fixtures}/${name}`)); + assert.equal(response.status, 200, name); + assert.equal(response.value.segments[0].text, expected, name); +} +const fontForm = await send( + 'pdf', + readFileSync(`${fixtures}/font-form-unicode.pdf`) +); +assert.equal(fontForm.status, 200); +assert.match(fontForm.value.segments[0].text, /First page text[\n]+\u03a9/); +const noResources = await send( + 'pdf', + readFileSync(`${fixtures}/no-resources-graphics.pdf`) +); +assert.equal(noResources.status, 200); +assert.deepEqual( + noResources.value.segments.map(segment => segment.text), + ['', 'Second page text'] +); +for (const mode of [ + 'null', + 'indirect-null', + 'null-unicode', + 'indirect-null-unicode', + 'null-base', +]) { + const response = await send( + 'pdf', + readFileSync(`${fixtures}/font-core-${mode}.pdf`) + ); + assert.equal(response.status, 200, mode); + assert.deepEqual(response.value.segments, result.value.segments, mode); +} +for (const [name, expected] of [ + ['font-null-with-unicode.pdf', '\u03a9'], + ['font-custom-standard.pdf', '\u00c1'], +]) { + const response = await send('pdf', readFileSync(`${fixtures}/${name}`)); + assert.equal(response.status, 200, name); + assert.equal(response.value.segments[0].text, expected, name); +} +const rejectedFixtures = [ + ['font-replacement.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-partial-simple.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-no-resources.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-graphics-state.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-type3-unmapped.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-simple-unmapped.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-form-unmapped.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-partial-map.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-empty-map.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-bad-unicode.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-composite-unmapped.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-codec-fallback.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-composite-partial.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-missing.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['merge-content.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['merge-orphan.docx', 'docx', 'INVALID_DOCUMENT'], + ['merge-span.docx', 'docx', 'INVALID_DOCUMENT'], + ['merge-legacy.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['wide-table.docx', 'docx', 'STRUCTURE_LIMIT'], + ['control.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['control-cell.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['control-inline.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['control-only.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['custom-wrapper.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['revision.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['simple-field.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['renamed-entities.docx', 'docx', 'INVALID_DOCUMENT'], + ['renamed-default-entities.docx', 'docx', 'INVALID_DOCUMENT'], + ['renamed-case-entities.docx', 'docx', 'INVALID_DOCUMENT'], + ['renamed-utf8.docx', 'docx', 'INVALID_DOCUMENT'], + ['empty.pdf', 'pdf', 'EMPTY_OUTPUT'], + ['empty.docx', 'docx', 'EMPTY_OUTPUT'], + ['inflate.pdf', 'pdf', 'DECOMPRESSION_LIMIT'], + ['aggregate.pdf', 'pdf', 'DECOMPRESSION_LIMIT'], + ['raw-limit.pdf', 'pdf', 'DECOMPRESSION_LIMIT'], + ['form-limit.pdf', 'pdf', 'DECOMPRESSION_LIMIT'], + ['form-filter.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['aggregate.docx', 'docx', 'DECOMPRESSION_LIMIT'], + ['bad.docx', 'docx', 'INVALID_DOCUMENT'], + ['entries.docx', 'docx', 'STRUCTURE_LIMIT'], + ['duplicate.docx', 'docx', 'INVALID_DOCUMENT'], + ['forged.docx', 'docx', 'INVALID_DOCUMENT'], + ['forged-bomb.docx', 'docx', 'DECOMPRESSION_LIMIT'], + ['utf8.docx', 'docx', 'INVALID_DOCUMENT'], + ['entities.docx', 'docx', 'INVALID_DOCUMENT'], + ['bomb.docx', 'docx', 'DECOMPRESSION_LIMIT'], + ['output.docx', 'docx', 'OUTPUT_LIMIT'], + ['encrypted.pdf', 'pdf', 'ENCRYPTED_DOCUMENT'], + ['too-many-pages.pdf', 'pdf', 'STRUCTURE_LIMIT'], +]; +for (const subtype of ['type1', 'truetype']) { + for (const mode of [ + 'absent', + 'null', + 'indirect-null', + 'empty-dict', + 'null-base', + 'invalid-number', + ]) { + rejectedFixtures.push([ + `font-${subtype}-${mode}.pdf`, + 'pdf', + 'UNSUPPORTED_ENCODING', + ]); + } +} +for (const tag of ['dir', 'bdo', 'smartTag', 'unknownWrapper']) { + for (const placement of ['paragraph', 'cell', 'hyperlink']) { + rejectedFixtures.push([ + `wrapper-${tag}-${placement}.docx`, + 'docx', + 'UNSUPPORTED_CONTENT', + ]); + } +} +for (const name of ['wrapper-block.docx', 'wrapper-run.docx']) { + rejectedFixtures.push([name, 'docx', 'UNSUPPORTED_CONTENT']); +} +for (const [name, format, code] of rejectedFixtures) { + const response = await send(format, readFileSync(`${fixtures}/${name}`)); + assert.equal( + response.value.error?.code, + code, + `${name}: ${JSON.stringify(response)}` + ); +} +assert.equal( + (await send('pdf', readFileSync(`${fixtures}/raw-exact.pdf`))).status, + 200 +); +const exact = await send('docx', readFileSync(`${fixtures}/exact.docx`)); +assert.equal(exact.status, 200); +assert.equal(exact.value.textBytes, 1024 * 1024); +assert.equal( + (await send('pdf', Buffer.from('not a pdf'))).value.error.code, + 'INVALID_DOCUMENT' +); +assert.equal((await send('docx', pages)).value.error.code, 'INVALID_DOCUMENT'); +assert.equal((await send('xls', pages)).status, 400); +assert.equal((await send('pdf', pages, '/exec')).status, 404); +assert.equal( + (await send('pdf', pages, '/v1/extract-text?script=x')).status, + 404 +); +assert.equal( + ( + await send('pdf', pages, '/v1/extract-text', { + 'X-Extraction-Url': 'http://example.com', + }) + ).status, + 400 +); +// Exact input cap is accepted; one byte more is rejected before parsing. +const padded = Buffer.alloc(10 * 1024 * 1024, 32); +pages.copy(padded); +assert.equal((await send('pdf', padded)).status, 200); +assert.equal((await send('pdf', Buffer.alloc(padded.length + 1))).status, 413); +console.log( + `Container endpoint: PDF/DOCX success, structure, input limits and ${rejectedFixtures.length} adversarial fixtures passed` +); diff --git a/packages/extraction/tests/container.sh b/packages/extraction/tests/container.sh new file mode 100755 index 00000000..839670c5 --- /dev/null +++ b/packages/extraction/tests/container.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/../../.." +run_id=$(node -e 'console.log(require("node:crypto").randomUUID())') +project="extraction-test-$run_id" +image="librechat-extraction-test:$run_id" +export EXTRACTION_IMAGE="librechat-extraction:$run_id" +compose=(docker compose -p "$project" -f packages/extraction/compose.yaml) +cleanup() { + "${compose[@]}" down -v --remove-orphans >/dev/null + docker image rm "$image" "$EXTRACTION_IMAGE" >/dev/null 2>&1 || true +} +trap cleanup EXIT +"${compose[@]}" build extraction +docker build --target qualification -t "$image" -f packages/extraction/Dockerfile . +"${compose[@]}" up -d --wait --wait-timeout 30 extraction +id=$("${compose[@]}" ps -q extraction) +volume="${project}_extraction-socket" +# Inspect exactly the shipped recipe, not a relaxed test variant. +docker inspect "$id" | python3 -c ' +import json,sys +c=json.load(sys.stdin)[0]; h=c["HostConfig"] +assert h["NetworkMode"] == "none" and h["ReadonlyRootfs"] and not h["Privileged"] +assert h["CapDrop"] == ["ALL"] and h["PidsLimit"] == 32 +assert h["Memory"] == 768*1024*1024 and h["NanoCpus"] == 2*10**9 +assert "no-new-privileges:true" in h["SecurityOpt"] +assert c["Config"]["User"] == "10001:10001" +assert len(c["Mounts"]) == 1 and c["Mounts"][0]["Destination"] == "/socket" +assert set(h["Tmpfs"]) == {"/jobs"} +for flag in ("noexec", "nosuid", "nodev", "size=96m", "nr_inodes=1024", "uid=10001", "gid=10001", "mode=0700"): + assert flag in h["Tmpfs"]["/jobs"] +assert not c["Config"]["ExposedPorts"] if "ExposedPorts" in c["Config"] else True +print("Shipped Compose mounts, resource limits and security options verified")' +docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ + --mount "type=volume,src=$volume,dst=/socket,readonly" "$image" \ + sh -c 'python /tests/adversarial.py && node /tests/container-client.mjs' +docker exec "$id" node -e 'require("node:fs").readdir("/jobs",(e,v)=>{if(e||v.length)process.exit(1); console.log("Endpoint scratch cleanup verified")})' +# Same outer boundary, with a test-only worker that attempts forbidden operations. +docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ + --read-only --memory 768m --cpus 2 --pids-limit 32 \ + --tmpfs /jobs:rw,noexec,nosuid,nodev,size=96m,nr_inodes=1024,uid=10001,gid=10001,mode=0700 \ + --tmpfs /socket:rw,noexec,nosuid,nodev,size=1m,uid=10001,gid=10001,mode=0700 \ + "$image" node /tests/isolation.mjs +# Denying the mandatory isolation syscall must fail closed, never start unsandboxed. +set +e +docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ + --read-only --memory 768m --cpus 2 --pids-limit 32 \ + --tmpfs /jobs:rw,noexec,nosuid,nodev,size=96m,uid=10001,gid=10001,mode=0700 \ + --tmpfs /socket:rw,noexec,nosuid,nodev,size=1m,uid=10001,gid=10001,mode=0700 \ + "$image" /tests/block-landlock > /dev/null 2>&1 +blocked=$? +set -e +if [ "$blocked" -ne 1 ]; then + printf 'FAIL: missing isolation did not reach fail-closed startup (exit %s)\n' "$blocked" >&2 + exit 1 +fi +printf 'Unavailable isolation fails closed\n' +# A stale socket left by SIGKILL must not prevent a safe restart. +docker kill -s KILL "$id" >/dev/null +"${compose[@]}" up -d --wait --wait-timeout 30 extraction +printf 'Hardened container restart passed\n' diff --git a/packages/extraction/tests/fixtures/aggregate.pdf b/packages/extraction/tests/fixtures/aggregate.pdf new file mode 100644 index 0000000000000000000000000000000000000000..70f476e3c7eb8a7c6236c212119a3d2c9d29752b GIT binary patch literal 37605 zcmeI*KTH#09L8~Ud>u4l(D7YJBt))v<$5h4kN{SV8c8u~Vn{sb3)a*gxgH8mXrgg3 zadE=INn;${j0~!_Yz9)z4zYtcc0$&$jy3!j>r+jNcG_eZm?EgW}AD3gz+Xx#{9$r|9n&s!}wEotkbJ=cbjkM*JyN z71mHNQ;DQ2th1$JRbG{@?pTms{F#eCZ8Jqd+UfY;Sk0iOC7F9=&>=ta%fXZ`#f&b^ ztc98rdKym12bD-zqch`CT$CBb7S^C&RpyAaPOI`Q6_qCZno&%N8>cn139F#0!Av-z z=QSAd`SBN9nt9_oG>COx6-)kLFsm;%)t!p}GPWbVoFm6f;i1wKuT-veDv{o;%1koC zs;AjcYRH$$kw&-jWj|8GY9c7=V$P|`RCHZ>cGfecSEEq*H^uDJ@9*xm>1|#<`P+H) zr7+*w5I_I{1Q0*~0R#|0009ILK%kWc=9ll^Zf`pv#kE&&3ta<`7N50$K6vlS!%II7 zzj^TZNGo4D*Mzw1WQ)6%Z`%4n&kc{F#tlgoB%FKdexeuz z5I_I{1Q0*~0R#|0009K{m;edq9{WKk2LS{SKmY**5I_KdtqU}naK=mKCgI|)MV$4k z?dnBs*Mn9f&aK}CjDP?F2q1s}0tg_000IagfI#aC5OKEdZ_AY-fB*srAbT7nKo^+<}T*q;} z`Z=6jHf`7U%jGgP<3*ugDyvZR^$BaVbVnIKs}KZ{G>LIRSZ6Aefz*SnEXP}o{4k28 P)1KpGL`TQT3wiMm|Keg% literal 0 HcmV?d00001 diff --git a/packages/extraction/tests/fixtures/body.docx b/packages/extraction/tests/fixtures/body.docx new file mode 100644 index 0000000000000000000000000000000000000000..85a4932cc073c3d721bdb7f4d90b5496aba1ccde GIT binary patch literal 36732 zcmagFWmsIvwm*yqcMC4T-QAsF!6mqB87GBBn9N544X1qDk=Bv%Gl67HCXiQWiRDxQ3UoKl zAEhy^5&_5jIV=6nk7T7@sN1)hT2DgqR9`2Mgls?aEGWKF>|&(}^$M|IVs zZ!|j$R(qo=?$6@^9uy#nK&hPPC1z^PY}-u4_8{K0D5DDv>2RaQ#6g!j_X&muXG;yI z`jx&SRL@}~0Q$_XeJ``em=?zoTzy&0jhgN{rcK!HDZn}J)d_1pCM-mi%z`RhsWj;wIzfMU*A=)6(L7)3e)W>~e*q`|K z$U)V2vQJ&bdu}Vi`KAhtDRg?z$;5>iJ)eyqeIAfXD&9&8_dP9^tnG=6-cjT~Vk;Uz zsQhCTy{O-d-h)SP9SQ;h3H;a4+1$>Bh57eqRpNv^G#h&0xvNMa`-2xGglvgc;VchV81ddam0PzvKu1eOizE{GSDi3`bJ422b;8aDgzu1u_GesyG82U06&3&gQ=-d3NHM z(f}*E)Sa)?oT6sbH9EYM96jXGCwW=ypry83PA)~eu^jG+jh#Y!-8=?&k_~~8Ye(TV zo~9OP)3C%a(umLJL1v$jbZ}l)OmJ5pbEN3`s**$d&wC$cuc$5gO-Fm9KM)`z*5*LV zWwPljEJX5C4%TyV&V^5sXSb>re?1ge!%$of*Az{pZKp&b+ry^&6%<4gl|-4?fKk>` zOahUGQTi1!DrL}EVbTC*cBuj(#CgjO#+{&AS9$wzU5|}mr5R<#wYL4i^olLOpI8?q z^h9RJNOeEUoVFH@o#@mZ1dEIY$szh7UDcyilYR5ksf)AI#*{m>eg$$%1Tt#M+pp87 zrIn(=U*T8v+}8nvMRg-{bK9#-E>wXcu4b(qwOG&1W4tt}#{OArUr|Rqge)TvvcKHj zTw1ebev!j25zB&+YnPp2?4HO!YzTOjd3n1;#Bk4Z34?!RiE2PM>ml*-)PF_z&vCg~ zS;0F7kBc1)1O)ou$Hmyu@sD9qA9Gk?MelrMfYEpTWc^0$1Ab!gg4ndg8bNJhR?FNd zh5TojHnwelKTsqNUpNaeN}xgDv2pZqQOx2^qmq>d&KM~Fn~KvKVpGkhk^Pm40}2|Cs zN}agZ-49+Qt28aGr+r{m{!{_|Tw_(?RXQcaC9?xO(n(21L6wFwe&&D~^Qy3>U5=E- zQzRo~GwQk+w}P=j@^J;$oBhJbv$WU46P%?KJtkpvyqZP6A4J^6GcDW{_5T5scC5%g zqg{c(`_y>cK&Dn-rWJV=dz#RrEjE(J#|8Z?K^yaZhBDd47|?fvk*giF>XguW+fXKu zfcpLx73M)eeCY{SlK!W}8(rdrq~>(2fVxq@^$8Cza#Gb5v^M0YtKG-_>{_fZqSi#q z?HKR;CZ~m^>c(m-BZvYauHy-Q*DL$ zw?Xa!XkfqC3gN5Bxa)3{-dPbmqcdOHvmI=L+=P&hJm7f!TN@BaQcn@=p1;S8kp#8c zZKNg(PiSqt6Nt#d)9|rV{+v3vA}3O$EYsDQ$@0WPa)FQux+Z*`2cD4rb7F{&HFV~I zFIGwTKQCAhfU_A3_^NfY2QN;)XGW36d%y}Sa`(9we)sO&Po;WBqzr7mPp+`sWL9^) z4E1(M$x>p8Nftf+U!LTDWd^(EV0}3egl!U#k_$`!d7Lz25l}* z%@4Vv zMDe#`DNdvL#D|;&!*VAc^3dkPM(tFe^-TKVszYag$cwL>vc{E^x-1(DTEcvibG68O zZ}MTDPx^2~UZ1R(A_dU8>3&{IO+4;SqNzJc5EC625uHvRwA_q+YV3Jxr`U*LA1WMX zM(jM>xH=oni6x6)NY{Gwcx%Avip%t^0+tlJL+$&0=hqFZdWmW19C4AV5cv-r0mN)k z%4v}yhZIU*9kUwDojpPB=e85ZF4x!IJ{4b8g$P8@PL+h+weQiN&@?#;24=>+%t#RP zFQDmvoiL~7eU~PqJn`IgFl+5yw1)n*F1Lq!irb;>WN()jN!>=oU;VH~kMh)0FwU6Fx@xO}4SF7gK5LrguB@&R1TI zha{WI}aVsArl&5TiEJ80*Q+b@^Kesmy3 z6JlK()nGm1yQZ%WqHQ?Uv4w}yYKh_XJi1UilGBN4#u2$WuHGFKmlM^kS6!91n<8r& zE`6Iy4VuCI7%+89^G5oVTvSHe=LbK4_hv)^&u(u?pM$GHx2Khzq;CHFoEy+-ckXXU8khNe zw#GBigVNomcdq$ZGI4g@0q}pu_Z0x`pLyz@<)o}UJ3OCdiY$%*UO@4$pvT4YxDt_; z;~o8b(As53>U_uJ!R#+!M?}QVk=Jv*;EVlh`+69Q|4&kbl#+|7frE(Dy{)m8ouT?vq5)snMzTPh7aK6Zx`qvxl23?=W`IRwmk53}6j@p0$jJuAV>wTc-+tZ3~ zR)%*GC5r?8O~NDtmj25*BR)C~?u*9ZO2&Dq6GycoXCm1zxWWp~GM6e_BM~9@hAbzYks^xo<+nbgif{A!J8R-AAaco5`W=$0n`CQuSmBYvzKSb9%DN; z4Iq4ZlsBHsp{8BAr*WB&7s3FnH?MB5pQ|F0;_(4t zEHB-w-5o8JW648o0rnlMe%&AS&byz15)T%tsU!nV8*40R8~$lhq`tB_r{u4LHJ~1o zzM8Rl>Xe2jSKSDIj0`FQ*A9fPi**Nl%(wf@VU8-hYr9_MEfsXS*n zJj%zfeg^0MLe;qri?3LN-6=7xKYLD<4p!m;+xNS#b-F#$IfUKqdeYn=x`$qF?>jas znwR^PeRrM%B)+oCq1TQ`BI;WcIWGg20{k9AO~j0C`1-URc@*Ocw5?J!HdWGo4N{Xz z;Xz5iNIKuhS${Y8?wgTQ>meFM19ebT@T|Ij{w8H|E^Vvi9t|Ri3b?6{LF7X%0Q}!R zrdM?nRN4C@T1JwVs!-=0 zfCIm3H~gxxO?7Nco>CVN_MBWQaSKZy;xR7T9v6+T)#+>o(=G2qUOV-x7p5LhXCEBz zoSsMIZm&g7&xNx-NH^)v&s_}cUtj+^8*s_uWC&{`D$veisGQl78f^pfBhr@PQ*F5vuPDZ6NJ$doLiX<7=Z@Y>O2X*XS8a3OpvTZf^KggD1s(#_92xZ*EW*~phv?vdssMOw58L&aba<4hsIfA`V*~yk!0Qtf0GPmWq zj37<-6XVO{nst{z_4#1o@~2uhjt#jbZTsyc^g?cEd3e=KL@JoR14$?HqojgR#wHzA z@r0kQbmZ`s>)5@9@YltG*r8r(qKH2`>Bx}m*DJFNWZ_jQ88Cw#6I%kwvUc?a;ym2}6>|gQK zO$1cTFf3k3-}>d&q5i@?m#14!@-CJXtr1rkme62r@t?K$mo(Be_=|Q7R~v=1W3p zYqMwfMnD^u-jclwk)-zAzEj8Y$e7&=V~h~9Udq$?(Ap!=txexy$VcpHU~%02tJRX{ zwch>kY;Mh#XU(s9VdaCp{qOl(_puuy`Yk|7(m020=4H|3k8$T*p9x+c9g!uNyrzuM z8CT1Q-`!!N={R9&)lHjLqruiq9gNkLj+mF;1)OVpG_IX=f2_`x(G_G#ieGDV_Zs0` zW=d$H-Es2u;7xyXz(e!cr8G8Tdi6T?n9qr|ZhC)k{BnP`?nEPZCfjv@zEyCnSKKTh z`^EKbO>I1*dyN@hy32D&M|4cViAHJo9(`Na{K{oxKvXs*0QhC*iki2_)kDo{67_p? z*s*-Q=Pc9+y$0@9nc+fa@bk;yMX~b182h>_NmQrYXr`+CSmx$aVmS{ioy5Yqq8uU# zd&8`kfN1jj(}jR6NldMG#dFJp92fK7yDJxKM8d8}nFF5CGhXf!fJr03l4tU{K#r)c zog?ncm0Li`H+}zmKOdYg?rEPUddFTnlg}01hz{FYnlC&+G7)lyTp4;D+yV{MoQlSIBi$0daEvC_d z${GBo3*^(P=V>2p3zz0i#Zm}&i9h)0rRDs#dY7U#al(y?a5mXaLjU_wI0GGj)PNi2 zn^`#`NSZbbCVRi(g?UTvgV*LQG0N&Uv8+N?b0}!eH4$qoxoYe6=;4nn1g758t%L@<1Y1&*WDgTQ({yt~6A{7=`OK0o$aoo! z`c}ce#FHFae`u59scwFRhC3i2yPx@T{9aALFvEQymHOcx(}cg@xlutb`8pZ7fjYxO zOzDm+?h$!tH6mc6q|{XDyU+-+n|#mv_UH{&je#oni`gA+x?&NBo(-v&XRb1j-1D1X z@q5pEvC%;H$DOOEvuB3asuf_#ENFhqsNeJ1DS-df@IE&l>E;QNq@L&6(GR}6+XKK7 ze=Kn&MRr%w(ds1qVeKHC;8RA5Vv5Kqi+(gdkE>VNs!)-0D%?h%e#IS##yUHym(#V(m(Nu0fQo(q~slPQ(6MBhe<@Ozf~`8Yz@(g6;imrquW ziP5oVA-j2;(;ZU6g;eHF%lqI23WhklAhpNtdNJ5vRy_c zE9G$lQPLYojvc(2SL->En7UIA7-ovOo+9jWba40xEVWMULXN2Kbmk`X#;WOj$J)q_ zv^Ym8a1HaZs5Jam;}nN~n-0cif1AH)g;JwN-Bd^nl*nnPX875uzPxVoLHQQp{Z05D z@)M?*;=0O58+V`bqalk*8CpXPP18-ogZ33)}K@c21mCX}iX_Ir%l|=wI}@V`$H_=NjZ>U@@eo62{}QHO;9O%T8~A^$M)k3HKBDq2`c_a)DoS3CY|qLKA%hrs#=pKIKtNL&eHXrUaWxNj zN4zt(NZ#q8ecM~+ITBij#CMIl@>XND3~RDl&Rf2*@aXiMJYs-I?k(3@3#T2 zNt`={b1_)&IF#CrhTXX{<$k*8+Dy?U3YF;BmV^^_FTq%U&iatsag3`U!-yOvvzbu$#teQL+F7sJLwHXZkhEL>F_dj788)xb+FXqE_@@-- ztI;Cf7g9wV6rm7F@c~+E%28$>v#IbQT7kl$WDU}j_FHV;QXj-6LZ2#x0%3uso$mHa z;gM^@`L9)P1{jwr=M>dxW@98YJtNdFc*-Zo#rID$GDWZaK5JZmLvOR;d8?3UPNLx= z9Qk3-FQ%eIT*PuOl%WY$l*6MP=(~SEdx}!-T~F@KAKn%~2*nw0fg3X2yJfXsFU*nJ zHox|%rp5%%-aM=<7+V8a$MZ-=&vgVnx$c^FgkD`KO{f#g*c90E@x{lUk4li1zEv>C}E4qPaLjZ70fx{0@P_oy%b<+C5s2_kh}X5$rjD zb7rCm*$@phRY?qMN`9e|Sa5ifUR17A|1U?KS*m;B{+@}&?vg#gJ%Xg<^FpnB;ctTx-w3_sxk&j6EtJ8UUwI})7 zvVkr{k6ChfBB4M>FMV5m`8?K)TCYPx)r62U-77tP>X^%7z?go(x5J&S{Gy0Paz2YS z=I`Ze4K_rTTl#BSOQLfU3E|$zvo9Adz^zW_y8b5x)2(B!#`9~petrAJF$t3O&;b9R zcBc7w&jwW_gLn-MX0E3necDLK-Z?aB3!UFZ^mx~h&~iSyr+jgdC;z#p!gU)&lVQpw z8qdZtozc5!kub-k3iocXcx{z=6UC~bBal(t-b_c_VOSJv>K1BaN-XIqYuWN7yqrwx zX80%6h$!GB+cBe{rOdRWdEl|SRN_!^^F5m^R1HzekIXoi&On~LiA$(@_^!|92wiWX zm7d5#X#VV8@k+ZyO*SN8PSk+yg#((&z+J{TMr4w z^4zNpU3bmef$fB}kHbrlI*es}e>g$pxc$)0V-k!d;L_A`h@vL;>9ERkm>a9B4sCUT zs4EuAl0T;o>qphQMaUcOx2==o+d{OwL0YKq4;4rF40;KF?0Z7$gsy+94_%Leq%qi3 zd_us=;!UsclLC|?s!9SV*>1SmF}juqqByb}uzsKjdO&pXT`!w#U6EjC^)f_I-4yQ- zv23DvRuq(*V|Y`JVS01C$`nkfy;s((L zyHj@T$XX+1%YSkNtu1{Eq8YnZ_wMTmYPB%K_IQlca>tE>xf2GSP|4zR_ zKhijD`09~FlCY?DoXecQo~6-H#qbd0O}7~am7~HkLGInE-c_1v7hxuVGdFf@|3$j% zVpuxVTjMe5aBUTS7dXdW7G26?Id3->FI-@N==n8}X70U9Bnh(y%MsNI`LYr$@XA}f z=mSe9IrdkB`EF)!p0MDm!% z-d#7;@W;)dAuuO4iCK1PQ0O_NS!nU=LL~}#6 zHXxj8?oj2DsECPyI_g+VB{O1^w*QDLtJc)^dZsK$5SB$hQk0WuzoL-e-HtEWv*Nl)>?QK$``u%!UM;|O{oav!vmXWYgmqB8S6a+YsboUv^aQw!Szp2cgdK-k?@5WCw z6r%(G%|k)<7mo*layvY{NNmKXK-f$*9{(V)B~h@Yf-g#F`S}swrze=8jxg74G~)Tu zEWT3YnP8}lc;-3gQRIc+ZNz_$lYRpy;8wUCTT#5O79L*lei={-bVl~Th zhK02ZF%o>0uyQL3;J*oEv4UNZD#=Rq22cEQr1&2;RJFo?vQ3p0{C}}YiVY)mMv1h= zWeLd$_-V|79Z6THE1Tw4)WFLMmqbm?e}LJ|p3h>2>YuVHrZ93s;?17d7NW1D$vbSV z&%FI*XHD;oPqg-w6POX&VuIC35Pw=kk!PbUin0^bAM`1ZU_C&DnsOpod4+;(S!u?d z9ro>5F6^?BT}#lc78es#BA`(o$a&jF-~*L7^=r$;4I5HFh+=c^!jvA00vqzzq0PU| zsUMcw(?hv;9vVBixOVA3I!7nNOpVgEUqrUJRA^a$#glD!o{=W=f#0Qus zr{_j5AJ9doFXDNvC+7uduVdFO%q57$r>5-5g8_t^ztP#$AQO_S)+-3HxKLysD1SPm zN|x+lO`W<=?-Xe}<~VmP}#KnSw()GfvjVdLxNwb`mE_cLg2Fw9?Umlq%0xU_NnnOqBRH^8o1f}<@0tY{0XJb#l#_OG=qaW{QJw}K@-Gf zM~s*QQ>6lZBzwdlS;%ZxxCCgv2V^oh_t1_;G7~m^V>aycQ%MWIu<`M>@MSEP$8nQy z(Pi`V5^W zPG%y?C~bgVmsVYi|7qNvqJbGbK_HFJ$4o0%2br%OGERw6dIO6z(}c}Co3E5AnvTS8 z#FXs<`XX`&yMBu^ZG1l~BQAtHOtdf%H4EljP?jhHI4ZO zPNP5LN-`ERFosh!WPhu)Hv}IVaBsHu{+mh;o{ZlrAM$>y6z&D9q=W~n6qr;z2w|fk zY`0Q_mqaXGAr(a}d?pn|KEa2!78^8a?>C`kUx^xT7Zt5;v1~rJ{TpKOmLe5v*H2B` zArpEJ-~i9Hz%v!o>g*@x7Bs;VTHp;#N-WeE6v^jsxW&S}UFtRj{#5JIZ2NUZ_PEd9 zT8j5|1I1HP^UF6wdHoU=ic&xg<7|$%&W+9H48{8G`PyYW88@?`JW>Ad>%||E;RCA+ zrt)0S*Umgz;x-d!qIj}{@SHzAdTS8b!qHeQa<^gI2ilkM^ins=*(EJmEUG4sn=o8w zpkr+evI}1-?kI$l)C}_4eqeG^+Rj>odppPnob6&isC+R|>(b}@-8jhCk&^h1+7?<* z8tVX@$HynjK!sXEor?V}kpa z?GwbZ)I`=8TqZ|D2pNGPgpn?Pv^RZu*AVr+b00LwR~Mt4HOrgyb%eobQdj-`ZzU~a za863kt~KL)4DrjgbMeb$|5k{6%6eW3y`5$Cr^4F*R+xH_C1Mx%WYIJojfi3+Gm-E< zMPiRXfd)0%K`h6eY9ftps(_vZ(H&m1shIzp3Dyp2@bj5#%+uK)EA-Us*^4=7<9X|u zF{WR3n?F>xVaFofe_FX(szyWxVCL`aCRoHg#iLAU&Z(E=7sMwtaa%sU!|GUU8t+Dm ztK_`MWHZi}P<6*IBVg~q7U}Q8dgB}M%eE<-YjHRhicLnN;ECf_r|kB!!RF(5d9gzt3{Z342%#m_88@Yu zkPeHnFN~_qA|_qCag*GDkFgJPz&I))tOMo!&zNlC0ZeFM9NhaSvaHkPO!)?_l?bZR zmU6iFt*6`#wJ__pNk$0FjNgIBql5V6nxb8zp1tVpiTi(3u^zd`U#LP*NN-Z_SJmI!)2~Fn61% zmi1rLwTb#9I-RwHslo>^$&ow=QySk2_=Qu!9TB6Y{IScPXk9C?_=~(o0!*ICkNyXF z>{l@P*8k}nYRaSYX+Py(_BpF40DPm1IL`*)P*kTU>oQ25ZI4dvG~#<2xFXE> z)POl~KR(&in0p_Ub`Y+2m!JCGC(egfm(k3h%7qac3jpg*n&}m&hi!w96-x1fi`J2X zXm#GuruSM{fOKlAh-b)ZXU`2V;68BZToa;Y9=)?nbW*-ip!B9T5>VsX;1x^CDd#`B z3Ri2qeN41cnmN)!pXNkYthn!2=;A@1ZbbdT8MKdjg8cX?0Qa4Ke!RFAo;B5UY$zP;7*f+YIOTR+@}r3`MNk?6M3%RCbhtClZqwQRd-wq*~RnK|1zofQ}vul@lTU{ zmA_4n==?T$ZvX#ha==>m(Qlor#2N@TnZc^~`z_hv=>>jQ>5nC+Rb&Q^s2$#r!&Jpp zX1pl}5wzYtVe7YfEainoqUF6a;?8`P25dd^48tnTG?|tf4=;#GVO-)pq7jI}HX@PG4*F%@a4C2(*j1_680cML;~BK# z#~F>cxh8|ag6)$Iy8}vVkXtti7b-O+5Ct!jG~i`2okSEFo&}7y4f`7n$wuf0e+(WN z%`>Qh$BH|rt>0jQ+Ca2RgR8UGJ|uMr>4Muc6c5o@qy;Y;!Lp@z0@(-NAW$}F6E>0v z!XPjbatpSVOs+-0s0pdc$?9WK=O zVC20ln1;qzL8bFIama~aal>Sxc%KX9Tw8AlGuOl(m7E8asD*w!75{`iOKwFBpoynj6#k8^{F; z)>X~67lIW63xbIY9Q;}~9N)+ugf@5R9j&ZIZMX$)@b#JT9-KA6vI_=@iJf|tM}*z% zl#$>p1j@L6T#gga*trcoZmM;1!?pN1`d`>wTuLJhNs9| zT{{zd+R3~lUp;mOYkyH3>kDq0A?u~k2A03Pi9cMRo{7IcCls3qQHS>WYG!l^5I5cM z5hZahJ(pe{G9HNqWv==+j~5dC`z}}D=PamthA*=Qo!1TINbSy zegf^tIEIy)e4Ev7)}H+pou>@#8k9dftr4#)uQZMg2Z~O9qV%v({dp9aVnxT%B0#a7y_V2E z7P^4Xpr(yR^=wdCKK4bc(VH^EKd72(Q@r)G|Dc-mZPso*fhQJA&+X02^Yfn{5WfX! z#3Qa8(C}BE2=|mbJ@!9k0N7S0g++MM5JjliM4)?_*m{@=g3?Go>361|ziIZ=1-RrI z9-o$A+=A$S6J7ZEH`%Va?rW`YuBZYo6vu0z$64@|^&_ypGJgRfAS2rF8VE5K_ILtI zYu&f=cShaOY6x0r#%sbXw(NZm4&I=az)X7o$545`%$cUPr&)JN?JvHFLH$)f0ohYs zo@7w+`FRN7IFnoACMI7|XNUSMyMZ~9KMDmoU)e-Lg7(YOw^|q~j%K?=%(ophuH|6^ z7~FiUjdT$*`3tJ=$f(0R_wX^V1?M7Zv5*;^F9e7o53_WY7q#3dGtRyND{L{U9{lZ@ zb+={8jpxy?r6&U(9z-?#U>3dkadprY^)(cKF(0z~ZGUj1sETVOpR&pBD~PGWVHCe{ zG?G$2Y$VZpZX}_<{1BXnX*lxUw<;^D0BMFtmxOc&2#4e!bRdugVAUSUR_jx)$twva?Jp#WDXAXV7Xe3d-u^Y1 z|66V92`BJ4CdEJCdVuTu#;@WSRz#gNmG0haY#^1CdqnN)C|;K%UJn%GN$i^m?Ayd; zZ7*s-E%#C%)D+pvcfJcyg7uy}D#32M0pt59iSeu@&RtB-5W6r)eyZ{#)m~3;(KXDj zk{Ulx1K}z$bSeq2V+81CYA2NPBEJ2reOWbSb&}F@qw5f8BNSR5_4qi4a!k>%hbnpa zZKZBQ(yPyRdmX^{R#-4J@NrJk%t4M&Tlz3tDW-3jeLh3>U2q$47(VV{ba{Kcva7sg(v3*$tXkw=)3u&GpXUM0uQXvfvd zwXZ`C^?(Lv{=Ul$6n5~lekSJ_q`u4QrT+kk_n`*^d{F)qprZ$ih=zAvir3-It>fU! z@T~EkYw!*ip&2&s?4`mg8O%s*i%h_#r`qTVEYXNYF%%IC$*@Y`&SnTceB2#E9MYWy z)xQ)F#8985VL`x*=xGZGXYtOqwTi*V5dz|0dM}H=EF|tf1Sw z%_o{zSjkCYjS$A*7BF?1tdK+Bq*Kq>bq!agqQPeai>AvyA2PydAO30PK64gX!}=s5 z*w8hx0_1QrRtW#g_F%Ybpv?M4k;-D_R5%AX-Jq}i>`e|6x9nihW(8#wIvRn-^+Uj( zKY4-cI5fgyPLFg`H5Zu~kHg!(jg6$YHO7Tdp~W5vNMlWlLgBTIlX)-WNZ=ZDAo2yB zHY#~!x@I#L>PWuBEtH{kk_mn@?44hgSR&NcwtM3^77K7VWw2Ml$fzyuOS%{iZ(46H z1THlz{UvL*R~nu+q)z|?8=HvFjp` zs~9_B9Y){JKK8ju6T{cN*I|HU?Q}r~25#=XI(E+hH4}5{T6XYyS*d^>mHf*Lg;R4s zem{xNr-0m6+%75EtItl*l%?NdjlPcEOH=a%FlK6g!QC=AnxLao?V(`-<(yZ^xR}e$ zQ92>tWi6=K9P<}f z$?EDC$F`3w47R>)z+eF8qaZDZXb6ET604Jb4(rgH6Ix%zD=;z&W0I<^BcqefR_2v_|!_Tf3r-|U#J2P zh^6@u+?Jpw+&Pv~C31Q)f+MDvCN%`!KSuOEXs3Z@{Dj8}T-kaJL=*EhV{;deUDTx? z(aRBOe`g#UC17Q(W6!P(8XgU9I~ZTiX4A$p$*g2dW3k9`nbBzhPm$1MDH`_6GVLe& zFaG-O?pV)9H;{_4JkQl`+jK(R-)&&K6#aZG8cMMCQ`U#_Eh@7*z9Mhao$&Xp$5J@& z)A#T%aFM@bj2x%*Cn@7%T|dAOxMUwkOeOublg$y{e6ScVclE&Yt7?O zL@J@=85PKNm8d*%kv#EhMdt@NoZ%POEm_XC3S#Y+wuyJUmnL+4 zy;3BC!GWhggu9NAD9M8dzxOjAi&4{}q0n%LuD|6`KW*)?3)=8HF#Dl&eH(y?{Ew znpxy>Zaq^xS4PVAiS?~yPw>l|!=gYqICKCuB5XO2g!H-U2D02zV69m8lBj)$iqE0Z z@7t4mQpV(V$V09UWvEd{ff4<7ntm@liXbC_d|K=Q`CGi(er_l5cT{?cu*9we>q3YP zBstlj1o3QHG5Z!3AK5>+%&KnX;kNQ1Pj@re?X-mY;+;2#J(Izr%ki;r=p zEF@49=d<{CrEi?7f#;#`3PokVY8K`-YLyBVs=PYQJdlU|_EE@+nDr~Ii!f|?r{AjZ zNAJKk@muZ3Ux(f#yVa4{^4#LZpP_sy-;u+LBf3$*HdTqAg7e~e=h6Oj0PDl+RD4o1 zl$QA+Mj8=ZiFN+4pUNlm*N(LavoPI`i z;*mUk3;49YXl@|pw|Loap{M?7bfRw)+BjTBP$TbO0$x15x*z@8)~dLV?>3NMx$pNK z+WeOUs~2)N0%+^jN5^yS)YzK-)g$n% zqoxM!{5l~rIdg4)_62~Jnd*_LoU4g8=3&UQGU735#LN5C zvv|61+_pkBydMobk6wE&YfD8jv}K8oxPEnm3z56Jv+=XuxsBtl8FO>KnzeYWU!nIMuD7f&GZgYtBhLMwI@j&+NOO|}ji}zNc#6mhE}fqQ%U%UR$l6bhy_UaNd;fso&?q_?>ahaHkjPeD&}jx zKT}opRANeqj4yP|`#D>*EuC6+%v1QV^Ao?H5-GxC4*=D!SfakE7@2?W$O5hze%I(> z$=TQtn4BmBRp0Dhn>>Sjo!ee+H%RO6()n)K*g!A-b+-yP%bG6Ax&YaAk~bI z9|Jx(fld8jzJq3MXJlOP0C95WH4VJncLzPvW9ZM7AQDhM&*{i^okschn=YTRcf!~={_ z*^E%ySW3>tDbG!&m1ZM+5KtR(zs-!GRBf;e_jeB=?ap26 z4FfkFZi1We!A%^uKoK8pqL*&amMk(?PxbB8*iWFnY+Y8)ZF+|x?#j@?={E2e^Bycu z@lgio;?`-(V^$(K^QCsfLG@P8bj%6fPKu>QIeN`T3lM9<>#wM@zP8a)F{0~Jkoa6R z+|M!<9kDmN4{~bjdUOMg7?gj`KTnbS@_|y$~LGV;lIJgIx&V6wT{&_30QPCzxN(BJjKgpQX07z5 z0)jT>Dim)eh6Cipy$;kuWOJbR`$0X32(f;qzyX3S++O2YN2JY*(XwU;R5f*Q2i;F_ z2iYmOBLUn|E8Pm~_-_cJ1^5<-=M2Igs_oxFe`Ws;@|>`TBq}cbjneoV{tuMD2mBr7 ze^-Ie)dK67mj4rqS7F)#k_gVa1p+lB6b#1o53T=_^b_ic2?GEQLjZ@_{wu5(;DEHr zD%JEGXAO!MdFD6H-vj>UxmLOnc219C9UO)U4*P8l)z-N}?g91o`YP@*Y>#|TkNT-b zE~yj3-*+y6Nrb6o3R6#*c} z*lrlT9z2~z?CoWb_TOQ}5Xy;oj`^><^SVEr@wfTD9@x$t+1_$hjqLCsm3WMy;l12@ zg5Xb`eSST!W_RNVW_PSV{$j3v)7)Y9DW{qQdAm)-ZRZ$;BxY`7dCx#eh}CRk0!Y|g z@f;s*R{7G#+_eR~iue2V{b2rK479s^^JHB&@zI$$@;Xed?Tj~T(ibEB325Ht`^?wY z;}K>mQnfIqf4d1B_#Qu|oM}Jfx4s)VIqUHymFvtAxNo29FA17GpGp;ITNOW}99>c1b$(y!pgfE%K@44Gif|ffJnb zbG!(}ZExN+{8_%Y2Qg;Cx!KrCX2LnyW__38!NeuKvo+EsU*gh zBX&wSU1*9-3oG`iUvfrZD+23^I|kh5ua(@lPep_e@-m;Eclfit43+!3caj8O>b{}4 zIWPNyo}S{RgfsX2uXUeicSh@<8bu7ai4@A2pIVpCd02>;9MAx_bDZgM8kgP5Bc7XW z{yw1A)f|)0MNR_}4k?Lsg47OQNu<-Ob`K`_4m6jma;B~)@tR;dRJNpnj1FE>PZ6_Vf+~C#F zIh2Gfh@8iHg@zR#F~r0|TY>OS-!ZnK$miidxQ@lusIk2Pddi20xLdU@yd|{QxK>-q zGYlCKob6C)7&xCLx-Rh;eUa}f?AS@k>Hgw^KWW-?J-O)N<-_Q|iR$^b9|8M#uFgwO zFxNBo(%~cO@PJdj(7Nl#2T7&11ixpp|VFDUefwQ=DP=a5i?{p5Bef9Unk~Im6LGC*(7(HMJqJ%;)ORC5+#H{qz43 z_7=c#Y)g}{n3f_nA7?)2GaOm@OKOcwnF08uAA{W3$b_O?#CLo4DIm?~+(vfGRzR-9+m#m@kk!ZO6>J)#->;%RaA)kQhKvv) ztm!+{e8V^*qhRr z@|V3^%)wY}>UJtT;k2v8(#9u!qYaFjQ{~b_qUwwvr9_-|AuIP~>pHRT-tK0_1!9WW zT1(l5DV>eh6U4QaJ`?<^+u14T*%?F+x>s`*w7zF^Bc)wBu`#;ma~|{e)^8eUki4#^ zhiHu*O%raW^A92J3+g=^nUZt6CZ zBDa`NPtjoaZdC~blobrFu!+C6yK(S3zB~Q!KV|Q+|1#bp^xal9SraLA&C%tdc;9xc z<5p#q(7h`a&>*JoQGOonPCO~+uIi)P{+KPT#yw5>*z%bWvRu9> z{kuYsp==#}zL(Lr%6Ws{PD(b<-Ta$blS%^@r+2_#Uq5-Z>py@c@=iXZ3}vja2_C(( zkTr&p@JT+L;KUc2EF!ydWb6w&ZoEA^{Td(L{5jf&pZH4@1%2OBK5Cqm6^bfDj+_)G zOy&~_)q!8yJvd5;nHh@sND~pRsq zsc;}u?`>I>7pTe1lcxZRWVO4g`R+kQ9nLdvj zI(v3D#GI8+Ku9!c{<>Ar#vibgJM>bi^#4|!sTzE7uI~EnYX0GgY0Wkl{mfu1|E}M1 zC)w5VJ@Kq{z(*SHI%BJc(Oi6W^G7?~laiVbQ>{hTdMj7e;Et&cSq|LyI$c zQt0^DQ+3K`3ag0jQd*LYS=qNMn<1n1l&N&h=g^b;G~wp@N9k#NljUcb)RIzUiOUHu zj!dfhifu_S5$oqym(Y-d1zM=g%YA2EckHt`K>f%xuIWaW_g*Hsh#-}&noZEkVTILE zR*7}P?ZE$b%fRw` z9T|BGn0(q%0Lz(*(P-%-Ah_|h>wyBUG((pRQoj>orKSxAhp+vBcVo~_On@D}`GGu2 zRae9@X;eOx8V}pN@n%fZqFXxTEtn%uB0ttBWlLpgVqE&q&~VYmp41i9v9iO8jyX* zkk`PI!D-tSsazm26ec^``1QB2!mO06RPnhE@{#q!sN5WX7{cw@fG3*zVn!MVT$orL)_*dFK~pGXpuJVx1%K=j9_xA z+Al6ryIWd(KyK00J6opgv0vu9afi2D1x$WIt>ghdHZsTiewhQbd&@exvn~g_{k_DH zwCxuElA7Y^@R!tIB`aNgQG+rZMz_|?UooztiwA{wwxS?5GHtmQ03F;{j(ZB=kM@pk zN-xJ-?)|oczyAN(0XQP>a=&gkbGC6~A%pvS?ZdZmuK{;B&tDg%GtdnRwLU_!k*R-o#uL?5msVE^9EmO9 zNT^&*D^_myOqB9wY5p>4uv(6i`EB2t+MgTSF3{N3-=5=8%7G^HYl*WXnP_j&cUbwo z&HeYSN2iBeOSTtR=*~t!Tk0Lyj|MB2`{PB!)H$T(6=u{#Y(|6BHuKc< zJB+#A==$2|`sv;&mV~@{_}|$Sx=#l98|V2430Pc_nl6h$aTvkHmUJuS`?4RFX&M|- zy=?9bf$Loi)a&IF7n4XuAu%?GNx&zH$B@?LqE?MQT|r#hu4bw(&ZnXkJ{2AAERIai zx!_jw;?@ve!xI9_bzN{49jK6sR*iH;f%9r*3iQy{l z(`wA?l$mNchiX^{1;NnP7=%s4V~|pP*WGIE(`q9m96snDkINuYDgOCq)#@cuGoCgW z?Q^@;X3RDMF73bX*(sPiE*8apxVhUXc-laA9jXow=-LeECMwF*kng$=(oUCSe|ERg z@U%$-5Bcuj(#;7{Xl7XEX_LIR)wqQxQSsH?W^EErafs=LtO3_*%dC{BVQf&>=DxF8 zVCy#B0Z4JTdfY;5C0?j|7vXX-lBtW6zMFFjRP0!IeGAWTj*UgFyLsZ^LE9mk%;i&8 z^MzUTaWZzUPl#!CYPP3z1hPZ5j6=0wt2dbekfOWIE$qHqQJ@3QuE0Y9yu-At1ENdc zEbnYfE0kdq0s^hW<U7>RkhWf4QipxH)``twQRYWdVLxYGYD8X)pSUu7`bg3rdp#Vq&*dek zBGMctwc0Q`Q}H>;UxO{sj>MFr&3{Z()6M6+x7#b2fIO;}OdgIo6|cBiy%}45zSI>t zc_77zJtZQ!3L22U6M6n=Osgx*;5D^ckk$9OQ>{cK#~8GZ5Cp%EYI~qK**oV9aO#${ zludGJ!7S!REmpx_-{tTb3L+J0v2=pbJMPKu*TW?(iwfOCrmB&6rR^#JlmO8y#k9_xh0fAW*b-_Di(eUeSYKAd z7ahovFBME=;8&VIY082nehX91&vNrr56I#Fv1u{|<=) z`tOkEs}{hJZWn)r@l&1A;K!hoY)D$de_W7+GEAd3G4{g<2d0!aAQf+8Kvlc*{Htgrt@DphFCTd<&%? zh7=543__nJb4JcuBNomlK%9=%f4LYf9JYx$QBpq7qg@AL07|?u=>}Ig41^Snp9Ay? zx6o_2;&bDK4Ju6fY`d)$CeG6eR}c&zTQIK;sup`NYoKmgKMbnY>sTK6V-00C94ZJ- zhCgXp4&NNZcWTq#j0xb`@ZUqTqyquc{y?%qNOtWZ1|cX4K@|eCceD6QcJ>ha72*b$ zX})9X70b%N_)8=QGbR}dH%XhjBi{eyBP#}BOy$a?s{<+d-i$~Z5F>BYgW~^56^Feo z7&i*Ve>fI_&+womrpO9SFTkS@&vGbmrseYU1c= z#-g4y8g=m+wMaYGN*2C=pRC5%p4$FI@;0h> z)G!)_*)P9Jr*kSM8eN<{UVSu_tN*y~dNfn}+F!{%Ur7wz*xyy;TnyE#Kb|$EQERGX zT&dYWeYiQP6Xc8h(S!P-?<336GuBi$#nC&9BevD$&W2DE8;2O*zu+*kFuHLuy%99J z-7!ksVMHK739tW1KQsojUn6(G|EqQ{}G_bNH0+HIU#Upz%~k#L4fb$cb1%$@|^ zbVh^myrGa%xVsKw8yndcOmvDa@WfK+E|0q5~>GDbJdcQFhg!%Lc#aXeDSH$&=(nW>%p&Tda zsYr1+XXEQz0@m&W5O3%_H5V`mg8e zS`WK{79(F}v&ioEg!yhWeIW@@1NXPJNlUojfWZDX)4*GLZklohgYhi2Z%CMdY=?Ym z7vq?qDNX+uJ!eRV@cRA#)Z^fbD@Wdfcz#ZybMkO!c2g+5mHM!^djQY-G}wShtEg%H zRptAa9g%@RiU*SR8!s_IZ8;jy0eFxK?P_K4YSge}67QG&Wp7%)q`=NY%`9?hIqWQ~ z4NP=wnPq8E&1fbcLJ&1VV^5*)30uCUFZ@|WA9Ty-7wL=edL_M93N0+}?aPyk{du4` zmQDJ*E^o~|8p-la-*P}OX!3*=2+=DXPn9&^md1F3}a8FWu1X`LO->!g4yNM3W@z0}7PobugWGsOrnBo=SDj6nOj%XUJ z>L7p z7m@jTA=??s^hGf3A6j3Sd4Oq5z}wcZ#RMQwD+;kLiQ`XHH8Ixn+pvL->qh zMk>zrA@$sZp;Z=+cw2#(UTYP~Y8VE$>tbS8yu!i=U^2uyB$Xb|wL7hZ#I#-)D@&dq z?_1H+U&Wh0#ewA>(n0zjG$*<3h&aMyXGl82yYVP~b5spe_3YoXC8ofpXhqHQKcZtJ zxbkVSr4Vrg4U#;uKM8TdU ze}5G55b?kV#iOYuTiCs_n3P0I!WOm+x|MVqKv`QGx|O=j@4_RN$-dP{HJVh_kHE9d z>k5WfDC`Qx!)Z4grp)Wgx(vJxxa9)lP|}wu8Qh#niKd`o_M1gGd?=(LV13HBp+8*u zC4H9E+7Tw3bE~4J1bWD*XNh;nXj=E3k)8!|&hj08cvebL(f{_B73gVytaw!U zH^gHZ&kn$ZBm-kDW>dqt@gLMGf31I^4&x;ko&AMc9}?d` zpCNFcjb)>Q0~N+efMP9EwOgjKqohIarv+XgQlt{Vv%*@06v)snnGPMsnFYg$X)}a2 zp({e)$+R4Xh&zNA#;N1C6sydwTcu139Z;^%cY#ke9BiEjumMAlGYX~I5|a~c_Z z2oDLSnPS;-k;{$7CQ}q>nwW~Fde%9PO0#;3ULg&`5FQ{XC4$rVkh(fA`7^f7PzZuC z;t(EV-`QwcCgXA$2J(m+Gofo3XOwmA%R$|O6`zownx$Mm4-Zr@ zRdan~xZm&|)kscqh1D+X5RWE(x=2`am+*cy52UNzE`3WxYA* z(#B9irWDFXG5%C|GUyijBuG!xX^7&=`{d=S!)Fnu9u?Bc@d#?HUbqT`weAE_a?Zqj zOObhMpAj%!RFZqt5GInWmFuY`Fku z1L@3}3u$?<|72>t7wXIbDv)|=Q!>d#s;o4ss?x*%L@k2-Cn^b22qp1i9aS2l+P^H& zs{>jP50CZ7g6YnP#3z#b)ySPPPt+t&P37UN#m)#iD6=hfwR-AbH3c0Kb6+tj|Kht3 z`3HHkGvd|-L2XXDf(JjiFzhV^+9qI47?V>lfls~ayLf781V;LaFlJ-w+<`+TRG`y1 z*@-5QGO!jrC1d6eq?}kF^*<<+f&G&*Puh&Rt^RO^A(xUwbS#A;Ca<7H{_hC0c|LaW z7-JJ&CMn{q3@qxVSG&nN&G%)>GCJRVl`gd=4-vG9B>@}@Xy;35Mr-FN&m*q_E`25${Ky?1(8Nrvv|@Q8wN$!ZNbKtM_GiG}1;=;%+_UeiwrCZ=3btVtG=s28KD z2lF_*k}A~5=R4G#&HNwD8ff7;m{SF{SN$w|Oa;MF?D*o}zDuuBMP;YG)k6j0!*D+f z(7|wv3ZK{iMBIhiiLb>V6faERp0Z1}zZe&irQ32$bngwL3rR)Ttf$) zGRqw;Sb1#IAkL9wS1)ACag+cJDGm&_^yiM#013kHCqGEEm`@)4GL!`NY)s6%*LUB* z?CPKXiWxt?nLl-a1PhG@@R%P1E;J}E3q;>PIdW;@jIl>aCVfdeX~Py;FjC<%hc zIr7CbZb`#hU?|FN=@>s?QxHh=q~>#Pu!aVK7Xc~py3aGjlJbl9AAn;?^E@t|lO#K~ zjkk3L4bJ?6?K!vc!SEvAbyU-GqM@%69`FxRJ=<4=>PeM>Ox9KF0y9BLbrjc0jdv9P zks9YHo{^g1ID`lz%aa8~vg=w95gCsbgFej4k27?}{KqLKF^4u2Abn~7paqv}@j{lE z)f~1&WG#~2RaL1-wSir#{aZ97p>0iQbNIqleAv*!+jKGQCj>MkimY2x+CPQdwaAIG zJ{$_?U6>}OWO%8(Z|QnGVAGR_1E^di8StSqAQV{uHBENqf$)LV*wT$RNT6BY%}x-% zPLb|{(8f7>BIP(@0XOcRfo6zvyeM~&$%GG0jhd9>dG*dE!3Ny@9hR(IOgkRqudsaS z`{h5A8R&fcS*4RyFfBhy9*lB_lJtd38m`R+-rE!2 zX#py4(`Fje$;{Y#HEdPc}U1=_%YVq<_M*0 zKg?4_{dVtTPU6UC9et$kpBu=?kivLK_WZ)cTFH>|@kc1jS5aL+_{FCBg{{@g*0XaP z!}v(fW zxQ`qo{JN_$gnI)$M2zB_#}V*55cDRL25B?yjg+vmOkfXr$0AIqNxtR?xSf{C%Qliw zY{h&Siv_B0Uskilcd~cC-+_oc60e{)&&Olf}RVP1TiCk+$w%O!m`glNxz5@9j62&R+r?$Fd>q6Lq17VUy$NiVo~@(#}OvIA?Hru|23$-EG9nXqr?LKzs0zC z{Qi9M&%0-fe~aOSCRz^Ym3q3*W*>7LY90W3rOE(IfwVJ zcw9Ref4q<%rP)Ky!}vi;Fx1aN;0W~Wzod}+{~m<-+cDwS{K^|>>Wty1B=_TBz7KVX zza7I7sp&H0%54;4BD}dPql<=d!McH-()K!H_(N>fPxX%^7Q#ZHY?YJR4;DBnRQ{lm ze`@je{~Z|nA0>0Y#ehXGnLM7EYoK5uvDbbWXUy0?OSs-|K1Axps78NAgV$!}kDME5 zd`IAD>>-z!0Ocsby`dl-H6|#!YuR(}A@_0aA)8Mz0|DDseV(kE*Bb+oPS-jItqyqx z-Q##S0a>JqDF>FCF|N2+oqq0FnP!25zQNip2JiJVh$ZI9ZIlV$`3#!V5jZcYN)qlp zZOhu@A^R*z*n-HZpa^UT8zWd}h+eLn-1X+;bW*O(#_nqgCG0F4k>Nl=F>rpUz*{5k zvOGE~=LrW0LYDfggASG%)b+7g<49`v{=Qe)sv8j!?2Y^C#<)W>sN6E7{J1yHVyIaa zQi-66ThqYcShX2SI1~Nt(JR$41$Ted%3|$7>Lw{00^-VfX5)~n4kWZ)4jeXhEyrKY zHG>dPy5_$Cyrj5FJp^$O1zHv^OheHD>{;eK)8AIZzm47yw3~4f7dL`lK>R_~&!FSw~hV5QVfZTkUJRK|p9+fD7>#W9HjXq|Sl* z$sggn9A8(hh+b4jv+w@TV4nb`yytTRRv`i5Gg;n!9bNvXsgSm!r@ArMgP4K**#S)kr<7-VYaK3OhtRBa?? z0ky5Uy*mSqTbRynC4Wf>*TvdI*f+fxJ)9wG_4?y+R|ueg_#%!>!Q=(wO_TAy?@a<% z?9(kz%lzy0(AnvhiRGi(+wN5ZLn8ic!dSv7sG$!>Eny(|ld25dLyvQ{>Inter5p_< z*|Zq_X}Sj52w)084b>dvpdrTSuVb96vp3cKl$Je*?HO)`#ITK|jWa(W=qtEUU_1|X zl$SmxQj@LtT?Zu0eu*M&>KJu~eXTHa3OGqiZp1dk}ZJU%Zg zEk7%15r;t)KqHGN%1f(fndPiP!yZSDsBiC)stkZj{&24D3|=-%LXUr>1Rjp^v+m`2 zafcJb2nEMhhW(w%s*L1Dx52i}AM9`0sgJWEh{lnEMxZ(lo;W?_Oj!LB66iO`^<^XT#Xxxa$< zR=L+w@gYU;yw2)y1PsJsJwDy-VsQp=u|P))BRN;wGTVWOw@R|)&OjTnW=io6Hi(eJ z>>1)T6ca3y?_R>OZ`IJ0S8A7JDJ880IjgkI@*EZ^>5xm|y>8qxCC`x?T(Ye^mjTm7 zNn~KU$mR*dfqYnGFd>66;xo0ia@U-fmjy#ly%Cx4X%TBeMl%WB;$mjx0tUAch9D_% z(n>Ws@n4*{*gc#4cxkAe<7}u4rQkv4a^?n+2g9=2qlfEkl>p|2ju`cjOv~gVysTop zw6rxkKuU{R!GBW{7bif`1c7F)9E$(P8D});eC#@x{05v9#0eRKDGT-%=K)U& z8frUQYN6^cFL><-j^Q2R2?|+o;XxW?{*%@+<*FvD;>n$St_U~6CGq?&Ynh+ezpZ%N z=X}+64R%tkf2g&4cIE9P!FC_eggak-XdwwZd0SA zck|)&`5pVqU+*1S#V9{37-2zu zEv&)@vh4U4nU7n2=8a+CEiM}db=GM^IcSP)MimA~NU~{T7laFcr$HaL+QoG~xTkW# zAEO;BPf(6iB|KP+EpQy_&R-P<6lx0~tbrlj{YAiIr|}?(3|8P@1gH+7crkIuoj9_+ zz$6sqDN2Ja#^M(;=fy}$NlIlUMF4I=5BrWP2{NJ0`C1|XDI>iEala?&>4b$2wN7nk z*?0@=gCk*c^u_R9*$qQiycT@nFgt63gwDcO>g z?lMFer~Ylo8p0nw$v}j^3@J$g2lA0w$qZuuA`qQEyDILi0~{ybZW($cXE^5yQ=m&y zx8p3gva;FrTp0LcR=3g%2?4HCQr3Q1@83E-t4NDOe`CgG`_-pSO}Ax zHjav?fDS&~q7NPmSP37MO>yOQ@bX#l??r&q;v|{uK74rW0Pe?NZdvNtx@=yk#3un` zi#e#T9Bt^;X0d;MDJ;N__*$RbGe1r8SB0dJd99C>QxFX_vTCv?$t)gYEwzsfh#~P9 zEFo*rQI{Ceu z8j&J1jCZq2XkbVnQ$W))-4(4H>#37IO~Pnt9Gi{Z@Ov|+Kkx4+Lj>+Vu6?kY%g3Z> z;4jfKui2q}Ab>E4`=M9wF99rrfMIE;&d^Mf(ca?vO90R2$P2{Cq-jDU{>ZGEUqo6@ewO@iASZWe$jBiUK{DFk_+RG7N=PeiIL)W%_w9pCPhH zS^xq|yA6e*ftf*tZ#?jlf$e&*jv|xbKZJ_-v)ol5JISZ?K)#vB+ubKW{vwmwr7=!* zH~cSkxB-9EiJ|;&b+!mVbv!WLes5=({{b^Scy@_*E&UtF9t#)xZY@fo{okb67;bNU z6SH4G{zrGXLLo_mUT_eQW7z-drQ~Af>S|?g@z--!%^I7|KX@>Dwo47$cFkd1oaFau zPbuoQ6sT;0n-73Ad;_t&=T1P#SKKCFj~GuePrJOpynXuK+v74cs>qy$_$3c9*;o*c zIbaS%oR*%B>uzqP)b`bggC!*_f`!&lkk1eITgUt5z96R|qjfr{CkoZ!p?6!eMsZ>| zZEIh6dmJ1@fK54X{qJ&abiln!^+@qSgqDty9g)RBAk&Ba+}X6|*kxcY|)ywgM+S-J_$PsVIpr76?U-A#pO5`LlRHBID0D(BNV4A)|Obk>sEtvM(2+ zsR+^!DS;am*r>3-h(~P59&Uaw$P1oioGWU|bCHm}Z;2bQ9`Dkp{7CuY&#<&Qv^)?; zC}6#+yec$5zl^6Ng_yiq&K37|BaGD0A|<^Q1IvdMWK!=?)|-IY24<}=(;J(wh_?O+ zAPHBkwt`fw8E^l@GFCT|a=(hu$83774@RPQFe4pV8W?~OcAox}8^DR3 z!djuBW>fxCJUH>GvY%>nC6c~@?fqIp_FO%1hbw2ggs`6A2YEjSVBmlv0b<f2ACM)PpX*S5?GD<;yn{1-J`lL?1djv2nRL|qgwk{yf!7@5=3-7? zz}>wK5X(*4vu^W!P$yFq{wO!A)^V*3w?5rhAKe`IU`*u-Gw*uXV=+l? zU3@y0YO6WR(tMfjA{O8J1Ulg~=4TWC{cEtwcn^0Vr=38ixlV7&UiOuLwh(Op^ zi?5CAr!fVo!3!;i$Kmy@QKqx*0GXU#FVQuFSq+}&`D7`-->yg6(!+fZ_qKp_3g0_$ z=I7(X0>((?z)1OBIn&}tU}}jqqUGulS{Pa?Ni-~i1}XC!u2tlUXRS|VPTvU+V$<)r z*K%bW^q=jIPIL}cL4d8MT7Z9l`h5P?YU-<*vFd+a@X@pV)`KhvAhTXQ0|yDJ`=(K> zq6us`gq=wuEy#`pl&sCbhoPpQf3a(G;dI_pk)D#4Z~5X)GwsOa+cmH;vq5R>04peyw=bV%}3v~((uH{4zN!&g5ocGT!~1bGEPs-#kOn|FrGjk zD}yO_jh;Bnz?k`_-@X%@Cf!T(ga{Bd+BK;-bulcDoh0FEcLT7xPdGWOg66r57<`m0 zqbJvIf*qdxq!6;(`q15+5D}q^NwI9+kJs&PA^+*v3vZr_39#4P3mmX%)W01w2lhc# zGcvaQt2^XoO27RN7J`wV-y&#Ivt(@(n=w%7Mnakc)&jD~gkFHtZ}t5~Quc1>Ykq5=IO!PLoU;XP2*gJDGwlCZLivdTGMW zGi+UL%!olkIEhWyT={5}<!$2I|_6 zlpv_)vK3Udnp$_GBiy}p#Xaf!^SOfx+=-bVs9%GX?mYasfN}HeMy#zV*W(sw8$2(G z(Da05&o%LIW5rLC&~VbZ9p_>6$iH(0D;pcv)#!3N?9(&Lvd6Yj7(_n5xhoxI zHuzA+=6YO6uyW_7!|q^4pU)jOMIMaXiZ^mI89)@C1z9nRhW&s*`P9nV$x}dOr7m(_ z933`4mQ8J0$6%RlFEfi|Io1pJ!}#5W^`p71*I=qQSI}M_A^dgbmI_TKylMiU#e+X? z)9N-ws6~HA&Cv0nKjls^MRl~QZqES{jCs2L=i#^T?{2TDJyhqVo(eUR72j_DM_qu{X4h12s(6Q$!|@p;`XqC?_h^>Voc z5v^4$wn<#qmPVQ6l%fam(y7nSXtQn*?xJO|u38xWV*V||p0EY!!~lnOErLeb`Dj?- z2PO0<*^_{GuYt<-(s?Km=u4QB{M)0>p^670$#x0S9(2LzLrBcR6W|{HpWoWUUBe)$#zsTB8=PkXgQ=n=!9kRV(az| z1`m`Mtn2*HAIqC_FzYY~7{zXvU?4835e`xT&Hp;R}$&klmh%*$(&pr$aWiyGO$Z9Q}uDavg zsXsqFOlDeHRb+5LUhCJcT&mA^Qt!^T6id)QVF0(lo|ln{0~_XmOmLlycHWduOfE@L zY}`6^#XG0U&g&$PeKpT0WecsGNLCH8ImK3}vN^y>Mc$PAs==DywRt(`jv ze%yspOSBpMYE_Dbg7p_h)R4Z zomwUs4Mr$oCBwKU81f+f&6H_Px0%rFBw}h8UtTJf*5?%*w;nf7VVZd5_W1;I*wbiR zAwY1?i%peewWL5xu;RGRGbJYYeJ`vdHeRbibaT*Tg&@k>d;rb`Q@J}tejKJ?=t~8u zxo76t*Hs~n-aFGyxpmv*H$Y4L%C_KJAKzw3laWzF>C3vPa7U#Zt;==j+#=WR(6%I- z|BTlU4@Lf;Ru*NYj8(FgKTSPXBCdSzIydLsW^`TG@r^{k3@7ES>pA%8WsICot`*K% z3LHc-7x>fnTK`Cj(h4VF$3X&<9H!Q4>tsSGlEM(IKSZTo9bRlQdc|O;^H!OvE=2gRk?CFj-qs|?0%Z( z!S73hQ`G390r~9bgLK6dX<76r&Bfvg$lC{5=qBGisqkx}i`I-lY% zErn$)<*Bp3nhL7BXnqU?IFxNhEOTSI?ga+6G;fu0F&GZ!*+m}^O8Gp1wG(SrqbAgE zx7ha2&skqUj{b_be{pjuC~^Ni$Cow`>9GKks(VaYC}P;qxWo52n(F zto5sVJMK!tF)2=J6Bu2601iY2PZQ%rkepd3yq@i^^sa?8wImIYhBR7m5Ye}5A+eY8 z5#Chly%}4%19ka}9acjvE~8zxpqAmOtq=PZ*(LWp82DA&MET&UAEMecENK>DkEpn1 z##?Ik4SxI%fVGawT^BGe&Z?E=-L{Y3ZnWj?!*V6ne4m8S9nU#+IwureAx1mJy{TNK zB2W2Nb$m#Pt6riba2mZl!2=IC_DmA`!XCa_MotPEpHj$BZFqle+g4=A;R%cml?0A=PLwAL7*)>y5Vzgb;56kJyk+vzA4XL5h$zx1uC=iIUC5)Lv zS`6A1Q=8_%a#J*q7AQ@wgS_dGQm?HphOb|9$MuW%^9}3xKX%g@V4Y)m-fBPphW=+R zmgC2_QVRUSg92u+fBMmzxVgAG*lF3>GF!Qt+5PpkR~=OkH^71c^2k6!Vc4W$&7=1|g0T>N8Uf6pztmHDk^uWpHg9T&AgW(9+uf&?Pu=xYP~Q zLk-kL^bcKscY*zny1!b$M$gm(^K%raoo+?aCd&#l3z~ebM>McCQAg6sxS5u)V)PC< z>8+juRd8IkEy9mqE_S}im2Dzav2$f570G@lj`vPnXBMlz_Q@nIPEGWZEN7ra&sT+? z`mNhdEpNibMaXd5SxOzM#~D!=)YC{(CWCmjm`rlq8m^2jW-~k9=bw5>JE>I(B+2iL zY>RM+V7Ka>GP(i@J8b;MnCY52(M)(kAJXbMBKnVTSEXlA-Zd1|<;y!Ygx*%(!2UU) z&7JZJyFkIlK;bz55dKH!tzLj)LZY}OI!vFwIoas4m}()Hw&(VQIvmu;AI}z=?sMIa#G>0| zLQ67n!lA1A+$HBT221`E3;0y2^Q$_yQ-+zc=()@MEZf@EiCmfnhuz+1?Oa9WX2xD8 zRFWpZ^QCw2R?R_p_119M2U1tWImNxj-~=(jnC1N=R5*2K zxnry>8&`6dCe!p)*+OLz?DA+|CZo}Pw|U>a0k4q1U#|lbwvfUxT)3_RbxXI!7>?wo zmiOO(rJhh-{8E8kLqq#M+vY25^|AXNJM)6NJ`agTS6fP>#rK2yX7z`3dXz1HG@#C>ag;i2za>S4osV1l-l(JNhd0$JxD4oA`VTZxr>oyjXLFiC)K5VR&w<|ODUQ3 zsdMK$(a3!9Fu8~+Qg$K-T4Of)8LY6f0A4w69^SL%Bn3!rN}03&|E z1hCAszUhH9G;B30N0`+@_M^+K9Of@7B<( zKz~Kj+?_jJp8ql|;9yvQgc2>KXfpz-hoT;@(p8!(eXie3-6mb^SO3~k+|B(}akBo< zl$K#w4s%_Et)<|CJx!-2W|lGjH~bIMTW4vd^g?VZn#}JD6bjO~S)P6Air=_n7aTJWLs8XRL=We@0C-&W zKExs~Uz&~29e){jQq>x)NsUyE)-_kY&>JaQ21CBY13pN+})qZ_@aKOBYQ3m8J5>6wB~Q{He23dIcF+ucv! z_Yb5d;&9_Ip2Ttu&glAQtQ+q&d#? zbeZAxI<_wuEif3Iy+!kaWkCQpa~HX9>Y21}u5sk4l)b>@as@eQZ6DYLu>rG0m?&sm zG}h^`ZI{`GXNU^m7z^P@)rG_6CBXCxwBGG-I)!1uKvpE)_^2hH2F`*YRFEEH4@oNf($i!5YC39sdb`srGp?+I#*RJ}=^!fVns zc-f#%o*U-vtJR|PVtODXfoB~VrwC~n>M@+4qdv2>epaa+Ab{RJ%Dir4{J3oR*({ka z_0krG(Q$<1WitWO3u-?;_ZIdMj*}QkK(s#k5&j9spojr0mp(P4!tRErTF|B#FBDy~ zV}fw3F}@aE>^mK&MU5&<65rDAjjg(iu*1FwXz*lAYk6Z7iekxOrWfmp#dgz_C}!ns zxfHKA%Ez=GMgpZrvLnfs2viF@d6!wrAF@k42LUKm(ZdH)o_fPgt?Y3#fx}Vo5HKQy zf}1EolZIf_Ndb_o2xOjJJE0&t5chDFH|$@9!GA!l@M1(yphtL`eG$rzu0*I&*5tuo z2$IHPP4wu14O*@y?$*pnwLh_w1%uuaD7a`?gEk0|= z^KuWa_oXlKxF&q{@kHT>2&FT}xni+t*C3R49 zaw=W^H=O}%LF@Om4U;X~E$jE23?}3sl=kSYfiHgRhW*B+$ z8u#bfj!k)o&i!+~cDb+$A3@O4%Eia3EnCO|+6!lAi{I5@o2bPC=6OV4QDz1G;^El@ zZR~MrWxvn7gWR4{=k03Y@aNv^RKPv41Ojmc$nYW zs-pT-KqP)T$zh#7fP;{akYfuEpa@e4uz;HVndd2P;7Yg6O?4+cK*bD^_5z=y4UOJ% zi0ZdO=~tBwtBaHqPOcAel5rRlrclqGYt#c0*Y)IaR_h9%qflBf7wN8l$_=Ksg%bU~ zi5soI6>y^nc7Y`RBHYsayBh=^dtQ}T#gApJZ(+@S#zYeN8jT7X*>OVKui1x$)yvHG zvSHN$a}9~B#xUn(XD>^=Xb#=5_=CvaW|pKCKQR0X$^)Udq16=l3AFJ+X!IoGJfgwDe5^roe5?83Q4(y* z=k-&bcmx-pes!%~uI|ajt*_;(YaTCKKCkZC&FAy#7Z#tsHa)H`_4exZdw*WL zZGEqDd-n5n{dV_0pS)N5+y3vH(}&}~%2-yd`Cf7VW&XLE-*>oywMc{Q$a()qbBpO}&m)p0+Ujw!0urM$PplGiLiomre78GD}wZrMndlmq# zap6F>diFNf+!A45hr0=@r@ zFk(U-*4{X}0q7lFgaH}#$Od5R_M)4G-giWpmDqr67HUrt-4yh$7Q&Q@Mrf#@biB~@ iqqheU`h{DOvKVUXFuambZ zPo=$j@^9(Ag$BHMiU4V!Ahm@ppM-90pXTvGK(Dksf>2Y&xToVibcg`8)hPMTV zGlpquc0cnFrJ3PqMU+WHd(=@0fp|rE&wRk3VT~6?ll%n3Zw0)${^GZtdi?O957^zL!M}5t=P8o{0b literal 0 HcmV?d00001 diff --git a/packages/extraction/tests/fixtures/empty.docx b/packages/extraction/tests/fixtures/empty.docx new file mode 100644 index 0000000000000000000000000000000000000000..38ea79cef69de713a138c159aa6d4c735f1d6973 GIT binary patch literal 36563 zcmagFWmp}_wm*yqcXvX7;1b+6Zoz^}aCdjuxN9J|ySuwXf+je@-JOlge`n^*xijb7 z`@UbAhpM%HvZ}hP=q?mLLc?G~KtRAly!92+sZ=Wbk_ZU_F#-nxfdy{W7PYf=HnDZq zQ}M7janfOOx3O+alv7v|LJv86#z|rpB=-_U!ztUee{aw5g;1g? z8?O3OF%*vJOGfIg59v~!K$mwDwYI3(iH=T&dj4DJV(4qt{w3&6w>ojTO>1s za5XI%-UO4JkStp@{k(dZ!Dk{lS^bo9~P$x;~Hjy1l&@^%Ku}USOhD%d|>IZ1d+%3D-YYl?c&qh#zpObseX#cQjLV zs9WkSM0FoRvO}NRw&`JV8&&^s2v=7cb*-$mifI|VePU-7L|rqz51pm{YRi{%w4_FF z-l3%#qvB(67$`;Pc`J&Vanut)OH)sw^A#?Qts}P5JLtCm_LjCV_MPDULYVID%(R@% zn5y45v=9zeK&?`{aBR}|4?JA24u_&|eLy)u+KW75@;)76dIInLg$BItE>ItL4Pbxa z-y!=~-AX-m7VWq!1>_mYG9=MzKPQqDV03>mc=UQeDlR7w6YPDOFJ9RJjNHD>d&HL0 zg;4y*D0)zT6uJYC-YOIX1QPhCo}-Di6EoBAYi0bn3^Xgc-zUWCRMS5MP0Fd zxdOqSl(EwyTcA|YuddEdVmcbQy`+0*TSlgH>G?zp^hMTz>V}%L0lD+N8f&x{4eff% z@JR?A{uJ&in?ah^X^|jGR3Q`UTTW9oazXl4tU+zs`qQ7nRU%?~bT(3`B|)VL7}rS7 z@6}y=7^m&v>?CBHI3kKGTeq#QCK^MRE7N3z*3%^2#7ni1kJdBW$!uDTiyWVFF+@!${(T!I3i%E;?Sa2PX?Oxpd(o8srC|PpL|7EoyA*5hqTL4r`OH&^qPF&7sJsNd&n+ zPf99;0&?M(wOv>31`2D3XJ$5+8J(#70M5oOA8N3kn?|{5k_~_vE4iq{ZUSbZ2$^4R zuP-cE)4xh%7Yk>=NViE%F?5aR?brLhO1=;*5YyjrUBKWUnxX2_PP>V|JoQ}?{c~Kd zmzMC3z~f>K0|9~l_i-_>xBp{UR7P!=SkOBj>0xx7BP`wsf5MLsm=pdfxb>xOe^aq(C0o3hbM#?6wX)A_t-G9JN6V9S6P1k;+*(Mg-?nb;YU?uebGvV;7`v?41I@C-`L_w8+GI;&~GwM-p@qRH( z0#A;dh*iJye9Roi63NRMRA+pEkz;0~jmJMtDRe~2V1GG{d^do&jb~J_E(H7mlCUq& zJbk|effvzmR8Ov4SE?R%8GRDhtsy*|!@~jnEc!m``xIrOrGZ`VHAc3E|FT0|%T0YL zUmWVk8&sGFK9PkdTrs*|qHnZF;u4xtv3zSs?5>WvxRDboFQGLcBQCcecQb3Sz6x0o zFScR4^O^W5C|)~SQ!cZ~R6!w7HAM3+iR96cjELTd_EdduPjp^OM_xM`C!2#1!Cg?18`x5hK$?7lVEz0(YLwK! z#d<9{QE*&+?Hyle2A-;ywZfO=fhB1`p@L**M>_KpGwC@(`u!Er>n!M)?4J_@)jDqI z7CbSK!RylR8`jOv(U=*0*Sgq%7pLDdqwwo^%r6%7;WM9z_sJRLs6W0+5Z&}d%#zaO zP6_%sWDNXTFj@@xD(FL#T~HRZ?-;RnwAVd6!7uO{pXjzbgs5nNJ}$lPFi4J7CKZ?s zaIJS)*OXM1pjYHaVOoZpOjl5FiHGEquF437F-g#>pJC3Nmd4jobCf>$jA+^;a=MFb z4DZ2&k7mW9*+PzCq5cbEYi`q3yKG|SK|sFGhh2KH<}-Sogm%f7i|V&OJ{z#Sp)3`h z#?~-cRqzavmCNLgkZJkYA0au$15%Kn5-{RCG`GaWj>|CNc-LRKEN}iwKsP4vwmm+% zYp1f~$2lTwU|r#l`7WQYA-#@%%lzltHH|T9u{sZtxg5Z%9a= z0~!voLEX2+UuDVko@?i1)3y2MmeiqXdI0qe1c{7^X+I8^uA|(4L_=vbJ-5ojZp;dKSWW1xkaOC4g z)0pv0F6WNW&SFCCW{rrY)M8L8_|D+)>=Aub^YKn1-G_LcC`L^u0z*8!z+6u23Vy=B zBZNLnA&;tBVM?s-s zPX{ZSVDg)xB_H+FOQ{N5Rgxp{*7S0AF7S)U{XD5nG_Tl``6~B>Sldc9P>1-gG1pe8 z6{jk?U|&KxKBSII3rbV$XMBo5Xm*yfXZzcW@v4@~&I+q_z)G4^@49@wT0kELOzpy~ zz78b^mHy`0-Ve~7F|qIS8(l=xhFN0Kq8pgAT#05bNPuN1lyAm6Cb!OF`eUSSwef;AGo_Xwv zo-Y-J5Sglys+vq$d>vn37Oo=gj(TQ-ipleHwr_{^8BmWbCjI$zg{!|CrK?r@Ozp9F z{Pe2b4)~1k&3C_h>aKN~m9+G1`+S-Xm>;%#xsQFlf1E#yDF!qjZRy!dAZfoD za4~DxOVieM-XKKYASZeJum*4n$b7*Sly#K6P}~>}4ZPE1UU~$Ja$ECVJwY2CK6zO2 z;r{Uq5E&OZQC|h`5p2A8C45PNg718(eFtBWZrZ0WPLJG1w=C=L@nulnxGx47b!MN$ zq(7b~f-c?DNVrC?BY}>Lp3gjT7Gq($nP!1+HKx#zt(DXOJosswP{Om-qSwxaC+XdQqjEJE$zB;?IH1$UJSmr zhY?fX7)pESGUwxW6R9DlZNk^3Y{{S)l%s8wps^~E^{J8>RtOG=`GnDWhfM=rT)VD^ zPb>y$5Or1V!vm&OfO+ebiP`U4#CB*9Nfp6Oz4U-jHFn_Nb{U=rED2t9?#84R-U!;t zFS(dN5z-a4y*pWZ?%-ztkR?@5y8T?%ic@0i3vC`wSg1suwFUJb)U5ebVH;^$8a^e@ z@9j7^Rp1tsJj7z0w?57rT&d7n4y2mh1-^FZRLxC3o=iX3-#R=GOW#}pPR;}~K1np{ z%+8$m?_OOUoc23qu+s;(5*MmRzxMCCmyu>%y;luHTKutdN9B7?Hm{K0oE&Kh^CQfP zKBA^foLdOv3#1sSK>=G=&vSdv1tn4B@~ei4xBp{yNtB&>EOVwsM_hA6@Vo}EE8gSD z%lUx~eFo{!6oxkr#SQ$Fus4a`d+De`O6~-~Gt+X_Z_8p&DipCYQlF$sFqOW3C=X)T z!KSD9$T%+pA1~k5UE#Y%%zURd(lLxZSJA~Hmenx`BLyol=PSZ_VLYrjSGnDfGx=R-DwW)#Qm!iX5*b5r0~Dus~QQZm|&PakiPXvuR{HW zeZn3OP9+D&(f*OD1x5=7lW9>X@pH@>#}~raM|)&ZMvqB-bcW?pl6SY5XqpaK>a{Tcv&TjA z*eO3cY;^fL`k2R#wQ6*?cl2_1y6Qk9eJa(thrW@2q+Qg+C-v2tpt>fO!L`~LFV*Qe zust#=|5&voWQVS`b9U*X!8bgU(hl@>>XMqf+u2Rod;;}*Q}B^Yo%=M@Fr6yyMycLh zdcgC`z%t2cJ;l$CEkV3^7dg zcSSRc10T+3zjsy4R|A4C$(Vef(9>S-;y?+*pyFqW8NUzVom+>T7fUyG#ou&*cRpS? zUtLoo#(PFzldR!lbhmBcCcH132)Nt4myn;_U(Pn4#LD5Jbx)^UC41Rj$M5o41(BZx z;6j>5F<@d$55%lW#|INe6K8fer7yQbFBNG)?;?++af@g)p|S?PX@Pju>$uuRT0~+io4nz`cTYhMe;VG?u zhKAcCB)^;bdh}6QRxi!9KbiXB4%3jg&#^&PI`JwIxt==BO<4YxJmwL3a5>a>t+>QU z{=2|1iHl74$F|5dCDr~)*YoKuPTC@XZTFh^%QHu*TlU%YLF~@+PIM&5^>OR+>GYZY zwQ>nmJbgdAq2K5J?BL6LqIZ`ai*)@2Nm|EsW$y#u)#YZ#9D5{sDNcS{-rnLM@o8l* zgfJp4NiGR+!mJaC&*khr+PXqM5&x6>qYPc&8yR`7HX7DK^Xj3_) zLa|OHTo7TandN}yQe#YJFV?Y?C;Fb@dNzg-ys(GEq^)u^5kc#+jnA{n4~KUcoa(Ekb2iW|9NV|_Ou z3diFmFAbv(IP8WckPtAWW;*OER%*9$bty5l0KW>&fT zSakAM!FPx;>S~gx)85whGpNKOxf40GuEUX&$P=r&;~h%_8`As?CEpdy=faYZ8`TpW z-c4E<%iT@h#wAMCY84{^;rn=YYh}GJ4s~U<6AubE2p_LQc95Sih2>TiKU=zbl^qV6 zR!F|rQ&lrs*V}7bLS{9dArAS*5qj*DY>qW5dFK*}NG7a??u%$Lk>Pi*G^jl_UF4&? zc;2z>6BCovrZ6|RC7=KE1FnKRYgLW$k^n56aJn>9)F!&)Hq{H| zCjlC5tnXI5`vx;nq})FpfVHdpHAPhdAQ+Vm&U{39=XYf#9K1oQtX0}Lk@g+qrWC$u^Ny-%a&T zmOfl$iFv0MYnIBH$31*x!}03B?k!axFz(YBNJ|+c7)zt4!=XNQCEyPLj(74(Y~8%r zif&Xo8BL(NrlJ&8OwkN=)ssQ|s8gP#p+p(d+s5-shxds~Tld}3)=Xa;HcMtADBsb? zwCN8%G@DBQnqX|qj5OwJMryM1Ovo&t8ZJyCrqpy?cKEny=bXU4RWK8U_3ndwoBoh1 zXS(z+CoRiK+IWFtotokhqOJuPi!T|UvYQWgjy>N?8!|8uvYr&TIi0J`VJtg1xkND_ z2TQKU)xI%?UxaqlZgLac5wuI#uKOIsx{wH)Q($2tOm_53oc&dQ9`7reoF$4tpqNPi zdke}DCNAU2kiqwS1%rvIWXEkc*xV&vhzmqsl?eHQe2rUOZ5M*WS9-I#m2dhP7Aj`s zRA{E7MAh6wRnED}CdNc|PtwwbE`7eJUVTGvwd5j@jW;1xbrKBwwBr+1UMvDI+X83J{MoZ%zFQ~wA-Q#SC8E085YNUWxHJG; z6;#XhNKVIb2tBdtoN|a>RUtv78O_k>*Zld#%fW&zOI^4PlIP=_#gl!^cLiBDRD-*e z)9g$b`=$=cGR{=Ik~t*_`-yj9kVP_6(ahWmfU+lD*!)8O&t_5gFXs({8QqaFd`2oc zy*$#aN_L|dFcc9%JL(|8@ewPeq=ONjhHQdm+3V^JN?4f~&SRvQpEVas=Vr5ZR3!>XNF(jUduf2z^ zwXSRyYf8Dtw!U&)z>)TqjxKrBY2J5K$M@U*)<#}oXafb0=?c^LvXy#E;))HO74-$7 z8PT{9Pvq&B^JdUShhuHuldRFk5l6$>l}n$F&HSh+>1vQK@Rzkw9^SKVCFuZOeZ8^s z$!D)tQu24UjT!=Hx1rsh)uiv)A6=8aI>}J{+EL`V@ux{M;t+~u{qQrbXWle!hEWOb z-9XXGGSfPWd40PdgNTi>riks35Z2@k)Y_zQ!c)eg*>Ol2x%lpzwEX5$ zqxPo$$Ep(1eYy3ItWr?b#7RHWW1KqtxN^oXpz7c|znCC&5a0auo+IvzhBD*Ls>S+I`EDNa znvc@S#VIJKcq942NkeWfO-|B)^qabN?*X5oNurjz)%YDS{N)VOA z>?m2UIoU8e7yH9MWY%N-K;d_T=;XOtG~Bo(#m?xV52d;;+9GCNM{zIDFEhdLq#VWc z{O~42mQmYNv!<%5Rt|d`~AiF}to4Cb*1WON`7e5S_4FrAPK`l}HNfiT#+K z(+)9s=Wq=?x-RCqYl^AaA0DH?C3+XV%KZi)@vu(%`W|YUPN8M! ztURBuEdJMGwcUrfDLBB4eu2Y!G9|6>Q^Nf9mLq8A%HS2(R0fXmd%?$zfy8?9t8&Uh zf(n%yZDBS}YU53q`ybj0{qPXoMOxWI_cxS4c{+6@@h2*Z%+>Re zGv%#jsMc4~KSX)bt_MS9DKd{!c($l?mLyw;8uQ`Ij2_v1mFPSlk_hrteN5P2S%%*R z&9Ie5mT;NPT93vG=IbK5=lapid~^yUWm086q*|g_l!pafdWsZ&V(y^8&efgmV)En) z4yZiJRZ}Jr>*Ogn)h@gy04en#!Sd(oriotLm!{2w;!o_Yivv78Ivb`+@K&WF6{{@) z#GUCD{vBCb&Mp|B-HJ3D)@ieukC4pR=Ruwps6YD)55+r;`Q9x_=db#ba4HZt5Ou@A zwkG3Dz&Mr)hkr|H&GIeDe+l8GgLL2#@_+}wB4H)mZuw;`C_(4r_FDIJ_HquyO`{<+ zAu)_vbf}l@-lv(*o6pYYVgt1DW+^4_r5DTPNr_`ynTnygpjzk>O*XYFafp^jML`|5 z&nJ`Xvr5=}#+6cTY<)dd;3o{upc^jCinm#k&FgBz7wevPn9j7Ho=-clmnSuVa+#i+ zWVo!ieIT%`tjb%nyv79q%T+q^fmU&!gvI$c9H@-GKJ+Ug^l$I);$Ar;*fnZNWL9#O z7D$eV&=S{O1y&8R(w%-f=O93yQPQSglEyaRUqbSJ<#oI<_q$e=8}->Hs#Fq)_NnTc zBD>F^7F3tSh?@rr>c9jwXBWTK@79xX2~^!v|oMyNx~RZG=aUNqC(w>gFwio@8+)r@@XS%2gDM zvdgRCr38z^CucvwY-i49FhTWAS{A+4cR=FKoYfGZtDwo*Z>dYaIk2{%^Ta1!dCKxj z3u-pRY9Nd~DSVq_sUU>1<=^KY;YYaY3!tVP4^UWoOTH*S<;n(2Fq#d!C~w{DKdsKe z2o-PFAOm8*X(jZ6il02#aB{(h)bW42zH@Fw2lW;k@?hWc-{w^IOKj+%Ts!s+Y@MAV z{n~<>wPr=%^OwRe$p4?!6ceUC_Z}wtEGds?ygc^53|{Cbm?wwl1`n_M^A2ysvl@5y z^ZT9lZI@ste`c@h(kD0id(^zOj>dY)z-;9neu(+GLK8oklPM+gL^lhH8c<-oVaX!q=n7>(`B@?>$7N%froZi-KI-*)qPq8^tR7 z{@vsYR4ukx5?j_J9MY*lq6XF*F+}4-zi*JTr0A^VO>ddQ?qt;O5XDXTzG&jXd?P{1 z5MXJW91A91fsmwuo4#8-YjeOKm+zR5AH+t}-K)jFyEqy!L|nATh}tuf&(}e+LG+h` z%yfo}gXVcaCWmtkYHuJnWYsZX#ZEmDGxZ4`8*2?&#A1FNGyL`w#opPF^?@{HSrKab z_?s@HrtHPG25F>!HrYppqBxYj8$;F&XT*k|Md)GJ->{*+K&Ob18wxQ<=%UxARMp@| zjJdw8XF`wTOQH2LR?pT%=4peBk!O%t!y-#JWcAGCDWQs_CG{CLVm*gG4;#d;+h9)_ z+s#Oe3FHhGD)2+ifcfU1A%uVqZOD2V5H{!>Jg8{ggq7;z@3;INbMb@Z!{0P&&dx-z zb)w2mfqhS&Zh(Cw2o4xV99hvs`tQ}$Z@~ZIn^6Ytn?HjLT$uk!nJdGKA(kwW385A| zlL;Xo<3n2r4;Z%f8NO#*3Lk3|5~^x8YdW&}8)DIh92HCFFEy({LpnE5Ki3uCGZo|V zbOci~8vpTo&^1g_G}I^*>6Z|=`GTBn>Q)5aWQ&qan^ifsm@l5{a(A`;MU&#Qi`RoW zeWIpv;viLn%nt;P4NWHWMLKPH8l_ul*V92<;lPh|A`gl1epUICIZo&+r*6$L>+w_J zT$%oOjuDTZs>D`sH0JZ1t=KkxHlLXF-|bek(uQ@MH6(lv@(f#8d+Za&`xi19^|-T`;QmFMIN=OszzV(7#7H0! z11OLv%;}Hz#;@<{!@qaz-VgB9Mk!=Wb0_2u(>qLPseJscq*)lwLH^mfdW?rYcCltA zc9Hzw3XxA(&Pt#+GtB=~So7ZsllL+J)-g|}jXxt1Q7k3L<31*d?(oLZpeEW1XSq_1 zr_fI3(~%;&!mBkF@qRPJ+9C^hK6Q?II{jmXo_sxhF}YuR-gst+>XX{!4U($gGED{k zRHzcK21wc&107xX3z;Ul6o^dNwG({&c?8C9%6@LK*cTbax{%?@JI*s&j`74*-ttQF z+1Rp%0i9T`y+aSI8Z$ZOhoYfaB~|mEKHMmt#Ve4*;00~-o=~GDS~emfosJkFgyEx# z1u-n!yq_{Xz9xvlFL7g++FaCKe;g|-vdw`3sm$Qa2KTY$gtNW|X4NV| zAAyPCJLqU+AP)uY0!*ERkNWX9wZVImV*`W^BeY)3?MA9ao!3+i;$G1XM~wig zkbX=GBsZd@1_C~x5GuGs611d0cG=@CEBU5>kynd?$^X+Io_($9(7G}33Q!xL-O*zS6cP-7ltP!`;e2+L{{&%=^- zqSdal6Q8^I*`TUYn%NWSU?KxPQ0;LOohhlE>FIY{S>|aH>~cM> zMV4I-Y19I$E;BnQ#TF;G7E>8Tv(@Bz7UXGatMsEvXNltMmC)>RIg8Ox%kye)NYM8W z+?ZJd&C@FGqeJCgrr5u?(4-Y%$YIrFmZsTd6iF+nlDK$GTWB5m961+TG}4Ee6NVKf z-l8_lc)h`tJA#nIGJl~0+&0dBLWc2tSkeATo6VF2^9oJFq)|ZOJ~hu6U^GqV&v%-l z{N=HYOOEMpVCie=)O^29$`xBw-g3-k7R^@u%cS-%r87pkKTYyf{5CnP`P<}~&Htat zehaNfpH+@x3lP|38jIZTuVe!!=lGo^KNcL8k?B8#Z*d3iC(A7};Z52C(0X%wV)l*xzVKmI6O`qwv6J?*8>$=A2oreY$hh zx-W{?_H1k}OYG2xcyi0PE4)_t%tIL{!)mq`(AGkUT>A#0*+J>I-M6%ZctSC$H$#G z^b#eW_Ylb{F$ho;0r@g-V=VWMpgYXHmkN~-{l@XTh@KGzx9sTz8ddelJ7{plWN{@- z4;=zC)vQZa%ZJ@5Yy2)$w{G#QwyWZF5NQQoFys!J8d7=dDfseNm5ev%0%ZO214#01 zeVW(o-$?BRHMMIWF0F*Gx%jQ|^cr*Toz}y$@%sx4pLmpo248QLlH$zyNjiUCjNwz= zx(PaJta0f=Y1)HT>(Pyq^p3qFy%4a867ohHWLjM&b5O%R7`ppqSDrg@uks1e{c}y= zOts5&s@@=;=VA26uzMH3dq%i?|Nd}Aq>(2WI9Lv)Qc~P0u0l^0jdbjv4kqn+D$z?= zyYnJgUvX3PSS|$Cuz)g#KsY~bL!b^j6f1zZU1N1QJ+c^to2vJS5$2igvLV-#QuOemg~rfA@1r|<%%HtYwd}s!H?B3PWZswN zVV}3?zbV!GgQ}?}$x~b752^|8CXJS3coN~%?4Fz)AK+}i$j!ZKEaK804R6)4V0W3r zW8YJn9qZDBAb=|c5kSQXfbL;r?PenMPa%uY=}0|$)8wvY=aj8?bW)6Qb5Hx5=-da` zXtmFvq-1w4H&zWj#*D9^6N;6~^z|MA8PSqkSAe0Q+Z|L=aY6oNu6kMCEh2Vodwz*rE`j2 zN9DJ5MNuFMF}N1EaS?3aEEWhvZ~&-yk5k$8U%0xB1Eta1Zz9%LD>bN7UNI{jpP zN`{IIvrDIA%ZMyo(+KX>j;y=9b_7+()amaM$)-%p$)+S=0WugW=SS|n)rt8(oO>U4 z-w5D5p;LpF{Y4k?a{{uvR_#W<&J*nw2V|GR09-KgD@c^!Z%5u3qb={%kEzclRkGCqQo=322kySBkXyTkv`)OX=Q3>gW&z&eQ)0 zev)Op`%^F?5G)wXGac)=qETHR`Mp`o>}rMnF885DtGD_|%2fMV%Jg5r!$`nNV(*H- zfTs#6ojS}p;Y!wvt8dfUc0fb-rR!#vy=Ve&4v%0}&P6#mah_x|fhsFf^^`~d0JeLl zcHW*5B@YXryR%4nTe3dv-g&gQaujeoj^;^sXVAbpnnjlx4M}%*aP(Xx>(S`|q`OPi zIv1VvfAE9^yN;HF1t=Gj?T`W~?Y8TWh8RwTV7pNt)WF_L1gq4}odywsOgK+qX?Ea} zkZpnBkbwSsd>M8u8pD~&y$aPi#Q|h}1*B2MRsFjFa3uT9UxWF-)uxVU9Ph(~$R}Jk zaDCtSlpnzYRLGKP?>t8PlgT)Tm2*e%I_>egp%{*%-;875#4l=iP}|jTF7!f8lD~ZC zIlqsy*pWdc+-}ol_!urama)LTjmaKp9Sq4!Rd%S<3EPAxKIUy+}i0 z1HbEMbc{mky{KCF4}e%NIxxT|g+Bq>yRnFAxL3uwZQtD354;Rb8|*j-Y;h17V{=bm z$S#w^3`aLh`mVbxj~v4i533eI5i^qxDf(@#2jauW+#%`CCY|ih0((@H#{7ElJ!nXy<3^vb& zHte&cgS^1OT=ZY*1l**nxDiKkjxw?W06F8czYTT0NUQLh%I^>~Ad)g6Um+ zCr=V(v*;Q%>liw(;3^eWc`RYkwAf|?hZ$@_BBt)rr;$}Hjza_Voa4*yZLdcQ;18_! zhAR6@Ew1ILOqWgsvp_%Vbu^wmDPUq2ZFO7Cq4a}BLeV&W@Y(Pt&T$+Cg__Q2ldUUd zBQxQB@U&@VCGBa6aw3v#wm|~XSiDDh>#>QGekW;9=sJD*D7c&Lp{*M>1HX3$X5K##1xerwFvRAC(Ml%5(0Txu4&3zkfe6g&+` zFJF3ARzpJ)6T`Va&8Wpp-mu70$Jz`>C*{5!$4=`CNN6mw>Fbrn&MwrEW2>&(oS%Pq;(;*=yV`E-ZUT>2K82NtbwZ*? zuQk6BbD!x7T`il3n%c45sFBGzXY;^FoTg@#o2n_4V@?Ibd^YEYl5v?%^L{9e)Si-o z5oc4A{J7-WBb&vLfJ`G3sM?Pe75Np`+p&_IkKRJ2L~@2Em90PoWkotK*kmU%@=vCRuUYcYE1R(lCiU>cL7ySZBEzNj!mqn|z)?a}k|Rst z*5*dbAWN9R8U5Y`$eASbJIC~XWAij*SutRl=*w#Fs5;<|E9*w4{);Nm0!*dv@;o5x_bMZkXRolJjV z(1Y?&kQM_~1;7=F)=WKvwe86Ys;lJY8y}e%o23wD!~I{W-T zGAsOtMgm&*#uhVKHLwiRD;QFkO*5RPG@HRwBq&jwhV7zM-q2+QZAb7 zxyogeR-o&R_H}MMirr@+R#uZ})2STgP3xZr(W#igyfQV{apb z*ugJ+0cJ}Ka6hZ{i|Q`Wn!afEV84W|mgu5Ig-485^YEl?W;N=RgU= z-b3pKqBGQV$LsHuapsEXdpOxK0n1%Pg5%ZfEJ`5Gt|Sz5$rN+%S9N>>5e)|Ju)=+^ z8%2YcQA~0~46N9MVFM7`gka}yJHOahSz^)M)$8xq65wB{9R(p$2_#M_Lar)?=ZFAu zM6ToK>N_YH~rkv64S zi$>+W#r2mE#B(Mx7##4*sgurR1qq|3D=u08isju-|9o^Brsry{HZ zX|GIeo^H$=P%~)PC|{yj_sEZev#vdfKz9vA*HDw$VyD>aO1&DT!)4OMUHX>_9e4OY zRlsDO*=t4DIjWXh;qd#8Sgen9s5+ONSNHFRqhV~S=YH#Ff)39pJV9g}#qz89)NvL}fQy;cbdhXHajPd5 z0}41_W%(R&mGcsG5V4&c90>D~s%cg9&1K&&B2OZmr%WJ(F!G%I>Z z^^%R#T-BcYxIxj*yp9o41cSega}wdQHdhfwV-vb-bQt9aF1iot>W^Rcaenz1P~{&$ zi2nwv2Lt847@dMd5kY`@2`PhPc}DGRgLc0^5VkZ0z|R&hgLElKkQ&Yxk?-=~*p>Xw zg5DJfN#&{)xpy5Kh%_5QP6cpRaD=Y%XqUYGUO4=TUth zo#cFpx|7FyoW{dJg{&k9itnR76PQp#x}#ecsW*3m<>fOvX;txuGIY)0)A~Z$ewg24 zrM?B708?m%2;y2lI1QtQ-M#p}xO;Rx`n0Z8avt5TA-{6o?b@~i7x=5@ve$gC&h)nI zXx5fbZX!Wjn;!MLU9Y>7JE!Td_h+vUFGp_e&)JisD>|2tpwss1YP7Sn>>KbA{?*PU z0e(h3-1hFy3eu+ZmEGwVJGAsC90v_NYGj2 z%5!OJGK!uRb7bh%s|#G9^yRIkkHywa3}^MIi{s_A>0{jjjjt_$rh5VC;q}aEaph90 z=kr;h`o_1&=04dJ=nLrzanCP)<#{z6bxIh{ z4oVMgPTukKZRvDd+HrF1PY0b^fILsQ7U$v<6!s+O$hfa+^z=fre&vKN4bE@-R-s%j zzieGdGEWZkRogmnuj7Ad2p?s3x23SV@}|)|_enfTv)%)$9W8a+aoJVRKfKDWF$ zTF`64iT&b@{&e%x)spLK!{Q=MHYYjs4EWTsYK=#hoxmsK`NSl?{p|ZXzoJveFJMdB zw^Ka0bOlT%9ZF>&3=P@^KPSh{xu8U$bO}`lft%_PIR+w4(Q2wVOF>ZH;eYy!5{6di%=o z`QF>H_2p)btnN0I=bDxE{smb3eA9pYtMB%VZN>h9a{g1ezq;C5C+>soZJWCWiLLR4 zMYrr+lkuH$`D8gZ`@{m^^`g4jc?{R?QSOCFx8CgwS1Ktj^5I77?f17g8!dQLcnkpW zSIoOQ@246C`HpUYh63u#z(uQjldacK7cf-`XDdkbDeHM|C7^0`Q9~6dsM9` zkzM%_BMBO|EE(#{YyHYbYyOC=03+T(l@C`%BPUhEE(;N!-7uBqFr}rL*i4MV%tT5_ zCc-B^eW7F?3=dS$y(Zhmv$lG6x!bud4E88hdQdBb zNUs`#Rjaxa0)xF@yp#K4lvUJ(iZoklHe#dpS)SFM+>UzrHacP(dZ9nCkmEAglphHS5o zO^fph#-So@cMhF;Rc@nvf(cBpUgIYPP=N|Hg@_uI3)-+17}j-AbJ%X$z>i`}Rer(& z&+0vdy%G%i(e{$N>pm)cY@xJeB&q$_Coy9NajufKL!p>866>-E8kEaW+!YwMkmGln zP;+5Tex4r(v>`%;`xyQD2{&+i45ICk*3U;un;=k?RlpszzrY>jC*Y1aa7T?q3#|RW zA&BSTn<1Xlh`OmZe+T`Q{X58g+y;`ksN^?F!*BRMQ2rk9ca;BK1wKbJtbIz}FDP!= zpSF<1a2CxFsDVLXFphs{{g0Ji;CVT-3x_|m zef9Bxsp?HryKzKT6)Ey&E5K#z2!%9iW^HjtS6+a{cx@a+)KvZ)8);ne(#q7i0lJL! zIrzRe`!IUHy?Fg(Q9J(GktFOYSh@9-J7dBdBlQVn((3)p)7tG8Yy_yB8`Zg42laoC z9aTuTnetiP_M4b?`JH{o*^uIXlA(mD}{>UcsN{A=ryD z7R=7XmNyp6$~5*mnh{0RcP;BZHixOFgxP(1FoXD}kp0Ffzj(PfAoq7LV3fS-bLy($;% zq1bYu+M@P;m)R?M*Ub}v;9gGp)AJT@riY$FZ`W1=|4Z#R6c@)u@B62xSaHGh9pIJL z^Yqq8-BSZVmy=kwjOnRm@r;X^n9&x^?q-HPHAeNKOJUf3y%p$n-?E%#_@&UHU(_}! zzLuZbHkVW)wQ_rJoM%sMu`+A&Y67nj_K6zl(4awqN3nC1teHpg!%Dk^+Wgt>(HFL#QHS~+>I7Du zKR<}cugtk0v~;`$I(})!V^}@TNY3UiMB4rKvq@hUQdh8cUu|TQzcN*3t|Ea17Q>6A zYdhNX?XII*zooZrJ5adjDh1Q7K~dTeH0FMGw2+t@Ij(bIuJ=J7(v6$Wzj!OLZ~CI4 z(t&3|B_PSR&`35A{Qq9{3odZr^WTiKF;xT-srIzS6gzqcwiN50#`Lp?GCSZ94_=`>uYObTOON8*n4syVT|Od&Xv( z`iuyexhPCADkkS{oi1FWs`$%)Cpw&`!S#^L1o;u4<#Iu1M-sU_0P`qvIo8lzGUd=} z2sdN_eKNZE-GU9@=(j3pw%r*!Dq$mpNvhfpYCh)?9VYBwcB*l)U%OX)Ik-Dt(C<#F z|LW+NCnmk$n`T0Nda=`1Jowds=W4Ek+V^a3q_j&rHb&=s&SU=G`du9boZI#E5T&uBX~NBP{vpJDp}n)> z==~XI-@He$l%wc}Id8M|nE}sI(>UW|IpoB)VC{#=P2FZvs5pU&mVne%PucYaoQKIl4R)@7s=b+$xXayLY9^H;Cwal%Get z6HdywtN7@&KW0m+a!!*!w&>x5*NZe=dXm*5>gj@fuYJ~la`sSV5&GU1#0BJ(Es@~4 z0ly#ONU_~Qq%n+$NBrRgE56WV5z&<` zV_(p5b|FJ)HpLU1Vx4nDKS)-)E6R(1HZI;P^1tuGbGWG zCIV~~XH(f%e>ks1ygK}==e@Y+jOdim1u2r!%$tT^We}L24PSN5RO=DDu{EolS&}s3 zjh-cjMTTP|YPUfyS~DAJZxoJAHK0RSbEB2Z!l}sdYgchoVSuJS+Oo(mkdqn7+wT#E z!S_@d^MuoQHPB`$X3ZS()k|G;UhWqcexFkFn#HkF$G*#MiUh}qd`;Lln5azOO&Eul zJgb);^>gM-t{bW2=YW zTy%BwXFJWaqN)!=twq*+hdjE6B#T)tl*EXggXu(vCVTRv!11xC%9PI(MiI@Wq&N$s zl5bZwT}JCEL+P5&p(p2Q!p-&1($n}ROFgO7l2Sym%Ly;GOp5x7ZE+AG>z6l|(2#=# zYKY9sePMN2!T=k0q|x)mnzu^{lTE~3i6FFt)Nbb<8!w*@LKb51(IX|T7umDw;C;v7*N|_`p}m73X?(L^fa!)5 zx7+4op+T`eS~jkJNh#_=Hl_GE>ayf8&|U9(w69{cd*2?HF5L<3NPX(X;S(Ypt!a5) zP!V@J8I30Qts5cvtlEAK5S;+#l(rAo*QEMuo{X+yQq`09uR>T!(`&o$uHY>4v^+o2 zp-cyh{bfX8wCswMFA(SolO1jR`dgTxR!UZ?cw7g$B4DaI#+G(AH2L7buH>wIP(!>_ z;*W}OucjAWzOT@FtrzxQ!5`}|&37{W(&2D)*x6DPQO9jY>ja%FBj3O^th0j~=ssny zjREZcW1yv#qH!EM&|U9}1+|emFB=H=R6??A$hnbOFW|Pp?ZxTnb+IKINQ5Of`69O2 zy=zN&1;2soJZ4>(V*JezyE(`U6s{#&sEzaeXvqgHn3SUStBb_$mL?B?TQud)mMLrO z*ZFSj;VoA_lV1=kdGendnd5y==JHy-Wu2T^mxJB@ULpuuc8h;WO>uPiOX^d}N*7Pm zpcI?Ytu^B}w5#aiLBXA^D6ox8TaE?!4$dpbJvq=vdq+3L*W)esep~*h|9^J!Y>{`l z-!`1t+c+^0LH)h!P#=x`83qM~F5u_3zHOqPptR>M8*(u?1)e<*R9h%FUjMQtm8`CzA%N z>bGjdxO5iN*`_Rzi&M{J!D$4y*NU5HUipG z??8SwSTQBH&gAgI$7Z(1uAE*~R+D?K2yN2lFE=CEhY;I~n^2GVf0tXO{OS$_r2|BK z>3L&XC0|=8s{wC7=cTnjUNlUZLtI{AMoGY8G)QSPPf5E&pWBV9uZ60g?ww+Z&z*<+ zgGH|UWPrDEo_7$B$rYjLvKSbP9#mvWr&6{r`(c@?!6DVl=FSi>-bF*cUOsU#iBu2} zVR4w0|3dZ{(z;yKs@|u=k4@dxOwq;uT(rWYtj(FlmgzYc+-hFj8p3UOf@is|1InZg z5mM2rp02=mUadri8rquDU3H_ny|*wEC%V_5g6J`^Tg`r2jeeanQw`%#4ecPuAKDrN zw+Vj?RI2Z~Tdj3kZG?cu1NrlD87L~nKmV*+twdtR(*~`5ZnxTu(T2~Z{r5d98Dq!A zqR3A-cN;lR8}P0}mEi#$n*p6f1*sa+UH3ui>2gdxcN=w2n>5gnAO0=f>_CNPhGm{M z$!lAUTR37B-`s81CUF#o7;Z=!u&uVtN(maq26b%iJDd5oZqpqA6nCq~Ei_l+1-f_P zE*B#iy4Y#E*{6U-js@4ZaQx<2m{hx)Ck`I89HL2FzH~KTm_;8aW9IsVm{zA|drC$i zI#f$JRQt7hlNbOfy4&1B@4FQRI&kgsJruw?OiMe!yY$U+&$hHe7&gJdQ9E4zT3<3r zPW{SNdzEE|n%KuJ!t;@`a6^d0sCqoR@aBM%5h`PqBhwzudH&8|=k-*l{hooa_4=AR z?8~uEXbz1uKavWqmvyNgc`<(CO84b6Nq_9^aL_)N8?TB$W0=@#!{|)e=Olj(xW%|oV&k!Ypusyr|r)Cax15t`y4T*y7*DWPlMNOO@2F%Xal zE7K6#!YuqgRmbYYO7%3sr92Z5)S|Ep&LNY!U)OXWkf6Q5m|G|mjZB2O2pAf<7&6J2 zen>(=ND`Xa^Jxbh6axWOM2609B$7Hk)Vk6NJ*%+3tcI`J;3Hov7)V5C_t3<|8L^w?`UB?M8<-`6vBogqyL!Pf%07AN5{1uYtF=Vmc z;$I=NAdpA|!}<{9`f`*fiits7tf_6#vDT>{qLG9mRnizm4QJ3s;OvbcNJR~ANCOd8 zJ5yX~M1c$~Awu#z&m@$%0BS2r?XN7q34;mdVPq^&h1P={*o8m=Ill_k@QUu-gg`aC zd=Um434;0^**!Vs7{6?G)GGOk(x_A9QI6!_2HZa!xBw&!28Fzu5A8UEwMnafnW2V5 zg2bBdvNlh{bX#i&h3skya~BMCkv@>tPyfdQIv>+nV6T+~`Cup@xamG56@@{rEYZZs z>MGB(!qCjBQ`P|XYShRv$=3ziYt%L_1B4DIjNc~X6AgCf>`b?QKGS=#`Fg^jI zG|c|X#VFy>O^k`+vUwiu+F%0^qK!#6*n(j|#2~zEz*pFXUc(i7jT1J=Q0cSnwpQp^ z&np~3P&_Qb+&0LXtijBII%)k-$eM3sd7zIqLDKv)_6#AP`=b96r_O?xvY0B6I0 zH_ehZ7*P8I(F#7vwTB28zc3g@2-M!q;*<33A?6#{4K~Ak$J85!m4WfsNH#`v5+qKd zHg`vy|H(&M1k9Mil|e@vT>PUMo;V;z)~E-`|BDJ1Yg;gO6q5gNEF6#FFYWnqBw)XM zY%Jq5D?D+1uC_l7=JS7UFnN5E-=nuq50AHidEc~+H$YavrSI~sakXm@CDptRr>bS_GhB^QQf14(J1tO*;N|tQxW0l;_UJ2 zqoG{=$9>nMncBDhO3wL8LdeGct|I4Rh+h5itSR+cQ$^!SjRwlY%}MPbU&PNItuA*KxSH5F`1t+>hmnQRjf?4xpwaD)QNj)*JTY=u{YNTqpF2-v zeczvhF=;Eg?~;i6z0CTbd<2mD3C61OhY;k=r?lybYq!a8(zd^)AB(ztv?8xxb{{zR z9wrsSM=;YdQqqqwaZFCuy_s;_ZELrcEUd3zxVwC&rnjc#+~2Fc-j%Z4tK`Q6Qoq2| zZg=Sit>iw|(H=cmeEI7BKH~oF&zq|teJ0bEjGMGrQfn%2KB>V%RliHSmU#POMn7;K zy0RU%!YDwFV_>SSX`W$V8I|v_Km;YOaZDw%t?O0-QJXt>{$*W= z_aSoyz;uD&aj5H~N(P~J8>`kAN5NbuobFQ1UQs!-CxJVi-e5d$D5Ta@Vd6Zs8LF00 z-1zC1#(S8U`JaeOCtQ6_vS$4YuGQW(zJ`1YQFvHJkP8#+(T zMTur)p{HY=W@HlXvU{=PP83G-V%u)obT@RpX7s$9-sJWKUE0(v0JA&=_CI681j)Bi=!8Nwl~e*Zu9 zIQU}AkhZ{|pOa~yJlvVx6iRNTKJ4utz;ZtiHo#LWXjp$!{_%B3Xdsa6fvEk)ON3ub zh6;E97N|n2S_!loIqaCo`*nZWo7yiau=7wOi&RnuGYew_9Tih*SrS+yn!yJjNEP4M zQ{YF!mT&1RZ&uMK&GPw0`Xa1uNw1Y$3)4sY^5kNF9x#?=lm4#Ddo!1MvTW1$93V8R zJV7~p)C$K_MfJ8I-IQ)PXY2!(F|&8w-C-=lSZQcsrbf&Xq$#4bwe~kRAm!{h$NLyI zQC62B@-XuUAkg#>O9(V9@(iaQZE-Cki~_AkmN0XqX`+k=mtag2zb_R-&~sDzX~JxQ zmgdK|E5MR&!UJ=>^GMTE$f-mbOQ3M3I0e{>hDnwq8U}0mRkd()bY9q^tbZ_G+6N+; z`v}k*f-TJ%Wr;I{*cx4)&w+|~h<8u$t-{QY!eo%9&wwFV;OF3|n55bEX%(&%Qz%0% z0U`?i!uXenGeHryKOz9E{@)^^heR7?y7m7Sz?=s?2S?jQV7^|+a)va05lsDu)>j5! zd_Pr6GQ4h5__--OCqRzzrBASgI0G15Jlwn|&GIYtHvpeM@gdlVGmL^)io~NzdqyA0 z-$15_Gs5@K3b3*N7UTSX6=SKv_G;B!Nu8dqFQiPt`l_wXd!|Xyzm4zeDRv^09 zT7{A-n!)Y5h{zSUpdcKG6rnazrN>L{PAfhkwb#YUlIQ2gR`m2Y(dI94Ai0M$;JydV zNp3qrj+;*PLxTngVERl-y}`}b@K$uP-Uk@NhIXjt&Bd|GVDgxr9G#82!`LY&Z; zLG@cu^rK^o|R++ zilZc93R(u;iaQM;t*s5+N?hi5;Sfq?->Rn?O{(Ze;MnGM1;fe}b_L^LwVMr-=XGUW z2HpnTa)7WY>Wh^OZqB4clTk7H&B7f%6jI?aKj+)f9xgpeo+Y(*gh}VzDyu32AJXeu z;vCYO*8QNTB>+A=v?WZ!O!}kZmWBoYZxtTELFW3lVS`etZy&dBieHugr9&m}7@*_B z<=;9!#y1*RNd6xr+zPa^e1{)i6jPM-zyDo4#fZ^C@bCe| zacD#0t+h9q$He5#8qV;Fb&nBT>lYuOuhntybeYz(8B^BW(os!zcI{xDy+1a8p&3lA zy|ZuE%X3H`sU=wQ;hcU|{={XOv5|--P=CjA z36WSEi#wP96>6o=b1*jNyMEqsAWEhr$qZGzc1o>=aEaGZk>l%PC`=8E+(m5yL` zBn!ogWG>n8HKGwu#5rIh!MX@9Sa2|WR@q;iu00<;so@+YdNod{$X~J$T=N~vtgYP~Y!$unmB8(jm$y%yv zw@iIUQJvOL6SO|0NI8IOg}DeJkgi=k9WsnP3yL1yW(Z|MM~JqQVL1#Qdk80tUE6Od zR*6%mN{JdWpj@Bl1jz=A#-ANswS#_{P2fLVl`KZ{s~SP;A&lXrq6wX9OeDdHwVb0t z!~kNDwP}{al7;I7)W7OgA>*?AQPc!fNmyv#qFJas2BXsLI!nCF)YDsE@~BZdoPiMn zYzz+>q>W<3MKi>mguX|#ELcTcE>C$g-iqD)%XFMw`dp}LSJcwu2TneTzWVG$O(eUq z2m~N_G=|}W$>KDIjjA-0aYZTu&k#2zKK`%IdsiJxrUS5IAG_AsqU?v(d6l`sFe-#1U0S zeAh7cDE-!wE$9DD76;Adqet(Hx`w}WEfMtxo4sDRF#A2@ga=EYIOD6kc;=_pl<$(FNn|05-xhf0~HL_9N+2gH@rvHlapMbwF)~# zqlup{64u-%s%YHIsw|Wu@PG^~PK4Y``97|%fJi8*87iWzHwRVH7(&35Ovxz5p8`h;)ncCr z;h8cGUQ}tHv|MHQEW*^ILQ*LnPLB}Qnhr);Oe-=k;-yl`kH-93L>wLc2&=3hUX?k@Vn2Q6OTTSi#Gn3)MtwzOTtzIx zlPWJ_ZBnw#c<0A)V#1V+{R0V=2zv6=xS|+&Ad0Hecn+ExSsj%mx*2@{&Do3@d_`J6 zBDR`pnn_-SxbmFR z*HxX8N-k1irczOn9R4S25%fP%i4a1_2^Z@q(%@D9Wr1!Tz=C*Kj6W7kcSa;W6Wy;y z?v!~VCwXcp4QDNOM$kZ*ZK?>_h+%fHgsMcK!q&wW=SYsihHU=_i8djj3}74w(>vPU9pe8UV_Gx!@@&Gj{;x!~%)` zL74>PpOm@MX3TB%hcgU06vd)r$rRAJ`7QE)N0`m?u!_bQn{YEo5N2gyP&U2UP1b3A zER&bf`0gursWy2Cp-e3CVOc;rUrN$jJ4Y!UfX^KeS~$R$o*f89f8ksW-YZ&H(h;=B zuqRVe?@KR;G{Tt0k=)oYD7tP7#zaEn9I>hS*rWC<{eC@#_<7?Dtbs2x=B5rmDGhRk zdXbxqraK1&OoU4)Afrq}d&2UTeu_6S<%(oYw3tM>7)?2t$L5t(p-MX6q3Uer|7g}g z4a>%u%CEKRXW3)Q4~k^R6aW50a*ZM?JMFz5A_y0X^M#KFic?tdy#5#bF2qiJEgHUP zVFKrrU9$bfxPUaxmSdum$8$*TuS(FyV5N)bpuxKcc(V~rNv=4@iMuoeanM>*B`26D zA(EwP!E@zwJMf>WQl_-CUAkM^MxX^&*3WQ)1E-?u+Nk7N?kK@ZW19wXjzqhsBsRIOPNECUG z`7yvigPi()1H>e5jFx7$LlsBqkJ1nrfKoD5fKquye$Y5ao_P8#NoWf+1?eqq;|EMK zJV~z9e9jH#&>+wv03~ksdAe9)UeW#oPz*_~$HjA^WXHDgwyvPTnJ4I;a~mHtFVbB{ zRZS-<+A6^T{~(pKeFcb~R4MRe9n~%n6Qoo}QSH=tN70|DagL%HsR@ol@KDlRSrA0K zt_2a1@hCBaJd#OWQtkMVM#>PB-&k7k%&|s*rnLN zML`hQ)_^pJEnLNg4lTS*7twl#LqQrDd|)-UbmJWyVAc<_6Zmga#JfP$agLq{IgS{B{jg^s z>EawO${nOKVM9}+CS|zZymN^#DW#|i-$I?K4eN0W^R6&dVR>X0*GgKz%byp*vsIQS`yq;&F=av zjH41YKY_$H_SyG%9PEo9W#BzwX`@{p0-`EzjCHp;Ty z1`-m4FfO7!zc7(j5`=u*5%Tg?WEUV_k*R(`YqhfV?A*pM9-^~Zy;K+%45v^G3RT}d zgqNq3oB_0WNExPxRZyznuTV21LU^~Gn_>JwD`EU;%)N{Ih%th1yUIh@H;_YwNWOV& z0lx!*Z$haMHsjuj@hi*t_7Hb0!UURRYmNZhX_>rkBM8J+%!e^qAp7=ZHCy~3`SAN4 z2+t+<27Cj#z#Q=#5anZI^#<_Jd+IOK_RTN>Kp=s?1M!8Y{J4QM)fEPOHTI1Jgtv}rAfJ?&!h&~K? z#F2f0if0LhVFw*Y7<7l6I{}|-kbRjaf(1n0`k|cBWB&}{dcXS+s1+j{{b>zuo0&gyZXj_T0j;rzSYiT@qX_ekgmBcD zpx~})&$)-#$G(SXKE((CY+LntwrXB)3`96x>m0N?TW(jjE7JI1TKY20F*8%~1TA=R$KYY!4PN!egvSI#pVha7c4q3trD(5Y)V{;IAS`0}M|{tJLhilfwn9}AwZ zW#Pg!6jh!z%baWa`)c_2(Hp#0Q!&4V4aQQBv-Y0zn?PWBmv7le0GQH0Fr|OP=*gEp zD^KG8`JGPIkrfhHA+^g^``T_002*7~h48B}<83HH=Rp1B&+uKgZ>v@WuPUS2cmHRw zFY=|_=W_#AAq%#_3sWkJnE#@)7QVTPAG7+)+oLKQOWq|k`$*|RZj{xBsn6=FW?=&Y zV~6dV711u+I%`~nyo*%+9d2oQ-Wc~EDe%4vQ%c|n{{hPGx7>@BQ3vCies|8;+#Sk< zcryfu)GKX=rvsqP3l3PdSP-FSA0?b+amup_naOiaJ7#3MWq3%Z_5p~&m9{hT-natR zkD1YeWF7f%Psg9nKwfA)}XpO+UgFl`VK{LJD-_%aX%VuUqgN zxPI%A-Zc49vd!_sgyH&lLElLwZcNo69bEnZ$BS*1?2S)Uqung7`&josWLwHy@_CoeY$0-8K2$`oti8?j)ALvV*hMP0h-&caG&2oqAf13A=`B-N8tlz&1}0zz`1raGf2 z6fn_LTDW&Y3v>HBgB*&&IhfN>?$@(^Tm1#kYC|l<-JZpv#!U@pWz@}0Xs^U=1?r+G ze?YV{@i-45=t@gDN0kP40VYZpVixF(GSYYx_3lA+9-o($mY>sw5AS{OMfX8N6(kgc|=y4mcd;XWh$kV-F{W;q#BJ4EsBiRAo6b zNQ&ZPy11K$O2LDY-R1^{@G2R;vI7>9SclDDl@>>+;I|{PEKE?>^puaSt z5dD({Msgy+nM2Y4IOB{0nU7iLlHY)p0zV;zH)X-v;ymCWi9gy>-QCJ`il9(N2K zcX8P;u(Ng>(m_*fGqRw(m^h0TW@*%EkwEkPivYzT1UEYNxD#8p7l@dGELmx=#aR48=DY|| zDN(7kxRAVC(8Io?a)MN7bH1jKyo8Z%f~eoKl8d?UvF<4C6qN6TbOuWz#C90he zk8VCrz9~ga{>#d@wW!SW>dG0qhG+l|$)Gz&PJJ(d*)%$?X3>ZQiDA5(RYC(@0*M@o zrs=M5-B?eZvEuk4&1zH{y-VnmG$ufB{n{$i?z%FkW7&v2NPy2G+l-vFv@S@p)^hP_VVc>izNBLFtpl`=o%R5lzGMj zFX>pW2kXc(@%}@oh&Rh!<*}1=N*Cz6dA!|y^5YYU#4eR_s=MKTslyKVqfP|rf2*^F z1E}MI>h^m-L;nw$>A{Ojyld&-K=v5ekaufQa_#>n#X@&`>zkPU_W3`Gt`!Q18}xz# z0UbmCSBWAQGgntDdyBu;Sv9L~I{)NC>)9?fY}+-5ZgG;`r#>aC+mfTO1#CV5)bJh1 z?w&INE?;4rbUk7`#XRlu0{!;+M{keI(5M1q7W~&d_+(>#7{-7(AW>>s8jicUl~UU` zBQ~a#um~ny}9_b)&fHo7tT0vZZKNY_x@+_UZ4ropch5 z>1+@NIFKoEu#Sj=*Obk=I6n-!iQ69Jzy3n6-=8|a9NdX=ep1$`#W*fHhD%Bz#|p$8 zN>_-B;CFlrEL>T|#OkE2cM3i-n$3Edg6E}-$MDPxrg(A80%u(4jE>7dgF6zLGIx)T zdZ8dEyjZ{&ItIteRN~Fz0*Z`3-#~$d!Ud1w_C%0@0L#8yh^D|xKO_fiRA8gb`YIZ+ zA$_>{qaZJMl76nJDbGbr`o1M@zd^8)96q1*s?w^!{QNSGwgi0g zW;sXP`;8z%LyLsuRtz)`MvzIpLs@SEdK-wf+)Qt5z5>enqdZZ#O0^ZZLd|&l7pAeg zku0A;T&2yG5#{?;xIRYHYkd$R-Gdp)$kM<7xUlo|=iC5x%oOGdbyb`4U!uW@&z1cY zqbrfL4J;qmV$$bofjb;I(A(H(O9Ar zF3wCLI5~H32c5lRkj zienD_wz&#}99qQ{8TbB;h`lZF)zyOMTU9W+QiYj!J@m1NIHwLS4{M|WdQauFM1439 zopoK=a0a4shp|#8?|sAAWXyeuKF6h5ed=)c#r!0Lm)4$Hc%q%t8+*-#MV5%K>us~C z>{b3t<)|-`+vJAalbl_#!T2Tw$EEoe3i#yLZBjxr+7kjG*45%`y7zK45TfO!hvJ5a`#JeHPS}IO3EQA6s^&6&D=!$EtPkBz?2?uP`@1@srWgGaP%PUW`4^@Bw zrKVZ{??2@;|0+oK&CFQkzb^Rb*?#LmCODv3ubzQ}1eJZ$C}!aV7A*YEB%u~WM?7-o zX24>o>6a%~Ee@>CdkW%H;_@wDoN0y~seHQz7Dg5b^}W1{MN`6Gp_?WnR8YK7(HI#Q zFiDr|Mfw%sF9^b6(L*H0o>s;&wdhc*=>EO zZcgyi>ewK+x}H_aWkdg{wEXO z$lo>Gq-_(M(U56ILYgKKaJ_&~@#-+v<1Yo<=U;#D+Qu?n)=72V+r@ca(oT^92S8!F zib#SIbAQe}8-L!v<7>BmqidiLrJ-s2)raFqHvr09yKCT}c7!?N#Nr;i`uo1wB|nVLT9x1v6g9k?MTFLMK3RV@9MI#g0E4 zc9tFq)YjmF>k$?SC4v*DK_ZE@%hSD`OvVxuP{|y|l1WA2jdZ4=RB9#N zNkmLkMj7yGRsRo~+1A~!;FnY74jKWArS*FL!Q7{TI`$(aa4NYh1y!x4*4?OZckf+s z&-(s6?!bI^BIXBb*C3@k4?i!U+&sJCYir7NIr-WK&r2dSJfT^0O+4I~anr=qowRSq zx#&Ie?;JtO#>REjyW9@@bj`BtbB0e(ch?sN5zlY#N=F$DK9w*z9v2d<+&O75I~Y;t zbB9e42jjNljoeHI;00%aR?MQIKf#c`v@&;c6;N2I37r>5hs}>=Q(D&1S!UZy%_3Ni z^}_r#{%~RbY;Nl{nCi{tx0i(rf1A0bK#>Zsn!si9;Emg~x=j&i(ce)ubUf%!x#Le! z8Lg_@a{vcnoUZ?M_&xlG+goZ6#d)cxT#b0e_nZGwZZb)Tq-_LHQLvK+2nhY(r=+W; znVlKqUq|M@Dw%1^IjnPF^*m`HbsQ|dth+^Yh+V8+E|ZosyN!>QK?sCQSSBsDZr`BuKzhZv&JX>$yg3K84h4r+ z>~;wPSohV70fvCadmHe#od#bcc25U96$Zq@{-;+r&bN~d* znV9mYI`>SdDXBP+VK(pt*U4z-P07UMk`#r;ty5Q=bBYY@kdhpQ1scwC zTs2g=FCq(6b#fMk>vkEIIM^BH{BobSo{jcJRuq=H0q@=3DCo_nf(t|_Oa%w#%rY_! zP0Dk%g^Q9|U!6!|$`~_T_~kk8$3j))CQ1OSRn5S z(1oG`jX@JRP6K&9DCQoS>mZ@ai3m7Dz(Y`F|CKDUznCAwJicS@xUo)H8?#3lb@O>c zxa4R`NZX+&j&Y>&e16p`P-_yF)Vkc;4M_Vc8T6{ZW3OVBJx!2jqTmq@!$#3LiCnTB zNg+&3d@Puviz{M2$s1@C`=_l@rjon_2wz_ws8c;Lq&wBrN#<%|Ob*5bgNStF$@2Hv z{36Fexu#kX(+*+fHfWLzO#8lYKmB_3SyHJnJE{#~v9G05%Xp)~a3#znX!m$S9>l*H zGOcMg6PlfbOzq;!OGVQ9yn^G_mG52W=A7G%j_W$Ek?_~yq`Y-q2S446k<-bw!Z}O6gGj~#f7)K_pGi@g;drcA z2q5Cal$vdw3~)uV=(Np)X*LZ*g+zI}Teg9 zCXRyh3+}$F?4=}iwbLvvE>jkZU*1}kD}`ezNXAO}NDgN40SjJSII{VaAP~AoKb0EN> zY%^k+6T@{cFtDY0tBixra4^p<`T$?T=K-XhP@@_-p??2NiFhZ9BJ2*O}K?g z+{2ir*^SMFrqh)KY27Uwl!*+r1+>`hOo@jlZ4y+=?W51szkG&8jq&;jB*o{KLz&C2 zup|iP4;Xpc-YrqrwWzQ|1+V_R+d0Px@yZ6hOZ=Y_V^bOc)?)Sy7Z@7Dq%ujM1$sg!#&wlW85vKKqdhMF8k zyDUL1!&6(I_AAm$?s-tKtG0=4?~+t8De*zpm+u~$lw^;4HcR6Hh9{JU=A46)m)tux=8F>v z&lw7|Q7Ud9gH z6~t!MNHvMka!o%hr!hy^CSx?Dgh(fiF{L7dBghsrW(;XDXje#WnghvA(KuQlH@Ob- zra?%(wz?R;e#;%#FW%2LtmFOMO{ar)j^%o<)%y+k&s;3Wk7uP6@P`Kh$X@@Hc{g!$ zadoiMw6kTjay7I2>u;|*svc&52@UAUM{GP{yo4tVZ9y4{5;-5HM(X7!L_$|Wk@4qN zSnrPeH^M-c{F-NSX0Mv@%fJj@76fcTqHXI4YBP;UWHD7gn zEJfrOh8hSiqswagoK5oJ+BTR>Q|F+iwf~`uf8ubd8=!|0po`!iy8f;L`yX|Gm4J<& zsR!ieNDw>S3dBv86=oJx`I?U?AZx;o#FcR~En&r|9Ws(zJq0SD*eqN4pHD7!zKE4= zLKLxcWhE8KekYFiP8??ztG@QhL@iEDwBk%>z(p@tgYXyW0`WU+{Kgn* znmSQTxI!P&>e(XtkFZxIXOKSB<f?@Q3rzR9ZbxF!Y%-uEnK)rlROxle_>4i5{=xvP zDs_HS<8;a}QxZORnV)4@yE>6cQ)jc=)6>dTP->>{bwVa;l7G4M4&JIc2(R864*Nvt ziZ~~`H~h`F#q7n_P&xtm`!0p#_x~8B#m$C>+W_Wd!U5Ucf5sg4Zg$3I&VX5&KiS>1 z=BE7~7gqOnsbIXgZgY4d8yH(tfuOoxv^Z(=9S{Xp-C6D!GtX@A5W<#6ccwwxpf&PgtH8KS_^so=N$<38QbiyaS|k=i#skZPQ^)&lGYc#LHP zjFS|!!wMZO_xInA(?#6q-!|pf1;Do{Bsex=F2lRkbt_Qc5Hxn@PM7Du4)Zw}79b!+ zODNckfa@Zu#VdD}=1QLHH&eDr7W>t|brg4Vep8sNKQyJL8yKK%3S3L_yVHZrF&O1nx)BzMYNPV=AlhM}6E2=@eskjz@>x+a z%}qNZo%0U|525s&oAtHZHTQyJ#$gDGdW-1coL6}+SKUvM$jjGe<8#L+<4%hDWbwo}b;QJ^kTGY+pg(`b^K{w43sVQ;^7JXx{IB`F?yNG!cdyhjAsAtFuSfzhK;W z=fTReoQPVzn1isE=hmaEMgX$iV5e^s`x{4>Q<(|{Yu-Pjoqj3InqYj{p60o2*5iDI z>Vcpz=sv1;f;duQJ3kT3@cy!m-FefjX7L79gCJWI60Mibo!Iw%>U9;}z+J<88~;1c zaYfA6&mLGgaX4JuT?9yAMSa~TcfA|e=G$EX9z8nN#Ox1y&wbnW+PXXw3&R?0jKw}a z){7d5$E(pt~?%*}b&-|@V*njp|1!Kz= z-QYk#K}0}6n1CJ&P@wy-ZVE8+0$iW=Mpm{A){bTtm+Cf-sA_2ZonD;OcGE~y^k|R+ zoh*Gt^un|t99BO|k~oU!Yn?)*66H`cD2%=>q{wH-zW_ai@_B#Oqp7-PIVsKs@F

A|a1+oFLgqtX6Tr}41ux*#xhNX)NU>gfzOVxqF z{%mBj8UNn1e|=!v}D9H z9y>>bk~-ZQ`W{)xj1*qW3GvIdJl_-87@>NTGKJfuYw)r`jWjpR+gGzi@zwM|Kn%w^ zGEM>9Fw|o>L0fHRYyGTJD}WEReUx$C#`tmBP|qxxC-u@6ir#UA?R7H&-3wwrKKB;- z5r&-*fls(T`VsaS%bj;@5OQPSYTT?mrIU{3VzI*x;3NkxkjW{Q74 z;Ae?ND{PLN!cIGv)XuGO`%3hKCd^Q^LI*D8VJ$vu$n$a!ruVfk@wg@fzk|VoI7h)5 zQg8raD0w0Ae0$~bV@x!PVBYO>_x`(`qFdv~t;KJj9?0D8xAC9=f&Sc~O3VnRR=)%# z(k&lyL~8251DuYVl>Xou{>9(~SB>fu_?j7ccT38k=HyhG{O{TW*8J8VYa1q8wp-R8 zHyI3wKgsP;TLWMH*2gdR_#shSGtZCs4<4Nz9Dk8_SK|JP_5W78h+CWD331*G1ROZb z^pmMy(H6@l&9qE%h}9I^8Qv^}`ULyS^;W`)OwBOj?Gqb zCQPoLHP@&ID6Z?-K|CAF%cMB={V-q`Ce=Fcd0ptQf^i{B>`FA%E zEatolp|T&-THnH&`;3V=;x!5dB%?PbHN1Ns^QM~z|5$)B(-xCvL@*ucKQhcR%H8a1<{9*nP$v2&-aPAMK-hxEEu{$ zC%Rqt97C%q&=W}GgV5+n`gwSRgZWs4)htk55T{k>7wLyOGKqw$iz zSluRNb(Z|-@>;$K81SkZZb_JD@$zt!yWAuH?YT+Nuk-7!Ht9f8#bxqrIh4TXV6OR` zz{`8Tq%_CpXze)1H{WG)IaS}sY`wC!`+0etcgv&I=C~%7f9r01>-(#~`^Ds-!2=$* zXVsJY_Nl`U@7K+zcMD%G-`&01UcHz1Sb@(1H|9S0Bz{Z1Rj?59AW~0l%s~zn|M=^D zXScx!j8nbJkk}cW!~~IOyjeE~IBeBIW6*7LRf~3Nk7*oz4Gihi?53PGxRY zePzY>QYOzrIlbj)0r0@(k`4UJn%JZ=7Bpfd{u_6EJ7gc=&B!Fe3>qHiU{I=eiCr7C zbmCNC3-uIm%nj7-fPn^}7z0DJGjK3CHLoN-q_QBjI2P18LpPvO&D-GzQ0YD3I5Ia( zH;`^%JO(rZ(#8ij?a?)Mv~dS#0+n0@Y7|7#_#7w#uX~d7i@+n%=-R*BzAgS5s6B^; zfk6O8dp%GDu064!0Gq4%awZ)64%E#9^fs#HGk_xCnFF8^rMW=AW7D43AT4AB)P5Rh zl?;mI`#7N5AwEYo1Zya;Us*r1SRdFF4rOKFM=?nYI4BQxNO4JJPAYi15cMi}wD7He-D-2n8CF2aEQb;t%_>-M6Xh2D2Wn6;xG z*(}taB)Tc+T`hzu=Nq7*g3|Fq*N@&FMCi|ILCRvNt-}CsR$vYSqWp D`Bw}5 literal 0 HcmV?d00001 diff --git a/packages/extraction/tests/fixtures/empty.pdf b/packages/extraction/tests/fixtures/empty.pdf new file mode 100644 index 0000000000000000000000000000000000000000..fc99c8291dbc2292fb29c408fdfbd0776a0fe03a GIT binary patch literal 660 zcmZWn-A=+V6uv)Cac-8tg?8Q8hza2WWYMURjNKV83hQtLSJHI`d`T~S1AP%sD?bY~ zS+agT=leOWH@!`+_;m>0&+peaaP(2lS1=d=n`)Ie3#qZQ*=_Qo3&SDED!0c$t1R1X zB(kYk${N^MHI+dQ?4itSe8GqcQ+#c;h8L}4VZ>T3Nzihjq{nYIK;NekLOY?>yw;pSwrFe%?hAiPoyjg*L zVh?=Ga_n?5Tj3Bu7NV8dXycDsP_VQRv`uAUB_VItyJWfh4~{pmu=94`d*AP;{??i+(RnfJv)ZrU zU%xW}9_Y=xY+-@%rqM~Sr3{?uZg-RRS+=;yRGzrc^~!HIx4Q~>Q+8Cr__FTh76jwh z(xiY}5a5q7+^)3xW39{5%1n1~p_>72c3f$=FxC+1+%mq=+qC6sCjd7t$wIj=z%Q%p zzOrdcR;D;1Lt}@E8IM(=d!~hZo#nE>d=cRGmO9Wjzpj#0F6lm6@@BpA2}K>|8g(>> z!i~bk|CJ+y+~T!>a5!N=&E1C@qBOIJmPeViYDKklq8MLOdB@&?dC_P%>4h~)Zn6He z-Ou9WN0>h5ozuIKl$}eTo_>4v=s3*0dwuoC#hvGSU+#T)$=9do5CI}U1c(3;AOb{y z2oM1xKm>>Y5g-B+OyI`Du7~!H9$9;#;neBXy{RW}PfW01vPJ}m01+SpM1Tko0U|&I zhyW2F0z_c42t5Az{!P&M%A%m}B0-fiMXC0NQX+6PSE(2lB&O zfZ2wTnh+Sw7_EhvXc(#0{qdR~R_0kF(@dE%L9&rPP$-YbT3c|zkC^c*d0T^vnUvXz S!b)RHAM>FfvRZ9*J!1RGaLMQZ literal 0 HcmV?d00001 diff --git a/packages/extraction/tests/fixtures/pages.pdf b/packages/extraction/tests/fixtures/pages.pdf new file mode 100644 index 0000000000000000000000000000000000000000..1e7acc2462718b0b8da896326f6a15871d0d0334 GIT binary patch literal 950 zcmcgr!A|2a5WUY=%%u_>z;V(fL_!=2M7!EuZI$MRIGAQqf=XODE-L&ahy8=)NAOq; zMCpYSmJ-?HnK$pvcqa3!{6wBcF!}%X`VS-qsP7)&{2avG=xVzpgGZa^O;sPm#RX7X z@pU+GE1x%nVy;%yfwy=B(NXQwALBRAN zAlI!0vDn^OPk?Di5Hr;g-=O$G>nF0!QVmd;PA~71H;95dy){dhmxcRyzmdp!Jr5^d z{GqB*Gre=dg44i%i3~B#LcC?-hFH9haFuASJ9e1#2>aUcx##ek+LgV>X~G;kYe+r9 ztVEGZbhu@WsY4F~EGs<98`D|bxay##oju0#0k2E$$Ubn7&&BTXzf0rUNF$$n4!y?o z3mO-+)NS|K{)x DfClz) literal 0 HcmV?d00001 diff --git a/packages/extraction/tests/isolation.mjs b/packages/extraction/tests/isolation.mjs new file mode 100644 index 00000000..d44ef800 --- /dev/null +++ b/packages/extraction/tests/isolation.mjs @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { mkdir, mkdtemp, writeFile, rm, readdir } from 'node:fs/promises'; +import { once } from 'node:events'; +import { RestrictedBackend } from '/opt/extraction/dist/backend.js'; + +await mkdir('/jobs/other'); +await writeFile('/jobs/other/secret', 'cross-job-canary'); +await writeFile('/socket/secret', 'transport-canary'); +const job = await mkdtemp('/jobs/job-'); +const child = spawn( + '/opt/extraction/guard', + [job, '/usr/local/bin/python3.13', '/tests/isolation.py', 'probe'], + { + env: { EXTRACTION_SECRET_CANARY: 'private-canary' }, + stdio: ['ignore', 'pipe', 'pipe'], + } +); +let output = '', + errors = ''; +child.stdout.on('data', bytes => { + output += bytes; +}); +child.stderr.on('data', bytes => { + errors += bytes; +}); +const [code] = await once(child, 'close'); +assert.equal(code, 0, errors); +assert.equal(output.trim(), 'ISOLATED'); +await rm(job, { recursive: true }); +const runtime = { + jobs: '/jobs', + guard: '/opt/extraction/guard', + python: '/usr/local/bin/python3.13', + worker: '/tests/isolation.py', +}; +const backend = new RestrictedBackend(runtime); +for (const abort of ['immediate', 'running']) { + const job = await backend.create(); + const signal = new AbortController(); + const pending = backend.run(job, 'hang', 'a'.repeat(64), signal.signal); + if (abort === 'running') + await new Promise(resolve => setTimeout(resolve, 100)); + signal.abort(new Error('CANCELLED')); + await assert.rejects(pending); + await backend.cleanup(job); +} +const job2 = await backend.create(); +await assert.rejects( + backend.run(job2, 'hang', 'a'.repeat(64), AbortSignal.timeout(100)) +); +await backend.cleanup(job2); +await rm('/jobs/other', { recursive: true }); +await rm('/socket/secret'); +assert.deepEqual(await readdir('/jobs'), []); +const missing = new RestrictedBackend({ ...runtime, guard: '/nonexistent' }); +const job3 = await missing.create(); +await assert.rejects( + missing.run(job3, 'pdf', 'a'.repeat(64), AbortSignal.timeout(1000)) +); +await missing.cleanup(job3); +console.log( + 'Container isolation: cross-job/credential paths, sockets, fork, signals, limits, startup/running cancellation, deadline and missing launcher passed' +); diff --git a/packages/extraction/tests/isolation.py b/packages/extraction/tests/isolation.py new file mode 100644 index 00000000..a50d0084 --- /dev/null +++ b/packages/extraction/tests/isolation.py @@ -0,0 +1,73 @@ +import ctypes +import errno +import fcntl +import os +import resource +import signal +import socket +import struct +import sys + +if sys.argv[1] == 'hang': + # Exercise supervisor cancellation/deadline termination, not a cooperative parser. + while True: + pass + +for path, mode in [('/jobs/other/secret', 'rb'), ('/jobs/other/secret', 'wb'), + ('/socket/secret', 'rb'), ('/proc/self/environ', 'rb'), ('/etc/passwd', 'rb')]: + try: + with open(path, mode) as file: + if mode == 'rb': + file.read() + except PermissionError: + pass + else: + raise RuntimeError('Cross-boundary file access') +for operation in [lambda: socket.socket(), lambda: socket.socket(socket.AF_UNIX), + lambda: os.fork(), lambda: os.kill(os.getppid(), 0), + lambda: os.chmod('/jobs/other/secret', 0o777), + lambda: fcntl.fcntl(1, fcntl.F_SETOWN, os.getppid()), + lambda: fcntl.fcntl(1, 15, struct.pack("ii", 1, os.getppid())), + lambda: fcntl.fcntl(1, fcntl.F_SETSIG, 0), + lambda: fcntl.ioctl(1, 0x8901, struct.pack("i", os.getppid()))]: + try: + operation() + except PermissionError: + pass + else: + raise RuntimeError('Forbidden syscall') +assert 'EXTRACTION_SECRET_CANARY' not in os.environ +assert resource.getrlimit(resource.RLIMIT_AS) == (512 * 1024 * 1024,) * 2 +assert resource.getrlimit(resource.RLIMIT_CPU) == (8, 8) +assert resource.getrlimit(resource.RLIMIT_FSIZE) == (4 * 1024 * 1024,) * 2 +assert resource.getrlimit(resource.RLIMIT_NOFILE) == (64, 64) + + +# Harmless signal-zero probes distinguish filtering from ordinary permissions. +class Sigval(ctypes.Union): + _fields_ = [("integer", ctypes.c_int), ("pointer", ctypes.c_void_p)] +libc = ctypes.CDLL(None, use_errno=True) +libc.sigqueue.argtypes = [ctypes.c_int, ctypes.c_int, Sigval] +assert libc.sigqueue(os.getppid(), 0, Sigval(integer=0)) == -1 +assert ctypes.get_errno() == errno.EPERM +number = {"x86_64": 297, "aarch64": 240}[os.uname().machine] +info = (ctypes.c_int * 32)() +info[2] = -1 # SI_QUEUE, with signal zero: never delivers a signal. +assert libc.syscall(number, os.getppid(), os.getppid(), 0, ctypes.byref(info)) == -1 +assert ctypes.get_errno() == errno.EPERM + + + +try: + fcntl.ioctl(1, 0, 0) +except PermissionError: + pass +else: + raise RuntimeError('Forbidden ioctl') + + + +libc.mq_open.argtypes = [ctypes.c_char_p, ctypes.c_int] +assert libc.mq_open(b"/extraction-negative-probe", os.O_RDONLY) == -1 +assert ctypes.get_errno() == errno.EPERM +print("ISOLATED", flush=True) diff --git a/packages/extraction/tsconfig.json b/packages/extraction/tsconfig.json new file mode 100644 index 00000000..aff02060 --- /dev/null +++ b/packages/extraction/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2023", + "lib": ["ES2024"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noUncheckedIndexedAccess": true, + "esModuleInterop": true, + "skipLibCheck": true, + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/extraction/worker/extract.py b/packages/extraction/worker/extract.py new file mode 100644 index 00000000..20f0360f --- /dev/null +++ b/packages/extraction/worker/extract.py @@ -0,0 +1,503 @@ +"""Request-local parser. Only the supervisor can choose its mode and files.""" +import hashlib +import io +import json +import logging +import os +from pathlib import PurePosixPath +import socket +import struct +import sys +import xml.etree.ElementTree as ET +import zipfile +import zlib + +INPUT_BYTES = 10 * 1024 * 1024 +TEXT_BYTES = 1024 * 1024 +RESULT_BYTES = 3 * 1024 * 1024 +EXPANDED_BYTES = 32 * 1024 * 1024 +ENTRY_BYTES = 8 * 1024 * 1024 +SEGMENTS = 128 + + +class Rejected(Exception): + def __init__(self, code): + self.code = code + + +class Text: + def __init__(self): + self.size = 0 + self.parts = [] + + def add(self, value): + size = len(value.encode("utf-8", errors="strict")) + if self.size + size > TEXT_BYTES: + raise Rejected("OUTPUT_LIMIT") + self.size += size + self.parts.append(value) + + def value(self): + return "".join(self.parts) + + +def pdf_segments(data): + import pypdf + from pypdf import filters, _page + from pypdf.generic import DecodedStreamObject, DictionaryObject, NameObject, NullObject, StreamObject + from pypdf._cmap import charset_encoding + from pypdf._codecs.core_font_metrics import CORE_FONT_METRICS + from pypdf._text_extraction._text_extractor import TextExtraction + + used = 0 + failure = None + credited = {} + shown_bytes = 0 + + def reject(code): + nonlocal failure + failure = failure or code + raise Rejected(failure) + + def check_failure(): + if failure is not None: + raise Rejected(failure) + + class Warnings(logging.Handler): + def emit(self, record): + nonlocal failure + failure = failure or "INVALID_DOCUMENT" + + logger = logging.getLogger("pypdf") + logger.addHandler(Warnings()) + logger.setLevel(logging.WARNING) + logger.propagate = False + original = filters.decode_stream_data + original_data = DecodedStreamObject.get_data + original_show = TextExtraction._handle_tj_operation + original_resources = _page._get_page_resources + + def inflate(value): + nonlocal used + # Strict bounded decoding, without pypdf's recovery/fallback paths. + cap = min(ENTRY_BYTES, EXPANDED_BYTES - used) + decoder = zlib.decompressobj() + try: + result = decoder.decompress(value, cap + 1) + except zlib.error: + reject("INVALID_DOCUMENT") + if len(result) > cap or decoder.unconsumed_tail: + reject("DECOMPRESSION_LIMIT") + if not decoder.eof or decoder.unused_data: + reject("INVALID_DOCUMENT") + used += len(result) + credited[id(result)] = result + return result + + def decoded_data(stream): + nonlocal used + value = original_data(stream) + if not getattr(stream, "_extraction_counted", False): + if credited.get(id(value)) is not value: + used += len(value) + if len(value) > ENTRY_BYTES or used > EXPANDED_BYTES: + reject("DECOMPRESSION_LIMIT") + stream._extraction_counted = True + return value + + def decode(stream): + names = stream.get("/Filter", []) + if not isinstance(names, list): + names = [names] + if len(names) > 1 or any(str(name) not in { + "/FlateDecode", "/Fl", "/ASCIIHexDecode", "/AHx", "/ASCII85Decode", "/A85" + } for name in names): + reject("UNSUPPORTED_ENCODING") + try: + return original(stream) + except Rejected: + raise + except MemoryError: + reject("RESOURCE_LIMIT") + except Exception: + reject("INVALID_DOCUMENT") + + def resources_for_text(obj): + resources = original_resources(obj) + if resources: + fonts = resources.get("/Font") + if fonts is not None and not isinstance(fonts.get_object(), NullObject): + for ref in fonts.get_object().values(): + resource = ref.get_object() + for key in ("/Encoding", "/ToUnicode"): + if key in resource and isinstance(resource[key], NullObject): + del resource[key] + encoding = resource.get("/Encoding") + if encoding is not None: + encoding = encoding.get_object() + if not isinstance(encoding, (NameObject, DictionaryObject)): + reject("UNSUPPORTED_ENCODING") + if isinstance(encoding, DictionaryObject): + base = encoding.get("/BaseEncoding") + if base is not None and isinstance(base.get_object(), NullObject): + del encoding["/BaseEncoding"] + unicode_map = resource.get("/ToUnicode") + if unicode_map is not None and not isinstance(unicode_map.get_object(), StreamObject): + reject("UNSUPPORTED_ENCODING") + for state in resources.get("/ExtGState", DictionaryObject()).get_object().values(): + if "/Font" in state.get_object(): + reject("UNSUPPORTED_ENCODING") + return resources + # Do not let the parser skip text operators solely because resources + # are absent. A used missing font must reach the rejection below. + return DictionaryObject({NameObject("/Font"): DictionaryObject()}) + + def show_text(extractor, operands): + nonlocal shown_bytes + if operands and operands[0]: + font, resource = extractor.font, extractor.font_resource + if resource is None or not font.interpretable: + reject("UNSUPPORTED_ENCODING") + if font.sub_type not in {"Type1", "MMType1", "TrueType", "Type3", "Type0"}: + reject("UNSUPPORTED_ENCODING") + encoding = resource.get("/Encoding") + if encoding is not None: + encoding = encoding.get_object() + if isinstance(encoding, DictionaryObject): + encoding = encoding.get("/BaseEncoding") + if encoding is not None: + encoding = encoding.get_object() + standard_encoding = isinstance(encoding, NameObject) and encoding in charset_encoding + require_map = ("/ToUnicode" in resource or font.sub_type in {"Type3", "Type0"} + or isinstance(font.encoding, str) + or (not standard_encoding and font.name not in CORE_FONT_METRICS)) + value = operands[0] + if isinstance(value, bytes): + try: + if isinstance(font.encoding, str): + # Never take pypdf's charmap recovery on codec failure. + characters = value.decode(font.encoding, errors="surrogatepass") + else: + characters = (font.encoding[code] for code in value) + for character in characters: + if require_map and character not in font.character_map: + reject("UNSUPPORTED_ENCODING") + mapped = font.character_map.get(character, character) + # Unknown Adobe glyph names are preserved as NameObject. + if (not isinstance(mapped, str) or isinstance(mapped, NameObject) + or not mapped or "\ufffd" in mapped + or any(ord(char) < 32 and char not in "\t\n\r" for char in mapped)): + reject("UNSUPPORTED_ENCODING") + shown_bytes += len(mapped.encode("utf-8", errors="strict")) + if shown_bytes > TEXT_BYTES: + reject("OUTPUT_LIMIT") + except (LookupError, UnicodeError, TypeError): + reject("UNSUPPORTED_ENCODING") + elif not isinstance(value, str): + reject("UNSUPPORTED_ENCODING") + return original_show(extractor, operands) + + _page._get_page_resources = resources_for_text + TextExtraction._handle_tj_operation = show_text + DecodedStreamObject.get_data = decoded_data + filters.decompress = inflate + filters.decode_stream_data = decode + reader = pypdf.PdfReader(io.BytesIO(data), strict=True) + check_failure() + if reader.is_encrypted: + raise Rejected("ENCRYPTED_DOCUMENT") + if len(reader.pages) > SEGMENTS: + raise Rejected("STRUCTURE_LIMIT") + result = [] + text = Text() + for index, page in enumerate(reader.pages): + value = page.extract_text() or "" + check_failure() + text.add(value) + result.append({"kind": "page", "index": index + 1, "text": value}) + check_failure() + return result, text.size + + +def validate_xml(value): + xml = value.decode("utf-8", errors="strict") + if "\x00" in xml or " 512: + raise Rejected("STRUCTURE_LIMIT") + for entry in archive.infolist(): + name = entry.filename + if entry.orig_filename != name: + raise Rejected("INVALID_DOCUMENT") + path = PurePosixPath(name) + if (name in names or path.is_absolute() or ".." in path.parts or "\\" in name + or "\x00" in name or (entry.external_attr >> 16) & 0o170000 == 0o120000 + or entry.flag_bits & 1 or entry.compress_type not in (0, 8)): + raise Rejected("INVALID_DOCUMENT") + names.add(name) + if entry.file_size > ENTRY_BYTES: + raise Rejected("DECOMPRESSION_LIMIT") + # Let ZipFile validate local names, flags and entry overlap, but do + # not use ZipExtFile.read(): it trims data to untrusted file_size. + with archive.open(entry): + offset = entry.header_offset + if offset < 0 or offset + 30 > len(data): + raise Rejected("INVALID_DOCUMENT") + method = struct.unpack_from(" len(data): + raise Rejected("INVALID_DOCUMENT") + decoder = zlib.decompressobj(-zlib.MAX_WBITS) if entry.compress_type == 8 else None + chunks = [] + size = 0 + crc = 0 + for position in range(start, end, 64 * 1024): + chunk = data[position:min(position + 64 * 1024, end)] + if decoder: + cap = min(ENTRY_BYTES - size, EXPANDED_BYTES - total) + chunk = decoder.decompress(chunk, cap + 1) + size += len(chunk) + total += len(chunk) + if size > ENTRY_BYTES or total > EXPANDED_BYTES: + raise Rejected("DECOMPRESSION_LIMIT") + crc = zlib.crc32(chunk, crc) + chunks.append(chunk) + if decoder and (not decoder.eof or decoder.unused_data or decoder.unconsumed_tail): + raise Rejected("INVALID_DOCUMENT") + if size != entry.file_size or crc != entry.CRC: + raise Rejected("INVALID_DOCUMENT") + value = b"".join(chunks) + parts[name] = value + xml_parts(parts) + output = io.BytesIO() + with zipfile.ZipFile(output, "w") as clean: + for name, value in parts.items(): + # The document parser sees only measured, validated parts. + clean.writestr(name, value, compress_type=zipfile.ZIP_STORED) + return output.getvalue() + + +def validate_docx_body(body): + from lxml.etree import _Element + + literal = _Element.text.__get__ + word = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" + # These are the exact content paths read by pinned python-docx. Unknown + # wrappers fail closed rather than hiding text from its direct-child XPath. + grammar = { + "body": {"p", "tbl", "sectPr"}, + "tbl": {"tblPr", "tblGrid", "tr"}, + "tr": {"trPr", "tc"}, + "tc": {"tcPr", "p", "tbl"}, + "p": {"pPr", "r", "hyperlink"}, + "hyperlink": {"r"}, + "r": {"rPr", "t", "tab", "br", "cr", "noBreakHyphen", "ptab", "lastRenderedPageBreak"}, + } + properties = {"sectPr", "tblPr", "tblGrid", "trPr", "tcPr", "pPr", "rPr"} + markers = {"bookmarkStart", "bookmarkEnd", "commentRangeStart", "commentRangeEnd", + "permStart", "permEnd", "proofErr"} + stack = [(body, "body")] + while stack: + parent, kind = stack.pop() + if literal(parent) and literal(parent).strip(): + raise Rejected("UNSUPPORTED_CONTENT") + for child in parent: + tag = child.tag.removeprefix(word) if isinstance(child.tag, str) else "" + marker = kind in {"body", "p", "hyperlink", "tc"} and tag in markers + if child.tag != word + tag or (tag not in grammar[kind] and not marker): + raise Rejected("UNSUPPORTED_CONTENT") + if child.tail and child.tail.strip(): + raise Rejected("UNSUPPORTED_CONTENT") + if tag in properties: + # Formatting is ignored, but cannot conceal textual content. + if any(node.tag in {word + "t", word + "instrText", word + "delText"} + or (literal(node) and literal(node).strip()) + or (node.tail and node.tail.strip()) for node in child.iter()): + raise Rejected("UNSUPPORTED_CONTENT") + elif tag in grammar: + stack.append((child, tag)) + elif len(child) or (tag != "t" and literal(child) and literal(child).strip()): + raise Rejected("UNSUPPORTED_CONTENT") + + +def docx_segments(data): + from docx import Document + from docx.table import Table, _Cell + + document = Document(io.BytesIO(safe_docx(data))) + validate_docx_body(document.element.body) + text = Text() + def blocks(container, depth=0, cell=False): + if depth > 32: + raise Rejected("STRUCTURE_LIMIT") + first = True + for block in container.iter_inner_content(): + if cell and not first: + text.add("\n") + first = False + if isinstance(block, Table): + width = len(block._tbl.tblGrid.gridCol_lst) + if width < 1 or width > 512: + raise Rejected("STRUCTURE_LIMIT") + previous = set() + for row_index, row in enumerate(block.rows): + if row_index: + text.add("\n") + before, after = row.grid_cols_before, row.grid_cols_after + if before < 0 or after < 0 or before + after >= width: + raise Rejected("INVALID_DOCUMENT") + column = before + text.add("\t" * before) + current = set() + for index, tc in enumerate(row._tr.tc_lst): + span, merge = tc.grid_span, tc.vMerge + if span < 1 or column + span > width - after: + raise Rejected("INVALID_DOCUMENT") + if tc.tcPr is not None and tc.tcPr.find( + "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}hMerge") is not None: + raise Rejected("UNSUPPORTED_CONTENT") + if index: + text.add("\t") + position = (column, span) + if merge == "continue": + if position not in previous: + raise Rejected("INVALID_DOCUMENT") + # Continuations occupy columns, not another copy of + # the originating cell. Hidden content is rejected. + if tc.xpath(".//w:t | .//w:tbl | .//w:tab | .//w:br | .//w:cr | .//w:noBreakHyphen | .//w:ptab"): + raise Rejected("UNSUPPORTED_CONTENT") + else: + blocks(_Cell(tc, block), depth + 1, cell=True) + if merge is not None: + if merge not in {"restart", "continue"}: + raise Rejected("INVALID_DOCUMENT") + current.add(position) + text.add("\t" * (span - 1)) + column += span + if column + after != width: + raise Rejected("INVALID_DOCUMENT") + text.add("\t" * after) + previous = current + if not cell: + text.add("\n") + else: + text.add(block.text) + if not cell: + text.add("\n") + + blocks(document) + return [{"kind": "document", "index": 1, "text": text.value()}], text.size + + +def probe(): + import pypdf + import docx + import lxml.etree + + for path in ("/socket", "/jobs", "/proc/self/environ", "/etc/passwd"): + try: + if path in ("/socket", "/jobs"): + os.listdir(path) + else: + open(path, "rb").read(1) + except PermissionError: + continue + raise Rejected("ISOLATION_UNAVAILABLE") + try: + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + except PermissionError: + pass + else: + raise Rejected("ISOLATION_UNAVAILABLE") + try: + os.fork() + except PermissionError: + pass + else: + raise Rejected("ISOLATION_UNAVAILABLE") + if "EXTRACTION_SECRET_CANARY" in os.environ: + raise Rejected("ISOLATION_UNAVAILABLE") + print("READY", flush=True) + + +def main(): + if sys.argv[1] == "probe": + probe() + return + data = open("input", "rb").read(INPUT_BYTES + 1) + if not data or len(data) > INPUT_BYTES: + raise Rejected("INPUT_LIMIT") + mode = sys.argv[1] + if mode == "pdf": + if not data.startswith(b"%PDF-"): + raise Rejected("INVALID_DOCUMENT") + segments, size = pdf_segments(data) + elif mode == "docx": + if not data.startswith(b"PK\x03\x04"): + raise Rejected("INVALID_DOCUMENT") + segments, size = docx_segments(data) + else: + raise Rejected("UNSUPPORTED_FORMAT") + if not any(item["text"].strip() for item in segments): + raise Rejected("EMPTY_OUTPUT") + result = json.dumps({"version": 1, "operation": "document.extract-text", "format": mode, + "inputSha256": hashlib.sha256(data).hexdigest(), "textBytes": size, + "segments": segments}, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + if len(result) > RESULT_BYTES: + raise Rejected("OUTPUT_LIMIT") + with open("result", "xb") as target: + target.write(result) + + +if __name__ == "__main__": + try: + main() + except Rejected as error: + print(error.code, flush=True) + sys.exit(2) + except MemoryError: + print("RESOURCE_LIMIT", flush=True) + sys.exit(2) + except Exception: + print("INVALID_DOCUMENT", flush=True) + sys.exit(2) diff --git a/packages/extraction/worker/requirements.txt b/packages/extraction/worker/requirements.txt new file mode 100644 index 00000000..25ad0ede --- /dev/null +++ b/packages/extraction/worker/requirements.txt @@ -0,0 +1,4 @@ +pypdf==6.19.0 +python-docx==1.2.0 +lxml==6.0.2 +typing_extensions==4.15.0 diff --git a/service/Dockerfile.node b/service/Dockerfile.node index 8a6006e4..dc964fce 100644 --- a/service/Dockerfile.node +++ b/service/Dockerfile.node @@ -1,5 +1,5 @@ # Base stage for shared steps -FROM node:22-slim AS base +FROM node:22.23.4-slim AS base WORKDIR /app COPY service/package*.json ./ RUN npm ci @@ -13,7 +13,7 @@ COPY service/tsconfig.json ./ RUN npx tsc -p tsconfig.json # Production stage -FROM node:22-slim AS production +FROM node:22.23.4-slim AS production ENV NODE_ENV=production WORKDIR /app COPY service/package*.json ./ diff --git a/service/openapi.yml b/service/openapi.yml index 07d2d7e3..929fab71 100644 --- a/service/openapi.yml +++ b/service/openapi.yml @@ -72,7 +72,12 @@ components: protocolVersion: { type: integer, enum: [1] } operation: type: string - enum: [list_files, search_text, read_file, write_file, preview_edit, edit_file, execute_command] + enum: [list_files, search_text, read_file, write_file, preview_edit, edit_file, execute_command, import_file] + description: >- + `import_file` copies a stored object the caller owns into the + workspace: `path`, `source` ({ storage_session_id, id }), `bytes`, + lowercase hex `sha256`, optional `overwrite`. Send it only when the + worker advertises it. Callers never send `egressGrant`. workspaceId: { type: string } additionalProperties: true diff --git a/service/rollup.config.js b/service/rollup.config.js index 059c71d1..e2024c6f 100644 --- a/service/rollup.config.js +++ b/service/rollup.config.js @@ -43,6 +43,7 @@ export default { '../shared/telemetry-core.ts', '../packages/code/src/protocol.ts', '../packages/code/src/identity.ts', + '../packages/code/src/browser-enrollment-protocol.ts', ], sourceMap: true, declaration: false, diff --git a/service/src/bridge/browser-enrollment-router.test.ts b/service/src/bridge/browser-enrollment-router.test.ts new file mode 100644 index 00000000..f0418d24 --- /dev/null +++ b/service/src/bridge/browser-enrollment-router.test.ts @@ -0,0 +1,184 @@ +import { randomBytes } from 'crypto'; +import { createServer, type Server } from 'http'; + +import { afterAll, afterEach, describe, expect, test } from 'bun:test'; +import express, { json } from 'express'; +import Redis from 'ioredis'; +import RedisMock from 'ioredis-mock'; + +import { + browserSetupRequestHash, + signEnrollmentRedemption, +} from '../../../packages/code/src/browser-enrollment-protocol'; +import type { BrowserEnrollmentApproval, BrowserEnrollmentRedemption } from '../../../packages/code/src/browser-enrollment-protocol'; +import { createBridgeIdentity } from '../../../packages/code/src/identity'; +import { RedisBridgePairingStore } from './pairing'; +import type { BridgeRecoveryOptions } from './pairing'; +import { createBridgeRouter } from './router'; +import type { BridgeRouterOptions } from './router'; +import { RedisBridgeStore } from './store'; + +const redis = process.env.BROWSER_ENROLLMENT_REDIS_PORT + ? new Redis({ host: '127.0.0.1', port: Number(process.env.BROWSER_ENROLLMENT_REDIS_PORT) }) + : new RedisMock() as unknown as Redis; +const serverId = 'https://code.example.test'; +const adminToken = 'browser-enrollment-test-admin'; +const identity = createBridgeIdentity(); +const outsider = createBridgeIdentity(); +const servers: Server[] = []; + +function pairings(options: Partial = {}): RedisBridgePairingStore { + return new RedisBridgePairingStore(redis, 600, 900, 5_000, '', { serverId, ...options }); +} + +async function router(options: Partial = {}): Promise { + const app = express(); + app.set('trust proxy', 1); + app.use(json()); + app.use('/bridge', createBridgeRouter({ + store: new RedisBridgeStore(redis), pairings: pairings(), authMode: 'paired', adminToken, + allowDynamicWorkers: true, configuredWorkerId: 'browser-http-worker', ...options, + })); + const server = createServer(app); + servers.push(server); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + return `http://127.0.0.1:${address.port}/bridge`; +} + +function approved(): BrowserEnrollmentApproval { + const request = { + version: 1 as const, instance: 'https://chat.example.test', attemptId: randomBytes(32).toString('base64url'), + createdAt: new Date().toISOString(), publicKey: identity.publicKey, machineName: 'HTTP worker', + workspace: { type: 'private' as const }, allowWorkspaceWrites: false, allowWorkspaceCommands: false, + }; + return { + protocolVersion: 1, operation: 'enrollment.redeem', serverId, instance: request.instance, + attemptId: request.attemptId, requestHash: browserSetupRequestHash(request), workerId: 'browser-http-worker', + expiresAt: new Date(Date.now() + 60_000).toISOString(), publicKey: identity.publicKey, + binding: { tenantId: 'tenant-one', principal: { type: 'user', id: 'owner-one' } }, + }; +} + +function proof(approval: BrowserEnrollmentApproval, privateKey = identity.privateKey): BrowserEnrollmentRedemption { + const { publicKey: _publicKey, binding: _binding, ...contract } = approval; + return { ...contract, signature: signEnrollmentRedemption(privateKey, contract) }; +} + +function post(url: string, body: object, token = adminToken, headers: Record = {}): Promise { + return fetch(url, { + method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${token}`, ...headers }, + body: JSON.stringify(body), + }); +} + +async function capability(url: string, token = adminToken): Promise { + return fetch(`${url}/enrollments/capabilities`, { headers: { Authorization: `Bearer ${token}` } }); +} + +afterEach(async () => { + await Promise.all(servers.splice(0).map(server => new Promise((resolve, reject) => { + server.close(error => error ? reject(error) : resolve()); + server.closeAllConnections(); + }))); + await redis.flushall(); +}); +afterAll(() => { redis.disconnect(); }); + +describe('browser enrollment HTTP boundary', () => { + test('advertises the version only for paired auth, durable recovery, dynamic workers and an authenticated admin', async () => { + const url = await router(); + const response = await capability(url); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ browserEnrollmentVersion: 1, serverId }); + expect((await capability(url, 'outsider')).status).toBe(401); + for (const options of [ + { enabled: false }, { authMode: 'static' as const }, { allowDynamicWorkers: false }, + { pairings: new RedisBridgePairingStore(redis) }, + ]) { + const disabled = await router(options); + expect((await capability(disabled)).status).toBe(404); + expect((await post(`${disabled}/enrollments`, approved())).status).toBe(404); + expect((await post(`${disabled}/enrollments/redeem`, proof(approved()))).status).toBe(404); + } + const unconfigured = await router({ adminToken: '' }); + expect((await capability(unconfigured)).status).toBe(503); + expect((await post(`${unconfigured}/enrollments`, approved())).status).toBe(503); + }); + + test('preserves old manual pairing when browser enrollment is disabled', async () => { + const url = await router({ allowDynamicWorkers: false, pairings: new RedisBridgePairingStore(redis) }); + const issue = await post(`${url}/pairings`, { workerId: 'browser-http-worker' }); + expect(issue.status).toBe(200); + const pairing = await issue.json() as { code: string }; + const result = await post(`${url}/pairings/redeem`, { + protocolVersion: 1, workerId: 'browser-http-worker', code: pairing.code, publicKey: identity.publicKey, + }, ''); + expect(result.status).toBe(200); + expect(await result.json()).toMatchObject({ protocolVersion: 1, workerId: 'browser-http-worker' }); + }); + + test('requires admin auth for approval, redemption and reconciliation, independently of machine proof', async () => { + const url = await router(); + const approval = approved(); + const redemption = proof(approval); + const { signature: _signature, ...statusRequest } = redemption; + for (const [path, body] of [ + ['enrollments', approval], ['enrollments/redeem', redemption], ['enrollments/status', statusRequest], + ] as const) expect((await post(`${url}/${path}`, body, 'outsider')).status).toBe(401); + expect(await redis.keys('codeapi:bridge:v1:browser-enrollment:*')).toHaveLength(0); + }); + + test('validates envelopes, binds approvals, and idempotently returns the same credential across HTTP replicas', async () => { + const first = await router(); + const second = await router(); + const approval = approved(); + for (const body of [ + { ...approval, protocolVersion: 2 }, { ...approval, publicKey: outsider.privateKey }, + { ...approval, binding: null }, { ...approval, extra: true }, + ]) expect((await post(`${first}/enrollments`, body)).status).toBe(400); + const issued = await post(`${first}/enrollments`, approval); + expect(issued.status).toBe(200); + expect(await issued.json()).toEqual(approval); + expect((await post(`${second}/enrollments`, approval)).status).toBe(200); + const { signature: _pendingSignature, ...pendingRequest } = proof(approval); + const pending = await post(`${second}/enrollments/status`, pendingRequest); + expect(pending.status).toBe(200); + expect(await pending.json()).toEqual({ protocolVersion: 1, status: 'pending', workerId: approval.workerId }); + expect((await post(`${second}/enrollments`, { ...approval, publicKey: outsider.publicKey })).status).toBe(409); + expect((await post(`${first}/enrollments/redeem`, { ...proof(approval), signature: '?' })).status).toBe(400); + expect((await post(`${first}/enrollments/redeem`, { ...proof(approval), operation: 'setup.complete' })).status).toBe(400); + expect((await post(`${first}/enrollments/redeem`, proof(approval, outsider.privateKey))).status).toBe(401); + expect((await post(`${first}/enrollments/redeem`, proof({ ...approval, serverId: 'https://other.example.test' }))).status).toBe(401); + expect((await post(`${first}/enrollments/redeem`, proof({ ...approval, instance: 'https://other.example.test' }))).status).toBe(409); + const responses = await Promise.all([first, second].map(url => post(`${url}/enrollments/redeem`, proof(approval)))); + expect(responses.map(response => response.status)).toEqual([200, 200]); + const [original, replay] = await Promise.all(responses.map(response => response.json())); + expect(original).toEqual(replay); + expect(original).toMatchObject({ protocolVersion: 1, workerId: approval.workerId }); + expect(JSON.stringify(original)).not.toMatch(/tenant-one|owner-one|publicKey|privateKey|binding/); + const { signature: _signature, ...statusRequest } = proof(approval); + expect((await post(`${second}/enrollments/status`, statusRequest)).status).toBe(200); + expect((await post(`${second}/enrollments/status`, { ...statusRequest, requestHash: 'f'.repeat(64) })).status).toBe(409); + expect((await post(`${first}/workers/${approval.workerId}/revoke`, {})).status).toBe(200); + expect((await post(`${second}/enrollments/redeem`, proof(approval))).status).toBe(409); + const revoked = await post(`${second}/enrollments/status`, statusRequest); + expect(revoked.status).toBe(200); + expect(await revoked.json()).toEqual({ protocolVersion: 1, status: 'revoked', workerId: approval.workerId }); + }); + + test('shares a connection-peer budget across replicas and ignores spoofed forwarded IPs', async () => { + const first = await router({ pairings: pairings({ maxUntrustedRequestsPerMinute: 2 }) }); + const second = await router({ pairings: pairings({ maxUntrustedRequestsPerMinute: 2 }) }); + const approval = approved(); + await post(`${first}/enrollments`, approval); + const forged = proof(approval, outsider.privateKey); + expect((await post(`${first}/enrollments/redeem`, forged, adminToken, { 'X-Forwarded-For': '198.51.100.1' })).status).toBe(401); + expect((await post(`${second}/enrollments/redeem`, forged, adminToken, { 'X-Forwarded-For': '198.51.100.2' })).status).toBe(401); + const blocked = await post(`${second}/enrollments/redeem`, proof(approved()), adminToken, { 'X-Forwarded-For': '198.51.100.3' }); + expect(blocked.status).toBe(429); + expect(blocked.headers.get('retry-after')).toBe('60'); + expect(await redis.keys('codeapi:bridge:v1:credential:*')).toHaveLength(0); + }); +}); diff --git a/service/src/bridge/browser-enrollment.test.ts b/service/src/bridge/browser-enrollment.test.ts new file mode 100644 index 00000000..ca2b6844 --- /dev/null +++ b/service/src/bridge/browser-enrollment.test.ts @@ -0,0 +1,447 @@ +import { createHash, randomBytes } from 'crypto'; + +import { afterAll, afterEach, describe, expect, test } from 'bun:test'; +import Redis from 'ioredis'; +import RedisMock from 'ioredis-mock'; + +import { + browserSetupRequestHash, + canonicalBrowserSetupProof, + canonicalBrowserSetupRequest, + canonicalEnrollmentRedemption, + isBrowserEnrollmentApproval, + isBrowserEnrollmentRedemption, + normalizeBrowserEnrollmentInstance, + signBrowserSetupProof, + signEnrollmentRedemption, + validateBrowserSetupProof, + validateBrowserSetupRequest, + verifyBrowserSetupProof, + verifyEnrollmentRedemption, +} from '../../../packages/code/src/browser-enrollment-protocol'; +import type { + BrowserEnrollmentApproval, + BrowserEnrollmentRedemption, + BrowserSetupProof, + BrowserSetupRequest, +} from '../../../packages/code/src/browser-enrollment-protocol'; +import { + createBridgeIdentity, + signBridgeRecovery, + signBridgeRecoveryStart, + signBridgeRequest, +} from '../../../packages/code/src/identity'; +import { RedisBridgePairingStore } from './pairing'; +import type { BridgeRecoveryOptions, BridgeWorkerCredential } from './pairing'; +import { RedisBridgeStore } from './store'; + +const redis = process.env.BROWSER_ENROLLMENT_REDIS_PORT + ? new Redis({ host: '127.0.0.1', port: Number(process.env.BROWSER_ENROLLMENT_REDIS_PORT) }) + : new RedisMock() as unknown as Redis; +const serverId = 'https://code.example.test'; +const adminToken = 'test-browser-enrollment-administrator-secret'; +const identity = createBridgeIdentity(); +const outsider = createBridgeIdentity(); +const workerId = 'browser-enrolled-worker'; + +function store(options: Partial = {}, credentialTtl = 900): RedisBridgePairingStore { + return new RedisBridgePairingStore(redis, 600, credentialTtl, 5_000, '', { serverId, ...options }); +} + +function request(): BrowserSetupRequest { + return { + version: 1, + instance: 'https://chat.example.test/base', + attemptId: randomBytes(32).toString('base64url'), + createdAt: new Date().toISOString(), + publicKey: identity.publicKey, + machineName: 'My VM (development)', + workspace: { type: 'directory' }, + allowWorkspaceWrites: true, + allowWorkspaceCommands: false, + }; +} + +function approval(overrides: Partial = {}): BrowserEnrollmentApproval { + const setup = request(); + return { + protocolVersion: 1, + operation: 'enrollment.redeem', + serverId, + instance: setup.instance, + attemptId: setup.attemptId, + requestHash: browserSetupRequestHash(setup), + workerId, + expiresAt: new Date(Date.now() + 60_000).toISOString(), + publicKey: identity.publicKey, + binding: { tenantId: 'tenant-one', principal: { type: 'user', id: 'owner-one' } }, + ...overrides, + }; +} + +function redemption(input: BrowserEnrollmentApproval, privateKey = identity.privateKey): BrowserEnrollmentRedemption { + const { operation, serverId, instance, attemptId, requestHash, workerId, expiresAt } = input; + const contract = { operation, serverId, instance, attemptId, requestHash, workerId, expiresAt }; + return { protocolVersion: 1, ...contract, signature: signEnrollmentRedemption(privateKey, contract) }; +} + +function authorized(credential: BridgeWorkerCredential): Parameters[0] { + const proof = { + credential: credential.credential, + method: 'POST', + path: `/bridge/workers/${workerId}/lease`, + timestamp: new Date().toISOString(), + nonce: randomBytes(32).toString('base64url'), + body: '{}', + }; + return { ...proof, workerId, signature: signBridgeRequest(identity.privateKey, proof) }; +} + +afterEach(async () => { await redis.flushall(); }); +afterAll(() => { redis.disconnect(); }); + +describe('browser enrollment proof contract', () => { + test('canonicalizes fixed request fields and preserves every authorization choice', () => { + const input = request(); + expect(normalizeBrowserEnrollmentInstance('https://CHAT.example.test:443/base///')).toBe(input.instance); + expect(() => normalizeBrowserEnrollmentInstance('http://untrusted.example.test')).toThrow(); + expect(() => normalizeBrowserEnrollmentInstance('https://user:password@chat.example.test')).toThrow(); + expect(() => normalizeBrowserEnrollmentInstance('https://chat.example.test/?query=1')).toThrow(); + const canonical = canonicalBrowserSetupRequest(input); + expect(Object.keys(JSON.parse(canonical))).toEqual([ + 'version', 'instance', 'attemptId', 'createdAt', 'publicKey', 'machineName', 'workspace', + 'allowWorkspaceWrites', 'allowWorkspaceCommands', + ]); + expect(browserSetupRequestHash(input)).toBe(createHash('sha256').update(canonical).digest('hex')); + const reordered = Object.fromEntries(Object.entries(input).reverse()) as unknown as BrowserSetupRequest; + expect(browserSetupRequestHash(reordered)).toBe(browserSetupRequestHash(input)); + for (const change of [ + { allowWorkspaceWrites: false }, { allowWorkspaceCommands: true }, { workspace: { type: 'private' as const } }, + { machineName: 'Different VM' }, { publicKey: outsider.publicKey }, + ]) expect(browserSetupRequestHash({ ...input, ...change })).not.toBe(browserSetupRequestHash(input)); + }); + + test('rejects noncanonical requests and malformed proofs without throwing from verification', () => { + const input = request(); + for (const malformed of [ + null, [], { ...input, extra: true }, { ...input, version: 2 }, { ...input, machineName: 'x\ny' }, + { ...input, machineName: 'x'.repeat(81) }, { ...input, machineName: ' leading' }, + { ...input, instance: `${input.instance}/` }, { ...input, attemptId: 'x'.repeat(43) }, + { ...input, createdAt: '2026-10-21' }, { ...input, publicKey: outsider.privateKey }, + { ...input, workspace: { type: 'directory', path: '/secret' } }, { ...input, allowWorkspaceWrites: 1 }, + ]) expect(validateBrowserSetupRequest(malformed)).toBe(false); + expect(verifyBrowserSetupProof('invalid key', {} as BrowserSetupProof, '!')).toBe(false); + }); + + test('binds operation, audience, attempt, request hash, time and nonce, with separate signing domains', () => { + const input = request(); + const proof: BrowserSetupProof = { + operation: 'setup.start', instance: input.instance, attemptId: input.attemptId, + requestHash: browserSetupRequestHash(input), timestamp: input.createdAt, + nonce: randomBytes(32).toString('base64url'), + }; + const signature = signBrowserSetupProof(identity.privateKey, proof); + expect(canonicalBrowserSetupProof(proof).split('\n')).toEqual([ + 'librechat-code:browser-setup:v1', proof.operation, proof.instance, proof.attemptId, + proof.requestHash, proof.timestamp, proof.nonce, + ]); + expect(validateBrowserSetupProof(proof)).toBe(true); + expect(verifyBrowserSetupProof(identity.publicKey, proof, signature)).toBe(true); + expect(verifyBrowserSetupProof(outsider.publicKey, proof, signature)).toBe(false); + for (const change of [ + { operation: 'setup.poll' as const }, { operation: 'setup.cancel' as const }, { operation: 'setup.complete' as const }, + { instance: 'https://different.example.test' }, { attemptId: randomBytes(32).toString('base64url') }, + { requestHash: 'f'.repeat(64) }, { timestamp: new Date(Date.now() + 1_000).toISOString() }, + { nonce: randomBytes(32).toString('base64url') }, + ]) expect(verifyBrowserSetupProof(identity.publicKey, { ...proof, ...change }, signature)).toBe(false); + const approved = approval({ instance: proof.instance, attemptId: proof.attemptId, requestHash: proof.requestHash }); + expect(canonicalEnrollmentRedemption(approved).split('\n')).toEqual([ + 'librechat-code:browser-enrollment-redeem:v1', approved.operation, approved.serverId, + approved.instance, approved.attemptId, approved.requestHash, approved.workerId, approved.expiresAt, + ]); + expect(verifyEnrollmentRedemption(identity.publicKey, approved, signature)).toBe(false); + expect(verifyBrowserSetupProof(identity.publicKey, proof, signEnrollmentRedemption(identity.privateKey, approved))).toBe(false); + expect(isBrowserEnrollmentRedemption({ ...redemption(approved), publicKey: identity.publicKey })).toBe(false); + expect(isBrowserEnrollmentApproval({ ...approved, binding: { tenantId: 'x', principal: { type: 'bad', id: 'x' } } })).toBe(false); + }); +}); + +describe('Redis browser enrollment ownership', () => { + test('retries lost issue and redemption responses across replicas without rotating keys or widening access', async () => { + const first = store(); + const approved = approval(); + await first.issueBrowserEnrollment(approved); + const second = store(); + expect(await second.issueBrowserEnrollment(approved)).toEqual(approved); + const original = await first.redeemBrowserEnrollment(redemption(approved), adminToken); + expect(await second.issueBrowserEnrollment(approved)).toEqual(approved); + expect(await second.redeemBrowserEnrollment(redemption(approved), adminToken)).toEqual(original); + const auth = await second.authorize(authorized(original)); + expect(auth).toMatchObject({ workerId, binding: approved.binding, pairingGeneration: 0 }); + const enrollmentRaw = await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`); + const allValues = await redis.mget(...await redis.keys('codeapi:bridge:v1:*')); + expect(allValues.join('\n')).not.toContain(original.credential); + expect(enrollmentRaw).toContain(identity.publicKey.trim().split('\n')[1]); + await expect(second.redeemBrowserEnrollment(redemption(approved), 'wrong-encryption-key')).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`)).toBe(enrollmentRaw); + }); + + test('concurrent duplicates commit exactly one enrollment and return the identical response', async () => { + const approved = approval(); + const replicas = [store(), store(), store()]; + const issued = await Promise.all(replicas.map(replica => replica.issueBrowserEnrollment(approved))); + expect(issued).toEqual([approved, approved, approved]); + const completed = await Promise.all(replicas.map(replica => replica.redeemBrowserEnrollment(redemption(approved), adminToken))); + expect(completed[1]).toEqual(completed[0]); + expect(completed[2]).toEqual(completed[0]); + expect(await redis.keys('codeapi:bridge:v1:credential:*')).toHaveLength(1); + expect(await redis.keys('codeapi:bridge:v1:enrollment:*')).toHaveLength(1); + }); + + test('does not depend on binding property order and retries after worker registration', async () => { + const replica = store(); + const approved = approval({ binding: { principal: { id: 'owner-one', type: 'user' }, tenantId: 'tenant-one' } }); + await replica.issueBrowserEnrollment(approved); + const credential = await replica.redeemBrowserEnrollment(redemption(approved), adminToken); + const auth = await replica.authorize(authorized(credential)); + const workers = new RedisBridgeStore(redis); + await workers.register({ + protocolVersion: 1, workerId, incarnationId: 'browser-test-incarnation', + identityId: auth.identityId, credentialId: auth.credentialId, binding: approved.binding, + capabilities: { statefulWorkspace: false, runtimes: [], sandboxProfile: 'native-srt' }, + }, auth); + expect(await replica.issueBrowserEnrollment(approved)).toEqual(approved); + expect(await replica.redeemBrowserEnrollment(redemption(approved), adminToken)).toEqual(credential); + }); + + test('conflicting concurrent approvals cannot replace the winner or allocate a second worker', async () => { + const approved = approval(); + const changed = { ...approved, workerId: 'other-worker', publicKey: outsider.publicKey }; + const results = await Promise.allSettled([store().issueBrowserEnrollment(approved), store().issueBrowserEnrollment(changed)]); + expect(results.filter(result => result.status === 'fulfilled')).toHaveLength(1); + expect(results.filter(result => result.status === 'rejected')).toHaveLength(1); + const winner = results.find(result => result.status === 'fulfilled'); + if (winner?.status !== 'fulfilled') throw new Error('Expected one committed approval'); + const key = winner.value.publicKey === identity.publicKey ? identity.privateKey : outsider.privateKey; + await store().redeemBrowserEnrollment(redemption(winner.value, key), adminToken); + expect(await redis.keys('codeapi:bridge:v1:enrollment:*')).toHaveLength(1); + }); + + test('bounded credential replay cannot remint after its TTL but durable status remains available', async () => { + const replica = store({}, 1); + const approved = approval(); + await replica.issueBrowserEnrollment(approved); + await replica.redeemBrowserEnrollment(redemption(approved), adminToken); + await new Promise(resolve => setTimeout(resolve, 1_100)); + await expect(store({}, 1).redeemBrowserEnrollment(redemption(approved), adminToken)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + const { signature: _signature, ...statusRequest } = redemption(approved); + expect(await store().browserEnrollmentStatus(statusRequest)).toMatchObject({ status: 'completed', workerId }); + expect(await redis.keys('codeapi:bridge:v1:credential:*')).toHaveLength(0); + }); + + test('rejects replacement of immutable approval fields and reuse of the allocated worker', async () => { + const replica = store(); + const approved = approval(); + await replica.issueBrowserEnrollment(approved); + for (const change of [ + { workerId: 'another-worker' }, { publicKey: outsider.publicKey }, { requestHash: 'f'.repeat(64) }, + { expiresAt: new Date(Date.now() + 120_000).toISOString() }, + { binding: { tenantId: 'tenant-two', principal: { type: 'deployment' as const, id: 'everyone' } } }, + ]) await expect(replica.issueBrowserEnrollment({ ...approved, ...change })).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(replica.issueBrowserEnrollment(approval())).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + const first = await replica.redeemBrowserEnrollment(redemption(approved), adminToken); + await expect(replica.issueBrowserEnrollment(approval())).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(await replica.authorize(authorized(first))).toMatchObject({ binding: approved.binding }); + }); + + test('rejects wrong key, audience, operation and every contract mismatch', async () => { + const replica = store(); + const approved = approval(); + await replica.issueBrowserEnrollment(approved); + await expect(replica.redeemBrowserEnrollment(redemption(approved, outsider.privateKey), adminToken)).rejects.toMatchObject({ code: 'PROOF_INVALID' }); + for (const change of [ + { serverId: 'https://other-code.example.test' }, { instance: 'https://other-chat.example.test' }, + { attemptId: randomBytes(32).toString('base64url') }, { requestHash: 'f'.repeat(64) }, + { workerId: 'another-worker' }, { expiresAt: new Date(Date.now() + 120_000).toISOString() }, + ]) await expect(replica.redeemBrowserEnrollment(redemption({ ...approved, ...change }), adminToken)).rejects.toBeDefined(); + await expect(replica.redeemBrowserEnrollment({ ...redemption(approved), operation: 'setup.complete' } as unknown as BrowserEnrollmentRedemption, adminToken)).rejects.toMatchObject({ code: 'PROOF_INVALID' }); + await expect(replica.redeemBrowserEnrollment({ ...redemption(approved), signature: '?' }, adminToken)).rejects.toMatchObject({ code: 'PROOF_INVALID' }); + expect(await redis.keys('codeapi:bridge:v1:credential:*')).toHaveLength(0); + await replica.redeemBrowserEnrollment(redemption(approved), adminToken); + }); + + test('never extends the immutable expiry or recreates an expired approval', async () => { + const replica = store(); + for (const expiresAt of [new Date(Date.now() - 1).toISOString(), new Date(Date.now() + 601_000).toISOString()]) { + await expect(replica.issueBrowserEnrollment(approval({ expiresAt }))).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + } + const approved = approval({ expiresAt: new Date(Date.now() + 2_000).toISOString() }); + await replica.issueBrowserEnrollment(approved); + await new Promise(resolve => setTimeout(resolve, 2_100)); + await expect(replica.issueBrowserEnrollment(approved)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(replica.redeemBrowserEnrollment(redemption(approved), adminToken)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + const { signature: _signature, ...statusRequest } = redemption(approved); + expect(await replica.browserEnrollmentStatus(statusRequest)).toMatchObject({ status: 'revoked' }); + await expect(replica.issueBrowserEnrollment({ ...approved, workerId: 'replacement-worker', expiresAt: new Date(Date.now() + 60_000).toISOString() })).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + }); + + test('durable enrollment outlives setup expiry and recovers after restart and access credential loss', async () => { + const replica = store(); + const approved = approval({ expiresAt: new Date(Date.now() + 2_000).toISOString() }); + await replica.issueBrowserEnrollment(approved); + const original = await replica.redeemBrowserEnrollment(redemption(approved), adminToken); + const originalAuth = await replica.authorize(authorized(original)); + await new Promise(resolve => setTimeout(resolve, 2_100)); + expect(await store().redeemBrowserEnrollment(redemption(approved), adminToken)).toEqual(original); + await redis.del( + `codeapi:bridge:v1:identity:${workerId}`, `codeapi:bridge:v1:stable-identity:${workerId}`, + `codeapi:bridge:v1:credential:${createHash('sha256').update(original.credential).digest('hex')}`, + ); + const restarted = store(); + const { signature: _signature, ...statusRequest } = redemption(approved); + expect(await restarted.browserEnrollmentStatus(statusRequest)).toEqual({ protocolVersion: 1, status: 'completed', workerId }); + await expect(restarted.browserEnrollmentStatus({ ...statusRequest, requestHash: 'f'.repeat(64) })).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + const start = { + operation: 'credential.challenge' as const, serverId, workerId, + timestamp: new Date().toISOString(), nonce: randomBytes(32).toString('base64url'), + }; + const challenge = await restarted.createRecoveryChallenge(workerId, start, signBridgeRecoveryStart(identity.privateKey, start)); + const recovered = await restarted.recoverCredential(workerId, challenge, signBridgeRecovery(identity.privateKey, challenge)); + expect(await restarted.authorize(authorized(recovered))).toMatchObject({ identityId: originalAuth.identityId, binding: approved.binding }); + await restarted.revoke(workerId); + expect(await restarted.browserEnrollmentStatus(statusRequest)).toMatchObject({ status: 'revoked', workerId }); + await expect(restarted.redeemBrowserEnrollment(redemption(approved), adminToken)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + }); + + test('revocation fences approval, completion, exact replay and recovery without clearing quarantine', async () => { + const replica = store(); + const approved = approval(); + const quarantine = `codeapi:bridge:v1:worker:${workerId}:workspace:root:quarantined`; + await redis.set(quarantine, 'existing-fence'); + await replica.issueBrowserEnrollment(approved); + await replica.revoke(workerId); + await expect(replica.issueBrowserEnrollment(approved)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(replica.redeemBrowserEnrollment(redemption(approved), adminToken)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(replica.issueBrowserEnrollment(approval())).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(await redis.get(quarantine)).toBe('existing-fence'); + expect(await redis.get(`codeapi:bridge:v1:pairing-generation:${workerId}`)).toBe('1'); + + const other = approval({ workerId: 'second-worker' }); + await replica.issueBrowserEnrollment(other); + const credential = await replica.redeemBrowserEnrollment(redemption(other), adminToken); + await replica.revoke(other.workerId); + await expect(replica.redeemBrowserEnrollment(redemption(other), adminToken)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(await redis.get(`codeapi:bridge:v1:enrollment:${other.workerId}`)).toBeNull(); + expect(credential.workerId).toBe(other.workerId); + }); + + test('status distinguishes pending, completed, revoked and unknown exact contracts without issuing credentials', async () => { + const replica = store(); + const approved = approval(); + const { signature: _signature, ...statusRequest } = redemption(approved); + await expect(replica.browserEnrollmentStatus(statusRequest)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await replica.issueBrowserEnrollment(approved); + expect(await store().browserEnrollmentStatus(statusRequest)).toEqual({ protocolVersion: 1, status: 'pending', workerId }); + expect(await redis.keys('codeapi:bridge:v1:credential:*')).toHaveLength(0); + await expect(replica.browserEnrollmentStatus({ ...statusRequest, workerId: 'different-worker' })).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await replica.revoke(workerId); + expect(await store().browserEnrollmentStatus(statusRequest)).toEqual({ protocolVersion: 1, status: 'revoked', workerId }); + await expect(replica.issueBrowserEnrollment(approved)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(await redis.keys('codeapi:bridge:v1:credential:*')).toHaveLength(0); + }); + + test('manual issue supersedes only the pending fence and cannot be undone by a retry', async () => { + const replica = store(); + const approved = approval(); + await replica.issueBrowserEnrollment(approved); + const manual = await replica.issue(workerId, approved.binding); + await expect(replica.issueBrowserEnrollment(approved)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(replica.redeemBrowserEnrollment(redemption(approved), adminToken)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + const issued = await replica.redeem({ workerId, code: manual.code, publicKey: outsider.publicKey }); + await expect(replica.issueBrowserEnrollment(approval())).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(issued.workerId).toBe(workerId); + }); + + for (const afterScanWindow of [false, true]) test(`unindexed legacy codes cannot supersede browser approval ${afterScanWindow ? 'after' : 'during'} migration`, async () => { + const replica = store(); + const approved = approval(); + const legacyCode = randomBytes(24).toString('base64url'); + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256').update(legacyCode).digest('hex')}`; + await redis.set(legacyKey, JSON.stringify({ workerId, expiresAt: new Date(Date.now() + 60_000).toISOString(), + binding: { tenantId: 'other-tenant', principal: { type: 'user', id: 'other-owner' } } }), 'EX', 60); + if (afterScanWindow) await redis.set('codeapi:bridge:v1:migration:legacy-pairing-scan-until', String(Date.now() - 1)); + await replica.issueBrowserEnrollment(approved); + expect(await redis.get(`codeapi:bridge:v1:pairing-generation:${workerId}`)).toBe('0'); + if (!afterScanWindow) expect(await redis.get(legacyKey)).toBeNull(); + await expect(replica.redeem({ workerId, code: legacyCode, publicKey: outsider.publicKey })).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); + const credential = await replica.redeemBrowserEnrollment(redemption(approved), adminToken); + expect(await replica.authorize(authorized(credential))).toMatchObject({ workerId, binding: approved.binding }); + }); + + test('cannot replace a preexisting manual enrollment or pending pairing', async () => { + const replica = store(); + const manual = await replica.issue(workerId); + await expect(replica.issueBrowserEnrollment(approval())).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await replica.redeem({ workerId, code: manual.code, publicKey: outsider.publicKey }); + await expect(replica.issueBrowserEnrollment(approval())).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(replica.issueBrowserEnrollment(approval({ serverId: 'https://wrong.example.test' }))).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(new RedisBridgePairingStore(redis).issueBrowserEnrollment(approval())).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + }); + + test('per-attempt budgets are shared by replicas and wrong signatures cannot create credentials', async () => { + const approved = approval(); + const first = store({ maxAttemptsPerMinute: 2 }); + const second = store({ maxAttemptsPerMinute: 2 }); + await first.issueBrowserEnrollment(approved); + const forged = redemption(approved, outsider.privateKey); + await expect(first.redeemBrowserEnrollment(forged, adminToken)).rejects.toMatchObject({ code: 'PROOF_INVALID' }); + await expect(second.redeemBrowserEnrollment(forged, adminToken)).rejects.toMatchObject({ code: 'PROOF_INVALID' }); + await expect(first.redeemBrowserEnrollment(redemption(approved), adminToken)).rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + expect(await redis.keys('codeapi:bridge:v1:credential:*')).toHaveLength(0); + }); + + test('manual issue between signature verification and completion permanently supersedes the browser attempt', async () => { + const replica = store(); + const approved = approval(); + await replica.issueBrowserEnrollment(approved); + const originalEval = redis.eval.bind(redis); + let manualCode = ''; + redis.eval = (async (...args: Parameters): Promise => { + if (!manualCode && String(args[0]).includes('local replay =')) { + manualCode = (await replica.issue(workerId)).code; + } + return originalEval(...args); + }) as Redis['eval']; + try { + await expect(replica.redeemBrowserEnrollment(redemption(approved), adminToken)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(manualCode.length).toBeGreaterThan(0); + await replica.redeem({ workerId, code: manualCode, publicKey: outsider.publicKey }); + await expect(replica.issueBrowserEnrollment(approved)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + } finally { + redis.eval = originalEval as Redis['eval']; + } + }); + + test('atomic generation comparison wins a revoke between proof validation and completion', async () => { + const replica = store(); + const approved = approval(); + await replica.issueBrowserEnrollment(approved); + const originalEval = redis.eval.bind(redis); + let intercepted = false; + redis.eval = (async (...args: Parameters): Promise => { + if (!intercepted && String(args[0]).includes('local replay =')) { + intercepted = true; + await replica.revoke(workerId); + } + return originalEval(...args); + }) as Redis['eval']; + try { + await expect(replica.redeemBrowserEnrollment(redemption(approved), adminToken)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(intercepted).toBe(true); + expect(await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`)).toBeNull(); + expect(await redis.keys('codeapi:bridge:v1:credential:*')).toHaveLength(0); + } finally { + redis.eval = originalEval as Redis['eval']; + } + }); +}); diff --git a/service/src/bridge/enabled.ts b/service/src/bridge/enabled.ts index 540a844a..e0cba569 100644 --- a/service/src/bridge/enabled.ts +++ b/service/src/bridge/enabled.ts @@ -7,3 +7,8 @@ export function isBridgeEnabled(): boolean { || env.BRIDGE_DYNAMIC_WORKERS || env.BRIDGE_TOKEN.length > 0; } + +/** `import_file` relays need the egress gateway to mint each one-object read grant. */ +export function isWorkspaceImportEnabled(): boolean { + return env.EGRESS_GATEWAY_URL.trim() !== ''; +} diff --git a/service/src/bridge/index.ts b/service/src/bridge/index.ts index f7ba03b4..93370dfa 100644 --- a/service/src/bridge/index.ts +++ b/service/src/bridge/index.ts @@ -3,7 +3,7 @@ import { env } from '../config'; import { RedisBridgePairingStore } from './pairing'; import { createBridgeRouter } from './router'; import { RedisBridgeStore } from './store'; -import { isBridgeEnabled } from './enabled'; +import { isBridgeEnabled, isWorkspaceImportEnabled } from './enabled'; export const bridgeStore = new RedisBridgeStore( connection, @@ -39,4 +39,5 @@ export default createBridgeRouter({ configuredWorkerId: env.BRIDGE_WORKER_ID, allowDynamicWorkers: env.BRIDGE_DYNAMIC_WORKERS, maxCommandTimeoutMs: env.JOB_TIMEOUT, + workspaceImportFile: isWorkspaceImportEnabled(), }); diff --git a/service/src/bridge/pairing.ts b/service/src/bridge/pairing.ts index b8a32162..c8170b69 100644 --- a/service/src/bridge/pairing.ts +++ b/service/src/bridge/pairing.ts @@ -1,4 +1,6 @@ import { + createCipheriv, + createDecipheriv, createHash, createPublicKey, randomBytes, @@ -6,6 +8,21 @@ import { import type Redis from 'ioredis'; +import { + BROWSER_ENROLLMENT_MAX_TTL_MS, + canonicalBrowserEnrollmentRedemption, + isBrowserEnrollmentApproval, + isBrowserEnrollmentRedemption, + isBrowserEnrollmentRedemptionContract, + isBrowserEnrollmentStatusRequest, + verifyBrowserEnrollmentRedemption, +} from '../../../packages/code/src/browser-enrollment-protocol'; +import type { + BrowserEnrollmentApproval, + BrowserEnrollmentRedemption, + BrowserEnrollmentRedemptionContract, + BrowserEnrollmentStatusRequest, +} from '../../../packages/code/src/browser-enrollment-protocol'; import { verifyBridgeRecovery, verifyBridgeRecoveryStart, @@ -74,6 +91,66 @@ else end return 1 `; +const ISSUE_BROWSER_ENROLLMENT_SCRIPT = ` +local now = redis.call('TIME') +local remaining = tonumber(ARGV[3]) - (tonumber(now[1]) * 1000.0 + math.floor(tonumber(now[2]) / 1000)) +if remaining <= 0 or remaining > tonumber(ARGV[4]) then return 0 end +if (redis.call('GET', KEYS[5]) or '0') ~= '0' then return 0 end +local saved = redis.call('GET', KEYS[1]) +local tombstone = redis.call('GET', KEYS[13]) +if tombstone and (not saved or tombstone ~= ARGV[5]) then return 0 end +if saved then + if saved ~= ARGV[1] or redis.call('GET', KEYS[4]) ~= KEYS[3] or redis.call('GET', KEYS[3]) ~= 'browser-enrollment' then return 0 end + local enrollment = redis.call('GET', KEYS[8]) + if enrollment then + if enrollment ~= redis.call('GET', KEYS[2]) then return 0 end + else + for i = 6, 12 do + if i ~= 8 and redis.call('EXISTS', KEYS[i]) == 1 then return 0 end + end + end + redis.call('SETNX', KEYS[5], '0') + return 1 +end +for i = 4, 12 do + if i ~= 5 and redis.call('EXISTS', KEYS[i]) == 1 then return 0 end +end +redis.call('SETNX', KEYS[5], '0') +redis.call('SET', KEYS[13], ARGV[5]) +redis.call('SET', KEYS[1], ARGV[1], 'PX', remaining) +redis.call('SET', KEYS[2], ARGV[2], 'PX', remaining) +redis.call('SET', KEYS[3], 'browser-enrollment', 'PX', remaining) +redis.call('SET', KEYS[4], KEYS[3], 'PX', remaining) +return 1 +`; +const REDEEM_BROWSER_ENROLLMENT_SCRIPT = ` +local now = redis.call('TIME') +local remaining = tonumber(ARGV[3]) - (tonumber(now[1]) * 1000.0 + math.floor(tonumber(now[2]) / 1000)) +if redis.call('GET', KEYS[1]) ~= ARGV[1] or redis.call('GET', KEYS[2]) ~= ARGV[2] or + redis.call('GET', KEYS[3]) ~= 'browser-enrollment' or redis.call('GET', KEYS[4]) ~= KEYS[3] or + (redis.call('GET', KEYS[5]) or '0') ~= '0' then return nil end +local replay = redis.call('GET', KEYS[11]) +if replay then + if redis.call('GET', KEYS[9]) ~= ARGV[2] or redis.call('GET', KEYS[10]) ~= ARGV[9] then return nil end + return replay +end +if remaining <= 0 or ARGV[11] ~= '1' then return nil end +for i = 7, 14 do + if i ~= 11 and redis.call('EXISTS', KEYS[i]) == 1 then return nil end +end +redis.call('SET', KEYS[6], ARGV[4], 'EX', ARGV[5]) +redis.call('SET', KEYS[7], ARGV[6], 'EX', ARGV[5]) +redis.call('SET', KEYS[8], ARGV[7], 'EX', ARGV[5]) +if tonumber(ARGV[8]) > 0 then + redis.call('SET', KEYS[9], ARGV[2], 'EX', ARGV[8]) +else + redis.call('SET', KEYS[9], ARGV[2]) +end +redis.call('SET', KEYS[10], ARGV[9]) +redis.call('SET', KEYS[11], ARGV[10], 'EX', ARGV[5]) +for i = 1, 4 do redis.call('EXPIRE', KEYS[i], ARGV[5]) end +return ARGV[10] +`; const ROTATE_CREDENTIAL_SCRIPT = ` if ARGV[6] ~= '' then if redis.call('GET', KEYS[5]) ~= ARGV[6] or (redis.call('GET', KEYS[6]) or '0') ~= ARGV[7] or redis.call('GET', KEYS[7]) ~= ARGV[8] then @@ -127,7 +204,7 @@ if redis.call('EXISTS', KEYS[6]) == 1 then return -2 end local count = redis.call('INCR', KEYS[2]) if count == 1 then redis.call('EXPIRE', KEYS[2], 60) end if count > tonumber(ARGV[2]) then return -1 end -if redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[5], 'NX') ~= 'OK' then return 0 end +if not redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[5], 'NX') then return 0 end redis.call('SET', KEYS[6], '1', 'EX', ARGV[7]) return 1 `; @@ -223,6 +300,7 @@ interface StoredEnrollment { pairingGeneration: number; publicKey: string; binding?: BridgeWorkerBinding; + browserEnrollment?: BrowserEnrollmentRedemptionContract; } export interface BridgeRecoveryOptions { @@ -277,6 +355,54 @@ function pairingKey(code: string): string { return `${PREFIX}:pairing:${digest(code)}`; } +function browserEnrollmentKey(instance: string, attemptId: string): string { + return `${PREFIX}:browser-enrollment:${digest(JSON.stringify([instance, attemptId]))}`; +} + +function canonicalApproval(approval: BrowserEnrollmentApproval): string { + return JSON.stringify({ + protocolVersion: 1, + operation: approval.operation, + serverId: approval.serverId, + instance: approval.instance, + attemptId: approval.attemptId, + requestHash: approval.requestHash, + workerId: approval.workerId, + expiresAt: approval.expiresAt, + publicKey: approval.publicKey, + binding: { + tenantId: approval.binding.tenantId, + principal: { type: approval.binding.principal.type, id: approval.binding.principal.id }, + }, + }); +} + +function enrollmentReplayKey(adminToken: string, serverId: string): Buffer { + if (!adminToken) throw new BridgePairingError('ENROLLMENT_INVALID', 'Browser enrollment requires an administrator token'); + return createHash('sha256').update('librechat-code:browser-enrollment-replay:v1\0') + .update(serverId).update('\0').update(adminToken).digest(); +} + +function encryptEnrollmentReplay(credential: BridgeWorkerCredential, approval: string, key: Buffer): string { + const iv = randomBytes(12); + const cipher = createCipheriv('aes-256-gcm', key, iv); + cipher.setAAD(Buffer.from(approval)); + const ciphertext = Buffer.concat([cipher.update(JSON.stringify(credential), 'utf8'), cipher.final()]); + return Buffer.concat([iv, cipher.getAuthTag(), ciphertext]).toString('base64url'); +} + +function decryptEnrollmentReplay(ciphertext: string, approval: string, key: Buffer): BridgeWorkerCredential { + try { + const bytes = Buffer.from(ciphertext, 'base64url'); + const decipher = createDecipheriv('aes-256-gcm', key, bytes.subarray(0, 12)); + decipher.setAAD(Buffer.from(approval)); + decipher.setAuthTag(bytes.subarray(12, 28)); + return JSON.parse(Buffer.concat([decipher.update(bytes.subarray(28)), decipher.final()]).toString('utf8')) as BridgeWorkerCredential; + } catch { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Browser enrollment replay is unavailable'); + } +} + function credentialDigestKey(credentialDigest: string): string { return `${PREFIX}:credential:${credentialDigest}`; } @@ -408,6 +534,215 @@ export class RedisBridgePairingStore { return this.recovery != null; } + get recoveryServerId(): string | undefined { + return this.recovery?.serverId; + } + + async limitBrowserEnrollmentPeer(peer: string): Promise { + if (this.recovery == null) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Browser enrollment is disabled'); + } + const allowed = await this.redis.eval( + LIMIT_RECOVERY_ATTEMPTS_SCRIPT, + 1, + `${PREFIX}:browser-enrollment:peer:${digest(peer)}`, + String(this.recovery.maxUntrustedRequestsPerMinute ?? DEFAULT_UNTRUSTED_RECOVERY_REQUESTS_PER_MINUTE), + ); + if (allowed !== 1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many browser enrollment requests from this peer'); + } + } + + async issueBrowserEnrollment(approval: BrowserEnrollmentApproval): Promise { + if (!isBrowserEnrollmentApproval(approval) || this.recovery == null || approval.serverId !== this.recovery.serverId || + Date.parse(approval.expiresAt) <= Date.now() || Date.parse(approval.expiresAt) - Date.now() > BROWSER_ENROLLMENT_MAX_TTL_MS) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Invalid browser enrollment approval'); + } + await this.removeLegacyPairings(approval.workerId); + const approvalRaw = canonicalApproval(approval); + const normalized = JSON.parse(approvalRaw) as BrowserEnrollmentApproval; + const enrollment: StoredEnrollment = { + workerId: approval.workerId, + serverId: approval.serverId, + identityId: randomBytes(18).toString('base64url'), + generation: randomBytes(18).toString('base64url'), + pairingGeneration: 0, + browserEnrollment: { + operation: approval.operation, + serverId: approval.serverId, + instance: approval.instance, + attemptId: approval.attemptId, + requestHash: approval.requestHash, + workerId: approval.workerId, + expiresAt: approval.expiresAt, + }, + publicKey: approval.publicKey, + binding: normalized.binding, + }; + const key = browserEnrollmentKey(approval.instance, approval.attemptId); + const workerPrefix = `${PREFIX}:worker:${encodeURIComponent(approval.workerId)}`; + // The manual-pairing index points at a separate fence, never the immutable + // approval. Manual issue/revoke consumes that fence without erasing the tombstone. + const accepted = await this.redis.eval( + ISSUE_BROWSER_ENROLLMENT_SCRIPT, + 13, + key, + `${key}:enrollment`, + `${key}:fence`, + workerPairingIndexKey(approval.workerId), + workerPairingGenerationKey(approval.workerId), + workerIdentityKey(approval.workerId), + workerStableIdentityKey(approval.workerId), + workerEnrollmentKey(approval.workerId), + workerEnrollmentRequiredKey(approval.workerId), + workerPrefix, + `${workerPrefix}:incarnation`, + `${workerPrefix}:registration-generation`, + `${key}:tombstone`, + approvalRaw, + JSON.stringify(enrollment), + String(Date.parse(approval.expiresAt)), + String(BROWSER_ENROLLMENT_MAX_TTL_MS), + `${digest(approvalRaw)}:${digest(canonicalBrowserEnrollmentRedemption(approval))}`, + ); + if (accepted !== 1) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Browser enrollment is expired, revoked, superseded, or conflicts with an existing worker'); + } + return normalized; + } + + async browserEnrollmentStatus(proof: BrowserEnrollmentStatusRequest): Promise<{ + protocolVersion: 1; + status: 'pending' | 'completed' | 'revoked'; + workerId: string; + }> { + if (!isBrowserEnrollmentStatusRequest(proof) || this.recovery == null || proof.serverId !== this.recovery.serverId) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Invalid browser enrollment status request'); + } + const key = browserEnrollmentKey(proof.instance, proof.attemptId); + const [raw, generation, requiredGeneration, approvalRaw, tombstone, fence, indexed, planned] = await this.redis.mget( + workerEnrollmentKey(proof.workerId), + workerPairingGenerationKey(proof.workerId), + workerEnrollmentRequiredKey(proof.workerId), + key, + `${key}:tombstone`, + `${key}:fence`, + workerPairingIndexKey(proof.workerId), + `${key}:enrollment`, + ); + const contract = canonicalBrowserEnrollmentRedemption(proof); + try { + const enrollment = this.checkedEnrollment(proof.workerId, raw, generation, requiredGeneration); + if (isBrowserEnrollmentRedemptionContract(enrollment.browserEnrollment) && + canonicalBrowserEnrollmentRedemption(enrollment.browserEnrollment) === contract) { + return { protocolVersion: 1, status: 'completed', workerId: proof.workerId }; + } + } catch (error) { + if (!(error instanceof BridgePairingError)) throw error; + } + if (tombstone?.split(':')[1] !== digest(contract)) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Browser enrollment does not match a known contract'); + } + let pending = false; + try { + const approval = JSON.parse(approvalRaw ?? 'null') as unknown; + pending = (generation ?? '0') === '0' && raw == null && requiredGeneration == null && planned != null && + isBrowserEnrollmentApproval(approval) && canonicalBrowserEnrollmentRedemption(approval) === contract && + Date.parse(approval.expiresAt) > Date.now() && fence === 'browser-enrollment' && indexed === `${key}:fence`; + } catch { + pending = false; + } + return { protocolVersion: 1, status: pending ? 'pending' : 'revoked', workerId: proof.workerId }; + } + + async redeemBrowserEnrollment( + proof: BrowserEnrollmentRedemption, + adminToken: string, + ): Promise { + if (!isBrowserEnrollmentRedemption(proof) || this.recovery == null || proof.serverId !== this.recovery.serverId) { + throw new BridgePairingError('PROOF_INVALID', 'Invalid browser enrollment proof'); + } + const replayKey = enrollmentReplayKey(adminToken, proof.serverId); + const key = browserEnrollmentKey(proof.instance, proof.attemptId); + const attempts = await this.redis.eval( + LIMIT_RECOVERY_ATTEMPTS_SCRIPT, + 1, + `${key}:attempts`, + String(this.recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE), + ); + if (attempts !== 1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many browser enrollment attempts'); + } + const [raw, enrollmentRaw] = await this.redis.mget(key, `${key}:enrollment`); + let approval: BrowserEnrollmentApproval; + let enrollment: StoredEnrollment; + try { + approval = JSON.parse(raw ?? 'null') as BrowserEnrollmentApproval; + enrollment = JSON.parse(enrollmentRaw ?? 'null') as StoredEnrollment; + if (!isBrowserEnrollmentApproval(approval) || canonicalBrowserEnrollmentRedemption(approval) !== canonicalBrowserEnrollmentRedemption(proof) || + enrollment?.workerId !== approval.workerId || enrollment.serverId !== approval.serverId || + enrollment.publicKey !== approval.publicKey || enrollment.pairingGeneration !== 0 || + enrollment.browserEnrollment == null || canonicalBrowserEnrollmentRedemption(enrollment.browserEnrollment) !== canonicalBrowserEnrollmentRedemption(proof) || + JSON.stringify(enrollment.binding) !== JSON.stringify(approval.binding) || + !/^[A-Za-z0-9_-]{24}$/.test(enrollment.identityId) || !/^[A-Za-z0-9_-]{24}$/.test(enrollment.generation)) { + throw new Error('Invalid approval'); + } + } catch { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Browser enrollment is unavailable or does not match its approval'); + } + if (!verifyBrowserEnrollmentRedemption(approval.publicKey, proof, proof.signature)) { + throw new BridgePairingError('PROOF_INVALID', 'Browser enrollment requires the approved machine key'); + } + const credential: BridgeWorkerCredential = { + workerId: approval.workerId, + credential: randomBytes(32).toString('base64url'), + expiresAt: new Date(Date.now() + this.credentialTtlSeconds * 1000).toISOString(), + }; + const credentialDigest = digest(credential.credential); + const stored: StoredCredential = { + workerId: approval.workerId, + identityId: enrollment.identityId, + publicKey: approval.publicKey, + expiresAt: credential.expiresAt, + binding: approval.binding, + enrollmentGeneration: enrollment.generation, + }; + const workerPrefix = `${PREFIX}:worker:${encodeURIComponent(approval.workerId)}`; + const encrypted = await this.redis.eval( + REDEEM_BROWSER_ENROLLMENT_SCRIPT, + 14, + key, + `${key}:enrollment`, + `${key}:fence`, + workerPairingIndexKey(approval.workerId), + workerPairingGenerationKey(approval.workerId), + credentialDigestKey(credentialDigest), + workerIdentityKey(approval.workerId), + workerStableIdentityKey(approval.workerId), + workerEnrollmentKey(approval.workerId), + workerEnrollmentRequiredKey(approval.workerId), + `${key}:result`, + workerPrefix, + `${workerPrefix}:incarnation`, + `${workerPrefix}:registration-generation`, + raw!, + enrollmentRaw!, + String(Date.parse(approval.expiresAt)), + JSON.stringify(stored), + String(this.credentialTtlSeconds), + credentialDigest, + enrollment.identityId, + String(this.recovery.enrollmentTtlSeconds ?? 0), + enrollment.generation, + encryptEnrollmentReplay(credential, raw!, replayKey), + Date.parse(proof.expiresAt) > Date.now() ? '1' : '0', + ); + if (typeof encrypted !== 'string') { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Browser enrollment is expired, revoked, or superseded'); + } + return decryptEnrollmentReplay(encrypted, raw!, replayKey); + } + async limitUntrustedRecovery( peer: string, operation: 'challenge' | 'recover', diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index 5e76876e..a478f623 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -216,6 +216,53 @@ describe('paired bridge HTTP API', () => { }); } + for (const workspaceImportFile of [false, true]) { + test(`registration ${workspaceImportFile ? 'offers' : 'omits'} import_file when grants ${workspaceImportFile ? 'can' : 'cannot'} be minted`, async () => { + const app = express(); + app.use(json()); + app.use( + '/v1/bridge', + createBridgeRouter({ + store: new RedisBridgeStore(redis), + pairings: new RedisBridgePairingStore(redis), + authMode: 'static', + adminToken: 'strong-administrator-bootstrap-token', + configuredWorkerId: 'vm-1', + workspaceImportFile, + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + const response = await fetch(`http://127.0.0.1:${address.port}/v1/bridge/workers/register`, { + method: 'POST', + headers: { + Authorization: 'Bearer strong-administrator-bootstrap-token', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['read_file', 'write_file', 'import_file'], + workspaces: [{ id: 'primary' }], + }, + }, + }), + }); + expect(response.status).toBe(200); + const body = (await response.json()) as { supportedWorkspaceToolOperations: string[] }; + expect(body.supportedWorkspaceToolOperations.includes('import_file')).toBe(workspaceImportFile); + }); + } + test('rejects a malformed optional binding for a configured worker', async () => { const app = express(); app.use(json()); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 123f8c75..c3b5aead 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -6,6 +6,12 @@ import type { BridgeWorkerRegistration } from '../../../packages/code/src/protoc import type { BridgePrincipalType, BridgeWorkerBinding } from './pairing'; import type { CodeBridgeAssignment, CodeBridgeSettlement } from './store'; +import { + BROWSER_ENROLLMENT_PROTOCOL_VERSION, + isBrowserEnrollmentApproval, + isBrowserEnrollmentRedemption, + isBrowserEnrollmentStatusRequest, +} from '../../../packages/code/src/browser-enrollment-protocol'; import { BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, BRIDGE_PROTOCOL_VERSION, @@ -39,6 +45,8 @@ export interface BridgeRouterOptions { configuredWorkerId?: string; allowDynamicWorkers?: boolean; maxCommandTimeoutMs?: number; + /** Offer `import_file`; true only when this Code API can mint its egress grant. */ + workspaceImportFile?: boolean; } function sameToken(left: string, right: string): boolean { @@ -243,6 +251,73 @@ export function createBridgeRouter(options: BridgeRouterOptions): Router { const workerAuth = options.authMode === 'paired' ? pairedWorkerAuth : staticWorkerAuth; + const browserEnrollmentEnabled = options.authMode === 'paired' && + options.pairings.recoveryEnabled && options.allowDynamicWorkers === true && + options.adminToken.length > 0; + + const browserEnrollmentAvailable: RequestHandler = (_req, res, next) => { + if (!browserEnrollmentEnabled) { + res.status(404).json({ error: 'Browser enrollment is disabled' }); + return; + } + next(); + }; + + const sendEnrollmentError = (error: unknown, res: Response): void => { + if (!(error instanceof BridgePairingError)) throw error; + if (error.code === 'RECOVERY_RATE_LIMITED') { + res.set('Retry-After', '60').status(429).json({ error: error.message, code: error.code }); + return; + } + res.status(error.code === 'PROOF_INVALID' ? 401 : 409) + .json({ error: error.message, code: error.code }); + }; + + router.get('/enrollments/capabilities', adminAuth, browserEnrollmentAvailable, (_req, res) => { + res.json({ + browserEnrollmentVersion: BROWSER_ENROLLMENT_PROTOCOL_VERSION, + serverId: options.pairings.recoveryServerId, + }); + }); + + router.post('/enrollments', adminAuth, browserEnrollmentAvailable, asyncRoute(async (req, res) => { + if (!isBrowserEnrollmentApproval(req.body)) { + res.status(400).json({ error: 'Invalid browser enrollment approval' }); + return; + } + try { + res.json(await options.pairings.issueBrowserEnrollment(req.body)); + } catch (error) { + sendEnrollmentError(error, res); + } + })); + + router.post('/enrollments/status', adminAuth, browserEnrollmentAvailable, asyncRoute(async (req, res) => { + if (!isBrowserEnrollmentStatusRequest(req.body)) { + res.status(400).json({ error: 'Invalid browser enrollment status request' }); + return; + } + try { + res.json(await options.pairings.browserEnrollmentStatus(req.body)); + } catch (error) { + sendEnrollmentError(error, res); + } + })); + + router.post('/enrollments/redeem', adminAuth, browserEnrollmentAvailable, asyncRoute(async (req, res) => { + try { + await options.pairings.limitBrowserEnrollmentPeer(req.socket.remoteAddress ?? ''); + if (!isBrowserEnrollmentRedemption(req.body)) { + res.status(400).json({ error: 'Invalid browser enrollment redemption' }); + return; + } + const credential = await options.pairings.redeemBrowserEnrollment(req.body, options.adminToken); + res.json({ protocolVersion: BROWSER_ENROLLMENT_PROTOCOL_VERSION, ...credential }); + } catch (error) { + sendEnrollmentError(error, res); + } + })); + router.post('/pairings', adminAuth, asyncRoute(async (req, res) => { if (options.authMode !== 'paired') { res.status(409).json({ error: 'Paired worker authentication is disabled' }); @@ -533,6 +608,7 @@ router.post( 'preview_edit', 'edit_file', 'execute_command', + ...(options.workspaceImportFile === true ? ['import_file' as const] : []), ], supportedWorkspaceWriteFileModes: ['replace', 'create'], supportedWorkspaceEditFileModes: ['single', 'batch'], diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 0ca74c45..f9919f49 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -15,6 +15,7 @@ import { BRIDGE_PROTOCOL_VERSION, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, + isWorkspaceMutationOperation, isWorkspaceToolRequest, isWorkspaceToolResult, workspaceIsolationKey, @@ -169,7 +170,7 @@ export interface BridgeWorkerStatus { } function isWorkspaceMutation(request: WorkspaceToolRequest): boolean { - return request.operation === 'write_file' || request.operation === 'edit_file' || request.operation === 'execute_command'; + return isWorkspaceMutationOperation(request.operation); } function supportsWorkspaceTool( diff --git a/service/src/config.spec.ts b/service/src/config.spec.ts index 3c2ca9d2..45ae5700 100644 --- a/service/src/config.spec.ts +++ b/service/src/config.spec.ts @@ -1,4 +1,6 @@ import { describe, expect, it } from 'bun:test'; +import { readFileSync } from 'fs'; +import * as path from 'path'; import { jobCompletionWaitTimeoutMs, jobDeadlineAtMs, @@ -22,11 +24,22 @@ describe('node language configuration', () => { it('maps Node.js requests to the node sandbox runtime', () => { expect(languageConfig[Languages.node]).toEqual({ language: 'node', - version: '24.15.0', + version: '24.22.1', fileName: 'index.js', }); }); + it('aligns package installers and Helm with the selected Node runtime', () => { + const root = path.resolve(__dirname, '../..'); + const version = languageConfig[Languages.node]!.version; + for (const file of ['build-packages.sh', 'docker/package-init.sh']) { + expect(readFileSync(path.join(root, file), 'utf8')) + .toContain(`NODE_VERSION="\${NODE_VERSION:-${version}}"`); + } + expect(readFileSync(path.join(root, 'helm/codeapi/values.yaml'), 'utf8')) + .toContain(`nodeVersion: "${version}"`); + }); + it('creates a valid Node.js execution payload', () => { const req = { body: { @@ -45,7 +58,7 @@ describe('node language configuration', () => { expect(payload).toMatchObject({ language: 'node', - version: '24.15.0', + version: '24.22.1', session_id: 'session-node', args: ['--trace-warnings'], files: [ diff --git a/service/src/config.ts b/service/src/config.ts index b3e397b8..6a49ea36 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -7,11 +7,12 @@ import { resolveExecutionProfile, resolveExecutionProfileSource, } from './execution-profile'; +import { BRIDGE_WORKSPACE_IMPORT_FILE_DEFAULT_MAX_BYTES } from '../../packages/code/src/protocol'; export const languageConfig: Record = { [Languages.bash]: { language: 'bash', version: '5.2.0', fileName: 'script.sh' }, [Languages.js]: { language: 'bun-js', version: '1.4.2', fileName: 'index.js' }, - [Languages.node]: { language: 'node', version: '24.15.0', fileName: 'index.js' }, + [Languages.node]: { language: 'node', version: '24.22.1', fileName: 'index.js' }, [Languages.py]: { language: 'python', version: '3.14.4', fileName: 'main.py' }, [Languages.ts]: { language: 'bun-ts', version: '1.4.2', fileName: 'main.ts' }, }; @@ -425,6 +426,14 @@ export const env = { * Off by default: enable only after the LibreChat talking to this service accepts the field. */ BRIDGE_LANE_GIT: process.env.CODEAPI_BRIDGE_LANE_GIT === 'true', + /** + * Largest stored object relayed into a worker by `import_file`. The operation + * is offered only when EGRESS_GATEWAY_URL can mint its one-object grant. + */ + BRIDGE_IMPORT_FILE_MAX_BYTES: resolvePositiveIntEnv( + process.env.CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES, + BRIDGE_WORKSPACE_IMPORT_FILE_DEFAULT_MAX_BYTES, + ), /** Outbound worker selected by the remote-bridge backend. */ BRIDGE_WORKER_ID: process.env.CODEAPI_BRIDGE_WORKER_ID ?? '', /** Static compatibility auth or short-lived proof-of-possession credentials. */ diff --git a/service/src/session-ownership.ts b/service/src/session-ownership.ts index bf4ba6dd..1841ed06 100644 --- a/service/src/session-ownership.ts +++ b/service/src/session-ownership.ts @@ -94,7 +94,7 @@ export type SessionOwnershipResult = * simply gone. */ export async function authorizeSessionOwnership( - store: SessionOwnershipStore, + store: Pick, args: { session_id: string; expectedSessionKey: string; diff --git a/service/src/workspace-tools/import.test.ts b/service/src/workspace-tools/import.test.ts new file mode 100644 index 00000000..7aa22aef --- /dev/null +++ b/service/src/workspace-tools/import.test.ts @@ -0,0 +1,295 @@ +import { createServer } from 'node:http'; +import type { Server } from 'node:http'; + +import { afterEach, expect, test } from 'bun:test'; +import express, { json } from 'express'; +import RedisMock from 'ioredis-mock'; +import type Redis from 'ioredis'; + +import type * as t from '../types'; +import type { CodeApiPrincipal } from '../auth/principal'; +import type { WorkspaceImportFileRequest, WorkspaceToolRequest } from '../../../packages/code/src/protocol'; + +import { applyPrincipal } from '../auth/principal'; +import { RedisBridgeStore } from '../bridge/store'; +import { resolveSessionKey } from '../session-key'; +import { recordSessionOwnership } from '../session-ownership'; +import { openEgressGrant, openEgressHandle, prepareSandboxEgress } from '../egress-grant'; +import { createWorkspaceToolsRouter } from './router'; +import { RedisWorkspaceRequests } from './requests'; +import { + WorkspaceImportError, + authorizeWorkspaceImportSource, + createWorkspaceImportService, +} from './import'; + +const SECRET = 'workspace-import-test-secret-of-sufficient-length'; +const SESSION = 'sessAAAAAAAAAAAAAAAAA'; +const OBJECT = 'fileAAAAAAAAAAAAAAAAA'; +const OTHER_SESSION = 'sessBBBBBBBBBBBBBBBBB'; +const OTHER_OBJECT = 'fileBBBBBBBBBBBBBBBBB'; +const owner: CodeApiPrincipal = { userId: 'user-1', tenantId: 'tenant-1', principalSource: 'librechat_jwt', codeWorkerId: 'worker' }; +const stranger: CodeApiPrincipal = { userId: 'user-2', tenantId: 'tenant-1', principalSource: 'librechat_jwt', codeWorkerId: 'worker' }; + +const importBody = (overrides: Record = {}): WorkspaceImportFileRequest => ({ + protocolVersion: 1, + operation: 'import_file', + workspaceId: 'primary', + path: 'uploads/report.pdf', + source: { storage_session_id: SESSION, id: OBJECT }, + bytes: 4096, + sha256: 'a'.repeat(64), + ...overrides, +}); + +function principalRequest(principal: CodeApiPrincipal): t.AuthenticatedRequest { + const req = { headers: {} } as t.AuthenticatedRequest; + applyPrincipal(req, principal); + return req; +} + +/** Uploads as LibreChat's code-environment upload does: user-kind session key plus upload record. */ +async function upload(redis: Redis, principal: CodeApiPrincipal, sessionId: string, objectId: string): Promise { + const sessionKey = resolveSessionKey(principalRequest(principal), { kind: 'user', id: principal.userId }); + await recordSessionOwnership(redis, sessionId, sessionKey); + await redis.set(`upload:${sessionKey}${sessionId}${objectId}`, '1'); +} + +function importService(redis: Redis, minted: string[] = [], revoked: string[] = [], maxBytes = 1024 * 1024) { + return createWorkspaceImportService({ + maxBytes, + store: redis, + async createGrant({ payload, claims }) { + const grantId = `grant-${minted.length + 1}`; + minted.push(grantId); + return { grant_id: grantId, ...prepareSandboxEgress({ payload, claims, grantId, secret: SECRET }) }; + }, + async revokeGrant(grantId) { + revoked.push(grantId); + }, + }); +} + +// ioredis-mock instances share one keyspace across test files; clear it like the bridge suites do. +const redis = new RedisMock() as unknown as Redis; +let server: Server | undefined; +afterEach(async () => { + server?.close(); + server = undefined; + await redis.flushall(); +}); + +async function listen(app: express.Express): Promise { + server = createServer(app); + await new Promise(resolve => server!.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Missing listener'); + return `http://127.0.0.1:${address.port}`; +} + +function appFor(router: express.Router): express.Express { + const app = express(); + app.use(json()); + app.use((req, _res, next) => { + applyPrincipal(req, req.header('X-Test-User') === 'user-2' ? stranger : owner); + next(); + }); + app.use(router); + return app; +} + +function post(url: string, body: unknown, headers: Record = {}, signal?: AbortSignal): Promise { + return fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json', ...headers }, body: JSON.stringify(body), signal }); +} + +test('ownership follows the download rule and rejects another principal\'s object', async () => { + await upload(redis, owner, SESSION, OBJECT); + await upload(redis, stranger, OTHER_SESSION, OTHER_OBJECT); + const req = principalRequest(owner); + + expect(await authorizeWorkspaceImportSource({ req, store: redis, source: { storage_session_id: SESSION, id: OBJECT } })) + .toBe(resolveSessionKey(req, { kind: 'user', id: 'user-1' })); + const denials = [ + { storage_session_id: OTHER_SESSION, id: OTHER_OBJECT }, + { storage_session_id: SESSION, id: OTHER_OBJECT }, + { storage_session_id: OTHER_SESSION, id: OBJECT }, + ]; + for (const source of denials) { + const error = await authorizeWorkspaceImportSource({ req, store: redis, source }).catch(caught => caught); + expect(error).toBeInstanceOf(WorkspaceImportError); + expect(error).toMatchObject({ status: 403, code: 'IMPORT_SOURCE_UNAUTHORIZED' }); + } + const malformed = await authorizeWorkspaceImportSource({ + req, store: redis, source: { storage_session_id: '../escape', id: OBJECT }, + }).catch(caught => caught); + expect(malformed).toMatchObject({ status: 400, code: 'INVALID_WORKSPACE_TOOL_REQUEST' }); + + await redis.del(`session:${SESSION}`); + const expired = await authorizeWorkspaceImportSource({ req, store: redis, source: { storage_session_id: SESSION, id: OBJECT } }) + .catch(caught => caught); + expect(expired).toMatchObject({ status: 403, code: 'IMPORT_SOURCE_UNAUTHORIZED' }); +}); + +test('the grant reads exactly one object, briefly, and can upload nothing', async () => { + await upload(redis, owner, SESSION, OBJECT); + const imports = importService(redis); + const req = principalRequest(owner); + const sessionKey = await imports.authorize(req, importBody()); + const before = Math.floor(Date.now() / 1000); + const prepared = await imports.prepare({ req, request: importBody(), sessionKey, ttlSeconds: 60 }); + + const grant = openEgressGrant(prepared.request.egressGrant, SECRET); + expect(grant.input_files).toEqual([{ id: OBJECT, session_id: SESSION, name: 'report.pdf' }]); + expect(grant.read_sessions).toEqual([SESSION]); + expect(grant).toMatchObject({ max_upload_bytes: 0, max_output_files: 0, max_requests: 2, session_key: sessionKey }); + expect(grant.exp - before).toBeGreaterThanOrEqual(90); + expect(grant.exp - before).toBeLessThanOrEqual(91); + expect(prepared.request.source.id).not.toBe(OBJECT); + expect(prepared.request.source.storage_session_id).not.toBe(SESSION); + expect(openEgressHandle(prepared.request.source.id, SECRET)).toMatchObject({ typ: 'object', object_id: OBJECT, session_id: SESSION }); + expect(openEgressHandle(prepared.request.source.storage_session_id, SECRET)).toMatchObject({ typ: 'session', dir: 'read', session_id: SESSION }); + const { egressGrant: _grant, source: _source, ...unchanged } = prepared.request; + const { source: _callerSource, ...callerFields } = importBody(); + expect(unchanged).toEqual(callerFields); +}); + +test('a synchronous import relays the granted request and revokes its grant', async () => { + await upload(redis, owner, SESSION, OBJECT); + await upload(redis, stranger, OTHER_SESSION, OTHER_OBJECT); + const minted: string[] = []; + const revoked: string[] = []; + const dispatched: Array<{ request: WorkspaceToolRequest; executionTimeoutMs?: number }> = []; + const url = await listen(appFor(createWorkspaceToolsRouter({ + backend: 'remote-bridge', configuredWorkerId: 'worker', dynamicWorkers: false, timeoutMs: 120_000, + imports: importService(redis, minted, revoked, 8192), + store: { async dispatchWorkspaceTool(args) { + dispatched.push({ request: args.request, executionTimeoutMs: args.executionTimeoutMs }); + const request = args.request as WorkspaceImportFileRequest; + return { protocolVersion: 1, generation: 1, leaseToken: 'lease', incarnationId: 'incarnation', status: 'fulfilled', result: { + protocolVersion: 1, operation: 'import_file', workspaceId: 'primary', path: request.path, + created: true, bytesWritten: request.bytes, sha256: request.sha256, + } }; + } }, + }))); + const endpoint = `${url}/workspace-tools/execute`; + + const response = await post(endpoint, importBody()); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ operation: 'import_file', created: true, bytesWritten: 4096 }); + expect(dispatched).toHaveLength(1); + const relayed = dispatched[0]!.request as WorkspaceImportFileRequest; + expect(typeof relayed.egressGrant).toBe('string'); + expect(relayed.source.id).not.toBe(OBJECT); + expect(dispatched[0]!.executionTimeoutMs).toBe(120_000); + await Bun.sleep(0); + expect(revoked).toEqual(minted); + + const callerGrant = await post(endpoint, { ...importBody(), egressGrant: 'ceg1.forged' }); + expect(callerGrant.status).toBe(400); + const foreign = await post(endpoint, importBody({ source: { storage_session_id: OTHER_SESSION, id: OTHER_OBJECT } })); + expect(foreign.status).toBe(403); + expect(await foreign.json()).toMatchObject({ code: 'IMPORT_SOURCE_UNAUTHORIZED' }); + const stolen = await post(endpoint, importBody(), { 'X-Test-User': 'user-2' }); + expect(stolen.status).toBe(403); + const oversized = await post(endpoint, importBody({ bytes: 8193 })); + expect(oversized.status).toBe(413); + expect(await oversized.json()).toMatchObject({ code: 'WRITE_LIMIT_EXCEEDED' }); + expect(dispatched).toHaveLength(1); + expect(minted).toHaveLength(1); +}); + +test('a caller that disconnects while its grant is minted never dispatches, and the grant is revoked', async () => { + await upload(redis, owner, SESSION, OBJECT); + const minted: string[] = []; + const revoked: string[] = []; + const base = importService(redis, minted, revoked); + const grantRequested = Promise.withResolvers(); + const releaseGrant = Promise.withResolvers(); + let dispatched = 0; + const url = await listen(appFor(createWorkspaceToolsRouter({ + backend: 'remote-bridge', configuredWorkerId: 'worker', dynamicWorkers: false, + imports: { ...base, async prepare(args) { + grantRequested.resolve(); + await releaseGrant.promise; + return base.prepare(args); + } }, + store: { async dispatchWorkspaceTool(): Promise { dispatched++; throw new Error('must not dispatch'); } }, + }))); + const caller = new AbortController(); + const response = post(`${url}/workspace-tools/execute`, importBody(), {}, caller.signal).catch(() => undefined); + await grantRequested.promise; + caller.abort(); + await response; + await Bun.sleep(50); + releaseGrant.resolve(); + for (let attempt = 0; attempt < 100 && revoked.length === 0; attempt++) await Bun.sleep(10); + expect(dispatched).toBe(0); + expect(revoked).toEqual(minted); + expect(minted).toHaveLength(1); +}); + +test('a Code API that cannot mint grants refuses import_file without dispatching', async () => { + let dispatched = 0; + const url = await listen(appFor(createWorkspaceToolsRouter({ + backend: 'remote-bridge', configuredWorkerId: 'worker', dynamicWorkers: false, + store: { async dispatchWorkspaceTool(): Promise { dispatched++; throw new Error('must not dispatch'); } }, + }))); + const response = await post(`${url}/workspace-tools/execute`, importBody()); + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ code: 'IMPORT_UNAVAILABLE' }); + expect(dispatched).toBe(0); +}); + +test('a durable import fingerprints the caller request and mints its grant once', async () => { + await upload(redis, owner, SESSION, OBJECT); + const bridge = new RedisBridgeStore(redis); + await bridge.register({ protocolVersion: 1, workerId: 'worker', incarnationId: 'incarnation-00000001', capabilities: { + sandboxProfile: 'anthropic-srt', statefulWorkspace: false, runtimes: [], + workspaceTools: { protocolVersion: 1, operations: ['read_file', 'import_file'], workspaces: [{ id: 'primary' }] }, + } }); + const minted: string[] = []; + const revoked: string[] = []; + const url = await listen(appFor(createWorkspaceToolsRouter({ + backend: 'remote-bridge', configuredWorkerId: 'worker', dynamicWorkers: false, + imports: importService(redis, minted, revoked), + requests: new RedisWorkspaceRequests(redis, bridge), + store: { async dispatchWorkspaceTool(): Promise { throw new Error('durable requests never dispatch synchronously'); } }, + }))); + const headers = { 'X-LibreChat-Workspace-Request-Id': 'import-request-000001' }; + for (let attempt = 0; attempt < 2; attempt++) { + const response = await post(`${url}/workspace-tools/requests`, importBody(), headers); + expect(response.status).toBe(202); + expect(await response.json()).toMatchObject({ state: 'queued', workspaceId: 'primary' }); + } + expect(minted).toHaveLength(1); + + // A retry of the same request answers from its record even once the upload is gone. + await redis.del(`session:${SESSION}`); + const retried = await post(`${url}/workspace-tools/requests`, importBody(), headers); + expect(retried.status).toBe(202); + expect(minted).toHaveLength(1); + + const keys = (await redis.keys('codeapi:workspace-requests:v1:*')).filter(key => !key.endsWith(':active')); + const records = await Promise.all(keys.map(key => redis.hget(key, 'record'))); + const stored = records.flatMap(record => record == null ? [] : [JSON.parse(record).request as WorkspaceImportFileRequest]); + expect(stored.length).toBeGreaterThan(0); + for (const request of stored) { + expect(openEgressGrant(request.egressGrant!, SECRET).grant_id).toBe(minted[0]); + expect(request.source.id).not.toBe(OBJECT); + } + + // Racing submissions of one new request ID: only the adopted grant survives. + const raceHeaders = { 'X-LibreChat-Workspace-Request-Id': 'import-request-000002' }; + await upload(redis, owner, SESSION, OBJECT); + const before = minted.length; + const raced = await Promise.all([0, 1].map(() => post(`${url}/workspace-tools/requests`, importBody(), raceHeaders))); + expect(raced.map(response => response.status)).toEqual([202, 202]); + const racedGrants = minted.slice(before); + expect(racedGrants.length).toBeGreaterThan(0); + expect(revoked).toHaveLength(racedGrants.length - 1); + + const mintedBeforeConflict = minted.length; + const conflict = await post(`${url}/workspace-tools/requests`, importBody({ bytes: 1 }), headers); + expect(conflict.status).toBe(409); + expect(minted).toHaveLength(mintedBeforeConflict); +}); diff --git a/service/src/workspace-tools/import.ts b/service/src/workspace-tools/import.ts new file mode 100644 index 00000000..6f22b55b --- /dev/null +++ b/service/src/workspace-tools/import.ts @@ -0,0 +1,182 @@ +import { posix } from 'path'; +import { nanoid } from 'nanoid'; + +import type * as t from '../types'; +import type { GatewayPreparedEgress } from '../egress-gateway-client'; +import type { ExecutionManifestClaims } from '../execution-manifest'; +import type { SessionOwnershipStore } from '../session-ownership'; +import type { WorkspaceImportFileRequest } from '../../../packages/code/src/protocol'; + +import { isValidId } from '../utils'; +import { authorizeSessionOwnership } from '../session-ownership'; +import { resolveSessionKey, SessionKeyResolutionError } from '../session-key'; +import { buildExecutionManifestClaims } from '../execution-manifest-claims'; + +/** One download plus one retry of the same object; nothing else is readable. */ +const IMPORT_GRANT_MAX_REQUESTS = 2; +/** Clock-skew slack beyond the request's own admission and execution budget. */ +const IMPORT_GRANT_TTL_SLACK_SECONDS = 30; + +export type WorkspaceImportErrorCode = + | 'INVALID_WORKSPACE_TOOL_REQUEST' + | 'IMPORT_SOURCE_UNAUTHORIZED' + | 'IMPORT_UNAVAILABLE'; + +export class WorkspaceImportError extends Error { + constructor( + readonly status: 400 | 403 | 503, + readonly code: WorkspaceImportErrorCode, + message: string, + ) { + super(message); + this.name = 'WorkspaceImportError'; + } +} + +/** Redis reads used to prove ownership: the session cache key and the upload record. */ +export interface WorkspaceImportSourceStore extends Pick { + exists(key: string): Promise; +} + +export type WorkspaceImportGrantFactory = (args: { + payload: t.PayloadBody; + claims: ExecutionManifestClaims; +}) => Promise; + +export interface PreparedWorkspaceImport { + grantId: string; + request: WorkspaceImportFileRequest & { egressGrant: string }; +} + +export interface WorkspaceImportService { + /** Largest file Code API relays, checked before ownership or dispatch. */ + readonly maxBytes: number; + /** Proves the caller owns `source` and returns the storage session key that owns it. */ + authorize(req: t.AuthenticatedRequest, request: WorkspaceImportFileRequest): Promise; + /** Mints the one-object grant and rewrites `source` into its opaque handles. */ + prepare(args: { + req: t.AuthenticatedRequest; + request: WorkspaceImportFileRequest; + sessionKey: string; + ttlSeconds: number; + }): Promise; + revoke(grantId: string): Promise; +} + +const unauthorized = (): WorkspaceImportError => + new WorkspaceImportError(403, 'IMPORT_SOURCE_UNAUTHORIZED', 'Unauthorized import source'); + +/** + * Applies the `GET /download/:session_id/:fileId` rule for a user-kind object: + * the live session cache key must name the caller's own storage key, and the + * object must have been uploaded under that key and session. + */ +export async function authorizeWorkspaceImportSource(args: { + req: t.AuthenticatedRequest; + source: WorkspaceImportFileRequest['source']; + store: WorkspaceImportSourceStore; +}): Promise { + const { req, source, store } = args; + if (!isValidId(source.storage_session_id) || !isValidId(source.id)) { + throw new WorkspaceImportError(400, 'INVALID_WORKSPACE_TOOL_REQUEST', 'Invalid import source'); + } + const userId = req.codeApiAuthContext?.userId; + if (!userId) throw unauthorized(); + let sessionKey: string; + try { + sessionKey = resolveSessionKey(req, { kind: 'user', id: userId }); + } catch (error) { + if (error instanceof SessionKeyResolutionError && error.status === 400) throw unauthorized(); + throw error; + } + const ownership = await authorizeSessionOwnership(store, { + session_id: source.storage_session_id, + expectedSessionKey: sessionKey, + allowExpiredCache: false, + }); + if (!ownership.authorized) throw unauthorized(); + const uploaded = await store.exists(`upload:${sessionKey}${source.storage_session_id}${source.id}`); + if (uploaded !== 1) throw unauthorized(); + return sessionKey; +} + +/** Claims for a read-only grant over exactly one object, with no upload capacity. */ +export function workspaceImportGrantClaims(args: { + req: t.AuthenticatedRequest; + request: WorkspaceImportFileRequest; + sessionKey: string; + ttlSeconds: number; + nowSeconds?: number; +}): { payload: t.PayloadBody; claims: ExecutionManifestClaims } { + const now = args.nowSeconds ?? Math.floor(Date.now() / 1000); + const payload: t.PayloadBody = { + language: 'workspace-import', + version: '1', + files: [{ + id: args.request.source.id, + storage_session_id: args.request.source.storage_session_id, + name: posix.basename(args.request.path), + }], + }; + const claims = buildExecutionManifestClaims({ + req: args.req, + executionId: `import_${nanoid()}`, + userId: args.req.codeApiAuthContext?.userId ?? '', + sessionKey: args.sessionKey, + outputSessionId: `import_${nanoid()}`, + payload, + nowSeconds: now, + }); + return { + payload, + claims: { + ...claims, + max_upload_bytes: 0, + max_output_files: 0, + max_requests: IMPORT_GRANT_MAX_REQUESTS, + tool_call_socket: false, + exp: now + Math.max(1, Math.ceil(args.ttlSeconds)) + IMPORT_GRANT_TTL_SLACK_SECONDS, + }, + }; +} + +function isPayloadFileRef(file: t.PayloadBody['files'][number] | undefined): file is t.PayloadFileRef { + return file != null && 'id' in file && typeof file.id === 'string' && + typeof file.storage_session_id === 'string'; +} + +export function createWorkspaceImportService(options: { + maxBytes: number; + store: WorkspaceImportSourceStore; + createGrant: WorkspaceImportGrantFactory; + revokeGrant: (grantId: string) => Promise; +}): WorkspaceImportService { + if (!Number.isSafeInteger(options.maxBytes) || options.maxBytes < 1) { + throw new RangeError('Workspace import file limit must be a positive safe integer'); + } + return { + maxBytes: options.maxBytes, + authorize: (req, request) => authorizeWorkspaceImportSource({ req, source: request.source, store: options.store }), + async prepare({ req, request, sessionKey, ttlSeconds }) { + let prepared: GatewayPreparedEgress; + try { + prepared = await options.createGrant(workspaceImportGrantClaims({ req, request, sessionKey, ttlSeconds })); + } catch { + throw new WorkspaceImportError(503, 'IMPORT_UNAVAILABLE', 'Import source grant could not be issued'); + } + const [file] = prepared.payload.files; + if (!isPayloadFileRef(file) || prepared.payload.files.length !== 1 || !prepared.egressGrantToken) { + throw new WorkspaceImportError(503, 'IMPORT_UNAVAILABLE', 'Import source grant is invalid'); + } + return { + grantId: prepared.grant_id, + request: { + ...request, + source: { storage_session_id: file.storage_session_id, id: file.id }, + egressGrant: prepared.egressGrantToken, + }, + }; + }, + revoke: options.revokeGrant, + }; +} diff --git a/service/src/workspace-tools/index.ts b/service/src/workspace-tools/index.ts index 3c7caee2..58c043d9 100644 --- a/service/src/workspace-tools/index.ts +++ b/service/src/workspace-tools/index.ts @@ -9,7 +9,9 @@ import { connection } from '../queue'; import logger from '../logger'; import { checkServiceStartUp, checkServiceShutDown } from '../lifecycle'; import { withSpan } from '../telemetry'; -import { isBridgeEnabled } from '../bridge/enabled'; +import { isBridgeEnabled, isWorkspaceImportEnabled } from '../bridge/enabled'; +import { createWorkspaceImportService } from './import'; +import { createGatewayEgressGrant, revokeGatewayEgressGrant } from '../egress-gateway-client'; import { workspaceRequestCoordinationScope } from './coordination'; const router = Router(); @@ -53,6 +55,14 @@ router.use(['/workspace-tools/execute', '/workspace-tools/requests'], (req, res, if (req.method === 'POST') executionLimiter(req, res, next); else next(); }); +const imports = isWorkspaceImportEnabled() + ? createWorkspaceImportService({ + maxBytes: env.BRIDGE_IMPORT_FILE_MAX_BYTES, + store: connection, + createGrant: args => createGatewayEgressGrant(args), + revokeGrant: grantId => revokeGatewayEgressGrant({ grantId, reason: 'completed' }), + }) + : undefined; router.use( createWorkspaceToolsRouter({ store: bridgeStore, @@ -61,6 +71,7 @@ router.use( configuredWorkerId: env.BRIDGE_WORKER_ID, dynamicWorkers: env.BRIDGE_DYNAMIC_WORKERS, timeoutMs: env.JOB_TIMEOUT, + imports, }), ); diff --git a/service/src/workspace-tools/requests.ts b/service/src/workspace-tools/requests.ts index 2988c7cc..902651fc 100644 --- a/service/src/workspace-tools/requests.ts +++ b/service/src/workspace-tools/requests.ts @@ -99,6 +99,13 @@ export class RedisWorkspaceRequests { request: WorkspaceToolRequest; queueWaitMs: number; executionTimeoutMs: number; + /** + * Derives the request a worker receives, such as an import carrying its + * egress grant. Runs only for a new request ID; the fingerprint, and so + * idempotent retries, cover the caller's request. `release` undoes the + * preparation when the queue does not adopt this submission. + */ + prepare?: () => Promise<{ request: WorkspaceToolRequest; release?: () => Promise }>; }): Promise { if (!WORKSPACE_REQUEST_ID_PATTERN.test(args.requestId)) { throw new BridgeStoreError('ASSIGNMENT_INVALID', 'Invalid durable workspace request ID'); @@ -117,21 +124,31 @@ export class RedisWorkspaceRequests { const registration = await this.bridge.prepareDurableWorkspaceTool({ ...args, tenantId: args.owner.tenantId, }); + const prepared = args.prepare == null ? undefined : await args.prepare(); + const release = async (): Promise => { await prepared?.release?.().catch(() => undefined); }; + const request = prepared?.request ?? args.request; const now = Date.now(); const record: StoredWorkspaceRequest = { id: `durable-${randomBytes(18).toString('base64url')}`, key, fingerprint, workerId: args.workerId, tenantId: args.owner.tenantId, requireTenantBinding: args.requireTenantBinding, - request: args.request, registration, createdAtMs: now, + request, registration, createdAtMs: now, queueDeadlineAtMs: now + args.queueWaitMs, executionTimeoutMs: args.executionTimeoutMs, state: 'queued', }; const queue = new BridgeAdmissionQueue(this.redis); - const accepted = await queue.submit({ - workerId: record.workerId, id: record.id, deadlineAtMs: record.queueDeadlineAtMs, - workspaceId: workspaceAdmissionId(record.request.workspaceId, record.request.workspaceInstanceId, record.request.worktree), - key, activeKey: this.activeKey, fingerprint, record: JSON.stringify(record), retentionMs: RETENTION_MS, - }); + let accepted: Awaited>; + try { + accepted = await queue.submit({ + workerId: record.workerId, id: record.id, deadlineAtMs: record.queueDeadlineAtMs, + workspaceId: workspaceAdmissionId(record.request.workspaceId, record.request.workspaceInstanceId, record.request.worktree), + key, activeKey: this.activeKey, fingerprint, record: JSON.stringify(record), retentionMs: RETENTION_MS, + }); + } catch (error) { + await release(); + throw error; + } + if (accepted !== 'accepted') await release(); if (accepted === 'conflict') throw new WorkspaceRequestConflict('Request ID was already used for different work'); if (accepted === 'full') throw new BridgeStoreError('WORKER_QUEUE_FULL', 'Bridge worker pending request limit reached'); const submitted = (await this.read(key))!; diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index d4d65d14..bf31e272 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -4,14 +4,19 @@ import type { RequestHandler, Response } from 'express'; import type { AuthenticatedRequest } from '../types'; import type { RedisBridgeStore } from '../bridge/store'; import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol'; +import type { PreparedWorkspaceImport, WorkspaceImportService } from './import'; +import logger from '../logger'; import { getWorkspaceToolOutcome } from './outcome'; +import { WorkspaceImportError } from './import'; import { getPrincipalOrReject } from '../auth/principal'; import { BridgeStoreError } from '../bridge/store'; import { checkServiceShutDown } from '../lifecycle'; import { isWorkspaceToolRequest, + isBridgeWorkspaceImportFileRequest, BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, + BRIDGE_WORKSPACE_IMPORT_FILE_TIMEOUT_MS, } from '../../../packages/code/src/protocol'; import { CODEAPI_BRIDGE_WORKER_HEADER, @@ -35,6 +40,8 @@ interface WorkspaceToolsRouterOptions { timeoutMs?: number; queueTimeoutMs?: number; isShuttingDown?: () => boolean; + /** Present only when this Code API can mint egress grants for `import_file`. */ + imports?: WorkspaceImportService; } function asyncRoute(handler: (req: AuthenticatedRequest, res: Response) => Promise): RequestHandler { @@ -82,13 +89,28 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) res.status(503).json({ error: 'Service is shutting down' }); return; } - if (!isWorkspaceToolRequest(req.body)) { + if ( + !isWorkspaceToolRequest(req.body) || + (req.body.operation === 'import_file' && !isBridgeWorkspaceImportFileRequest(req.body)) + ) { outcome.errorCode = 'INVALID_WORKSPACE_TOOL_REQUEST'; res.status(400).json({ error: 'Invalid workspace tool request', }); return; } + if (req.body.operation === 'import_file') { + if (options.imports == null) { + outcome.errorCode = 'IMPORT_UNAVAILABLE'; + res.status(503).json({ error: 'Workspace file import is not available on this Code API', code: 'IMPORT_UNAVAILABLE' }); + return; + } + if (req.body.bytes > options.imports.maxBytes) { + outcome.errorCode = 'WRITE_LIMIT_EXCEEDED'; + res.status(413).json({ error: 'Imported file exceeds the Code API file limit', code: 'WRITE_LIMIT_EXCEEDED' }); + return; + } + } const advertisedQueueWait = req.header(WORKSPACE_QUEUE_WAIT_HEADER); if (advertisedQueueWait !== undefined && !/^[1-9]\d*$/.test(advertisedQueueWait)) { outcome.errorCode = 'INVALID_WORKSPACE_QUEUE_WAIT'; @@ -121,9 +143,14 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) options.timeoutMs ?? Number.MAX_SAFE_INTEGER, ) } : principalRequest; - const executionBudgetMs = request.operation === 'execute_command' - ? request.timeoutMs! + 5_000 - : Math.min(options.timeoutMs ?? 30_000, 30_000); + let executionBudgetMs = Math.min(options.timeoutMs ?? 30_000, 30_000); + if (request.operation === 'execute_command') executionBudgetMs = request.timeoutMs! + 5_000; + if (request.operation === 'import_file') { + executionBudgetMs = Math.min( + options.timeoutMs ?? BRIDGE_WORKSPACE_IMPORT_FILE_TIMEOUT_MS, + BRIDGE_WORKSPACE_IMPORT_FILE_TIMEOUT_MS, + ); + } outcome.deadlineBudgetMs = queueBudgetMs + executionBudgetMs; let selection: { workerId: string; explicit: boolean } | undefined; @@ -152,20 +179,47 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) } outcome.workerId = selection.workerId; + const sendImportError = (error: unknown): boolean => { + if (!(error instanceof WorkspaceImportError)) return false; + outcome.errorCode = error.code; + res.status(error.status).json({ error: error.message, code: error.code }); + return true; + }; + const imports = options.imports; + const importRequest = request.operation === 'import_file' ? request : undefined; + /** Ownership and the grant are settled only for work about to be queued or dispatched, + * so a durable retry of an existing request never depends on the upload still existing. */ + const prepareImport = imports == null || importRequest == null + ? undefined + : async (): Promise => { + const sessionKey = await imports.authorize(req, importRequest); + // The grant must outlive admission and execution, never more. + const ttlSeconds = Math.ceil((queueBudgetMs + executionBudgetMs) / 1000); + return imports.prepare({ req, request: importRequest, sessionKey, ttlSeconds }); + }; + if (durable) { if (options.requests == null) { res.status(404).json({ error: 'Durable workspace requests are unavailable' }); return; } try { + const prepare = prepareImport; const status = await options.requests.submit({ owner: principal, requestId: req.header('X-LibreChat-Workspace-Request-Id') ?? '', workerId: selection.workerId, requireTenantBinding: selection.explicit && (options.dynamicWorkers || selection.workerId !== options.configuredWorkerId), request, queueWaitMs: queueBudgetMs, executionTimeoutMs: executionBudgetMs, + ...(prepare == null || imports == null ? {} : { + prepare: async () => { + const prepared = await prepare(); + return { request: prepared.request, release: () => imports.revoke(prepared.grantId) }; + }, + }), }); res.status(202).json(status); } catch (error) { + if (sendImportError(error)) return; if (error instanceof WorkspaceRequestConflict) { res.status(409).json({ error: error.message, code: 'REQUEST_CONFLICT' }); } else if (error instanceof BridgeStoreError) { @@ -182,7 +236,24 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) if (!res.writableEnded) abort(); }; res.once('close', abortClosedResponse); + // Events that fired before these listeners existed are visible only on the socket. + if (req.socket?.destroyed === true) abort(); + let preparedImport: PreparedWorkspaceImport | undefined; try { + if (prepareImport != null) { + try { + preparedImport = await prepareImport(); + } catch (error) { + if (sendImportError(error)) return; + throw error; + } + // A caller that left while ownership or the grant was settled must not start a mutation. + if (controller.signal.aborted) { + outcome.errorCode = 'CLIENT_DISCONNECTED'; + return; + } + } + const dispatchRequest: WorkspaceToolRequest = preparedImport?.request ?? request; outcome.dispatchPending = true; const dispatchStartedAt = performance.now(); const settlement = await options.store.dispatchWorkspaceTool({ @@ -190,7 +261,7 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) tenantId: principal.tenantId, requireTenantBinding: selection.explicit && (options.dynamicWorkers || selection.workerId !== options.configuredWorkerId), - request, + request: dispatchRequest, deadlineAtMs: Date.now() + queueBudgetMs, executionTimeoutMs: executionBudgetMs, signal: controller.signal, @@ -243,6 +314,12 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) outcome.flush(); req.removeListener('aborted', abort); res.removeListener('close', abortClosedResponse); + if (preparedImport != null && options.imports != null) { + const { grantId } = preparedImport; + void options.imports.revoke(grantId).catch(error => { + logger.warn('Failed to revoke workspace import grant', { grantId, error }); + }); + } } }); router.post('/workspace-tools/execute', execute(false)); diff --git a/tests/compose-bridge-config.cjs b/tests/compose-bridge-config.cjs index bafea2fe..79268b52 100644 --- a/tests/compose-bridge-config.cjs +++ b/tests/compose-bridge-config.cjs @@ -16,6 +16,7 @@ function render(overrides) { CODEAPI_BRIDGE_TOKEN: '', CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '', CODEAPI_BRIDGE_LANE_GIT: '', + CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES: '', CODEAPI_BRIDGE_RECOVERY_SERVER_ID: '', CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS: '', CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: '', @@ -36,6 +37,7 @@ for (const overrides of [ CODEAPI_BRIDGE_WORKER_ID: 'test-worker', CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '4', CODEAPI_BRIDGE_LANE_GIT: 'true', + CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES: '1048576', CODEAPI_BRIDGE_RECOVERY_SERVER_ID: 'https://code.example.test', CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS: '86400', CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: '90', @@ -61,6 +63,10 @@ for (const overrides of [ assert.equal(env.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE, overrides.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE ?? '30'); assert.equal(env.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE, overrides.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE ?? '240'); } + assert.equal( + config.services.api.environment.CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES, + overrides.CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES ?? '104857600', + ); for (const name of ['egress_gateway', 'sandbox-runner']) { assert.equal(config.services[name].environment.CODEAPI_BRIDGE_TOKEN, undefined); assert.equal(config.services[name].environment.CODEAPI_BRIDGE_RECOVERY_SERVER_ID, undefined); diff --git a/tests/package_init_runtime_rollout.sh b/tests/package_init_runtime_rollout.sh index 3bb8cf6c..9c09d542 100755 --- a/tests/package_init_runtime_rollout.sh +++ b/tests/package_init_runtime_rollout.sh @@ -15,6 +15,11 @@ render() { --set workerSandbox.enabled=true \ --set workerSandbox.packages.source=pvc "$@" } +# The hook installs the runtime selected by service requests. +render | grep -A 1 'name: NODE_VERSION' | grep -q 'value: "24.22.1"' +# Explicit runtime selections still override the chart default. +render --set workerSandbox.packages.initJob.nodeVersion=24.22.0 \ + | grep -A 1 'name: NODE_VERSION' | grep -q 'value: "24.22.0"' # Default mutable image must refresh on every pre-install/pre-upgrade hook. render | grep -q 'imagePullPolicy: Always' # Reused historical values must not bypass the runtime migration. diff --git a/tests/test_tool_call_pipes.py b/tests/test_tool_call_pipes.py new file mode 100644 index 00000000..3907ce05 --- /dev/null +++ b/tests/test_tool_call_pipes.py @@ -0,0 +1,36 @@ +import unittest +from unittest.mock import patch + +from tool_call_pipes import wait_for_killed_descendant + + +class DescendantReapingTest(unittest.TestCase): + def test_already_reaped_by_parent(self): + with patch('tool_call_pipes.os.waitpid', side_effect=ChildProcessError), \ + patch('tool_call_pipes.os.path.exists', return_value=False): + wait_for_killed_descendant(123) + + def test_waits_for_adoption(self): + with patch('tool_call_pipes.os.waitpid', side_effect=[ChildProcessError, (123, 9)]), \ + patch('tool_call_pipes.os.path.exists', return_value=True), \ + patch('tool_call_pipes.time.sleep'): + wait_for_killed_descendant(123) + + def test_rejects_wrong_exit_status(self): + with patch('tool_call_pipes.os.waitpid', return_value=(123, 0)): + with self.assertRaises(AssertionError): + wait_for_killed_descendant(123) + + def test_live_nonchild_still_fails_at_deadline(self): + with patch('tool_call_pipes.os.waitpid', side_effect=ChildProcessError), \ + patch('tool_call_pipes.os.path.exists', return_value=True), \ + patch('tool_call_pipes.time.monotonic', side_effect=[0, 0, 6]), \ + patch('tool_call_pipes.time.sleep'), \ + patch('tool_call_pipes.os.kill') as kill: + with self.assertRaisesRegex(AssertionError, 'orphan survived'): + wait_for_killed_descendant(123) + kill.assert_called_once_with(123, 9) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/tool_call_pipes.py b/tests/tool_call_pipes.py index c9388530..df37fc39 100644 --- a/tests/tool_call_pipes.py +++ b/tests/tool_call_pipes.py @@ -36,6 +36,25 @@ def do_POST(self): self.wfile.write(body) +def wait_for_killed_descendant(pid, timeout=5): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + reaped, status = os.waitpid(pid, os.WNOHANG) + except ChildProcessError: + # Adoption is asynchronous, and the original parent may reap it. + # ECHILD alone does not prove that a live descendant has exited. + if not os.path.exists(f'/proc/{pid}'): + return + else: + if reaped: + assert os.waitstatus_to_exitcode(status) == -9 + return + time.sleep(0.01) + os.kill(pid, 9) + raise AssertionError('orphan survived broker death') + + def main(): policy, broker, preamble = sys.argv[1:] server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), Upstream) @@ -148,16 +167,7 @@ async def check(): running.kill() running.communicate(timeout=5) for pid in [job_pid, relay_pid, broker_pid]: - deadline = time.monotonic() + 5 - while time.monotonic() < deadline: - reaped, status = os.waitpid(pid, os.WNOHANG) - if reaped: - assert os.waitstatus_to_exitcode(status) == -9 - break - time.sleep(0.01) - else: - os.kill(pid, 9) - raise AssertionError('orphan survived broker death') + wait_for_killed_descendant(pid) finally: if running.poll() is None: running.kill()