diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml index 7c626767..ad1c36f3 100644 --- a/.github/workflows/images.yml +++ b/.github/workflows/images.yml @@ -33,6 +33,7 @@ on: - javascript-packages.txt - launcher/** - packages/code/src/** + - packages/extraction/** - service/** - shared/** pull_request: @@ -70,6 +71,8 @@ jobs: run: | images=$(jq -c . <<'JSON' [ + {"image": "code-interpreter-extraction", "dockerfile": "packages/extraction/Dockerfile", "target": "runtime", + "description": "Authenticated isolated PDF/DOCX extraction service", "timeout": 15}, {"image": "code-interpreter-api", "dockerfile": "service/Dockerfile", "target": "api", "description": "Code API HTTP server (Compose service api)", "timeout": 30}, {"image": "code-interpreter-worker", "dockerfile": "service/Dockerfile", "target": "worker", diff --git a/packages/extraction/Dockerfile b/packages/extraction/Dockerfile index cfc48c14..0ae3d5ce 100644 --- a/packages/extraction/Dockerfile +++ b/packages/extraction/Dockerfile @@ -6,8 +6,8 @@ COPY packages/extraction/tsconfig.json ./ COPY packages/extraction/src ./src RUN npm run build RUN apt-get update && apt-get install -y --no-install-recommends gcc libc6-dev linux-libc-dev && rm -rf /var/lib/apt/lists/* -COPY packages/extraction/native/guard.c ./guard.c -RUN gcc -O2 -Wall -Wextra -Werror -o guard guard.c +COPY packages/extraction/native/guard.c packages/extraction/native/gateway.c ./ +RUN gcc -O2 -Wall -Wextra -Werror -o guard guard.c && gcc -O2 -Wall -Wextra -Werror -o gateway gateway.c COPY packages/extraction/tests/block-landlock.c ./block-landlock.c RUN gcc -O2 -Wall -Wextra -Werror -o block-landlock block-landlock.c @@ -17,14 +17,16 @@ COPY packages/extraction/worker/requirements.txt /opt/extraction/requirements.tx RUN pip install --no-cache-dir -r /opt/extraction/requirements.txt COPY --from=build /usr/local/bin/node /usr/local/bin/node COPY --from=build --chmod=0555 /build/guard /opt/extraction/guard +COPY --from=build --chmod=0555 /build/gateway /opt/extraction/gateway COPY --from=build --chmod=0555 /build/dist /opt/extraction/dist COPY --chmod=0444 packages/extraction/package.json /opt/extraction/package.json COPY --chmod=0444 packages/extraction/worker/extract.py /opt/extraction/extract.py RUN groupadd -g 10001 extraction && useradd -u 10001 -g extraction -M extraction && \ - mkdir /socket /jobs && chown extraction:extraction /socket /jobs && chmod 0700 /jobs && chmod 0770 /socket + mkdir -p /run/extraction /jobs && chown extraction:extraction /run/extraction /jobs && chmod 0700 /jobs && chmod 0755 /run/extraction USER 10001:10001 WORKDIR /opt/extraction -HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --retries=3 CMD ["node", "dist/health.js"] +HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --retries=3 CMD ["/opt/extraction/gateway", "node", "dist/health.js"] +ENTRYPOINT ["/opt/extraction/gateway"] CMD ["node", "dist/main.js"] FROM runtime AS qualification diff --git a/packages/extraction/compose.yaml b/packages/extraction/compose.yaml index f6bbf9f4..87d90964 100644 --- a/packages/extraction/compose.yaml +++ b/packages/extraction/compose.yaml @@ -1,22 +1,47 @@ services: + extraction-init: + image: ${EXTRACTION_IMAGE:-librechat-extraction:local} + build: + context: ../.. + dockerfile: packages/extraction/Dockerfile + target: runtime + command: [node, dist/provision.js] + user: '10001:10001' + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + network_mode: none + mem_limit: 128m + cpus: 1 + pids_limit: 16 + healthcheck: + disable: true + volumes: + - extraction-token:/run/extraction extraction: image: ${EXTRACTION_IMAGE:-librechat-extraction:local} build: context: ../.. dockerfile: packages/extraction/Dockerfile target: runtime + depends_on: + extraction-init: + condition: service_completed_successfully init: true user: '10001:10001' read_only: true cap_drop: [ALL] security_opt: [no-new-privileges:true] - network_mode: none + networks: [extraction] mem_limit: 768m cpus: 2 pids_limit: 32 tmpfs: - /jobs:rw,noexec,nosuid,nodev,size=96m,nr_inodes=1024,uid=10001,gid=10001,mode=0700 volumes: - - extraction-socket:/socket + - extraction-token:/run/extraction:ro +networks: + extraction: + internal: true volumes: - extraction-socket: + extraction-token: diff --git a/packages/extraction/native/gateway.c b/packages/extraction/native/gateway.c new file mode 100644 index 00000000..a91bbaa7 --- /dev/null +++ b/packages/extraction/native/gateway.c @@ -0,0 +1,63 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__x86_64__) +#define ARCH AUDIT_ARCH_X86_64 +#elif defined(__aarch64__) +#define ARCH AUDIT_ARCH_AARCH64 +#else +#error Unsupported architecture +#endif + +#define DENY(nr) BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, nr, 0, 1), BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM) +int main(int argc, char **argv) { + if (argc < 2 || geteuid() == 0) goto fail; + /* No inherited socket, including a caller-supplied socket on standard IO. */ + for (int fd = 0; fd < 3; fd++) { + struct stat stat; + if (fstat(fd, &stat)) goto fail; + if (S_ISSOCK(stat.st_mode)) { + int null = open("/dev/null", O_RDWR | O_CLOEXEC); + if (null < 0 || dup2(null, fd) < 0) goto fail; + close(null); + } + } + if (syscall(SYS_close_range, 3, ~0U, 0)) goto fail; + struct sock_filter filter[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, arch)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, ARCH, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), +#ifdef __x86_64__ + BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 0x40000000, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), +#endif + DENY(__NR_socketpair), + DENY(__NR_sendmsg), DENY(__NR_recvmsg), DENY(__NR_sendmmsg), DENY(__NR_recvmmsg), + /* Only IP sockets. Docker's default filter remains in force underneath. */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_socket, 0, 5), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, AF_INET, 2, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, AF_INET6, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW) + }; + struct sock_fprog program = {.len = sizeof(filter) / sizeof(filter[0]), .filter = filter}; + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) || prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program)) goto fail; + execvp(argv[1], &argv[1]); +fail: + fputs("EXTRACTION_GATEWAY_UNAVAILABLE\n", stderr); + return 125; +} diff --git a/packages/extraction/native/guard.c b/packages/extraction/native/guard.c index ee1d2cbc..77d11e91 100644 --- a/packages/extraction/native/guard.c +++ b/packages/extraction/native/guard.c @@ -59,6 +59,9 @@ static void restrict_syscalls(void) { BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), #endif DENY(__NR_socket), DENY(__NR_socketpair), DENY(__NR_connect), + DENY(__NR_sendmsg), DENY(__NR_recvmsg), DENY(__NR_sendmmsg), DENY(__NR_recvmmsg), + DENY(__NR_sendto), DENY(__NR_recvfrom), DENY(__NR_setsockopt), DENY(__NR_getsockopt), + DENY(__NR_shutdown), DENY(__NR_getsockname), DENY(__NR_getpeername), DENY(__NR_bind), DENY(__NR_listen), DENY(__NR_accept), DENY(__NR_accept4), DENY(__NR_clone), DENY(__NR_clone3), #ifdef __NR_fork @@ -119,6 +122,11 @@ static void restrict_syscalls(void) { int main(int argc, char **argv) { if (argc != 5 || geteuid() == 0) fail(); /* Paths are supervisor-owned, never supplied in the HTTP protocol. */ + struct stat input, output, errors, null; + if (fstat(0, &input) || fstat(1, &output) || fstat(2, &errors) || stat("/dev/null", &null)) fail(); + if (!S_ISCHR(input.st_mode) || input.st_rdev != null.st_rdev || + !S_ISCHR(errors.st_mode) || errors.st_rdev != null.st_rdev || + !S_ISREG(output.st_mode) || output.st_nlink != 1 || output.st_size != 0) fail(); if (chdir(argv[1])) fail(); pid_t parent = getppid(); if (parent == 1 || prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() != parent) fail(); diff --git a/packages/extraction/src/backend.ts b/packages/extraction/src/backend.ts index a3831c65..b6d3c430 100644 --- a/packages/extraction/src/backend.ts +++ b/packages/extraction/src/backend.ts @@ -46,67 +46,79 @@ export class RestrictedBackend implements Backend { signal: AbortSignal ): Promise { signal.throwIfAborted(); - const child = spawn( - this.runtime.guard, - [job, this.runtime.python, this.runtime.worker, mode], - { - detached: true, - stdio: ['ignore', 'pipe', 'pipe'], - env: {}, - } + const status = await open( + join(job, 'status'), + constants.O_CREAT | + constants.O_EXCL | + constants.O_NOFOLLOW | + constants.O_RDWR, + 0o600 ); - let output = ''; - let count = 0; - let overflow = false; - const kill = () => { - if ( - child.pid && - child.exitCode === null && - child.signalCode === null - ) { - try { - process.kill(-child.pid, 'SIGKILL'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ESRCH') - this.healthy = false; + try { + signal.throwIfAborted(); + const child = spawn( + this.runtime.guard, + [job, this.runtime.python, this.runtime.worker, mode], + { + detached: true, + stdio: ['ignore', status.fd, 'ignore'], + env: {}, } - } - }; - signal.addEventListener('abort', kill, { once: true }); - if (signal.aborted) kill(); - let spawnError = false; - child.on('error', () => { - spawnError = true; - }); - child.stdout.on('data', (chunk: Buffer) => { - count += chunk.length; - if (count > 1024) { - overflow = true; - kill(); - } else output += chunk.toString('ascii'); - }); - child.stderr.on('data', (chunk: Buffer) => { - count += chunk.length; - if (count > 2048) { - overflow = true; - kill(); - } - }); - const code = await new Promise(resolve => - child.once('close', resolve) - ); - signal.removeEventListener('abort', kill); - signal.throwIfAborted(); - if (spawnError || overflow) throw new ExtractionError('UNAVAILABLE'); - if (code !== 0) { - const value = output.trim(); - if (parserErrors.has(value)) - throw new ExtractionError(value as 'INVALID_DOCUMENT'); - throw new ExtractionError( - code === null ? 'RESOURCE_LIMIT' : 'UNAVAILABLE' ); + const kill = () => { + if ( + child.pid && + child.exitCode === null && + child.signalCode === null + ) { + try { + process.kill(-child.pid, 'SIGKILL'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') + this.healthy = false; + } + } + }; + signal.addEventListener('abort', kill, { once: true }); + if (signal.aborted) kill(); + let spawnError = false; + child.on('error', () => { + spawnError = true; + }); + const code = await new Promise(resolve => + child.once('close', resolve) + ); + signal.removeEventListener('abort', kill); + signal.throwIfAborted(); + const stat = await status.stat(); + if ( + spawnError || + !stat.isFile() || + stat.nlink !== 1 || + stat.size > 1024 + ) + throw new ExtractionError('UNAVAILABLE'); + const buffer = Buffer.alloc(1025); + const { bytesRead } = await status.read( + buffer, + 0, + buffer.length, + 0 + ); + if (bytesRead > 1024) throw new ExtractionError('UNAVAILABLE'); + const output = buffer.toString('ascii', 0, bytesRead); + if (code !== 0) { + const value = output.trim(); + if (parserErrors.has(value)) + throw new ExtractionError(value as 'INVALID_DOCUMENT'); + throw new ExtractionError( + code === null ? 'RESOURCE_LIMIT' : 'UNAVAILABLE' + ); + } + return output; + } finally { + await status.close(); } - return output; } async probe(): Promise { const job = await this.create(); @@ -131,7 +143,7 @@ export class RestrictedBackend implements Backend { await this.execute(job, format, signal); const file = await open( join(job, 'result'), - constants.O_RDONLY | constants.O_NOFOLLOW + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK ); try { const stat = await file.stat(); @@ -142,7 +154,11 @@ export class RestrictedBackend implements Backend { ) { throw new ExtractionError('OUTPUT_LIMIT'); } - const bytes = await file.readFile(); + const buffer = Buffer.alloc(stat.size + 1); + const { bytesRead } = await file.read(buffer, 0, buffer.length, 0); + if (bytesRead > policy.resultBytes) + throw new ExtractionError('OUTPUT_LIMIT'); + const bytes = buffer.subarray(0, bytesRead); signal.throwIfAborted(); validateResult(bytes, format, digest); return bytes; diff --git a/packages/extraction/src/config.ts b/packages/extraction/src/config.ts new file mode 100644 index 00000000..f2d94238 --- /dev/null +++ b/packages/extraction/src/config.ts @@ -0,0 +1,9 @@ +import { isIP } from 'node:net'; + +export function endpoint() { + const host = process.env.EXTRACTION_HOST ?? '0.0.0.0'; + const port = process.env.EXTRACTION_PORT ?? '8000'; + if (!isIP(host) || !/^[1-9]\d{0,4}$/.test(port) || Number(port) > 65535) + throw new Error('INVALID_ENDPOINT'); + return { host, port: Number(port) }; +} diff --git a/packages/extraction/src/credential.test.ts b/packages/extraction/src/credential.test.ts new file mode 100644 index 00000000..5298c089 --- /dev/null +++ b/packages/extraction/src/credential.test.ts @@ -0,0 +1,94 @@ +import assert from 'node:assert/strict'; +import { + chmod, + mkdir, + mkdtemp, + readFile, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { test } from 'node:test'; +import { provision } from './provision.js'; +import { readToken } from './credential.js'; +import { endpoint } from './config.js'; + +test('atomic provisioning is concurrent, idempotent and restart safe', async () => { + const root = await mkdtemp(join(tmpdir(), 'credential-')); + await chmod(root, 0o755); + const path = join(root, 'token'); + try { + await Promise.all(Array.from({ length: 8 }, () => provision(path))); + const token = await readToken(path); + assert.match(token, /^[a-f0-9]{64}$/); + await provision(path); + assert.equal(await readToken(path), token); + } finally { + await rm(root, { recursive: true }); + } +}); + +test('existing invalid tokens, symlinks and non-files fail closed without regeneration', async () => { + const root = await mkdtemp(join(tmpdir(), 'credential-')); + await chmod(root, 0o755); + try { + const target = join(root, 'valid'); + await writeFile(target, 'a'.repeat(64), { mode: 0o444 }); + for (const [name, value, mode] of [ + ['short', 'a', 0o444], + ['uppercase', 'A'.repeat(64), 0o444], + ['oversize', 'a'.repeat(4096), 0o444], + ['writable', 'a'.repeat(64), 0o644], + ] as const) { + const path = join(root, name); + await writeFile(path, value, { mode }); + await assert.rejects(provision(path)); + assert.equal(await readFile(path, 'utf8'), value); + } + for (const [name, destination] of [ + ['link', target], + ['dangling', join(root, 'missing')], + ]) { + const path = join(root, name!); + await symlink(destination!, path); + await assert.rejects(provision(path)); + } + const directory = join(root, 'directory'); + await mkdir(directory); + await assert.rejects(readToken(directory)); + } finally { + await rm(root, { recursive: true }); + } +}); + +test('endpoint defaults and deployment overrides are bounded literal IP addresses', () => { + const previous = { + host: process.env.EXTRACTION_HOST, + port: process.env.EXTRACTION_PORT, + }; + try { + delete process.env.EXTRACTION_HOST; + delete process.env.EXTRACTION_PORT; + assert.deepEqual(endpoint(), { host: '0.0.0.0', port: 8000 }); + process.env.EXTRACTION_HOST = '::'; + process.env.EXTRACTION_PORT = '9000'; + assert.deepEqual(endpoint(), { host: '::', port: 9000 }); + for (const port of ['0', '65536', '08', '-1', 'x']) { + process.env.EXTRACTION_PORT = port; + assert.throws(endpoint); + } + process.env.EXTRACTION_PORT = '8000'; + process.env.EXTRACTION_HOST = '/socket/forbidden'; + assert.throws(endpoint); + } finally { + for (const [key, value] of [ + ['EXTRACTION_HOST', previous.host], + ['EXTRACTION_PORT', previous.port], + ]) { + if (value === undefined) delete process.env[key!]; + else process.env[key!] = value; + } + } +}); diff --git a/packages/extraction/src/credential.ts b/packages/extraction/src/credential.ts new file mode 100644 index 00000000..afee844f --- /dev/null +++ b/packages/extraction/src/credential.ts @@ -0,0 +1,25 @@ +import { constants } from 'node:fs'; +import { open } from 'node:fs/promises'; + +export const tokenFile = () => + process.env.EXTRACTION_TOKEN_FILE ?? '/run/extraction/token'; + +export async function readToken(path: string = tokenFile()): Promise { + const file = await open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK + ); + try { + const stat = await file.stat(); + if (!stat.isFile() || stat.size !== 64 || (stat.mode & 0o777) !== 0o444) + throw new Error('INVALID_TOKEN_FILE'); + const bytes = Buffer.alloc(65); + const { bytesRead } = await file.read(bytes, 0, bytes.length, 0); + const token = bytes.toString('utf8', 0, bytesRead); + if (bytesRead !== 64 || !/^[a-f0-9]{64}$/.test(token)) + throw new Error('INVALID_TOKEN_FILE'); + return token; + } finally { + await file.close(); + } +} diff --git a/packages/extraction/src/health.ts b/packages/extraction/src/health.ts index 641575c4..fa18be07 100644 --- a/packages/extraction/src/health.ts +++ b/packages/extraction/src/health.ts @@ -1,15 +1,26 @@ import { request } from 'node:http'; -const req = request( - { - socketPath: '/socket/extraction.sock', - path: '/v1/capabilities', - timeout: 1000, - }, - res => { - res.resume(); - res.once('end', () => process.exit(res.statusCode === 200 ? 0 : 1)); - } -); -req.on('timeout', () => req.destroy()); -req.on('error', () => process.exit(1)); -req.end(); +import { endpoint } from './config.js'; +import { readToken } from './credential.js'; + +try { + const { host, port } = endpoint(); + const req = request( + { + host: + host === '::' ? '::1' : host === '0.0.0.0' ? '127.0.0.1' : host, + port, + path: '/v1/capabilities', + headers: { Authorization: `Bearer ${await readToken()}` }, + timeout: 1000, + }, + res => { + res.resume(); + res.once('end', () => process.exit(res.statusCode === 200 ? 0 : 1)); + } + ); + req.on('timeout', () => req.destroy()); + req.on('error', () => process.exit(1)); + req.end(); +} catch { + process.exit(1); +} diff --git a/packages/extraction/src/main.ts b/packages/extraction/src/main.ts index 424a6ed6..2d103a81 100644 --- a/packages/extraction/src/main.ts +++ b/packages/extraction/src/main.ts @@ -1,9 +1,9 @@ -import { chmod, lstat, readdir, unlink } from 'node:fs/promises'; -import { connect } from 'node:net'; +import { readdir } from 'node:fs/promises'; import { RestrictedBackend } from './backend.js'; +import { endpoint } from './config.js'; +import { readToken } from './credential.js'; import { extractionServer } from './server.js'; -// Deployment-owned locations, not caller-selected paths or runtimes. const backend = new RestrictedBackend({ jobs: '/jobs', guard: '/opt/extraction/guard', @@ -11,46 +11,16 @@ const backend = new RestrictedBackend({ worker: '/opt/extraction/extract.py', }); try { + const address = endpoint(); + const token = await readToken(); if ((await readdir('/jobs')).length) throw new Error('DIRTY_SCRATCH'); - try { - const stat = await lstat('/socket/extraction.sock'); - if (!stat.isSocket()) throw new Error('INVALID_SOCKET'); - await new Promise((resolve, reject) => { - const client = connect('/socket/extraction.sock'); - client.setTimeout(1000, () => { - client.destroy(); - reject(new Error('SOCKET_EXISTS')); - }); - client.once('connect', () => { - client.destroy(); - reject(new Error('SOCKET_EXISTS')); - }); - client.once('error', error => { - if ((error as NodeJS.ErrnoException).code === 'ECONNREFUSED') - resolve(); - else reject(error); - }); - }); - await unlink('/socket/extraction.sock'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - } await backend.probe(); - const server = extractionServer(backend); + const server = extractionServer(backend, token); server.on('error', () => { console.error('EXTRACTION_UNAVAILABLE'); process.exit(1); }); - process.umask(0o007); - server.listen('/socket/extraction.sock', async () => { - try { - await chmod('/socket/extraction.sock', 0o660); - } catch { - console.error('EXTRACTION_UNAVAILABLE'); - process.exit(1); - } - console.log('Extraction service ready'); - }); + server.listen(address, () => console.log('Extraction service ready')); const shutdown = () => { server.close(); server.closeAllConnections(); diff --git a/packages/extraction/src/provision.ts b/packages/extraction/src/provision.ts new file mode 100644 index 00000000..0bdc329b --- /dev/null +++ b/packages/extraction/src/provision.ts @@ -0,0 +1,70 @@ +import { randomBytes } from 'node:crypto'; +import { constants } from 'node:fs'; +import { link, lstat, open, unlink } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { readToken, tokenFile } from './credential.js'; + +export async function provision(path: string = tokenFile()): Promise { + try { + await readToken(path); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + const directory = dirname(path); + const stat = await lstat(directory); + if (!stat.isDirectory() || (stat.mode & 0o777) !== 0o755) + throw new Error('INVALID_TOKEN_DIRECTORY'); + const temporary = join( + directory, + `.token-${randomBytes(16).toString('hex')}` + ); + const file = await open( + temporary, + constants.O_WRONLY | + constants.O_CREAT | + constants.O_EXCL | + constants.O_NOFOLLOW, + 0o600 + ); + try { + try { + await file.writeFile(randomBytes(32).toString('hex'), 'ascii'); + await file.chmod(0o444); + await file.sync(); + } finally { + await file.close(); + } + try { + // Publish complete bytes without replacing a concurrent provisioner's token. + await link(temporary, path); + const directoryFile = await open( + directory, + constants.O_RDONLY | + constants.O_DIRECTORY | + constants.O_NOFOLLOW + ); + try { + await directoryFile.sync(); + } finally { + await directoryFile.close(); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + await readToken(path); + } finally { + await unlink(temporary); + } +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + provision().catch(() => { + console.error('EXTRACTION_PROVISION_UNAVAILABLE'); + process.exitCode = 1; + }); +} diff --git a/packages/extraction/src/server.test.ts b/packages/extraction/src/server.test.ts index 60f9bc63..8aa55c37 100644 --- a/packages/extraction/src/server.test.ts +++ b/packages/extraction/src/server.test.ts @@ -27,15 +27,15 @@ async function fixture(deadline = 500) { await rm(job, { recursive: true }); }, }; - const server = extractionServer(backend, deadline); - const socketPath = join(root, 'service.sock'); - server.listen(socketPath); + const server = extractionServer(backend, 'a'.repeat(64), deadline); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const port = (server.address() as import('node:net').AddressInfo).port; return { backend, events, root, - socketPath, + port, server, close: async () => { server.closeAllConnections(); @@ -45,23 +45,25 @@ async function fixture(deadline = 500) { }; } const headers = { + Authorization: `Bearer ${'a'.repeat(64)}`, 'Content-Type': 'application/octet-stream', 'Content-Length': '4', 'X-Extraction-Version': '1', 'X-Extraction-Format': 'pdf', }; -async function send( - socketPath: string, - overrides = {}, - path = '/v1/extract-text' -) { +async function send(port: number, overrides = {}, path = '/v1/extract-text') { return new Promise<{ status: number; body: string }>((resolve, reject) => { const req = request( { - socketPath, + host: '127.0.0.1', + port, path, - method: 'POST', - headers: { ...headers, ...overrides }, + method: path === '/v1/capabilities' ? 'GET' : 'POST', + headers: Object.fromEntries( + Object.entries({ ...headers, ...overrides }).filter( + ([, value]) => value !== undefined + ) + ), }, res => { let body = ''; @@ -78,9 +80,9 @@ async function send( test('binary upload executes once and cleanup completes before success', async () => { const f = await fixture(); try { - assert.equal((await send(f.socketPath)).status, 200); + assert.equal((await send(f.port)).status, 200); assert.deepEqual(f.events, ['create', 'run', 'cleanup']); - assert.deepEqual(await readdir(f.root), ['service.sock']); + assert.deepEqual(await readdir(f.root), []); } finally { await f.close(); } @@ -94,15 +96,15 @@ test('closed parameters and declared input limit are rejected before admission', { 'X-Extraction-Url': 'https://example.com' }, { 'Content-Type': 'text/plain' }, ]) { - assert.equal((await send(f.socketPath, changes)).status, 400); + assert.equal((await send(f.port, changes)).status, 400); } assert.equal( - (await send(f.socketPath, {}, '/v1/extract-text?path=x')).status, + (await send(f.port, {}, '/v1/extract-text?path=x')).status, 404 ); assert.equal( ( - await send(f.socketPath, { + await send(f.port, { 'Content-Length': String(policy.inputBytes + 1), }) ).status, @@ -120,8 +122,8 @@ test('cleanup failure suppresses success and quarantines backend', async () => { throw new ExtractionError('UNAVAILABLE'); }; try { - assert.equal((await send(f.socketPath)).status, 503); - assert.equal((await send(f.socketPath)).status, 503); + assert.equal((await send(f.port)).status, 503); + assert.equal((await send(f.port)).status, 503); assert.equal(f.events.filter(e => e === 'run').length, 1); } finally { await f.close(); @@ -146,7 +148,7 @@ test('deadline aborts execution and waits for its close before cleanup', async ( return Buffer.alloc(0); }; try { - assert.equal((await send(f.socketPath)).status, 504); + assert.equal((await send(f.port)).status, 504); assert.deepEqual(f.events, ['create', 'closed', 'cleanup']); } finally { await f.close(); @@ -157,7 +159,8 @@ test('upload disconnect and stalled upload both release scratch and admission', try { for (const disconnect of [true, false]) { const req = request({ - socketPath: f.socketPath, + host: '127.0.0.1', + port: f.port, path: '/v1/extract-text', method: 'POST', headers, @@ -168,10 +171,10 @@ test('upload disconnect and stalled upload both release scratch and admission', await new Promise(resolve => setTimeout(resolve, 20)); if (disconnect) req.destroy(); await new Promise(resolve => setTimeout(resolve, 80)); - assert.deepEqual(await readdir(f.root), ['service.sock']); + assert.deepEqual(await readdir(f.root), []); } assert.equal(f.events.includes('run'), false); - assert.equal((await send(f.socketPath)).status, 200); + assert.equal((await send(f.port)).status, 200); } finally { await f.close(); } @@ -188,11 +191,11 @@ test('two active jobs bound concurrency without a queue', async () => { return Buffer.alloc(0); }; try { - const a = send(f.socketPath), - b = send(f.socketPath); + const a = send(f.port), + b = send(f.port); while (active !== 2) await new Promise(resolve => setTimeout(resolve, 5)); - assert.equal((await send(f.socketPath)).status, 429); + assert.equal((await send(f.port)).status, 429); assert.equal((await a).status, 504); assert.equal((await b).status, 504); } finally { @@ -205,7 +208,8 @@ test('deadline closes stalled output readers while keeping their slots admitted' const clients = [0, 1].map(() => { const req = request( { - socketPath: f.socketPath, + host: '127.0.0.1', + port: f.port, path: '/v1/extract-text', method: 'POST', headers, @@ -222,10 +226,10 @@ test('deadline closes stalled output readers while keeping their slots admitted' try { while (f.events.filter(e => e === 'cleanup').length !== 2) await new Promise(resolve => setTimeout(resolve, 5)); - assert.equal((await send(f.socketPath)).status, 429); + assert.equal((await send(f.port)).status, 429); await new Promise(resolve => setTimeout(resolve, 300)); f.backend.run = async () => Buffer.from('{}'); - assert.equal((await send(f.socketPath)).status, 200); + assert.equal((await send(f.port)).status, 200); } finally { clients.forEach(client => client.destroy()); await f.close(); @@ -236,7 +240,7 @@ test('duplicate extraction control headers are rejected before worker creation', try { assert.equal( ( - await send(f.socketPath, { + await send(f.port, { 'X-Extraction-Format': ['pdf', 'pdf'], }) ).status, @@ -254,13 +258,95 @@ test('unsupported document content returns a safe failure after cleanup', async throw new ExtractionError('UNSUPPORTED_CONTENT'); }; try { - const response = await send(f.socketPath); + const response = await send(f.port); assert.equal(response.status, 422); assert.deepEqual(JSON.parse(response.body), { error: { code: 'UNSUPPORTED_CONTENT' }, }); assert.deepEqual(f.events, ['create', 'cleanup']); - assert.deepEqual(await readdir(f.root), ['service.sock']); + assert.deepEqual(await readdir(f.root), []); + } finally { + await f.close(); + } +}); + +test('authentication fails before validation, health checks or admission', async () => { + const f = await fixture(); + f.backend.healthy = false; + try { + for (const authorization of [ + undefined, + '', + `Bearer ${'b'.repeat(64)}`, + `Bearer ${'A'.repeat(64)}`, + [`Bearer ${'a'.repeat(64)}`, `Bearer ${'a'.repeat(64)}`], + ]) { + for (const path of ['/v1/capabilities', '/v1/extract-text']) { + const response = await send( + f.port, + { + Authorization: authorization, + 'X-Extraction-Format': 'invalid', + }, + path + ); + assert.equal(response.status, 401); + assert.deepEqual(JSON.parse(response.body), { + error: { code: 'UNAUTHORIZED' }, + }); + } + } + assert.deepEqual(f.events, []); + const response = await new Promise((resolve, reject) => { + const req = request( + { + host: '127.0.0.1', + port: f.port, + path: '/v1/capabilities', + headers: { Authorization: headers.Authorization }, + }, + res => { + res.resume(); + res.on('end', () => resolve(res.statusCode!)); + } + ); + req.on('error', reject); + req.end(); + }); + assert.equal(response, 503); + } finally { + await f.close(); + } +}); + +test('unauthenticated expectations and stalled bodies close without admission', async () => { + const f = await fixture(); + try { + for (const expectation of [undefined, '100-continue', 'unsupported']) { + await new Promise((resolve, reject) => { + const req = request( + { + host: '127.0.0.1', + port: f.port, + method: 'POST', + path: '/v1/extract-text', + headers: { + 'Content-Length': '1000', + ...(expectation ? { Expect: expectation } : {}), + }, + }, + res => { + assert.equal(res.statusCode, 401); + assert.equal(res.headers.connection, 'close'); + res.resume(); + res.once('end', resolve); + } + ); + req.on('error', reject); + req.flushHeaders(); + }); + } + assert.deepEqual(f.events, []); } finally { await f.close(); } diff --git a/packages/extraction/src/server.ts b/packages/extraction/src/server.ts index b3bd37c8..54428438 100644 --- a/packages/extraction/src/server.ts +++ b/packages/extraction/src/server.ts @@ -1,4 +1,4 @@ -import { createHash } from 'node:crypto'; +import { createHash, timingSafeEqual } from 'node:crypto'; import { createServer } from 'node:http'; import { open } from 'node:fs/promises'; import { join } from 'node:path'; @@ -95,8 +95,31 @@ function requestFormat(req: IncomingMessage): Format { } export function extractionServer( backend: Backend, + token: string, deadlineMs: number = policy.deadlineMs ) { + if (!/^[a-f0-9]{64}$/.test(token)) throw new Error('INVALID_TOKEN'); + const expected = Buffer.from(`Bearer ${token}`, 'ascii'); + function authorized(req: IncomingMessage): boolean { + let count = 0; + for (let i = 0; i < req.rawHeaders.length; i += 2) + if (req.rawHeaders[i]!.toLowerCase() === 'authorization') count++; + const value = req.headers.authorization; + if ( + count !== 1 || + typeof value !== 'string' || + value.length !== expected.length + ) + return false; + const actual = Buffer.from(value, 'utf8'); + return ( + actual.length === expected.length && + timingSafeEqual(actual, expected) + ); + } + function unauthorized(res: ServerResponse): void { + reply(res, 401, Buffer.from('{"error":{"code":"UNAUTHORIZED"}}')); + } if ( !Number.isInteger(deadlineMs) || deadlineMs <= 0 || @@ -120,12 +143,18 @@ export function extractionServer( server.requestTimeout = policy.deadlineMs; server.setTimeout(policy.deadlineMs, socket => socket.destroy()); server.on('checkContinue', (req, res) => { + if (!authorized(req)) return unauthorized(res); + reply(res, 400, Buffer.from('{"error":{"code":"INVALID_REQUEST"}}')); + }); + server.on('checkExpectation', (req, res) => { + if (!authorized(req)) return unauthorized(res); reply(res, 400, Buffer.from('{"error":{"code":"INVALID_REQUEST"}}')); }); async function handle( req: IncomingMessage, res: ServerResponse ): Promise { + if (!authorized(req)) return unauthorized(res); if (req.method === 'GET' && req.url === '/v1/capabilities') { reply( res, diff --git a/packages/extraction/tests/block-landlock.c b/packages/extraction/tests/block-landlock.c index 1185ec95..c4f5fbee 100644 --- a/packages/extraction/tests/block-landlock.c +++ b/packages/extraction/tests/block-landlock.c @@ -7,15 +7,18 @@ #include /* Negative startup probe: add a restriction to Docker's existing filter. */ -int main(void) { +int main(int argc, char **argv) { + (void)argv; + int gateway = argc == 2; struct sock_filter filter[] = { BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_landlock_create_ruleset, 0, 1), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, gateway ? SYS_prctl : SYS_landlock_create_ruleset, 0, 1), BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | 38), BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW) }; struct sock_fprog program = {.len = sizeof(filter) / sizeof(filter[0]), .filter = filter}; if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) || prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program)) return 125; - execl("/usr/local/bin/node", "node", "/opt/extraction/dist/main.js", (char *)NULL); + if (gateway) execl("/opt/extraction/gateway", "gateway", "node", "dist/main.js", (char *)NULL); + else execl("/usr/local/bin/node", "node", "/opt/extraction/dist/main.js", (char *)NULL); return 125; } diff --git a/packages/extraction/tests/container-client.mjs b/packages/extraction/tests/container-client.mjs index 14e70f74..d053a9e9 100644 --- a/packages/extraction/tests/container-client.mjs +++ b/packages/extraction/tests/container-client.mjs @@ -3,21 +3,27 @@ import { readFileSync } from 'node:fs'; import { request } from 'node:http'; import { createHash } from 'node:crypto'; -const socketPath = '/socket/extraction.sock'; +const host = process.env.EXTRACTION_HOST ?? 'extraction'; +const token = readFileSync('/run/extraction/token', 'ascii'); function send(format, bytes, path = '/v1/extract-text', extra = {}) { return new Promise((resolve, reject) => { const req = request( { - socketPath, + host, + port: 8000, path, - method: 'POST', - headers: { - 'Content-Type': 'application/octet-stream', - 'Content-Length': bytes.length, - 'X-Extraction-Version': '1', - 'X-Extraction-Format': format, - ...extra, - }, + method: path === '/v1/capabilities' ? 'GET' : 'POST', + family: 4, + headers: Object.fromEntries( + Object.entries({ + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/octet-stream', + 'Content-Length': bytes.length, + 'X-Extraction-Version': '1', + 'X-Extraction-Format': format, + ...extra, + }).filter(([, value]) => value !== undefined) + ), }, res => { const chunks = []; @@ -37,6 +43,15 @@ function send(format, bytes, path = '/v1/extract-text', extra = {}) { req.end(bytes); }); } +for (const Authorization of [undefined, `Bearer ${'0'.repeat(64)}`]) { + for (const path of ['/v1/capabilities', '/v1/extract-text']) { + const response = await send('pdf', Buffer.from('test'), path, { + Authorization, + }); + assert.equal(response.status, 401); + assert.deepEqual(response.value, { error: { code: 'UNAUTHORIZED' } }); + } +} const fixtures = '/tests/fixtures'; const pages = readFileSync(`${fixtures}/pages.pdf`); const result = await send('pdf', pages); diff --git a/packages/extraction/tests/container.sh b/packages/extraction/tests/container.sh index 839670c5..520b7805 100755 --- a/packages/extraction/tests/container.sh +++ b/packages/extraction/tests/container.sh @@ -13,40 +13,65 @@ cleanup() { trap cleanup EXIT "${compose[@]}" build extraction docker build --target qualification -t "$image" -f packages/extraction/Dockerfile . -"${compose[@]}" up -d --wait --wait-timeout 30 extraction -id=$("${compose[@]}" ps -q extraction) -volume="${project}_extraction-socket" -# Inspect exactly the shipped recipe, not a relaxed test variant. -docker inspect "$id" | python3 -c ' +"${compose[@]}" up -d --scale extraction=2 --wait --wait-timeout 30 extraction +mapfile -t ids < <("${compose[@]}" ps -q extraction) +[ "${#ids[@]}" -eq 2 ] +volume="${project}_extraction-token" +network="${project}_extraction" +docker network inspect "$network" | python3 -c 'import json,sys; assert json.load(sys.stdin)[0]["Internal"]' +for id in "${ids[@]}"; do + # Inspect the shipped recipe, not a relaxed test variant. + docker inspect "$id" | python3 -c ' import json,sys c=json.load(sys.stdin)[0]; h=c["HostConfig"] -assert h["NetworkMode"] == "none" and h["ReadonlyRootfs"] and not h["Privileged"] +assert h["ReadonlyRootfs"] and not h["Privileged"] assert h["CapDrop"] == ["ALL"] and h["PidsLimit"] == 32 assert h["Memory"] == 768*1024*1024 and h["NanoCpus"] == 2*10**9 assert "no-new-privileges:true" in h["SecurityOpt"] assert c["Config"]["User"] == "10001:10001" -assert len(c["Mounts"]) == 1 and c["Mounts"][0]["Destination"] == "/socket" +assert c["Config"]["Entrypoint"] == ["/opt/extraction/gateway"] +assert len(c["NetworkSettings"]["Networks"]) == 1 and not h["PortBindings"] +assert len(c["Mounts"]) == 1 and c["Mounts"][0]["Destination"] == "/run/extraction" and not c["Mounts"][0]["RW"] assert set(h["Tmpfs"]) == {"/jobs"} for flag in ("noexec", "nosuid", "nodev", "size=96m", "nr_inodes=1024", "uid=10001", "gid=10001", "mode=0700"): assert flag in h["Tmpfs"]["/jobs"] -assert not c["Config"]["ExposedPorts"] if "ExposedPorts" in c["Config"] else True -print("Shipped Compose mounts, resource limits and security options verified")' -docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ - --mount "type=volume,src=$volume,dst=/socket,readonly" "$image" \ - sh -c 'python /tests/adversarial.py && node /tests/container-client.mjs' -docker exec "$id" node -e 'require("node:fs").readdir("/jobs",(e,v)=>{if(e||v.length)process.exit(1); console.log("Endpoint scratch cleanup verified")})' -# Same outer boundary, with a test-only worker that attempts forbidden operations. +assert not c["Config"].get("ExposedPorts") +print("Shipped Compose mounts, private network, resource limits and security options verified")' + docker exec "$id" /opt/extraction/gateway node dist/health.js + docker exec "$id" node -e 'require("node:fs").readdir("/jobs",(e,v)=>{if(e||v.length)process.exit(1)})' +done +# One-shot init is idempotent; arbitrary caller UIDs can read but not write. +before=$(docker exec "${ids[0]}" node -e 'console.log(require("node:fs").statSync("/run/extraction/token").ino)') +"${compose[@]}" run --rm --no-deps extraction-init +after=$(docker exec "${ids[0]}" node -e 'console.log(require("node:fs").statSync("/run/extraction/token").ino)') +[ "$before" = "$after" ] +docker run --rm --user 12345:12345 --network none --cap-drop ALL --security-opt no-new-privileges \ + --mount "type=volume,src=$volume,dst=/run/extraction,readonly" "$image" \ + node --input-type=module -e 'import {readToken} from "/opt/extraction/dist/credential.js"; await readToken(); console.log("Cross-UID credential read verified")' +docker run --rm --init --network "$network" --cap-drop ALL --security-opt no-new-privileges \ + --mount "type=volume,src=$volume,dst=/run/extraction,readonly" "$image" \ + sh -c 'python /tests/gateway.py && node --input-type=module -e '\''import {lookup, resolve4} from "node:dns/promises"; const a=await lookup("extraction", {all:true, family:4}); const b=await resolve4("extraction"); if(a.length !== 2 || b.length !== 2) process.exit(1); console.log("Node IPv4 libc and c-ares DNS qualified for two replicas")'\'' && python /tests/adversarial.py && node /tests/container-client.mjs' +for id in "${ids[@]}"; do + docker exec "$id" node -e 'require("node:fs").readdir("/jobs",(e,v)=>{if(e||v.length)process.exit(1); console.log("Endpoint scratch cleanup verified")})' +done +# Same outer boundary; the worker independently exercises its stronger filter. docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ --read-only --memory 768m --cpus 2 --pids-limit 32 \ --tmpfs /jobs:rw,noexec,nosuid,nodev,size=96m,nr_inodes=1024,uid=10001,gid=10001,mode=0700 \ - --tmpfs /socket:rw,noexec,nosuid,nodev,size=1m,uid=10001,gid=10001,mode=0700 \ + --mount "type=volume,src=$volume,dst=/run/extraction,readonly" \ "$image" node /tests/isolation.mjs -# Denying the mandatory isolation syscall must fail closed, never start unsandboxed. +# A test controller without the gateway supplies socket stdio deliberately. +# Both native launchers must reject/remove it before running their target. +docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ + --read-only --memory 768m --cpus 2 --pids-limit 32 \ + --tmpfs /jobs:rw,noexec,nosuid,nodev,size=96m,uid=10001,gid=10001,mode=0700 \ + --entrypoint node "$image" /tests/stdio.mjs +# Mandatory isolation unavailable: fail closed, never start unsandboxed. set +e docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ --read-only --memory 768m --cpus 2 --pids-limit 32 \ --tmpfs /jobs:rw,noexec,nosuid,nodev,size=96m,uid=10001,gid=10001,mode=0700 \ - --tmpfs /socket:rw,noexec,nosuid,nodev,size=1m,uid=10001,gid=10001,mode=0700 \ + --mount "type=volume,src=$volume,dst=/run/extraction,readonly" \ "$image" /tests/block-landlock > /dev/null 2>&1 blocked=$? set -e @@ -55,7 +80,17 @@ if [ "$blocked" -ne 1 ]; then exit 1 fi printf 'Unavailable isolation fails closed\n' -# A stale socket left by SIGKILL must not prevent a safe restart. -docker kill -s KILL "$id" >/dev/null -"${compose[@]}" up -d --wait --wait-timeout 30 extraction -printf 'Hardened container restart passed\n' +set +e +docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ + --read-only --memory 128m --pids-limit 16 \ + "$image" /tests/block-landlock gateway > /dev/null 2>&1 +blocked=$? +set -e +[ "$blocked" -eq 125 ] +printf 'Unavailable pre-runtime gateway fails closed\n' +# Restart uses the same credential and fresh per-replica scratch. +for id in "${ids[@]}"; do docker kill -s KILL "$id" >/dev/null; done +"${compose[@]}" up -d --scale extraction=2 --wait --wait-timeout 30 extraction +after=$(docker exec "${ids[0]}" node -e 'console.log(require("node:fs").statSync("/run/extraction/token").ino)') +[ "$before" = "$after" ] +printf 'Hardened two-replica restart and credential persistence passed\n' diff --git a/packages/extraction/tests/gateway.py b/packages/extraction/tests/gateway.py new file mode 100644 index 00000000..0d947345 --- /dev/null +++ b/packages/extraction/tests/gateway.py @@ -0,0 +1,21 @@ +"""Probe the pre-runtime guard independently of the stronger worker filter.""" +import ctypes +import errno +import socket +import os +import stat + +libc = ctypes.CDLL(None, use_errno=True) +for fd in (0, 1, 2): + assert not stat.S_ISSOCK(os.fstat(fd).st_mode) +for name, args in [ + ('socket', (socket.AF_UNIX, socket.SOCK_STREAM, 0)), + ('socketpair', (socket.AF_UNIX, socket.SOCK_STREAM, 0, 0)), + ('sendmsg', (-1, 0, 0)), ('recvmsg', (-1, 0, 0)), + ('sendmmsg', (-1, 0, 0, 0)), ('recvmmsg', (-1, 0, 0, 0, 0)), +]: + assert getattr(libc, name)(*args) == -1, name + assert ctypes.get_errno() == errno.EPERM, (name, ctypes.get_errno()) +with socket.socket(socket.AF_INET, socket.SOCK_STREAM): + pass +print('Pre-runtime guard: AF_UNIX, socketpair and descriptor-message syscalls denied; IP available') diff --git a/packages/extraction/tests/isolation.mjs b/packages/extraction/tests/isolation.mjs index d44ef800..fb93cef9 100644 --- a/packages/extraction/tests/isolation.mjs +++ b/packages/extraction/tests/isolation.mjs @@ -1,31 +1,42 @@ import assert from 'node:assert/strict'; +import { + mkdir, + mkdtemp, + writeFile, + rm, + readdir, + open, + readFile, +} from 'node:fs/promises'; +import { constants } from 'node:fs'; import { spawn } from 'node:child_process'; -import { mkdir, mkdtemp, writeFile, rm, readdir } from 'node:fs/promises'; import { once } from 'node:events'; import { RestrictedBackend } from '/opt/extraction/dist/backend.js'; await mkdir('/jobs/other'); await writeFile('/jobs/other/secret', 'cross-job-canary'); -await writeFile('/socket/secret', 'transport-canary'); + const job = await mkdtemp('/jobs/job-'); +const status = await open( + `${job}/status`, + constants.O_RDWR | + constants.O_CREAT | + constants.O_EXCL | + constants.O_NOFOLLOW, + 0o600 +); const child = spawn( '/opt/extraction/guard', [job, '/usr/local/bin/python3.13', '/tests/isolation.py', 'probe'], { env: { EXTRACTION_SECRET_CANARY: 'private-canary' }, - stdio: ['ignore', 'pipe', 'pipe'], + stdio: ['ignore', status.fd, 'ignore'], } ); -let output = '', - errors = ''; -child.stdout.on('data', bytes => { - output += bytes; -}); -child.stderr.on('data', bytes => { - errors += bytes; -}); const [code] = await once(child, 'close'); -assert.equal(code, 0, errors); +await status.close(); +const output = await readFile(`${job}/status`, 'ascii'); +assert.equal(code, 0, output); assert.equal(output.trim(), 'ISOLATED'); await rm(job, { recursive: true }); const runtime = { @@ -45,13 +56,19 @@ for (const abort of ['immediate', 'running']) { await assert.rejects(pending); await backend.cleanup(job); } +const flood = await backend.create(); +await assert.rejects( + backend.run(flood, 'flood', 'a'.repeat(64), AbortSignal.timeout(1000)), + /UNAVAILABLE/ +); +await backend.cleanup(flood); const job2 = await backend.create(); await assert.rejects( backend.run(job2, 'hang', 'a'.repeat(64), AbortSignal.timeout(100)) ); await backend.cleanup(job2); await rm('/jobs/other', { recursive: true }); -await rm('/socket/secret'); + assert.deepEqual(await readdir('/jobs'), []); const missing = new RestrictedBackend({ ...runtime, guard: '/nonexistent' }); const job3 = await missing.create(); diff --git a/packages/extraction/tests/isolation.py b/packages/extraction/tests/isolation.py index a50d0084..e46119b4 100644 --- a/packages/extraction/tests/isolation.py +++ b/packages/extraction/tests/isolation.py @@ -6,15 +6,44 @@ import signal import socket import struct +import stat import sys +for fd in (0, 2): + assert stat.S_ISCHR(os.fstat(fd).st_mode) + assert os.fstat(fd).st_rdev == os.stat('/dev/null').st_rdev +assert stat.S_ISREG(os.fstat(1).st_mode) +for fd in range(3, 64): + try: + os.fstat(fd) + except OSError as error: + assert error.errno == errno.EBADF + else: + raise RuntimeError('Inherited descriptor') + +libc = ctypes.CDLL(None, use_errno=True) +# Invalid descriptors would return EBADF without filtering. No messages are sent. +for name, args in [ + ('socketpair', (socket.AF_UNIX, socket.SOCK_STREAM, 0, 0)), + ('sendmsg', (-1, 0, 0)), ('recvmsg', (-1, 0, 0)), + ('sendmmsg', (-1, 0, 0, 0)), ('recvmmsg', (-1, 0, 0, 0, 0)), + ('sendto', (-1, 0, 0, 0, 0, 0)), ('recvfrom', (-1, 0, 0, 0, 0, 0)), + ('setsockopt', (-1, 0, 0, 0, 0)), ('getsockopt', (-1, 0, 0, 0, 0)), + ('sendmsg', (1, 0, 0)), ('recvmsg', (1, 0, 0)), +]: + assert getattr(libc, name)(*args) == -1, name + assert ctypes.get_errno() == errno.EPERM, (name, ctypes.get_errno()) + +if sys.argv[1] == 'flood': + os.write(1, b'x' * 2048) + sys.exit(0) if sys.argv[1] == 'hang': # Exercise supervisor cancellation/deadline termination, not a cooperative parser. while True: pass for path, mode in [('/jobs/other/secret', 'rb'), ('/jobs/other/secret', 'wb'), - ('/socket/secret', 'rb'), ('/proc/self/environ', 'rb'), ('/etc/passwd', 'rb')]: + ('/run/extraction/token', 'rb'), ('/proc/self/environ', 'rb'), ('/etc/passwd', 'rb')]: try: with open(path, mode) as file: if mode == 'rb': diff --git a/packages/extraction/tests/stdio.mjs b/packages/extraction/tests/stdio.mjs new file mode 100644 index 00000000..d6260d98 --- /dev/null +++ b/packages/extraction/tests/stdio.mjs @@ -0,0 +1,31 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtemp, rm } from 'node:fs/promises'; + +const job = await mkdtemp('/jobs/job-'); +try { + const worker = spawn( + '/opt/extraction/guard', + [job, '/usr/local/bin/python3.13', '/tests/isolation.py', 'probe'], + { stdio: ['pipe', 'pipe', 'pipe'] } + ); + worker.stdin.end(); + worker.stdout.resume(); + worker.stderr.resume(); + assert.equal((await once(worker, 'close'))[0], 125); + const gateway = spawn( + '/opt/extraction/gateway', + ['python', '/tests/gateway.py'], + { stdio: ['pipe', 'pipe', 'pipe'] } + ); + gateway.stdin.end(); + gateway.stdout.resume(); + gateway.stderr.resume(); + assert.equal((await once(gateway, 'close'))[0], 0); + console.log( + 'Socket standard IO rejected by worker and removed by pre-runtime gateway' + ); +} finally { + await rm(job, { recursive: true }); +} diff --git a/packages/extraction/worker/extract.py b/packages/extraction/worker/extract.py index 20f0360f..65022025 100644 --- a/packages/extraction/worker/extract.py +++ b/packages/extraction/worker/extract.py @@ -434,9 +434,9 @@ def probe(): import docx import lxml.etree - for path in ("/socket", "/jobs", "/proc/self/environ", "/etc/passwd"): + for path in ("/run/extraction", "/jobs", "/proc/self/environ", "/etc/passwd"): try: - if path in ("/socket", "/jobs"): + if path in ("/run/extraction", "/jobs"): os.listdir(path) else: open(path, "rb").read(1)