diff --git a/.github/workflows/release-model-probe.yml b/.github/workflows/release-model-probe.yml new file mode 100644 index 00000000..727ffa07 --- /dev/null +++ b/.github/workflows/release-model-probe.yml @@ -0,0 +1,75 @@ +name: Release Model Probe + +on: + workflow_dispatch: + inputs: + commit: + description: Exact main commit of the candidate + required: true + type: string + candidate_run_id: + description: Successful candidate Release workflow run + required: true + type: string + +concurrency: + group: braid-protected-account-proof + cancel-in-progress: false + +jobs: + probe: + runs-on: ubuntu-latest + timeout-minutes: 20 + environment: release-live + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Check out the exact candidate source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.commit }} + path: candidate-source + fetch-depth: 0 + - name: Verify candidate source + env: + REQUESTED_COMMIT: ${{ inputs.commit }} + run: | + set -euo pipefail + test "$(git -C candidate-source rev-parse HEAD)" = "$REQUESTED_COMMIT" + test "$(printf '%s' "$REQUESTED_COMMIT" | wc -c)" = 40 + git -C candidate-source fetch origin main + git -C candidate-source merge-base --is-ancestor "$REQUESTED_COMMIT" origin/main + - name: Verify the candidate workflow run + env: + GH_TOKEN: ${{ github.token }} + BRAID_RELEASE_RUN_ID: ${{ inputs.candidate_run_id }} + BRAID_RELEASE_EXPECT_WORKFLOW: .github/workflows/release.yml + BRAID_RELEASE_EXPECT_COMMIT: ${{ inputs.commit }} + BRAID_RELEASE_EXPECT_BRANCH: main + run: node scripts/release/verify-workflow-run.mjs + - name: Download exact candidate + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: braid-release-candidate-${{ inputs.commit }} + path: ${{ runner.temp }}/braid-model-probe + github-token: ${{ github.token }} + run-id: ${{ inputs.candidate_run_id }} + - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22.19.0 + cache: pnpm + - name: Install and build exact candidate source + run: | + cd candidate-source + pnpm install --frozen-lockfile + pnpm run build + - name: One bounded authenticated Gemini task from installed candidate + env: + BRAID_LIVE_TANGLE_ENV_JSON: ${{ secrets.BRAID_LIVE_TANGLE_ENV_JSON }} + BRAID_PROBE_SOURCE: ${{ github.workspace }}/candidate-source + BRAID_PROBE_ARTIFACT_ROOT: ${{ runner.temp }}/braid-model-probe + BRAID_PROBE_COMMIT: ${{ inputs.commit }} + run: node scripts/release/model-probe.mjs diff --git a/scripts/release/model-probe.mjs b/scripts/release/model-probe.mjs new file mode 100644 index 00000000..fa34fd2e --- /dev/null +++ b/scripts/release/model-probe.mjs @@ -0,0 +1,122 @@ +import { readFile, writeFile } from 'node:fs/promises' +import { join, resolve } from 'node:path' + +import { installPackedBraid } from '../packed-binary.mjs' +import { readCandidateIdentity } from './build-identity.mjs' +import { connectionConfiguration } from '../live-required/configuration.mjs' +import { safeMessage } from '../live-required/contracts.mjs' +import { + closeSession, + prepareProductionWorkspace, + runHeadlessTurn, +} from '../live-required/headless.mjs' + +const model = 'gemini-2.5-flash-lite' +const marker = 'BRAID_GEMINI_PROBE_OK' +const source = resolve(process.env.BRAID_PROBE_SOURCE ?? '') +const artifactRoot = resolve(process.env.BRAID_PROBE_ARTIFACT_ROOT ?? '') +const commit = process.env.BRAID_PROBE_COMMIT +const rawSecret = process.env.BRAID_LIVE_TANGLE_ENV_JSON +if (!/^[a-f0-9]{40}$/u.test(commit ?? '') || !rawSecret) { + throw new Error('Exact candidate commit and protected environment are required') +} +const protectedEnvironment = JSON.parse(rawSecret) +if ( + protectedEnvironment === null || + typeof protectedEnvironment !== 'object' || + Array.isArray(protectedEnvironment) +) { + throw new Error('Protected environment is invalid') +} +const environment = { ...process.env, ...protectedEnvironment } +delete environment.BRAID_LIVE_TANGLE_ENV_JSON +const values = connectionConfiguration(environment, { + prefix: 'BRAID_TANGLE', + kind: 'tangle-inference', + endpointNames: ['BRAID_TANGLE_INFERENCE_ENDPOINT'], + modelNames: ['BRAID_TANGLE_INFERENCE_MODEL'], + runnerNames: ['BRAID_TANGLE_INFERENCE_RUNNER'], + providerNames: ['BRAID_TANGLE_INFERENCE_PROVIDER'], + fallbackModelProvider: 'tangle-router', + fallbackRunner: 'cli-base', +}) +if (values.credentialValue === undefined) { + throw new Error('Probe requires the existing protected credential value') +} +if (values.modelProvider !== 'tangle-router') { + throw new Error('Protected credential is not configured for Tangle Router') +} +const catalogUrl = new URL('/v1/models', values.endpoint) +const auth = values.credentialValue.replace(/^Bearer\s+/iu, '') +const catalogResponse = await fetch(catalogUrl, { + headers: { Authorization: `Bearer ${auth}` }, + signal: AbortSignal.timeout(10_000), +}) +if (!catalogResponse.ok) { + throw new Error(`Protected credential model listing returned HTTP ${catalogResponse.status}`) +} +const catalog = await catalogResponse.json() +if (!Array.isArray(catalog.data) || !catalog.data.some((item) => item?.id === model)) { + throw new Error('Protected credential catalog does not list the selected model') +} +const { identity } = await readCandidateIdentity({ + repository: source, + artifactRoot, + expectedCommit: commit, + expectedVersion: '0.3.4', +}) +const packed = await installPackedBraid(source, { + tarballPath: join(artifactRoot, identity.tarballPath), +}) +let config +let session +try { + if (packed.tarballSha256 !== identity.tarballSha256) { + throw new Error('Installed archive differs from the candidate') + } + config = await prepareProductionWorkspace({ + repository: source, + environment, + ...values, + model, + }) + const profilePath = join(config.workspace, '.braid', 'profiles', 'profile-tangle-inference.json') + const profile = JSON.parse(await readFile(profilePath, 'utf8')) + profile.model.maxVisibleOutputTokens = 32 + profile.model.maxTotalOutputTokens = 64 + await writeFile(profilePath, `${JSON.stringify(profile, null, 2)}\n`, { mode: 0o600 }) + process.env.BRAID_LIVE_REQUIRED_TIMEOUT_MS = '60000' + const turn = await runHeadlessTurn({ + binary: packed.binary, + config, + marker, + prompt: `Reply with exactly ${marker}.`, + }) + session = turn.session + const receipt = { + schema: 'braid.release-model-probe.v1', + commit, + version: identity.braidVersion, + tarballSha256: packed.tarballSha256, + model: turn.run.model ?? model, + status: turn.run.status, + runId: turn.run.id, + materializationDigest: turn.run.materializationDigest, + costStatus: turn.run.costStatus ?? 'unknown', + costUsd: turn.run.costUsd ?? null, + maxVisibleOutputTokens: 32, + maxTotalOutputTokens: 64, + markerMatched: true, + } + process.stdout.write(`${JSON.stringify(receipt)}\n`) + if (typeof receipt.costUsd === 'number' && receipt.costUsd > 0.01) { + throw new Error('One-turn cost exceeded the $0.01 probe ceiling') + } +} catch (error) { + process.stderr.write(`Protected model probe failed: ${safeMessage(error, environment)}\n`) + process.exitCode = 1 +} finally { + if (session) await closeSession(session) + if (config) await config.cleanup() + await packed.cleanup() +}