ci: make the canary report reason over the planned matrix #2
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Dependency canary | |
| # Weekly: re-resolve the runtime dependencies of `mcp[cli,rich]` to the newest | |
| # versions our (floors-only) specifiers allow, ignoring uv.lock, run the test | |
| # suite against them, and keep ONE tracking issue in sync with the result — | |
| # opened (and assigned) when newest-allowed breaks, refreshed weekly while it | |
| # stays broken, closed automatically once it passes again. | |
| # | |
| # Why this exists: users who `pip install mcp` get the newest release of every | |
| # dependency the day it ships, while PR CI only ever sees uv.lock (`locked`) and | |
| # the floors (`lowest-direct`). This is deliberately NOT part of PR CI, so an | |
| # upstream release can never turn an unrelated PR red; the price is up to a | |
| # week of latency, which the incident history says is fine. | |
| # | |
| # What it does not do, on purpose: | |
| # - open a PR adding a ceiling. A cap only helps once released, resolvers | |
| # route around retroactive caps by picking an older uncapped mcp, and the | |
| # bot cannot tell which package (or interaction) is at fault. The issue's | |
| # "What to do" section is the runbook; a human decides. | |
| # - float test tooling (pytest, ruff, pyright, coverage, ...). Only the | |
| # runtime closure is upgraded; everything else keeps its uv.lock version as | |
| # a preference and moves only when a floated runtime dependency forces it | |
| # (the report tags such rows "tooling"). Dependabot owns moving tooling. | |
| # - test pre-releases on the schedule. `workflow_dispatch` with | |
| # `prerelease: true` does that on demand and never files an issue. | |
| # - bisect. The issue lists what changed since the last green run (usually | |
| # one to three packages) and the one-line command to pin a suspect back. | |
| # | |
| # Known blind spots: Python 3.11-3.13 and macOS are not run; runtime deps that a | |
| # *dev* dependency caps (e.g. logfire pins opentelemetry-sdk, which pins | |
| # opentelemetry-api) cannot reach their newest release here — the issue's "Not | |
| # tested at their newest release" section lists them each run. Dependency | |
| # groups that `uv sync` does not install (translate, codegen) are stripped | |
| # before resolving so their caps (anthropic: pydantic<3) do not apply. | |
| # | |
| # Notifications: assignees get the issue traffic. GitHub additionally e-mails | |
| # scheduled-run failures only to whoever last edited the `cron:` line below. | |
| # The P0 label is added automatically when every cell hard-fails and is never | |
| # removed automatically; de-escalation is a human call. | |
| on: | |
| schedule: | |
| - cron: "23 5 * * 1" # Mondays 05:23 UTC | |
| workflow_dispatch: | |
| inputs: | |
| prerelease: | |
| description: "Also consider pre-releases (investigative run; never files an issue)" | |
| type: boolean | |
| default: false | |
| file-issue: | |
| description: "Create/update/close the tracking issue exactly as a scheduled run would" | |
| type: boolean | |
| default: false | |
| # TEMPORARY while this workflow is under review: exercise it on the PR branch. | |
| # Report-only (push runs never touch issues). Remove before merging. | |
| push: | |
| branches: ["ci/dependency-canary"] | |
| permissions: {} | |
| concurrency: | |
| # One canary at a time, and never cancel one in flight: a half-finished run | |
| # must not be what decides the tracking issue's fate. Runs are short; queue. | |
| group: ${{ github.workflow }} | |
| cancel-in-progress: false | |
| defaults: | |
| run: | |
| shell: bash # -eo pipefail everywhere, and Git-Bash on the Windows cell | |
| env: | |
| COLUMNS: 150 | |
| UV_VERSION: "0.11.33" | |
| # Releases younger than this are invisible to the run: skips half-uploaded | |
| # releases (the ruff 0.14.12 incident that got the per-PR "highest" leg | |
| # removed in #1869) and same-day yanks. A weekly job loses nothing by it. | |
| CANARY_LAG: "24 hours" | |
| # Single source of truth for the test matrix; the report checks every one of | |
| # these produced a result. Oldest and newest supported Python bracket the | |
| # marker forks in the lock (deps drop 3.10 first; 3.14 gets wheels last), and | |
| # Windows/newest is where a fresh release most often lacks a wheel. | |
| CANARY_CELLS: >- | |
| [ | |
| {"cell": "ubuntu-3.10", "os": "ubuntu-latest", "python": "3.10", "smoke": "", "pyright": ""}, | |
| {"cell": "ubuntu-3.14", "os": "ubuntu-latest", "python": "3.14", "smoke": "1", "pyright": "1"}, | |
| {"cell": "windows-3.14", "os": "windows-latest", "python": "3.14", "smoke": "", "pyright": ""} | |
| ] | |
| CANARY_LABEL: dependency-canary | |
| CANARY_ASSIGNEES: "maxisbey,Kludex" | |
| jobs: | |
| resolve: | |
| # Don't run the schedule on forks. | |
| if: github.repository == 'modelcontextprotocol/python-sdk' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| permissions: | |
| contents: read | |
| actions: read # find and download the last green scheduled run's resolution to diff against | |
| outputs: | |
| cells: ${{ steps.plan.outputs.cells }} | |
| steps: | |
| - name: Publish the test matrix | |
| id: plan | |
| run: | | |
| { | |
| echo 'cells<<EOF' | |
| echo "$CANARY_CELLS" | |
| echo 'EOF' | |
| } >>"$GITHUB_OUTPUT" | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| persist-credentials: false | |
| # setup-uv's manifest fetch is a single request with a hard 5s timeout | |
| # (astral-sh/setup-uv#869); retry once. Drop when upstream adds a retry. | |
| - name: Install uv | |
| id: setup-uv | |
| continue-on-error: true | |
| uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 | |
| with: | |
| enable-cache: false | |
| version: ${{ env.UV_VERSION }} | |
| - name: Install uv (retry) | |
| if: steps.setup-uv.outcome == 'failure' | |
| uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 | |
| with: | |
| enable-cache: false | |
| version: ${{ env.UV_VERSION }} | |
| - name: Fetch the last green run's resolution | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| mkdir -p out/canary-resolve | |
| cd out/canary-resolve | |
| date -u -d "-$CANARY_LAG" +%Y-%m-%dT%H:%M:%SZ >cutoff.txt | |
| (uv self version --short 2>/dev/null || uv --version | awk '{print $2}') >uv-version.txt | |
| : >baseline.txt | |
| # Baseline = the uv.lock the last successful *scheduled* run uploaded, so the | |
| # report can list only what moved since then. Best effort (artifacts expire). | |
| gh run list --repo "$GITHUB_REPOSITORY" --workflow dependency-canary.yml --branch main \ | |
| --event schedule --status success --limit 1 --json databaseId,startedAt,url >last-green.json || echo '[]' >last-green.json | |
| run_id=$(jq -r '.[0].databaseId // empty' last-green.json) | |
| if [ -n "$run_id" ] && gh run download "$run_id" --repo "$GITHUB_REPOSITORY" -n canary-resolve -D baseline-dl; then | |
| if [ -f baseline-dl/canary-resolve/uv.lock ]; then | |
| cp baseline-dl/canary-resolve/uv.lock baseline.lock | |
| jq -r '.[0] | "[\(.startedAt[:10]) run](\(.url))"' last-green.json >baseline.txt | |
| fi | |
| fi | |
| rm -rf baseline-dl | |
| echo "cutoff=$(cat cutoff.txt) uv=$(cat uv-version.txt) baseline=$(cat baseline.txt)" | |
| - name: Work out what to float | |
| run: | | |
| r=out/canary-resolve | |
| # The runtime closure of mcp[cli,rich] as currently locked: exactly the set | |
| # `pip install "mcp[cli,rich]"` pulls in. New transitive deps that a newer | |
| # release introduces have no lock entry and so resolve to newest anyway. | |
| uv export --frozen --no-default-groups --all-extras --no-emit-workspace \ | |
| --no-hashes --no-header --no-annotate | grep -E '^[A-Za-z0-9]' | sed -E 's/[=; @[].*//' | sort -u >$r/closure.txt | |
| if [ ! -s $r/closure.txt ]; then echo "::error::uv export produced an empty runtime closure"; exit 1; fi | |
| echo "Floating $(wc -l <$r/closure.txt) packages:"; tr '\n' ' ' <$r/closure.txt; echo | |
| # Dependency groups that `uv sync` does not install still constrain the | |
| # resolution (uv.lock is universal). Strip them so e.g. the translate | |
| # group's `anthropic` cannot hold pydantic below a new major. | |
| python3 - <<'EOF' >$r/strip.sh | |
| import re, tomllib | |
| project = tomllib.load(open("pyproject.toml", "rb")) | |
| groups = project.get("dependency-groups", {}) | |
| default = project.get("tool", {}).get("uv", {}).get("default-groups", ["dev"]) | |
| keep = set(groups) if default == "all" else set(default) | |
| todo = list(keep) | |
| while todo: # groups pulled in via {include-group = "..."} are installed too | |
| for entry in groups.get(todo.pop(), []): | |
| if isinstance(entry, dict) and entry.get("include-group") not in keep | {None}: | |
| keep.add(entry["include-group"]) | |
| todo.append(entry["include-group"]) | |
| for group, entries in groups.items(): | |
| names = [re.match(r"[A-Za-z0-9._-]+", e).group(0) for e in entries if isinstance(e, str)] | |
| if group not in keep and names: | |
| print("uv remove --frozen --group", group, *names) | |
| EOF | |
| cat $r/strip.sh | |
| bash -e $r/strip.sh | |
| - name: Resolve newest allowed versions | |
| env: | |
| PRERELEASE: ${{ inputs.prerelease && 'allow' || '' }} | |
| run: | | |
| r=out/canary-resolve | |
| cutoff=$(cat $r/cutoff.txt) | |
| cp uv.lock $r/committed.lock | |
| # Per-package cutoffs rather than a global --exclude-newer, so an exact pin | |
| # elsewhere (docs group, build constraints) bumped the day before the run | |
| # cannot make the resolution fail. | |
| args=() | |
| if [ -n "$PRERELEASE" ]; then args+=(--prerelease "$PRERELEASE"); fi | |
| while read -r pkg; do args+=(-P "$pkg" --exclude-newer-package "$pkg=$cutoff"); done <$r/closure.txt | |
| { printf 'uv lock'; printf ' %q' "${args[@]}"; echo; } >$r/lock-cmd.sh | |
| if uv lock "${args[@]}" 2>&1 | tee $r/lock.log; then | |
| echo ok >$r/lock-status | |
| cp uv.lock $r/uv.lock | |
| else | |
| # Distinguishes "uv ran and could not resolve" from every other way this job can fail. | |
| echo failed >$r/lock-status | |
| exit 1 | |
| fi | |
| - name: Summarise what moved | |
| run: | | |
| r=out/canary-resolve | |
| python3 scripts/ci/canary_lock_diff.py $r/committed.lock uv.lock \ | |
| --old-label "uv.lock" --new-label "this run" --suspects $r/suspects-vs-lock.txt >$r/vs-lock.md | |
| if [ -f $r/baseline.lock ]; then | |
| python3 scripts/ci/canary_lock_diff.py $r/baseline.lock uv.lock \ | |
| --old-label "last green" --new-label "this run" --suspects $r/suspects-since-green.txt >$r/since-green.md | |
| fi | |
| # Direct runtime deps that could not reach their newest release (capped by something else in the resolution). | |
| uv tree --frozen --outdated --universal --depth 1 --package mcp >$r/tree.txt | |
| { | |
| echo "| Package | Resolved | Latest |" | |
| echo "| --- | --- | --- |" | |
| sed -nE 's/^[^A-Za-z0-9]*([A-Za-z0-9._-]+)(\[[^]]*\])? v([^ ]+) .*\(latest: v([^)]+)\)$/\1 \3 \4/p' $r/tree.txt | | |
| sort -u | while read -r name have latest; do | |
| if grep -qxF "$name" $r/closure.txt; then echo "| $name | $have | $latest |"; fi | |
| done | |
| } >$r/held-back.md | |
| if [ "$(wc -l <$r/held-back.md)" -le 2 ]; then : >$r/held-back.md; fi | |
| { | |
| echo "## Resolution (cutoff $(cat $r/cutoff.txt))" | |
| echo | |
| if [ -s $r/since-green.md ]; then echo "### Since last green ($(cat $r/baseline.txt))"; cat $r/since-green.md; echo; fi | |
| echo "### vs uv.lock"; cat $r/vs-lock.md; echo | |
| if [ -s $r/held-back.md ]; then echo "### Held back below latest"; cat $r/held-back.md; fi | |
| } >>"$GITHUB_STEP_SUMMARY" | |
| - name: Upload resolution | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: canary-resolve | |
| path: out/ | |
| retention-days: 90 | |
| if-no-files-found: error | |
| test: | |
| name: test (${{ matrix.cell }}) | |
| needs: resolve | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 25 | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJSON(needs.resolve.outputs.cells) }} | |
| steps: | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| persist-credentials: false | |
| - name: Install uv | |
| id: setup-uv | |
| continue-on-error: true | |
| uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 | |
| with: | |
| enable-cache: false | |
| version: ${{ env.UV_VERSION }} | |
| - name: Install uv (retry) | |
| if: steps.setup-uv.outcome == 'failure' | |
| uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 | |
| with: | |
| enable-cache: false | |
| version: ${{ env.UV_VERSION }} | |
| - name: Fetch the resolved lock | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: canary-resolve | |
| path: . | |
| - name: Install and test | |
| # The suite takes ~2 min; a dependency that makes it hang should surface as | |
| # this cell's "incomplete" status in the report, not as a 25-minute job kill. | |
| timeout-minutes: 15 | |
| env: | |
| CANARY_CELL: ${{ matrix.cell }} | |
| CANARY_PYTHON: ${{ matrix.python }} | |
| CANARY_PYRIGHT: ${{ matrix.pyright }} | |
| # Same switches as PR CI: real stdio/uvicorn subprocess smoke tests on one | |
| # cell, and PEP 597 EncodingWarnings surfaced (as an env var so xdist workers inherit it). | |
| MCP_EXAMPLES_SMOKE: ${{ matrix.smoke }} | |
| PYTHONWARNDEFAULTENCODING: "1" | |
| run: | | |
| cp canary-resolve/uv.lock uv.lock | |
| bash scripts/ci/canary_cell.sh | |
| - name: Upload cell result | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: canary-cell-${{ matrix.cell }} | |
| path: out/ | |
| retention-days: 30 | |
| if-no-files-found: error | |
| report: | |
| needs: [resolve, test] | |
| # !cancelled(): a red resolve/test job is exactly when this must run, but a | |
| # cancelled run must never decide the issue's fate from partial results. | |
| if: ${{ !cancelled() && github.repository == 'modelcontextprotocol/python-sdk' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| permissions: | |
| contents: read # checkout, for scripts/ci/canary_report.sh | |
| issues: write # open / refresh / close the tracking issue | |
| steps: | |
| - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 | |
| with: | |
| persist-credentials: false | |
| sparse-checkout: scripts/ci | |
| - name: Collect results | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| pattern: canary-* | |
| # Every artifact already carries its own top-level directory, so merging | |
| # yields artifacts/canary-resolve/ and artifacts/canary-cell-*/ no matter | |
| # how many artifacts exist (a lone match would otherwise be flattened). | |
| merge-multiple: true | |
| path: artifacts | |
| - name: Report | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| CANARY_ARTIFACTS: artifacts | |
| CANARY_RESOLVE_RESULT: ${{ needs.resolve.result }} | |
| CANARY_TEST_RESULT: ${{ needs.test.result }} | |
| CANARY_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| # Only the schedule (or an explicit dispatch asking for it) touches issues, and never a pre-release run. | |
| CANARY_FILE_ISSUES: ${{ !inputs.prerelease && (github.event_name == 'schedule' || inputs.file-issue) && 'true' || 'false' }} | |
| run: bash scripts/ci/canary_report.sh |