From 044ab899ee5d9af36749ea9356bf9b272482d495 Mon Sep 17 00:00:00 2001 From: pelikhan Date: Sat, 17 Oct 2026 23:25:06 -0700 Subject: [PATCH] fix(workflows): report PR content integrity blockers accurately Keep approved integrity, stop after the first content denial, and distinguish reviewed, blocked, unavailable, and unattempted PRs. Preserve incomplete status and explicitly retain the gateway content-trust dependency. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../workflows/daily-pr-review-cursor.lock.yml | 36 ++++-- .github/workflows/daily-pr-review-cursor.md | 80 ++++++++++--- scripts/test_daily_pr_review_integrity.py | 106 ++++++++++++++++++ 3 files changed, 196 insertions(+), 26 deletions(-) create mode 100644 scripts/test_daily_pr_review_integrity.py diff --git a/.github/workflows/daily-pr-review-cursor.lock.yml b/.github/workflows/daily-pr-review-cursor.lock.yml index ce599442078..b6533bf0755 100644 --- a/.github/workflows/daily-pr-review-cursor.lock.yml +++ b/.github/workflows/daily-pr-review-cursor.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"5debcfbb08f0b0f724062d142083ceaf784db94c954d8bf8885678f1e4f64219","body_hash":"bf1cf21f4246f5ae31a56495c39972606baec878241331ad7f299696d6230278","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"3f4d411f9c0263730a5cd38581b3ceeed4c09626779fdce4af78b0a121908cd5","body_hash":"db43a60ccbd8abbac6ba8fe521dc3aec4687f2d94b002e9886f3a2b1982504f2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.98","copilot-sdk":"1.0.16"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"9000827ccba6bdab643e8b6fd33ac0654aef8333","version":"v8.0.2"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"949feb2413d6458794dcd2491c4babbbce0c15c1","version":"v7.1.0"},{"repo":"actions/upload-artifact","sha":"cf430e030ddbb5b0abf93d22962f4752f3646cd9","version":"v7.0.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62","digest":"sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.62@sha256:80c4ddd927647b636f49807d2557b4cd3ec175c894194e432c9374f5a69e890f"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62","digest":"sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.62@sha256:ebb9e7b5bec475f6b6c2eb6e999ada7d41d57d0acbacb8493c5bebc0106fd44d"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62","digest":"sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.62@sha256:b145850ed8bdffd2d4fcd8f1f3ea5793b56cadee6989700b91f42cff490630ef"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.35","digest":"sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f","pinned_image":"ghcr.io/github/gh-aw-node@sha256:11c2c544876cdbaffefe0f7f1283b74838c209654b4934a78e23ed11f750490f"},{"image":"ghcr.io/github/github-mcp-server:v2.0.2","digest":"sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796","pinned_image":"ghcr.io/github/github-mcp-server:v2.0.2@sha256:ffced0d76e77428532a2ced185516992d77fd146d69eec2f244e6237d1a97796"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_pull_request","get_pull_request_comments","get_pull_request_diff","get_pull_request_files","get_pull_request_review_comments","get_pull_request_reviews","get_pull_request_status","get_release_by_tag","get_tag","list_branches","list_commits","list_pull_requests","list_releases","list_starred_repositories","list_tags","pull_request_read","search_code","search_pull_requests","search_repositories"]},{"name":"safeoutputs","tools":["create_issue","missing_data","missing_tool","noop","report_incomplete"]}],"threat_detection":{"mode":"enabled"}} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -23,7 +23,7 @@ # # For more information: https://github.github.com/gh-aw/introduction/overview/ # -# Daily review of recently opened pull requests for code quality issues +# Daily code quality review of accessible recent pull requests with explicit coverage reporting # # Resolved workflow manifest: # Imports: @@ -563,12 +563,20 @@ jobs: env: GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + GH_AW_GITHUB_MIN_INTEGRITY: 'approved' with: script: | const path = require('path'); const actionsDir = path.join(process.env.RUNNER_TEMP, 'gh-aw', 'actions'); const determineAutomaticLockdown = require(path.join(actionsDir, 'determine_automatic_lockdown.cjs')); await determineAutomaticLockdown(github, context, core); + - name: Parse integrity filter lists + id: parse-guard-vars + env: + GH_AW_BLOCKED_USERS_VAR: ${{ vars.GH_AW_GITHUB_BLOCKED_USERS || '' }} + GH_AW_TRUSTED_USERS_VAR: ${{ vars.GH_AW_GITHUB_TRUSTED_USERS || '' }} + GH_AW_APPROVAL_LABELS_VAR: ${{ vars.GH_AW_GITHUB_APPROVAL_LABELS || '' }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/parse_guard_list.sh" - name: Restore agent config folders from base branch if: steps.checkout-pr.outcome == 'success' env: @@ -743,13 +751,14 @@ jobs: id: start-mcp-gateway env: DEBUG: ${{ env.DEBUG || runner.debug == '1' && '*' || '' }} + GH_AW_GUARD_APPROVAL_LABELS: ${{ steps.parse-guard-vars.outputs.approval_labels }} + GH_AW_GUARD_BLOCKED_USERS: ${{ steps.parse-guard-vars.outputs.blocked_users }} + GH_AW_GUARD_TRUSTED_USERS: ${{ steps.parse-guard-vars.outputs.trusted_users }} GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST: ${{ vars.GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST || 'true' }} GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} GH_AW_SAFE_OUTPUTS_CONFIG_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_CONFIG_PATH }} GH_AW_SAFE_OUTPUTS_TOOLS_PATH: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS_TOOLS_PATH }} GH_AW_SINK_VISIBILITY: ${{ steps.determine-automatic-lockdown.outputs.visibility }} - GITHUB_MCP_GUARD_MIN_INTEGRITY: ${{ steps.determine-automatic-lockdown.outputs.min_integrity }} - GITHUB_MCP_GUARD_REPOS: ${{ steps.determine-automatic-lockdown.outputs.repos }} GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -783,11 +792,11 @@ jobs: MCP_GATEWAY_UID=$(id -u 2>/dev/null || echo '0') MCP_GATEWAY_GID=$(id -g 2>/dev/null || echo '0') source "${RUNNER_TEMP}/gh-aw/actions/resolve_docker_socket_gid.sh" - export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network bridge -p 127.0.0.1:'"${MCP_GATEWAY_PORT}"':'"${MCP_GATEWAY_PORT}"' --name awmg-mcpg --add-host host.docker.internal:host-gateway --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_AGENT_ID -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_TARGET_REPO_SLUG -e GH_AW_PR_HEAD_BASE_BRANCH -e GH_AW_PR_HEAD_BASE_SHA -e GH_AW_PR_HEAD_BASE_REPO -e GH_AW_PR_HEAD_BASE_PR_NUMBER -e GH_AW_PR_HEAD_BASE_REF -e GH_AW_PR_HEAD_REPO -e GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GH_AW_SINK_VISIBILITY -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e RUNNER_TEMP -e RUNNER_TOOL_CACHE -e MCP_GATEWAY_ALLOWED_MOUNT_ROOTS -e GITHUB_AW_OTEL_TRACE_ID -e GITHUB_AW_OTEL_PARENT_SPAN_ID -e OTEL_EXPORTER_OTLP_HEADERS -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw -v '"${RUNNER_TEMP}"'/gh-aw/safeoutputs:'"${RUNNER_TEMP}"'/gh-aw/safeoutputs:rw ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858' + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network bridge -p 127.0.0.1:'"${MCP_GATEWAY_PORT}"':'"${MCP_GATEWAY_PORT}"' --name awmg-mcpg --add-host host.docker.internal:host-gateway --user '"${MCP_GATEWAY_UID}"':'"${MCP_GATEWAY_GID}"' --group-add '"${DOCKER_SOCK_GID}"' -v '"${DOCKER_SOCK_PATH}"':/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_AGENT_ID -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DOCKER_HOST=unix:///var/run/docker.sock -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_TARGET_REPO_SLUG -e GH_AW_PR_HEAD_BASE_BRANCH -e GH_AW_PR_HEAD_BASE_SHA -e GH_AW_PR_HEAD_BASE_REPO -e GH_AW_PR_HEAD_BASE_PR_NUMBER -e GH_AW_PR_HEAD_BASE_REF -e GH_AW_PR_HEAD_REPO -e GH_AW_POLICY_ALLOW_CREATE_PULL_REQUEST -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GH_AW_SINK_VISIBILITY -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e RUNNER_TEMP -e RUNNER_TOOL_CACHE -e MCP_GATEWAY_ALLOWED_MOUNT_ROOTS -e GITHUB_AW_OTEL_TRACE_ID -e GITHUB_AW_OTEL_PARENT_SPAN_ID -e OTEL_EXPORTER_OTLP_HEADERS -e GH_AW_GUARD_APPROVAL_LABELS -e GH_AW_GUARD_BLOCKED_USERS -e GH_AW_GUARD_TRUSTED_USERS -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw -v '"${RUNNER_TEMP}"'/gh-aw/safeoutputs:'"${RUNNER_TEMP}"'/gh-aw/safeoutputs:rw ghcr.io/github/gh-aw-mcpg:v0.4.35@sha256:35e28671836380c0eae52660ea2ad7e5f6659f63cb20d1abae9271656c670858' mkdir -p "$HOME/.copilot" GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_9357db9d571a0359_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_62e9b52f25a22aed_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "github": { @@ -802,8 +811,11 @@ jobs: }, "guard-policies": { "allow-only": { - "min-integrity": "$GITHUB_MCP_GUARD_MIN_INTEGRITY", - "repos": "$GITHUB_MCP_GUARD_REPOS" + "approval-labels": ${GH_AW_GUARD_APPROVAL_LABELS}, + "blocked-users": ${GH_AW_GUARD_BLOCKED_USERS}, + "min-integrity": "approved", + "repos": "all", + "trusted-users": ${GH_AW_GUARD_TRUSTED_USERS} } } }, @@ -863,7 +875,7 @@ jobs: } } } - GH_AW_MCP_CONFIG_9357db9d571a0359_EOF + GH_AW_MCP_CONFIG_62e9b52f25a22aed_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true @@ -1230,6 +1242,8 @@ jobs: /tmp/gh-aw/sandbox/agent/logs/ /tmp/gh-aw/redacted-urls.log /tmp/gh-aw/mcp-logs/ + /tmp/gh-aw/proxy-logs/ + !/tmp/gh-aw/proxy-logs/proxy-tls/ /tmp/gh-aw/agent_usage.json /tmp/gh-aw/runtime/aw_info.json /tmp/gh-aw/agent_usage.jsonl @@ -1729,7 +1743,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: WORKFLOW_NAME: "Daily PR Code Quality Review" - WORKFLOW_DESCRIPTION: "Daily review of recently opened pull requests for code quality issues" + WORKFLOW_DESCRIPTION: "Daily code quality review of accessible recent pull requests with explicit coverage reporting" HAS_PATCH: ${{ needs.agent.outputs.has_patch }} GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" GH_AW_DETECTION_SKIP_PROMPT_SUMMARY: "true" @@ -1801,7 +1815,7 @@ jobs: RUNNER_TEMP: ${{ runner.temp }} TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} WORKFLOW_NAME: "Daily PR Code Quality Review" - WORKFLOW_DESCRIPTION: "Daily review of recently opened pull requests for code quality issues" + WORKFLOW_DESCRIPTION: "Daily code quality review of accessible recent pull requests with explicit coverage reporting" HAS_PATCH: ${{ needs.agent.outputs.has_patch }} GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" run: | diff --git a/.github/workflows/daily-pr-review-cursor.md b/.github/workflows/daily-pr-review-cursor.md index ee8de97bdf3..155eb67757e 100644 --- a/.github/workflows/daily-pr-review-cursor.md +++ b/.github/workflows/daily-pr-review-cursor.md @@ -1,7 +1,7 @@ --- private: true emoji: "🖱️" -description: Daily review of recently opened pull requests for code quality issues +description: Daily code quality review of accessible recent pull requests with explicit coverage reporting on: schedule: daily workflow_dispatch: @@ -25,6 +25,7 @@ tools: github: mode: local toolsets: [repos, pull_requests] + min-integrity: approved bash: - cat - grep @@ -52,28 +53,60 @@ features: # Daily PR Code Quality Review -Review recently opened pull requests for common code quality issues and produce a concise daily -report. +Review accessible recently opened pull requests for common code quality issues and produce a +concise daily report that accounts for every selected PR. The GitHub MCP integrity policy is the +authority for content access; discovering a PR does not mean its diff is trusted. ## Step 1 — Find recent open PRs -Use GitHub MCP tools to list open pull requests from `${{ github.repository }}` with at most -`perPage: 5`, ordered by `created_at` descending. Record: +Use GitHub MCP `list_pull_requests` for `${{ github.repository }}` with `state: open`, +`sort: created`, `direction: desc`, and `perPage: 5`. Request only metadata fields: +`number`, `title`, `user`, `html_url`. Select at most these five returned candidates; do not +expand the search to work around a denial. Record: - PR number - Title - Author -## Step 2 — Review each PR +PR metadata visibility, same-repository branches, author association, and GitHub review approval +are not proof that diff content passes the guard. Do not invent approval labels, promote users, +or change the integrity policy. If the list call fails or is denied, call `report_incomplete` +with that exact discovery failure; do not claim that no open PRs exist. An empty successful list +means only that no candidates were returned under the current policy. -For each PR, fetch its diff using the GitHub MCP `pull_request_read` tool with `method: get_diff`. -If you inspect local files, use the `view` tool (not `read`). -Look for: +## Step 2 — Attempt and review each candidate + +Attempt candidates in selection order, making at most one GitHub MCP `pull_request_read` call +with `method: get_diff` per PR. Stop content reads on the first integrity-filtering denial. +The pinned gateway (`gh-aw-mcpg v0.4.35`, `labels/response_paths.rs`) labels `get_diff` and +`get_files` as `pull-request-content` with `none` integrity, independently of PR metadata trust. +Repeated probes cannot resolve that boundary. Do not attempt the remaining candidates after +that denial; mark them **Not attempted** with the shared gateway blocker as the reason, not +**Blocked** as though each had returned a denial. + +Assign exactly one coverage status to every selected PR: +- **Reviewed**: the complete diff was returned and inspected. +- **Blocked**: the diff call explicitly reported integrity filtering. Record the actual denial + and required integrity level, not an authentication or model failure. +- **Unavailable**: the attempted diff read failed for another reason, or returned incomplete, + truncated, or missing content. Record the concrete error or coverage limit. +- **Not attempted**: no diff call was made, for example because the time budget expired. Record + why; never describe this PR as blocked or reviewed. + +On integrity filtering, do not retry with `get_files`, other GitHub tools, shell commands, +local checkouts, raw URLs, or pre-fetched patches to obtain the denied content. Do not infer +findings from PR titles or metadata. The pinned gateway can allow PR metadata while labeling +`pull-request-content` below `approved`, including for trusted authors. If no diff is accessible, +report that content-access blocker; do not prescribe a label or membership change as a fix. + +Review only changed code in successfully returned diffs. This is static analysis: do not +execute PR code, tests, or builds. Look for: - Missing error handling in Go (`if err != nil` blocks that are absent where errors could occur) - Exported functions without doc comments - Test files without assertions (only `t.Log` or no checks) - Oversized functions (> 80 lines in a single function body) -Keep each per-PR analysis to 3–5 bullet points. +Keep each reviewed PR's analysis to 3–5 bullet points, with file/line evidence for findings. +Do not pad a clean review with invented findings. ## Step 3 — Produce report issue @@ -81,13 +114,30 @@ Use the `reporting` skill to format the report body. Use `###` (or lower) header `#` or `##`. Keep the overall quality signal and PR table visible at the top; wrap the full per-PR findings in a `
Full Findings` block. -Use the `create_issue` safe-output tool to post a daily report: +If candidates were selected, use the `create_issue` safe-output tool to post a daily report, +even when all diff reads were blocked. A coverage report is not evidence of completed reviews. - **Title**: `[pr-review] Daily PR Code Quality Review — ${{ github.run_id }}` - **Body**: - - `### Summary` with a markdown table of PR number + link, top issue count, and overall - quality signal: 🟢 (≤1 issue/PR) / 🟡 (2–3 issues/PR) / 🔴 (>3 issues/PR) + - `### Summary` with reviewed/selected counts and whether coverage is complete or partial. + Include a markdown table of PR number + link, coverage status, issue count, quality signal, + and any access failure or coverage limit. + - For **Reviewed** PRs only, use 🟢 (≤1 issue/PR) / 🟡 (2–3 issues/PR) / 🔴 (>3 issues/PR). + For all other statuses, issue count and quality signal must be `N/A`, never zero or green. + If none were reviewed, state "Code quality not assessed" instead of an overall quality signal. - `
Full Findings` wrapping the detailed per-PR issue lists - (or "No issues" if clean) + (or "No issues" only for a completely inspected, clean diff), plus the exact reasons for + blocked, unavailable, and not-attempted PRs. When integrity filtering prevents review, state + that meaningful diff review remains unavailable until the gateway supports content-specific + trust compatible with `approved`; bounded attempts and a coverage report do not fix that + dependency. + +After the report is queued, call `report_incomplete` if any selected PR is not **Reviewed**. +Give the exact PR numbers grouped by coverage status and the concrete blocker. Report only +observed denials as blocked; unattempted candidates remain not attempted. A queued report +does not make incomplete code review successful. If report creation fails, call +`report_incomplete` with the publication error as well; do not claim the report was published. -If no open PRs were found, call the `noop` MCP tool with `"No open PRs to review today."`. \ No newline at end of file +Only when discovery succeeded and returned no candidates, call the `noop` MCP tool with +`"No review candidates returned under the current integrity policy."`. Do not call `noop` +when candidates exist but their content could not be reviewed. \ No newline at end of file diff --git a/scripts/test_daily_pr_review_integrity.py b/scripts/test_daily_pr_review_integrity.py new file mode 100644 index 00000000000..0a70c1166cd --- /dev/null +++ b/scripts/test_daily_pr_review_integrity.py @@ -0,0 +1,106 @@ +"""Regression checks for the daily PR review's integrity and coverage contract.""" + +import re +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SOURCE = ROOT / ".github/workflows/daily-pr-review-cursor.md" +LOCK = ROOT / ".github/workflows/daily-pr-review-cursor.lock.yml" + + +class DailyPRReviewIntegrityTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.source = SOURCE.read_text() + cls.frontmatter = cls.source.split("---", 2)[1] + cls.prompt = cls.source.split("---", 2)[2] + cls.text = re.sub(r"\s+", " ", cls.prompt) + + def test_approved_integrity_remains_explicit_without_trust_promotion(self): + self.assertIn(" min-integrity: approved\n", self.frontmatter) + for field in ("trusted-users:", "approval-labels:", "lockdown:", "integrity-proxy:"): + self.assertNotIn(field, self.frontmatter) + self.assertIn("strict: true\n", self.frontmatter) + self.assertIn(" id: awf\n", self.frontmatter) + + def test_candidate_selection_is_bounded_and_metadata_is_not_authorization(self): + for setting in ("state: open", "sort: created", "direction: desc", "perPage: 5"): + self.assertIn(setting, self.prompt) + self.assertIn("Select at most these five returned candidates", self.text) + self.assertIn("not proof that diff content passes the guard", self.text) + self.assertIn("do not expand the search to work around a denial", self.text) + + def test_integrity_denial_stops_repeated_probes_without_fabricating_denials(self): + self.assertIn("making at most one", self.text) + self.assertIn("`method: get_diff`", self.prompt) + self.assertIn("Stop content reads on the first integrity-filtering denial", self.text) + self.assertIn("Do not attempt the remaining candidates after that denial", self.text) + self.assertIn("mark them **Not attempted**", self.text) + self.assertIn("not **Blocked** as though each had returned a denial", self.text) + self.assertIn("gh-aw-mcpg v0.4.35", self.text) + self.assertIn("meaningful diff review remains unavailable", self.text) + + def test_filtered_content_has_no_alternate_transport_or_trust_workaround(self): + self.assertIn("On integrity filtering, do not retry", self.text) + for transport in ( + "`get_files`", + "other GitHub tools", + "shell commands", + "local checkouts", + "raw URLs", + "pre-fetched patches", + ): + self.assertIn(transport, self.text) + self.assertIn("do not prescribe a label or membership change as a fix", self.text) + self.assertIn("not an authentication or model failure", self.text) + + def test_reporting_distinguishes_all_four_coverage_outcomes(self): + coverage = self.prompt.split("Assign exactly one coverage status", 1)[1].split( + "On integrity filtering", 1 + )[0] + self.assertEqual( + re.findall(r"^- \*\*([^*]+)\*\*:", coverage, re.MULTILINE), + ["Reviewed", "Blocked", "Unavailable", "Not attempted"], + ) + self.assertIn("complete diff was returned and inspected", self.text) + self.assertIn("incomplete, truncated, or missing content", self.text) + self.assertIn("never describe this PR as blocked or reviewed", self.text) + self.assertIn("unattempted candidates remain not attempted", self.text) + + def test_unread_content_cannot_receive_a_clean_quality_signal(self): + self.assertIn("For **Reviewed** PRs only", self.text) + self.assertIn("must be `N/A`, never zero or green", self.text) + self.assertIn('"Code quality not assessed"', self.text) + self.assertIn('"No issues" only for a completely inspected, clean diff', self.text) + self.assertIn("Do not infer findings from PR titles or metadata", self.text) + self.assertIn("do not execute PR code, tests, or builds", self.text) + + def test_partial_and_all_blocked_reviews_publish_coverage_and_remain_incomplete(self): + self.assertIn("even when all diff reads were blocked", self.text) + self.assertIn( + "After the report is queued, call `report_incomplete` if any selected PR is not **Reviewed**", + self.text, + ) + self.assertIn("exact PR numbers grouped by coverage status", self.text) + self.assertIn("If report creation fails, call `report_incomplete`", self.text) + + def test_noop_requires_successful_empty_discovery(self): + self.assertIn("If the list call fails or is denied, call `report_incomplete`", self.text) + self.assertIn("Only when discovery succeeded and returned no candidates", self.text) + self.assertIn("No review candidates returned under the current integrity policy", self.text) + self.assertNotIn("No open PRs to review today", self.prompt) + self.assertIn("Do not call `noop` when candidates exist", self.text) + + def test_generated_lock_enforces_approved_integrity(self): + lock = LOCK.read_text() + self.assertRegex(lock, r'"min-integrity":\s*"approved"') + self.assertNotIn('"min-integrity": "none"', lock) + self.assertIn('"guard-policies":', lock) + self.assertIn('"ghcr.io/github/gh-aw-mcpg:', lock) + self.assertIn(".github/workflows/daily-pr-review-cursor.md", lock) + + +if __name__ == "__main__": + unittest.main()