Skip to content

[spec-review] Update Safe Outputs conformance checker for recent spec changes #70150

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pelikhan merged 1 commit into from
Oct 19, 2026
Merged

[spec-review] Update Safe Outputs conformance checker for recent spec changes #70150

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
335 changes: 330 additions & 5 deletions scripts/check-safe-outputs-conformance.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ set +o histexpand
# Safe Outputs Specification Conformance Checker
# This script implements automated checks for the Safe Outputs specification
# Specification: docs/src/content/docs/specs/safe-outputs-specification.md
# Spec Version: 1.29.8 (2026-10-01; reviewed through 2026-10-09 changes)
# Script Version: 1.29.8 (2026-10-12)
# Spec Version: 1.29.13 (2026-10-18; reviewed through 2026-10-19 changes)
# Script Version: 1.29.13 (2026-10-19)

set -euo pipefail

Expand Down Expand Up @@ -672,7 +672,8 @@ check_mce_constraint_disclosure() {
fi

# Per spec Section 8.3 MCE2: add_comment MUST surface its constraint limits in description
# Required: 65536 char limit, 10 mentions, 50 links (checks the combined tool JSON file)
# Required: 65536 char limit, 10 mentions, and the link limit (checks the combined tool JSON file).
# Since spec v1.29.9 the link limit is the configured max-links (default: 50) in agent content.
if ! grep -iE "65536" "$tools_json" > /dev/null 2>&1; then
log_medium "MCE-001: add_comment tool description may be missing 65536 character limit"
failed=1
Expand All @@ -681,8 +682,8 @@ check_mce_constraint_disclosure() {
log_medium "MCE-001: add_comment tool description may be missing 10 mention limit"
failed=1
fi
if ! grep -iE "50 link" "$tools_json" > /dev/null 2>&1; then
log_medium "MCE-001: add_comment tool description may be missing 50 link limit"
if ! grep -iE "50 link|max-links[^\"]*default: 50" "$tools_json" > /dev/null 2>&1; then
log_medium "MCE-001: add_comment tool description may be missing the link limit (configured max-links, default: 50)"
failed=1
fi

Expand Down Expand Up @@ -2487,6 +2488,330 @@ check_resolve_review_thread_semantics() {
}
check_resolve_review_thread_semantics

# TYPE-016: add_comment Configurable Link Limit (Section 7.1 Enforced Constraints, Section 8.3 MCE5, v1.29.9)
# safe-outputs.add-comment.max-links accepts a positive integer (default 50), counts URL
# occurrences in agent content only (generated footer/metadata links excluded), and MUST be
# enforced at both tool invocation (MCP server) and posting time (handler) with E008.
echo "Running TYPE-016: add_comment Configurable Link Limit..."
check_add_comment_max_links() {
local handler="actions/setup/js/add_comment.cjs"
local mcp_handlers="actions/setup/js/safe_outputs_handlers.cjs"
local helpers="actions/setup/js/comment_limit_helpers.cjs"
local schema="pkg/parser/schemas/main_workflow_schema.json"
local failed=0

if [ ! -f "$helpers" ] || ! grep -qE "function enforceCommentLimits\(body, \{ maxLinks = MAX_LINKS, linkBody = body \}" "$helpers"; then
log_high "TYPE-016: enforceCommentLimits does not accept a configurable maxLinks and agent-only linkBody (generated links must be excluded)"
failed=1
elif ! grep -qE "E008: Comment contains .* links, maximum is \\\$\{maxLinks\}" "$helpers"; then
log_medium "TYPE-016: E008 link-limit error does not report the configured max-links value"
failed=1
fi

if [ ! -f "$handler" ] || ! grep -qE "enforceCommentLimits\(processedBody, \{ maxLinks, linkBody \}\)" "$handler"; then
log_high "TYPE-016: add_comment handler does not enforce max-links at posting time against agent content only"
failed=1
fi

if [ ! -f "$mcp_handlers" ] || ! grep -qE "enforceCommentLimits\(body, \{ maxLinks: addCommentConfig\.max_links" "$mcp_handlers"; then
log_high "TYPE-016: MCP server does not enforce configured max-links at tool invocation (Section 8.3 MCE4)"
failed=1
fi

if [ ! -f "$schema" ] || ! python3 - "$schema" <<'PYEOF' > /dev/null 2>&1
import json, sys
s = json.load(open(sys.argv[1]))
v = s["properties"]["safe-outputs"]["properties"]["add-comment"]
alts = [v] + v.get("oneOf", []) + v.get("anyOf", [])
ml = next((a["properties"]["max-links"] for a in alts if "max-links" in a.get("properties", {})), None)
sys.exit(0 if ml and ml.get("type") == "integer" and ml.get("minimum") == 1 else 1)
PYEOF
then
log_medium "TYPE-016: Schema does not define safe-outputs.add-comment.max-links as a positive integer"
failed=1
fi

if [ $failed -eq 0 ]; then
log_pass "TYPE-016: add_comment max-links is configurable, excludes generated links, and is enforced at invocation and posting time"
fi
}
check_add_comment_max_links

# TYPE-017: add_comment replace-older-comments (Section 7.1 Operational Semantics 8, v1.29.9)
# replace-older-comments MUST NOT be combined with hide-older-comments; matching is limited
# to comments by the authenticated identity (bot-only fallback) carrying the caller workflow
# marker; discussions are excluded; staged mode MUST NOT perform the lookup or update.
echo "Running TYPE-017: add_comment replace-older-comments..."
check_add_comment_replace_older() {
local handler="actions/setup/js/add_comment.cjs"
local schema="pkg/parser/schemas/main_workflow_schema.json"
local failed=0

if [ ! -f "$handler" ]; then
log_high "TYPE-017: add_comment handler missing: $handler"
return
fi

if ! grep -qE "replace-older-comments cannot be combined with hide-older-comments" "$handler"; then
log_high "TYPE-017: Handler does not reject replace-older-comments combined with hide-older-comments"
failed=1
fi
if ! grep -qE 'required": \["hide-older-comments", "replace-older-comments"\]' "$schema" 2>/dev/null; then
log_medium "TYPE-017: Schema does not forbid combining hide-older-comments and replace-older-comments"
failed=1
fi

if ! grep -qE 'comment\.user\.login === authenticatedLogin : comment\.user\.type === "Bot"' "$handler"; then
log_high "TYPE-017: Replacement candidates are not restricted to the authenticated identity (bot-only fallback)"
failed=1
fi

if ! grep -qE "replaceOlderCommentsEnabled && .*!isDiscussion && \(callerWorkflowId \|\| workflowId\)" "$handler"; then
log_high "TYPE-017: Replacement is not limited to issue/PR comments with a caller/workflow ID marker"
failed=1
fi

local staged_line replace_line
staged_line=$(grep -nE "^\s*if \(isStaged\)" "$handler" | head -1 | cut -d: -f1 || true)
replace_line=$(grep -nE "if \(replaceOlderCommentsEnabled" "$handler" | head -1 | cut -d: -f1 || true)
if [ -z "$staged_line" ] || [ -z "$replace_line" ] || [ "$staged_line" -gt "$replace_line" ]; then
log_high "TYPE-017: Staged mode does not return before the replace-older-comments lookup/update"
failed=1
fi

if [ $failed -eq 0 ]; then
log_pass "TYPE-017: replace-older-comments is exclusive with hide-older-comments, identity/marker scoped, and skipped in staged mode"
fi
}
check_add_comment_replace_older

# SCD-001: Signed Commit Delivery (Section 7.3 Signed Commit Delivery Requirements SCD-001..003, v1.29.9)
# Stale-commit rebases MUST keep HEAD attached to the named delivery branch (no explicit
# "HEAD" branch argument), and allowGitPushFallback: false MUST refuse unsigned pushes.
echo "Running SCD-001: Signed Commit Delivery..."
check_signed_commit_delivery() {
local helper="actions/setup/js/push_signed_commits.cjs"
local failed=0

if [ ! -f "$helper" ]; then
log_high "SCD-001: Signed delivery helper missing: $helper"
return
fi

local rebase_lines
rebase_lines=$(grep -E '\["rebase", "--onto"' "$helper" || true)
if [ -z "$rebase_lines" ]; then
log_medium "SCD-001: Could not locate stale-commit rebase in $helper"
failed=1
elif echo "$rebase_lines" | grep -qE '"HEAD"\]'; then
log_high "SCD-001: Rebase passes an explicit HEAD argument, detaching HEAD and leaving the named branch stale"
failed=1
fi

if ! grep -qE "allowGitPushFallback = true" "$helper" || [ "$(grep -cE 'if \(allowGitPushFallback === false\)' "$helper" || true)" -lt 1 ]; then
log_high "SCD-003: allowGitPushFallback: false is not honored to refuse unsigned git-push fallback"
failed=1
fi

if [ ! -f "actions/setup/js/push_signed_commits.test.cjs" ]; then
log_medium "SCD-001: Real-git conformance evidence (push_signed_commits.test.cjs) missing"
failed=1
fi

if [ $failed -eq 0 ]; then
log_pass "SCD-001: Signed delivery rebases keep HEAD attached and honor allowGitPushFallback: false"
fi
}
check_signed_commit_delivery

# PTI-001: Patch and Bundle Transport Integrity (Section 7.3 PTI-001..007, v1.29.12)
# Both PR handlers MUST resolve qualified transport paths, capture bytes with sha256 digest and
# dev/inode identity via nonblocking no-follow opens, consume private read-only snapshots,
# classify failures as ERR_TRANSPORT_INTEGRITY, reject that code before custom retry
# classifiers, and dispose snapshots.
echo "Running PTI-001: Patch and Bundle Transport Integrity..."
check_transport_integrity() {
local ti="actions/setup/js/transport_integrity.cjs"
local resolver="actions/setup/js/resolve_transport_paths.cjs"
local recovery="actions/setup/js/error_recovery.cjs"
local failed=0

if [ ! -f "$ti" ] || [ ! -f "$resolver" ]; then
log_critical "PTI-001: Transport integrity modules missing ($ti, $resolver)"
return
fi

local h
for h in actions/setup/js/create_pull_request.cjs actions/setup/js/push_to_pull_request_branch.cjs; do
if ! grep -q 'require("./transport_integrity.cjs")' "$h" 2>/dev/null || ! grep -q 'require("./resolve_transport_paths.cjs")' "$h" 2>/dev/null; then
log_critical "PTI-001: $h does not use transport integrity guard and trusted path resolution"
failed=1
fi
done

if ! grep -qE "O_NOFOLLOW" "$ti" || ! grep -qE "O_NONBLOCK" "$ti"; then
log_high "PTI-001: Captures do not use nonblocking, no-follow opens (symlinks/FIFOs)"
failed=1
fi
if ! grep -qE 'createHash\("sha256"\)' "$ti" || ! grep -qE "stat\.ino !== identity\.ino" "$ti"; then
log_high "PTI-001: Captures do not bind content digest and device/inode identity"
failed=1
fi
if ! grep -qE "isFile\(\)" "$ti"; then
log_high "PTI-001: Nonregular transport artifacts are not rejected"
failed=1
fi

if ! grep -qE "mkdtempSync" "$ti" || ! grep -qE "mode: 0o400" "$ti"; then
log_high "PTI-002: Git does not consume private read-only snapshots"
failed=1
fi

if ! grep -qE 'this\.code = "ERR_TRANSPORT_INTEGRITY"' "$ti"; then
log_high "PTI-007: Integrity errors do not expose stable ERR_TRANSPORT_INTEGRITY classification"
failed=1
fi

if [ ! -f "$recovery" ] || ! grep -qE 'error\?\.code === "ERR_TRANSPORT_INTEGRITY" \|\| !fullConfig\.shouldRetry\(error\)' "$recovery"; then
log_high "PTI-005: Retry loop does not reject integrity failures before invoking custom retry classifiers"
failed=1
fi

if ! grep -qE "function dispose\(\)" "$ti" || ! grep -qE "rmSync\(directory" "$ti"; then
log_high "PTI-006: Private snapshots are not disposed"
failed=1
fi

if [ $failed -eq 0 ]; then
log_pass "PTI-001: Patch/bundle transport binds captured bytes, uses private snapshots, and fails permanently on integrity errors"
fi
}
check_transport_integrity

# RCR-001: Runtime Repository-List Checkouts (Section 9.6.1 RCR8-RCR12, v1.29.10-1.29.12)
# Only literal multi: true enables list expansion; single-mode selectors MUST resolve to one
# repository; max-repos defaults to 10 and fails in full; private/internal targets require
# allow-private-repos: true; runtime lists MUST NOT fetch LFS or submodules.
echo "Running RCR-001: Runtime Repository-List Checkouts..."
check_runtime_repository_checkouts() {
local parser="pkg/workflow/checkout_config_parser.go"
local runtime="actions/setup/js/checkout_dynamic.cjs"
local failed=0

if [ ! -f "$parser" ] || ! grep -qE "checkout\.multi must be a literal boolean" "$parser"; then
log_high "RCR-001: Compiler does not require checkout.multi to be a literal boolean (RCR8)"
failed=1
fi
if [ -f "$parser" ] && ! grep -qE "must select a single repository unless checkout\.multi is true" "$parser"; then
log_high "RCR-001: Single-mode repository selectors are not restricted to one repository (RCR8)"
failed=1
fi

if [ ! -f "$runtime" ]; then
log_high "RCR-001: Runtime checkout preparation missing: $runtime"
return
fi
if ! grep -qE 'GH_AW_DYNAMIC_MAX_REPOS \?\? "10"' "$runtime" || ! grep -qE "exceeding checkout\.max-repos" "$runtime"; then
log_high "RCR-001: max-repos (default 10) is not enforced before checkout (RCR9)"
failed=1
fi
if ! grep -qE "requires checkout\.allow-private-repos: true" "$runtime" || ! grep -qE "allow-private-repos must be a literal boolean" "$runtime"; then
log_critical "RCR-001: Private/internal runtime targets are not default-denied behind literal allow-private-repos (RCR9)"
failed=1
fi
if ! grep -qE "isPublic && \(data\.private \|\| data\.visibility !== \"public\"\)" "$runtime"; then
log_critical "RCR-001: Public hosting repositories are not prevented from checking out private/internal targets (RCR9)"
failed=1
fi
if ! grep -qE "submodules: false" "$runtime" || ! grep -qE "lfs: false" "$runtime" || ! grep -qE '"--no-recurse-submodules"' "$runtime"; then
log_high "RCR-001: Runtime lists do not hardcode lfs/submodules false and --no-recurse-submodules (RCR12)"
failed=1
fi

if [ $failed -eq 0 ]; then
log_pass "RCR-001: Runtime repository-list checkouts enforce explicit multi mode, bounded preflight, private opt-in, and no secondary fetches"
fi
}
check_runtime_repository_checkouts

# BGO-001: Bundle Git Operations and Validation Baselines (Section 7.3 BGO-001..008, v1.29.13)
# Bundle inspection MUST be offline (GIT_ALLOW_PROTOCOL="" + GIT_NO_LAZY_FETCH=1);
# allow-full-history-fetch MUST be a literal boolean defaulting to false per handler and gate
# prerequisite recovery with a no-byte-limit warning; allow-merged-base-baseline MUST be a
# literal opt-in; optional discovery MUST be bounded (5s, 1 MiB, 100(+1) records, 4 REST calls,
# retries disabled).
echo "Running BGO-001: Bundle Git Operations and Validation Baselines..."
check_bundle_git_operations() {
local helpers="actions/setup/js/git_helpers.cjs"
local push="actions/setup/js/push_to_pull_request_branch.cjs"
local create="actions/setup/js/create_pull_request.cjs"
local schema="pkg/parser/schemas/main_workflow_schema.json"
local failed=0

if [ ! -f "$helpers" ] || ! grep -qE 'GIT_ALLOW_PROTOCOL: "", GIT_NO_LAZY_FETCH: "1"' "$helpers"; then
log_critical "BGO-001: Bundle inspection does not deny all Git transports (GIT_ALLOW_PROTOCOL=\"\" and GIT_NO_LAZY_FETCH=1)"
failed=1
fi

if [ -f "$helpers" ]; then
if ! grep -qE "if \(allowFullHistoryFetch !== true\) throw" "$helpers"; then
log_critical "BGO-002: Bundle prerequisite recovery is not gated on literal allow-full-history-fetch: true"
failed=1
fi
if ! grep -qE "without a byte limit" "$helpers"; then
log_medium "BGO-002: Opted-in recovery does not warn that downloads have no byte limit"
failed=1
fi
fi

if ! grep -qE "config\.allow_full_history_fetch === true" "$create" 2>/dev/null || ! grep -qE "allowFullHistoryFetch: config\.allow_full_history_fetch === true" "$push" 2>/dev/null; then
log_high "BGO-002: Handlers do not scope allow-full-history-fetch to their own trusted configuration"
failed=1
fi

if [ ! -f "$schema" ] || ! python3 - "$schema" <<'PYEOF' > /dev/null 2>&1
import json, sys
s = json.load(open(sys.argv[1]))
so = s["properties"]["safe-outputs"]["properties"]
for key in ("create-pull-request", "push-to-pull-request-branch"):
v = so[key]
alts = [v] + v.get("oneOf", []) + v.get("anyOf", [])
p = next((a["properties"]["allow-full-history-fetch"] for a in alts if "allow-full-history-fetch" in a.get("properties", {})), None)
if not p or p.get("type") != "boolean" or p.get("default") is not False:
sys.exit(1)
PYEOF
then
log_high "BGO-002: Schema does not define allow-full-history-fetch as a boolean defaulting to false for both PR handlers"
failed=1
fi

if [ ! -f "$push" ]; then
log_high "BGO-004: push_to_pull_request_branch handler missing: $push"
return
fi
if ! grep -qE "allowMergedBaseBaseline = config\.allow_merged_base_baseline === true" "$push"; then
log_high "BGO-004: Older-ancestor baseline discovery is not disabled by default behind literal allow-merged-base-baseline: true"
failed=1
fi
if ! grep -qE "MAX_BUNDLE_PARENT_COMPARISONS = 4;" "$push" || ! grep -qE "MAX_BUNDLE_HISTORY_COMMITS = 100;" "$push" || ! grep -qE "MAX_BUNDLE_HISTORY_COMMITS \+ 1" "$push"; then
log_high "BGO-005: Discovery is not bounded to 4 REST comparisons and 100 records with 101st-record overflow detection"
failed=1
fi
if ! grep -qE "BUNDLE_VALIDATION_TIMEOUT_MS = 5000;" "$push" || ! grep -qE "BUNDLE_VALIDATION_MAX_OUTPUT = 1024 \* 1024;" "$push"; then
log_high "BGO-005: Optional discovery commands lack the 5-second timeout and 1 MiB output limit"
failed=1
fi
if ! grep -qE "retries: 0, signal: AbortSignal\.timeout\(BUNDLE_VALIDATION_TIMEOUT_MS\)" "$push"; then
log_high "BGO-005: REST comparisons do not disable retries and enforce a 5-second abort deadline"
failed=1
fi

if [ $failed -eq 0 ]; then
log_pass "BGO-001: Bundle Git operations are offline by default, recovery is literal opt-in, and baseline discovery is bounded"
fi
}
check_bundle_git_operations

# Summary
echo ""
echo "=================================================="
Expand Down
Loading