From d125e5c451d4e425ce7ea96f09c9aa54d166de06 Mon Sep 17 00:00:00 2001 From: che cheng Date: Thu, 12 Nov 2026 11:59:49 +0800 Subject: [PATCH 1/2] fix: give the edit-run suite a time limit of its own (#508) The suite uses about 65 s of CPU, so its wall time grows with the load of the machine; at a load of 50 it reached the 180 s default of run-all-tests.sh and was cut off with no failing check. It now has a timeout file of 1800 s. run-all-tests-self checks that the file is there, holds one number of at least 900, and is applied by the runner. --- .claude/.idd/attachments/issue-508/_manifest.json | 6 ++++++ plugins/issue-driven-dev/CHANGELOG.md | 13 +++++++++++++ plugins/issue-driven-dev/scripts/run-all-tests.sh | 4 +++- .../scripts/tests/edit-run/timeout | 1 + .../scripts/tests/run-all-tests-self/test.sh | 15 +++++++++++++++ 5 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 .claude/.idd/attachments/issue-508/_manifest.json create mode 100644 plugins/issue-driven-dev/scripts/tests/edit-run/timeout diff --git a/.claude/.idd/attachments/issue-508/_manifest.json b/.claude/.idd/attachments/issue-508/_manifest.json new file mode 100644 index 00000000..43ab1bcb --- /dev/null +++ b/.claude/.idd/attachments/issue-508/_manifest.json @@ -0,0 +1,6 @@ +{ + "issue": 508, + "fetched_at": "2026-11-12T03:48:51Z", + "fetched_by": "idd-diagnose", + "files": [] +} diff --git a/plugins/issue-driven-dev/CHANGELOG.md b/plugins/issue-driven-dev/CHANGELOG.md index d92bf37d..65743d92 100644 --- a/plugins/issue-driven-dev/CHANGELOG.md +++ b/plugins/issue-driven-dev/CHANGELOG.md @@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +Version number left to the release. + +### Fixed + +- **The edit-run suite has a time limit of its own (#508).** Its run time is almost all CPU time (about 65 s on + an 18-core Mac), so its wall time grows with the load of the machine. At a load of 50 it reached the 180 s + default of `scripts/run-all-tests.sh`, and the full run reported one failed suite although no check had failed. + The suite now has a `timeout` file of 1800 s, which the runner already reads. `run-all-tests-self` checks that + the file is there, holds one number of at least 900, and is applied by the runner. No other suite got one: + apart from merge-completeness, none took more than 60 s in three full runs. + ## [4.0.1] - 2026-11-11 Ships #499. diff --git a/plugins/issue-driven-dev/scripts/run-all-tests.sh b/plugins/issue-driven-dev/scripts/run-all-tests.sh index 3296c792..293c7e00 100755 --- a/plugins/issue-driven-dev/scripts/run-all-tests.sh +++ b/plugins/issue-driven-dev/scripts/run-all-tests.sh @@ -11,7 +11,9 @@ # needs longer puts the seconds in a `timeout` file next to its test.sh; the # larger of the two applies (#399: the merge-completeness suite uses about seven # minutes of CPU, and took 11 minutes of wall time on a Mac, where process start-up -# is slow, hours at a load of 500, and about two minutes on Linux) +# is slow, hours at a load of 500, and about two minutes on Linux; #508: the edit-run +# suite uses about 65 seconds of CPU, and at a load of 50 on 18 cores it reached the +# default and was cut off with no failing check) # - exit 0 iff every suite passed; 1 otherwise # # Env: diff --git a/plugins/issue-driven-dev/scripts/tests/edit-run/timeout b/plugins/issue-driven-dev/scripts/tests/edit-run/timeout new file mode 100644 index 00000000..bb08106d --- /dev/null +++ b/plugins/issue-driven-dev/scripts/tests/edit-run/timeout @@ -0,0 +1 @@ +1800 diff --git a/plugins/issue-driven-dev/scripts/tests/run-all-tests-self/test.sh b/plugins/issue-driven-dev/scripts/tests/run-all-tests-self/test.sh index 92bc4bce..9b5bf43a 100755 --- a/plugins/issue-driven-dev/scripts/tests/run-all-tests-self/test.sh +++ b/plugins/issue-driven-dev/scripts/tests/run-all-tests-self/test.sh @@ -44,6 +44,21 @@ printf '1\n' > "$W/tests/slow/timeout" IDD_TESTS_DIR="$W/tests" IDD_SUITE_TIMEOUT=20 bash "$AGG" > "$W/out6" 2>&1 assert_exit "suite timeout file never lowers the limit → 0" 0 $? +# edit-run declares a limit of its own (#508). Its run time is almost all CPU time (about 65 s on an 18-core +# Mac), so its wall time grows with the load of the machine: at a load of 50 it reached the default 180 s and was +# cut off with no failing check. The file is read the way the aggregator reads it, and then handed to the aggregator. +ER_TIMEOUT="$HERE/../edit-run/timeout" +assert_file_exists "edit-run has a timeout file" "$ER_TIMEOUT" +ER_SECS="$(tr -cd '0-9' < "$ER_TIMEOUT" 2>/dev/null)" +assert_eq "the file holds one number and nothing else (the aggregator keeps every digit it finds)" "${ER_SECS:-no number}" "$(cat "$ER_TIMEOUT" 2>/dev/null)" +[ "${ER_SECS:-0}" -ge 900 ] 2>/dev/null +assert_exit "edit-run's limit is at least 900 s, five times the default" 0 $? +rm -rf "$W/tests/slow" +mk_suite edit-run 'sleep 3' +cp "$ER_TIMEOUT" "$W/tests/edit-run/timeout" 2>/dev/null +IDD_TESTS_DIR="$W/tests" IDD_SUITE_TIMEOUT=1 bash "$AGG" > "$W/out7" 2>&1 +assert_exit "the aggregator applies edit-run's own limit, not the global one → 0" 0 $? + # empty dir → non-zero (zero suites must not read as success) mkdir -p "$W/empty" IDD_TESTS_DIR="$W/empty" bash "$AGG" > "$W/out4" 2>&1 From 66c0cd61286cb4932f00dfcf78b282e87a554f00 Mon Sep 17 00:00:00 2001 From: che cheng Date: Thu, 12 Nov 2026 12:49:26 +0800 Subject: [PATCH 2/2] fix: keep the edit-run limit under the CI job limit and check it in the suite itself (#508) Verify round 1: the stand-in suite in run-all-tests-self was named edit-run, so the nested runner emptied the real suite log; 1800 s was longer than the 20-minute CI job limit, so a hung run on CI would cancel the job with no summary. The checks now live in edit-run/test.sh with a stand-in name no real suite has, the limit is 480 s (checked to stay between 360 and 600), and run-all-tests-self is back to what it was. --- plugins/issue-driven-dev/CHANGELOG.md | 9 ++++-- .../scripts/tests/edit-run/test.sh | 28 +++++++++++++++++++ .../scripts/tests/edit-run/timeout | 2 +- .../scripts/tests/run-all-tests-self/test.sh | 15 ---------- 4 files changed, 35 insertions(+), 19 deletions(-) diff --git a/plugins/issue-driven-dev/CHANGELOG.md b/plugins/issue-driven-dev/CHANGELOG.md index 65743d92..82706fd4 100644 --- a/plugins/issue-driven-dev/CHANGELOG.md +++ b/plugins/issue-driven-dev/CHANGELOG.md @@ -14,9 +14,12 @@ Version number left to the release. - **The edit-run suite has a time limit of its own (#508).** Its run time is almost all CPU time (about 65 s on an 18-core Mac), so its wall time grows with the load of the machine. At a load of 50 it reached the 180 s default of `scripts/run-all-tests.sh`, and the full run reported one failed suite although no check had failed. - The suite now has a `timeout` file of 1800 s, which the runner already reads. `run-all-tests-self` checks that - the file is there, holds one number of at least 900, and is applied by the runner. No other suite got one: - apart from merge-completeness, none took more than 60 s in three full runs. + The suite now has a `timeout` file of 480 s, which the runner already reads. The suite checks its own file: it + is there, it holds one number between 360 and 600, and the runner applies it. The upper bound keeps the limit + under half of the CI job's 20 minutes, so a hung run is still reported by the runner, with its summary, before + the job is cancelled. A longer limit buys little: at a load near 150 on the same machine, actionability-gate + alone took 193 s, so at that load the full run already fails on another suite. No other suite got a file: + apart from merge-completeness, none took more than 60 s in three full runs at ordinary load. ## [4.0.1] - 2026-11-11 diff --git a/plugins/issue-driven-dev/scripts/tests/edit-run/test.sh b/plugins/issue-driven-dev/scripts/tests/edit-run/test.sh index 166c5a62..d2aae2f5 100644 --- a/plugins/issue-driven-dev/scripts/tests/edit-run/test.sh +++ b/plugins/issue-driven-dev/scripts/tests/edit-run/test.sh @@ -21,6 +21,34 @@ HELPER="$PLUGIN/scripts/idd-edit-helper.py" SKILL="$PLUGIN/skills/idd-edit/SKILL.md" FAKE="$HERE/../correct-flow/fake-gh.py" +# ── This suite's own time limit (#508) ── +# Almost all of this suite's run time is CPU time (about 65 s on an 18-core Mac), so its wall time grows with the +# load of the machine: at a load of 50 it reached the aggregator's 180 s default and was cut off with no failing +# check. The `timeout` file next to this script raises the limit for this suite only. The limit has two bounds. +# It has to be well above the default. And it has to leave the aggregator time to report a hung run, with its +# summary, before the CI job's own 20-minute limit cancels the job (the other suites take up to nine minutes there). +LIMIT_FILE="$HERE/timeout" +assert_file_exists "this suite has a timeout file" "$LIMIT_FILE" +LIMIT_SECS="$(tr -cd '0-9' < "$LIMIT_FILE" 2>/dev/null)" +assert_eq "the file holds one number and nothing else (the aggregator keeps every digit it finds)" "${LIMIT_SECS:-no number}" "$(cat "$LIMIT_FILE" 2>/dev/null)" +[ "${LIMIT_SECS:-0}" -ge 360 ] 2>/dev/null +assert_exit "the limit is at least 360 s, twice the default" 0 $? +[ "${LIMIT_SECS:-99999}" -le 600 ] 2>/dev/null +assert_exit "the limit is at most 600 s, half the CI job's limit" 0 $? +# The aggregator applies the file. The stand-in suite must not carry the name of a real suite: the aggregator +# names each suite's log after the suite alone, so a stand-in called edit-run empties the log of this very run (#518). +STANDIN="er-limit" +[ ! -e "$HERE/../$STANDIN" ] +assert_exit "the stand-in suite's name is not the name of a real suite" 0 $? +LIMIT_SB="$(mktemp -d /tmp/idd-er-limit-XXXXXX)" +mkdir -p "$LIMIT_SB/tests/$STANDIN" +printf '#!/usr/bin/env bash\nsleep 3\n' > "$LIMIT_SB/tests/$STANDIN/test.sh" +cp "$LIMIT_FILE" "$LIMIT_SB/tests/$STANDIN/timeout" 2>/dev/null +IDD_TESTS_DIR="$LIMIT_SB/tests" IDD_SUITE_TIMEOUT=1 bash "$PLUGIN/scripts/run-all-tests.sh" > "$LIMIT_SB/out" 2>&1 +assert_exit "the aggregator applies this suite's limit, not the global one → 0" 0 $? +rm -rf "$LIMIT_SB" +# ── end of the time limit checks ── + WORK="$(mktemp -d /tmp/idd-er-XXXXXX)" export TMPDIR="$(mktemp -d /var/tmp/idd-er-tmp-XXXXXX)" cleanup() { chmod -R u+rwx "$WORK" "$TMPDIR" 2>/dev/null; rm -rf "$WORK" "$TMPDIR"; } diff --git a/plugins/issue-driven-dev/scripts/tests/edit-run/timeout b/plugins/issue-driven-dev/scripts/tests/edit-run/timeout index bb08106d..36e08261 100644 --- a/plugins/issue-driven-dev/scripts/tests/edit-run/timeout +++ b/plugins/issue-driven-dev/scripts/tests/edit-run/timeout @@ -1 +1 @@ -1800 +480 diff --git a/plugins/issue-driven-dev/scripts/tests/run-all-tests-self/test.sh b/plugins/issue-driven-dev/scripts/tests/run-all-tests-self/test.sh index 9b5bf43a..92bc4bce 100755 --- a/plugins/issue-driven-dev/scripts/tests/run-all-tests-self/test.sh +++ b/plugins/issue-driven-dev/scripts/tests/run-all-tests-self/test.sh @@ -44,21 +44,6 @@ printf '1\n' > "$W/tests/slow/timeout" IDD_TESTS_DIR="$W/tests" IDD_SUITE_TIMEOUT=20 bash "$AGG" > "$W/out6" 2>&1 assert_exit "suite timeout file never lowers the limit → 0" 0 $? -# edit-run declares a limit of its own (#508). Its run time is almost all CPU time (about 65 s on an 18-core -# Mac), so its wall time grows with the load of the machine: at a load of 50 it reached the default 180 s and was -# cut off with no failing check. The file is read the way the aggregator reads it, and then handed to the aggregator. -ER_TIMEOUT="$HERE/../edit-run/timeout" -assert_file_exists "edit-run has a timeout file" "$ER_TIMEOUT" -ER_SECS="$(tr -cd '0-9' < "$ER_TIMEOUT" 2>/dev/null)" -assert_eq "the file holds one number and nothing else (the aggregator keeps every digit it finds)" "${ER_SECS:-no number}" "$(cat "$ER_TIMEOUT" 2>/dev/null)" -[ "${ER_SECS:-0}" -ge 900 ] 2>/dev/null -assert_exit "edit-run's limit is at least 900 s, five times the default" 0 $? -rm -rf "$W/tests/slow" -mk_suite edit-run 'sleep 3' -cp "$ER_TIMEOUT" "$W/tests/edit-run/timeout" 2>/dev/null -IDD_TESTS_DIR="$W/tests" IDD_SUITE_TIMEOUT=1 bash "$AGG" > "$W/out7" 2>&1 -assert_exit "the aggregator applies edit-run's own limit, not the global one → 0" 0 $? - # empty dir → non-zero (zero suites must not read as success) mkdir -p "$W/empty" IDD_TESTS_DIR="$W/empty" bash "$AGG" > "$W/out4" 2>&1