diff --git a/.github/workflows/CronModelingToolkit.yml b/.github/workflows/CronModelingToolkit.yml index 37d930767..92fcd029b 100644 --- a/.github/workflows/CronModelingToolkit.yml +++ b/.github/workflows/CronModelingToolkit.yml @@ -2,11 +2,12 @@ name: Cron ModelingToolkit on: schedule: - - cron: "0 4 * * 6" + # daily: a rotation (#435), each run tests the problems tested longest ago for an hour + - cron: "0 4 * * *" workflow_dispatch: inputs: problems: - description: "Only these problems, comma-separated (e.g. beam): publishes their results, leaves the reports, the state and the issues" + description: "These problems first, comma-separated (e.g. beam), then the others while the budget lasts" required: false default: '' type: string @@ -26,5 +27,6 @@ jobs: with: environment: ModelingToolkit problems: ${{ inputs.problems }} - note: "`CronModelingToolkit.yml` · weekly (Saturday 04:00 UTC) · Julia 1.13, Ubuntu + Windows, CPU" + budget: 3600 + note: "`CronModelingToolkit.yml` · daily (04:00 UTC), a rotation with a budget of 1 h (#435) · Julia 1.13, Ubuntu + Windows, CPU" secrets: inherit diff --git a/.github/workflows/README.md b/.github/workflows/README.md index baf4ce864..3df9e7288 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -9,7 +9,7 @@ scope labels. They are staggered at 04:00 UTC: | `CronOptimalControl.yml` | `OptimalControl` | Wednesday | Yes | | `CronOptimalControlScalarized.yml` | `OptimalControlScalarized` | Thursday | Yes | | `CronComparison.yml` | `comparison` | Friday | No | -| `CronModelingToolkit.yml` | `ModelingToolkit` | Saturday | No | +| `CronModelingToolkit.yml` | `ModelingToolkit` | Daily, a rotation (see below) | No | | `CronRobustness.yml` | `JuMP-robustness`, `OptimalControl-robustness` | 1st of the month | No | Each thin workflow calls `cron-reusable.yml` (inputs `mode`: `cron`, or `robustness` for @@ -69,8 +69,34 @@ a run restricted to some problems (`problem=`) leaves the other problems as they environment is removed once no entry refers to it. Everything is written by `tools/cron/report.py` (`--results-dir`), tested in `tools/cron/test/`. +## Rotation with a time budget + +A cron with a `budget` input (seconds; 0, the default, runs every problem) is a rotation +(#435): `ModelingToolkit` for now, with 1 h, daily. Its `select` job orders the problems of +the modeling, those with a file in `ext/Models/`, from the one tested longest ago to the +last tested. The date is that of its last result on `results`, and a problem never tested +comes first (`tools/cron/select.py`). The tests then run each problem in its own Julia +process, in that order, and start no new one once the budget is spent +(`test_args`: `problem= budget=`, see `test/README.md`). A process that crashes +(e.g. out of memory) or runs past its timeout (45 min) is reported as an `error` of its +problem only, so the problem at fault is named and the others are tested. A runner killed +before it uploads its report (run 44747569284: the Ubuntu runner died during a problem) is a +`workflow` error of its platform: the report expects one report from each runner of the +`runs_on` input (`--runners` of `tools/cron/report.py`). + +A run of a rotation tests some problems only, so its reports and issues are built from the +**last result of every problem** (`--rotation` of `tools/cron/report.py`, the `latest` entry +of `cron/state.json`): +- a problem that did not run keeps its findings, and is never "resolved" by a run that + did not test it; +- the regression issues of a problem change only when it runs; +- the scope report ends with the date each problem was last tested. + +A run that crashed wrote no result, so its problems come first again in the next run. The +`problems` input of a rotation puts those problems first; the run then goes on as usual. + A convergence cron dispatched by hand takes an optional `problems` input (`beam` or -`beam,chain`): it solves those problems only, and publishes their results and environments. +`beam,chain`). Without a budget, it solves those problems only, and publishes their results and environments. Its report and its state are written aside, unpublished, and the issues are left as they are: a restricted run cannot tell about the problems it did not solve, so `cron/.md`, `cron/state.json` and the issues stay those of the last complete run. Use it to check a diff --git a/.github/workflows/cron-reusable.yml b/.github/workflows/cron-reusable.yml index a23459ba2..c2ff292fb 100644 --- a/.github/workflows/cron-reusable.yml +++ b/.github/workflows/cron-reusable.yml @@ -40,6 +40,23 @@ on: required: false default: '' type: string + runs_on: + description: >- + The runners of the CPU job, a JSON list. The report expects one report from each: + a runner that leaves none (killed, #435) is a finding. macOS is left out for now + (scarce runners, long queues); it comes back with #344. + required: false + type: string + default: '["ubuntu-latest", "windows-latest"]' + budget: + description: >- + Time budget of a run, in seconds (#435). 0, the default: every problem, in one + process per environment. Otherwise a rotation: the problems of the modeling, tested + longest ago first (tools/cron/select.py), one process each, while the budget lasts; + the reports and issues keep the last result of every problem. + required: false + default: 0 + type: number secrets: SSH_KEY: required: false @@ -52,11 +69,64 @@ permissions: contents: read jobs: + # The arguments of the tests: in a rotation (budget > 0), the problems of the modeling, the + # ones tested longest ago first, read from the `results` branch (#435). + select: + if: inputs.enabled + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + test_args: ${{ steps.args.outputs.test_args }} + problems: ${{ steps.args.outputs.problems }} + steps: + - name: Checkout package + uses: actions/checkout@v4 + + - name: Test arguments + id: args + shell: bash + env: + ENVIRONMENT: ${{ inputs.environment }} + MODE: ${{ inputs.mode }} + PROBLEMS: ${{ inputs.problems }} + BUDGET: ${{ inputs.budget }} + SCOPE: ${{ inputs.scope || inputs.environment }} + run: | + set -euo pipefail + args="$ENVIRONMENT $MODE" + problems="" + if [ "$BUDGET" != "0" ]; then + results="" + if git ls-remote --exit-code origin results >/dev/null 2>&1; then + git fetch --depth 1 origin results + git worktree add --detach "$RUNNER_TEMP/results" FETCH_HEAD + results="$RUNNER_TEMP/results" + fi + problems=$(python3 tools/cron/select.py --scope "$SCOPE" \ + --models-dir "ext/${ENVIRONMENT}Models" --metadata-dir ext/MetaData \ + ${results:+--results-dir "$results"}) + if [ -z "$problems" ]; then + echo "::error::a budget needs the problems of ext/${ENVIRONMENT}Models/" + exit 1 + fi + # problems given by hand come first, in their order + order="${PROBLEMS:+$PROBLEMS,}$problems" + order=$(echo "$order" | tr ',' '\n' | awk 'NF && !seen[$0]++' | paste -sd, -) + args="$args problem=$order budget=$BUDGET" + elif [ -n "$PROBLEMS" ]; then + args="$args problem=$PROBLEMS" + fi + echo "test_args=$args" >> "$GITHUB_OUTPUT" + echo "problems=$problems" >> "$GITHUB_OUTPUT" + echo "Test arguments: $args" + # The called CTActions workflows request `actions`, `contents` and `pull-requests` read # (their `changes` job) and `statuses` write (their `status-start` and `status-end` jobs); a # nested job cannot be granted less than it requests. cpu: if: inputs.enabled + needs: select permissions: actions: read contents: read @@ -65,12 +135,11 @@ jobs: uses: control-toolbox/CTActions/.github/workflows/ci.yml@main with: versions: '["1.13"]' - # macOS is left out for now (scarce runners, long queues); it comes back with #344. - runs_on: '["ubuntu-latest", "windows-latest"]' + runs_on: ${{ inputs.runs_on }} # keep every platform running after a failure: one report per platform is needed fail_fast: false use_ct_registry: true - test_args: ${{ inputs.environment }} ${{ inputs.mode }}${{ inputs.problems && format(' problem={0}', inputs.problems) || '' }} + test_args: ${{ needs.select.outputs.test_args }} upload_artifact_path: test/reports upload_artifact_name: cron-${{ inputs.scope || inputs.environment }} secrets: @@ -78,6 +147,7 @@ jobs: gpu: if: inputs.enabled && inputs.gpu + needs: select permissions: actions: read contents: read @@ -87,7 +157,7 @@ jobs: with: versions: '["1.13"]' use_ct_registry: true - test_args: ${{ inputs.environment }} ${{ inputs.mode }}${{ inputs.problems && format(' problem={0}', inputs.problems) || '' }} + test_args: ${{ needs.select.outputs.test_args }} upload_artifact_path: test/reports upload_artifact_name: cron-${{ inputs.scope || inputs.environment }}-gpu fail_fast: false @@ -98,7 +168,7 @@ jobs: publish: if: always() && inputs.enabled - needs: [cpu, gpu] + needs: [select, cpu, gpu] runs-on: ubuntu-latest permissions: contents: write @@ -113,8 +183,12 @@ jobs: RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} TESTS_FAILED: ${{ needs.cpu.result != 'success' || needs.gpu.result == 'failure' }} RESULTS_DIR: /tmp/cron-results - # a restricted run writes its report and state aside, unpublished (`problems` input) - PARTIAL: ${{ inputs.problems != '' }} + RUNNERS: ${{ join(fromJSON(inputs.runs_on), ',') }} + # a restricted run writes its report and state aside, unpublished (`problems` input); + # not in a rotation, where every run tests some problems and updates their results + PARTIAL: ${{ inputs.problems != '' && inputs.budget == 0 }} + # a rotation (#435): every problem of the scope, for the reports and issues + ROTATION: ${{ needs.select.outputs.problems }} steps: - name: Checkout package uses: actions/checkout@v4 @@ -178,6 +252,8 @@ jobs: --julia-version '1.13' \ --package-version "$package_version" \ --scope-note "$SCOPE_NOTE" \ + --rotation "$ROTATION" \ + --runners "$RUNNERS" \ "${test_failed_arg[@]}" - name: Publish report, results and observed state @@ -216,6 +292,8 @@ jobs: --julia-version '1.13' \ --package-version "$(sed -n 's/^version = \"\(.*\)\"/\1/p' Project.toml | head -1)" \ --scope-note "$SCOPE_NOTE" \ + --rotation "$ROTATION" \ + --runners "$RUNNERS" \ --issues \ "${test_failed_arg[@]}" diff --git a/test/README.md b/test/README.md index ceb1cf8f2..45ed60dc6 100644 --- a/test/README.md +++ b/test/README.md @@ -11,7 +11,7 @@ test/ ├── Project.toml # light environment of the dispatcher and of core (Test, Aqua, CTBase) ├── common/ # shared code, included by the tests (configurations, statuses, solvers) ├── suite// # fast tests, run on every pull request -├── cron// # slow tests of the same group, run by the weekly cron +├── cron// # slow tests of the same group, run by the crons └── robustness// # robustness report of the group, run by the monthly cron (#378) ``` @@ -42,6 +42,7 @@ Pkg.test(; test_args=["OptimalControl", "convergence"]) # suite and convergence Pkg.test(; test_args=["JuMP", "cron", "*convergence*"]) # environments, mode and file globs Pkg.test(; test_args=["problem=beam"]) # one problem, in every environment Pkg.test(; test_args=["JuMP", "problem=beam,robot"]) # some problems, in one environment +Pkg.test(; test_args=["ModelingToolkit", "cron", "problem=beam,chain", "budget=3600"]) # a rotation ``` `test_args` is a list of: @@ -57,6 +58,13 @@ Pkg.test(; test_args=["JuMP", "problem=beam,robot"]) # some problems, in one - `problem=a,b`: test only these problems, for instance the one you add. A name that is not in the catalogue is an error (a selection matching nothing would test nothing and pass). `core` still runs entirely. +- `budget=` (with `problem=`), and optionally `timeout=` (45 min by + default): the rotation of the crons (#435). Each problem runs in its own Julia process, + in the order of `problem=`, and no new one starts once the budget is spent; a process past + its timeout is stopped. A problem whose process crashed or was stopped gets the report row + `process (crashed)` or `process (timeout after … s)`, an `error` + (`TestStatus.write_process_report`). Each problem writes its own report, + `test/reports/-cron-.md`. In cron mode, an environment without a `cron//` directory is skipped. @@ -275,7 +283,7 @@ the `System` of every problem is read as the data of a JuMP problem (`common/continuous_modelingtoolkit.jl`) and compared with the OptimalControl model by `Continuous.compare`, as `suite/comparison/test_modelings` does for JuMP. It runs in this environment, which therefore loads OptimalControl too, so that the `comparison` environment -stays free of ModelingToolkit. Its convergence report (`cron/ModelingToolkit`, weekly) has no +stays free of ModelingToolkit. Its convergence report (`cron/ModelingToolkit`, daily, a rotation with a budget, #435) has no reference objective for a shooting discretization: the gap is to the objective of the analytic solution, when the problem has one, and `violation` is the largest violation of the constraints of the NLP solved. The costates are not computed by Corleone and are left empty. diff --git a/test/common/selection.jl b/test/common/selection.jl index 038ef3c52..7cf0ed92d 100644 --- a/test/common/selection.jl +++ b/test/common/selection.jl @@ -90,6 +90,81 @@ end const MODES = ("cron", "convergence", "robustness") +# The time a problem may take in a rotation (#435) before its process is stopped, by default. +const DEFAULT_TIMEOUT = 2700.0 + +""" + split_budget(args) -> (budget, rest) + +Take the `budget=` and `timeout=` arguments out of `args` (#435). With a +budget, the problems of an environment run one per process, in the order of `problem=`, +while the budget lasts (see [`run_within_budget`](@ref)); `timeout` is the time a problem +may take before its process is stopped ([`DEFAULT_TIMEOUT`](@ref) by default). `budget` is +`nothing` without `budget=`, a `NamedTuple` `(seconds, timeout)` otherwise. +""" +function split_budget(args) + values = Dict{String,Float64}() + rest = String[] + for arg in args + key = findfirst(k -> startswith(arg, k * "="), ("budget", "timeout")) + if isnothing(key) + push!(rest, arg) + continue + end + name = ("budget", "timeout")[key] + value = tryparse(Float64, chopprefix(arg, name * "=")) + (isnothing(value) || value <= 0) && + throw(ArgumentError("`$arg`: $name must be a positive number of seconds")) + values[name] = value + end + haskey(values, "budget") || return nothing, rest + return (seconds=values["budget"], timeout=get(values, "timeout", DEFAULT_TIMEOUT)), rest +end + +""" + run_within_budget(f, problems, seconds; clock=time) -> (outcomes, skipped) + +Call `f(problem)` for the `problems` in their order while less than `seconds` have passed +since the start (#435): the first one always runs, a problem starts only if the budget is +not spent. Returns the `problem => f(problem)` pairs, in order, and the problems left for a +later run. +""" +function run_within_budget(f, problems, seconds::Real; clock=time) + start = clock() + outcomes = Pair{String,Symbol}[] + skipped = String[] + for problem in problems + if clock() - start >= seconds + push!(skipped, problem) + else + push!(outcomes, problem => f(problem)) + end + end + return outcomes, skipped +end + +""" + run_process(cmd; timeout) -> Symbol + +Run `cmd`, its output shown, and stop it after `timeout` seconds. Returns `:success`, +`:failed` (a non-zero exit code: a test failed), `:crashed` (stopped by a signal, e.g. out of +memory) or `:timeout`. +""" +function run_process(cmd::Cmd; timeout::Real) + process = run(ignorestatus(cmd); wait=false) + start = time() + while process_running(process) + if time() - start > timeout + kill(process) + wait(process) + return :timeout + end + sleep(1) + end + success(process) && return :success + return process.termsignal == 0 ? :failed : :crashed +end + """ directories(group, suite, cron, robustness; mode) -> Vector{String} diff --git a/test/common/status.jl b/test/common/status.jl index 17d60ea97..44acf98c6 100644 --- a/test/common/status.jl +++ b/test/common/status.jl @@ -243,7 +243,7 @@ function _trajectory(x) return Dict(name => [r(f(τ)[i]) for τ in t] for (i, name) in enumerate(names)) end xs, us = O.state_components(x), O.control_components(x) - scalar(f) = τ -> (y=f(τ); y isa Number ? [y] : y) + scalar(f) = τ -> (y = f(τ); y isa Number ? [y] : y) return Dict{String,Any}( "time" => r.(t), "state" => components(scalar(O.state(x)), xs), @@ -328,6 +328,40 @@ end const REPORTS = joinpath(dirname(@__DIR__), "reports") +""" + write_process_report(report, problem, environment, outcome; timeout) + +The report of a problem whose process gave no report of its own (#435): stopped by a signal +(`:crashed`, e.g. out of memory) or after `timeout` seconds (`:timeout`). One row, an +`error` of the test `process`, so that the cron reports it for this problem only. +""" +function write_process_report( + report::AbstractString, + problem::AbstractString, + environment::AbstractString, + outcome::Symbol; + timeout::Real, +) + file = joinpath(REPORTS, "$report.md") + isfile(file) && return file + reason = outcome === :timeout ? "timeout after $(round(Int, timeout)) s" : "crashed" + status = get( + OptimalControlProblems.status(Symbol(problem)), Symbol(environment), :valid + ) + mkpath(REPORTS) + write( + file, + """ + # Test report: $report + + | test | problem | modelings | status | outcome | + | --- | --- | --- | --- | --- | + | process ($reason) | $problem | $environment | $status | error | + """, + ) + return file +end + """ Print the report and write it to `test/reports/.md`, with the results of the solves (#360) as a second table and as `test/reports/-results.tsv`, the environment of the run diff --git a/test/runtests.jl b/test/runtests.jl index e87db973e..9a708f7e5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -20,6 +20,7 @@ # julia --project -e 'using Pkg; Pkg.test(; test_args=["robustness"])' # robustness/ dirs # julia --project -e 'using Pkg; Pkg.test(; test_args=["JuMP", "cron", "*beam*"])' # julia --project -e 'using Pkg; Pkg.test(; test_args=["JuMP", "problem=beam,robot"])' +# julia --project -e 'using Pkg; Pkg.test(; test_args=["ModelingToolkit", "cron", "problem=beam,chain", "budget=3600"])' # # `test_args` is a list of: environment names (none given: every environment except the # opt-in ones below), the word `cron` (run `cron//test_*.jl` instead of @@ -29,6 +30,11 @@ # `test_` and `.jl`); # - `problem=a,b`: test only these problems (a name not in the catalogue is an error), for # instance the one you add; `core` and the environments are unchanged otherwise. +# - `budget=` (and `timeout=`): the rotation of the crons (#435). Each +# problem of `problem=` runs in its own Julia process, in the order given (the cron lists +# the problems tested longest ago first), while the budget lasts; a process past `timeout` +# is stopped, and a problem whose process crashed or was stopped is reported as an error of +# this problem (`TestStatus.write_process_report`). # # Test files follow TST1: a module and an entry function `test_()`. Their discovery, # their selection by the patterns and their execution are `CTBase.DevTools.run_tests`; the @@ -73,6 +79,7 @@ function run_environment( dirs::Vector{String}, report::AbstractString, patterns::Vector{String}, + budget=nothing, ) # In GitHub Actions the project is kept (see `finally`), also past the exit of Julia. keep = get(ENV, "GITHUB_ACTIONS", "false") == "true" @@ -87,13 +94,31 @@ function run_environment( setup = "using Pkg: Pkg; Pkg.develop(path=ARGS[1]); Pkg.instantiate()" run(`$julia --project=$project -e $setup $ROOT`) script = joinpath(@__DIR__, "common", "run_environment.jl") - problem_args = - isempty(problem_names) ? String[] : ["problem=" * join(problem_names, ",")] tests = join(dirs, "|") - run( - `$julia --project=$project $script $tests $name $report $problem_args $patterns` - ) - return true + if isnothing(budget) + problem_args = + isempty(problem_names) ? String[] : ["problem=" * join(problem_names, ",")] + run( + `$julia --project=$project $script $tests $name $report $problem_args $patterns`, + ) + return true + end + # the rotation (#435): one process per problem, in order, while the budget lasts + outcomes, skipped = + TestSelection.run_within_budget(problem_names, budget.seconds) do p + problem_report = "$report-$p" + rm(joinpath(TestStatus.REPORTS, "$problem_report.md"); force=true) + cmd = `$julia --project=$project $script $tests $name $problem_report problem=$p $patterns` + outcome = TestSelection.run_process(cmd; timeout=budget.timeout) + outcome in (:crashed, :timeout) && TestStatus.write_process_report( + problem_report, p, name, outcome; timeout=budget.timeout + ) + println("Problem $p: $outcome") + return outcome + end + isempty(skipped) || + println("Budget spent, left for a later run: ", join(skipped, ", ")) + return all(o -> last(o) === :success, outcomes) catch err @error "Environment $name failed" exception = err return false @@ -107,7 +132,11 @@ function run_environment( end mode, args = TestSelection.split_mode(String.(ARGS)) +budget, args = TestSelection.split_budget(args) problem_names, rest = TestSelection.split_problems(args) +isnothing(budget) || + !isempty(problem_names) || + throw(ArgumentError("`budget=` needs `problem=`: the problems to run, in order")) TestStatus.select_problems!(problem_names) available = environments() selected, patterns = TestSelection.select_environments(rest, available, OPT_IN; core=CORE) @@ -135,7 +164,7 @@ Test.@testset "OptimalControlProblems tests" verbose = Main.TestOptions.VERBOSE Test.@testset "environment $name" begin report = mode in (:cron, :robustness) ? "$name-$mode" : name Test.@test run_environment( - name; dirs=dirs(name), report=report, patterns=patterns + name; dirs=dirs(name), report=report, patterns=patterns, budget=budget ) end end diff --git a/test/suite/core/test_runner.jl b/test/suite/core/test_runner.jl index 85f66110a..68b337c4f 100644 --- a/test/suite/core/test_runner.jl +++ b/test/suite/core/test_runner.jl @@ -67,6 +67,50 @@ function test_runner() (["a", "b", "c"], String[]) end + Test.@testset "budget (#435)" begin + Test.@test R.split_budget(["JuMP", "problem=a"]) == + (nothing, ["JuMP", "problem=a"]) + Test.@test R.split_budget(["budget=3600", "cron"]) == + ((seconds=3600.0, timeout=R.DEFAULT_TIMEOUT), ["cron"]) + Test.@test R.split_budget(["timeout=60", "budget=600"]) == + ((seconds=600.0, timeout=60.0), String[]) + # a timeout alone changes nothing: the rotation needs a budget + Test.@test R.split_budget(["timeout=60"]) == (nothing, String[]) + Test.@test_throws ArgumentError R.split_budget(["budget=0"]) + Test.@test_throws ArgumentError R.split_budget(["budget=one"]) + + # a clock that advances 10 s per problem: the budget is checked before each one + now = Ref(0.0) + clock() = now[] + ran = String[] + outcomes, skipped = R.run_within_budget(["a", "b", "c", "d"], 25; clock) do p + push!(ran, p) + now[] += 10 + return :success + end + Test.@test ran == ["a", "b", "c"] + Test.@test outcomes == ["a" => :success, "b" => :success, "c" => :success] + Test.@test skipped == ["d"] + # the first problem always runs, even past the budget + now[] = 0.0 + outcomes, skipped = R.run_within_budget(["a", "b"], 5; clock) do p + now[] += 10 + return :failed + end + Test.@test outcomes == ["a" => :failed] + Test.@test skipped == ["b"] + + julia = Base.julia_cmd() + Test.@test R.run_process(`$julia --startup-file=no -e 0`; timeout=60) === + :success + Test.@test R.run_process( + `$julia --startup-file=no -e "exit(1)"`; timeout=60 + ) === :failed + Test.@test R.run_process( + `$julia --startup-file=no -e "sleep(60)"`; timeout=2 + ) === :timeout + end + Test.@testset "run" begin RAN = Main.TEST_RAN mktempdir() do dir diff --git a/tools/cron/report.py b/tools/cron/report.py index bb66a8e63..45ddc021c 100644 --- a/tools/cron/report.py +++ b/tools/cron/report.py @@ -81,7 +81,7 @@ "ModelingToolkit": { "title": "ModelingToolkit — Nominal Convergence", "focus": "Symbolic modeling backend with ModelingToolkit.jl", - "cadence": "Weekly (Saturday 04:00 UTC) · Julia 1.13, Ubuntu + Windows (CPU)", + "cadence": "Daily (04:00 UTC), a rotation with a budget of 1 h (#435) · Julia 1.13, Ubuntu + Windows (CPU)", }, } @@ -235,23 +235,103 @@ def result_key(row: dict[str, str]) -> str: def remember_results( - state: dict, scope: str, results: list[dict[str, str]], run_url: str + state: dict, + scope: str, + results: list[dict[str, str]], + run_url: str, + rotation: bool = False, ) -> dict[str, dict[str, str]]: """Store the results of this run in `state` and return those of the previous run. The report is written twice per run (report, then issues): the previous results are - kept with the run they belong to, so that both calls compare with the same run.""" + kept with the run they belong to, so that both calls compare with the same run. In a + rotation (#435) a run tests some problems only: the results of the others are kept, and + those of a problem that ran replace all of its own.""" entry = state.setdefault("convergence", {}).setdefault(scope, {}) if entry.get("run") != run_url: entry["previous"] = entry.get("results", {}) entry["run"] = run_url - entry["results"] = { + current = { result_key(row): {"result": row["result"], "objective": row["objective"]} for row in results } + if rotation: + ran = {row["problem"] for row in results} + kept = { + key: value + for key, value in entry.get("previous", {}).items() + if key.split("|")[1] not in ran + } + current = {**kept, **current} + entry["results"] = current return entry.get("previous", {}) +def remember_latest( + state: dict, + scope: str, + rows: list[dict[str, str]], + results: list[dict[str, str]], + degradation_rows: list[dict[str, str]], + run_url: str, + generated: str, + scope_problems: list[str], +) -> dict[str, dict]: + """The last test rows, results and convergence findings of every problem of `scope`, this + run's for the problems it tested (#435): a rotating cron tests some problems per run, and + its reports and issues are built from the last result of each problem, so that a problem + that did not run keeps its findings instead of looking resolved. Problems no longer in + `scope_problems` (no model file any more) are dropped. Returns the entry, by problem.""" + entry = state.setdefault("latest", {}).setdefault(scope, {}) + ran = ({row["problem"] for row in rows} | {row["problem"] for row in results}) - { + "workflow" + } + for problem in ran: + entry[problem] = { + "run": run_url, + "date": generated, + "rows": [row for row in rows if row["problem"] == problem], + "results": [row for row in results if row["problem"] == problem], + "degradations": [row for row in degradation_rows if row["problem"] == problem], + } + for problem in [p for p in entry if p not in scope_problems]: + del entry[problem] + return entry + + +def latest_lists(entry: dict[str, dict], kind: str) -> list[dict[str, str]]: + """The `kind` (`rows`, `results` or `degradations`) of every problem of a `latest` entry, + by problem.""" + return [row for problem in sorted(entry) for row in entry[problem][kind]] + + +def rotation_markdown( + entry: dict[str, dict], run_url: str, scope_problems: list[str] | None = None +) -> str: + """The table of the problems of a rotating cron (#435): when each was last tested, those + of `scope_problems` not tested yet first.""" + lines = [ + "", + "## Rotation (#435)", + "", + "Each run tests the problems tested longest ago, while its time budget lasts; the " + "tables above show the last result of every problem.", + "", + "| problem | last tested | run |", + "| --- | --- | --- |", + ] + for problem in sorted(p for p in scope_problems or [] if p not in entry): + lines.append(f"| {problem} | not tested yet | |") + for problem in sorted(entry, key=lambda p: (entry[p]["date"], p)): + item = entry[problem] + mark = " (this run)" if item["run"] == run_url else "" + lines.append( + f"| {problem} | {item['date'][:16].replace('T', ' ')}{mark} | " + f"[{run_id(item['run'])}]({item['run']}) |" + ) + return "\n".join(lines) + "\n" + + def remember_run(state: dict, scope: str, run_url: str) -> str: """Store this run of `scope` in `state` and return the previous run ("" if none). @@ -537,9 +617,11 @@ def issue_section( previous_run: str = "", note: str = "", history: list[str] | None = None, + rotation: str = "", ) -> str: """One scope section of a summary issue: title with scope name, focus and cadence, - status line with run link, and the findings table.""" + status line with run link, the rotation line of a rotating cron, and the findings + table.""" meta = scope_meta(scope) word = STATUS_WORD[kind] status = f"**⚠️ {len(rows)} {word}{'s' * (len(rows) != 1)}**" if rows else "**✅ Clean**" @@ -562,6 +644,8 @@ def issue_section( " · ".join(bits), "", ] + if rotation: + lines += [rotation, ""] if rows: outcome_header = "Change" if kind == "convergence" else "Outcome" lines += [ @@ -921,6 +1005,7 @@ def update_summary_issue( generated: str = "", previous_run: str = "", note: str = "", + rotation: str = "", ) -> None: label = SUMMARY_LABELS[kind] candidates = [issue for issue in all_issues if label in labels(issue)] @@ -932,7 +1017,7 @@ def update_summary_issue( if issue is None: history = [change_entry(generated, run_url, rows, [])] if rows else [] section = issue_section( - scope, kind, rows, run_url, generated, previous_run, note, history + scope, kind, rows, run_url, generated, previous_run, note, history, rotation ) body = initial_issue_body(kind, scope, section) github.create_issue( @@ -956,7 +1041,7 @@ def update_summary_issue( ), ) section = issue_section( - scope, kind, rows, run_url, generated, previous_run, note, history + scope, kind, rows, run_url, generated, previous_run, note, history, rotation ) body = replace_section(old_body, scope, section) total = sum(count for _, count in section_statuses(body)) @@ -1252,17 +1337,27 @@ def prune_environments(results_dir: pathlib.Path) -> None: shutil.rmtree(folder) -def missing_report_row(scope: str) -> dict[str, str]: +def missing_report_row(scope: str, platform: str = "workflow") -> dict[str, str]: return { "test": "report", "problem": "workflow", "modelings": scope, "status": "valid", "outcome": "error", - "platform": "workflow", + "platform": platform, } +def missing_platforms( + rows: list[dict[str, str]], scope: str, julia_version: str, runners: list[str] +) -> list[str]: + """The platforms of the CPU job (`cron---`) with no report in + this run: a job killed before uploading it (#435, a runner that died).""" + seen = {row.get("platform", "") for row in rows} + expected = [f"cron-{scope}-{julia_version}-{runner}" for runner in runners] + return [platform for platform in expected if platform not in seen] + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--reports-dir", type=pathlib.Path, required=True) @@ -1278,7 +1373,13 @@ def main() -> None: parser.add_argument("--issues", action="store_true") parser.add_argument("--no-github", action="store_true") parser.add_argument("--scope-note", default="") + # a rotating cron (#435): the problems of the scope, all of them, comma-separated + parser.add_argument("--rotation", default="") + # the runners of the CPU job, comma-separated: one report expected from each + parser.add_argument("--runners", default="") args = parser.parse_args() + scope_problems = [p for p in args.rotation.split(",") if p] + rotation = bool(scope_problems) now = dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat() rows = parse_reports(args.reports_dir) @@ -1287,8 +1388,29 @@ def main() -> None: state = {} if args.state_file.exists(): state = json.loads(args.state_file.read_text()) - previous = remember_results(state, args.scope, results, args.run_url) + previous = remember_results(state, args.scope, results, args.run_url, rotation) previous_run = remember_run(state, args.scope, args.run_url) + # the rows, results and convergence findings the reports and issues are built from: this + # run's, or in a rotation the last of every problem (#435) + run_rows = rows + convergence_rows = degradations(results, previous) + rotation_line = "" + if rotation: + latest = remember_latest( + state, args.scope, rows, results, convergence_rows, args.run_url, now, + scope_problems, + ) + rows = latest_lists(latest, "rows") + report_results = latest_lists(latest, "results") + convergence_rows = latest_lists(latest, "degradations") + tested = sorted(p for p, item in latest.items() if item["run"] == args.run_url) + rotation_line = ( + f"**Rotation (#435):** this run tested {len(tested)} of {len(scope_problems)} " + f"problems ({', '.join(f'`{p}`' for p in tested) or 'none'}); the others keep " + "their last result." + ) + else: + report_results = results args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text( @@ -1300,10 +1422,11 @@ def main() -> None: args.sha, args.julia_version, args.package_version, - results, + report_results, previous, note=args.scope_note, ) + + (rotation_markdown(latest, args.run_url, scope_problems) if rotation else "") ) if args.results_dir is not None and results: publish_results( @@ -1312,13 +1435,19 @@ def main() -> None: grouped = findings(rows) failure_rows = grouped["failure"] - if not rows: - failure_rows = [missing_report_row(args.scope)] - elif args.tests_failed and not failure_rows: - failure_rows = [missing_report_row(args.scope)] + # this run gave no report, or failed without saying where: the workflow is the finding; + # a platform without a report is one, even when the others reported failures + runners = [r for r in args.runners.split(",") if r] + missing = missing_platforms(run_rows, args.scope, args.julia_version, runners) + if not run_rows: + failure_rows = failure_rows + [missing_report_row(args.scope)] + elif missing: + failure_rows = failure_rows + [missing_report_row(args.scope, m) for m in missing] + elif args.tests_failed and not findings(run_rows)["failure"]: + failure_rows = failure_rows + [missing_report_row(args.scope)] if not args.issues: - prepare_state(rows, args.scope, args.tests_failed, state) + prepare_state(run_rows, args.scope, args.tests_failed, state) args.state_file.parent.mkdir(parents=True, exist_ok=True) args.state_file.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n") return @@ -1333,7 +1462,7 @@ def main() -> None: ("failure", failure_rows), ("draft", grouped["draft"]), ("capability", grouped["capability"]), - ("convergence", degradations(results, previous)), + ("convergence", convergence_rows), ): update_summary_issue( github, @@ -1345,12 +1474,14 @@ def main() -> None: generated=now, previous_run=previous_run, note=args.scope_note, + rotation=rotation_line, ) + # the regression issues of a problem change only when it runs update_bug_issues( github, all_issues, - rows, - grouped["failure"], + run_rows, + findings(run_rows)["failure"], args.scope, args.run_url, now, diff --git a/tools/cron/select.py b/tools/cron/select.py new file mode 100644 index 000000000..d59270487 --- /dev/null +++ b/tools/cron/select.py @@ -0,0 +1,62 @@ +"""The problems a rotating cron runs, in order (#435). + +A cron with a time budget runs the problems of its modeling one per process, while the budget +lasts. This script gives their order: the problems tested longest ago first, by the date of +their last result for the scope on the `results` branch (`problems//result.json`, +written by `report.py`). A problem with no result comes first; ties are broken by name. A run +that crashed wrote no result, so its problems stay first for the next run. + +Usage: + python3 tools/cron/select.py --scope ModelingToolkit \ + --models-dir ext/ModelingToolkitModels --metadata-dir ext/MetaData \ + [--results-dir ] + +Prints the problems, comma-separated, ready for `problem=`. +""" +from __future__ import annotations + +import argparse +import json +import pathlib + + +def problems_of(models_dir: pathlib.Path, metadata_dir: pathlib.Path) -> list[str]: + """The problems of a modeling: a file `.jl` in its directory (the single source + of truth, `ext/README.md`) for a problem of the collection (a metadata file).""" + collection = {path.stem for path in metadata_dir.glob("*.jl")} + return sorted(path.stem for path in models_dir.glob("*.jl") if path.stem in collection) + + +def last_date(results_dir: pathlib.Path | None, problem: str, scope: str) -> str: + """The date of the last result of `problem` for `scope` (ISO 8601), "" if none.""" + if results_dir is None: + return "" + path = results_dir / "problems" / problem / "result.json" + if not path.is_file(): + return "" + entries = json.loads(path.read_text()).get("entries", []) + return max((e.get("date", "") for e in entries if e.get("scope") == scope), default="") + + +def rotation(problems: list[str], dates: dict[str, str]) -> list[str]: + """`problems` ordered for the rotation: no result first, then the oldest; by name in ties. + ISO 8601 dates in UTC sort as strings, and "" sorts first.""" + return sorted(problems, key=lambda problem: (dates.get(problem, ""), problem)) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--scope", required=True) + parser.add_argument("--models-dir", type=pathlib.Path, required=True) + parser.add_argument("--metadata-dir", type=pathlib.Path, required=True) + parser.add_argument("--results-dir", type=pathlib.Path) + args = parser.parse_args() + + results_dir = args.results_dir if args.results_dir and args.results_dir.is_dir() else None + problems = problems_of(args.models_dir, args.metadata_dir) + dates = {problem: last_date(results_dir, problem, args.scope) for problem in problems} + print(",".join(rotation(problems, dates))) + + +if __name__ == "__main__": + main() diff --git a/tools/cron/test/test_report.py b/tools/cron/test/test_report.py index 471f8223d..672050255 100644 --- a/tools/cron/test/test_report.py +++ b/tools/cron/test/test_report.py @@ -249,6 +249,20 @@ def test_a_missing_report_row_does_not_get_a_bogus_reproduce_command(self): self.assertNotIn("problem=workflow", section) self.assertIn("failed or produced no report", section) + def test_a_platform_without_report_is_a_finding(self): + # #435: the ubuntu runner died, windows reported a failure of its own + rows = [{"platform": "cron-MTK-1.13-windows-latest", "problem": "beam"}] + self.assertEqual( + ["cron-MTK-1.13-ubuntu-latest"], + REPORT.missing_platforms(rows, "MTK", "1.13", ["ubuntu-latest", "windows-latest"]), + ) + self.assertEqual([], REPORT.missing_platforms(rows, "MTK", "1.13", ["windows-latest"])) + self.assertEqual([], REPORT.missing_platforms(rows, "MTK", "1.13", [])) + row = REPORT.missing_report_row("MTK", "cron-MTK-1.13-ubuntu-latest") + section = REPORT.issue_section("MTK", "failure", [row], "run") + self.assertIn("1.13-ubuntu-latest", section) + self.assertIn("failed or produced no report", section) + def test_initial_issue_body_prepopulates_all_known_scopes(self): active = "\nACTIVE_JUMP_SECTION" body = REPORT.initial_issue_body("convergence", "JuMP", active) @@ -595,6 +609,111 @@ def test_a_solve_without_trajectory_or_environment_is_published(self): self.assertIsNone(entry["environment"]) self.assertEqual("JuMP-robustness", entry["scope"]) + # ------- rotation (#435) ------- + + def write_run(self, directory, problems): + """Reports of one platform for `problems`, a {problem: outcome} dict, as the per-problem + processes of a rotating cron write them (`-cron-.md`).""" + platform = pathlib.Path(directory) / "cron-MTK-1.13-ubuntu-latest" + platform.mkdir(parents=True, exist_ok=True) + for problem, outcome in problems.items(): + (platform / f"MTK-cron-{problem}.md").write_text( + "| test | problem | modelings | status | outcome |\n" + "| --- | --- | --- | --- | --- |\n" + f"| test_convergence | {problem} | MTK | valid | {outcome} |\n" + ) + + def run_report(self, reports, state, output, run_url, rotation="beam,chain,robot"): + import subprocess + import sys + + subprocess.run( + [ + sys.executable, str(ROOT / "tools/cron/report.py"), + "--reports-dir", str(reports), + "--scope", "MTK", + "--run-url", run_url, + "--state-file", str(state), + "--output", str(output), + "--rotation", rotation, + ], + check=True, + ) + + def test_a_rotation_keeps_the_last_result_of_the_problems_that_did_not_run(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + state, output = root / "state.json", root / "MTK.md" + self.write_run(root / "run1", {"beam": "fail", "chain": "pass"}) + self.run_report(root / "run1", state, output, "https://x/runs/1") + # robot has not run yet: listed as such + self.assertIn("| robot | not tested yet | |", output.read_text()) + self.write_run(root / "run2", {"robot": "pass"}) + self.run_report(root / "run2", state, output, "https://x/runs/2") + + latest = json.loads(state.read_text())["latest"]["MTK"] + self.assertEqual(sorted(latest), ["beam", "chain", "robot"]) + self.assertEqual(latest["beam"]["run"], "https://x/runs/1") + self.assertEqual(latest["robot"]["run"], "https://x/runs/2") + # the failure of beam is still a finding: it did not run again + rows = REPORT.latest_lists(latest, "rows") + failures = REPORT.findings(rows)["failure"] + self.assertEqual([row["problem"] for row in failures], ["beam"]) + report = output.read_text() + self.assertIn("| test_convergence | beam | MTK | valid | fail |", report) + self.assertIn("## Rotation (#435)", report) + self.assertRegex(report, r"\| robot \| [^|]*\(this run\)") + + # beam runs again and passes: now it is resolved + self.write_run(root / "run3", {"beam": "pass"}) + self.run_report(root / "run3", state, output, "https://x/runs/3") + latest = json.loads(state.read_text())["latest"]["MTK"] + self.assertEqual(REPORT.findings(REPORT.latest_lists(latest, "rows"))["failure"], []) + + def test_a_problem_left_out_of_the_scope_is_dropped(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + state, output = root / "state.json", root / "MTK.md" + self.write_run(root / "run1", {"beam": "pass", "chain": "fail"}) + self.run_report(root / "run1", state, output, "https://x/runs/1") + # chain has no model file any more + self.write_run(root / "run2", {"beam": "pass"}) + self.run_report(root / "run2", state, output, "https://x/runs/2", rotation="beam") + latest = json.loads(state.read_text())["latest"]["MTK"] + self.assertEqual(sorted(latest), ["beam"]) + + def test_a_crashed_process_is_a_failure_of_its_problem(self): + # the row that `TestStatus.write_process_report` writes for a crashed or stopped process + row = { + "test": "process (timeout after 2700 s)", + "problem": "chain", + "modelings": "MTK", + "status": "valid", + "outcome": "error", + "platform": "cpu", + } + self.assertEqual(REPORT.findings([row])["failure"], [row]) + + def test_rotation_results_replace_those_of_the_problems_that_ran(self): + state = {} + first = [ + {"platform": "p", "problem": "beam", "modeling": "M", "configuration": "c", + "result": "converged", "objective": "1.0"}, + {"platform": "p", "problem": "chain", "modeling": "M", "configuration": "c", + "result": "converged", "objective": "2.0"}, + ] + REPORT.remember_results(state, "MTK", first, "run-1", rotation=True) + second = [ + {"platform": "p", "problem": "beam", "modeling": "M", "configuration": "c", + "result": "converged", "objective": "1.5"}, + ] + previous = REPORT.remember_results(state, "MTK", second, "run-2", rotation=True) + results = state["convergence"]["MTK"]["results"] + self.assertEqual(results["p|beam|M|c"]["objective"], "1.5") + # chain did not run: its result is kept, and it is no change + self.assertEqual(results["p|chain|M|c"]["objective"], "2.0") + self.assertEqual(previous["p|beam|M|c"]["objective"], "1.0") + def test_slug(self): self.assertEqual("modeler-adnlp_solver-ipopt", REPORT.slug("modeler=adnlp solver=ipopt")) self.assertEqual("mu_init-0.001", REPORT.slug("mu_init=0.001")) diff --git a/tools/cron/test/test_select.py b/tools/cron/test/test_select.py new file mode 100644 index 000000000..3e283b01e --- /dev/null +++ b/tools/cron/test/test_select.py @@ -0,0 +1,112 @@ +import importlib.util +import json +import pathlib +import subprocess +import sys +import tempfile +import unittest + + +ROOT = pathlib.Path(__file__).parents[3] +SPEC = importlib.util.spec_from_file_location("cron_select", ROOT / "tools/cron/select.py") +SELECT = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(SELECT) + + +def write_result(results: pathlib.Path, problem: str, entries: list[dict]) -> None: + folder = results / "problems" / problem + folder.mkdir(parents=True, exist_ok=True) + (folder / "result.json").write_text(json.dumps({"problem": problem, "entries": entries})) + + +class TestSelect(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + root = pathlib.Path(self.tmp.name) + self.models = root / "ext" / "MTKModels" + self.metadata = root / "ext" / "MetaData" + self.results = root / "results" + for directory in (self.models, self.metadata, self.results): + directory.mkdir(parents=True) + for problem in ("alpha", "beta", "gamma", "delta"): + (self.metadata / f"{problem}.jl").write_text("") + (self.metadata / f"{problem}.md").write_text("") + for problem in ("alpha", "beta", "gamma", "delta"): + (self.models / f"{problem}.jl").write_text("") + # a helper file of the modeling is not a problem + (self.models / "helpers.jl").write_text("") + + def tearDown(self): + self.tmp.cleanup() + + def test_problems_of_a_modeling(self): + self.assertEqual( + ["alpha", "beta", "delta", "gamma"], + SELECT.problems_of(self.models, self.metadata), + ) + + def test_last_date_of_the_scope(self): + write_result( + self.results, + "alpha", + [ + {"scope": "MTK", "date": "2026-11-10T04:00:00+00:00"}, + {"scope": "MTK", "date": "2026-11-12T04:00:00+00:00"}, + # another scope does not count + {"scope": "JuMP", "date": "2026-11-19T04:00:00+00:00"}, + ], + ) + self.assertEqual( + "2026-11-12T04:00:00+00:00", SELECT.last_date(self.results, "alpha", "MTK") + ) + self.assertEqual("", SELECT.last_date(self.results, "beta", "MTK")) + self.assertEqual("", SELECT.last_date(None, "alpha", "MTK")) + + def test_rotation_oldest_first(self): + dates = { + "alpha": "2026-11-12T04:00:00+00:00", + "beta": "2026-11-10T04:00:00+00:00", + "gamma": "", # never tested: first + "delta": "2026-11-12T04:00:00+00:00", # tie with alpha: by name + } + self.assertEqual( + ["gamma", "beta", "alpha", "delta"], + SELECT.rotation(["alpha", "beta", "gamma", "delta"], dates), + ) + + def test_command_line(self): + write_result(self.results, "alpha", [{"scope": "MTK", "date": "2026-11-12T04:00:00+00:00"}]) + write_result(self.results, "beta", [{"scope": "MTK", "date": "2026-11-11T04:00:00+00:00"}]) + out = subprocess.run( + [ + sys.executable, + str(ROOT / "tools/cron/select.py"), + "--scope", "MTK", + "--models-dir", str(self.models), + "--metadata-dir", str(self.metadata), + "--results-dir", str(self.results), + ], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + self.assertEqual("delta,gamma,beta,alpha", out) + # no results branch yet: every problem by name + out = subprocess.run( + [ + sys.executable, + str(ROOT / "tools/cron/select.py"), + "--scope", "MTK", + "--models-dir", str(self.models), + "--metadata-dir", str(self.metadata), + "--results-dir", str(self.results / "missing"), + ], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + self.assertEqual("alpha,beta,delta,gamma", out) + + +if __name__ == "__main__": + unittest.main()