Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,8 @@ plans and reports.
the proposal the versions you apply: ct-agents, ct-conventions, and the commit of the
CTActions clone `release.jl` runs from.
- **Add a problem, or a modeling of one.** Specific to this repository: follow
[`ext/README.md`](ext/README.md). Before the tests, run the linter,
`julia --project tools/lint/problems.jl <problem>`, until it reports no ❌. No ❌ does not
mean the problem is right: paste the ℹ️ checklist of its report in the pull request and
tick each box, or say why it is not ticked.
[`ext/README.md`](ext/README.md). Start from the skeleton of `tools/new/problem.jl` (`new`,
or `add` for a modeling) and write in place of its `TODO`s. Before the tests, run the
linter, `julia --project tools/lint/problems.jl <problem>`, until it reports no ❌. No ❌
does not mean the problem is right: paste the ℹ️ checklist of its report in the pull
request and tick each box, or say why it is not ticked.
6 changes: 6 additions & 0 deletions ext/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,12 @@ ext/

## Add a problem

0. **Skeleton**: `julia --project tools/new/problem.jl new <problem> <modelings...>` writes
the files of the steps below from the templates of `tools/new/templates/`, an example
problem in each modeling given, `:draft`; `add <problem> <modelings...>` adds a modeling
to an existing problem (a model file, a `:draft` status). Their `TODO`s mark what to write
in place of the example; the linter (step 4) fails until none is left. A new modeling of
the package needs its template there (a test checks it).
1. **Metadata**, `MetaData/<problem>.jl`: an `OrderedDict` named `<problem>_meta` with
- `:grid_size`, the default number of steps;
- `:parameters`, a `NamedTuple` of the default parameters, or `nothing`; every bound of
Expand Down
91 changes: 91 additions & 0 deletions test/suite/core/test_new.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
module TestNew

using Test: Test
using OptimalControlProblems: OptimalControlProblems

const VERBOSE = isdefined(Main, :TestOptions) ? Main.TestOptions.VERBOSE : true
const SHOWTIMING = isdefined(Main, :TestOptions) ? Main.TestOptions.SHOWTIMING : true

const TOOLS = joinpath(pkgdir(OptimalControlProblems), "tools")
include(joinpath(TOOLS, "new", "NewProblem.jl"))
include(joinpath(TOOLS, "lint", "ProblemLint.jl"))
const N = NewProblem
const L = ProblemLint

failures(problem, root) = [
r.message for r in L.checks(L.source(problem; root)) if r.status === :fail
]

# what the person writes in place of each `TODO`
function fill_todos(root)
for (dir, _, files) in walkdir(joinpath(root, "ext")), file in files
path = joinpath(dir, file)
write(path, replace(read(path, String), "TODO" => "Written"))
end
end

statuses(problem, root) = L.metadata(problem; root)[:status]

function test_new()
Test.@testset "new problem" verbose = VERBOSE showtiming = SHOWTIMING begin
Test.@testset "a template per modeling, nothing else" begin
Test.@test sort(readdir(N.TEMPLATES)) == sort([
"metadata.jl", "description.md", ("$(m).jl" for m in N.modelings())...
])
end

Test.@testset "every modeling: lint fails on the TODOs only" begin
mktempdir() do root
paths = N.create(:foo, N.modelings(); root)
Test.@test length(paths) == 2 + length(N.modelings())
for path in paths
endswith(path, ".jl") || continue
Test.@test Meta.parseall(read(path, String)) isa Expr
end
Test.@test keys(statuses(:foo, root)) == Tuple(N.modelings())
Test.@test all(==(:draft), values(statuses(:foo, root)))
failed = failures(:foo, root)
Test.@test length(failed) == 1
Test.@test startswith(only(failed), "nothing left to fill (`TODO`)")
fill_todos(root)
Test.@test isempty(failures(:foo, root))
end
end

Test.@testset "some modelings, then add one" begin
mktempdir() do root
N.create(:foo, [:JuMP, :OptimalControl]; root)
Test.@test keys(statuses(:foo, root)) == (:OptimalControl, :JuMP)
fill_todos(root)
Test.@test isempty(failures(:foo, root))
# a status set by hand is kept, the new modeling comes in its place
path = N.metadata_path(root, :foo)
write(path, replace(read(path, String), "JuMP=:draft" => "JuMP=:valid"))
N.add(:foo, [:ModelingToolkit]; root)
Test.@test statuses(:foo, root) ==
(OptimalControl=:draft, JuMP=:valid, ModelingToolkit=:draft)
failed = failures(:foo, root)
Test.@test failed == ["nothing left to fill (`TODO`): model ModelingToolkit"]
end
end

Test.@testset "refusals" begin
mktempdir() do root
Test.@test_throws ArgumentError N.create(:foo, Symbol[]; root)
Test.@test_throws ArgumentError N.create(:foo, [:Fortran]; root)
Test.@test_throws ArgumentError N.create(:foo, [:JuMP, :JuMP]; root)
Test.@test_throws ArgumentError N.create(Symbol("foo bar"), [:JuMP]; root)
Test.@test_throws ArgumentError N.add(:foo, [:JuMP]; root)
N.create(:foo, [:JuMP]; root)
Test.@test_throws ArgumentError N.create(:foo, [:OptimalControl]; root)
Test.@test_throws ArgumentError N.add(:foo, [:JuMP]; root)
# nothing written by a refusal
Test.@test !isfile(N.model_path(root, :foo, :OptimalControl))
end
end
end
end

end # module TestNew

test_new() = TestNew.test_new()
33 changes: 29 additions & 4 deletions tools/lint/ProblemLint.jl
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
# failure here does not mean that a problem is right.
#
# Used by `tools/lint/problems.jl` (the command line) and by `test/suite/core/test_lint.jl`
# (no failure allowed). The checks read the files of the problem as text and its metadata
# through the package; they are pure functions of a `ProblemSource`, which the tests build by
# hand.
# (no failure allowed). The checks read the files of the problem as text, and its metadata
# by evaluating its file; they are pure functions of a `ProblemSource`, which the tests build
# by hand.
module ProblemLint

using OptimalControlProblems: OptimalControlProblems
Expand Down Expand Up @@ -38,6 +38,18 @@ struct ProblemSource
models::Vector{Model}
end

"""
The metadata of `problem` in `ext/MetaData/<problem>.jl` under `root`: the file evaluated in
a module of its own, as the package does, so that a problem not yet known to the loaded
package (just created, or under another root) can be linted.
"""
function metadata(problem::Symbol; root::AbstractString=pkgdir(OCP))
sandbox = Module()
Core.eval(sandbox, :(const OrderedDict = $(OCP.Metadata.OrderedDict)))
Base.include(sandbox, joinpath(root, "ext", "MetaData", "$(problem).jl"))
return Base.invokelatest(getglobal, sandbox, Symbol(problem, :_meta))
end

"""The `ProblemSource` of `problem`, read from the files of the package under `root`."""
function source(problem::Symbol; root::AbstractString=pkgdir(OCP))
ext = joinpath(root, "ext")
Expand All @@ -51,7 +63,7 @@ function source(problem::Symbol; root::AbstractString=pkgdir(OCP))
]
return ProblemSource(
problem,
OCP.problem_metadata(problem),
metadata(problem; root),
read(joinpath(ext, "MetaData", "$(problem).jl"), String),
read_or_nothing(joinpath(ext, "MetaData", "$(problem).md")),
models,
Expand Down Expand Up @@ -233,6 +245,19 @@ function checks(src::ProblemSource)
"a model in one modeling at least",
)

# nothing left to fill: the skeleton of `tools/new/problem.jl` marks what to write
texts = [
"metadata" => src.metadata_text,
"description" => src.description,
("model $(m.modeling)" => m.text for m in src.models)...,
]
todo = [name for (name, text) in texts if !isnothing(text) && occursin("TODO", text)]
add(
isempty(todo),
"files",
"nothing left to fill (`TODO`)" * (isempty(todo) ? "" : ": $(join(todo, ", "))"),
)

# the description: one outline, sources
if !isnothing(src.description)
add(
Expand Down
107 changes: 107 additions & 0 deletions tools/new/NewProblem.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
# The skeleton of a new problem (#449, T3): its metadata, its description and a model per
# modeling asked for, from the templates of `templates/`, the same for every problem. The
# skeleton passes the linter (`tools/lint/ProblemLint.jl`) but for its `TODO`s, which mark
# what is left to write: fill them, lint again, until no failure.
#
# Used by `tools/new/problem.jl` (the command line) and by `test/suite/core/test_new.jl`.
# A modeling is one of `MODELING_SPECS`; each has a template `templates/<modeling>.jl`, a
# test checks it, so adding a modeling to the package means adding its template.
module NewProblem

using OptimalControlProblems: OptimalControlProblems

const OCP = OptimalControlProblems
const TEMPLATES = joinpath(@__DIR__, "templates")

"""The modelings of the package, in the order of `MODELING_SPECS`."""
modelings() = [spec.name for spec in OCP.Modelings.MODELING_SPECS]

modeling_spec(m::Symbol) = only(s for s in OCP.Modelings.MODELING_SPECS if s.name === m)

function template(name::AbstractString, problem::Symbol)
text = read(joinpath(TEMPLATES, name), String)
return replace(text, "{{problem}}" => string(problem))
end

metadata_path(root, problem) = joinpath(root, "ext", "MetaData", "$(problem).jl")
description_path(root, problem) = joinpath(root, "ext", "MetaData", "$(problem).md")
function model_path(root, problem, m)
return joinpath(root, "ext", modeling_spec(m).directory, "$(problem).jl")
end

"""The `:status` entry of a metadata file: one `:draft` (or given) status per modeling."""
function status_entry(statuses)
lines = [" $(m)=:$(statuses[m])," for m in modelings() if haskey(statuses, m)]
return join([" :status => (", lines..., " ),"], "\n")
end

function check_modelings(ms)
isempty(ms) && throw(ArgumentError("give at least one modeling, among $(modelings())"))
unknown = setdiff(ms, modelings())
isempty(unknown) ||
throw(ArgumentError("unknown modelings $(unknown), expected some of $(modelings())"))
allunique(ms) || throw(ArgumentError("a modeling is given twice: $(ms)"))
return nothing
end

function write_new(path, text)
isfile(path) && throw(ArgumentError("$(path) exists already"))
mkpath(dirname(path))
write(path, text)
return path
end

"""
create(problem, modelings; root) -> paths

The skeleton of a new `problem` in `modelings`, each `:draft`: the metadata, the description
and one model file per modeling, under `root`. Refuses a problem that exists.
"""
function create(
problem::Symbol, ms::AbstractVector{Symbol}; root::AbstractString=pkgdir(OCP)
)
Base.isidentifier(problem) ||
throw(ArgumentError("$(problem) is not a valid name of a function"))
check_modelings(ms)
isfile(metadata_path(root, problem)) &&
throw(ArgumentError("$(problem) exists already: add a modeling with `add`"))
status = status_entry(Dict(m => :draft for m in ms))
paths = [
write_new(
metadata_path(root, problem),
replace(template("metadata.jl", problem), "{{status}}" => status),
),
write_new(description_path(root, problem), template("description.md", problem)),
]
for m in ms
push!(paths, write_new(model_path(root, problem, m), template("$(m).jl", problem)))
end
return paths
end

"""
add(problem, modelings; root) -> paths

A model file per modeling for an existing `problem`, from the template of the modeling, and
a `:draft` status for each in its metadata. The other fields of the metadata do not depend on
the modeling. Refuses a modeling the problem has already.
"""
function add(problem::Symbol, ms::AbstractVector{Symbol}; root::AbstractString=pkgdir(OCP))
check_modelings(ms)
path = metadata_path(root, problem)
isfile(path) || throw(ArgumentError("$(problem) does not exist: create it with `new`"))
text = read(path, String)
found = match(r"^[ \t]*:status => \(([^()]*)\),[ \t]*$"m, text)
isnothing(found) && throw(ArgumentError("no `:status => (…),` entry found in $(path)"))
statuses = Dict(
Symbol(e[1]) => Symbol(e[2]) for e in eachmatch(r"(\w+)\s*=\s*:(\w+)", found[1])
)
present = filter(m -> haskey(statuses, m) || isfile(model_path(root, problem, m)), ms)
isempty(present) || throw(ArgumentError("$(problem) has already $(present)"))
paths = [write_new(model_path(root, problem, m), template("$(m).jl", problem)) for m in ms]
merge!(statuses, Dict(m => :draft for m in ms))
write(path, replace(text, found.match => status_entry(statuses)))
return [path; paths]
end

end # module NewProblem
34 changes: 34 additions & 0 deletions tools/new/problem.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# The skeleton of a new problem, or of a new modeling of a problem (#449, T3).
#
# julia --project tools/new/problem.jl new <problem> <modelings...> # a new problem
# julia --project tools/new/problem.jl add <problem> <modelings...> # a new modeling of it
#
# The modelings are those of `MODELING_SPECS`: OptimalControl, OptimalControlScalarized, JuMP,
# ModelingToolkit. The files are those of `ext/README.md`, from the templates of `templates/`,
# each modeling `:draft`; their `TODO`s mark what is left to write. Then lint the problem,
# `julia --project tools/lint/problems.jl <problem>`, fill what it reports, and again.
using OptimalControlProblems: OptimalControlProblems

include(joinpath(@__DIR__, "NewProblem.jl"))

const USAGE = "usage: julia --project tools/new/problem.jl new|add <problem> <modelings...>"

length(ARGS) >= 2 && ARGS[1] in ("new", "add") || (println(stderr, USAGE); exit(2))
command, problem, modelings = ARGS[1], Symbol(ARGS[2]), Symbol.(ARGS[3:end])
paths = try
if command == "new"
NewProblem.create(problem, modelings)
else
NewProblem.add(problem, modelings)
end
catch e
e isa ArgumentError || rethrow()
println(stderr, e.msg, "\n", USAGE)
exit(2)
end
println("Written:")
foreach(p -> println("- ", relpath(p, pkgdir(OptimalControlProblems))), paths)
println(
"\nNext: fill the `TODO`s, then `julia --project tools/lint/problems.jl $(problem)` ",
"until no failure, then the tests (ext/README.md).",
)
51 changes: 51 additions & 0 deletions tools/new/templates/JuMP.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
"""TODO: the first sentence of the description.

Builds the `{{problem}}` problem with the JuMP backend.

```julia-repl
julia> using OptimalControlProblems
julia> {{problem}}(JuMPBackend(); grid_size=100)
```

The keyword `scheme` chooses the discretization: `:trapeze` (default), `:euler`,
`:euler_implicit` or `:midpoint`, as in `OptimalControl.Collocation`.
The keyword `ordering` chooses the order of the variables and constraints:
`:interleaved` (default, as in CTDirect) or `:blocks` (grouped by kind).

Default parameters and the modeling status are available through
`problem_metadata(:{{problem}})` and `OptimalControlProblems.status(:{{problem}})`.
"""
function OptimalControlProblems.{{problem}}(
::JuMPBackend,
args...;
grid_size::Int=grid_size_data(:{{problem}}),
scheme::Symbol=:trapeze,
ordering::Symbol=:interleaved,
parameters::Union{Nothing,NamedTuple}=nothing,
init=nothing,
kwargs...,
)
OptimalControlProblems.Catalogue._warn_if_draft(:{{problem}}, :JuMP)

# parameters
params = parameters_data(:{{problem}}, parameters)
t0 = params[:t0]
tf = params[:tf]
x_t0 = params[:x_t0]
x_u = params[:x_u]

# the continuous problem, discretized by `build`. TODO: the problem, in place of this
# example
data = ProblemData(;
state=[Component("x"; upper=x_u)],
control=[Component("u")],
costate=["∂x"],
times=v -> (t0, tf),
dynamics=(x, u, v, t) -> [u[1]],
lagrange=(x, u, v, t) -> u[1]^2,
boundary=[(x_t0, (x0, xf, v) -> x0[1], x_t0)],
)
# initial guess: from the metadata (#377), unless given
init = OptimalControlProblems.Metadata._initial_guess(:{{problem}}, params, init)
return build(data, grid_size, scheme, ordering, args...; init=init, kwargs...)
end
Loading
Loading