Skip to content

Commit 069f14e

Browse files
maleadtmaleadt
committed
Rework args to support running filtered tests.
1 parent 1c3428c commit 069f14e

2 files changed

Lines changed: 163 additions & 91 deletions

File tree

‎README.md‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,18 +58,23 @@ testsuite = Dict(
5858
runtests(MyModule, ARGS; testsuite)
5959
```
6060

61-
You can also use `find_tests` to automatically discover tests and then filter or modify them:
61+
You can also use `find_tests` to automatically discover tests and then filter or modify them. This requires manually parsing arguments so that filtering is only applied when the user did not request specific tests to run:
6262

6363
```julia
6464
# Start with autodiscovered tests
6565
testsuite = find_tests(pwd())
6666

67-
# Remove tests that shouldn't run on Windows
68-
if Sys.iswindows()
69-
delete!(testsuite, "ext/specialfunctions")
67+
# Parse arguments
68+
args = parse_args(ARGS)
69+
70+
if filter_tests!(testsuite, args)
71+
# Remove tests that shouldn't run on Windows
72+
if Sys.iswindows()
73+
delete!(testsuite, "ext/specialfunctions")
74+
end
7075
end
7176

72-
runtests(MyModule, ARGS; testsuite)
77+
runtests(MyModule, args; testsuite)
7378
```
7479

7580
### Provide defaults

‎src/ParallelTestRunner.jl‎

Lines changed: 153 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
module ParallelTestRunner
22

3-
export runtests, addworkers, addworker, find_tests
3+
export runtests, addworkers, addworker, find_tests, parse_args, filter_tests!
44

55
using Malt
66
using Dates
@@ -35,28 +35,6 @@ end
3535

3636
const max_worker_rss = JULIA_TEST_MAXRSS_MB * 2^20
3737

38-
# parse some command-line arguments
39-
function extract_flag!(args, flag, default = nothing; typ = typeof(default))
40-
for f in args
41-
if startswith(f, flag)
42-
# Check if it's just `--flag` or if it's `--flag=foo`
43-
if f != flag
44-
val = split(f, '=')[2]
45-
if !(typ === Nothing || typ <: AbstractString)
46-
val = parse(typ, val)
47-
end
48-
else
49-
val = default
50-
end
51-
52-
# Drop this value from our args
53-
filter!(x -> x != f, args)
54-
return (true, val)
55-
end
56-
end
57-
return (false, default)
58-
end
59-
6038
function with_testset(f, testset)
6139
@static if VERSION >= v"1.13.0-DEV.1044"
6240
Test.@with_testset testset f()
@@ -487,21 +465,139 @@ function find_tests(dir::String)
487465
return tests
488466
end
489467

468+
struct ParsedArgs
469+
jobs::Union{Some{Int}, Nothing}
470+
verbose::Union{Some{Nothing}, Nothing}
471+
quickfail::Union{Some{Nothing}, Nothing}
472+
list::Union{Some{Nothing}, Nothing}
473+
474+
custom::Dict{String,Any}
475+
476+
positionals::Vector{String}
477+
end
478+
479+
# parse some command-line arguments
480+
function extract_flag!(args, flag; typ = Nothing)
481+
for f in args
482+
if startswith(f, flag)
483+
# Check if it's just `--flag` or if it's `--flag=foo`
484+
val = if f == flag
485+
nothing
486+
else
487+
parts = split(f, '=')
488+
if typ === Nothing || typ <: AbstractString
489+
parts[2]
490+
else
491+
parse(typ, parts[2])
492+
end
493+
end
494+
495+
# Drop this value from our args
496+
filter!(x -> x != f, args)
497+
return Some(val)
498+
end
499+
end
500+
return nothing
501+
end
502+
503+
"""
504+
parse_args(args; [custom::Array{String}]) -> ParsedArgs
505+
506+
Parse command-line arguments for `runtests`. Typically invoked by passing `Base.ARGS`.
507+
508+
Fields of this structure represent command-line options, containing `nothing` when the
509+
option was not specified, or `Some(optional_value=nothing)` when it was.
510+
511+
Custom arguments can be specified via the `custom` keyword argument, which should be
512+
an array of strings representing custom flag names (without the `--` prefix). Presence
513+
of these flags will be recorded in the `custom` field of the returned `ParsedArgs` object.
514+
"""
515+
function parse_args(args; custom::Array{String} = String[])
516+
args = copy(args)
517+
518+
help = extract_flag!(args, "--help")
519+
if help !== nothing
520+
usage =
521+
"""
522+
Usage: runtests.jl [--help] [--list] [--jobs=N] [TESTS...]
523+
524+
--help Show this text.
525+
--list List all available tests.
526+
--verbose Print more information during testing.
527+
--quickfail Fail the entire run as soon as a single test errored.
528+
--jobs=N Launch `N` processes to perform tests."""
529+
530+
if !isempty(custom)
531+
usage *= "\n\nCustom arguments:"
532+
for flag in custom
533+
usage *= "\n --$flag"
534+
end
535+
end
536+
usage *= "\n\nRemaining arguments filter the tests that will be executed."
537+
println(usage)
538+
exit(0)
539+
end
540+
541+
jobs = extract_flag!(args, "--jobs"; typ = Int)
542+
verbose = extract_flag!(args, "--verbose")
543+
quickfail = extract_flag!(args, "--quickfail")
544+
list = extract_flag!(args, "--list")
545+
546+
custom_args = Dict{String,Any}()
547+
for flag in custom
548+
custom_args[flag] = extract_flag!(args, "--$flag")
549+
end
550+
551+
## no options should remain
552+
optlike_args = filter(startswith("-"), args)
553+
if !isempty(optlike_args)
554+
error("Unknown test options `$(join(optlike_args, " "))` (try `--help` for usage instructions)")
555+
end
556+
557+
return ParsedArgs(jobs, verbose, quickfail, list, custom_args, args)
558+
end
559+
560+
"""
561+
filter_tests!(testsuite, args::ParsedArgs) -> Bool
562+
563+
Filter tests in `testsuite` based on command-line arguments in `args`.
564+
565+
Returns `true` if additional filtering may be done by the caller, `false` otherwise.
566+
"""
567+
function filter_tests!(testsuite, args::ParsedArgs)
568+
# the user did not request specific tests, so let the caller do its own filtering
569+
isempty(args.positionals) && return true
570+
571+
# only select tests matching positional arguments
572+
tests = collect(keys(testsuite))
573+
for test in tests
574+
if !any(arg -> startswith(test, arg), args.positionals)
575+
delete!(testsuite, test)
576+
end
577+
end
578+
579+
# the user requested specific tests, so don't allow further filtering
580+
return false
581+
end
582+
490583
"""
491-
runtests(mod::Module, ARGS; testsuite::Dict{String,Expr}=find_tests(pwd()),
492-
RecordType = TestRecord,
493-
init_code = :(),
494-
test_worker = Returns(nothing),
495-
stdout = Base.stdout,
496-
stderr = Base.stderr)
584+
runtests(mod::Module, args::ParsedArgs;
585+
testsuite::Dict{String,Expr}=find_tests(pwd()),
586+
RecordType = TestRecord,
587+
init_code = :(),
588+
test_worker = Returns(nothing),
589+
stdout = Base.stdout,
590+
stderr = Base.stderr)
591+
runtests(mod::Module, ARGS; ...)
497592
498593
Run Julia tests in parallel across multiple worker processes.
499594
500595
## Arguments
501596
502597
- `mod`: The module calling runtests
503598
- `ARGS`: Command line arguments array, typically from `Base.ARGS`. When you run the tests
504-
with `Pkg.test`, this can be changed with the `test_args` keyword argument.
599+
with `Pkg.test`, this can be changed with the `test_args` keyword argument. If the caller
600+
needs to accept args too, consider using `parse_args` to parse the arguments first.
505601
506602
Several keyword arguments are also supported:
507603
@@ -542,87 +638,57 @@ runtests(MyModule, ARGS)
542638
# Run only tests matching "integration"
543639
runtests(MyModule, ["integration"])
544640
545-
# Customize the test suite
546-
testsuite = find_tests(pwd())
547-
delete!(testsuite, "slow_test") # Remove a specific test
548-
runtests(MyModule, ARGS; testsuite)
549-
550-
# Define a custom test suite manually
641+
# Define a custom test suite
551642
testsuite = Dict(
552643
"custom" => quote
553644
@test 1 + 1 == 2
554645
end
555646
)
556647
runtests(MyModule, ARGS; testsuite)
557648
558-
# Use custom test record type
559-
runtests(MyModule, ARGS; RecordType = MyCustomTestRecord)
649+
# Customize the test suite
650+
testsuite = find_tests(pwd())
651+
args = parse_args(ARGS)
652+
if filter_tests!(testsuite, args)
653+
# Remove a specific test
654+
delete!(testsuite, "slow_test")
655+
end
656+
runtests(MyModule, args; testsuite)
560657
```
561658
562659
## Memory Management
563660
564661
Workers are automatically recycled when they exceed memory limits to prevent out-of-memory
565662
issues during long test runs. The memory limit is set based on system architecture.
566663
"""
567-
function runtests(mod::Module, ARGS; testsuite::Dict{String,Expr} = find_tests(pwd()),
664+
function runtests(mod::Module, args::ParsedArgs;
665+
testsuite::Dict{String,Expr} = find_tests(pwd()),
568666
RecordType = TestRecord, init_code = :(), test_worker = Returns(nothing),
569667
stdout = Base.stdout, stderr = Base.stderr)
570668
#
571669
# set-up
572670
#
573671

574-
do_help, _ = extract_flag!(ARGS, "--help")
575-
if do_help
576-
println(
577-
"""
578-
Usage: runtests.jl [--help] [--list] [--jobs=N] [TESTS...]
579-
580-
--help Show this text.
581-
--list List all available tests.
582-
--verbose Print more information during testing.
583-
--quickfail Fail the entire run as soon as a single test errored.
584-
--jobs=N Launch `N` processes to perform tests.
585-
586-
Remaining arguments filter the tests that will be executed."""
587-
)
672+
# list tests, if requested
673+
if args.list !== nothing
674+
println(stdout, "Available tests:")
675+
for test in keys(testsuite)
676+
println(stdout, " - $test")
677+
end
588678
exit(0)
589679
end
590-
set_jobs, jobs = extract_flag!(ARGS, "--jobs"; typ = Int)
591-
do_verbose, _ = extract_flag!(ARGS, "--verbose")
592-
do_quickfail, _ = extract_flag!(ARGS, "--quickfail")
593-
do_list, _ = extract_flag!(ARGS, "--list")
594-
## no options should remain
595-
optlike_args = filter(startswith("-"), ARGS)
596-
if !isempty(optlike_args)
597-
error("Unknown test options `$(join(optlike_args, " "))` (try `--help` for usage instructions)")
598-
end
680+
681+
# filter tests
682+
filter_tests!(testsuite, args)
599683

600684
# determine test order
601685
tests = collect(keys(testsuite))
602686
Random.shuffle!(tests)
603687
historical_durations = load_test_history(mod)
604688
sort!(tests, by = x -> -get(historical_durations, x, Inf))
605689

606-
# list tests, if requested
607-
if do_list
608-
println(stdout, "Available tests:")
609-
for test in sort(tests)
610-
println(stdout, " - $test")
611-
end
612-
exit(0)
613-
end
614-
615-
# filter tests based on command-line arguments
616-
if !isempty(ARGS)
617-
filter!(tests) do test
618-
any(arg -> startswith(test, arg), ARGS)
619-
end
620-
end
621-
622690
# determine parallelism
623-
if !set_jobs
624-
jobs = default_njobs()
625-
end
691+
jobs = something(args.jobs, default_njobs())
626692
jobs = clamp(jobs, 1, length(tests))
627693
println(stdout, "Running $jobs tests in parallel. If this is too many, specify the `--jobs=N` argument to the tests, or set the `JULIA_CPU_THREADS` environment variable.")
628694
workers = addworkers(min(jobs, length(tests)))
@@ -761,7 +827,7 @@ function runtests(mod::Module, ARGS; testsuite::Dict{String,Expr} = find_tests(p
761827
test_name, wrkr = msg[2], msg[3]
762828

763829
# Optionally print verbose started message
764-
if do_verbose
830+
if args.verbose !== nothing
765831
clear_status()
766832
print_test_started(RecordType, wrkr, test_name, io_ctx)
767833
end
@@ -868,7 +934,7 @@ function runtests(mod::Module, ARGS; testsuite::Dict{String,Expr} = find_tests(p
868934
# One of Malt.TerminatedWorkerException, Malt.RemoteException, or ErrorException
869935
@assert result isa Exception
870936
put!(printer_channel, (:crashed, test, worker_id(wrkr)))
871-
if do_quickfail
937+
if args.quickfail !== nothing
872938
stop_work()
873939
end
874940

@@ -977,7 +1043,7 @@ function runtests(mod::Module, ARGS; testsuite::Dict{String,Expr} = find_tests(p
9771043
return testset
9781044
end
9791045
t1 = time()
980-
o_ts = create_testset("Overall"; start=t0, stop=t1, verbose=do_verbose)
1046+
o_ts = create_testset("Overall"; start=t0, stop=t1, verbose=!isnothing(args.verbose))
9811047
function collect_results()
9821048
with_testset(o_ts) do
9831049
completed_tests = Set{String}()
@@ -1054,6 +1120,7 @@ function runtests(mod::Module, ARGS; testsuite::Dict{String,Expr} = find_tests(p
10541120
end
10551121

10561122
return
1057-
end # runtests
1123+
end
1124+
runtests(mod::Module, ARGS; kwargs...) = runtests(mod, parse_args(ARGS); kwargs...)
10581125

1059-
end # module ParallelTestRunner
1126+
end

0 commit comments

Comments
 (0)