[breaking] move BenchmarkTools.jl to a package extension (#2982)

This is a breaking change because it requires the user to install
and load BenchmarkTools.jl.

We justify releasing it in a minor release of MathOptInterface
because we assess that there are very few users of it, and because
it should be used only during package development by advanced users.

We reserve the right to revert this commit in a future release of MOI
if it causes problems in the ecosystem.

Author: odow

Project.toml

@@ -3,7 +3,6 @@ uuid = "b8f27783-ece8-5eb3-8dc8-9495eed66fee"
 version = "1.50.1"
 
 [deps]
-BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 CodecBzip2 = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd"
 CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
@@ -18,6 +17,12 @@ SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
+[weakdeps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+
+[extensions]
+MathOptInterfaceBenchmarkToolsExt = "BenchmarkTools"
+
 [compat]
 BenchmarkTools = "1"
 CodecBzip2 = "0.6, 0.7, 0.8"
@@ -38,8 +43,9 @@ Test = "1"
 julia = "1.10"
 
 [extras]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 JSONSchema = "7d188eb4-7ad8-530c-ae41-71a32a6d4692"
 ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc"
 
 [targets]
-test = ["JSONSchema", "ParallelTestRunner"]
+test = ["BenchmarkTools", "JSONSchema", "ParallelTestRunner"]

docs/src/submodules/Benchmarks/overview.md

@@ -9,13 +9,29 @@ DocTestFilters = [r"MathOptInterface|MOI"]
 # The `Benchmarks` submodule
 
 To aid the development of efficient solver wrappers, MathOptInterface provides
-benchmarking capability. Benchmarking a wrapper follows a two-step process.
+a suite of benchmarks in the `MOI.Benchmarks` submodule.
+
+!!! warning
+    To use this submodule you must first install and load
+    [BenchmarkTools.jl](https://github.com/juliaci/benchmarktools.jl).
+    ```julia
+    import Pkg
+    Pkg.add("BenchmarkTools")
+    import BenchmarkTools
+    ```
+
+## Benchmarking a solver wrapper
+
+Benchmarking a wrapper follows a two-step process.
 
 First, prior to making changes, create a baseline for the benchmark results on a
 given benchmark suite as follows:
 
 ```julia
-using SolverPackage  # Replace with your choice of solver.
+# You must load BenchmarkTools.jl to enable MOI.Benchmarks
+import BenchmarkTools
+# Replace `SolverPackage` with your choice of solver
+using SolverPackage
 import MathOptInterface as MOI
 
 suite = MOI.Benchmarks.suite() do
@@ -33,6 +49,7 @@ Second, after making changes to the package, re-run the benchmark suite and
 compare to the prior saved results:
 
 ```julia
+import BenchmarkTools
 using SolverPackage
 import MathOptInterface as MOI
 

ext/MathOptInterfaceBenchmarkToolsExt.jl

@@ -0,0 +1,79 @@
+# Copyright (c) 2017: Miles Lubin and contributors
+# Copyright (c) 2017: Google Inc.
+#
+# Use of this source code is governed by an MIT-style license that can be found
+# in the LICENSE.md file or at https://opensource.org/licenses/MIT.
+
+module MathOptInterfaceBenchmarkToolsExt
+
+import BenchmarkTools
+import MathOptInterface as MOI
+
+function MOI.Benchmarks.suite(
+    new_model::Function;
+    exclude::Vector{Regex} = Regex[],
+)
+    group = BenchmarkTools.BenchmarkGroup()
+    for (name, func) in MOI.Benchmarks.BENCHMARKS
+        if any(occursin.(exclude, Ref(name)))
+            continue
+        end
+        group[name] = BenchmarkTools.@benchmarkable $func($new_model)
+    end
+    return group
+end
+
+function MOI.Benchmarks.create_baseline(
+    suite::BenchmarkTools.BenchmarkGroup,
+    name::String;
+    directory::String = "",
+    kwargs...,
+)
+    BenchmarkTools.tune!(suite)
+    BenchmarkTools.save(
+        joinpath(directory, name * "_params.json"),
+        BenchmarkTools.params(suite),
+    )
+    results = BenchmarkTools.run(suite; kwargs...)
+    BenchmarkTools.save(joinpath(directory, name * "_baseline.json"), results)
+    return
+end
+
+function MOI.Benchmarks.compare_against_baseline(
+    suite::BenchmarkTools.BenchmarkGroup,
+    name::String;
+    directory::String = "",
+    report_filename::String = "report.txt",
+    kwargs...,
+)
+    params_filename = joinpath(directory, name * "_params.json")
+    baseline_filename = joinpath(directory, name * "_baseline.json")
+    if !isfile(params_filename) || !isfile(baseline_filename)
+        error("You create a baseline with `create_baseline` first.")
+    end
+    BenchmarkTools.loadparams!(
+        suite,
+        BenchmarkTools.load(params_filename)[1],
+        :evals,
+        :samples,
+    )
+    new_results = BenchmarkTools.run(suite; kwargs...)
+    old_results = BenchmarkTools.load(baseline_filename)[1]
+    open(joinpath(directory, report_filename), "w") do io
+        println(stdout, "\n========== Results ==========")
+        println(io, "\n========== Results ==========")
+        for key in keys(new_results)
+            judgement = BenchmarkTools.judge(
+                BenchmarkTools.median(new_results[key]),
+                BenchmarkTools.median(old_results[key]),
+            )
+            println(stdout, "\n", key)
+            println(io, "\n", key)
+            show(stdout, MIME"text/plain"(), judgement)
+            show(io, MIME"text/plain"(), judgement)
+        end
+    end
+    return
+end
+
+end # module

src/Benchmarks/Benchmarks.jl

@@ -6,7 +6,6 @@
 
 module Benchmarks
 
-import BenchmarkTools
 import MathOptInterface as MOI
 
 const BENCHMARKS = Dict{String,Function}()
@@ -22,26 +21,24 @@ arguments, and returns a new instance of the optimizer you wish to benchmark.
 
 Use `exclude` to exclude a subset of benchmarks.
 
+## BenchmarkTools
+
+To use this function you must first install and load the `BenchmarkTools.jl`
+package.
+
 ## Example
 
 ```julia
-julia> MOI.Benchmarks.suite() do
-           return GLPK.Optimizer()
-       end
+julia> import BenchmarkTools, GLPK, Gurobi
+
+julia> MOI.Benchmarks.suite(GLPK.Optimizer)
 
 julia> MOI.Benchmarks.suite(; exclude = [r"delete"]) do
            return Gurobi.Optimizer()
        end
 ```
 """
-function suite(new_model::Function; exclude::Vector{Regex} = Regex[])
-    group = BenchmarkTools.BenchmarkGroup()
-    for (name, func) in BENCHMARKS
-        any(occursin.(exclude, Ref(name))) && continue
-        group[name] = BenchmarkTools.@benchmarkable $func($new_model)
-    end
-    return group
-end
+function suite end
 
 """
     create_baseline(suite, name::String; directory::String = ""; kwargs...)
@@ -50,12 +47,17 @@ Run all benchmarks in `suite` and save to files called `name` in `directory`.
 
 Extra `kwargs` are based to `BenchmarkTools.run`.
 
+## BenchmarkTools
+
+To use this function you must first install and load the `BenchmarkTools.jl`
+package.
+
 ## Example
 
 ```julia
-julia> import GLPK
+julia> import BenchmarkTools, GLPK
 
-julia> my_suite = MOI.Benchmarks.suite(() -> GLPK.Optimizer());
+julia> my_suite = MOI.Benchmarks.suite(GLPK.Optimizer);
 
 julia> MOI.Benchmarks.create_baseline(
            my_suite,
@@ -65,21 +67,7 @@ julia> MOI.Benchmarks.create_baseline(
        )
 ```
 """
-function create_baseline(
-    suite::BenchmarkTools.BenchmarkGroup,
-    name::String;
-    directory::String = "",
-    kwargs...,
-)
-    BenchmarkTools.tune!(suite)
-    BenchmarkTools.save(
-        joinpath(directory, name * "_params.json"),
-        BenchmarkTools.params(suite),
-    )
-    results = BenchmarkTools.run(suite; kwargs...)
-    BenchmarkTools.save(joinpath(directory, name * "_baseline.json"), results)
-    return
-end
+function create_baseline end
 
 """
     compare_against_baseline(
@@ -95,12 +83,17 @@ A report summarizing the comparison is written to `report_filename` in
 
 Extra `kwargs` are based to `BenchmarkTools.run`.
 
+## BenchmarkTools
+
+To use this function you must first install and load the `BenchmarkTools.jl`
+package.
+
 ## Example
 
 ```julia
-julia> import GLPK
+julia> import BenchmarkTools, GLPK
 
-julia> my_suite = MOI.Benchmarks.suite(() -> GLPK.Optimizer());
+julia> my_suite = MOI.Benchmarks.suite(GLPK.Optimizer);
 
 julia> MOI.Benchmarks.compare_against_baseline(
            my_suite,
@@ -110,42 +103,7 @@ julia> MOI.Benchmarks.compare_against_baseline(
        )
 ```
 """
-function compare_against_baseline(
-    suite::BenchmarkTools.BenchmarkGroup,
-    name::String;
-    directory::String = "",
-    report_filename::String = "report.txt",
-    kwargs...,
-)
-    params_filename = joinpath(directory, name * "_params.json")
-    baseline_filename = joinpath(directory, name * "_baseline.json")
-    if !isfile(params_filename) || !isfile(baseline_filename)
-        error("You create a baseline with `create_baseline` first.")
-    end
-    BenchmarkTools.loadparams!(
-        suite,
-        BenchmarkTools.load(params_filename)[1],
-        :evals,
-        :samples,
-    )
-    new_results = BenchmarkTools.run(suite; kwargs...)
-    old_results = BenchmarkTools.load(baseline_filename)[1]
-    open(joinpath(directory, report_filename), "w") do io
-        println(stdout, "\n========== Results ==========")
-        println(io, "\n========== Results ==========")
-        for key in keys(new_results)
-            judgement = BenchmarkTools.judge(
-                BenchmarkTools.median(new_results[key]),
-                BenchmarkTools.median(old_results[key]),
-            )
-            println(stdout, "\n", key)
-            println(io, "\n", key)
-            show(stdout, MIME"text/plain"(), judgement)
-            show(io, MIME"text/plain"(), judgement)
-        end
-    end
-    return
-end
+function compare_against_baseline end
 
 ###
 ### Benchmarks

test/Benchmarks/test_Benchmarks.jl

@@ -8,6 +8,7 @@ module TestBenchmarks
 
 using Test
 
+import BenchmarkTools
 import MathOptInterface as MOI
 
 function runtests()