From 835a770fdc24f9230624f26ab3ac3bfd942c9f74 Mon Sep 17 00:00:00 2001 From: Daniel Weindl Date: Tue, 20 Oct 2026 13:30:48 +0200 Subject: [PATCH] Add v2 overview table to the site and slim the README table The GitHub Pages landing page now shows sortable and filterable overview tables for the PEtab v2 and v1 encodings (Tabulator, rendered via a Jinja2 template in the internal `_site` package), with problem descriptions from `metadata.yaml`, summary tiles, column groups, header tooltips, copy to clipboard with headers, CSV download, and a light/dark toggle. It replaces the bokeh table generated by `bmp-create-overview --html-file`. v2 statistics are computed by the new `overview_v2` module, using v2-specific definitions where v1 ones don't apply (experiments instead of simulation conditions; preequilibration and postequilibration counted per experiment; v2 noise and prior distribution names). Problems with multiple SBML models are supported. `v1.MODELS` and `v2.MODELS` list the problems with a v1 or v2 encoding. The README overview table is reduced to version-independent columns: ID, links to the available v1/v2 encodings, and the short description. Per-problem statistics are now only on the site. `bmp-create-overview --markdown` prints the new README table. Co-Authored-By: Claude Opus 5.5 --- README.md | 88 +-- src/python/MANIFEST.in | 1 + .../benchmark_models_petab/_site/__init__.py | 3 + .../benchmark_models_petab/_site/build.py | 204 ++++++ .../_site/templates/index.html.j2 | 584 ++++++++++++++++++ src/python/benchmark_models_petab/overview.py | 293 ++++----- .../benchmark_models_petab/overview_v2.py | 164 +++++ src/python/benchmark_models_petab/v1.py | 17 +- src/python/benchmark_models_petab/v2.py | 15 +- src/python/pyproject.toml | 2 +- src/python/test/test_overview.py | 18 + src/python/test/test_overview_v2.py | 115 ++++ src/python/test/test_v1.py | 5 + src/python/test/test_v2.py | 5 + 14 files changed, 1308 insertions(+), 206 deletions(-) create mode 100644 src/python/benchmark_models_petab/_site/__init__.py create mode 100644 src/python/benchmark_models_petab/_site/build.py create mode 100644 src/python/benchmark_models_petab/_site/templates/index.html.j2 create mode 100644 src/python/benchmark_models_petab/overview_v2.py create mode 100644 src/python/test/test_overview.py create mode 100644 src/python/test/test_overview_v2.py diff --git a/README.md b/README.md index 48ad16e..85729fd 100644 --- a/README.md +++ b/README.md @@ -6,49 +6,55 @@ Contributions to the collection are very welcome. Please see [CONTRIBUTING.md](C ## Overview -The following table summarizes the problems in the collection. -The full set of PEtab files are available in the respective subfolders. -A sortable version of the table is available -[here](https://benchmarking-initiative.github.io/Benchmark-Models-PEtab/). +The collection spans a wide variety of problems, in terms of application +as well as the numbers of observables, parameters, measurements, and +experimental conditions, and features such as events, preequilibration, +and noise and prior distributions. +For these details, see the sortable and filterable +[overview tables](https://benchmarking-initiative.github.io/Benchmark-Models-PEtab/). + +The following table lists the problems in the collection and the +available encodings in PEtab v1 and v2. +The PEtab files are available in the respective subdirectories. -| PEtab Problem ID | Conditions | Estimated Parameters | Events | Possible Discontinuities | Preequilibration | Postequilibration | Measurements | Observables | Noise distribution(s) | Objective prior distribution(s) | Species | References | SBML4Humans | -|:----------------------------------------------------------------------------------------|-------------:|-----------------------:|---------:|:---------------------------|-------------------:|--------------------:|---------------:|--------------:|:------------------------|:----------------------------------|----------:|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [Alkan_SciSignal2018](problems/Alkan_SciSignal2018/v1/) | 73 | 44 | 0 | ✓ | 0 | 0 | 1733 | 12 | normal | | 36 | [\[1\]](http://identifiers.org/doi/10.1126/scisignal.aat0229) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Alkan_SciSignal2018/v1/model.xml) | -| [Armistead_CellDeathDis2024](problems/Armistead_CellDeathDis2024/v1/) | 2 | 14 | 0 | | 0 | 0 | 58 | 4 | normal | | 4 | [\[1\]](https://doi.org/10.1038/s41419-024-07134-2) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Armistead_CellDeathDis2024/v1/model.xml) | -| [Bachmann_MSB2011](problems/Bachmann_MSB2011/v1/) | 36 | 113 | 0 | | 0 | 0 | 541 | 20 | normal; log10-normal | parameterScaleNormal | 25 | [\[1\]](http://identifiers.org/doi/10.1038/msb.2011.50) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Bachmann_MSB2011/v1/model.xml) | -| [Beer_MolBioSystems2014](problems/Beer_MolBioSystems2014/v1/) | 19 | 72 | 0 | ✓ | 0 | 0 | 27132 | 2 | normal | | 4 | [\[1\]](http://identifiers.org/doi/10.1039/c3mb70594c) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Beer_MolBioSystems2014/v1/model.xml) | -| [Bertozzi_PNAS2020](problems/Bertozzi_PNAS2020/v1/) | 2 | 8 | 0 | | 0 | 0 | 22 | 2 | normal | | 3 | [\[1\]](http://identifiers.org/doi/10.1073/pnas.2006520117) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Bertozzi_PNAS2020/v1/model.xml) | -| [Blasi_CellSystems2016](problems/Blasi_CellSystems2016/v1/) | 1 | 9 | 0 | | 0 | 1 | 252 | 15 | log-normal | | 16 | [\[1\]](http://identifiers.org/doi/10.1016/j.cels.2016.01.002) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Blasi_CellSystems2016/v1/model.xml) | -| [Boehm_JProteomeRes2014](problems/Boehm_JProteomeRes2014/v1/) | 1 | 9 | 0 | | 0 | 0 | 48 | 3 | normal | | 8 | [\[1\]](http://identifiers.org/doi/10.1021/pr5006923) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Boehm_JProteomeRes2014/v1/model.xml) | -| [Borghans_BiophysChem1997](problems/Borghans_BiophysChem1997/v1/) | 1 | 23 | 0 | | 0 | 0 | 111 | 1 | log10-normal | | 3 | [\[1\]](http://identifiers.org/doi/10.1016/s0301-4622(97)00010-0) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Borghans_BiophysChem1997/v1/model.xml) | -| [Brannmark_JBC2010](problems/Brannmark_JBC2010/v1/) | 8 | 22 | 0 | ✓ | 1 | 0 | 43 | 3 | normal | | 9 | [\[1\]](http://identifiers.org/doi/10.1074/jbc.M110.106849) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Brannmark_JBC2010/v1/model.xml) | -| [Bruno_JExpBot2016](problems/Bruno_JExpBot2016/v1/) | 6 | 13 | 0 | | 0 | 0 | 77 | 5 | normal | | 7 | [\[1\]](http://identifiers.org/doi/10.1093/jxb/erw356) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Bruno_JExpBot2016/v1/model.xml) | -| [Chen_MSB2009](problems/Chen_MSB2009/v1/) | 4 | 155 | 0 | ✓ | 0 | 0 | 120 | 3 | normal | | 500 | [\[1\]](http://identifiers.org/doi/10.1038/msb.2008.74) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Chen_MSB2009/v1/model.xml) | -| [Crauste_CellSystems2017](problems/Crauste_CellSystems2017/v1/) | 1 | 12 | 0 | | 0 | 0 | 21 | 4 | normal | | 5 | [\[1\]](http://identifiers.org/doi/10.1016/j.cels.2017.01.014) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Crauste_CellSystems2017/v1/model.xml) | -| [Elowitz_Nature2000](problems/Elowitz_Nature2000/v1/) | 1 | 21 | 0 | | 0 | 0 | 58 | 1 | log10-normal | | 8 | [\[1\]](http://identifiers.org/doi/10.1038/35002125) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Elowitz_Nature2000/v1/model.xml) | -| [Fiedler_BMCSystBiol2016](problems/Fiedler_BMCSystBiol2016/v1/) | 3 | 22 | 0 | | 0 | 0 | 72 | 2 | normal | | 6 | [\[1\]](http://identifiers.org/doi/10.1186/s12918-016-0319-7) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Fiedler_BMCSystBiol2016/v1/model.xml) | -| [Froehlich_CellSystems2018](problems/Froehlich_CellSystems2018/v1/) | 9169 | 4231 | 0 | | 0 | 9169 | 9169 | 1 | normal | | 1396 | [\[1\]](http://identifiers.org/doi/10.1016/j.cels.2018.10.013) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Froehlich_CellSystems2018/v1/model.xml) | -| [Fujita_SciSignal2010](problems/Fujita_SciSignal2010/v1/) | 6 | 19 | 0 | ✓ | 0 | 0 | 144 | 3 | normal | | 9 | [\[1\]](http://identifiers.org/doi/10.1126/scisignal.2000810) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Fujita_SciSignal2010/v1/model.xml) | -| [Giordano_Nature2020](problems/Giordano_Nature2020/v1/) | 1 | 50 | 0 | ✓ | 0 | 0 | 313 | 7 | normal | | 13 | [\[1\]](http://identifiers.org/pubmed/32322102) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Giordano_Nature2020/v1/model.xml) | -| [Isensee_JCB2018](problems/Isensee_JCB2018/v1/) | 123 | 46 | 0 | ✓ | 1 | 0 | 687 | 3 | normal | parameterScaleNormal | 25 | [\[1\]](http://identifiers.org/doi/10.1083/jcb.201708053) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Isensee_JCB2018/v1/model.xml) | -| [Lang_PLOSComputBiol2024](problems/Lang_PLOSComputBiol2024/v1/) | 1 | 294 | 0 | | 0 | 0 | 9600 | 16 | normal | laplace; uniform | 124 | [\[1\]](http://identifiers.org/doi/10.1371/journal.pcbi.1011151) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Lang_PLOSComputBiol2024/v1/model.xml) | -| [Laske_PLOSComputBiol2019](problems/Laske_PLOSComputBiol2019/v1/) | 3 | 13 | 0 | | 0 | 0 | 42 | 13 | normal; log-normal | | 41 | [\[1\]](http://identifiers.org/biomodels.db/BIOMD0000000463) [\[2\]](http://identifiers.org/biomodels.db/MODEL1307270000) [\[3\]](http://identifiers.org/doi/10.1371/journal.pcbi.1006944) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Laske_PLOSComputBiol2019/v1/model.xml) | -| [Liu_IFACPapersOnLine2025](problems/Liu_IFACPapersOnLine2025/v1/) | 2 | 9 | 1 | ✓ | 0 | 0 | 100 | 1 | normal | | 0 | [\[1\]](http://identifiers.org/doi/10.1016/j.ifacol.2025.07.160) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Liu_IFACPapersOnLine2025/v1/model.xml) | -| [Lucarelli_CellSystems2018](problems/Lucarelli_CellSystems2018/v1/) | 16 | 84 | 0 | | 0 | 0 | 1755 | 65 | normal; log10-normal | | 33 | [\[1\]](http://identifiers.org/doi/10.1016/j.cels.2017.11.010) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Lucarelli_CellSystems2018/v1/model.xml) | -| [Okuonghae_ChaosSolitonsFractals2020](problems/Okuonghae_ChaosSolitonsFractals2020/v1/) | 1 | 16 | 0 | | 0 | 0 | 92 | 2 | normal | | 9 | [\[1\]](http://identifiers.org/doi/10.1016/j.chaos.2020.110032) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Okuonghae_ChaosSolitonsFractals2020/v1/model.xml) | -| [Oliveira_NatCommun2021](problems/Oliveira_NatCommun2021/v1/) | 1 | 12 | 0 | ✓ | 0 | 0 | 120 | 2 | normal | | 9 | [\[1\]](http://identifiers.org/doi/10.1038/s41467-020-19798-3) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Oliveira_NatCommun2021/v1/model.xml) | -| [Perelson_Science1996](problems/Perelson_Science1996/v1/) | 1 | 3 | 0 | | 0 | 0 | 16 | 1 | log10-normal | | 4 | [\[1\]](http://identifiers.org/doi/10.1126/science.271.5255.1582) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Perelson_Science1996/v1/model.xml) | -| [Rahman_MBS2016](problems/Rahman_MBS2016/v1/) | 1 | 9 | 0 | | 0 | 0 | 23 | 1 | normal | | 7 | [\[1\]](http://identifiers.org/doi/10.1016/j.mbs.2016.07.009) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Rahman_MBS2016/v1/model.xml) | -| [Raia_CancerResearch2011](problems/Raia_CancerResearch2011/v1/) | 4 | 39 | 0 | | 0 | 0 | 205 | 8 | normal | | 14 | [\[1\]](http://identifiers.org/doi/10.1158/0008-5472.CAN-10-2987) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Raia_CancerResearch2011/v1/model.xml) | -| [Raimundez_PCB2020](problems/Raimundez_PCB2020/v1/) | 170 | 136 | 0 | ✓ | 4 | 0 | 627 | 79 | normal | parameterScaleNormal; laplace | 22 | [\[1\]](http://identifiers.org/doi/10.1371/journal.pcbi.1007147) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Raimundez_PCB2020/v1/model.xml) | -| [SalazarCavazos_MBoC2020](problems/SalazarCavazos_MBoC2020/v1/) | 4 | 6 | 0 | | 0 | 0 | 18 | 3 | normal | | 75 | [\[1\]](http://identifiers.org/doi/10.1091/mbc.E19-09-0548) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/SalazarCavazos_MBoC2020/v1/model.xml) | -| [Schwen_PONE2014](problems/Schwen_PONE2014/v1/) | 19 | 30 | 0 | | 0 | 0 | 286 | 4 | log10-normal | parameterScaleNormal | 11 | [\[1\]](http://identifiers.org/doi/10.1371/journal.pone.0133653) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Schwen_PONE2014/v1/model.xml) | -| [Smith_BMCSystBiol2013](problems/Smith_BMCSystBiol2013/v1/) | 35 | 25 | 3 | ✓ | 0 | 0 | 62 | 9 | normal | | 133 | [\[1\]](http://identifiers.org/doi/10.1186/1752-0509-7-41) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Smith_BMCSystBiol2013/v1/model.xml) | -| [Sneyd_PNAS2002](problems/Sneyd_PNAS2002/v1/) | 9 | 15 | 0 | | 0 | 0 | 135 | 1 | normal | | 6 | [\[1\]](http://identifiers.org/doi/10.1073/pnas.032281999) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Sneyd_PNAS2002/v1/model.xml) | -| [Weber_BMC2015](problems/Weber_BMC2015/v1/) | 2 | 36 | 0 | ✓ | 1 | 0 | 135 | 8 | normal | | 7 | [\[1\]](http://identifiers.org/doi/10.1186/s12918-015-0147-1) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Weber_BMC2015/v1/model.xml) | -| [Zhao_QuantBiol2020](problems/Zhao_QuantBiol2020/v1/) | 7 | 28 | 0 | | 0 | 0 | 82 | 1 | normal | | 5 | [\[1\]](http://identifiers.org/pubmed/32219006) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Zhao_QuantBiol2020/v1/model.xml) | -| [Zheng_PNAS2012](problems/Zheng_PNAS2012/v1/) | 1 | 46 | 0 | | 1 | 0 | 60 | 15 | normal | | 15 | [\[1\]](http://identifiers.org/doi/10.1073/pnas.1205707109) | [\[1\]](https://sbml4humans.de/report?url=https://raw.githubusercontent.com/Benchmarking-Initiative/Benchmark-Models-PEtab/master/problems/Zheng_PNAS2012/v1/model.xml) | +| ID | PEtab v1 | PEtab v2 | Description | +|:-------------------------------------------------------------------------------------|:-------------------------------------------------------|:------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [Alkan_SciSignal2018](problems/Alkan_SciSignal2018/) | [v1](problems/Alkan_SciSignal2018/v1/) | | Reaction network model of the DNA damage response and cell fate in cancer cells treated with chemotherapeutics, fitted to signaling and cell count data. | +| [Armistead_CellDeathDis2024](problems/Armistead_CellDeathDis2024/) | [v1](problems/Armistead_CellDeathDis2024/v1/) | | Small reaction network model of sphingolipid metabolism with ceramide feedback, fitted to lipidomics data of wild-type and mutant zebrafish embryos. | +| [Bachmann_MSB2011](problems/Bachmann_MSB2011/) | [v1](problems/Bachmann_MSB2011/v1/) | | Reaction network model of Epo-induced JAK2/STAT5 signaling with two negative feedback regulators (SOCS3, CIS) in murine erythroid progenitor cells. | +| [Beer_MolBioSystems2014](problems/Beer_MolBioSystems2014/) | [v1](problems/Beer_MolBioSystems2014/v1/) | | Small kinetic model of bacterial growth and production of the blue pigment indigoidine by engineered variants of a non-ribosomal peptide synthetase. | +| [Bertozzi_PNAS2020](problems/Bertozzi_PNAS2020/) | [v1](problems/Bertozzi_PNAS2020/v1/) | | SIR compartmental epidemiological model of the early COVID-19 outbreak in New York and California. | +| [Blasi_CellSystems2016](problems/Blasi_CellSystems2016/) | [v1](problems/Blasi_CellSystems2016/v1/) | | Reaction network model of combinatorial histone H4 acetylation states generated by motif-specific acetylation reactions. | +| [Boehm_JProteomeRes2014](problems/Boehm_JProteomeRes2014/) | [v1](problems/Boehm_JProteomeRes2014/v1/) | [v2](problems/Boehm_JProteomeRes2014/v2/) | Reaction network model of Epo-induced STAT5A/STAT5B phosphorylation and homo- and heterodimerization, fitted to quantitative mass spectrometry data. | +| [Borghans_BiophysChem1997](problems/Borghans_BiophysChem1997/) | [v1](problems/Borghans_BiophysChem1997/v1/) | | Three-variable ODE model of complex calcium oscillations with Ca2+-activated IP3 degradation, fitted to measured calcium bursting in a single hepatocyte. | +| [Brannmark_JBC2010](problems/Brannmark_JBC2010/) | [v1](problems/Brannmark_JBC2010/v1/) | | Reaction network model of insulin receptor and IRS1 phosphorylation with receptor endocytosis and feedback in primary human adipocytes. | +| [Bruno_JExpBot2016](problems/Bruno_JExpBot2016/) | [v1](problems/Bruno_JExpBot2016/v1/) | | Enzyme kinetic model of the in vitro cleavage of carotenoids and apocarotenoids by Arabidopsis carotenoid cleavage dioxygenase 4 (CCD4). | +| [Chen_MSB2009](problems/Chen_MSB2009/) | [v1](problems/Chen_MSB2009/v1/) | | Large mass-action reaction network model of ErbB receptor signaling to ERK and AKT in response to EGF and heregulin. | +| [Crauste_CellSystems2017](problems/Crauste_CellSystems2017/) | [v1](problems/Crauste_CellSystems2017/v1/) | | Cell population model of CD8 T cell differentiation (naive, early/late effector, memory), fitted to cell counts in mice after vaccinia virus infection. | +| [Elowitz_Nature2000](problems/Elowitz_Nature2000/) | [v1](problems/Elowitz_Nature2000/v1/) | | Gene regulatory network model of the repressilator, a synthetic three-gene transcriptional oscillator in E. coli, fitted to a fluorescence time course. | +| [Fiedler_BMCSystBiol2016](problems/Fiedler_BMCSystBiol2016/) | [v1](problems/Fiedler_BMCSystBiol2016/v1/) | | Reaction network model of RAF/MEK/ERK signaling with ERK feedback in HeLa cells, fitted to phospho-MEK/ERK data without and with RAF or MEK inhibitors. | +| [Froehlich_CellSystems2018](problems/Froehlich_CellSystems2018/) | [v1](problems/Froehlich_CellSystems2018/v1/) | | Large-scale reaction network model of cancer signaling pathways, parameterized by cell-line-specific gene expression and fitted to drug response viability data. | +| [Fujita_SciSignal2010](problems/Fujita_SciSignal2010/) | [v1](problems/Fujita_SciSignal2010/v1/) | | Reaction network model of EGF-induced EGFR, Akt and S6 phosphorylation in PC12 cells under different EGF stimulation profiles. | +| [Giordano_Nature2020](problems/Giordano_Nature2020/) | [v1](problems/Giordano_Nature2020/v1/) | | Compartmental epidemiological model (SIDARTHE, distinguishing detected/undetected and symptom severity) of the COVID-19 epidemic in Italy. | +| [Isensee_JCB2018](problems/Isensee_JCB2018/) | [v1](problems/Isensee_JCB2018/v1/) | | Reaction network model of cAMP-induced PKA activation and phosphorylation of its regulatory subunit in rat sensory neurons under pharmacological perturbations. | +| [Lang_PLOSComputBiol2024](problems/Lang_PLOSComputBiol2024/) | [v1](problems/Lang_PLOSComputBiol2024/v1/) | | Reaction network model of the human cell cycle, generated from a rule-based model and fitted to compartment-resolved single-cell imaging data of RPE-1 cells. | +| [Laske_PLOSComputBiol2019](problems/Laske_PLOSComputBiol2019/) | [v1](problems/Laske_PLOSComputBiol2019/v1/) | | Reaction network model of intracellular influenza A virus replication, fitted to viral RNA, genome import and virus release data in human A549 cells. | +| [Liu_IFACPapersOnLine2025](problems/Liu_IFACPapersOnLine2025/) | [v1](problems/Liu_IFACPapersOnLine2025/v1/) | | Population model of oncolytic virus therapy with uninfected and infected tumor cells and virus, fitted to tumor size data in zebrafish. | +| [Lucarelli_CellSystems2018](problems/Lucarelli_CellSystems2018/) | [v1](problems/Lucarelli_CellSystems2018/v1/) | | Reaction network model of TGF-beta-induced SMAD complex formation and target gene expression. | +| [Okuonghae_ChaosSolitonsFractals2020](problems/Okuonghae_ChaosSolitonsFractals2020/) | [v1](problems/Okuonghae_ChaosSolitonsFractals2020/v1/) | | Compartmental epidemiological model of COVID-19 in Lagos, Nigeria, including asymptomatic and detected infections. | +| [Oliveira_NatCommun2021](problems/Oliveira_NatCommun2021/) | [v1](problems/Oliveira_NatCommun2021/v1/) | | Compartmental epidemiological model of COVID-19 in Bahia, Brazil (state excluding the capital Salvador), with non-detected cases, hospitalization and deaths. | +| [Perelson_Science1996](problems/Perelson_Science1996/) | [v1](problems/Perelson_Science1996/v1/) | | Within-host model of HIV-1 viral dynamics, fitted to plasma HIV-1 RNA levels after treatment with the protease inhibitor ritonavir. | +| [Rahman_MBS2016](problems/Rahman_MBS2016/) | [v1](problems/Rahman_MBS2016/v1/) | | Compartmental epidemiological model of HIV transmission with CD4+ T cell count-based infection stages and treatment, fitted to HIV prevalence in South Africa. | +| [Raia_CancerResearch2011](problems/Raia_CancerResearch2011/) | [v1](problems/Raia_CancerResearch2011/v1/) | | Reaction network model of IL-13-induced JAK2/STAT5 signaling in the primary mediastinal B-cell lymphoma cell line MedB-1. | +| [Raimundez_PCB2020](problems/Raimundez_PCB2020/) | [v1](problems/Raimundez_PCB2020/v1/) | | Reaction network model of EGFR and MET signaling to ERK and AKT in two gastric cancer cell lines in response to EGF and the anti-EGFR antibody cetuximab. | +| [SalazarCavazos_MBoC2020](problems/SalazarCavazos_MBoC2020/) | [v1](problems/SalazarCavazos_MBoC2020/v1/) | | Reaction network model of EGFR dimerization, multisite phosphorylation and binding of the adaptor proteins GRB2 and SHC1, generated from a rule-based model. | +| [Schwen_PONE2014](problems/Schwen_PONE2014/) | [v1](problems/Schwen_PONE2014/v1/) | [v2](problems/Schwen_PONE2014/v2/) | Reaction network model of insulin binding, internalization and degradation in two subpopulations of isolated hepatocytes with low and high insulin binding. | +| [Smith_BMCSystBiol2013](problems/Smith_BMCSystBiol2013/) | [v1](problems/Smith_BMCSystBiol2013/v1/) | | Large reaction network model of insulin signaling in rodent adipocytes and its regulation by oxidative stress. | +| [Sneyd_PNAS2002](problems/Sneyd_PNAS2002/) | [v1](problems/Sneyd_PNAS2002/v1/) | | Kinetic model of the type-2 IP3 receptor, fitted to time courses of calcium release from liver microsomes at varying IP3 and calcium concentrations. | +| [Weber_BMC2015](problems/Weber_BMC2015/) | [v1](problems/Weber_BMC2015/v1/) | | Reaction network model of interactions of protein kinase D, a lipid kinase (PI4KIIIbeta) and a ceramide transfer protein (CERT) at the trans-Golgi network. | +| [Zhao_QuantBiol2020](problems/Zhao_QuantBiol2020/) | [v1](problems/Zhao_QuantBiol2020/v1/) | | Compartmental epidemiological model of COVID-19 in Wuhan, Hubei and the rest of China, with quarantined and unquarantined infected individuals. | +| [Zheng_PNAS2012](problems/Zheng_PNAS2012/) | [v1](problems/Zheng_PNAS2012/v1/) | | Reaction network model of the combinatorial methylation of lysines 27 and 36 on histone H3. | ## License diff --git a/src/python/MANIFEST.in b/src/python/MANIFEST.in index 2d06440..945aae1 100644 --- a/src/python/MANIFEST.in +++ b/src/python/MANIFEST.in @@ -1,2 +1,3 @@ recursive-include benchmark_models_petab/problems * include benchmark_models_petab/schemas/*.json +include benchmark_models_petab/_site/templates/*.j2 diff --git a/src/python/benchmark_models_petab/_site/__init__.py b/src/python/benchmark_models_petab/_site/__init__.py new file mode 100644 index 0000000..f28af58 --- /dev/null +++ b/src/python/benchmark_models_petab/_site/__init__.py @@ -0,0 +1,3 @@ +"""Generation of the GitHub Pages site.""" + +from .build import create_overview_page diff --git a/src/python/benchmark_models_petab/_site/build.py b/src/python/benchmark_models_petab/_site/build.py new file mode 100644 index 0000000..07f8dc0 --- /dev/null +++ b/src/python/benchmark_models_petab/_site/build.py @@ -0,0 +1,204 @@ +"""Build the overview page of the GitHub Pages site.""" + +import json +from pathlib import Path + +import pandas as pd + +from .. import overview, overview_v2, v1, v2 +from ..C import MODELS +from ..C import ( + GITHUB_DEFAULT_BRANCH, + GITHUB_REPO, + PROBLEMS_DIRNAME, + V1_DIRNAME, + V2_DIRNAME, +) +from ..metadata import get_metadata + +REPO_URL = f"https://github.com/{GITHUB_REPO}" +LOGO_URL = ( + f"https://raw.githubusercontent.com/{GITHUB_REPO}" + f"/{GITHUB_DEFAULT_BRANCH}/gfx/logo.svg" +) +CONTRIBUTING_URL = f"{REPO_URL}/blob/{GITHUB_DEFAULT_BRANCH}/CONTRIBUTING.md" + +#: Rendering of the columns that are not plain text, by column name. +#: Other columns of integer dtype are rendered as numbers. +column_kinds = { + "petab_problem_id": "id", + "short_description": "description", + "possible_discontinuities": "bool", + "noise_distributions": "tags", + "objective_prior_distributions": "tags", + "prior_distributions": "tags", + "reference_uris": "links", + "sbml4humans_urls": "links", +} + +#: Column groups, in display order, with their columns. +#: Columns without a group are shown first. +column_groups = { + "Links": ["reference_uris", "sbml4humans_urls"], + "Size": [ + "conditions", + "experiments", + "estimated_parameters", + "measurements", + "observables", + "species", + ], + "Features": [ + "events", + "preequilibration", + "postequilibration", + "possible_discontinuities", + ], + "Distributions": [ + "noise_distributions", + "objective_prior_distributions", + "prior_distributions", + ], +} + +column_group = { + col: group for group, cols in column_groups.items() for col in cols +} + +#: Statistics shown as summary tiles, as (v1 column, v2 column, label). +summary_columns = [ + ("conditions", "experiments", "simulation conditions"), + ("estimated_parameters", "estimated_parameters", "estimated parameters"), + ("measurements", "measurements", "measurements"), + ("observables", "observables", "observables"), +] + + +def get_table( + df: pd.DataFrame, + labels: dict[str, str], + descriptions: dict[str, str], + version_dirname: str, +) -> dict: + """Get the data and column specification for one overview table. + + :param df: Overview table as returned by `get_overview_df`. + :param labels: Column labels, by column name. + :param descriptions: Column descriptions, by column name. + :param version_dirname: The encoding's directory (`v1`, `v2`). + """ + df = df.reset_index() + df.insert( + 1, + "short_description", + [get_metadata(id_).short_description for id_ in df.petab_problem_id], + ) + labels = {"short_description": "Description"} | labels + descriptions = { + "short_description": ( + "Model type, biological system, and, where informative, the data." + ) + } | descriptions + # ungrouped columns first, in their original order, then the groups + ungrouped = [col for col in df.columns if col not in column_group] + ordered_columns = ungrouped + [ + col for cols in column_groups.values() for col in cols if col in df + ] + columns = [ + { + "field": col, + "title": labels.get(col, col), + "tooltip": descriptions.get(col), + "group": column_group.get(col), + "kind": column_kinds.get( + col, + "int" if pd.api.types.is_integer_dtype(df[col]) else "text", + ), + } + for col in ordered_columns + ] + return { + "columns": columns, + "rows": json.loads(df[ordered_columns].to_json(orient="records")), + "tree_url": ( + f"{REPO_URL}/tree/{GITHUB_DEFAULT_BRANCH}/{PROBLEMS_DIRNAME}" + f"/{{id}}/{version_dirname}" + ), + } + + +def get_summary_tiles(v1_df: pd.DataFrame, v2_df: pd.DataFrame) -> list: + """Get the summary statistics shown above the tables. + + The ranges are computed from the v1 statistics, and from the v2 + statistics for problems without a v1 encoding. + + :param v1_df: v1 overview table. + :param v2_df: v2 overview table. + :return: List of ``(value, label, tooltip)`` tuples. + """ + tiles = [ + ( + f"{len(MODELS)}", + "problems in total", + "Number of benchmark problems, in any PEtab version.", + ), + ( + f"{len(v1.MODELS)}", + "in PEtab v1", + "Number of problems with a PEtab v1 encoding.", + ), + ( + f"{len(v2.MODELS)}", + "in PEtab v2", + "Number of problems with a PEtab v2 encoding.", + ), + ] + v2_only = ~v2_df.index.isin(v1_df.index) + for v1_col, v2_col, label in summary_columns: + values = pd.concat([v1_df[v1_col], v2_df.loc[v2_only, v2_col]]) + tooltip = f"Smallest and largest number of {label} per problem." + if v1_col != v2_col: + tooltip += ( + f" For problems without a PEtab v1 encoding, the number of " + f"{overview_v2.columns[v2_col].lower()}." + ) + tiles.append((f"{values.min():,}–{values.max():,}", label, tooltip)) + return tiles + + +def create_overview_page(dest: Path) -> None: + """Create the HTML page with the v1 and v2 overview tables. + + :param dest: Path to the output HTML file. + """ + from jinja2 import Environment, PackageLoader, select_autoescape + + env = Environment( + loader=PackageLoader(__package__), + autoescape=select_autoescape(["html", "j2"]), + ) + v1_df = overview.get_overview_df() + v2_df = overview_v2.get_overview_df() + html = env.get_template("index.html.j2").render( + repo_url=REPO_URL, + logo_url=LOGO_URL, + contributing_url=CONTRIBUTING_URL, + summary_tiles=get_summary_tiles(v1_df, v2_df), + tables={ + "v2": get_table( + v2_df, + overview_v2.columns, + overview_v2.column_descriptions, + V2_DIRNAME, + ), + "v1": get_table( + v1_df, + overview.columns, + overview.column_descriptions, + V1_DIRNAME, + ), + }, + ) + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(html, encoding="utf-8") diff --git a/src/python/benchmark_models_petab/_site/templates/index.html.j2 b/src/python/benchmark_models_petab/_site/templates/index.html.j2 new file mode 100644 index 0000000..278ddd8 --- /dev/null +++ b/src/python/benchmark_models_petab/_site/templates/index.html.j2 @@ -0,0 +1,584 @@ + + + + + +Benchmark Problems + + + + + + +
+
+
+
+ PEtab Benchmark Collection logo +
+

Benchmark Problems

+

+ Parameter estimation problems with published models and experimental + data in the PEtab format, + from the Benchmark-Models-PEtab + collection. +

+
+
+ +
+
+{%- for value, label, tooltip in summary_tiles %} +
+
{{ value }}
+
{{ label }}
+
+{%- endfor %} +
+
+
+
+

+ Click a column header to sort; hover over it for a definition. Filter with + the fields below the headers; numeric columns accept, e.g., 10, + >10, <=5, or 10-100. + Click a table and press Ctrl+C to copy its visible + rows, including the column headers. +

+ + +

PEtab v2

+
+

+ +
+
+ +

PEtab v1

+
+

+ +
+
+
+ + + + + + diff --git a/src/python/benchmark_models_petab/overview.py b/src/python/benchmark_models_petab/overview.py index b146975..f1fb4d6 100644 --- a/src/python/benchmark_models_petab/overview.py +++ b/src/python/benchmark_models_petab/overview.py @@ -11,23 +11,29 @@ from petab.v1.yaml import load_yaml from sbmlmath import sbml_math_to_sympy from sbmlmath.csymbol import TimeSymbol -from . import MODELS, get_problem, get_problem_yaml_path -from .C import GITHUB_DEFAULT_BRANCH, GITHUB_REPO, PROBLEMS_DIRNAME, V1_DIRNAME +from . import MODELS, get_problem, get_problem_yaml_path, v1, v2 +from .C import ( + GITHUB_DEFAULT_BRANCH, + GITHUB_REPO, + PROBLEMS_DIRNAME, + V1_DIRNAME, + V2_DIRNAME, +) +from .metadata import get_metadata import sympy as sp from sympy.core.relational import Relational -REPO_URL = f"https://github.com/{GITHUB_REPO}/" - readme_md = Path(__file__).resolve().parent.parent / "README.md" -markdown_columns = { +#: Column labels for the overview table. +columns = { "conditions": "Conditions", "estimated_parameters": "Estimated Parameters", "events": "Events", "preequilibration": "Preequilibration", "postequilibration": "Postequilibration", "measurements": "Measurements", - "petab_problem_id": "PEtab Problem ID", + "petab_problem_id": "ID", "observables": "Observables", "species": "Species", "noise_distributions": "Noise distribution(s)", @@ -37,6 +43,42 @@ "possible_discontinuities": "Possible Discontinuities", } +#: Column descriptions for the overview table. +#: Shown as HTML on the site, so they must not contain markup characters. +column_descriptions = { + "petab_problem_id": ( + "Problem ID: first author, journal, and year of the publication." + ), + "conditions": ( + "Number of simulation conditions (combinations of preequilibration " + "and simulation condition) in the measurement table." + ), + "estimated_parameters": "Number of estimated parameters.", + "events": "Number of SBML events.", + "preequilibration": "Number of distinct preequilibration conditions.", + "postequilibration": ( + "Number of simulation conditions with steady-state measurements " + "(time = inf)." + ), + "measurements": "Number of measurements (data points).", + "observables": "Number of observables with measurements.", + "species": "Number of species in the SBML model.", + "noise_distributions": ( + "Noise distributions, including the observable transformation." + ), + "objective_prior_distributions": ( + "Prior distributions of estimated parameters that enter the " + "objective function." + ), + "reference_uris": "Publications, from the SBML model annotation.", + "sbml4humans_urls": "The SBML model rendered by SBML4Humans.", + "possible_discontinuities": ( + "Heuristic: events with assignments, or discontinuous functions " + "(piecewise, min, max, ...) with time-dependent arguments in the " + "model." + ), +} + index_column = "petab_problem_id" @@ -110,16 +152,29 @@ def get_sbml4humans_urls(petab_problem_id: str) -> List[str]: """Get URL to SBML4humans model""" yaml_file = get_problem_yaml_path(petab_problem_id) yaml_dict = load_yaml(yaml_file) + model_filenames = [ + model_filename + for problem_dict in yaml_dict[petab.PROBLEMS] + for model_filename in problem_dict[petab.SBML_FILES] + ] + return make_sbml4humans_urls(petab_problem_id, V1_DIRNAME, model_filenames) + + +def make_sbml4humans_urls( + petab_problem_id: str, version_dirname: str, model_filenames: list[str] +) -> list[str]: + """Get SBML4humans URLs for the given model files of a problem. + + :param petab_problem_id: Problem ID. + :param version_dirname: The encoding's directory (`v1`, `v2`). + :param model_filenames: Model file paths relative to that directory. + """ repo_root = f"https://raw.githubusercontent.com/{GITHUB_REPO}/{GITHUB_DEFAULT_BRANCH}" - urls = [] - for problem_dict in yaml_dict[petab.PROBLEMS]: - for model_filename in problem_dict[petab.SBML_FILES]: - gh_raw_url = ( - f"{repo_root}/{PROBLEMS_DIRNAME}/{petab_problem_id}" - f"/{V1_DIRNAME}/{model_filename}" - ) - urls.append(f"https://sbml4humans.de/report?url={gh_raw_url}") - return urls + return [ + f"https://sbml4humans.de/report?url={repo_root}/{PROBLEMS_DIRNAME}" + f"/{petab_problem_id}/{version_dirname}/{model_filename}" + for model_filename in model_filenames + ] def get_noise_distributions(observable_df): @@ -191,15 +246,21 @@ def guess_discontinuities(petab_problem: petab.Problem) -> bool: discontinuities might play a role in the simulation; the presence of discontinuities might be parameter-dependent; ... . """ - model: libsbml.Model = petab_problem.sbml_model + return sbml_model_has_discontinuities(petab_problem.sbml_model) + + +def sbml_model_has_discontinuities(model: libsbml.Model) -> bool: + """Guess whether the SBML model has discontinuities. + See :func:`guess_discontinuities`. + """ for event in model.getListOfEvents(): for ea in event.getListOfEventAssignments(): if ea.getMath(): return True # convert reactions to rate rules - sbml_doc = petab_problem.sbml_model.getSBMLDocument().clone() + sbml_doc = model.getSBMLDocument().clone() model = sbml_doc.getModel() conversion_config = libsbml.ConversionProperties() conversion_config.addOption("replaceReactions") @@ -271,7 +332,7 @@ def expr_maybe_time_dependent(expr: sp.Expr) -> bool: def get_overview_df() -> pd.DataFrame: """Get overview table with stats for all benchmark PEtab problems""" data = [] - for petab_problem_id in MODELS: + for petab_problem_id in v1.MODELS: petab_problem = get_problem(petab_problem_id) summary = get_summary(petab_problem, petab_problem_id) data.append(summary) @@ -284,153 +345,58 @@ def get_overview_df() -> pd.DataFrame: end_overview_table = "\n\n" -def show_overview_table( - markdown: bool = False, - update_readme: bool = False, -): - df = get_overview_df() - pd.options.display.width = 0 - - if markdown or update_readme: - # directory as markdown link - df.rename( - index=lambda x: f"[{x}]({PROBLEMS_DIRNAME}/{x}/{V1_DIRNAME}/)", - inplace=True, - ) - # references to markdown links - for field in "reference_uris", "sbml4humans_urls": - df[field] = df[field].apply( - lambda x: " ".join( - [f"[\\[{i + 1}\\]]({uri})" for i, uri in enumerate(x)] - ) - ) - df["possible_discontinuities"] = df["possible_discontinuities"].apply( - lambda x: "✓" if x else "" - ) - df.index.rename(markdown_columns[index_column], inplace=True) - df.rename(columns=markdown_columns, inplace=True) - markdown_overview = df.to_markdown() - if update_readme: - with open(readme_md, "r") as f: - readme_content = f.read() - before_table = readme_content.split(start_overview_table)[0] - after_table = readme_content.split(end_overview_table)[1] - new_readme_content = ( - before_table - + start_overview_table - + markdown_overview - + end_overview_table - + after_table - ) - with open(readme_md, "w") as f: - f.write(new_readme_content) - else: - print(markdown_overview) - else: - print(df) - - -def create_html_table(dest: Path) -> None: - """Create HTML table with stats for all benchmark PEtab problems. +def get_readme_table() -> str: + """Get the version-independent overview table for the README. - :param dest: Path to the output HTML file. + Lists each problem's short description and links to its PEtab v1 and + v2 encodings, where available. """ - from bokeh.io import output_file, save - from bokeh.layouts import column - from bokeh.models import ( - ColumnDataSource, - DataTable, - TableColumn, - Div, - StringFormatter, - NumberFormatter, - HTMLTemplateFormatter, - InlineStyleSheet, - ) - - dest.parent.mkdir(parents=True, exist_ok=True) - - # get the overview dataframe and prettify it - df = get_overview_df() - df["possible_discontinuities"] = df["possible_discontinuities"].apply( - lambda x: "✓" if x else "" - ) - df.fillna({"objective_prior_distributions": ""}, inplace=True) - - def get_formatter(col: str): - """Get the appropriate formatter for the column.""" - if col not in df.columns: - # index column - return HTMLTemplateFormatter( - template=f""" - <%= value %> - """ - ) - if pd.api.types.is_integer_dtype(df[col].dtype): - return NumberFormatter(text_align="right") - if col in ("reference_uris", "sbml4humans_urls"): - icon = "📚" if col == "reference_uris" else "🌠" - return HTMLTemplateFormatter( - template=""" - <% - if (Array.isArray(value)) { - urls = value; - } else { - const sanitizedValue = value.replace(/'/g, '"'); - const urls = JSON.parse(sanitizedValue); - console.log('Parsed JSON:', urls); - } - for (let i = 0; i < urls.length; i++) { - %> - LINK_TEXT - <% } %> - """.replace("LINK_TEXT", icon) + rows = [] + for petab_problem_id in MODELS: + problem_dir = f"{PROBLEMS_DIRNAME}/{petab_problem_id}" + row = {"ID": f"[{petab_problem_id}]({problem_dir}/)"} + for label, version, dirname in ( + ("PEtab v1", v1, V1_DIRNAME), + ("PEtab v2", v2, V2_DIRNAME), + ): + row[label] = ( + f"[{dirname}]({problem_dir}/{dirname}/)" + if petab_problem_id in version.MODELS + else "" ) + row["Description"] = get_metadata(petab_problem_id).short_description + rows.append(row) + return pd.DataFrame(rows).to_markdown(index=False) - return StringFormatter() - - columns = [ - TableColumn( - field=col, - title=markdown_columns.get(col, col), - formatter=get_formatter(col), - width=len(col), - ) - for col in df.reset_index().columns - ] - source = ColumnDataSource(df) +def show_overview_table( + markdown: bool = False, + update_readme: bool = False, +): + """Print the overview table or write the README table. - css = InlineStyleSheet( - css=""" - .slick-header-column { - background-color: #f4f4f4; - font-weight: bold; - } + :param markdown: Print the README table instead of the v1 statistics. + :param update_readme: Write the README table to `README.md`. """ - ) - - data_table = DataTable( - source=source, - columns=columns, - sortable=True, - sizing_mode="stretch_both", - ) - data_table.stylesheets.append(css) - - heading = Div(text="

Benchmark Problems

") - preamble = Div( - text=f""" -

- This table provides an overview of the benchmark problems - available in the Benchmark-Models-PEtab - repository. -

- """ - ) - layout = column(heading, preamble, data_table, sizing_mode="stretch_both") - output_file(dest, title="Benchmark Problems") - save(layout) + if update_readme: + with open(readme_md) as f: + readme_content = f.read() + before_table = readme_content.split(start_overview_table)[0] + after_table = readme_content.split(end_overview_table)[1] + new_readme_content = ( + before_table + + start_overview_table + + get_readme_table() + + end_overview_table + + after_table + ) + with open(readme_md, "w") as f: + f.write(new_readme_content) + elif markdown: + print(get_readme_table()) + else: + pd.options.display.width = 0 + print(get_overview_df()) def main(): @@ -439,7 +405,9 @@ def main(): ) group1 = parser.add_mutually_exclusive_group() group1.add_argument( - "--markdown", action="store_true", help="Output in markdown format" + "--markdown", + action="store_true", + help="Print the README overview table (markdown)", ) group1.add_argument( "--update", @@ -449,12 +417,15 @@ def main(): group2 = parser.add_mutually_exclusive_group() group2.add_argument( - "--html-file", help="Output the overview table to an HTML file" + "--html-file", + help="Write the v1 and v2 overview tables to an HTML file", ) args = parser.parse_args() if args.html_file: - create_html_table(dest=Path(args.html_file)) + from ._site import create_overview_page + + create_overview_page(dest=Path(args.html_file)) else: show_overview_table(markdown=args.markdown, update_readme=args.update) diff --git a/src/python/benchmark_models_petab/overview_v2.py b/src/python/benchmark_models_petab/overview_v2.py new file mode 100644 index 0000000..29ffb36 --- /dev/null +++ b/src/python/benchmark_models_petab/overview_v2.py @@ -0,0 +1,164 @@ +"""Stats for the PEtab v2 encodings of the benchmark problems.""" + +from pathlib import Path + +import libsbml +import numpy as np +import pandas as pd +import petab.v2 as petab + +from . import v2 +from .C import V2_DIRNAME +from .overview import column_descriptions as v1_column_descriptions +from .overview import columns as v1_columns +from .overview import ( + get_reference_uris, + index_column, + make_sbml4humans_urls, + sbml_model_has_discontinuities, +) + +_labels = v1_columns | { + "experiments": "Experiments", + "preequilibration": "Experiments with preequilibration", + "prior_distributions": "Prior distribution(s)", +} + +#: Column labels for the overview table, in column order. +columns = { + col: _labels[col] + for col in ( + "petab_problem_id", + "experiments", + "preequilibration", + "postequilibration", + "estimated_parameters", + "measurements", + "observables", + "species", + "events", + "possible_discontinuities", + "noise_distributions", + "prior_distributions", + "reference_uris", + "sbml4humans_urls", + ) +} + +#: Column descriptions for the overview table. +column_descriptions = v1_column_descriptions | { + "experiments": ( + "Number of experiments in the measurement table. Measurements " + "without an experiment count as one." + ), + "preequilibration": ( + "Number of experiments with a preequilibration period." + ), + "postequilibration": ( + "Number of experiments with steady-state measurements (time = inf)." + ), + "noise_distributions": "Noise distributions of the observables.", + "prior_distributions": "Prior distributions of estimated parameters.", + "possible_discontinuities": ( + "Heuristic: events with assignments, discontinuous functions " + "(piecewise, min, max, ...) with time-dependent arguments in the " + "model, or condition changes after the start of an experiment." + ), +} + + +def get_summary(problem: petab.Problem, petab_problem_id: str) -> dict: + """Get dictionary with stats for the given PEtab v2 problem. + + Measurements without an experiment ID count as one experiment. + Preequilibration and postequilibration are counted as the number of + experiments with a preequilibration period (starting at ``-inf``) or + with steady-state measurements (at ``inf``), respectively. + For problems with multiple models, species and events are the maximum + over the models, and references are collected from all models. + """ + sbml_models = get_sbml_models(problem) + experiments = {e.id: e for e in problem.experiments} + measured_experiment_ids = {m.experiment_id for m in problem.measurements} + + return { + "petab_problem_id": petab_problem_id, + "experiments": len(measured_experiment_ids), + "preequilibration": sum( + experiments[exp_id].has_preequilibration + for exp_id in measured_experiment_ids + if exp_id is not None + ), + "postequilibration": len( + {m.experiment_id for m in problem.measurements if m.time == np.inf} + ), + "estimated_parameters": problem.n_estimated, + "measurements": len(problem.measurements), + "observables": len({m.observable_id for m in problem.measurements}), + "species": max(m.getNumSpecies() for m in sbml_models), + "events": max(m.getNumEvents() for m in sbml_models), + "possible_discontinuities": ( + any(sbml_model_has_discontinuities(m) for m in sbml_models) + or any( + has_period_changes(experiments[exp_id]) + for exp_id in measured_experiment_ids + if exp_id is not None + ) + ), + "noise_distributions": "; ".join( + sorted({str(o.noise_distribution) for o in problem.observables}) + ), + "prior_distributions": "; ".join( + sorted( + { + str(p.prior_distribution) + for p in problem.parameters + if p.estimate and p.prior_distribution is not None + } + ) + ), + "reference_uris": list( + dict.fromkeys( + uri for m in sbml_models for uri in get_reference_uris(m) + ) + ), + "sbml4humans_urls": make_sbml4humans_urls( + petab_problem_id, + V2_DIRNAME, + [Path(m.rel_path).as_posix() for m in problem.models], + ), + } + + +def get_sbml_models(problem: petab.Problem) -> list[libsbml.Model]: + """Get the SBML models of the problem. + + :raises ValueError: If the problem has no model or a non-SBML model. + """ + if not problem.models: + raise ValueError("The problem has no model.") + if non_sbml := [m.model_id for m in problem.models if m.type_id != "sbml"]: + raise ValueError(f"Only SBML models are supported, not: {non_sbml}") + return [m.sbml_model for m in problem.models] + + +def has_period_changes(experiment: petab.Experiment) -> bool: + """Check whether the experiment changes conditions after its start. + + Preequilibration periods are not considered. + """ + return ( + sum(not period.is_preequilibration for period in experiment.periods) + > 1 + ) + + +def get_overview_df() -> pd.DataFrame: + """Get overview table with stats for all PEtab v2 benchmark problems.""" + data = [ + get_summary(v2.get_problem(petab_problem_id), petab_problem_id) + for petab_problem_id in v2.MODELS + ] + df = pd.DataFrame(data, columns=list(columns)) + df.set_index([index_column], inplace=True) + return df diff --git a/src/python/benchmark_models_petab/v1.py b/src/python/benchmark_models_petab/v1.py index b3f6679..a1929ab 100644 --- a/src/python/benchmark_models_petab/v1.py +++ b/src/python/benchmark_models_petab/v1.py @@ -4,10 +4,23 @@ import petab.v1 as petab -from .C import MODELS_DIR, PROBLEM_FILENAME, SIMULATIONS_FILENAME, V1_DIRNAME - import pandas as pd +from .C import ( + MODELS as ALL_MODELS, + MODELS_DIR, + PROBLEM_FILENAME, + SIMULATIONS_FILENAME, + V1_DIRNAME, +) + +#: IDs of the problems that have a PEtab v1 encoding. +MODELS: list[str] = [ + id_ + for id_ in ALL_MODELS + if Path(MODELS_DIR, id_, V1_DIRNAME, PROBLEM_FILENAME).is_file() +] + def get_problem_yaml_path(id_: str) -> Path: """Get the path to the PEtab problem YAML file. diff --git a/src/python/benchmark_models_petab/v2.py b/src/python/benchmark_models_petab/v2.py index 3a459ef..d8fff25 100644 --- a/src/python/benchmark_models_petab/v2.py +++ b/src/python/benchmark_models_petab/v2.py @@ -5,7 +5,20 @@ import pandas as pd import petab.v2 as petab -from .C import MODELS_DIR, PROBLEM_FILENAME, SIMULATIONS_FILENAME, V2_DIRNAME +from .C import ( + MODELS as ALL_MODELS, + MODELS_DIR, + PROBLEM_FILENAME, + SIMULATIONS_FILENAME, + V2_DIRNAME, +) + +#: IDs of the problems that have a PEtab v2 encoding. +MODELS: list[str] = [ + id_ + for id_ in ALL_MODELS + if Path(MODELS_DIR, id_, V2_DIRNAME, PROBLEM_FILENAME).is_file() +] def get_problem_yaml_path(id_: str) -> Path: diff --git a/src/python/pyproject.toml b/src/python/pyproject.toml index 5b1a3a0..a90a937 100644 --- a/src/python/pyproject.toml +++ b/src/python/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ [project.optional-dependencies] dev = ["pre-commit", "pytest", "ruff"] -site = ["bokeh>=3.7.3"] +site = ["jinja2"] [project.scripts] bmp-petablint = "benchmark_models_petab.check_petablint:main" diff --git a/src/python/test/test_overview.py b/src/python/test/test_overview.py new file mode 100644 index 0000000..1269427 --- /dev/null +++ b/src/python/test/test_overview.py @@ -0,0 +1,18 @@ +"""Tests for the README overview table.""" + +import benchmark_models_petab as models +from benchmark_models_petab.overview import get_readme_table + +PROBLEM_ID = "Boehm_JProteomeRes2014" + + +def test_get_readme_table(): + table = get_readme_table() + (row,) = [line for line in table.splitlines() if f"[{PROBLEM_ID}]" in line] + assert models.get_metadata(PROBLEM_ID).short_description in row + assert f"[v1](problems/{PROBLEM_ID}/v1/)" in row + assert f"[v2](problems/{PROBLEM_ID}/v2/)" in row + # a problem without a v2 encoding + no_v2 = next(id_ for id_ in models.MODELS if id_ not in models.v2.MODELS) + (row,) = [line for line in table.splitlines() if f"[{no_v2}]" in line] + assert "/v2/" not in row diff --git a/src/python/test/test_overview_v2.py b/src/python/test/test_overview_v2.py new file mode 100644 index 0000000..d1d12b6 --- /dev/null +++ b/src/python/test/test_overview_v2.py @@ -0,0 +1,115 @@ +"""Tests for the v2 overview table and the site's overview page data.""" + +import numpy as np +import pandas as pd +import petab.v2 as petab + +import benchmark_models_petab as models +from benchmark_models_petab import overview, overview_v2 +from benchmark_models_petab._site.build import ( + column_group, + get_summary_tiles, + get_table, +) + +PROBLEM_ID = "Boehm_JProteomeRes2014" + + +def test_get_summary(): + summary = overview_v2.get_summary( + models.v2.get_problem(PROBLEM_ID), PROBLEM_ID + ) + assert set(summary) == set(overview_v2.columns) + assert summary["experiments"] == 1 + assert summary["estimated_parameters"] == 9 + assert summary["measurements"] == 48 + assert summary["observables"] == 3 + assert summary["noise_distributions"] == "normal" + assert summary["prior_distributions"] == "" + assert summary["possible_discontinuities"] is False + assert summary["sbml4humans_urls"][0].endswith( + f"/{PROBLEM_ID}/v2/model.xml" + ) + + +def test_get_table(): + table = get_table( + overview_v2.get_overview_df(), + overview_v2.columns, + overview_v2.column_descriptions, + "v2", + ) + kinds = {col["field"]: col["kind"] for col in table["columns"]} + assert kinds["petab_problem_id"] == "id" + assert kinds["short_description"] == "description" + assert kinds["measurements"] == "int" + assert kinds["possible_discontinuities"] == "bool" + assert kinds["reference_uris"] == "links" + assert kinds["noise_distributions"] == "tags" + + # columns of a group are contiguous, ungrouped columns come first + groups = [col["group"] for col in table["columns"]] + assert groups[:2] == [None, None] + assert None not in groups[2:] + for group in set(groups) - {None}: + idx = [i for i, g in enumerate(groups) if g == group] + assert idx == list(range(idx[0], idx[-1] + 1)) + assert all(col["tooltip"] for col in table["columns"]) + + (row,) = [r for r in table["rows"] if r["petab_problem_id"] == PROBLEM_ID] + assert ( + row["short_description"] + == models.get_metadata(PROBLEM_ID).short_description + ) + assert row["measurements"] == 48 + assert table["tree_url"].endswith("/problems/{id}/v2") + + +def test_has_period_changes(): + def experiment(*times): + return petab.Experiment( + id="e", + periods=[petab.ExperimentPeriod(time=t) for t in times], + ) + + assert not overview_v2.has_period_changes(experiment(0)) + assert not overview_v2.has_period_changes(experiment(-np.inf, 0)) + assert overview_v2.has_period_changes(experiment(0, 10)) + assert overview_v2.has_period_changes(experiment(-np.inf, 0, 10)) + + +def test_get_summary_tiles(): + v1_df = pd.DataFrame( + { + "estimated_parameters": [3, 1200], + "measurements": [10, 27132], + "conditions": [1, 9169], + "observables": [1, 4], + }, + index=["a", "b"], + ) + # "b" is ignored, as it has a v1 encoding; "c" extends the ranges + v2_df = pd.DataFrame( + { + "estimated_parameters": [1, 2], + "measurements": [1, 30000], + "experiments": [9170, 1], + "observables": [5, 5], + }, + index=["b", "c"], + ) + tiles = { + label: value for value, label, _ in get_summary_tiles(v1_df, v2_df) + } + assert tiles["problems in total"] == str(len(models.MODELS)) + assert tiles["in PEtab v1"] == str(len(models.v1.MODELS)) + assert tiles["in PEtab v2"] == str(len(models.v2.MODELS)) + assert tiles["measurements"] == "10\u201330,000" + assert tiles["simulation conditions"] == "1\u20139,169" + assert tiles["estimated parameters"] == "2\u20131,200" + assert tiles["observables"] == "1\u20135" + + +def test_all_columns_grouped(): + for labels in (overview.columns, overview_v2.columns): + assert set(labels) - set(column_group) == {"petab_problem_id"} diff --git a/src/python/test/test_v1.py b/src/python/test/test_v1.py index 42aedfe..dbf00a8 100644 --- a/src/python/test/test_v1.py +++ b/src/python/test/test_v1.py @@ -26,3 +26,8 @@ def test_v1_submodule_matches_top_level(): assert models.get_problem is models.v1.get_problem assert models.get_problem_yaml_path is models.v1.get_problem_yaml_path assert models.get_simulation_df is models.v1.get_simulation_df + + +def test_models(): + assert "Boehm_JProteomeRes2014" in models.v1.MODELS + assert set(models.v1.MODELS) <= set(models.MODELS) diff --git a/src/python/test/test_v2.py b/src/python/test/test_v2.py index 1a2d6e8..c01d519 100644 --- a/src/python/test/test_v2.py +++ b/src/python/test/test_v2.py @@ -31,3 +31,8 @@ def test_get_problem(): assert "log10_k_phos" in problem.parameter_df.index assert "k_phos" not in problem.parameter_df.index assert len(problem.measurement_df) > 0 + + +def test_models(): + assert "Boehm_JProteomeRes2014" in models.v2.MODELS + assert set(models.v2.MODELS) <= set(models.MODELS)