From 5cf88358dccad7312c9be94a6367b729aa223440 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Sat, 17 Oct 2026 20:52:55 +0000 Subject: [PATCH 01/10] TEMP: [CI] Run only mac-beta, only cppjit tests, 300 copies of test_datatypes Measurement commit for the sporadic crash of test_datatypes on the mac-beta ARM64 runner: disable the Windows and Linux jobs, drop the other macOS platforms from the matrix, restrict ctest on mac-beta to the cppjit tests, and register 300 extra copies of test_datatypes (-rep1 to -rep300) that share the dictionary fixture. ctest runs them 8 at a time in the common test directory, like the real suite runs the cppjit tests, instead of the sequential and otherwise idle repetitions --repeat gives. The crash rate measured over 201 PR jobs and 13 nightlies is about 3%, so 300 clean runs without the fix have a probability below one in a thousand, while 300 runs with it should show no failure. About 6 seconds per copy, so a few minutes of test time. Not to be merged. --- .../root-ci-config/buildconfig/mac-beta.txt | 1 + .github/workflows/root-ci.yml | 21 ++++--------------- bindings/pyroot/cppjit/test/CMakeLists.txt | 11 ++++++++-- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/.github/workflows/root-ci-config/buildconfig/mac-beta.txt b/.github/workflows/root-ci-config/buildconfig/mac-beta.txt index f7de3ebcdb542..c4e6c9003106e 100644 --- a/.github/workflows/root-ci-config/buildconfig/mac-beta.txt +++ b/.github/workflows/root-ci-config/buildconfig/mac-beta.txt @@ -22,3 +22,4 @@ cocoa=ON davix=OFF minuit2_omp=OFF x11=OFF +ROOT_CTEST_CUSTOM_FLAGS=-R ^pyunittests-bindings-cppjit- diff --git a/.github/workflows/root-ci.yml b/.github/workflows/root-ci.yml index 0af636192a117..9477dde216f5a 100644 --- a/.github/workflows/root-ci.yml +++ b/.github/workflows/root-ci.yml @@ -120,11 +120,10 @@ jobs: matrix: # Specify platform + optional build option overrides # Build options: https://root.cern/install/build_from_source/#all-build-options - platform: [mac26, mac27, mac-beta] + # TEMP: only mac-beta, to measure the sporadic cppjit test_datatypes crash + platform: [mac-beta] arch: [ARM64] include: - - platform: mac15 - arch: X86 - platform: mac-beta is_special: true @@ -246,13 +245,7 @@ jobs: build-windows: # For any event that is not a PR, the CI will always run. In PRs, the CI # can be skipped if the tag [skip-ci] or [skip ci] is written in the title. - if: | - (github.repository_owner == 'root-project' && github.event_name != 'pull_request') || - (github.event_name == 'pull_request' && !( - contains(github.event.pull_request.title, '[skip-ci]') || - contains(github.event.pull_request.title, '[skip ci]') || - contains(github.event.pull_request.labels.*.name, 'skip ci') - )) + if: false # TEMP: only mac-beta runs, see build-macos permissions: contents: read @@ -372,13 +365,7 @@ jobs: build-linux: # For any event that is not a PR, the CI will always run. In PRs, the CI # can be skipped if the tag [skip-ci] or [skip ci] is written in the title. - if: | - (github.repository_owner == 'root-project' && github.event_name != 'pull_request') || - (github.event_name == 'pull_request' && !( - contains(github.event.pull_request.title, '[skip-ci]') || - contains(github.event.pull_request.title, '[skip ci]') || - contains(github.event.pull_request.labels.*.name, 'skip ci') - )) + if: false # TEMP: only mac-beta runs, see build-macos permissions: contents: read diff --git a/bindings/pyroot/cppjit/test/CMakeLists.txt b/bindings/pyroot/cppjit/test/CMakeLists.txt index 482c2abb89102..84bf4d10be550 100644 --- a/bindings/pyroot/cppjit/test/CMakeLists.txt +++ b/bindings/pyroot/cppjit/test/CMakeLists.txt @@ -37,7 +37,7 @@ endif() function(CPPJIT_ADD_PYTEST name) cmake_parse_arguments(ARG - "DROP_ROOTMAP" "" "DICTS;FIXTURES_SETUP;FIXTURES_REQUIRED" ${ARGN}) + "DROP_ROOTMAP" "SUFFIX" "DICTS;FIXTURES_SETUP;FIXTURES_REQUIRED" ${ARGN}) set(extra) if(ARG_DICTS) @@ -60,7 +60,7 @@ function(CPPJIT_ADD_PYTEST name) endif() string(REGEX REPLACE "[_]" "-" good_name "${name}") - set(test_name pyunittests-bindings-cppjit-${good_name}) + set(test_name pyunittests-bindings-cppjit-${good_name}${ARG_SUFFIX}) ROOT_ADD_TEST(${test_name} # Upstream marks platform xfails without explicit strict=; the in-tree @@ -115,6 +115,13 @@ CPPJIT_ADD_PYTEST(test_basic_api FIXTURES_REQUIRED cppjit_example01_dict_fixture) CPPJIT_ADD_PYTEST(test_datatypes FIXTURES_REQUIRED cppjit_datatypes_dict_fixture) +# TEMP: 300 more copies of test_datatypes, run by ctest in parallel in this +# directory like the real suite does, to measure its sporadic crash on the +# mac-beta CI runner (about 3% of runs). +foreach(i RANGE 1 300) + CPPJIT_ADD_PYTEST(test_datatypes SUFFIX -rep${i} + FIXTURES_REQUIRED cppjit_datatypes_dict_fixture) +endforeach() CPPJIT_ADD_PYTEST(test_api) CPPJIT_ADD_PYTEST(test_boost) From 076640457478b50519adaa2e0c3e78748525ccb3 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Fri, 16 Oct 2026 13:18:15 +0000 Subject: [PATCH 02/10] [cppjit] Collect crash diagnostics for test_datatypes in CI pyunittests-bindings-cppjit-test-datatypes crashes in roughly one of six runs on the mac-beta ARM64 runner, and nowhere else: 2 of 13 nightlies and 4 of 25 PR jobs since the test was introduced on 2026-10-03. Every instance is a segmentation violation or bus error in Python's cyclic garbage collector (gc_collect_main -> list_traverse -> visit_decref) during test26_function_pointers. That is the signature of a Python object that was freed, or whose memory was corrupted, while a container still referred to it, and the damage can come from any earlier test since the collection that trips over it only runs when the allocation count crosses the GC threshold. The crash does not reproduce on Linux, not even with the debug allocator and a full collection forced after every Python line, and the local build uses the free-threaded interpreter whose GC differs from the mac's default build. So instead of guessing, make the next CI failure informative: - Run the test with PYTHONMALLOC=debug, so a dangling reference fails loudly (freed blocks are filled with 0xDD, guard bytes are checked, a double free or overrun reports the block) instead of corrupting whatever reused the memory. - Load a small pytest plugin, root_ci_diagnostics.py, that performs a full gc.collect() after every test, which moves the crash into the teardown of the test that leaves the damage behind, and re-arms Python's faulthandler before and after each test. ROOT's signal handlers replace the one pytest installs at session start and print a C stack without symbols on the macOS runners; with faulthandler back in place the report carries the Python traceback and, on Python 3.14+, a symbolized C stack. The plugin is loaded only for test files registered with the new CRASH_DIAGNOSTICS option of CPPJIT_ADD_PYTEST, currently test_datatypes alone, and lives next to the CMake file rather than in the vendored conftest.py so that syncing cppjit from upstream does not drop it. It can be removed once the crash is understood. Verified locally: the test passes with the plugin active; an injected segfault yields the faulthandler report instead of ROOT's; on a default-build Python 3.14.7, an injected over-decref of a list element crashes in the teardown of the injecting test with a "Garbage-collecting" marker. --- bindings/pyroot/cppjit/test/CMakeLists.txt | 25 ++++-- .../pyroot/cppjit/test/root_ci_diagnostics.py | 79 +++++++++++++++++++ 2 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 bindings/pyroot/cppjit/test/root_ci_diagnostics.py diff --git a/bindings/pyroot/cppjit/test/CMakeLists.txt b/bindings/pyroot/cppjit/test/CMakeLists.txt index 84bf4d10be550..f013e87560ce1 100644 --- a/bindings/pyroot/cppjit/test/CMakeLists.txt +++ b/bindings/pyroot/cppjit/test/CMakeLists.txt @@ -9,7 +9,8 @@ # ACLiC into the test working directory (the suite's own Makefile is disabled # via CPPJIT_TEST_SKIP_MAKE). -set(pythonpaths ${localruntimedir} $ENV{PYTHONPATH}) +# The test directory is on the path for the root_ci_diagnostics pytest plugin. +set(pythonpaths ${localruntimedir} ${CMAKE_CURRENT_SOURCE_DIR} $ENV{PYTHONPATH}) cmake_path(CONVERT "${pythonpaths}" TO_NATIVE_PATH_LIST pythonpaths_native) set(common_env PYTHONPATH=${pythonpaths_native} CPPJIT_TEST_SKIP_MAKE=1) @@ -35,11 +36,21 @@ elseif(MSVC) -e "gSystem->AddLinkedLibs(\"${Python3_LIBRARIES}\")") endif() +# CRASH_DIAGNOSTICS loads root_ci_diagnostics.py (next to this file) as a +# pytest plugin and runs the test under Python's debug allocator hooks, to get +# an actionable report out of a crash that only happens in CI. See the plugin's +# docstring for what it does and why. function(CPPJIT_ADD_PYTEST name) cmake_parse_arguments(ARG - "DROP_ROOTMAP" "SUFFIX" "DICTS;FIXTURES_SETUP;FIXTURES_REQUIRED" ${ARGN}) + "DROP_ROOTMAP;CRASH_DIAGNOSTICS" "SUFFIX" "DICTS;FIXTURES_SETUP;FIXTURES_REQUIRED" ${ARGN}) set(extra) + set(env ${common_env}) + set(pytest_args) + if(ARG_CRASH_DIAGNOSTICS) + list(APPEND pytest_args -p root_ci_diagnostics) + list(APPEND env PYTHONMALLOC=debug) + endif() if(ARG_DICTS) set(copy_files) set(precmd_args) @@ -67,10 +78,10 @@ function(CPPJIT_ADD_PYTEST name) # patches fix many of them on the ROOT stack, so an xpass must not fail # the suite (pyproject.toml sets xfail_strict=true for upstream CI). COMMAND ${Python3_EXECUTABLE} -B -m pytest -v -p no:cacheprovider - -o xfail_strict=false + -o xfail_strict=false ${pytest_args} ${CMAKE_CURRENT_SOURCE_DIR}/${name}.py WORKING_DIR ${CMAKE_CURRENT_BINARY_DIR} - ENVIRONMENT ${common_env} + ENVIRONMENT ${env} LABELS python PYTHON_DEPS pytest ${extra}) @@ -113,13 +124,15 @@ CPPJIT_ADD_PYTEST(test_aclassloader FIXTURES_REQUIRED cppjit_example01_dict_fixture) CPPJIT_ADD_PYTEST(test_basic_api FIXTURES_REQUIRED cppjit_example01_dict_fixture) -CPPJIT_ADD_PYTEST(test_datatypes +# Sporadic crash in the garbage collector during test26_function_pointers on +# the mac-beta ARM64 CI runner; collect diagnostics until it is understood. +CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS FIXTURES_REQUIRED cppjit_datatypes_dict_fixture) # TEMP: 300 more copies of test_datatypes, run by ctest in parallel in this # directory like the real suite does, to measure its sporadic crash on the # mac-beta CI runner (about 3% of runs). foreach(i RANGE 1 300) - CPPJIT_ADD_PYTEST(test_datatypes SUFFIX -rep${i} + CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS SUFFIX -rep${i} FIXTURES_REQUIRED cppjit_datatypes_dict_fixture) endforeach() diff --git a/bindings/pyroot/cppjit/test/root_ci_diagnostics.py b/bindings/pyroot/cppjit/test/root_ci_diagnostics.py new file mode 100644 index 0000000000000..e26b6cfebd1f9 --- /dev/null +++ b/bindings/pyroot/cppjit/test/root_ci_diagnostics.py @@ -0,0 +1,79 @@ +"""Crash diagnostics for cppjit tests that fail sporadically in ROOT's CI. + +Not part of upstream cppjit: ROOT's CMakeLists.txt loads this as a pytest +plugin (``-p root_ci_diagnostics``) for selected test files, together with +``PYTHONMALLOC=debug``. It is meant to turn an unexplained crash into +an actionable report and can be dropped once the crash is understood. + +Context: pyunittests-bindings-cppjit-test-datatypes dies in about one of six +runs on the mac-beta ARM64 runner (and nowhere else) with a segmentation +violation or bus error inside Python's cyclic garbage collector while it +traverses a list, during test26_function_pointers. That is the signature of a +Python object that was freed, or whose memory was corrupted, while a container +still referenced it. The damage can originate in any earlier test, because the +collection that trips over it only runs when the allocation count crosses the +GC threshold. + +Three measures: + +* ``PYTHONMALLOC=debug`` (set by CMake, reported in the header) keeps the + build's default allocator but adds the debug hooks: freed blocks are filled + with 0xDD, the guard bytes around every block are checked, and a double free + or an overrun reports the offending block, so a dangling reference fails + loudly and recognizably instead of corrupting whatever reused the memory. +* A full ``gc.collect()`` after every test moves the crash into the teardown + of the test that actually leaves the damage behind. +* Python's faulthandler is re-armed before and after every test. ROOT installs + its own signal handlers when the interpreter is initialized, which replaces + the handler pytest's faulthandler plugin set up at session start; ROOT's + handler prints a C stack without symbols on the macOS runners. Re-arming + restores the Python traceback (and, with Python 3.14+, a C stack with + symbols) at the point of the crash. +""" + +import faulthandler +import gc +import os +import sys + +_stderr_fd = None + + +def _rearm_faulthandler(): + # enable() is a no-op while already enabled, so cycle it to take the + # signal handlers back from ROOT. + faulthandler.disable() + faulthandler.enable(file=_stderr_fd, all_threads=True) + + +def pytest_configure(config): + # Capture starts after configuration, so fd 2 is still the real stderr. + global _stderr_fd + _stderr_fd = os.dup(2) + _rearm_faulthandler() + + +def pytest_report_header(config): + return ( + "ROOT CI diagnostics: faulthandler re-armed and gc.collect() after " + f"each test, PYTHONMALLOC={os.environ.get('PYTHONMALLOC', '')}" + ) + + +def pytest_runtest_setup(item): + _rearm_faulthandler() + + +def pytest_runtest_teardown(item, nextitem): + _rearm_faulthandler() + gc.collect() + _rearm_faulthandler() + + +def pytest_unconfigure(config): + global _stderr_fd + if _stderr_fd is not None: + faulthandler.disable() + faulthandler.enable(file=sys.__stderr__, all_threads=True) + os.close(_stderr_fd) + _stderr_fd = None From 7a41fe70080529007e7348c314229940242cc768 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Sun, 18 Oct 2026 07:16:52 +0000 Subject: [PATCH 03/10] TEMP: [cppjit] Collect after every line around test26, 600 copies Sharpen the crash diagnostics for the measurement runs on mac-beta: the plugin now also forces a full gc.collect() after every line of test24 to test29 of test_datatypes, so a corrupted container is found at the statement that leaves it behind rather than at the next threshold-triggered collection, and reports how many collections it forced. The number of test_datatypes copies goes from 300 to 600: the crash rate in this setup is about 0.5%, so this yields a few reports per run (or, if the debug allocator and the forced collections make the damage deterministic, a few hundred). Not to be merged. --- bindings/pyroot/cppjit/test/CMakeLists.txt | 4 +-- .../pyroot/cppjit/test/root_ci_diagnostics.py | 32 ++++++++++++++++++- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/bindings/pyroot/cppjit/test/CMakeLists.txt b/bindings/pyroot/cppjit/test/CMakeLists.txt index f013e87560ce1..86d19b0e999de 100644 --- a/bindings/pyroot/cppjit/test/CMakeLists.txt +++ b/bindings/pyroot/cppjit/test/CMakeLists.txt @@ -128,10 +128,10 @@ CPPJIT_ADD_PYTEST(test_basic_api # the mac-beta ARM64 CI runner; collect diagnostics until it is understood. CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS FIXTURES_REQUIRED cppjit_datatypes_dict_fixture) -# TEMP: 300 more copies of test_datatypes, run by ctest in parallel in this +# TEMP: 600 more copies of test_datatypes, run by ctest in parallel in this # directory like the real suite does, to measure its sporadic crash on the # mac-beta CI runner (about 3% of runs). -foreach(i RANGE 1 300) +foreach(i RANGE 1 600) CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS SUFFIX -rep${i} FIXTURES_REQUIRED cppjit_datatypes_dict_fixture) endforeach() diff --git a/bindings/pyroot/cppjit/test/root_ci_diagnostics.py b/bindings/pyroot/cppjit/test/root_ci_diagnostics.py index e26b6cfebd1f9..92ed159237b3e 100644 --- a/bindings/pyroot/cppjit/test/root_ci_diagnostics.py +++ b/bindings/pyroot/cppjit/test/root_ci_diagnostics.py @@ -38,6 +38,27 @@ _stderr_fd = None +# The crash always surfaces in test26_function_pointers, in a collection that +# an allocation happens to trigger; collecting after every line of the tests +# around it pins the damage to the statement that first leaves a corrupted +# container behind, instead of wherever the GC threshold is crossed. +_TRACE_FILE = "test_datatypes.py" +_TRACE_TESTS = ("test24_", "test25_", "test26_", "test27_", "test28_", "test29_") +_forced_collections = 0 + + +def _line_tracer(frame, event, arg): + code = frame.f_code + if not code.co_filename.endswith(_TRACE_FILE) or not code.co_name.startswith( + _TRACE_TESTS + ): + return None + if event == "line": + global _forced_collections + gc.collect() + _forced_collections += 1 + return _line_tracer + def _rearm_faulthandler(): # enable() is a no-op while already enabled, so cycle it to take the @@ -51,12 +72,20 @@ def pytest_configure(config): global _stderr_fd _stderr_fd = os.dup(2) _rearm_faulthandler() + sys.settrace(_line_tracer) def pytest_report_header(config): return ( "ROOT CI diagnostics: faulthandler re-armed and gc.collect() after " - f"each test, PYTHONMALLOC={os.environ.get('PYTHONMALLOC', '')}" + f"each test and after every line of {_TRACE_FILE} {_TRACE_TESTS}, " + f"PYTHONMALLOC={os.environ.get('PYTHONMALLOC', '')}" + ) + + +def pytest_terminal_summary(terminalreporter): + terminalreporter.write_line( + f"ROOT CI diagnostics: {_forced_collections} collections forced by the line tracer" ) @@ -72,6 +101,7 @@ def pytest_runtest_teardown(item, nextitem): def pytest_unconfigure(config): global _stderr_fd + sys.settrace(None) if _stderr_fd is not None: faulthandler.disable() faulthandler.enable(file=sys.__stderr__, all_threads=True) From a3e062395ae133e9c72c5f53a10aa9832c572082 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Sun, 18 Oct 2026 08:11:20 +0000 Subject: [PATCH 04/10] TEMP: [cppjit] Measure with macOS malloc debugging and with test26 alone The 600 copies under PYTHONMALLOC=debug and forced collections all passed, where the production configuration fails about 0.5% of the time, so the Python debug allocator changes the memory layout enough to hide the bug. That points away from a plain dangling reference to a Python object and towards the C++ heap, or a stale pointer whose effect depends on where a block lands. Replace the 600 copies by two groups of 300 that keep Python's production allocator and no forced collections (new ROOT_CI_DIAGNOSTICS_NO_GC switch in the plugin, which then only re-arms the faulthandler): - -malloc: MallocScribble, MallocPreScribble and MallocGuardEdges, macOS's debugging for the system allocator, which Python's small objects do not use. A use of freed or uninitialized C++ memory then reads 0x55/0xAA patterns and crashes at the culprit instead of corrupting a neighbour. - -only26: pytest -k test26_function_pointers, to learn whether the test that always notices the damage also causes it. CPPJIT_ADD_PYTEST gains ENV and PYTEST_ARGS for these per-copy overrides; the test driver applies environment entries in order, so the overrides win. Not to be merged. --- bindings/pyroot/cppjit/test/CMakeLists.txt | 28 +++++++++++++++---- .../pyroot/cppjit/test/root_ci_diagnostics.py | 22 +++++++++++---- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/bindings/pyroot/cppjit/test/CMakeLists.txt b/bindings/pyroot/cppjit/test/CMakeLists.txt index 86d19b0e999de..f460e565527ed 100644 --- a/bindings/pyroot/cppjit/test/CMakeLists.txt +++ b/bindings/pyroot/cppjit/test/CMakeLists.txt @@ -42,7 +42,8 @@ endif() # docstring for what it does and why. function(CPPJIT_ADD_PYTEST name) cmake_parse_arguments(ARG - "DROP_ROOTMAP;CRASH_DIAGNOSTICS" "SUFFIX" "DICTS;FIXTURES_SETUP;FIXTURES_REQUIRED" ${ARGN}) + "DROP_ROOTMAP;CRASH_DIAGNOSTICS" "SUFFIX" + "DICTS;FIXTURES_SETUP;FIXTURES_REQUIRED;ENV;PYTEST_ARGS" ${ARGN}) set(extra) set(env ${common_env}) @@ -51,6 +52,9 @@ function(CPPJIT_ADD_PYTEST name) list(APPEND pytest_args -p root_ci_diagnostics) list(APPEND env PYTHONMALLOC=debug) endif() + # TEMP: per-test overrides; later entries win in the test driver + list(APPEND env ${ARG_ENV}) + list(APPEND pytest_args ${ARG_PYTEST_ARGS}) if(ARG_DICTS) set(copy_files) set(precmd_args) @@ -128,11 +132,23 @@ CPPJIT_ADD_PYTEST(test_basic_api # the mac-beta ARM64 CI runner; collect diagnostics until it is understood. CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS FIXTURES_REQUIRED cppjit_datatypes_dict_fixture) -# TEMP: 600 more copies of test_datatypes, run by ctest in parallel in this -# directory like the real suite does, to measure its sporadic crash on the -# mac-beta CI runner (about 3% of runs). -foreach(i RANGE 1 600) - CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS SUFFIX -rep${i} +# TEMP: two groups of 300 copies of test_datatypes, run by ctest in parallel in +# this directory like the real suite does, to measure its sporadic crash on the +# mac-beta CI runner. Both groups keep Python's production allocator and no +# forced collections, because the debug allocator made the crash disappear. +# -malloc: macOS malloc debugging for the C++ heap (scribble freed and fresh +# blocks, guard pages around large ones), to turn a C++ heap misuse +# into a crash at the culprit. +# -only26: test26_function_pointers alone, to learn whether the damage is +# done by test26 itself or left behind by an earlier test. +foreach(i RANGE 1 300) + CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS SUFFIX -malloc${i} + ENV PYTHONMALLOC=default ROOT_CI_DIAGNOSTICS_NO_GC=1 + MallocScribble=1 MallocPreScribble=1 MallocGuardEdges=1 + FIXTURES_REQUIRED cppjit_datatypes_dict_fixture) + CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS SUFFIX -only26-${i} + ENV PYTHONMALLOC=default ROOT_CI_DIAGNOSTICS_NO_GC=1 + PYTEST_ARGS -k test26_function_pointers FIXTURES_REQUIRED cppjit_datatypes_dict_fixture) endforeach() diff --git a/bindings/pyroot/cppjit/test/root_ci_diagnostics.py b/bindings/pyroot/cppjit/test/root_ci_diagnostics.py index 92ed159237b3e..87bbcf9722acc 100644 --- a/bindings/pyroot/cppjit/test/root_ci_diagnostics.py +++ b/bindings/pyroot/cppjit/test/root_ci_diagnostics.py @@ -42,6 +42,9 @@ # an allocation happens to trigger; collecting after every line of the tests # around it pins the damage to the statement that first leaves a corrupted # container behind, instead of wherever the GC threshold is crossed. +# ROOT_CI_DIAGNOSTICS_NO_GC=1 keeps only the faulthandler part, for runs that +# must leave the allocation pattern of the test untouched. +_NO_GC = bool(os.environ.get("ROOT_CI_DIAGNOSTICS_NO_GC")) _TRACE_FILE = "test_datatypes.py" _TRACE_TESTS = ("test24_", "test25_", "test26_", "test27_", "test28_", "test29_") _forced_collections = 0 @@ -72,14 +75,20 @@ def pytest_configure(config): global _stderr_fd _stderr_fd = os.dup(2) _rearm_faulthandler() - sys.settrace(_line_tracer) + if not _NO_GC: + sys.settrace(_line_tracer) def pytest_report_header(config): + gc_mode = ( + "forced collections disabled" + if _NO_GC + else f"gc.collect() after each test and after every line of {_TRACE_FILE} {_TRACE_TESTS}" + ) return ( - "ROOT CI diagnostics: faulthandler re-armed and gc.collect() after " - f"each test and after every line of {_TRACE_FILE} {_TRACE_TESTS}, " - f"PYTHONMALLOC={os.environ.get('PYTHONMALLOC', '')}" + f"ROOT CI diagnostics: faulthandler re-armed, {gc_mode}, " + f"PYTHONMALLOC={os.environ.get('PYTHONMALLOC', '')}, " + f"Malloc*={[k for k in os.environ if k.startswith('Malloc')]}" ) @@ -95,8 +104,9 @@ def pytest_runtest_setup(item): def pytest_runtest_teardown(item, nextitem): _rearm_faulthandler() - gc.collect() - _rearm_faulthandler() + if not _NO_GC: + gc.collect() + _rearm_faulthandler() def pytest_unconfigure(config): From e445082e7c191bdc4b5b1ffeb6cb7d568559bd62 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Sun, 18 Oct 2026 08:47:06 +0000 Subject: [PATCH 05/10] TEMP: [cppjit] Measure with forced collections on the production allocator Both 300-copy groups of the previous round passed (macOS malloc debugging, and test26 alone), as did the 600 copies under PYTHONMALLOC=debug, while every configuration with the production allocator and the whole test file has crashed, always with the collector tripping in test26. Python 3.14's incremental collector reaches an old container at a reproducible point of the allocation sequence, so a list damaged at any earlier point can be found consistently in test26; the test may just be where the damage is noticed, not where it is done. The previous round could not tell, because the forced collections were bundled with the debug allocator, which hides the bug. Separate them: - -gc (700 copies): production allocator, full gc.collect() after every test and after every line of test24 to test29. A crash in the teardown of a test, or at a line, names where the damage is left behind. - -plain (300 copies): production allocator, faulthandler only, for the base rate and the Python line at which the collector trips. Not to be merged. --- bindings/pyroot/cppjit/test/CMakeLists.txt | 31 +++++++++++----------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/bindings/pyroot/cppjit/test/CMakeLists.txt b/bindings/pyroot/cppjit/test/CMakeLists.txt index f460e565527ed..f9ec6142a2e49 100644 --- a/bindings/pyroot/cppjit/test/CMakeLists.txt +++ b/bindings/pyroot/cppjit/test/CMakeLists.txt @@ -132,23 +132,24 @@ CPPJIT_ADD_PYTEST(test_basic_api # the mac-beta ARM64 CI runner; collect diagnostics until it is understood. CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS FIXTURES_REQUIRED cppjit_datatypes_dict_fixture) -# TEMP: two groups of 300 copies of test_datatypes, run by ctest in parallel in -# this directory like the real suite does, to measure its sporadic crash on the -# mac-beta CI runner. Both groups keep Python's production allocator and no -# forced collections, because the debug allocator made the crash disappear. -# -malloc: macOS malloc debugging for the C++ heap (scribble freed and fresh -# blocks, guard pages around large ones), to turn a C++ heap misuse -# into a crash at the culprit. -# -only26: test26_function_pointers alone, to learn whether the damage is -# done by test26 itself or left behind by an earlier test. -foreach(i RANGE 1 300) - CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS SUFFIX -malloc${i} - ENV PYTHONMALLOC=default ROOT_CI_DIAGNOSTICS_NO_GC=1 - MallocScribble=1 MallocPreScribble=1 MallocGuardEdges=1 +# TEMP: copies of test_datatypes, run by ctest in parallel in this directory +# like the real suite does, to measure its sporadic crash on the mac-beta CI +# runner. All keep Python's production allocator, because the debug allocator +# made the crash disappear. +# -gc: a full gc.collect() after every test and after every line of +# test24 to test29, so the collector finds a damaged container at +# the test or line that leaves it behind instead of wherever the +# incremental collector happens to reach it (always test26 so far). +# -plain: faulthandler only, the production behaviour, for the base rate +# and the Python line at which the collector trips. +foreach(i RANGE 1 700) + CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS SUFFIX -gc${i} + ENV PYTHONMALLOC=default FIXTURES_REQUIRED cppjit_datatypes_dict_fixture) - CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS SUFFIX -only26-${i} +endforeach() +foreach(i RANGE 1 300) + CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS SUFFIX -plain${i} ENV PYTHONMALLOC=default ROOT_CI_DIAGNOSTICS_NO_GC=1 - PYTEST_ARGS -k test26_function_pointers FIXTURES_REQUIRED cppjit_datatypes_dict_fixture) endforeach() From 0ea96aa24fcd69174f1b8d5243293b44d3a805b6 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Sun, 18 Oct 2026 09:34:26 +0000 Subject: [PATCH 06/10] TEMP: [cppjit] Watch the reference counts of pytest's reports In the 700 copies with a forced collection after every test, 4 crashed, all identically: every test passed, then a bus error at the session summary in pytest's _get_reports_to_display, on a TestReport taken from the terminal reporter's stats lists. The 300 copies without forced collections all passed. The collections had traversed those lists after every test without tripping, so the report looked valid until late and was garbage by the end: the signature of a report freed early, through a stale Py_DECREF on a pointer to a dead object whose memory the report had reused, and then overwritten once its block was reused again. Under the debug allocator such a pointer lands in a block's padding instead of a reference count, which is why that configuration was silent. So watch for exactly that: after every test, and once more at session finish, the plugin walks the stats lists, records each report's reference count once its own test has finished, and exits the process with a message if a report's count ever drops, or if a report can no longer be read. Each check announces itself on stderr first, so a hard crash inside the scan still shows which test preceded it. The copies become one group of 700 (-rc) with the production allocator and forced collections, the configuration that produced the hits. Verified with a default-build Python 3.14: a clean run stays silent, and an injected Py_DecRef on a report is reported after the test that did it. Not to be merged. --- bindings/pyroot/cppjit/test/CMakeLists.txt | 22 ++----- .../pyroot/cppjit/test/root_ci_diagnostics.py | 57 +++++++++++++++++++ 2 files changed, 63 insertions(+), 16 deletions(-) diff --git a/bindings/pyroot/cppjit/test/CMakeLists.txt b/bindings/pyroot/cppjit/test/CMakeLists.txt index f9ec6142a2e49..494ea37e5ec73 100644 --- a/bindings/pyroot/cppjit/test/CMakeLists.txt +++ b/bindings/pyroot/cppjit/test/CMakeLists.txt @@ -132,26 +132,16 @@ CPPJIT_ADD_PYTEST(test_basic_api # the mac-beta ARM64 CI runner; collect diagnostics until it is understood. CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS FIXTURES_REQUIRED cppjit_datatypes_dict_fixture) -# TEMP: copies of test_datatypes, run by ctest in parallel in this directory -# like the real suite does, to measure its sporadic crash on the mac-beta CI -# runner. All keep Python's production allocator, because the debug allocator -# made the crash disappear. -# -gc: a full gc.collect() after every test and after every line of -# test24 to test29, so the collector finds a damaged container at -# the test or line that leaves it behind instead of wherever the -# incremental collector happens to reach it (always test26 so far). -# -plain: faulthandler only, the production behaviour, for the base rate -# and the Python line at which the collector trips. +# TEMP: 700 copies of test_datatypes, run by ctest in parallel in this +# directory like the real suite does, to measure its sporadic crash on the +# mac-beta CI runner. Production allocator (the debug allocator hides the bug), +# a full gc.collect() after every test, and the plugin's reference count watch +# on pytest's TestReport objects, which the crash was seen to hit. foreach(i RANGE 1 700) - CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS SUFFIX -gc${i} + CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS SUFFIX -rc${i} ENV PYTHONMALLOC=default FIXTURES_REQUIRED cppjit_datatypes_dict_fixture) endforeach() -foreach(i RANGE 1 300) - CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS SUFFIX -plain${i} - ENV PYTHONMALLOC=default ROOT_CI_DIAGNOSTICS_NO_GC=1 - FIXTURES_REQUIRED cppjit_datatypes_dict_fixture) -endforeach() CPPJIT_ADD_PYTEST(test_api) CPPJIT_ADD_PYTEST(test_boost) diff --git a/bindings/pyroot/cppjit/test/root_ci_diagnostics.py b/bindings/pyroot/cppjit/test/root_ci_diagnostics.py index 87bbcf9722acc..4113b62b44a56 100644 --- a/bindings/pyroot/cppjit/test/root_ci_diagnostics.py +++ b/bindings/pyroot/cppjit/test/root_ci_diagnostics.py @@ -36,6 +36,8 @@ import os import sys +import pytest + _stderr_fd = None # The crash always surfaces in test26_function_pointers, in a collection that @@ -98,6 +100,55 @@ def pytest_terminal_summary(terminalreporter): ) +# Watch the reference counts of pytest's own TestReport objects. The crash on +# mac-beta tripped over a report in the terminal reporter's stats lists: a +# plain Python object cpyrt never sees, so the suspect is a stale Py_DECREF +# through a pointer to a dead object whose memory the report reused. Such a +# decrement shows up as a refcount drop of a report that is still referenced +# from the stats list, and checking after every test names the test during +# which it happened, without depending on a later crash. +_report_refcounts = {} + + +def _check_reports(config, where, current_nodeid=None): + tr = config.pluginmanager.get_plugin("terminalreporter") + if tr is None: + return + # announce first: if the scan itself crashes on a dead report, the + # faulthandler output follows this line + os.write(_stderr_fd, f"ROOT CI diagnostics: {where}\n".encode()) + for key, reports in tr.stats.items(): + for rep in reports: + ident = id(rep) + known = _report_refcounts.get(ident) + try: + tname = type(rep).__name__ + rc = sys.getrefcount(rep) + if tname not in ("TestReport", "CollectReport"): + raise TypeError(f"{tname} is not a report") + if known is None: + # the runner still holds temporaries to the reports of the + # test being torn down; take their baseline at the next check + if rep.nodeid != current_nodeid: + _report_refcounts[ident] = (rc, rep.nodeid, rep.when) + continue + except Exception as exc: # a freed report may raise anything + _report( + f"report at {ident:#x} in stats[{key!r}] is unreadable ({exc!r}), {where}" + ) + if rc < known[0]: + _report( + f"refcount of the {known[2]} report of {known[1]} dropped " + f"from {known[0]} to {rc}, {where}" + ) + + +def _report(msg): + os.write(_stderr_fd, f"ROOT CI diagnostics: {msg}\n".encode()) + faulthandler.dump_traceback(file=_stderr_fd, all_threads=False) + os._exit(99) # make ctest show this output + + def pytest_runtest_setup(item): _rearm_faulthandler() @@ -107,6 +158,12 @@ def pytest_runtest_teardown(item, nextitem): if not _NO_GC: gc.collect() _rearm_faulthandler() + _check_reports(item.config, f"checked after {item.nodeid}", item.nodeid) + + +@pytest.hookimpl(tryfirst=True) +def pytest_sessionfinish(session, exitstatus): + _check_reports(session.config, "checked at session finish") def pytest_unconfigure(config): From f342b4792330282a4a6dc40f72cb1baff6ffea0e Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Sun, 18 Oct 2026 10:47:00 +0000 Subject: [PATCH 07/10] TEMP: [cppjit] Check the reports after every line of test19 to test22 The report watch fired in 33 of 700 copies, always at the same place: in the check after test21 (24 copies, a report in stats['passed'] whose type pointer now pointed at a list) or as a bus error inside the check after test20 (9 copies). The check after test19 was clean in all of them. So the report dies during test20_object_comparisons_with_cpp__eq__, or at the latest during test21_object_validity. Narrow that to a statement: the line tracer now works on test19 to test22 and runs the report check after every line. The check reads the stats lists' item pointers, reference counts and type pointers as raw memory through ctypes, because a freed report has a zero reference count and any Python-level touch of it, even iterating the list, deallocates the block a second time. Baselines are taken at the setup of the next test and at teardown, where the runner holds no temporaries to the reports, so the message can say whose report died. Only default (GIL) builds have the object layout the raw reads assume; elsewhere the check is skipped. Verified with a default-build Python 3.14: a clean run is silent, and an injected Py_DecRef on a report is reported at the following line with the report named. Not to be merged. --- .../pyroot/cppjit/test/root_ci_diagnostics.py | 95 ++++++++++++++----- 1 file changed, 73 insertions(+), 22 deletions(-) diff --git a/bindings/pyroot/cppjit/test/root_ci_diagnostics.py b/bindings/pyroot/cppjit/test/root_ci_diagnostics.py index 4113b62b44a56..6927b7554f4f7 100644 --- a/bindings/pyroot/cppjit/test/root_ci_diagnostics.py +++ b/bindings/pyroot/cppjit/test/root_ci_diagnostics.py @@ -31,10 +31,12 @@ symbols) at the point of the crash. """ +import ctypes import faulthandler import gc import os import sys +import sysconfig import pytest @@ -48,8 +50,11 @@ # must leave the allocation pattern of the test untouched. _NO_GC = bool(os.environ.get("ROOT_CI_DIAGNOSTICS_NO_GC")) _TRACE_FILE = "test_datatypes.py" -_TRACE_TESTS = ("test24_", "test25_", "test26_", "test27_", "test28_", "test29_") +# The report watch below found the damage done during test20 (or test21), so +# the tracer now works there, and runs the report check after every line. +_TRACE_TESTS = ("test19_", "test20_", "test21_", "test22_") _forced_collections = 0 +_config = None def _line_tracer(frame, event, arg): @@ -62,6 +67,13 @@ def _line_tracer(frame, event, arg): global _forced_collections gc.collect() _forced_collections += 1 + if _config is not None: + _check_reports( + _config, + f"checked before line {frame.f_lineno} of {code.co_name}", + quiet=True, + record=False, + ) return _line_tracer @@ -74,7 +86,8 @@ def _rearm_faulthandler(): def pytest_configure(config): # Capture starts after configuration, so fd 2 is still the real stderr. - global _stderr_fd + global _stderr_fd, _config + _config = config _stderr_fd = os.dup(2) _rearm_faulthandler() if not _NO_GC: @@ -109,37 +122,72 @@ def pytest_terminal_summary(terminalreporter): # which it happened, without depending on a later crash. _report_refcounts = {} +# Everything below reads object memory through ctypes instead of touching the +# objects: a freed report has a zero reference count, and even iterating the +# list or calling sys.getrefcount on it would increment and decrement that +# count and deallocate the block a second time. The raw layout is that of a +# default (GIL) build: ob_refcnt, ob_type, ob_size, ob_item. +_RAW_LAYOUT_KNOWN = not sysconfig.get_config_var("Py_GIL_DISABLED") + + +def _raw_items(lst): + ob_item = ctypes.c_void_p.from_address( + id(lst) + 3 * ctypes.sizeof(ctypes.c_void_p) + ).value + n = len(lst) + if not ob_item or n == 0: + return [] + return list((ctypes.c_void_p * n).from_address(ob_item)) + + +def _raw_refcnt(p): + return ctypes.c_ssize_t.from_address(p).value + + +def _raw_type(p): + return ctypes.c_void_p.from_address(p + ctypes.sizeof(ctypes.c_void_p)).value -def _check_reports(config, where, current_nodeid=None): + +def _check_reports(config, where, current_nodeid=None, quiet=False, record=True): + if not _RAW_LAYOUT_KNOWN: + return tr = config.pluginmanager.get_plugin("terminalreporter") if tr is None: return # announce first: if the scan itself crashes on a dead report, the # faulthandler output follows this line - os.write(_stderr_fd, f"ROOT CI diagnostics: {where}\n".encode()) + if not quiet: + os.write(_stderr_fd, f"ROOT CI diagnostics: {where}\n".encode()) + from _pytest.reports import CollectReport, TestReport + + report_types = {id(TestReport), id(CollectReport)} for key, reports in tr.stats.items(): - for rep in reports: - ident = id(rep) - known = _report_refcounts.get(ident) - try: - tname = type(rep).__name__ - rc = sys.getrefcount(rep) - if tname not in ("TestReport", "CollectReport"): - raise TypeError(f"{tname} is not a report") - if known is None: - # the runner still holds temporaries to the reports of the - # test being torn down; take their baseline at the next check - if rep.nodeid != current_nodeid: - _report_refcounts[ident] = (rc, rep.nodeid, rep.when) - continue - except Exception as exc: # a freed report may raise anything + for index, p in enumerate(_raw_items(reports)): + known = _report_refcounts.get(p) + tp = _raw_type(p) + rc = _raw_refcnt(p) + if tp not in report_types or rc <= 0: + was = ( + f"the {known[2]} report of {known[1]}" + if known + else "a report not seen before" + ) _report( - f"report at {ident:#x} in stats[{key!r}] is unreadable ({exc!r}), {where}" + f"stats[{key!r}][{index}] at {p:#x} is no longer a report " + f"(type {tp:#x}, refcount {rc}); it was {was}; {where}" ) - if rc < known[0]: + if known is None: + # baselines only at the teardown checks, and not for the + # reports of the test being torn down: the runner still + # holds temporaries to recent reports + if record: + rep = reports[index] # alive: type and refcount checked above + if rep.nodeid != current_nodeid: + _report_refcounts[p] = (rc, rep.nodeid, rep.when) + elif rc < known[0]: _report( f"refcount of the {known[2]} report of {known[1]} dropped " - f"from {known[0]} to {rc}, {where}" + f"from {known[0]} to {rc}; {where}" ) @@ -151,6 +199,9 @@ def _report(msg): def pytest_runtest_setup(item): _rearm_faulthandler() + # the previous test's protocol is complete here: a stable point to take + # the baselines of its reports + _check_reports(item.config, f"checked at setup of {item.nodeid}") def pytest_runtest_teardown(item, nextitem): From 3d5c111d0fe2d5bc26ec68c156d0e71642c780a1 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Sun, 18 Oct 2026 11:46:21 +0000 Subject: [PATCH 08/10] TEMP: [cppjit] Report the stack of the decrement that kills a pytest report The previous round checked the reports after every line of test19 to test22, with a full collection before each check. The collections changed the allocation pattern enough that the damage became rare (2 in 700 copies instead of 33 in 700); both remaining copies crashed in the collection right after `d = gbl.CppjitTestPod()` in test21, and ctest still reported them as passed because RootTestDriver.cmake runs a test a second time after a raw segmentation fault. This round goes back to the configuration of the run that found the damage (collections only after each test and in test24 to test29) and adds a finalizer to pytest's TestReport and CollectReport classes. It runs inside the Py_DECREF that drops the last reference, so when a report that the terminal reporter's stats lists still hold is deallocated, the output carries the Python traceback (the statement of the test) and, with Python 3.14, the C stack of the decrement, which names the cpyrt code that performed it. The process then exits with code 98. The second attempt after a segmentation fault in RootTestDriver.cmake is disabled so that such crashes fail the test instead of hiding behind a passing retry. --- bindings/pyroot/cppjit/test/CMakeLists.txt | 6 +- .../pyroot/cppjit/test/root_ci_diagnostics.py | 66 +++++++++++++++---- cmake/modules/RootTestDriver.cmake | 7 +- 3 files changed, 64 insertions(+), 15 deletions(-) diff --git a/bindings/pyroot/cppjit/test/CMakeLists.txt b/bindings/pyroot/cppjit/test/CMakeLists.txt index 494ea37e5ec73..81c199a3c3244 100644 --- a/bindings/pyroot/cppjit/test/CMakeLists.txt +++ b/bindings/pyroot/cppjit/test/CMakeLists.txt @@ -135,8 +135,10 @@ CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS # TEMP: 700 copies of test_datatypes, run by ctest in parallel in this # directory like the real suite does, to measure its sporadic crash on the # mac-beta CI runner. Production allocator (the debug allocator hides the bug), -# a full gc.collect() after every test, and the plugin's reference count watch -# on pytest's TestReport objects, which the crash was seen to hit. +# a full gc.collect() after every test, and the plugin's watch on pytest's +# TestReport objects, which the crash was seen to hit: their reference counts +# are checked around every test, and a finalizer reports the stack of the +# decrement that deallocates one while the stats lists still hold it. foreach(i RANGE 1 700) CPPJIT_ADD_PYTEST(test_datatypes CRASH_DIAGNOSTICS SUFFIX -rc${i} ENV PYTHONMALLOC=default diff --git a/bindings/pyroot/cppjit/test/root_ci_diagnostics.py b/bindings/pyroot/cppjit/test/root_ci_diagnostics.py index 6927b7554f4f7..c65b26da50e84 100644 --- a/bindings/pyroot/cppjit/test/root_ci_diagnostics.py +++ b/bindings/pyroot/cppjit/test/root_ci_diagnostics.py @@ -50,11 +50,15 @@ # must leave the allocation pattern of the test untouched. _NO_GC = bool(os.environ.get("ROOT_CI_DIAGNOSTICS_NO_GC")) _TRACE_FILE = "test_datatypes.py" -# The report watch below found the damage done during test20 (or test21), so -# the tracer now works there, and runs the report check after every line. -_TRACE_TESTS = ("test19_", "test20_", "test21_", "test22_") +# The report watch below found the damage done during test20 (or test21); +# collecting after every line there changed the allocation pattern enough to +# make the damage rare (2 in 700 instead of 33 in 700), so the tracer stays +# where it was for the run that found it, and the report finalizer below +# catches the culprit in the act instead. +_TRACE_TESTS = ("test24_", "test25_", "test26_", "test27_", "test28_", "test29_") _forced_collections = 0 _config = None +_session_active = False def _line_tracer(frame, event, arg): @@ -67,13 +71,6 @@ def _line_tracer(frame, event, arg): global _forced_collections gc.collect() _forced_collections += 1 - if _config is not None: - _check_reports( - _config, - f"checked before line {frame.f_lineno} of {code.co_name}", - quiet=True, - record=False, - ) return _line_tracer @@ -86,10 +83,12 @@ def _rearm_faulthandler(): def pytest_configure(config): # Capture starts after configuration, so fd 2 is still the real stderr. - global _stderr_fd, _config + global _stderr_fd, _config, _session_active _config = config _stderr_fd = os.dup(2) _rearm_faulthandler() + _install_report_finalizer() + _session_active = True if not _NO_GC: sys.settrace(_line_tracer) @@ -101,7 +100,8 @@ def pytest_report_header(config): else f"gc.collect() after each test and after every line of {_TRACE_FILE} {_TRACE_TESTS}" ) return ( - f"ROOT CI diagnostics: faulthandler re-armed, {gc_mode}, " + f"ROOT CI diagnostics: faulthandler re-armed, report finalizer " + f"installed, {gc_mode}, " f"PYTHONMALLOC={os.environ.get('PYTHONMALLOC', '')}, " f"Malloc*={[k for k in os.environ if k.startswith('Malloc')]}" ) @@ -197,6 +197,45 @@ def _report(msg): os._exit(99) # make ctest show this output +# The checks above only see the damage after the fact. A finalizer on the +# report classes runs inside the very Py_DECREF that drops the last reference, +# so when a report that the stats lists still hold is deallocated, the Python +# traceback names the statement of the test and the C stack (Python 3.14+) +# names the cpyrt code that performed the decrement. +def _install_report_finalizer(): + from _pytest.reports import CollectReport, TestReport + + for cls in (TestReport, CollectReport): + cls.__del__ = _report_finalizer + + +def _report_finalizer(self): + if not _session_active or _config is None: + return + tr = _config.pluginmanager.get_plugin("terminalreporter") + if tr is None: + return + for key, reports in tr.stats.items(): + for index, rep in enumerate(reports): + if rep is self: + break + else: + continue + os.write( + _stderr_fd, + ( + f"ROOT CI diagnostics: the {getattr(self, 'when', '?')} report of " + f"{getattr(self, 'nodeid', '?')} at {id(self):#x} is being " + f"deallocated while stats[{key!r}][{index}] still holds it; " + f"Python and C stacks of the decrement follow\n" + ).encode(), + ) + faulthandler.dump_traceback(file=_stderr_fd, all_threads=False) + if hasattr(faulthandler, "dump_c_stack"): + faulthandler.dump_c_stack(file=_stderr_fd) + os._exit(98) # make ctest show this output + + def pytest_runtest_setup(item): _rearm_faulthandler() # the previous test's protocol is complete here: a stable point to take @@ -214,7 +253,10 @@ def pytest_runtest_teardown(item, nextitem): @pytest.hookimpl(tryfirst=True) def pytest_sessionfinish(session, exitstatus): + global _session_active _check_reports(session.config, "checked at session finish") + # from here on pytest drops its reports legitimately + _session_active = False def pytest_unconfigure(config): diff --git a/cmake/modules/RootTestDriver.cmake b/cmake/modules/RootTestDriver.cmake index 5545d6b25f835..2d421f96fe59d 100644 --- a/cmake/modules/RootTestDriver.cmake +++ b/cmake/modules/RootTestDriver.cmake @@ -248,7 +248,12 @@ if(CMD) else() execute_process(COMMAND ${_cmd} ${_out} ${_err} ${_cwd} RESULT_VARIABLE _rc) - if(_rc STREQUAL "Segmentation fault" OR _rc EQUAL 11) + # TEMP: no second attempt after a segmentation fault. The cppjit crash + # diagnostics re-arm Python's faulthandler, so the process dies by the + # raw signal instead of ROOT's handler, and the retry below then hid two + # crashes in 700 copies behind a passing second run (the first attempt's + # output stays in the log, but ctest reports success). + if(FALSE AND (_rc STREQUAL "Segmentation fault" OR _rc EQUAL 11)) message(STATUS "Got ${_rc}, retrying again once more... with coredump enabled") string (REPLACE ";" " " _cmd_str "${_cmd}") file(WRITE run_with_coredump.sh " From 99f5832f218a5376c4df7160d9a70d5443c9813a Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Sun, 18 Oct 2026 12:11:50 +0000 Subject: [PATCH 09/10] TEMP: [cppjit] Compare the stats lists before every line of test19 to test22 The report finalizer did not fire in the copy that crashed: the collection at the teardown of test21 died in list_traverse without any report having been deallocated. Together with the earlier run, where a report's type slot pointed at a list instance while the stats list still held the report, this says the damage is a stray write into live memory, not a stray decrement. So the tracer now snapshots the raw state of the terminal reporter's stats lists (list identity, item array, item pointers, type slots and reference counts of the reports) when the body of test19 to test22 starts, and compares it before every line and at the return, without collecting. Pytest does not touch its reports inside a test body, so any difference is the damage, and the message names the statement that did it. On a difference the plugin dumps the memory around the affected addresses and looks up which live object owns each address (the object itself, a list's item array, or the inside of an object), so that a pointer written into the report can be identified. It also prints the address of every new local of these tests, with the C++ address for cppjit proxies, to relate the damage to the objects the test creates. --- .../pyroot/cppjit/test/root_ci_diagnostics.py | 251 +++++++++++++++++- 1 file changed, 237 insertions(+), 14 deletions(-) diff --git a/bindings/pyroot/cppjit/test/root_ci_diagnostics.py b/bindings/pyroot/cppjit/test/root_ci_diagnostics.py index c65b26da50e84..9a06ed59b868a 100644 --- a/bindings/pyroot/cppjit/test/root_ci_diagnostics.py +++ b/bindings/pyroot/cppjit/test/root_ci_diagnostics.py @@ -50,28 +50,70 @@ # must leave the allocation pattern of the test untouched. _NO_GC = bool(os.environ.get("ROOT_CI_DIAGNOSTICS_NO_GC")) _TRACE_FILE = "test_datatypes.py" -# The report watch below found the damage done during test20 (or test21); +# The report watch below found the damage done during test21 (or test20); # collecting after every line there changed the allocation pattern enough to -# make the damage rare (2 in 700 instead of 33 in 700), so the tracer stays -# where it was for the run that found it, and the report finalizer below -# catches the culprit in the act instead. -_TRACE_TESTS = ("test24_", "test25_", "test26_", "test27_", "test28_", "test29_") +# make the damage rare (2 in 700 instead of 33 in 700), so the collections +# stay where they were for the run that found it. In the tests around the +# damage the tracer only reads memory: it snapshots the stats lists when the +# test body starts and compares them before every line, which pins the +# statement without a collection. +_GC_TESTS = ("test24_", "test25_", "test26_", "test27_", "test28_", "test29_") +_WATCH_TESTS = ("test19_", "test20_", "test21_", "test22_") _forced_collections = 0 _config = None _session_active = False +_last_line = None +_seen_locals = set() def _line_tracer(frame, event, arg): + global _forced_collections, _last_line code = frame.f_code - if not code.co_filename.endswith(_TRACE_FILE) or not code.co_name.startswith( - _TRACE_TESTS - ): + if not code.co_filename.endswith(_TRACE_FILE): return None - if event == "line": - global _forced_collections - gc.collect() - _forced_collections += 1 - return _line_tracer + name = code.co_name + if name.startswith(_GC_TESTS): + if event == "line": + gc.collect() + _forced_collections += 1 + return _line_tracer + if name.startswith(_WATCH_TESTS) and _config is not None: + if event == "call": + _snapshot_stats(f"snapshot at the start of {name}") + _seen_locals.clear() + _last_line = None + elif event == "line": + _compare_stats( + f"checked after line {_last_line} (before line {frame.f_lineno}) " + f"of {name}" + ) + _note_new_locals(frame) + _last_line = frame.f_lineno + elif event == "return": + _compare_stats(f"checked at the return of {name}") + return _line_tracer + return None + + +def _note_new_locals(frame): + # the addresses of the objects a test creates, to relate them to the + # addresses of the damage + for name, obj in list(frame.f_locals.items()): + if id(obj) in _seen_locals: + continue + _seen_locals.add(id(obj)) + cpp = "" + try: + import cppjit + + cpp = f", C++ object at {cppjit.addressof(obj):#x}" + except Exception: + pass + os.write( + _stderr_fd, + f"ROOT CI diagnostics: local {name} is a {type(obj).__name__} at " + f"{id(obj):#x}{cpp}\n".encode(), + ) def _rearm_faulthandler(): @@ -97,7 +139,8 @@ def pytest_report_header(config): gc_mode = ( "forced collections disabled" if _NO_GC - else f"gc.collect() after each test and after every line of {_TRACE_FILE} {_TRACE_TESTS}" + else f"gc.collect() after each test and after every line of {_TRACE_FILE} {_GC_TESTS}, " + f"stats lists compared before every line of {_WATCH_TESTS}" ) return ( f"ROOT CI diagnostics: faulthandler re-armed, report finalizer " @@ -197,6 +240,186 @@ def _report(msg): os._exit(99) # make ctest show this output +# Within a test body pytest does not touch its reports, so the stats lists, +# their item arrays and the reports' headers must stay exactly as they were +# when the body started. The run of 2026-10-18 showed a report whose type +# slot had been overwritten with a pointer to a list while the report was +# still referenced, i.e. a stray write rather than a stray decrement, which +# is why the comparison covers the raw item pointers and type slots. +_stats_snapshot = {} + + +def _raw_ob_item(lst): + return ctypes.c_void_p.from_address( + id(lst) + 3 * ctypes.sizeof(ctypes.c_void_p) + ).value + + +def _write(msg): + os.write(_stderr_fd, f"ROOT CI diagnostics: {msg}\n".encode()) + + +def _snapshot_stats(where): + if not _RAW_LAYOUT_KNOWN: + return + tr = _config.pluginmanager.get_plugin("terminalreporter") + if tr is None: + return + _stats_snapshot.clear() + for key, reports in tr.stats.items(): + items = _raw_items(reports) + _stats_snapshot[key] = ( + id(reports), + _raw_ob_item(reports), + items, + [_raw_type(p) for p in items], + [_raw_refcnt(p) for p in items], + ) + _write( + f"{where}: stats[{key!r}] is the list at {id(reports):#x} with " + f"{len(items)} items at {_raw_ob_item(reports) or 0:#x}" + + (f", first report at {items[0]:#x}" if items else "") + ) + + +def _compare_stats(where): + if not _RAW_LAYOUT_KNOWN or not _stats_snapshot: + return + tr = _config.pluginmanager.get_plugin("terminalreporter") + if tr is None: + return + for key, (list_id, ob_item, items, types, refcnts) in _stats_snapshot.items(): + reports = tr.stats.get(key) + if reports is None or id(reports) != list_id: + _corruption(f"stats[{key!r}] is a different list; {where}", items) + now = _raw_items(reports) + if _raw_ob_item(reports) != ob_item or len(now) != len(items): + _corruption( + f"stats[{key!r}] changed from {len(items)} items at {ob_item:#x} to " + f"{len(now)} items at {_raw_ob_item(reports) or 0:#x}; {where}", + items, + ) + for index, (p, q) in enumerate(zip(items, now)): + if q != p: + _corruption( + f"stats[{key!r}][{index}] changed from {p:#x} to {q:#x}; {where}", + [p, q], + ) + tp = _raw_type(p) + rc = _raw_refcnt(p) + if tp != types[index]: + _corruption( + f"the type slot of the report at {p:#x} (stats[{key!r}][{index}]) " + f"changed from {types[index]:#x} to {tp:#x}; {where}", + [p, tp], + ) + if rc <= 0 or rc < refcnts[index]: + _corruption( + f"the refcount of the report at {p:#x} (stats[{key!r}][{index}]) " + f"changed from {refcnts[index]} to {rc}; {where}", + [p], + ) + + +def _corruption(msg, addresses): + _write(msg) + # the lookups only compare addresses; the dumps read the memory, which + # may be unmapped for a garbage pointer, so they come last + for addr in addresses: + for line in _locate(addr): + _write(f"{addr:#x} {line}") + for addr in addresses: + _write(f"memory around {addr:#x}: " + _dump_words(addr)) + _report("stopping here") + + +def _is_real_type(tp): + # whether the type slot value tp points at a type object: its own type + # is `type` or a metaclass. Everything the lookup below does with an + # object goes through its type slot, so a corrupted one must be skipped. + if not tp: + return False + try: + mt = _raw_type(tp) + if mt == id(type): + return True + if _raw_type(mt) != id(type): + return False + return issubclass(ctypes.cast(mt, ctypes.py_object).value, type) + except Exception: + return False + + +def _dump_words(addr, before=8, after=8): + try: + words = (ctypes.c_void_p * (before + after)).from_address( + addr - before * ctypes.sizeof(ctypes.c_void_p) + ) + return " ".join( + ("|" if i == before else "") + f"{(w or 0):#018x}" + for i, w in enumerate(words) + ) + except Exception as exc: + return f"unreadable ({exc!r})" + + +def _type_name(obj): + try: + return type(obj).__name__ + except Exception: + return f"" + + +def _locate(addr): + # which live object owns the memory at addr: the object itself, a list's + # item array, or the inside of an object + found = [] + tr = _config.pluginmanager.get_plugin("terminalreporter") + stats_lists = {id(v): k for k, v in tr.stats.items()} if tr else {} + list_type = id(list) + for o in gc.get_objects(): + oid = id(o) + if not _is_real_type(_raw_type(oid)): + found.append( + f"(skipped the object at {oid:#x}, whose type slot {_raw_type(oid):#x} " + f"is not a type)" + ) + continue + if oid == addr: + desc = f"is a live {_type_name(o)} object" + elif _raw_type(oid) == list_type: + ob_item = _raw_ob_item(o) + n = len(o) + if ob_item and ob_item <= addr < ob_item + n * ctypes.sizeof( + ctypes.c_void_p + ): + desc = ( + f"is item {(addr - ob_item) // ctypes.sizeof(ctypes.c_void_p)} " + f"of the item array of the list of {n} at {oid:#x}" + ) + else: + continue + else: + try: + size = sys.getsizeof(o) + except Exception: + continue + # large sizes are external buffers, not the object's own memory + if size <= 1024 and oid < addr < oid + size: + desc = f"is {addr - oid} bytes into the {_type_name(o)} of size {size} at {oid:#x}" + else: + continue + if oid in stats_lists: + desc += f" (stats[{stats_lists[oid]!r}])" + if _raw_type(oid) == list_type: + try: + desc += f" holding {[_type_name(x) for x in o[:4]]}" + except Exception: + pass + found.append(desc) + return found or ["matches no live GC-tracked object"] + + # The checks above only see the damage after the fact. A finalizer on the # report classes runs inside the very Py_DECREF that drops the last reference, # so when a report that the stats lists still hold is deallocated, the Python From 41e8bb8ae083c6fc4a4bc7c99557d7e837bf3c98 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Sun, 18 Oct 2026 13:05:52 +0000 Subject: [PATCH 10/10] [cppjit] Unregister proxies whose C++ object was replaced through T** When a C++ function takes a proxied object as `T**` (or `void**`), the converter passes the address of the proxy's own pointer slot, so the function can delete the object and store a new one in the proxy, as `CppjitTestData::set_pod_ptrptr_out` does in test_datatypes. The memory regulator still tracks the proxy under the old address, and when the proxy dies, `UnregisterPyObject` looks it up under the new one. That erases nothing, or erases the entry of an unrelated proxy that is tracked under the new address, and leaves the old entry pointing at the dead proxy. Once the allocator hands the old address to a new object of the same class, `RegisterPyObject` finds the dangling entry and clears `kIsRegulated` in the `fFlags` of the dead proxy, i.e. it clears bit 0x800 somewhere in whatever object reused that memory. This was the sporadic crash of pyunittests-bindings-cppjit-test-datatypes on the mac-beta runner: the constructor call in test21 got the address of the pod that test16 had replaced, and the write landed in the item array of pytest's list of passed reports, which the garbage collector later traversed. Only macOS's allocator reused the address at that point. `UnregisterPyObject` now erases an entry only if it belongs to the proxy, and searches the map for the proxy otherwise, which also covers proxies whose held pointer was nulled. The search runs only when a registered proxy is not found under the address it holds now. --- .../cppjit/src/cpyrt/MemoryRegulator.cxx | 28 +++++++++--- .../pyroot/cppjit/test/test_regression.py | 44 +++++++++++++++++++ 2 files changed, 66 insertions(+), 6 deletions(-) diff --git a/bindings/pyroot/cppjit/src/cpyrt/MemoryRegulator.cxx b/bindings/pyroot/cppjit/src/cpyrt/MemoryRegulator.cxx index a7a06c63b20d8..bc4d376bb1c1b 100644 --- a/bindings/pyroot/cppjit/src/cpyrt/MemoryRegulator.cxx +++ b/bindings/pyroot/cppjit/src/cpyrt/MemoryRegulator.cxx @@ -8,6 +8,7 @@ using namespace cppjit; #include "ProxyWrappers.h" // Standard +#include #include #include #include @@ -199,10 +200,8 @@ bool cpyrt::MemoryRegulator::UnregisterPyObject(CPPInstance* pyobj, interop::TCppObject_t cppobj = pyobj->IsSmart() ? pyobj->GetObjectRaw() : pyobj->GetObject(); - if (!cppobj) - return false; - if (unregisterHook) { + if (cppobj && unregisterHook) { auto res = unregisterHook(cppobj, ((CPPClass*)pyclass)->fCppType); if (!res.second) return res.first; @@ -212,9 +211,26 @@ bool cpyrt::MemoryRegulator::UnregisterPyObject(CPPInstance* pyobj, if (!cppobjs) return false; - // erase if tracked - if (cppobjs->erase(cppobj)) { - pyobj->fFlags &= ~CPPInstance::kIsRegulated; + // The proxy is normally tracked under the address it holds now. That + // address can have changed since registration, though: a function taking + // the object as T** (or void**) may delete it and write a new object into + // the proxy, and cross-inheritance can null the held pointer. Only erase an + // entry that belongs to this proxy, because another proxy may be tracked + // under the new address by now, and otherwise search the map, because an + // entry left behind under the old address would dangle once this proxy is + // gone: a later C++ object at that address would be matched with the dead + // proxy, and re-registering the address writes into its freed memory. + auto ppo = cppobj ? cppobjs->find(cppobj) : cppobjs->end(); + if (ppo == cppobjs->end() || ppo->second != (PyObject*)pyobj) { + ppo = std::find_if(cppobjs->begin(), cppobjs->end(), + [pyobj](const CppToPyMap_t::value_type& entry) { + return entry.second == (PyObject*)pyobj; + }); + } + + pyobj->fFlags &= ~CPPInstance::kIsRegulated; + if (ppo != cppobjs->end()) { + cppobjs->erase(ppo); return true; } diff --git a/bindings/pyroot/cppjit/test/test_regression.py b/bindings/pyroot/cppjit/test/test_regression.py index cbd5f566667d4..c63323e085ba1 100644 --- a/bindings/pyroot/cppjit/test/test_regression.py +++ b/bindings/pyroot/cppjit/test/test_regression.py @@ -1733,3 +1733,47 @@ def test56_enum_arg_typedefed_underlying_type(self): assert ns.pick(ns.S.A) == 1 assert ns.pick(ns.S.B) == 5 + + def test57_regulator_follows_replaced_object(self): + """A proxy whose object a T** argument replaced unregisters cleanly""" + + import gc + + import cppjit + + cppjit.cppdef("""\ + namespace RegulatorReplaced { + struct Pod { int m_int; }; + // allocate the replacement first, so its address always differs + void replace(Pod** pp) { + Pod* fresh = new Pod(**pp); + delete *pp; + *pp = fresh; + } + }""") + + ns = cppjit.gbl.RegulatorReplaced + + p = ns.Pod() + old = cppjit.addressof(p) + ns.replace(p) + new = cppjit.addressof(p) + assert new != old + + # a second proxy for the object at the new address + q = cppjit.bind_object(new, ns.Pod) + assert q is not p + + # p must not take q's entry along when it dies (the C++ object stays + # alive for q) + p.__python_owns__ = False + del p + gc.collect() + assert cppjit.bind_object(new, ns.Pod) is q + + # nor leave its own entry behind under the old address, where a later + # object at that address would be matched with the dead proxy (binding + # does not dereference the address) + r = cppjit.bind_object(old, ns.Pod) + assert cppjit.addressof(r) == old + assert r is not q