diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 93af052cf..3f5d3300d 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -382,6 +382,11 @@ jobs: failed=0 for pkg_dir in build/ros2_medkit_*/; do pkg=$(basename "$pkg_dir") + # graph_watchdog is tested in sanitizer-graph-watchdog. Its + # end-to-end suite is 24 minutes under instrumentation, which is + # more than the headroom left here, and a package that overruns + # this step takes every package after it down with it. + if [ "$pkg" = "ros2_medkit_graph_watchdog" ]; then continue; fi echo "::group::Testing $pkg" (cd "$pkg_dir" && ctest -LE "linter" --output-on-failure) || failed=1 echo "::endgroup::" @@ -392,6 +397,7 @@ jobs: if: always() run: | for pkg_dir in build/ros2_medkit_*/; do + if [ "$(basename "$pkg_dir")" = "ros2_medkit_graph_watchdog" ]; then continue; fi colcon test-result --test-result-base "$pkg_dir" --verbose 2>/dev/null || true done @@ -500,6 +506,11 @@ jobs: failed=0 for pkg_dir in build/ros2_medkit_*/; do pkg=$(basename "$pkg_dir") + # graph_watchdog is tested in sanitizer-graph-watchdog. Its + # end-to-end suite is 24 minutes under instrumentation, which is + # more than the headroom left here, and a package that overruns + # this step takes every package after it down with it. + if [ "$pkg" = "ros2_medkit_graph_watchdog" ]; then continue; fi echo "::group::Testing $pkg" (cd "$pkg_dir" && ctest -j1 -LE "linter" --output-on-failure) || failed=1 echo "::endgroup::" @@ -510,5 +521,152 @@ jobs: if: always() run: | for pkg_dir in build/ros2_medkit_*/; do + if [ "$(basename "$pkg_dir")" = "ros2_medkit_graph_watchdog" ]; then continue; fi colcon test-result --test-result-base "$pkg_dir" --verbose 2>/dev/null || true done + + # ros2_medkit_graph_watchdog's end-to-end suite runs 24 minutes under + # instrumentation, against a 45-minute test budget the rest of the workspace + # already spends 22 to 38 of. Testing it here instead of in the workspace + # sweeps gives it a budget of its own, and stops a suite that grows with every + # new detector from deciding whether the packages behind it get to run at all. + # It is not path-filtered: the plugin drives the gateway, the fault manager and + # discovery, so the changes most likely to break it are not in its own tree. + sanitizer-graph-watchdog: + name: Sanitizer ${{ matrix.name }} (graph_watchdog) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - sanitizer: asan + name: ASan + UBSan + cmake_sanitizer: asan,ubsan + # Both sizes are the ones the matching workspace job argued for. + # This job restores that job's cache and must not evict what it + # restored, so it cannot be smaller here. + ccache_size: 2G + - sanitizer: tsan + name: TSan + cmake_sanitizer: tsan + ccache_size: 1.5G + container: + image: ubuntu:noble + # See sanitizer-asan: the instrumented build tree does not fit the + # container overlay on small GitHub runners, so build on /mnt. + volumes: + - "/mnt:/mnt" + # A cold cache builds the chain from scratch, which is the ~20 min the TSan + # workspace job measures, before the 45-minute test budget starts. + timeout-minutes: 90 + defaults: + run: + shell: bash + + steps: + - name: Install Git + run: | + apt-get update + apt-get install -y git + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Pre-install ROS 2 apt source + uses: ./.github/actions/ros-apt-source + + - name: Set up ROS 2 Jazzy + uses: ros-tooling/setup-ros@v0.7 + with: + required-ros-distributions: jazzy + + - name: Install ccache + run: apt-get install -y ccache + + - name: Restore ccache + # Restore, never save. This job compiles a subset of the sources + # sanitizer-${{ matrix.sanitizer }} compiles, with the same flags, so + # that job's cache is a superset of what this one needs and serves its + # misses. A second copy would buy nothing and would spend the + # repository's 10 GB Actions cache quota twice for one set of objects. + uses: actions/cache/restore@v4 + with: + path: /root/.cache/ccache + key: ccache-jazzy-${{ matrix.sanitizer }}-${{ github.sha }} + restore-keys: | + ccache-jazzy-${{ matrix.sanitizer }}- + + - name: Install dependencies + run: | + apt-get update + apt-get install -y ros-jazzy-test-msgs + source /opt/ros/jazzy/setup.bash + rosdep update + rosdep install --from-paths src --ignore-src -y + + - name: Redirect heavy build output to /mnt + run: | + # Same reasoning as the workspace sanitizer jobs, and the same need: + # the chain built here still includes the gateway. + mkdir -p /mnt/gw/build /mnt/gw/tmp + ln -sfn /mnt/gw/build build + df -h / /mnt + + - name: Build with ${{ matrix.name }} + env: + CCACHE_DIR: /root/.cache/ccache + CCACHE_MAXSIZE: ${{ matrix.ccache_size }} + CCACHE_SLOPPINESS: pch_defines,time_macros + TMPDIR: /mnt/gw/tmp + run: | + source /opt/ros/jazzy/setup.bash + ccache -z + # --packages-up-to rather than the workspace: the chain reaches the + # gateway, the fault manager and ros2_medkit_integration_tests, which + # is a test dependency and is where the launch helpers and the demo + # nodes the end-to-end scenarios start actually live. + colcon build --symlink-install \ + --packages-up-to ros2_medkit_graph_watchdog \ + --cmake-args -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DSANITIZER=${{ matrix.cmake_sanitizer }} \ + --event-handlers console_direct+ + ccache -s + ./scripts/ccache_report.sh jazzy-${{ matrix.sanitizer }}-graph-watchdog + df -h / /mnt + + - name: Extend test timeouts for sanitizer overhead + run: | + # The same generic x3 rewrite the workspace sanitizer jobs apply - see + # the comment in sanitizer-asan for why it is not a list of literals. + find build/ -name "CTestTestfile.cmake" -exec \ + perl -pi -e 's/\bTIMEOUT "(\d+)"/sprintf(q{TIMEOUT "%d"}, $1 * 3)/ge' {} + + find build/ -name "CTestTestfile.cmake" -exec cat {} + \ + | grep -oE 'TIMEOUT "[0-9]+"' | sort -t'"' -k2 -n | uniq -c + + - name: Run graph_watchdog tests with ${{ matrix.name }} + timeout-minutes: 45 + env: + # Same factor as the ctest TIMEOUT rewrite above, for the wall-clock + # budgets tests assert internally - ctest's clock cannot reach those. + MEDKIT_TEST_TIME_SCALE: 3 + # Lets tests size instrumented-only-expensive resources down. + MEDKIT_TEST_SANITIZED: 1 + run: | + if [ "${{ matrix.sanitizer }}" = "tsan" ]; then + export TSAN_OPTIONS="halt_on_error=0:history_size=4:suppressions=$(pwd)/tsan_suppressions.txt" + else + # detect_leaks=0: FastDDS allocator leaks on shutdown (not our code) + # new_delete_type_mismatch=0: ROS 2 DDS scalar/array new/delete mismatch + export ASAN_OPTIONS=halt_on_error=1:detect_leaks=0:new_delete_type_mismatch=0 + export UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 + fi + source /opt/ros/jazzy/setup.bash + source install/setup.bash + cd build/ros2_medkit_graph_watchdog + ctest -j1 -LE "linter" --output-on-failure + + - name: Show test results + if: always() + run: | + colcon test-result --test-result-base build/ros2_medkit_graph_watchdog \ + --verbose 2>/dev/null || true diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/CMakeLists.txt b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/CMakeLists.txt index 9ccd6c794..e491290f6 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/CMakeLists.txt +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/CMakeLists.txt @@ -163,6 +163,7 @@ if(BUILD_TESTING) src/detector_registry.cpp src/reliability_gate.cpp src/lifecycle_watcher.cpp + src/detectors/node_death_detector.cpp ${LIFECYCLE_STATE_READER_SOURCES}) target_include_directories(test_graph_watchdog_plugin PRIVATE include ${GATEWAY_SRC_INCLUDE_DIR}) medkit_target_dependencies(test_graph_watchdog_plugin @@ -211,6 +212,60 @@ if(BUILD_TESTING) medkit_target_dependencies(test_aggregated_fault ros2_medkit_gateway rclcpp ros2_medkit_msgs) target_link_libraries(test_aggregated_fault nlohmann_json::nlohmann_json) + # Pure interface/logic: DetectorContext is only ever default-constructed (every pointer + # null), so no rclcpp::init() is needed anywhere in this one. + medkit_add_gtest(test_suppressor test/test_suppressor.cpp) + target_include_directories(test_suppressor PRIVATE include) + medkit_target_dependencies(test_suppressor rclcpp ros2_medkit_msgs) + target_link_libraries(test_suppressor nlohmann_json::nlohmann_json) + + # AllowlistSuppressor::suppresses() ignores ctx entirely - same "no rclcpp::init()" shape + # as test_suppressor above. + medkit_add_gtest(test_allowlist_suppressor test/test_allowlist_suppressor.cpp) + target_include_directories(test_allowlist_suppressor PRIVATE include) + medkit_target_dependencies(test_allowlist_suppressor rclcpp ros2_medkit_msgs) + target_link_libraries(test_allowlist_suppressor nlohmann_json::nlohmann_json) + + # Reads a REAL ReliabilityGate (needs a real rclcpp::Node for its LifecycleWatcher), + # driven through set_departed_lifecycle_state_for_test() rather than a live managed node. + medkit_add_gtest(test_lifecycle_shutdown_suppressor + test/test_lifecycle_shutdown_suppressor.cpp + src/reliability_gate.cpp + src/lifecycle_watcher.cpp + ${LIFECYCLE_STATE_READER_SOURCES}) + target_include_directories(test_lifecycle_shutdown_suppressor PRIVATE include ${GATEWAY_SRC_INCLUDE_DIR}) + medkit_target_dependencies(test_lifecycle_shutdown_suppressor ros2_medkit_gateway rclcpp ros2_medkit_msgs lifecycle_msgs) + target_link_libraries(test_lifecycle_shutdown_suppressor nlohmann_json::nlohmann_json) + + # Pure logic over hand-built present/armed sets: no rclcpp::Node anywhere in it. Depends + # on ros2_medkit_gateway/nlohmann_json only because the D1 case assembles a description + # through the real AggregatedFault::describe_ordered(), the same reason + # test_lifecycle_expectation_tracker above needs it. + medkit_add_gtest(test_node_liveness_tracker test/test_node_liveness_tracker.cpp) + target_include_directories(test_node_liveness_tracker PRIVATE include) + medkit_target_dependencies(test_node_liveness_tracker ros2_medkit_gateway rclcpp ros2_medkit_msgs) + target_link_libraries(test_node_liveness_tracker nlohmann_json::nlohmann_json) + + # Registers a fake /fault_manager/report_fault service like the QoS/orphan/param_drift/ + # lifecycle_expectation integration tests above, so it shares their RESOURCE_LOCK. + medkit_add_gtest(test_node_death_integration + test/test_node_death_integration.cpp + src/detectors/node_death_detector.cpp + src/detector_registry.cpp + src/reliability_gate.cpp + src/lifecycle_watcher.cpp + ${LIFECYCLE_STATE_READER_SOURCES}) + target_include_directories(test_node_death_integration PRIVATE include ${GATEWAY_SRC_INCLUDE_DIR}) + medkit_target_dependencies(test_node_death_integration + ros2_medkit_gateway rclcpp rcutils ros2_medkit_msgs lifecycle_msgs) + target_link_libraries(test_node_death_integration nlohmann_json::nlohmann_json) + # N11's churn sweep and the two retention-boundary cases each drive dozens of ticks; none + # needs real DDS endpoint discovery (every snapshot is hand-built), so this is generous + # against CI slowness rather than against any real per-tick cost. + set_tests_properties(test_node_death_integration PROPERTIES TIMEOUT 120) + set_property(TEST test_node_death_integration APPEND PROPERTY + RESOURCE_LOCK graph_watchdog_integration_domain) + # Real DDS endpoints read through the live graph API. The three that ALSO register a fake # /fault_manager/report_fault service share a RESOURCE_LOCK, because they spin real nodes for # many seconds and running them concurrently on this box starves the discovery each one waits @@ -586,6 +641,284 @@ if(BUILD_TESTING) LABELS "integration;e2e" ENV "GATEWAY_TEST_PORT=19300" "WATCHDOG_E2E_SCENARIO=restart_departed" "${_WATCHDOG_E2E_ENV}") + # === node_death e2e (launch_testing) === + # + # Exercises the node_death detector end to end against a real gateway + fault_manager + + # demo-node stack. See test/e2e/test_node_death_e2e.test.py's own module docstring for the + # full scenario list and what each row proves. + + # 60 (arming gate, app_id-scoped - already implies presence, so this is not a separate + # budget) + 30 (departure poll) + 60 (raise poll) + 30 (entity-scoped fault poll) = 180 + # internal, plus the two harness self-tests (a handful of seconds against a local + # stand-in server, no ROS graph involved). Rounded to 300 to also cover the launch's own + # teardown for TWO processes (sigterm 30 + sigkill 15 each, not fully serial). + medkit_add_launch_test(test_node_death_e2e_raise + test/e2e/test_node_death_e2e.test.py TIMEOUT 300 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19310" "WATCHDOG_E2E_SCENARIO=raise" "${_WATCHDOG_E2E_ENV}") + + # 60 (arming, app_id-scoped - already implies presence) + 30 (departure) + 60 (raise + # poll) + 42.5 (presence poll after the respawn: DEPARTURE_TIMEOUT_SEC + RESPAWN_DELAY_SEC) + # + 60 (heal poll) = 252.5 internal. Rounded to 420 to also cover the launch's own teardown + # for TWO processes. + medkit_add_launch_test(test_node_death_e2e_clear_on_return + test/e2e/test_node_death_e2e.test.py TIMEOUT 420 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19320" "WATCHDOG_E2E_SCENARIO=clear_on_return" "${_WATCHDOG_E2E_ENV}") + + # Same drive as clear_on_return (60 arming, app_id-scoped + 30 faults-live + 30 + # departure + 60 raise poll + 42.5 presence-after-respawn = 222.5), plus the persistence + # window itself (20, SUSTAINED_WINDOW_SEC) instead of a heal poll = 242.5 internal. + # Rounded to 420 to also cover the launch's own teardown for TWO processes - same + # budget as clear_on_return since the internal sum is close and both drive one + # kill-then-return cycle. + medkit_add_launch_test(test_node_death_e2e_no_heal_standalone + test/e2e/test_node_death_e2e.test.py TIMEOUT 420 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19330" "WATCHDOG_E2E_SCENARIO=no_heal_standalone" "${_WATCHDOG_E2E_ENV}") + + # 60 (arming) + 30 (faults-live) + 30 (label poll to "active") + 30 (the ChangeState + # call itself) + 30 (separate label poll to "inactive") + 20 (silence window) + 15 + # (end-of-window label check) = 215 internal, 260 with the launch's own teardown for + # TWO processes - three independent 30 s budgets, not one combined 30 s. + medkit_add_launch_test(test_node_death_e2e_deactivated_not_dead + test/e2e/test_node_death_e2e.test.py TIMEOUT 300 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19340" "WATCHDOG_E2E_SCENARIO=deactivated_not_dead" "${_WATCHDOG_E2E_ENV}") + + # 60 (arming - proves the hybrid-mode manifest actually loaded rather than silently + # degrading to runtime_only) + 30 (faults-live) + 30 (presence poll for the one online + # app) + 30 (a SEPARATE presence poll for the never-online app's manifest-derived + # snapshot entry) + 20 (silence window) = 170 internal, 215 with the launch's own + # teardown for TWO processes - two independent 30 s budgets, not one. + medkit_add_launch_test(test_node_death_e2e_manifest_never_online + test/e2e/test_node_death_e2e.test.py TIMEOUT 240 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19350" "WATCHDOG_E2E_SCENARIO=manifest_never_online" "${_WATCHDOG_E2E_ENV}") + + # test_01 (load-bearing): 60 (arming) + 30 (faults-live) + three renamed-node arm/kill + # cycles (30 presence poll on GET /apps, visible only because this scenario's own + # discovery.runtime.filter_internal_nodes is off + 60 app_id-scoped arming poll + 30 + # departure poll + 15 process-reap budget each = 135 worst case, *3 = 405) + 20 (final + # silence window, SUSTAINED_WINDOW_SEC) = 515 internal. + # test_02: a constant comparison against the installed ros2cli package - no I/O, no + # polling, negligible budget. + # Rounded to 660 to also cover the launch's own teardown for the ONE launch-managed + # process (the gateway; this scenario declares no demo_nodes), plus however many + # renamed `demo_engine_temp_sensor` processes test_01 spawns and reaps itself across + # its three cycles (not launch-managed). + medkit_add_launch_test(test_node_death_e2e_ros2cli_ignored + test/e2e/test_node_death_e2e.test.py TIMEOUT 660 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19360" "WATCHDOG_E2E_SCENARIO=ros2cli_ignored" "${_WATCHDOG_E2E_ENV}") + + # 120 (two app_id-scoped arming gates, 60 each - one per colliding node, each already + # implying presence) + 30 (departure poll for the killed one) + 0 (the survivor's + # liveness check is a single non-blocking signal-0 probe, not a poll) + 60 (raise poll) + # + 5 (the describes-only window, MUTUAL_NAMING_WINDOW_SEC) = 215 internal. Rounded to + # 360 to also cover the launch's own teardown for THREE processes. + medkit_add_launch_test(test_node_death_e2e_bare_name_collision + test/e2e/test_node_death_e2e.test.py TIMEOUT 360 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19370" "WATCHDOG_E2E_SCENARIO=bare_name_collision" "${_WATCHDOG_E2E_ENV}") + + # 60 (arming) + 30 (faults-live) + 30 (presence poll) + 20 (silence window) + 5 + # (end-of-window presence re-check) = 145 internal. Rounded to 240 to also cover the + # launch's own teardown for TWO processes. + medkit_add_launch_test(test_node_death_e2e_fast_tick_floor + test/e2e/test_node_death_e2e.test.py TIMEOUT 240 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19380" "WATCHDOG_E2E_SCENARIO=fast_tick_floor" "${_WATCHDOG_E2E_ENV}") + + # 60 (arming, app_id-scoped - already implies presence) + THREE kill cycles (see + # RESTART_LOOP_OCCURRENCES_TARGET's own comment for why three), two of which also + # acknowledge and re-arm on the respawned instance (30 departure + 60 occurrence-count + # poll + 42.5 app_id-scoped re-arm after the respawn [DEPARTURE_TIMEOUT_SEC + + # RESPAWN_DELAY_SEC] = 132.5 each, *2 = 265) plus the third cycle's kill-and-confirm alone + # (30 + 60 = 90) + 30 (final record poll) = 445 internal. Rounded to 900, matching + # cap_pressure's own budget for a similarly multi-cycle scenario, to also cover the + # launch's own teardown for TWO processes plus CI-slowness margin. + medkit_add_launch_test(test_node_death_e2e_restart_loop_occurrences + test/e2e/test_node_death_e2e.test.py TIMEOUT 900 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19390" "WATCHDOG_E2E_SCENARIO=restart_loop_occurrences" "${_WATCHDOG_E2E_ENV}") + + # 60 (arming, app_id-scoped - already implies presence) + 30 (departure) + 60 (raise + # poll) + 30 (record poll) = 180 (test_01); 60 (port-down wait) + 90 (global arming gate + # after the restart - TARGET_NODE is gone by design here, so this cannot be app_id-scoped + # the way test_01's is) + 30 (faults-live, proving the restarted gateway reconnected to + # the fault_manager before the persistence window below could mean anything) + 5 + # (not-present check) + 20 (persistence window) + 30 (record poll) = 235 (test_02) - 415 + # internal. Rounded to 600, the same budget as the sibling detector's restart_departed + # scenario (an identically-shaped gateway-restart test), to also cover the launch's own + # teardown for TWO processes. + medkit_add_launch_test(test_node_death_e2e_restart_rebaseline + test/e2e/test_node_death_e2e.test.py TIMEOUT 600 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19400" "WATCHDOG_E2E_SCENARIO=restart_rebaseline" "${_WATCHDOG_E2E_ENV}") + + # === node_death suppression e2e (launch_testing) === + # + # Exercises the node_death detector's suppression framework (allowlist, self-suppression, + # pruning) against a real gateway + fault_manager + demo-node stack - see + # test/e2e/test_node_death_suppression_e2e.test.py's own module docstring for the full + # scenario list and what each row proves. + + # 60 (arming, app_id-scoped) + 30 (faults-live) + 30 (departure poll) + 20 (absence window, + # SUSTAINED_WINDOW_SEC) = 140 internal, plus the harness self-test for assert_fault_never_names + # (a handful of seconds against a local stand-in server, no ROS graph involved). Rounded to + # 240 to also cover the launch's own teardown for TWO processes. + medkit_add_launch_test(test_node_death_suppression_e2e_allowlist_suppresses + test/e2e/test_node_death_suppression_e2e.test.py TIMEOUT 240 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19410" "WATCHDOG_E2E_SCENARIO=allowlist_suppresses" + "${_WATCHDOG_E2E_ENV}") + + # 60 (arming, app_id-scoped) + 30 (departure poll) + 60 (raise poll) + 30 (startup-warning + # wait on the gateway's own stderr via proc_output, WARNING_TIMEOUT_SEC - unreachable today + # since the raise poll above fails first, but written and counted on its own budget) = 180 + # internal. Rounded to 300 to also cover the launch's own teardown for TWO processes. + medkit_add_launch_test(test_node_death_suppression_e2e_allowlist_not_named_is_inert + test/e2e/test_node_death_suppression_e2e.test.py TIMEOUT 300 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19420" "WATCHDOG_E2E_SCENARIO=allowlist_not_named_is_inert" + "${_WATCHDOG_E2E_ENV}") + + # 60 (global arming) + 30 (faults-live) + 60 (watchdog-entity poll, CLEAN_NODE "unconfigured") + # + 60 (watchdog-entity poll, KILLED_NODE "active") + 60 (the ChangeState call itself: up to + # 30 waiting for the service plus another 30 for the response - _call_change_state_once pays + # both, not one) + 30 (watchdog-entity poll, CLEAN_NODE "finalized") + 30 (departure poll, + # CLEAN_NODE) + 30 (departure poll, KILLED_NODE) + 60 (raise poll) + 20 (describes-only window, + # SUSTAINED_WINDOW_SEC) = 440 internal - ten independent budgets, not merged. Rounded to 720 + # to also cover the launch's own teardown for THREE processes (gateway plus both lifecycle + # fixtures). + medkit_add_launch_test(test_node_death_suppression_e2e_lifecycle_clean_shutdown + test/e2e/test_node_death_suppression_e2e.test.py TIMEOUT 720 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19430" "WATCHDOG_E2E_SCENARIO=lifecycle_clean_shutdown" + "${_WATCHDOG_E2E_ENV}") + + # 60 (arming, app_id-scoped) + 30 (departure poll) + 60 (raise poll) = 150 internal. Rounded + # to 240 to also cover the launch's own teardown for TWO processes. + medkit_add_launch_test(test_node_death_suppression_e2e_suppression_is_opt_in + test/e2e/test_node_death_suppression_e2e.test.py TIMEOUT 240 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19440" "WATCHDOG_E2E_SCENARIO=suppression_is_opt_in" + "${_WATCHDOG_E2E_ENV}") + + # Two independent nodes (TARGET_NODE, allowlisted; SECOND_NODE, not) - see the module + # docstring's own note on why a single unsuppressed node cannot exercise pruning at all. + # 60+60 (arming, app_id-scoped, one per node) + 30 (faults-live) + 40 (baseline settle, + # STABLE_TRACKED_COUNT_TIMEOUT_SEC - node_death's own tick has to catch up to the gate + # before the reclaim delta below means anything, see _poll_stable_tracked_count) + 30+30 + # (departure poll, one per node) + 60 (raise poll, SECOND_NODE) + 60 (reclaim poll on + # detectors.node_death.tracked_count via GET /x-medkit-watchdog - PRUNE_GRACE ticks elapse + # comfortably inside it) + 20 (describes-only window, SUSTAINED_WINDOW_SEC) = 390 internal. + # Rounded to 600 to also cover the launch's own teardown for THREE processes. + medkit_add_launch_test(test_node_death_suppression_e2e_prune_no_false_heal + test/e2e/test_node_death_suppression_e2e.test.py TIMEOUT 600 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19450" "WATCHDOG_E2E_SCENARIO=prune_no_false_heal" + "${_WATCHDOG_E2E_ENV}") + + # === node_death boundary, config and instrument e2e (launch_testing) === + # + # Exercises the seam between node_death and lifecycle_expectation, plus config/instrument + # boundary cases, against a real gateway + fault_manager + demo-node stack - see + # test/e2e/test_node_death_boundary_e2e.test.py's own module docstring for the full scenario + # list and which rows (B1, B3) are satisfied by lifecycle_expectation alone. + + # 60 (global arming) + 30 (faults-live) + 30 (presence poll) + 60 (raise poll, INACTIVE) + 20 + # (absence window, DISAPPEARED, SUSTAINED_WINDOW_SEC) + 5 (end-of-window presence recheck) = + # 205 internal. Rounded to 300 to also cover the launch's own teardown for TWO processes. + medkit_add_launch_test(test_node_death_boundary_e2e_b1_inactive_present + test/e2e/test_node_death_boundary_e2e.test.py TIMEOUT 300 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19460" "WATCHDOG_E2E_SCENARIO=b1_inactive_present" + "${_WATCHDOG_E2E_ENV}") + + # 60 (global arming) + 30 (faults-live) + 60 (tracked_nodes==1 poll) + 30 (departure poll) + + # 20 (absence window, INACTIVE, SUSTAINED_WINDOW_SEC) + 60 (raise poll, DISAPPEARED - + # unreachable today since the absence window above fails first, but written and counted on + # its own budget) = 260 internal. Rounded to 360 to also cover the launch's own teardown for + # TWO processes. + medkit_add_launch_test(test_node_death_boundary_e2e_b2_inactive_below_grace_then_gone + test/e2e/test_node_death_boundary_e2e.test.py TIMEOUT 360 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19470" "WATCHDOG_E2E_SCENARIO=b2_inactive_below_grace_then_gone" + "${_WATCHDOG_E2E_ENV}") + + # 60 (global arming) + 30 (faults-live) + 60 (raise poll, INACTIVE) + 30 (departure poll) + 20 + # (persistence window, INACTIVE, SUSTAINED_WINDOW_SEC) + 60 (raise poll, DISAPPEARED) = 260 + # internal. Rounded to 360 to also cover the launch's own teardown for TWO processes. + medkit_add_launch_test(test_node_death_boundary_e2e_b3_matured_then_gone + test/e2e/test_node_death_boundary_e2e.test.py TIMEOUT 360 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19480" "WATCHDOG_E2E_SCENARIO=b3_matured_then_gone" + "${_WATCHDOG_E2E_ENV}") + + # 60 (arming, app_id-scoped) + 30 (faults-live) + 30 (departure poll) + 20 (absence window, + # INACTIVE, SUSTAINED_WINDOW_SEC) + 60 (raise poll, DISAPPEARED) = 200 internal. Rounded to + # 300 to also cover the launch's own teardown for TWO processes. + medkit_add_launch_test(test_node_death_boundary_e2e_b4_healthy_then_gone + test/e2e/test_node_death_boundary_e2e.test.py TIMEOUT 300 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19490" "WATCHDOG_E2E_SCENARIO=b4_healthy_then_gone" + "${_WATCHDOG_E2E_ENV}") + + # 60 (global arming) + 30 (faults-live) + 30 (presence poll) + THREE restart cycles (see + # B5_CYCLES's own comment for why three), each (30 departure poll + 60 raise poll + 42.5 + # presence-after-respawn poll [DEPARTURE_TIMEOUT_SEC + B5_RESPAWN_DELAY_SEC] + 60 clear + # poll = 192.5), *3 = 577.5, plus the 120 preamble = 697.5 internal - four independent + # per-cycle budgets, not one merged figure, times three cycles. B5_RESPAWN_DELAY_SEC + # (12.5 s) replaces the launch-wide RESPAWN_DELAY_SEC (1.5 s) for this scenario alone: the + # outage has to stay safely above the detector's own wall-clock floor for every cycle to be + # reportable by a correct implementation - see that constant's own comment for the margin + # arithmetic. Rounded to 1500, well past the 900 this package's own restart-loop-shaped + # scenarios already use for a smaller sum, to also cover the launch's own teardown for TWO + # processes plus CI-slowness margin. + medkit_add_launch_test(test_node_death_boundary_e2e_b5_restart_loop_still_caught + test/e2e/test_node_death_boundary_e2e.test.py TIMEOUT 1500 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19500" "WATCHDOG_E2E_SCENARIO=b5_restart_loop_still_caught" + "${_WATCHDOG_E2E_ENV}") + + # 60 (global arming) + 30 (faults-live) + 30 (presence poll) + 60 (tracked_nodes==1 poll) + + # 30 (departure poll) + 60 (raise poll, INACTIVE) + 20 (absence window, DISAPPEARED, + # SUSTAINED_WINDOW_SEC) = 290 internal. Rounded to 360 to also cover the launch's own + # teardown for TWO processes. + medkit_add_launch_test(test_node_death_boundary_e2e_b6_never_armed_below_grace_then_gone + test/e2e/test_node_death_boundary_e2e.test.py TIMEOUT 360 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19530" "WATCHDOG_E2E_SCENARIO=b6_never_armed_below_grace_then_gone" + "${_WATCHDOG_E2E_ENV}") + + # 60 (arming, app_id-scoped) + 30 (faults-live) + 30 (departure poll) + 25 (early silence + # window, C4_EARLY_WINDOW_SEC, needle-scoped) + 100 (late raise poll, + # C4_LATE_RAISE_TIMEOUT_SEC) = 245 internal - five independent budgets, not merged. One + # gateway, one target, one domain: rounded to 360 to also cover the launch's own teardown for + # TWO processes. + medkit_add_launch_test(test_node_death_boundary_e2e_c4_config_endpoint_e2e + test/e2e/test_node_death_boundary_e2e.test.py TIMEOUT 360 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19510" "WATCHDOG_E2E_SCENARIO=c4_config_endpoint_e2e" + "${_WATCHDOG_E2E_ENV}") + + # test_01: 60 (arming, app_id-scoped) + 30 (departure poll) + 60 (raise poll) + 30 (record + # poll) = 180. test_02: 60 (port-down wait) + 30 (faults-live gate BEFORE the ungated window + # opens - a restarted gateway's HTTP server is not up the instant the old port goes down, + # confirmed live) + 3.5 (ungated pre-arm window, D2_UNGATED_WATCH_SEC) + 90 (global arming + # after the restart - TARGET_NODE is gone by design here, so this cannot be app_id-scoped) + + # 30 (faults-live, again, gating the persistence window below) + 5 (not-present check) + 20 + # (persistence window, SUSTAINED_WINDOW_SEC) + 30 (record poll) = 268.5 - 448.5 internal, nine + # independent budgets across both test methods, not merged. Rounded to 660 to also cover the + # launch's own teardown for TWO processes. + medkit_add_launch_test(test_node_death_boundary_e2e_d2_ungated_clear + test/e2e/test_node_death_boundary_e2e.test.py TIMEOUT 660 + LABELS "integration;e2e" + ENV "GATEWAY_TEST_PORT=19520" "WATCHDOG_E2E_SCENARIO=d2_ungated_clear" + "${_WATCHDOG_E2E_ENV}") + ros2_medkit_relax_vendor_warnings() endif() diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/README.md b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/README.md index 5d878b839..344467c43 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/README.md +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/README.md @@ -10,10 +10,11 @@ and is driven by the plugin's own executor from the tick thread, never by the ga ROS executor. This package carries the plugin skeleton, the central reliability gate that holds raises -until the graph has quiesced, and four detectors, `qos_mismatch`, `orphan`, -`param_drift` and `lifecycle_expectation`. The remaining -silent-fault classes land in follow-up changes, each against its own issue; their fault -codes are already reserved in the frozen `GRAPH_*` namespace (see "Fault codes"). +until the graph has quiesced, and five detectors, `qos_mismatch`, `orphan`, +`param_drift`, `lifecycle_expectation` and `node_death`. Two silent-fault classes remain +undelivered, `GRAPH_TF_STALE` and `GRAPH_LATENCY_BUDGET`; they land in follow-up changes, +each against its own issue, and their fault codes are already reserved in the frozen +`GRAPH_*` namespace (see "Fault codes"). ## Build @@ -53,10 +54,69 @@ id itself is warned about the same way, with the registered ids listed. |-----|------|---------|---------| | `mode` | string \| bool | `raise` | As above. | | `require_active` | string[] | `[]` | Node names that must be in the `active` lifecycle state, each matched against a live node by its `App::id`, its full FQN (`/ns/name`), or the bare leaf of that FQN. A bare name matches that node in EVERY namespace (a fleet-wide "all `controller_server`s must be active"); use a full FQN to pin one robot's. Both the bare and the FQN form match against the node's stable FQN, so they survive the `App::id` renaming that a same-bare-name collision triggers; an entry written as an `App::id` also works, but can stop matching on a multi-robot graph for exactly that reason. Empty = the detector checks nothing and emits nothing. A `require_active` that is not a string array warns and is ignored; an empty-string entry warns and is skipped - never dropped silently, since the operator would go on believing the node is covered. | -| `grace` | int | `5` | Consecutive not-active ticks a required node tolerates before being reported inactive - bringup time for a managed node to reach `active` (configure + activate) before the expectation is enforced. Counted per NODE, not per matching entry: a node named by both a bare-name and a full-FQN entry still advances its streak once per tick, so mixing the two documented forms cannot halve the grace that was configured. Past the absence grace the streak keeps advancing while the node is ABSENT, so a node that leaves the graph while violating is still confirmed. Accepted range 0..300 - five minutes at the shipped 1 s cadence, and already an extravagant allowance for a managed node to reach `active`. The upper end is a real bound rather than a formality: `grace` also decides how long a node that LEFT the graph while not-active sits unsettled in the tracker, and `GRAPH_NODE_INACTIVE`'s clear is withheld for EVERY node while it does, so at the old maximum of `INT_MAX - 1` the fault could neither raise nor heal for about 24 days. The check runs on the wide integer, so a value that only fits after truncation is rejected rather than silently turned into a hair-trigger. Anything outside the range warns and keeps the default. | +| `grace` | int | `5` | Consecutive not-active ticks a required node tolerates before being reported inactive - bringup time for a managed node to reach `active` (configure + activate) before the expectation is enforced. Counted per NODE, not per matching entry: a node named by both a bare-name and a full-FQN entry still advances its streak once per tick, so mixing the two documented forms cannot halve the grace that was configured. An ALREADY-CONFIRMED streak keeps its content while the node is ABSENT, so a node that leaves the graph after being confirmed stays confirmed; a streak that has not yet crossed `grace` on a node the presence detector could also report (it was armed at least once) is HELD rather than advanced while absent, so a node is never confirmed out of ticks gathered while nobody could observe it - only a PRESENT tick may still cross `grace` for that node. A node that was NEVER armed gets no such hold: `node_death` can never track it either, so absence keeps advancing its streak too, the same as a present tick would. Accepted range 0..300 - five minutes at the shipped 1 s cadence, and already an extravagant allowance for a managed node to reach `active`. The upper end is a real bound rather than a formality: `grace` decides how long a PRESENT node may go unreported, and how long a node returning from a departure still inactive takes to (re-)mature, so at the old maximum of `INT_MAX - 1` that PRESENT-side silence lasted about 24 days at the shipped cadence with no warning. Whether it also bounds how long a node that leaves the graph BELOW `grace` sits unsettled depends on the same split: for an armed node that lasts for as long as the node stays gone, independent of `grace` (see "Bounded by evidence, not by age" below); for a never-armed one `grace` bounds that too, since absence matures it within `grace + absence_grace + 1` ticks either way. The check runs on the wide integer, so a value that only fits after truncation is rejected rather than silently turned into a hair-trigger. Anything outside the range warns and keeps the default. | | `prune_grace` | int | `60` | Consecutive ticks an IDLE tracked node - both clocks at zero and no matured ownership, so nothing to lose - may stay ABSENT from the graph before its bookkeeping is reclaimed. A node carrying evidence is never reclaimed by age at all, however long it stays gone, so this key cannot erase a fault's own state; the map is bounded instead by `tracked_node_cap` (see "Bounded by evidence, not by age"). Injected for every detector at plugin scope and overridable per detector; a value outside 0..3600 warns and this detector keeps its own default of 60. The range check runs on the wide integer, so an out-of-int-range value is rejected rather than truncated. The value is used as written - there is no `grace + 1` clamp, because there is no longer anything for one to protect. | | `tracked_node_cap` | int | `512` | The most nodes this detector keeps bookkeeping for at once. It is what bounds the map, since evidence is never reclaimed by age (see "Bounded by evidence, not by age"): at the cap, idle entries are reclaimed first, then entries for DEPARTED nodes are collapsed into a count, and only if every tracked node is PRESENT and carrying evidence is a newly matched node refused - which withholds `GRAPH_NODE_INACTIVE`'s clear and is reported both in the log and on `GET /x-medkit-watchdog`. 512 is comfortably above every node in a realistic graph (a full Nav2 stack plus perception is roughly a hundred nodes; a ten-robot fleet sharing one domain a few hundred), so raising it is only needed where `require_active` legitimately matches more PRESENT nodes than that. Accepted range 1..16384 - 16384 is about 8 MB of bookkeeping at roughly 500 bytes per tracked node, and a deployment needing more distinct required-node identities alive at once has identity churn rather than a large fleet. Anything outside the range, including 0 (a cap of nothing would mean checking nothing), warns and keeps the default; the check runs on the wide integer, so an out-of-int-range value is rejected rather than truncated. | +### `node_death` keys + +Zero-config, unlike every detector above: there is no `require_active`-style list of nodes +to watch, because every armed App in the graph is a candidate. + +| Key | Type | Default | Meaning | +|-----|------|---------|---------| +| `mode` | string \| bool | `raise` | As above. | +| `miss_grace` | int | `2` | Consecutive missed ticks a tracked node tolerates before being reported dead - a death is reported once misses EXCEED this, i.e. after `miss_grace + 1` ticks. Accepted range 0..3600, and additionally floored to whatever wall-clock window the configured `tick_interval_ms` needs (see "The wall-clock floor" below); a value below that floor is silently raised to it, with a warning naming the ms window it actually spans. | +| `prune_grace` | int | `60` | Plugin-injected default (overridable here); how long a key being DURABLY suppressed (see "Suppression" below) may sit unreported before its bookkeeping is reclaimed. Accepted range 0..3600. The value actually used is `max(prune_grace, miss_grace + 1)`, a silent internal floor rather than a rejection: without it, a durably-suppressed key could be reclaimed the very tick it would first have become eligible to report, losing the report rather than merely suppressing it. | +| `allowlist` | string[] | `[]` | Node identities never reported dead. A dead key matches if it - or a name the same node is also known by - is present verbatim: the key's own fqn, its bare leaf, or (captured while the node was still present) its `App::id`. Has no effect unless named in `suppress`; naming it without a configured list is a no-op. | +| `suppress` | string[] | `[]` | Suppression mechanisms to activate: `"allowlist"` and `"lifecycle"`. Any other entry warns as unrecognized and is ignored; a non-string or non-array entry warns the same way. | +| `tracked_node_cap` | int | `512` | The most DEPARTED (not-yet-confirmed-dead or already-collapsed) identities this detector keeps individual bookkeeping for at once - a present/armed node is never counted against it and never refused tracking. Accepted range 1..16384; `0` is refused rather than clamped, since a cap of nothing would mean tracking nothing. See "Bounded by evidence, not by age" below for what it actually bounds and why, unlike the sibling detector's identical-looking cap, a newcomer here is never the thing refused. | + +`tick_interval_ms` is not an own key of this detector, but this is the one detector that +reads the plugin-injected value directly, to compute `miss_grace`'s wall-clock floor below. + +**Liveness, and why membership in the snapshot is not enough.** `App::is_online` is what +this detector tracks, never mere presence in the entity snapshot. In runtime-only discovery +the two happen to coincide - a dead node's App leaves the snapshot entirely - but a manifest +keeps a bound App present with only `is_online` cleared once its ROS binding disappears, and +hybrid discovery inherits that same shape; counting snapshot membership alone would make a +manifest-declared node immortal. A managed lifecycle node that merely deactivates keeps +`is_online: true` - its process is still running, only its ROS 2 lifecycle state changed - +so a deactivation is never mistaken for a death either; that is `lifecycle_expectation`'s +concern, not this one's (see "The boundary with `lifecycle_expectation`" below). + +Tracking is keyed on the STABLE fqn (`App::effective_fqn()`), never `App::id`: an id is +recomputed every sweep and only gains a namespace prefix once a same-bare-name collision +currently exists anywhere in the graph, so a live node's id can change out from under a key +built from it. + +**What is never tracked.** + +| Excluded | Why | +|---|---| +| Peer-aggregated apps (`app.source` starting `peer:`) | A peer app carries no ROS binding of its own, so `effective_fqn()` is empty for every one of them; tracking them would collapse an entire peer fleet onto one `""` key, and one online peer app would mask every other peer app's departure. | +| `_ros2cli_` nodes | Every `ros2` CLI invocation (`ros2 topic echo`, `ros2 param get`, ...) spins up a real, short-lived node under this prefix - rcl's own hidden-node naming convention, matched structurally rather than guessed at. Letting these through would accumulate one permanently "dead" entry per CLI invocation for the life of the gateway. | +| An App that never comes online | A key is admitted to tracking only once it has been armed by the reliability gate at least once. A manifest App whose binding never starts is never armed and so can never be falsely called dead; a node still inside its `warmup_cycles` window gets the identical protection. Once tracked, though, a node's continued life-or-death judgement rests on PRESENCE alone, not on staying armed - a tracked node that later goes lifecycle-inactive is not thereby mistaken for dead. | + +Composable/component nodes hosted in one process die together: if the container process +dies, every node it hosted leaves the snapshot on the same tick, and all of them are folded +into the one aggregated fault rather than raising one fault each - individually named up to +the description's 480-character cap and up to 3 at once under `tracked_node_cap` pressure, +whichever runs out first; past either limit the remainder still count toward the fault, just +not by name (see "Bounding the tracked-key map" below). + +**The wall-clock floor on `miss_grace`.** The entity snapshot a tick reads is not rebuilt +every tick: runtime discovery rebuilds it off a graph event debounced to about one refresh +per second, so several consecutive ticks between two refreshes see the IDENTICAL snapshot. A +`miss_grace` counted purely in ticks does not count independent samples of the graph - +shorten `tick_interval_ms` enough and one stale cache generation gets re-counted as several +misses. The floor raises `miss_grace`, for the configured `tick_interval_ms`, to the +smallest value whose `(miss_grace + 1) * tick_interval_ms` window is still at least 3000 ms - +one graph-cache refresh cycle plus margin. At the shipped 1000 ms tick the floor is exactly +the shipped default (`miss_grace: 2`), so the default configuration is unaffected; only a +faster-than-default tick ever raises the effective grace, and it does so with a warning +naming the window it actually spans. + ### `orphan` keys | Key | Type | Default | Meaning | @@ -603,8 +663,8 @@ lifecycle state (`inactive`/`unconfigured`/`finalized`), or its lifecycle promis not be MEASURED at all - a managed node whose label has never been read, or a node with no tracked lifecycle whatsoever. On a Nav2 stack a `controller_server` stuck inactive means the robot silently will not act - no crash, no log, nothing on `/diagnostics` or -`/rosout`. Presence itself is a different fault class (`GRAPH_NODE_DISAPPEARED`, reserved -in the frozen namespace for a follow-up change): this detector only ever STARTS reporting a +`/rosout`. Presence itself is a different fault class (`GRAPH_NODE_DISAPPEARED`, this +package's own `node_death` detector): this detector only ever STARTS reporting a node it has measured while PRESENT. A node that leaves the graph having only ever been measured healthy is left entirely to the presence class - but one that leaves while already under one of these three faults keeps it, because a node the operator declared must-be- @@ -659,7 +719,9 @@ node's SETTLED observation had started, and resets nothing: | Settled observation | What sustained absence does | |---|---| -| `inactive` (or any non-active label) | the violation streak keeps climbing, so the node is eventually reported under `GRAPH_NODE_INACTIVE` even though it is gone | +| `inactive`, ALREADY reported under `GRAPH_NODE_INACTIVE` | the streak continues exactly as it did on its last present tick, so the fault stays raised and keeps naming a node that is no longer there | +| `inactive`, not yet past `grace`, node WAS armed at some point | the streak is HELD - neither advanced (no fault is born while nobody can observe the node) nor erased (a node that returns still inactive resumes rather than re-earning `grace`). The presence detector could report this exact departure instead (`node_death` tracks any node the reliability gate has armed at least once), so this detector does not need to. | +| `inactive`, not yet past `grace`, node was NEVER armed | the streak keeps climbing on absence exactly as it would on a present tick. `node_death` only ever tracks a node the gate has armed, so a node that never reached that bar is structurally invisible to it no matter what happens afterwards - if this detector held the streak too, the departure would be reported by nothing at all. | | unread label, or no tracked lifecycle | the unmeasured clock keeps climbing under that same cause, so the node is eventually reported under `GRAPH_NODE_UNREADABLE` / `GRAPH_NODE_NOT_MANAGED` | | `active` | nothing. Anything the node had started but not corroborated is released, the entry becomes idle and is reclaimed silently | @@ -681,11 +743,13 @@ leaves has corroborated that long before the hold that reports it. Three consequences worth stating plainly: -- **A departure never heals a fault, within one gateway lifetime.** A node measured - not-active, or corroborated as unmeasurable, that then leaves the graph keeps its fault - - the operator declared it must be ACTIVE, it was not, and being gone is not an answer. The - fault's description switches to saying the node has since left the graph, so nobody is - sent looking for it. **Across a gateway restart it does not hold**, and that is a real +- **A departure never heals a fault it has already earned, within one gateway lifetime.** A + node CONFIRMED not-active, or corroborated as unmeasurable, that then leaves the graph + keeps its fault - the operator declared it must be ACTIVE, it was not, and being gone is + not an answer. The fault's description switches to saying the node has since left the + graph, so nobody is sent looking for it. A node that had NOT yet been confirmed when it + left earns nothing from leaving either - see "Absence continues the last CORROBORATED + observation" above. **Across a gateway restart it does not hold**, and that is a real boundary rather than an oversight: the restarted detector has no measurements at all, the departed node is not in the graph, and its `require_active` entry therefore matches nothing - which is indistinguishable from a misspelt entry, since the only component that @@ -698,19 +762,41 @@ Three consequences worth stating plainly: - **A departure never STARTS one.** A node measured `active` that shuts down raises nothing, and neither does one whose lifecycle services went missing from a sweep or two immediately before it did. Reporting a healthy node that left is the presence class's job - (`GRAPH_NODE_DISAPPEARED`), which still has no detector in this package - the one gap - left here, and a much narrower one than "a node absent past the grace is invisible - whichever clock it was on". + (`GRAPH_NODE_DISAPPEARED`, this package's own `node_death` detector), not this one's - + a much narrower boundary than "a node absent past the grace is invisible whichever clock + it was on". - **A departed node never crowds out a present one.** Entries for departed nodes are collapsed into a count when the tracked-node cap needs the slot - see "Bounded by evidence, not by age" below. -Why: every erasure horizon is an evasion for a node that touches it periodically. A node -in a restart loop - start, crash, respawn delay, start - touches absence by construction, -and that is the node this detector most exists to catch. Discarding its evidence on -absence let it alternate `(unreadable, absent x N)`, `(not-managed, absent x N)` or -`(inactive, absent x N)` forever without ever accumulating enough of anything to be -reported. +Why, for the two UNMEASURED causes: every erasure horizon is an evasion for a node that +touches it periodically. A node in a restart loop - start, crash, respawn delay, start - +touches absence by construction, and discarding its evidence on absence would let it +alternate `(unreadable, absent x N)` or `(not-managed, absent x N)` forever without ever +accumulating enough of anything to be reported. + +The VIOLATION streak used to need the identical rule for the identical reason - a node +alternating `(inactive, absent x N)` would otherwise never accumulate `grace` + 1 either. +Where the presence class CAN also report the same departure, that is no longer the only +thing standing between such a node and invisibility: `GRAPH_NODE_DISAPPEARED` (this +package's `node_death` detector) independently reports it, whether or not the node was +ever measured not-active first. But `node_death` only ever tracks an App once the +reliability gate has armed it, and the gate refuses to arm a MANAGED node that is not +`active` (`LifecycleWatcher::node_ok()`) - so a `require_active` node that never reaches +`active` is never armed, is never tracked by `node_death`, and its departure can never +raise `GRAPH_NODE_DISAPPEARED`, whatever kills it. The tracker therefore splits on whether +a node was EVER armed, read from the reliability gate itself rather than guessed from the +observed label: once armed, a below-`grace` streak that goes absent is simply HELD - +maturing it would raise `GRAPH_NODE_INACTIVE` from ticks gathered while nobody could +observe the node, a fault its presence never earned, and `GRAPH_NODE_DISAPPEARED` is there +to report the departure instead. A node that was NEVER armed gets no such backstop - +`GRAPH_NODE_DISAPPEARED` structurally cannot report it - so absence keeps advancing its +streak exactly as a present tick would, maturing it within the same bound an armed node's +UNMEASURED clock already has (`grace` + `absence_grace` + 1 ticks). An ALREADY-matured +streak still continues through absence exactly as before, for either kind of node: two +codes standing at once for the same node (`GRAPH_NODE_INACTIVE` because it was measured +not-active, `GRAPH_NODE_DISAPPEARED` because it is gone) is not a problem to fix - they say +different, both-true things. **Content follows the clocks, not the snapshot.** Whether a node happened to be in this tick's matches decides nothing about what it reports: a node whose streak is past `grace` @@ -792,14 +878,25 @@ at a different N. So: in the same tick, never partially. There is no `grace + 1` clamp any more, and none is needed: a node carrying evidence is exempt by construction rather than by arithmetic, so `prune_grace: 0` means exactly 0. -- **A non-idle entry is never pruned by age at all.** It cannot grow without bound in time - either: past the absence grace its clock advances every tick, so it matures within at most - `grace + absence_grace + 1` ticks (a violation streak) or `60 + absence_grace + 1` (an - unmeasured clock) and gets reported. Those two numbers are also the longest - `GRAPH_NODE_INACTIVE`'s clear can be withheld by one departed node, which is why `grace` is - capped at 300 rather than accepted up to the int maximum: at the old maximum the bound was - about 24 days at the shipped cadence, i.e. the fault could neither raise nor heal for - anybody, with no warning and no way to tell it from a working detector finding nothing. +- **A non-idle entry is never pruned by age at all.** An entry whose UNMEASURED clock is + still climbing cannot grow without bound in time either: past the absence grace it + advances every tick, so it matures within at most `60 + absence_grace + 1` ticks and gets + reported - the longest `GRAPH_NODE_UNREADABLE` or `GRAPH_NODE_NOT_MANAGED`'s clear can be + withheld by one departed node. A below-`grace` VIOLATION streak on a node that was NEVER + armed shares that same bound, for the same reason it advances at all while absent: it + matures within at most `grace + absence_grace + 1` ticks, because nothing else will ever + report that node's departure either. Only once a node HAS been armed does its below-grace + streak lose the bound: absence then holds it rather than advancing it, so it neither + matures nor becomes idle for as long as the node is away, however long that is. That is + not a new way to withhold `GRAPH_NODE_INACTIVE`'s clear - the clear was already gated on + every required node's status being settled, so one node this indecisive already blocked + it before this design; what changes is only that the fault never NAMES an ARMED node's + departure, since that node's evidence belongs to `GRAPH_NODE_DISAPPEARED` instead. `grace` + is still capped at 300 rather than accepted up to the int maximum, because it independently + bounds how long a PRESENT node may + go unreported and how long a returning node takes to re-mature: at the old maximum of + `INT_MAX - 1` that PRESENT-side bound was about 24 days at the shipped cadence, with no + warning and no way to tell a silently-withheld detector from one finding nothing. - **The map is bounded by `tracked_node_cap`** (default 512, accepted range 1..16384). At the cap, idle entries are reclaimed first (free - they carry nothing), then entries for DEPARTED nodes are collapsed into a count so a PRESENT node always wins a slot; only if @@ -886,13 +983,14 @@ re-established the node is fine. What never blocks `GRAPH_NODE_INACTIVE`'s raise: a RAISE is never withheld (a violation read from the nodes that did answer is real regardless of the unmeasured ones). An entry -that HAS matched and later stops matching does not re-enter reason 1 either. Every hold is -bounded, and every one of them releases by SETTLING the node's status rather than by -giving up on it: the never-matched hold and both unmeasured-clock causes release after 60 -consecutive ticks (a minute at the shipped 1s cadence), whether the node is present or -gone; a below-`grace` violation streak releases as soon as the node reads `active` or its -streak passes `grace` and the fault is raised - which, past the absence grace, happens -while the node is absent too. +that HAS matched and later stops matching does not re-enter reason 1 either. The +never-matched hold and both unmeasured-clock causes are bounded (60 consecutive ticks, a +minute at the shipped cadence) and release by SETTLING the node's status rather than by +giving up on it, whether the node is present or gone. A below-`grace` violation streak +releases the same way - as soon as the node reads `active`, or a PRESENT tick pushes its +streak past `grace` and the fault is raised - but while the node stays absent that release +has no timeout of its own: absence holds the streak rather than advancing it, so the hold +lasts for exactly as long as the node does not return. Because a withheld clear is indistinguishable from a detector that is working and finding nothing, a hold that lives past 10 consecutive ticks is explained in the log once per episode, naming every reason in force and, for the node-keyed ones, how many nodes @@ -998,12 +1096,20 @@ still fit inside the 480-char cap (`3 * 150 + 2 * 2 = 454 <= 480`). read and a not-managed one across absence gaps longer than the absence grace; that the violation streak survives non-maturing unmeasured ticks and RESUMES rather than restarting (counted exactly, and read through `pending_violation` - the only field that - differs during the climbing window); absence CONTINUING whichever clock the node's last - real observation started (a blink holds it unchanged; past the blink tolerance it - advances, so a node absent long enough matures or crosses grace on absence alone and its - detail says the node has left the graph), a matured fault surviving a departure, a node - measured ACTIVE that vanishes raising nothing and being reclaimed, and the three - `(X, absent x N)` restart-loop shapes for N at the absence grace and past it; the + differs during the climbing window); absence CONTINUING an already-matured fault exactly + as it was on its last present tick (a blink holds either clock unchanged; past the blink + tolerance an unmeasured clock still climbing keeps climbing and matures on absence alone, + its detail then saying the node has left the graph, while a below-grace violation streak + is instead HELD - neither advanced nor erased - and RESUMES rather than restarts once the + node returns, proven both from a single long absence and from many short ones + interleaved with matched reads), a matured violation fault surviving a departure, a node + measured ACTIVE that vanishes raising nothing and being reclaimed, and the two + `(unreadable/not-managed, absent x N)` restart-loop shapes for N at the absence grace and + past it (their `inactive` sibling now pinned to the opposite claim, at the same two N: it + never crosses grace from absence alone, for a node that was armed at some point - a node + that was NEVER armed instead matures from absence alone within `grace + absence_grace + 1` + ticks, exactly like the unmeasured clock does, and resumes rather than restarts on return + the same way an armed node's held streak does); the `pending` set and its per-reason breakdown; new-first ordering for all three fault-shaped maps, including the lexicographic tie-break when several cross together; the remote-supplied label's own trim budget ahead of the whole-detail backstop, and the @@ -1037,8 +1143,9 @@ still fit inside the 480-char cap (`3 * 150 + 2 * 2 = 454 <= 480`). warning, the withheld-clear guard's releases and its once-per-episode log line, the blink-plus-unread-re-seed sequence, the no-match warning, a filler batch sized (from the real detail-building code) to exceed the 480-char cap aggregating into one fault - with the fresh crossing named first, a PRESENT node crossing on the same tick as a batch - of departed ones being named ahead of them (and not truncated away by them), a required + with the fresh crossing named first, a PRESENT node crossing grace fresh while a batch of + already-departed, already-matured ones sits absent-and-content being named ahead of them + (and not truncated away by them), a required node appearing mid-run, a re-bind under the same `App::id`, and the reconfigure/config-validation edge cases. The unmeasured clock's own split is pinned directly, for BOTH codes symmetrically: a @@ -1058,8 +1165,11 @@ still fit inside the 480-char cap (`3 * 150 + 2 * 2 = 454 <= 480`). filler batch, the same new-first ordering; an already-reported node of either cause KEEPING its own record once it vanishes (and its description switching to say the node has left the graph), a node returning from that absence staying reported with no - clear/re-raise churn, each of the three restart-loop shapes raising its own code, the - `inactive` <-> `not-managed` alternation across absence gaps, a node measured ACTIVE + clear/re-raise churn, each of the two UNMEASURED restart-loop shapes raising its own code + from absence alone (their `inactive` sibling instead pinned to counting only real reads, + never crossing from any of the interleaved absence gaps), the + `inactive` <-> `not-managed` alternation across absence gaps now counting only the + MEASURED not-active legs, a node measured ACTIVE that vanishes raising nothing under any of the three, and a withheld `GRAPH_NODE_INACTIVE` clear releasing when the absent node's clock MATURES into its sibling's content rather than when the node is given up on; the two wire strings pinned as hand-typed literals @@ -1199,6 +1309,266 @@ still fit inside the 480-char cap (`3 * 150 + 2 * 2 = 454 <= 480`). crash loop is the case this detector most exists to catch, and it is the one that a design discarding evidence on absence makes permanently silent. +#### `node_death` (GRAPH_NODE_DISAPPEARED) + +Watches every armed App in the graph for one that goes offline, and raises +`GRAPH_NODE_DISAPPEARED` naming it. Zero-config, unlike `lifecycle_expectation`'s +operator-declared `require_active` list: every armed App is a candidate. See the +`node_death` key table and the "Liveness", "What is never tracked" and "The wall-clock +floor on `miss_grace`" notes above the `orphan` key table for what counts as alive, what +this detector never tracks, and why `miss_grace` has a floor - none of that is repeated +here. + +**Suppression.** Nothing is suppressed unless the operator names the mechanism in +`suppress` - a configured `allowlist` that `suppress` does not name has NO effect, and a +startup warning says so. Two mechanisms exist, activated independently of each other: + +| Name | Mechanism | Durable | +|---|---|---| +| `"allowlist"` | `AllowlistSuppressor` over the configured `allowlist` set - the same three-way match (fqn, bare leaf, `App::id`) `lifecycle_expectation`'s `require_active` uses, so an operator who has one working can expect the other to accept the same shapes. Exact match only, never a prefix or a shared suffix. | Yes | +| `"lifecycle"` | `LifecycleShutdownSuppressor` - suppresses a departure that the reliability gate classifies as a clean managed-lifecycle shutdown. See "Clean-shutdown suppression" below for what counts as clean. | Yes | + +Every field is re-validated from scratch on every `configure()` call: a malformed +`allowlist` entry, an unrecognized `suppress` name, or a `suppress`/`allowlist` entry of the +wrong type each produce their own named warning rather than being silently dropped. A +candidate is suppressed the moment ANY active mechanism votes yes, order-independent - +which one runs first never changes the result. Suppression is independent of `mode`: a +suppressed key never becomes content in the first place, while `mode` separately decides +whether non-suppressed content is actually sent (`advisory` observes without sending, +`off` disables). + +**Durability, and why only a durable veto may reclaim bookkeeping.** Whether a suppressor is +durable answers whether its "yes" is a standing fact about the key rather than a condition +that can later stop holding. Both mechanisms here are durable: an operator-declared allowlist +entry does not start matching and then stop on its own, and a departure's observed shutdown +shape does not change after the fact. Durability is what makes reclaiming a suppressed key's +tracker bookkeeping (`prune_grace` consecutive suppressed ticks) sound - a NON-durable veto +could lift later even after its key's bookkeeping was discarded, leaving a live, unsuppressed +death with nothing left to report it: a false heal that silently outlives the very condition +that produced it. A durable veto never lifts for a given key once it has fired, so reclaiming +under it loses nothing. Durable defaults to false, the safe assumption for a suppressor +nobody has reasoned about yet; both suppressors here override it explicitly to true. +Reclaiming is re-checked against the durable suppressors specifically, never merely "no +longer in this tick's report", because a key can be dropped from a tick's report by ANY +suppressor in the chain while a durable one also happens to cover it - whether a key may be +reclaimed depends on WHICH suppressor vetoed it, not on whether it survived the filter. + +**Clean-shutdown suppression - what counts as clean.** `LifecycleShutdownSuppressor` reads +the lifecycle watcher's cached label for the departed node's LAST observed transition: + +| Last observed label | Suppressed? | +|---|---| +| `shuttingdown` | Yes - only reachable via a deliberate SHUTDOWN transition, so the label alone is enough. | +| `finalized`, with an observed transition and never through the error branch | Yes. | +| `finalized`, with no observed transition, or reached through the error branch | No. | +| `unconfigured` | No, deliberately - it is also the resting state of a node whose `configure()` failed or that never activated at all, and suppressing on it would hide exactly that startup failure. | +| Any other label, or no departure on record at all | No (abstains). | + +`finalized` needs the extra corroboration because it is also where a node's `on_error` +override lands after `ON_ERROR_FAILURE`/`ON_ERROR_ERROR` out of `errorprocessing` - the +standard way a driver reports a hardware fault it cannot recover from, which is precisely the +death an operator needs reported, not silenced. Without an observed transition history the +departure is unclassified and stays reported - the safe direction. This mechanism is +stateless by construction: a detector's `configure()` (where the suppressor chain is built) +runs before any per-tick context exists, so it stores nothing at construction and reads the +gate fresh on every call. Its retention window is not this detector's own to size, either: +before this detector's own `configure()` has run, the plugin predicts the same +`prune_ticks_` (`max(prune_grace, miss_grace + 1)`) this detector will compute, then sizes +the lifecycle watcher's departed-node retention from it with enough margin that a +clean-shutdown label is still cached at this detector's own reclaim tick, one tick to +spare - see `GraphWatchdogPlugin::compute_departed_retention_ticks()` for the exact +arithmetic, which is larger than `prune_ticks_` alone. + +**Bounded by evidence, not by age.** This detector is zero-config over every armed App in +the graph - a strictly larger scope than `lifecycle_expectation`'s named `require_active` +set - so identity churn (nodes reappearing under ever-new names, one per run or per +namespace) would grow the tracker's map without a bound if nothing capped it. An +unsuppressed, still-dead entry is never reclaimed by age - `prune_grace` is the only reclaim +path there is, and it only ever applies to a durably-suppressed key - so `tracked_node_cap` +is what actually bounds memory: specifically, the DEPARTED subset of it (identities carrying +a nonzero miss count). A PRESENT entry never counts against the cap and is never evicted to +make room for anything, at any map size: it costs nothing to keep (an idle entry is simply +re-tracked a moment later if it is still armed), and it is bounded by the live graph, not by +churn, which is bounded by reality rather than by this cap. So a graph carrying far more +nodes than `tracked_node_cap` is not, by itself, capacity pressure at all - every one of them +is still tracked and every genuine death among them still reported; only a departed set that +is itself larger than the cap is. + +At that point, the cap collapses departed entries into one synthetic count, keeping at most +three individually named - but ONLY entries that have actually crossed `miss_grace` (a +confirmed death). An entry that is merely mid-grace is never a collapse candidate: folding it +into the count would report a death the node has not earned, permanently, since collapsing +erases the identity and a node returning cannot un-report what was never real. Under +sustained pressure from still-maturing churn the departed set may therefore exceed +`tracked_node_cap` for a few ticks - bounded by how fast new departures arrive times +`miss_grace`, not by how long the process has been running, which is the growth this cap +exists to prevent in the first place. `tracking_saturated` reports exactly this: the departed +set still exceeds `tracked_node_cap` even after collapsing every confirmed death it safely +could, because immature entries alone account for the excess. + +Unlike `lifecycle_expectation`'s identical-looking cap, a PRESENT/armed key here is never the +thing evicted to free a slot. `LifecycleExpectationTracker` carries two clocks per node, so a +node can be simultaneously present and mid-violation - a third state this tracker does not +have, and what makes evicting a present entry there lose nothing (the evidence lives in the +clocks). `NodeLivenessTracker` carries exactly one piece of per-key state (a miss count), so +a tracked identity is always either present-and-empty or absent-and-evidential; evicting a +present one here would only ever discard a key that could simply be re-admitted next tick +anyway, at the cost of losing any death that happens to land in the eviction window - only +ONLINE nodes re-enter `armed`, so an evicted-while-present node that then dies is never +re-admitted at all. The sibling's eviction order does not transfer. + +**The boundary with `lifecycle_expectation`.** `GRAPH_NODE_INACTIVE` and +`GRAPH_NODE_DISAPPEARED` can both be raised for the same node at the same time, and that is +CORRECT, not a defect this detector removes: `GRAPH_NODE_INACTIVE` says a required node is +present but not active, `GRAPH_NODE_DISAPPEARED` says a node is gone, and an operator facing +each has a different repair - reactivate it, or find out why it left. A node CONFIRMED +`GRAPH_NODE_INACTIVE` that then departs keeps that fault (its description switches to say +the node has since left the graph) AND now also raises `GRAPH_NODE_DISAPPEARED`, since this +detector tracks every armed App independently of whatever `lifecycle_expectation` thinks of +it. + +What changed is narrower than that, and it only applies to a node this detector could ever +have tracked. Before this detector existed, `lifecycle_expectation`'s own violation streak +let a SUSTAINED absence mature an unconfirmed (below-`grace`) streak into a confirmed +`GRAPH_NODE_INACTIVE` - the only way a node stuck in a restart loop, never observed long +enough to cross `grace` while present, would ever be reported at all. Where this detector +can independently report the same departure - which needs the node to have been ARMED at +least once, since `node_death` only ever tracks an App the reliability gate has armed, and +the gate refuses to arm a MANAGED node that is not `active` - that absence-alone maturity is +gone: absence CONTINUES a violation that has already matured past `grace` (the fault stays +raised, still naming the node), but no longer CREATES one that has not. A node that was +briefly non-active and then died is reported as gone (`GRAPH_NODE_DISAPPEARED`) rather than +also acquiring an inactive fault born from ticks gathered while nobody could observe it. A +`require_active` node that never reaches `active` is never armed, is never tracked here, and +its departure can never raise `GRAPH_NODE_DISAPPEARED` - for that one node, +`lifecycle_expectation` keeps the older behaviour: absence still matures a below-`grace` +streak, because it is structurally the only detector that will ever get to report it. See +`lifecycle_expectation`'s own "Absence continues the last CORROBORATED observation, it never +erases" section above for the mechanism. + +**Repeated failures: what `occurrence_count` and the captured evidence answer, and don't.** +An operator asking "how many times did this node die in the last hour" reads it off the +fault record itself, not off this detector: `GRAPH_NODE_DISAPPEARED`'s own +`occurrence_count` (`GET /api/v1/apps/graph_watchdog/faults`, or the scoped +`GET /api/v1/apps/graph_watchdog/faults/GRAPH_NODE_DISAPPEARED`) starts at 1 on the first +raise and increments by exactly one each time a FAILED report reactivates a record that was +CLEARED - never merely re-raised while still CONFIRMED, and never merely HEALED. Healing +(the fault manager's debounce counter crossing `healing_threshold` on clean sweeps, see +"Closing the loop" above) and clearing are different things: `DELETE +/api/v1/apps/graph_watchdog/faults/GRAPH_NODE_DISAPPEARED` - the fault manager's own +`~/clear_fault` underneath - is what acknowledges an occurrence and closes its cycle. A +still-CONFIRMED, or still-HEALED-but-unacknowledged, fault that fires FAILED again is the +SAME occurrence continuing, not a new one, so restarting the dead node fast enough to heal it +before anyone acknowledges it will not move the count. + +The honest limits, read off this branch's fault-manager storage rather than assumed: the +per-fault rosbag store enforces `fault_code` as UNIQUE, so a fault can hold at most one +recording at a time - a later confirmation's capture replaces the earlier one on disk rather +than accumulating a history. The freeze frame is the same shape: one row per `fault_code`, +overwritten on every capture, so a fifth occurrence's captured values overwrite the first's. +What survives every occurrence by default is the count itself; a hash-chained record of +every raise/clear/heal transition also exists, but only once the fault manager's own +`audit_log.enabled` is turned on, which it is not by default. Recordings and the freeze +frame do not accumulate a per-occurrence history either way. + +**A second death while the first is still outstanding gets no evidence of its own.** This +detector folds every currently-affected node into ONE `GRAPH_NODE_DISAPPEARED` record (see +"The boundary with `lifecycle_expectation`" above) - so a node dying while another's death is +still CONFIRMED is added to the description, but the report that adds it is a re-report on an +already-active fault, not a new occurrence: the same distinction "Repeated failures" above +draws for one node dying twice applies just as much across two different nodes sharing the +one code. `occurrence_count` does not move, no state transition fires, and - for the same +reason "The honest limits" above gives - neither a freeze frame nor a recording is captured +for the second node; only whichever death actually confirmed the record has either. +Acknowledging between the two deaths avoids this: `DELETE +/api/v1/apps/graph_watchdog/faults/GRAPH_NODE_DISAPPEARED` closes the first occurrence, so the +second node's next report reactivates a CLEARED record instead of updating a CONFIRMED one - a +genuine new occurrence, with its own `occurrence_count` and its own capture. + +**Test tiers.** + +1. **Unit**: `test_node_liveness_tracker.cpp` pins the pure presence/absence state machine - + a key present but never armed is never tracked; once armed, presence alone (not + continued arming) keeps a tracked key's miss counter at zero; a key is reported dead only + once misses exceed `miss_grace`; freshness ordering keeps a brand-new death out of a + capped description's blind spot; `prune()` reclaims a key only once it has been + suppressed for MORE than `prune_ticks` CONSECUTIVE calls, resets the streak the moment a + veto lifts even once, and never reclaims an unsuppressed death no matter how long it + stays dead. On the cap: a graph carrying ten times `tracked_node_cap` present keys is + never refused or evicted, and every genuine death among them is still individually + reported; two entries carrying only a below-grace miss each are never collapsed or + reported under cap pressure, however much the departed count exceeds the cap, and + `tracking_saturated` reports exactly that condition; matured (confirmed-dead) entries ARE + collapsed into a monotonically-accumulating count once the departed set exceeds the cap + and no present or immature entry is available to spare; and at `miss_grace: 0`, where + every departure matures instantly, saturation never fires at all - a narrower, honestly- + scoped property than a blanket "can never saturate" claim. `test_suppressor.cpp` + pins the `Suppressor` interface (`durable()` defaults false) and the free + `apply_suppressors()` helper (order-independent, null-safe, returns the dropped count) - + this detector's own filtering is hand-inlined rather than a call to that helper, because + it layers an id-form check the helper's plain string-keyed signature cannot express. + `test_allowlist_suppressor.cpp` and `test_lifecycle_shutdown_suppressor.cpp` pin each + suppressor's own matching rules independently of any detector. +2. **Integration** (`test_node_death_integration.cpp`): the full config contract + (`miss_grace`, `prune_grace` and `tick_interval_ms` range checks and their floor + interaction; malformed `allowlist`/`suppress` entries; `tracked_node_cap` range checks, + including a value far past `INT_MAX` rejected rather than wrapped; an unknown top-level + key), that a peer-aggregated app and an `is_online: false` app are never tracked, that + the tracked count stays bounded and shrinks after a reclaim under sustained identity + churn (and, without any suppression configured at all, that `tracked_node_cap` still + bounds memory under unsuppressed churn while the fault stays raised), that the + `prune_grace` clamp is not merely never-reclaimed-yet but actually reclaims once its + clamped horizon passes, that Advisory mode keeps accumulating misses (provable by + switching to Raise mode without letting the node return and observing an immediate raise) + rather than merely declining to send, that a clean managed-lifecycle departure is + suppressed exactly AT the `miss_grace` boundary and stays reclaimed - not re-raised - past + the retention window the plugin sizes for it, that the allowlist's id-form suppresses only + the colliding node and not its namesake, and the ungated-clear guard's four properties: a + stored fault stays unhealed while an unrelated node arms alone, a clear flows once this + process instance has itself handed a FAILED request to the fault client, the guard tracks + that handoff rather than intent (an attempted-but-never-sent raise never earns a clear), + and a reconfigure while the node is absent does not withhold a clear the process had + already earned. Three more rows sit past the cap specifically: 513 armed nodes against the + default `tracked_node_cap` (512) still report the one that dies; three below-grace entries + under `tracked_node_cap: 2` never fabricate a death and still mature into a genuine one + once the pressure passes; and a cap-forced collapse stays raised through a genuine + recovery but clears once reconfigured, mirroring the ungated-clear guard's own reconfigure + test for an ordinary (uncollapsed) fault. +3. **E2e**, three files, each launching its own real gateway + fault_manager + demo-node + stack: + - `test/e2e/test_node_death_e2e.test.py` (ten scenarios): raise and name the node; clear + once it returns; no heal with the fault manager's healing disabled; a lifecycle + DEACTIVATE is never mistaken for a death; a manifest app that never comes online is + never called dead; a `_ros2cli_*`-prefixed node is never tracked, checked against + ros2cli's own naming constant; a bare-name collision names only the node that actually + exited; a fast tick alone, nothing else perturbing the graph, raises nothing; a + three-cycle restart loop, acknowledged between cycles, reaches `occurrence_count: 3`; + and a fault confirmed before a gateway restart stays CONFIRMED across it. + - `test/e2e/test_node_death_suppression_e2e.test.py` (five scenarios): the allowlist + suppresses the node it names; the SAME allowlist left unnamed in `suppress` is inert, + and a startup warning says so; a managed lifecycle node that reached a clean shutdown + is never named while an active sibling that was simply killed still is; naming a node + on the allowlist alone, with `suppress` unset, does not suppress it; and an ALLOWLISTED + death (the only shape pruning ever applies to - see "Bounded by evidence, not by age") + held well past `prune_grace` is actually reclaimed (`tracked_count` on + `GET /x-medkit-watchdog` drops to 0, not merely "no fault ever appeared", which a + deleted `prune()` would also produce), while never once raising `GRAPH_NODE_DISAPPEARED` + for it - proving pruning a suppressed key's bookkeeping is not itself an event the + aggregate reacts to. + - `test/e2e/test_node_death_boundary_e2e.test.py` (eight scenarios, the seam with + `lifecycle_expectation`): a node stuck inactive but never gone is + `GRAPH_NODE_INACTIVE`'s alone; the same node, ARMED first, then killed while still below + `grace`, raises `GRAPH_NODE_DISAPPEARED` alone, with no `GRAPH_NODE_INACTIVE` ever born + from the departure; a node killed AFTER `GRAPH_NODE_INACTIVE` has confirmed raises BOTH + codes at once, the confirmed one still naming the node; a healthy node that is simply + killed raises `GRAPH_NODE_DISAPPEARED` alone; a restart-looping required node is caught + every cycle regardless of whether it ever matures under `lifecycle_expectation`; a node + that is NEVER armed, killed while still below `grace`, raises `GRAPH_NODE_INACTIVE` + alone instead - `node_death` cannot track a node the gate never armed, so absence has to + mature the violation here; a large `miss_grace` delays the report but does not swallow + it; and a restarted gateway's own warmup window never produces a spurious PASSED for a + node it has not yet re-measured. + ## Reliability (bringup-quiesce) Silent-fault detectors are prone to bringup noise: a node joining the graph, a @@ -1324,6 +1694,11 @@ two share one cause-blind unmeasured clock internally (see the detector section but are always two DISTINCT codes on the wire - the clock's blindness to which cause it is seeing never leaks into which fault code a node ends up reported under. +Of these nine, seven currently have a detector raising them: `qos_mismatch` +(`GRAPH_QOS_MISMATCH`), `orphan` (`GRAPH_ORPHAN`), `param_drift` (`GRAPH_PARAM_DRIFT`), +`node_death` (`GRAPH_NODE_DISAPPEARED`), and `lifecycle_expectation`'s three above. Two +remain undelivered: `GRAPH_TF_STALE` and `GRAPH_LATENCY_BUDGET`. + Splitting the unmeasured cases out of `GRAPH_NODE_INACTIVE` has two consequences nothing else states. Anything downstream that filters or correlates on `GRAPH_NODE_INACTIVE` alone no longer sees either unmeasured case at all - they live under diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/design/graph_watchdog.rst b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/design/graph_watchdog.rst index b51f8dc3b..9f79f3300 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/design/graph_watchdog.rst +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/design/graph_watchdog.rst @@ -138,8 +138,9 @@ Reliability core Detectors --------- -``qos_mismatch``, ``orphan``, ``param_drift`` and ``lifecycle_expectation`` are the -detectors this package ships so far. The remaining silent-fault classes land in +``qos_mismatch``, ``orphan``, ``param_drift``, ``lifecycle_expectation`` and +``node_death`` are the detectors this package ships so far. Two silent-fault classes +remain undelivered, ``GRAPH_TF_STALE`` and ``GRAPH_LATENCY_BUDGET``; they land in follow-up changes, each against its own issue. ``qos_mismatch`` raises ``GRAPH_QOS_MISMATCH``. It @@ -415,14 +416,24 @@ situation never changed. **Absence continues the last CORROBORATED observation, it never erases.** A node not matched at all this tick is ABSENT, a fact separate from any observed state (there is no label to classify). For up to ``absence_grace`` (a fixed 3 ticks) consecutive absent ticks a node's -whole state is held unchanged - the blink tolerance. Past ``absence_grace`` absence ADVANCES -whichever clock the node's SETTLED observation had started, and resets nothing: a -settled-``INACTIVE`` node keeps climbing its violation streak, a -settled-``UNREADABLE``/``NOT_MANAGED`` node keeps climbing its unmeasured clock under that -same cause, and a settled-``ACTIVE`` node advances nothing at all - anything it had started -but not corroborated is released, so the entry becomes idle and is reclaimed silently. So a -departure never heals a fault and never starts one; what it changes is the detail phrase, -which then says the node has since left the graph. +whole state is held unchanged - the blink tolerance. Past ``absence_grace`` absence resets +nothing, but what it ADVANCES depends on whether the node's SETTLED observation is already +CONTENT, and - for a below-grace violation streak only - on whether the node was ever ARMED: a +settled-``INACTIVE`` node ALREADY reported under ``GRAPH_NODE_INACTIVE`` keeps its streak +exactly as it was, so the fault stays raised, regardless of arming. One not yet past ``grace`` +splits on that fact: a node the presence detector COULD have tracked (armed at least once) is +HELD - neither advanced (no fault born from evidence gathered while nobody could observe the +node, since ``node_death`` is able to report this exact departure instead) nor erased (it +resumes rather than restarts on return) - while a node the presence detector could NEVER have +tracked keeps climbing on absence exactly as it would on a present tick, because nothing else +in the plugin will ever report its departure either. A settled-``UNREADABLE``/``NOT_MANAGED`` +node keeps climbing its unmeasured clock under that same cause regardless of maturity or +arming - that clock's own absence behaviour is unrelated to this distinction. A +settled-``ACTIVE`` node advances nothing at all - anything it had started but not corroborated +is released, so the entry becomes idle and is reclaimed silently. So a departure never heals a +fault it has already earned, and never starts one out of evidence gathered while the node +could not be observed AND could be reported some other way; what it changes, for an +already-raised fault, is the detail phrase, which then says the node has since left the graph. **What "settled" means, and why an unmeasured reading needs corroborating.** A real measurement (``ACTIVE`` or a non-active label) settles at once - a label is a fact about the @@ -451,15 +462,30 @@ record heals with nothing having been measured. Re-seeding the tracker from the startup would change that and is not implemented; the boundary is pinned by the ``restart_departed`` e2e scenario rather than left to be discovered. -Why the erasure went away rather than moving: every erasure horizon is an evasion for a -node that touches it periodically. A node in a restart loop - start, crash, respawn delay, -start - touches absence by construction, and that is the node this detector most exists to -catch, so discarding its evidence let it alternate ``(UNREADABLE, ABSENT x N)``, -``(NOT_MANAGED, ABSENT x N)`` or ``(INACTIVE, ABSENT x N)`` forever without ever -accumulating enough of anything to be reported. Reporting a HEALTHY node that left remains -the presence class's job (``GRAPH_NODE_DISAPPEARED``, still no detector of its own) - the -single gap this class keeps, and a far narrower one than "a node absent past the grace is -invisible whichever clock it was on". +Why the erasure went away rather than moving, for the two UNMEASURED causes: every erasure +horizon is an evasion for a node that touches it periodically. A node in a restart loop - +start, crash, respawn delay, start - touches absence by construction, and discarding its +evidence would let it alternate ``(UNREADABLE, ABSENT x N)`` or ``(NOT_MANAGED, ABSENT x N)`` +forever without ever accumulating enough of anything to be reported. + +The VIOLATION streak needed the identical rule for the identical reason once - a node +alternating ``(INACTIVE, ABSENT x N)`` would otherwise never accumulate ``grace`` + 1 either. +Where the presence class CAN also report the same departure, it no longer does: +``GRAPH_NODE_DISAPPEARED`` (this package's own ``node_death`` detector) independently +reports it, whether or not the node was ever measured not-active first. But ``node_death`` +only ever tracks an App once the reliability gate has armed it, and the gate refuses to arm a +MANAGED node that is not ``active`` (``LifecycleWatcher::node_ok()``) - so a +``require_active`` node that never reaches ``active`` is never armed, is never tracked by +``node_death``, and its departure can never raise ``GRAPH_NODE_DISAPPEARED``, whatever kills +it. So the split is on whether a node was EVER armed - a fact read from the reliability gate +itself, not guessed from the observed label: once armed, a below-``grace`` streak that goes +absent is simply HELD rather than matured on the strength of the absence alone - two codes +standing at once for the same node is not a problem to fix, it is ``GRAPH_NODE_INACTIVE`` and +``GRAPH_NODE_DISAPPEARED`` saying different, both-true things. A node that was NEVER armed +gets no such backstop - ``GRAPH_NODE_DISAPPEARED`` structurally cannot report it - so absence +keeps advancing its streak exactly as a present tick would. Reporting a HEALTHY node that left +remains ``GRAPH_NODE_DISAPPEARED``'s job as well, for either kind of node - a healthy +departure never starts a violation regardless of arming. **Content follows the clocks, not the snapshot.** Whether a node was in this tick's matches decides nothing about what it reports: a node past ``grace`` stays in @@ -516,14 +542,25 @@ different N. So ``prune_ticks`` (the operator's ``prune_grace``, used as written no ``grace + 1`` clamp any more, because there is nothing left for one to protect) reclaims IDLE entries only: both clocks at zero and no matured ownership, i.e. nothing to lose, and still atomically - ONE map entry per node, gone in the same tick, never partially. A -non-idle entry is never pruned by age, and cannot grow without bound in time either: past -the absence grace its clock advances every tick, so it matures within at most -``grace + absence_grace + 1`` ticks (a violation streak) or ``60 + absence_grace + 1`` (an -unmeasured clock) and is reported. Those are also the longest ``GRAPH_NODE_INACTIVE``'s clear -can be withheld by one departed node, which is why ``grace`` is capped at 300 instead of -being accepted up to ``INT_MAX - 1``: at the old maximum the bound was roughly 24 days at the -shipped cadence, during which the fault could neither raise nor heal for ANY node - silence -indistinguishable from a working detector finding nothing. +non-idle entry is never pruned by age. An entry whose UNMEASURED clock is still climbing +cannot grow without bound in time either: past the absence grace it advances every tick, so +it matures within at most ``60 + absence_grace + 1`` ticks and is reported - the longest +``GRAPH_NODE_UNREADABLE`` or ``GRAPH_NODE_NOT_MANAGED``'s clear can be withheld by one +departed node. A below-``grace`` VIOLATION streak on a node that was NEVER armed shares that +same bound, for the same reason it advances at all while absent: it matures within at most +``grace + absence_grace + 1`` ticks, because nothing else will ever report that node's +departure either. Only once a node HAS been armed does its below-grace streak lose the +bound: absence then holds it rather than advancing it, so it neither matures nor becomes +idle for as long as the node is away. That is not a new way to withhold +``GRAPH_NODE_INACTIVE``'s clear - the clear was already gated on every required node's +status being settled, so one node this indecisive already blocked it; what changes is only +that the fault never NAMES an ARMED node's departure, since that node's evidence belongs to +``GRAPH_NODE_DISAPPEARED`` instead. ``grace`` is still capped at 300 instead of being +accepted up to ``INT_MAX - 1``, because it independently +bounds how long a PRESENT node may go unreported and how long a returning node takes to +re-mature: at the old maximum that PRESENT-side bound was roughly 24 days at the shipped +cadence, with no warning - silence indistinguishable from a working detector finding +nothing. What bounds the map is ``tracked_node_cap`` (default 512, ``kDefaultTrackedNodeCap``, accepted range 1..16384): at the cap idle entries are reclaimed first, then entries for @@ -583,12 +620,14 @@ started over (a restarted gateway, a reconfigure - ``configure()`` rebuilds the Either reason withholds the emission entirely, neither raise nor clear. A raise is never withheld: a violation read from the nodes that DID answer is real regardless of the -unmeasured ones. Every hold is bounded: the never-matched leg and both unmeasured -causes all release after 60 consecutive ticks (a minute at the shipped cadence, -mirroring ``param_drift``'s frozen hold), whether the node is present or gone - the pending -leg releases as soon as the node reads ``active``, or its streak passes grace and the fault -is raised again, which past the absence grace happens while the node is absent too. Every -hold releases by SETTLING the node's status, never by giving up on it. Because a correctly +unmeasured ones. The never-matched leg and both unmeasured causes are bounded: they release +after 60 consecutive ticks (a minute at the shipped cadence, mirroring ``param_drift``'s +frozen hold), whether the node is present or gone. The pending leg releases the same +way - as soon as the node reads ``active``, or a PRESENT tick pushes its streak past grace +and the fault is raised again - but while the node stays absent that release has no timeout +of its own: absence holds a below-grace streak rather than advancing it, so the hold lasts +for exactly as long as the node does not return. Every hold releases by SETTLING the node's +status, never by giving up on it. Because a correctly withheld clear and a detector with nothing to report look identical from outside, a hold that lives past 10 consecutive ticks is explained in the log once per episode, naming every reason in force and, for the node-keyed ones, the count behind each and one node @@ -684,13 +723,23 @@ overlapping - plus the shared-watcher seam the re-bind behaviour lives in: alternation this redesign closes - a node alternating between unreadable and not-managed, indefinitely and on every single tick, still matures the shared clock; the ``INACTIVE``/``NOT_MANAGED`` alternation across absence gaps longer than the - absence grace; the violation streak surviving non-maturing unmeasured ticks and - RESUMING rather than restarting, counted exactly and read through - ``pending_violation``; absence CONTINUING whichever clock the last real observation - started - a blink holds it unchanged, past the blink tolerance it advances, a matured - fault survives a departure, a node measured ACTIVE that vanishes raises nothing and is - reclaimed, and the three ``(X, ABSENT x N)`` restart-loop shapes are swept at the - absence grace and past it; the ``pending`` set and its per-reason breakdown; new-first + absence grace, now counting only the MEASURED not-active legs; the violation streak + surviving non-maturing unmeasured ticks and RESUMING rather than restarting, counted + exactly and read through ``pending_violation``; absence CONTINUING an already-matured + fault exactly as it was on its last present tick - a blink holds either clock unchanged; + past the blink tolerance an unmeasured clock still climbing keeps climbing and matures on + absence alone, while a below-grace violation streak is instead HELD and RESUMES rather + than restarts once the node returns, proven both from one long absence and from many + short ones interleaved with matched reads; a matured violation fault survives a + departure, a node measured ACTIVE that vanishes raises nothing and is reclaimed, and the + two ``(UNREADABLE/NOT_MANAGED, ABSENT x N)`` restart-loop shapes are swept at the + absence grace and past it (their ``INACTIVE`` sibling pinned to the opposite claim at the + same two N: it never crosses grace from absence alone, for a node that was armed at some + point - a node that was NEVER armed gets the opposite result instead, maturing from + absence alone within ``grace + absence_grace + 1`` ticks exactly like the unmeasured + clock does, and resuming rather than restarting on return the same way an armed node's + held streak does); the ``pending`` set and its + per-reason breakdown; new-first ordering for all three fault-shaped maps; the remote-supplied label's own trim budget ahead of the whole-detail backstop, reapplied to a matured unreadable node's detail (which carries no label, only a fqn and a "required by" list); the age horizon @@ -725,8 +774,9 @@ overlapping - plus the shared-watcher seam the re-bind behaviour lives in: releases and its once-per-episode log line; the blink-plus-unread-re-seed sequence; a filler batch sized (from the real detail-building code) to exceed the 480-char cap aggregating into one fault with the fresh crossing named first; a PRESENT node crossing - on the same tick as a batch of departed ones being named ahead of them rather than - truncated away by them; a required node appearing mid-run; a re-bind under the same + grace fresh while a batch of already-departed, already-matured ones sits + absent-and-content, named ahead of them rather than truncated away by them; a required + node appearing mid-run; a re-bind under the same ``App::id``; and the reconfigure/config-validation edge cases. The unmeasured clock's own split is pinned directly, for BOTH codes symmetrically: a managed node whose ``GetState`` genuinely never answers (through ``set_managed_app`` and a real, failing seed - proven by asserting @@ -744,8 +794,13 @@ overlapping - plus the shared-watcher seam the re-bind behaviour lives in: new-first ordering; an already-reported node of either cause KEEPING its own record once it vanishes, with its description switching to say the node has left the graph; a node returning from that absence staying reported with no clear/re-raise churn; each of - the three restart-loop shapes raising its own code, and the ``inactive``/``not-managed`` - alternation across absence gaps; a withheld ``GRAPH_NODE_INACTIVE`` clear releasing when + the two UNMEASURED restart-loop shapes raising its own code from absence alone (their + ``INACTIVE`` sibling instead pinned to counting only real reads, never crossing from any + of the interleaved absence gaps), and the ``inactive``/``not-managed`` + alternation across absence gaps now counting only the MEASURED not-active legs; a + below-grace streak surviving the tightest ``prune_grace`` without being confirmed while + the node stays absent, and resuming (not restarting) once it returns; a withheld + ``GRAPH_NODE_INACTIVE`` clear releasing when the absent node's clock MATURES into its sibling's content rather than when the node is given up on; the two wire strings pinned as hand-typed literals; and the independence claim in both directions. @@ -761,9 +816,11 @@ overlapping - plus the shared-watcher seam the re-bind behaviour lives in: non-string entries each warning, the unclamped prune horizon reaching idle bookkeeping at exactly the configured ``prune_grace`` (0, 1 and 4 - the smallest positive value included, since neither documented endpoint sweeps it) while a node carrying evidence - survives it - including the ``grace: 0, prune_grace: 0`` corner and a wide ``grace`` - beside the tightest ``prune_grace``, whose instrument is the CONFIRMATION rather than the - map size - ``tracked_node_cap`` validated at both range endpoints and one value past each + survives it - including the ``grace: 0, prune_grace: 0`` corner (an UNMEASURED clock, the + only kind that keeps climbing while absent) and a wide ``grace`` beside the tightest + ``prune_grace`` for a below-grace VIOLATION streak, whose instrument is that it is neither + confirmed nor pruned while the node stays absent, and resumes rather than restarts once it + returns - ``tracked_node_cap`` validated at both range endpoints and one value past each with the key proven IN FORCE at both ends, boundedness under identity churn at the real 512-node cap by collapsing the departed rather than refusing the live node, and saturation reported once per EPISODE with a second episode reported again after the first ends. @@ -866,9 +923,289 @@ overlapping - plus the shared-watcher seam the re-bind behaviour lives in: detector cannot tell an entry for a departed node from a misspelt one. +``node_death`` raises ``GRAPH_NODE_DISAPPEARED``. It watches every ARMED App in the graph +for one that goes offline - zero-config, unlike ``lifecycle_expectation``'s +operator-declared ``require_active`` list, because every armed App is a candidate. +Liveness is ``App::is_online``, never mere membership in the entity snapshot: in +runtime-only discovery the two happen to coincide (a dead node's App leaves the snapshot +entirely), but a manifest keeps a bound App present with only ``is_online`` cleared once +its ROS binding disappears, and hybrid discovery inherits that shape - counting snapshot +membership alone would make a manifest-declared node immortal. A managed lifecycle node +that merely deactivates keeps ``is_online: true`` (its process is still running, only its +ROS 2 lifecycle state changed), so a deactivation is never mistaken for a death either - +that is ``lifecycle_expectation``'s concern, not this one's. Tracking is keyed on the +STABLE fqn (``App::effective_fqn()``), never ``App::id``: an id is recomputed every sweep +and only gains a namespace prefix once a same-bare-name collision currently exists +anywhere in the graph, so a live node's id can change out from under a key built from it. + +**What is never tracked.** A peer-aggregated app (``app.source`` starting ``peer:``) +carries no ROS binding of its own, so ``effective_fqn()`` is empty for every one of them; +tracking them would collapse an entire peer fleet onto one ``""`` key. A +``_ros2cli_`` node - rcl's own hidden-node naming convention for every ``ros2`` CLI +invocation, matched structurally rather than guessed at - is excluded too, or every CLI +invocation would accumulate one permanently "dead" entry for the life of the gateway. An +App that never comes online is never armed and so is never admitted to tracking in the +first place - the identical protection a node still inside ``warmup_cycles`` gets. Once +tracked, though, a node's continued life-or-death judgement rests on PRESENCE alone, not +on staying armed: a tracked node that later goes lifecycle-inactive is not thereby +mistaken for dead. Composable/component nodes hosted in one process die together - if the +container process dies, every node it hosted leaves the snapshot on the same tick, and all +of them are named in the one aggregated fault rather than raising one fault each. + +**The wall-clock floor on** ``miss_grace``. The entity snapshot a tick reads is not +rebuilt every tick: runtime discovery rebuilds it off a graph event debounced to about one +refresh per second, so several consecutive ticks between two refreshes see the IDENTICAL +snapshot. A ``miss_grace`` counted purely in ticks does not count independent samples of +the graph - shorten ``tick_interval_ms`` enough and one stale cache generation gets +re-counted as several misses. The floor raises ``miss_grace``, for the configured +``tick_interval_ms``, to the smallest value whose ``(miss_grace + 1) * tick_interval_ms`` +window is still at least 3000 ms - one graph-cache refresh cycle plus margin. At the +shipped 1000 ms tick the floor is exactly the shipped default (``miss_grace: 2``), so the +default configuration is unaffected; only a faster-than-default tick ever raises the +effective grace, with a warning naming the window it actually spans. + +**Suppression: opt-in, and only by name.** Nothing is suppressed unless the operator +names the mechanism in ``suppress`` - a configured ``allowlist`` that ``suppress`` does +not name has NO effect, and a startup warning says so. Two mechanisms exist, activated +independently of each other: + +- ``"allowlist"`` builds an ``AllowlistSuppressor`` over the configured ``allowlist`` set + - the same three-way match (fqn, bare leaf, ``App::id``) ``lifecycle_expectation``'s + ``require_active`` uses, so an operator who has one working can expect the other to + accept the same shapes. Exact match only, never a prefix or a shared suffix. +- ``"lifecycle"`` builds a (stateless) ``LifecycleShutdownSuppressor``: it suppresses a + departure the reliability gate classifies as a clean managed-lifecycle shutdown - see + "Clean-shutdown suppression" below for what counts as clean. + +Every field is re-validated from scratch on every ``configure()`` call, so a malformed +``allowlist`` entry, an unrecognized ``suppress`` name, or a wrongly-typed entry each +produce their own named warning rather than being silently dropped. A candidate is +suppressed the moment ANY active mechanism votes yes, order-independent - which one runs +first never changes the result. Suppression is independent of ``mode``: a suppressed key +never becomes content in the first place, while ``mode`` separately decides whether +non-suppressed content is actually sent. + +**Durability, and why only a durable veto may reclaim bookkeeping.** Whether a suppressor +is durable answers whether its "yes" is a standing fact about the key rather than a +condition that can later stop holding. Both mechanisms here are durable: an +operator-declared allowlist entry does not start matching and then stop on its own, and a +departure's observed shutdown shape does not change after the fact. Durability is what +makes reclaiming a suppressed key's tracker bookkeeping (``prune_grace`` consecutive +suppressed ticks) sound - a NON-durable veto could lift later even after its key's +bookkeeping was discarded, leaving a live, unsuppressed death with nothing left to report +it: a false heal that silently outlives the very condition that produced it. A durable +veto never lifts for a given key once it has fired, so reclaiming under it loses nothing. +``Suppressor::durable()`` defaults to false, the safe assumption for a suppressor nobody +has reasoned about yet; both suppressors here override it explicitly to true. Reclaiming +is re-checked against the durable suppressors specifically, never merely "no longer in +this tick's report", because a key can be dropped from a tick's report by ANY suppressor +in the chain while a durable one also happens to cover it - whether a key may be reclaimed +depends on WHICH suppressor vetoed it, not on whether it survived the filter. The generic +chain is reusable (``Suppressor`` plus the free function ``apply_suppressors()``, +independently unit-tested), but this detector's own filtering is hand-inlined rather than +a call to that helper, because it has to interleave the id-form check above - answerable +only while the node is still present - with the ordinary fqn/leaf dispatch the helper's +plain string-keyed signature does not have room for. + +**Clean-shutdown suppression - what counts as clean.** ``LifecycleShutdownSuppressor`` +reads the lifecycle watcher's cached label for the departed node's LAST observed +transition. ``shuttingdown`` alone suppresses - it is only reachable via a deliberate +SHUTDOWN transition, so the label alone is enough. ``finalized`` suppresses only WITH an +observed transition and never through the error branch; without an observed transition, +or reached through the error branch, it does not, because ``finalized`` is also where a +node's ``on_error`` override lands after ``ON_ERROR_FAILURE``/``ON_ERROR_ERROR`` out of +``errorprocessing`` - the standard way a driver reports a hardware fault it cannot +recover from, which is precisely the death an operator needs reported, not silenced. +``unconfigured`` is deliberately excluded even though it looks quiet: it is also the +resting state of a node whose ``configure()`` failed or that never activated at all, and +suppressing on it would hide exactly that startup failure. Any other label, or no +departure on record at all, abstains. The mechanism is stateless by construction - a +detector's ``configure()`` runs before any per-tick context exists, so it stores nothing +at construction and reads the gate fresh on every call - and its retention window is not +this detector's own to size: before this detector's own ``configure()`` has run, the +plugin predicts the same ``prune_ticks_`` (``max(prune_grace, miss_grace + 1)``) this +detector will compute, then sizes the lifecycle watcher's departed-node retention from it +with enough margin that a clean-shutdown label is still cached at this detector's own +reclaim tick, one tick to spare +(``GraphWatchdogPlugin::compute_departed_retention_ticks()`` - the resulting window is +larger than ``prune_ticks_`` alone). + +**Bounded by evidence, not by age - and why this cap, unlike the sibling's, cannot +actually saturate.** This detector is zero-config over every armed App in the graph - a +strictly larger scope than ``lifecycle_expectation``'s named ``require_active`` set - so +identity churn would grow the tracker's map without a bound if nothing capped it. An +unsuppressed, still-dead entry is never reclaimed by age (``prune_grace`` only ever +reclaims a DURABLY suppressed key), so ``tracked_node_cap`` (default 512, accepted range +1..16384) is what bounds memory. At the cap: idle entries (present, carrying no evidence) +are evicted first - free, since a still-armed one is simply re-tracked a moment later - +then departed entries are collapsed into one synthetic count, keeping at most three +individually named, and only if even that leaves no room is every departed entry +collapsed. Unlike ``LifecycleExpectationTracker``, whose two clocks let a node be +simultaneously present and mid-violation (neither idle nor departed, and so un-evictable - +exactly what lets ITS cap genuinely saturate and withhold ``GRAPH_NODE_INACTIVE``'s +clear), ``NodeLivenessTracker`` carries exactly one piece of per-key state, so every +tracked identity is always either idle or departed; collapsing every departed entry always +empties enough room, so a newcomer is never actually refused while the cap is at least 1 +(``NodeLivenessTrackerCap.SaturationNeverFiresBecauseEveryKeyIsEitherIdleOrCollapsible`` +pins this directly). ``tracking_saturated`` and ``tracked_node_cap`` still appear in this +detector's own ``GET /x-medkit-watchdog`` block, in the same shape the sibling reports +them, but the field is never observed true here - the shared shape exists for consistency +with the sibling detector, not because saturation is reachable in this one. + +**The boundary with** ``lifecycle_expectation``. ``GRAPH_NODE_INACTIVE`` and +``GRAPH_NODE_DISAPPEARED`` can both be raised for the same node at the same time, and that +is CORRECT, not a defect this detector removes: ``GRAPH_NODE_INACTIVE`` says a required +node is present but not active, ``GRAPH_NODE_DISAPPEARED`` says a node is gone, and an +operator facing each has a different repair - reactivate it, or find out why it left. A +node CONFIRMED ``GRAPH_NODE_INACTIVE`` that then departs keeps that fault (its description +switches to say the node has since left the graph) AND now also raises +``GRAPH_NODE_DISAPPEARED``, since this detector tracks every armed App independently of +whatever ``lifecycle_expectation`` thinks of it. + +What changed is narrower than that, and it only applies to a node this detector could ever +have tracked. Before this detector existed, ``lifecycle_expectation``'s own violation streak +let a SUSTAINED absence mature an unconfirmed (below-``grace``) streak into a confirmed +``GRAPH_NODE_INACTIVE`` - the only way a node stuck in a restart loop, never observed long +enough to cross ``grace`` while present, would ever be reported at all. Where this detector +can independently report the same departure - which needs the node to have been ARMED at +least once, since it only ever tracks an App the reliability gate has armed, and the gate +refuses to arm a MANAGED node that is not ``active`` - that absence-alone maturity is gone: +absence CONTINUES a violation that has already matured past ``grace`` (the fault stays +raised, still naming the node), but no longer CREATES one that has not. A node that was +briefly non-active and then died is reported as gone (``GRAPH_NODE_DISAPPEARED``) rather +than also acquiring an inactive fault born from ticks gathered while nobody could observe +it. A ``require_active`` node that never reaches ``active`` is never armed, is never +tracked here, and its departure can never raise ``GRAPH_NODE_DISAPPEARED`` - for that one +node, ``lifecycle_expectation`` keeps the older behaviour: absence still matures a +below-``grace`` streak, because it is structurally the only detector that will ever get to +report it. See ``lifecycle_expectation``'s own "Absence continues the last CORROBORATED +observation, it never erases" section above for the mechanism this rests on. + +**Repeated failures: what** ``occurrence_count`` **and the captured evidence answer, and +don't.** An operator asking "how many times did this node die in the last hour" reads it +off the fault record itself, not off this detector: ``GRAPH_NODE_DISAPPEARED``'s own +``occurrence_count`` starts at 1 on the first raise and increments by exactly one each +time a FAILED report reactivates a record that was CLEARED - never merely re-raised while +still CONFIRMED, and never merely HEALED. Healing (the fault manager's debounce counter +crossing ``healing_threshold`` on clean sweeps, see "Closing the loop" in the README) and +clearing are different things: ``DELETE +/api/v1/apps/graph_watchdog/faults/GRAPH_NODE_DISAPPEARED`` - the fault manager's own +``~/clear_fault`` underneath - is what acknowledges an occurrence and closes its cycle. A +still-CONFIRMED, or still-HEALED-but-unacknowledged, fault that fires FAILED again is the +SAME occurrence continuing, not a new one, so restarting the dead node fast enough to heal +it before anyone acknowledges it will not move the count. + +The honest limits, read off this branch's fault-manager storage rather than assumed: the +per-fault rosbag store enforces ``fault_code`` as UNIQUE, so a fault can hold at most one +recording at a time - a later confirmation's capture replaces the earlier one on disk +rather than accumulating a history. The freeze frame is the same shape: one row per +``fault_code``, overwritten on every capture, so a fifth occurrence's captured values +overwrite the first's. What survives every occurrence by default is the count itself; a +hash-chained record of every raise/clear/heal transition also exists, but only once the +fault manager's own ``audit_log.enabled`` is turned on, which it is not by default. +Recordings and the freeze frame do not accumulate a per-occurrence history either way. + +**A second death while the first is still outstanding gets no evidence of its own.** This +detector folds every currently-affected node into ONE ``GRAPH_NODE_DISAPPEARED`` record via +``AggregatedFault::emit()`` (see "The boundary with" ``lifecycle_expectation`` above): a +second node dying while the fault is still CONFIRMED is added to the description on the next +tick, but the ``ReportFault`` call that carries it lands on ``fault_storage.cpp``'s +already-CONFIRMED, still-FAILED branch - the same re-report-not-a-new-occurrence path +"Repeated failures" above describes for one node dying twice, reached here by two different +nodes sharing one code instead. ``occurrence_count`` does not move, the fault manager +publishes ``EVENT_UPDATED`` rather than ``EVENT_CONFIRMED``, and ``just_confirmed`` in +``fault_manager_node.cpp`` stays false - which is what gates ``capture_pool_``, so neither a +freeze frame nor a recording is captured for the second node. Acknowledging between the two +deaths avoids this: the DELETE route's ``~/clear_fault`` closes the first occurrence, so the +second node's next FAILED report reactivates a CLEARED record instead of updating a CONFIRMED +one - a genuine new occurrence, with its own ``occurrence_count`` and its own capture. + +Two fixes were measured and rejected here, so this is not open for reconsideration without +new evidence: + +- **Clear-then-raise from inside the detector does nothing.** ``AggregatedFault`` already has + both halves available - ``emit()`` sends a PASSED-shaped ``clear_fault()`` when ``affected`` + is empty, a FAILED-shaped ``raise_fault()`` otherwise - so sending one right after the other + on the same tick looks like a free fix. It is not: ``compute_debounce_status()``'s + hysteresis latch holds a CONFIRMED (or HEALED) status until the debounce counter itself + crosses the OPPOSITE threshold, and a single PASSED event only moves that counter by one + step. Unless the fault happened to sit exactly one step short of ``healing_threshold`` + already, the clear changes nothing the status can see, and the raise that follows lands + right back on the same already-CONFIRMED branch as before. +- **Calling the real** ``~/clear_fault`` **service instead is worse than doing nothing.** + ``SqliteFaultStorage::clear_fault()`` runs ``DELETE FROM snapshots WHERE fault_code = ?`` - + it deletes the per-topic readings captured for the fault it clears, keeping only the + freeze-frame row. And ``ClearFault.srv``'s own ``skip_correlation_auto_clear`` defaults to + false, so clearing a root cause also clears every symptom the correlation engine attributes + to it unless the caller explicitly opts out (the gateway's own REST DELETE routes do; a + detector reaching for this service directly would have to remember to as well). Having the + detector call this automatically the moment a second node dies would destroy the FIRST + node's evidence - snapshots and correlated symptoms alike - to chase evidence for the + second, before anyone has necessarily seen the first. + +A correct fix does not belong in this detector at all: it needs an operation in the fault +manager that re-confirms a CONFIRMED record - bumping ``occurrence_count``, publishing +``EVENT_CONFIRMED`` again, and re-arming ``just_confirmed`` for a fresh capture - without +routing through CLEARED and without deleting anything the first occurrence already earned. + +**Test tiers.** Three tiers each prove a different layer, deliberately not overlapping, +plus the suppression chain the detector shares no code with any sibling for: + +1. **Unit** (``test_node_liveness_tracker.cpp``): the pure presence/absence state machine + - a key present but never armed is never tracked; once armed, presence alone (not + continued arming) keeps a tracked key's miss counter at zero; a key is reported dead + only once misses exceed ``miss_grace``; freshness ordering keeps a brand-new death out + of a capped description's blind spot; ``prune()`` reclaims a key only once it has been + suppressed for MORE than ``prune_ticks`` CONSECUTIVE calls, resets the streak the + moment a veto lifts even once, and never reclaims an unsuppressed death no matter how + long it stays dead; the cap evicts idle entries before collapsing departed ones, keeps + the collapsed count monotone within one tracker lifetime, and - the property most at + odds with a naive read of the sibling detector - can never actually report + ``tracking_saturated`` at all, since every tracked key here is always either idle or + collapsible, never both at once the way the sibling's two clocks allow. + ``test_suppressor.cpp`` pins the ``Suppressor`` interface and the free + ``apply_suppressors()`` helper (order-independent, null-safe, returns the dropped + count); ``test_allowlist_suppressor.cpp`` and ``test_lifecycle_shutdown_suppressor.cpp`` + pin each suppressor's own matching rules independently of any detector. +2. **Integration** (``test_node_death_integration.cpp``): the full config contract + (``miss_grace``, ``prune_grace`` and ``tick_interval_ms`` range checks and their floor + interaction; malformed ``allowlist``/``suppress`` entries; ``tracked_node_cap`` range + checks, including a value far past ``INT_MAX`` rejected rather than wrapped; an unknown + top-level key), that a peer-aggregated app and an ``is_online: false`` app are never + tracked, that the tracked count stays bounded and shrinks after a reclaim under + sustained identity churn, that a clean managed-lifecycle departure is suppressed + exactly AT the ``miss_grace`` boundary and stays reclaimed - not re-raised - past the + retention window the plugin sizes for it, that the allowlist's id-form suppresses only + the colliding node and not its namesake, and the ungated-clear guard's four properties: + a stored fault stays unhealed while an unrelated node arms alone, a clear flows once + this process instance has genuinely raised, the guard tracks DELIVERY rather than + intent, and a reconfigure while the node is absent does not withhold a clear the + process had already earned. +3. **E2e**, three files, each launching its own real gateway + fault_manager + demo-node + stack: ``test/e2e/test_node_death_e2e.test.py`` (ten scenarios covering the raise/clear + round trip, no-heal-standalone, the lifecycle-deactivate/manifest-never-online/ros2cli + non-cases, a bare-name collision naming only the node that exited, a fast tick alone + raising nothing, a three-cycle restart loop reaching ``occurrence_count: 3``, and a + fault surviving a gateway restart); ``test/e2e/test_node_death_suppression_e2e.test.py`` + (five scenarios covering the allowlist, its inertness when unnamed in ``suppress``, the + clean-shutdown/still-active contrast, opt-in suppression, and pruning without a false + heal); and ``test/e2e/test_node_death_boundary_e2e.test.py`` (eight scenarios proving + the seam with ``lifecycle_expectation`` directly: a node stuck inactive but never gone + raises ``GRAPH_NODE_INACTIVE`` alone, an ARMED node killed below ``grace`` raises + ``GRAPH_NODE_DISAPPEARED`` alone, a node killed AFTER ``GRAPH_NODE_INACTIVE`` has + confirmed raises BOTH at once, a healthy node that is simply killed raises + ``GRAPH_NODE_DISAPPEARED`` alone, a restart-looping required node is caught every cycle + regardless of lifecycle maturity, a NEVER-armed node killed below ``grace`` raises + ``GRAPH_NODE_INACTIVE`` alone instead - node_death cannot track a node the gate never + armed, so absence has to mature it here - a large ``miss_grace`` delays but does not + swallow the report, and a restarted gateway's warmup window never produces a spurious + PASSED). + + Status --------------- The plugin loads, ticks the graph, and shuts down cleanly. The reliability core is real -and already ticking. Four silent-fault detector classes raise through it today, -``qos_mismatch``, ``orphan``, ``param_drift`` and ``lifecycle_expectation``. The -remaining classes land in follow-up changes, each against its own issue. +and already ticking. Five silent-fault detector classes raise through it today, +``qos_mismatch``, ``orphan``, ``param_drift``, ``lifecycle_expectation`` and +``node_death``. Two classes remain undelivered, ``GRAPH_TF_STALE`` and +``GRAPH_LATENCY_BUDGET``; they land in follow-up changes, each against its own issue. diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/aggregated_fault.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/aggregated_fault.hpp index 7fda64c0c..a1fc860fc 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/aggregated_fault.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/aggregated_fault.hpp @@ -29,8 +29,8 @@ namespace ros2_medkit_graph_watchdog { /// external entity from its own IntrospectionProvider (graph_watchdog_plugin.cpp). /// /// It must be an entity the plugin owns, not a borrowed one. Scoping these faults to the -/// host Component - the obvious choice, and what this used to do - makes them reachable -/// from NO endpoint: `collect_component_app_fqns` (fault_scope.cpp) only puts a +/// host Component instead - the obvious-looking alternative - makes them reachable from NO +/// endpoint: `collect_component_app_fqns` (fault_scope.cpp) only puts a /// component's bare id in the scope set when `external` is true, and a runtime host /// Component built by HostInfoProvider never sets it, so `/components//faults` and /// the scoped detail route both drop the fault. There is no server-level @@ -72,16 +72,21 @@ class AggregatedFault { AggregatedFault(const char * code, std::uint8_t severity) : code_(code), severity_(severity) { } - void emit(DetectorContext & ctx, const std::map & affected) const { + /// Returns whatever ctx.raise_fault()/ctx.clear_fault() returned - true only if the + /// request actually reached async_send_request(), never merely because `affected` was + /// non-empty. That proves local enqueue, not fault_manager receipt (the client is + /// fire-and-forget) - see DetectorContext::raise_fault's own doc for exactly what this + /// return value does and does not prove. + bool emit(DetectorContext & ctx, const std::map & affected) const { const std::string source = graph_source_id(ctx); if (affected.empty()) { - ctx.clear_fault(code_, source); - return; + return ctx.clear_fault(code_, source); } - ctx.raise_fault(code_, severity_, describe(affected), source); + return ctx.raise_fault(code_, severity_, describe(affected), source); } - /// emit() with a caller-chosen ORDER for the description. + /// emit() with a caller-chosen ORDER for the description. Return value: see emit()'s own + /// doc. /// /// The map overload above lists entities lexicographically, which is fine when every /// entry is equally interesting. It is not fine when the set mixes long-standing entries @@ -89,14 +94,13 @@ class AggregatedFault { /// `order` names the keys of `affected` in the order they should appear; any key of /// `affected` missing from `order` is appended afterwards, so a caller can never silently /// drop an entity by getting the ordering wrong. - void emit_ordered(DetectorContext & ctx, const std::map & affected, + bool emit_ordered(DetectorContext & ctx, const std::map & affected, const std::vector & order) const { const std::string source = graph_source_id(ctx); if (affected.empty()) { - ctx.clear_fault(code_, source); - return; + return ctx.clear_fault(code_, source); } - ctx.raise_fault(code_, severity_, describe_ordered(affected, order), source); + return ctx.raise_fault(code_, severity_, describe_ordered(affected, order), source); } /// Join the affected entities into one description, capped at kMaxDescriptionChars. diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/allowlist_suppressor.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/allowlist_suppressor.hpp new file mode 100644 index 000000000..10a9ccae7 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/allowlist_suppressor.hpp @@ -0,0 +1,94 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include + +#include "ros2_medkit_graph_watchdog/suppressor.hpp" + +namespace ros2_medkit_graph_watchdog { + +/// Operator-declared veto. Suppresses a key iff it - or a name the SAME entity is also +/// known by - is present, verbatim, in the configured allow set: never a prefix or a +/// substring, so allowlisting `/r1/x` can never reach into `/r2/x` on a fleet where the two +/// share a suffix. +/// +/// Deliberately mirrors lifecycle_expectation_detector.cpp's own require_active matching +/// (`id != app.id && id != fqn && id != leaf`) rather than inventing a second convention: +/// a bare name is the natural operator input, and `App::id` is unstable - it gains a +/// namespace prefix the moment a same-bare-name collision exists anywhere in the graph - +/// so the two config surfaces (require_active, allowlist) need to accept the same shapes or +/// an operator who has one working would reasonably expect the other to behave the same way +/// and be wrong. +/// +/// The two surfaces reach that same three-way match through different mechanics, though: +/// lifecycle_expectation matches while the node is still PRESENT, so `app.id` and its fqn +/// are both in hand from the very same `ctx.snapshot->apps` entry it is iterating. This +/// class is asked about a key ONLY after the entity is already gone - present-tense +/// `ctx.snapshot` cannot answer "what was this dead key's id" at all. suppresses() itself +/// therefore still only ever sees a single string and covers the two forms derivable from +/// that string alone (the key verbatim, and its own bare leaf); the id form needs the +/// caller to have captured `App::id` while the entity was still alive and to re-offer it +/// through allows() below - see node_death_detector.cpp's tick() for where that capture +/// happens and why. +/// +/// Shared by any detector whose candidate keys are plain strings (node_death's are +/// `App::effective_fqn()`; a future tf_stale conversion would use its own "parent->child" +/// pair strings) - the exact-match contract does not care which. +/// +/// The empty string is never treated specially: a caller that means "match nothing" for +/// an empty entity_key has to actually insert "" into the allow set for that to happen. +/// Detector configure() code is expected to have already dropped empty entries when +/// building the set it hands here. +class AllowlistSuppressor : public Suppressor { + public: + explicit AllowlistSuppressor(std::set allow) : allow_(std::move(allow)) { + } + + /// True iff `candidate` is present, verbatim, in the configured allow set. Public (beyond + /// what the Suppressor interface requires) so a caller holding a form of an entity's + /// identity that suppresses() itself has no way to reach - `App::id`, captured while the + /// entity was still present - can still check it against the same set. See the class doc. + bool allows(const std::string & candidate) const { + return allow_.count(candidate) > 0; + } + + /// Matches `entity_key` verbatim, or the bare leaf of it (the substring after the last + /// '/', or the whole key if it carries no '/') - the two forms answerable from the key + /// alone. The id form is NOT checked here; see allows() and the class doc. + bool suppresses(const std::string & entity_key, const DetectorContext & /*ctx*/) const override { + if (allows(entity_key)) { + return true; + } + const auto slash = entity_key.rfind('/'); + if (slash == std::string::npos) { + return false; // entity_key has no '/': it already IS its own bare leaf, checked above + } + return allows(entity_key.substr(slash + 1)); + } + + /// An operator-declared entry is a standing fact about that key for as long as this + /// configuration is loaded - it does not start matching and then stop on its own, so a + /// key it vetoes may safely have its tracker bookkeeping reclaimed. + bool durable() const override { + return true; + } + + private: + std::set allow_; +}; + +} // namespace ros2_medkit_graph_watchdog diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector.hpp index b16e7df8b..18ad46637 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector.hpp @@ -74,10 +74,22 @@ struct DetectorContext { nullptr; ///< entities this tick (id + bound_fqn); null in bare-context tests const std::atomic * cancelled = nullptr; ///< plugin shutdown flag; a long sweep polls it (null => never) - void raise_fault(const std::string & code, uint8_t severity, const std::string & description, + /// Returns true only once `async_send_request` has actually been called - false for every + /// suppression path (Advisory/Off, no client, empty source_id, reliability gate, service + /// not ready). This proves the request was handed to rclcpp's client library for sending, + /// nothing more: the client is deliberately fire-and-forget (see + /// GraphWatchdogPlugin::set_context()'s own note on why - nothing consumes the future), so + /// a true return does NOT prove fault_manager received or processed the request, only that + /// this call was not one of the silent-decline paths above. A caller that needs to + /// distinguish "genuinely attempted" from "merely warranted" (report non-empty) - not + /// receipt - must use this return value rather than inferring it from its own inputs: + /// node_death_detector.cpp's ever_raised_ guard is exactly that caller, and every one of + /// these suppression paths is silent by design (no detector should have to duplicate them + /// to know whether it may later trust its own silence as a clear). + bool raise_fault(const std::string & code, uint8_t severity, const std::string & description, const std::string & source_id) { if (!mode_emits(mode) || !fault_client) { - return; // Advisory/Off suppressed, or client not yet wired. + return false; // Advisory/Off suppressed, or client not yet wired. } if (source_id.empty()) { if (gateway_node) { @@ -85,21 +97,24 @@ struct DetectorContext { "graph_watchdog: dropping fault '%s' with empty source_id (detector contract violation)", code.c_str()); } - return; + return false; } if (!reliability_allows(gate, source_id)) { - return; // entity warming up or lifecycle-inactive: suppressed by the reliability core. + return false; // entity warming up or lifecycle-inactive: suppressed by the reliability core. } if (!fault_client->service_is_ready()) { - return; // fault_manager not reachable yet; avoid unbounded pending_requests_ growth. + return false; // fault_manager not reachable yet; avoid unbounded pending_requests_ growth. } fault_client->async_send_request(std::make_shared( make_fault_report(source_id, code, severity, description))); + return true; } - void clear_fault(const std::string & code, const std::string & source_id) { + /// See raise_fault()'s own doc on the return value - identical contract, minus the + /// reliability-gate check clear_fault() has never applied. + bool clear_fault(const std::string & code, const std::string & source_id) { if (!mode_emits(mode) || !fault_client) { - return; // Advisory/Off suppressed, or client not yet wired. + return false; // Advisory/Off suppressed, or client not yet wired. } if (source_id.empty()) { if (gateway_node) { @@ -107,13 +122,14 @@ struct DetectorContext { "graph_watchdog: dropping fault-clear '%s' with empty source_id (detector contract violation)", code.c_str()); } - return; + return false; } if (!fault_client->service_is_ready()) { - return; // fault_manager not reachable yet; avoid unbounded pending_requests_ growth. + return false; // fault_manager not reachable yet; avoid unbounded pending_requests_ growth. } fault_client->async_send_request( std::make_shared(make_fault_clear(source_id, code))); + return true; } }; diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector_config_keys.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector_config_keys.hpp index ae675aded..873e66f5d 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector_config_keys.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/detector_config_keys.hpp @@ -13,6 +13,8 @@ // limitations under the License. #pragma once +#include +#include #include #include #include @@ -50,6 +52,50 @@ inline constexpr double min_jump_thresh_sec(int tick_interval_ms) { return kJumpThreshTickPeriods * static_cast(tick_interval_ms) / 1000.0; } +/// Floor for node_death's own `miss_grace`, in milliseconds of wall clock. +/// +/// The entity cache a detector reads (ctx.snapshot) is not rebuilt every tick: it is +/// rebuilt on a graph event debounced to about one refresh per second, so every tick +/// between two refreshes sees the SAME snapshot. A tick-counted miss_grace therefore does +/// not measure independent samples of the graph - it can re-count one stale cache +/// generation as several misses - and that collapses silently the moment the tick period +/// is shortened: at a 200ms tick, a miss_grace of 2 ticks is only 600ms of nominal grace, +/// well under a single refresh cycle, so one omitted refresh alone can already cross it. +/// 3000ms is one debounce cycle plus margin, chosen so the shipped default (miss_grace 2 at +/// the 1000ms default tick) is unaffected and only a faster-than-default tick ever raises +/// the effective grace. +inline constexpr int kMinNodeDeathWindowMs = 3000; + +/// Ceiling node_death applies to BOTH `miss_grace` and `prune_grace` - ticks-before- +/// something-happens knobs that, at the 1s default tick, already mean an hour of silence +/// before either takes effect at 3600; a value past it is a typo or a unit mix-up, not a +/// real operator choice. Shared (not detector-local) so graph_watchdog_plugin.cpp's own +/// compute_departed_retention_ticks() - which has to predict node_death's eventual +/// prune_ticks_ before that detector's own configure() has run - clamps to the IDENTICAL +/// bound rather than risking a plugin-side window sized for a value the detector will +/// itself reject. +inline constexpr std::int64_t kMaxNodeDeathGraceTicks = 3600; + +/// Smallest `miss_grace` (in ticks) that keeps kMinNodeDeathWindowMs of wall clock, +/// whatever `tick_interval_ms` is configured to. A death is reported once misses EXCEED +/// miss_grace, i.e. after miss_grace + 1 ticks, so the ceiling divides the window by the +/// tick period and subtracts the one tick that boundary already buys back. +/// +/// Computed in int64_t throughout: `tick_interval_ms` is validated only against +/// `> 0 && <= INT_MAX` (node_death_detector.cpp's own configure()), and at that documented- +/// valid endpoint `kMinNodeDeathWindowMs + tick_interval_ms - 1` overflows a 32-bit int +/// before the division ever runs - undefined behaviour at a value the config contract +/// explicitly accepts. The final result is always small (it shrinks as tick_interval_ms +/// grows), so narrowing it back to int at the end never loses anything. +inline int min_node_death_miss_grace(int tick_interval_ms) { + if (tick_interval_ms <= 0) { + return 0; // not a real cadence; nothing meaningful to floor against + } + const std::int64_t wide_tick = tick_interval_ms; + const std::int64_t window = static_cast(kMinNodeDeathWindowMs) + wide_tick - 1; + return static_cast(std::max(0, window / wide_tick - 1)); +} + /// Append one warning per key in `detector_cfg` that the detector does not read. /// /// extract_detector_config() copies EVERY key under detectors. verbatim, and a diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/graph_watchdog_plugin.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/graph_watchdog_plugin.hpp index 3228db4c0..8507b492b 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/graph_watchdog_plugin.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/graph_watchdog_plugin.hpp @@ -92,6 +92,15 @@ class GraphWatchdogPlugin : public ros2_medkit_gateway::GatewayPlugin, /// clears the pending map as it reads it. std::size_t prune_pending_fault_requests_for_test(); + /// Test-only: exposes the otherwise-private compute_departed_retention_ticks() so a unit + /// test can drive its config-validation directly (malformed/oversized miss_grace or + /// prune_grace) without constructing a full ROS gate/node - it has no ROS dependency of + /// its own, only tick_interval_ms_/prune_grace_ (set via configure()+load_parameters()) + /// and the JSON it is handed. + int compute_departed_retention_ticks_for_test(const nlohmann::json & config_snapshot) const { + return compute_departed_retention_ticks(config_snapshot); + } + private: void load_parameters(); void run_tick_loop(); ///< Body of tick_thread_: tick() + interruptible wait, until shutdown. diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/lifecycle_expectation_tracker.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/lifecycle_expectation_tracker.hpp index de1a3593d..be5a0e4ad 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/lifecycle_expectation_tracker.hpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/lifecycle_expectation_tracker.hpp @@ -35,9 +35,14 @@ inline constexpr int kDefaultNoMatchWarnTicks = 10; /// clocks start moving again. Inside this budget a node's whole state is simply HELD: /// zeroing it on the first blink would mean a node dropping out of one snapshot in every /// few never accumulates enough consecutive present ticks on either clock below to be -/// reported at all. PAST this budget absence CONTINUES whatever the node's last real -/// observation had started - it never restarts a clock and never discards one. See -/// "Absence continues, it never erases" in LifecycleExpectationTracker's class doc. +/// reported at all. PAST this budget absence still never DISCARDS a clock, but it may no +/// longer be the one thing MOVING it either: an already-matured fault CONTINUES exactly as +/// it did on its last present tick, while a violation streak that has not yet matured is +/// simply held at whatever it already reached, for a node the presence detector could also +/// have reported - only a present tick can still advance it there. A node the presence +/// detector could structurally never report gets no such backstop: absence keeps advancing +/// its streak too, exactly like a still-climbing unmeasured clock does. See "Absence +/// continues, it never erases" in LifecycleExpectationTracker's class doc. inline constexpr int kDefaultAbsenceGrace = 3; /// Default bound on how many nodes this tracker keeps state for at once - the operator's @@ -132,6 +137,16 @@ struct LifecycleMatch { std::string entry; ///< the require_active entry that matched std::string fqn; ///< the node's stable App::effective_fqn() std::optional state; + /// Whether the reliability gate currently allows a fault to be raised for this node's + /// App::id - the same predicate node_death's presence detector requires before it will + /// ever track a key (NodeLivenessTracker: "a key becomes TRACKED the first time it is + /// armed, and stays tracked from then on"). Read off the gate by the caller, not + /// re-derived here: whether a MANAGED node counts as armed also depends on + /// LifecycleWatcher::node_ok(), which this header has no access to and must not + /// reimplement. Defaults true - the narrower of the two behaviours NodeState::ever_armed + /// then selects between - so a caller that does not wire this through keeps the + /// already-shipped absence handling rather than silently gaining a new reporting path. + bool armed = true; }; /// The one observed fact this whole state machine runs on: what a MATCHED node's read @@ -300,23 +315,47 @@ struct LifecycleExpectationReport { /// be measured at all. PAST `absence_grace` absence advances whichever clock the node's /// `NodeState::settled_observed` says it was on, and resets nothing: /// -/// - settled kInactive: the violation streak keeps climbing, so a node measured not-active -/// that then vanishes eventually raises GRAPH_NODE_INACTIVE about a node that is no -/// longer there. That is the honest reading of the expectation the operator wrote: the -/// node must be ACTIVE, it was not, and now it is gone. Any UNCORROBORATED unmeasured -/// spell it had also started is dropped here rather than held: nothing is left to read, -/// and a clock that can never mature would keep the entry non-idle - and therefore -/// `pending` - for the life of the process. +/// - settled kInactive, ALREADY REPORTED under GRAPH_NODE_INACTIVE (its streak past +/// `grace`): the streak continues exactly as it did on its last present tick - frozen at +/// `grace` + 1 either way, see advance_violation_streak's own "frozen one past itself" - +/// so a node measured not-active and confirmed that then vanishes keeps its fault raised, +/// still naming a node that is no longer there. That is the honest reading of the +/// expectation the operator wrote: the node must be ACTIVE, it was not and was already +/// said so, and now it is gone too. +/// - settled kInactive, NOT YET REPORTED (streak at or below `grace`), on a node that WAS +/// ARMED at some point (`NodeState::ever_armed`): the streak is HELD, neither advanced +/// nor erased. Maturing it here would raise GRAPH_NODE_INACTIVE from ticks gathered while +/// the node could not be observed at all - evidence the operator's own graph never +/// witnessed - and the presence detector is ABLE to report this exact departure instead: +/// node_death tracks any node the gate has armed at least once (NodeLivenessTracker's own +/// "a key becomes TRACKED the first time it is armed"), so starting a NEW violation from +/// a departure nobody here could watch is its job, not this one's. The entry keeps what +/// it already earned, so a node that RETURNS still inactive resumes its streak rather +/// than re-earning `grace` from zero. +/// - settled kInactive, NOT YET REPORTED, on a node that was NEVER armed: the streak keeps +/// climbing on absence exactly as it would on a present tick. The reasoning above +/// inverts: node_death only ever tracks a node the gate has armed at least once, so one +/// that never reached that bar - a `require_active` entry that comes up `unconfigured` +/// and is killed before its own `grace` elapses is exactly this shape - is structurally +/// invisible to the presence detector no matter what happens to it afterwards. Holding +/// this streak too would mean nothing in the plugin ever reports the departure. This +/// keeps the same bound a still-climbing UNMEASURED clock already has regardless of +/// arming (see "Bounded by evidence, not by age" below): the streak matures within +/// `grace` + `absence_grace` + 1 ticks, never staying unsettled forever the way an ARMED +/// node's below-grace streak may. +/// +/// Either way, any UNCORROBORATED unmeasured spell the node had also started is dropped +/// here rather than held: nothing is left to read, and a clock that can never mature +/// would keep the entry non-idle - and therefore `pending` - for the life of the process. /// - settled kUnreadable or kNotManaged: the unmeasured clock keeps climbing, under the /// cause that observation set - absence carries no label, so it cannot change one. /// - settled kActive: nothing advances, and anything the node had started but not /// corroborated is released, so the entry becomes idle and is reclaimed silently. An /// entry that is already CONTENT (a matured unmeasured clock, or a streak past `grace`) /// is never released - a departure heals nothing. Starting a NEW violation from a healthy -/// departure is the presence class's job (GRAPH_NODE_DISAPPEARED), which still has no -/// detector in this package and is still out of scope - the single remaining gap here, -/// and a far narrower one than "a node absent past the grace is invisible whichever clock -/// it was on". +/// departure is the presence class's job (GRAPH_NODE_DISAPPEARED, this package's own +/// node_death detector), not this class's - a far narrower boundary than "a node absent +/// past the grace is invisible whichever clock it was on". /// /// **What "settled" means, and the alternatives rejected for it.** A real measurement /// (kActive or kInactive) settles IMMEDIATELY: a lifecycle label is a fact about the node, @@ -350,17 +389,32 @@ struct LifecycleExpectationReport { /// ABSENT x N)` forever and never accumulate enough of anything to be reported, which is /// precisely the node this detector most exists to catch. /// -/// **Bounded by evidence, not by age.** Since absence no longer erases anything, an entry -/// carrying a live clock is never reclaimed by age either - the two are the same defect -/// seen twice, and moving the horizon rather than removing it would leave the evasion in -/// place at a different N. `prune_ticks` therefore reclaims only IDLE entries (both clocks -/// zero, no matured ownership), which carry nothing to lose. What bounds the map instead is -/// `tracked_node_cap`. A non-idle entry cannot grow without bound in time either way: past -/// `absence_grace` its clock advances every tick, so it matures within at most -/// `grace` + `absence_grace` + 1 ticks (violation) or `unmeasured_hold_ticks` + -/// `absence_grace` + 1 (unmeasured) and is reported. Both bounds are why the detector caps -/// the `grace` it accepts: they are also the longest GRAPH_NODE_INACTIVE's clear can be -/// withheld by one departed node. +/// **Bounded by evidence, not by age.** Since absence no longer erases anything a MATURED +/// entry carries, that entry is never reclaimed by age - a departure must not heal what it +/// cannot un-happen. `prune_ticks` therefore reclaims only IDLE entries (both clocks zero, +/// no matured ownership, no live-but-unmatured streak), which carry nothing to lose. What +/// bounds the map's SIZE instead is `tracked_node_cap`. +/// +/// A node whose UNMEASURED clock is still climbing keeps the bound this class shipped with: +/// past `absence_grace` it advances every tick regardless of maturity - unlike the violation +/// streak above, this clock's own absence behaviour did not change, see the two unmeasured +/// causes above - so it matures within at most `unmeasured_hold_ticks` + `absence_grace` + 1 +/// ticks and is reported, which is also the longest GRAPH_NODE_UNREADABLE or +/// GRAPH_NODE_NOT_MANAGED's clear can be withheld by one departed node. +/// +/// A VIOLATION streak on a node that was NEVER armed shares that same bound, for the same +/// reason it advances at all while absent (see the kInactive case above): it matures within +/// at most `grace` + `absence_grace` + 1 ticks, because nothing else will ever report that +/// node's departure either. Only once a node HAS been armed does its below-grace streak +/// lose the bound: absence then holds it rather than advancing it, so it neither matures +/// nor becomes idle for as long as the node is away, however long that is. This is not a +/// new way to withhold GRAPH_NODE_INACTIVE's clear - an aggregate, level-triggered fault +/// already could not assert "every required node is healthy" while any one of them was +/// unsettled, whether that unsettled node was HELD (`pending`) or climbing toward CONTENT +/// (`affected`), so a node this indecisive already blocked the same clear before this +/// design. What changes is only that GRAPH_NODE_INACTIVE never NAMES an ARMED node's +/// departure: that node's evidence belongs to GRAPH_NODE_DISAPPEARED, not to an entry +/// nobody here could measure. /// /// **A present node always wins a slot.** At the cap the order is: reclaim IDLE entries /// (they carry nothing); then collapse DEPARTED entries - lexicographically last first, @@ -439,6 +493,7 @@ class LifecycleExpectationTracker { struct NodeTick { std::vector entries; ///< every entry that named this node, in match order std::optional state; ///< the label enforced for it this tick + bool armed = false; ///< whether the gate allowed a raise for this node on THIS tick }; std::set matched_entries; std::map nodes; @@ -458,6 +513,11 @@ class LifecycleExpectationTracker { observed_rank(classify_observed_state(match.state)) > observed_rank(classify_observed_state(node.state))) { node.state = match.state; } + // Same app.id, same tick, so every duplicate match necessarily carries the same + // arming fact in practice - OR rather than overwrite only so a future caller that + // ever legitimately disagrees cannot make this node look LESS armed than any one + // match already said it was. + node.armed = node.armed || match.armed; } // Clocks that cross their bound on THIS tick. Collected here rather than pushed @@ -479,6 +539,10 @@ class LifecycleExpectationTracker { NodeState & node = *tracked; node.absent_ticks = 0; node.entries = node_tick.entries; // refreshed on every matched tick, held through a blink + // Sticky: once the gate has armed this node on any one tick, the presence detector + // could have picked up its departure from that tick onward, for the rest of this + // node's life - see NodeState::ever_armed. + node.ever_armed = node.ever_armed || node_tick.armed; const LifecycleObservedState state = classify_observed_state(node_tick.state); switch (state) { @@ -532,12 +596,16 @@ class LifecycleExpectationTracker { } // Absence loop: every tracked node NOT matched this tick. Inside absence_grace_ - // everything is HELD unchanged (a blink). Past it, absence CONTINUES whichever clock - // the node's last REAL observation had started, and resets nothing - see "Absence - // continues, it never erases" in the class doc. A node last measured kActive continues - // nothing: a healthy node shutting down is GRAPH_NODE_DISAPPEARED's business, which - // still has no detector in this package and stays out of scope. Only IDLE bookkeeping - // is reclaimed by age, because only an idle entry has nothing to lose. + // everything is HELD unchanged (a blink). Past it, absence resets nothing but no longer + // advances everything either - see "Absence continues, it never erases" in the class + // doc. A node last measured kInactive continues an ALREADY-matured violation exactly as + // it was, but a below-grace one is simply held, never matured on the strength of absence + // alone. A node last measured kUnreadable/kNotManaged keeps climbing its unmeasured + // clock regardless of maturity - that clock's own absence behaviour did not change. A node + // last measured kActive continues nothing: a healthy node shutting down is + // GRAPH_NODE_DISAPPEARED's business, which this package's node_death detector now + // covers. Only IDLE bookkeeping is reclaimed by age, because only an idle entry has + // nothing to lose. for (auto it = nodes_.begin(); it != nodes_.end();) { const std::string & fqn = it->first; NodeState & node = it->second; @@ -555,7 +623,29 @@ class LifecycleExpectationTracker { // the node: there is nothing left to read, and a clock that can never mature // would keep this entry non-idle - and therefore `pending` - forever. node.unmeasured_clock = 0; - advance_violation_streak(node, fqn, crossed_violation); + // Two independent questions decide whether this tick may advance the streak. + // First, is_content(node): "has this node's violation already been reported", + // not "how many ticks has it accumulated" - already reported, absence continues + // it exactly like any other tick would (advance_violation_streak() no-ops past + // `grace` regardless, see its own "frozen one past itself"), so this call only + // documents that maturity, once earned, survives a departure. Second, + // `!node.ever_armed`: who ELSE could ever report this node's departure. A node + // the presence detector could have tracked (armed at least once) needs no help + // from here even below grace: maturing it would raise GRAPH_NODE_INACTIVE from + // ticks gathered while nobody could observe the node, evidence the presence + // detector owns instead (node_death tracks any node the gate has armed at least + // once - NodeLivenessTracker's own "a key becomes TRACKED the first time it is + // armed") - so the streak is left untouched: neither advanced (no fault born + // from an absence the presence class will report anyway) nor erased (a node + // that RETURNS still inactive resumes from here rather than re-earning `grace` + // from zero). A node that was NEVER armed is structurally invisible to the + // presence detector no matter what happens to it afterwards, so nothing else in + // the plugin will ever report this departure either - absence keeps advancing + // the streak for it exactly as a present tick would, the same way it already + // does for a still-climbing unmeasured clock below. + if (is_content(node) || !node.ever_armed) { + advance_violation_streak(node, fqn, crossed_violation); + } break; case LifecycleObservedState::kUnreadable: case LifecycleObservedState::kNotManaged: @@ -679,6 +769,17 @@ class LifecycleExpectationTracker { /// restart loop, the case this detector most exists to catch) would have its clock /// released on every absence and never mature. bool ever_measured = false; + /// Whether this node has been armed by the reliability gate on at least one matched + /// tick, ever. Sticky: once true it stays true, because the fact it stands in for - + /// "the presence detector could report this node's departure" - is itself sticky + /// (NodeLivenessTracker tracks a key from its first armed tick onward, regardless of + /// its arm state afterwards). Read here rather than re-derived from `settled_observed` + /// or the live label: this node's LIFECYCLE state alone cannot say whether it is + /// armed, since arming also needs warmup, which this class does not track - guessing + /// from state would either miss a warming-up node that later arms, or wrongly treat a + /// node this class never measured active as armed. See the absence loop's kInactive + /// case for the one place this decides anything. + bool ever_armed = false; /// The label of the last kInactive observation, already trimmed to /// kMaxLifecycleLabelChars. Kept because the detail for a CONFIRMED violation names the /// state the node is stuck in, and absence carries no label of its own to name. Cleared @@ -779,8 +880,19 @@ class LifecycleExpectationTracker { /// unmeasured clock STILL CLIMBING counts under its cause rather than being dropped: /// absence advances it every tick and nothing can reset it any more, so it would have /// matured under that cause within a bounded number of ticks anyway, and dropping it - /// instead would let freeing a slot heal a fault. An idle entry contributes nothing - - /// idle entries are reclaimed before this ever runs. + /// instead would let freeing a slot heal a fault. A violation streak counts only once it + /// has already matured (past `grace`): for a node the presence detector could also have + /// reported (`ever_armed`), absence does not keep advancing a streak that has not yet + /// matured (see the kInactive case in update()'s absence loop), so counting one that had + /// not crossed `grace` would fabricate a violation the node never earned. A NEVER-armed + /// node's below-grace streak is the one case where that is not true - absence keeps + /// advancing it too, so left alone it would eventually mature the same way a climbing + /// unmeasured clock does - but this tally still leaves it uncounted rather than folding + /// it in: reaching this function with one still below `grace` needs `tracked_node_cap` + /// genuinely saturated by departed identities, and losing that one entry's evidence is + /// the safer direction to be incomplete in than risking a wrong count for every OTHER + /// entry this function has to keep classifying correctly. An idle entry, or a + /// live-but-unmatured streak, contributes nothing. void count_collapsed(const NodeState & node) { if (node.unmeasured_matured || node.unmeasured_clock > 0) { if (node.cause == LifecycleUnmeasuredCause::kUnreadable) { @@ -788,7 +900,7 @@ class LifecycleExpectationTracker { } else { ++collapsed_not_managed_; } - } else if (node.violation_streak > 0) { + } else if (node.violation_streak > grace_) { ++collapsed_inactive_; } } diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/lifecycle_shutdown_suppressor.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/lifecycle_shutdown_suppressor.hpp new file mode 100644 index 000000000..db2e3e80f --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/lifecycle_shutdown_suppressor.hpp @@ -0,0 +1,77 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include + +#include "ros2_medkit_graph_watchdog/reliability_gate.hpp" // DepartedLifecycle +#include "ros2_medkit_graph_watchdog/suppressor.hpp" + +namespace ros2_medkit_graph_watchdog { + +/// Self-suppression: a managed lifecycle node that reached a clean shutdown before it left +/// the graph was stopped on purpose, so node_death must not report its departure. Reads +/// `ReliabilityGate::departed_lifecycle_state_of(fqn)` - the seam that header documents for +/// exactly this - rather than owning any state of its own. +/// +/// A departure counts as clean only for two labels, and one of them still needs corroborating +/// evidence: +/// - `shuttingdown`: only reachable via a deliberate SHUTDOWN transition, so the label alone +/// is enough. +/// - `finalized`: NOT enough on its own. It is also where a node's on_error override lands +/// after ON_ERROR_FAILURE/ON_ERROR_ERROR out of `errorprocessing` - the standard way a +/// driver reports a hardware fault it cannot recover from, which is precisely the death +/// an operator needs reported, not silenced. So `finalized` only counts when the watcher +/// actually saw a transition into it (`saw_transition`) and never saw it pass through the +/// error branch (`!error_terminated`). Without an observed transition history the +/// departure is unclassified and stays reported. +/// +/// `unconfigured` is deliberately excluded: it is also the resting state of a node whose +/// configure() failed or that never activated at all, and suppressing on it would hide exactly +/// that startup failure. +/// +/// Stateless by design: a detector's own `configure()` (where the suppressor chain is built) +/// runs before any DetectorContext exists, so this class stores nothing at construction and +/// reads the gate fresh on every call instead. +class LifecycleShutdownSuppressor : public Suppressor { + public: + bool suppresses(const std::string & fqn, const DetectorContext & ctx) const override { + if (ctx.gate == nullptr) { + return false; // not yet wired (e.g. a bare-context test) -> nothing to read, abstain + } + const auto departed = ctx.gate->departed_lifecycle_state_of(fqn); + if (!departed.has_value()) { + return false; // never departed within retention, or this fqn was never lifecycle-tracked + } + if (departed->label == "shuttingdown") { + return true; + } + if (departed->label == "finalized") { + return departed->saw_transition && !departed->error_terminated; + } + return false; + } + + /// A departure's shutdown shape does not change after the fact - once a fqn's last + /// observed transition qualifies as clean, it stays clean for as long as the gate's + /// retention window remembers it at all. Reclaiming a suppressed key's tracker + /// bookkeeping is therefore sound PROVIDED that window outlives node_death's own reclaim + /// tick, which is what GraphWatchdogPlugin::compute_departed_retention_ticks() sizes it + /// for. + bool durable() const override { + return true; + } +}; + +} // namespace ros2_medkit_graph_watchdog diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/node_liveness_tracker.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/node_liveness_tracker.hpp new file mode 100644 index 000000000..b5cc979e0 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/node_liveness_tracker.hpp @@ -0,0 +1,327 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ros2_medkit_graph_watchdog { + +/// One sweep's verdict. `dead` maps every reported key to a human-readable detail phrase; +/// `keys_by_freshness` names the same keys ordered by MISS COUNT ascending - the most +/// recently departed key first, with the collapsed-count synthetic entry (see +/// NodeLivenessTracker::kCollapsedKey) always first of all when present. +/// +/// The ordering matters because the description built from `dead` is capped +/// (AggregatedFault::kMaxDescriptionChars) while `dead` itself is a std::map and so walks +/// lexicographically. On a graph carrying several long-dead entries, a fresh death whose +/// key sorts late alphabetically would otherwise be cut from the text - and, being fresh, +/// it is also the one thing the operator has not already been told. Freshest-first is what +/// keeps a capped description still naming the thing that just happened. +struct NodeDeathReport { + std::map dead; + std::vector keys_by_freshness; + /// LEVEL: true on every tick the DEPARTED (misses_ > 0) subset of the tracked map still + /// exceeds `tracked_key_cap` after collapsing every entry it safely could - i.e. entries + /// still mid-grace, which collapsing must never touch (see collapse_matured()'s own doc), + /// are alone enough to keep it over the cap. Stays true for as long as the condition + /// lasts. NOT the same condition LifecycleExpectationReport::tracking_saturated names for + /// the sibling tracker: an ARMED/PRESENT key is never refused here (see the class doc), so + /// this can no longer mean "a newly-armed key went unwatched." + bool tracking_saturated = false; + /// EDGE: the first tick of a saturation episode, for a caller's one-time warning. Re-arms + /// when the episode ends. + bool saturation_started = false; +}; + +/// Pure presence/absence state machine for "armed nodes that vanish." +/// +/// Each call to update() is handed two sets over the SAME key space: +/// present - every key visible in this tick's graph snapshot (the liveness signal) +/// armed - the subset the reliability gate currently allows a raise for +/// +/// A key becomes TRACKED (added to `known_`) the first time it is armed, and stays tracked +/// from then on regardless of its later arm state - so a node that is present but not +/// currently armed (still warming up, or lifecycle-inactive) is never mistaken for dead: +/// presence alone keeps a tracked key's miss counter at zero. A key is reported dead once +/// its consecutive-miss counter exceeds `miss_grace`. +/// +/// update() never removes a key BY AGE. An unsuppressed, still-dead entry has to stay +/// reported for as long as it is actually dead - the alternative is a detector that quietly +/// stops saying so once enough time has passed, which is the one failure mode this exists +/// to rule out. prune() reclaims a key only once it has been DURABLY suppressed for long +/// enough (see its own doc below); ordinary, unsuppressed churn - unique identities that +/// arm once and are never seen again, which this detector's zero-config "every armed App is +/// a candidate" scope makes the common case on any fleet with per-run or per-namespace +/// names - has no age-based or suppression-based removal path at all. `tracked_key_cap` is +/// what bounds the map against exactly that growth. +/// +/// **What the cap bounds, and why it differs from its own past shape.** The unbounded +/// growth is entirely in DEPARTED keys: a dead node's entry is never +/// reclaimed by anything but a durable suppressor, so unique dead identities accumulate for +/// the life of the process. A PRESENT key does not have that problem - its count at any +/// instant is bounded by the live graph, which is bounded by reality (a large single-robot +/// graph runs to roughly a hundred nodes, a ten-robot fleet to a few hundred - see +/// kDefaultTrackedKeyCap) - and it carries no state worth reclaiming anyway (misses_ == 0). +/// So the cap here bounds only the DEPARTED subset (misses_ > 0): admission of an +/// ARMED/PRESENT key is therefore NEVER refused and never evicts anything, at any map size. +/// A tracked_key_cap smaller than the live graph is consequently not saturation - the extra +/// present entries simply coexist with a full departed set - only a departed set that stays +/// oversized even after every eligible collapse is. +/// +/// Collapsing (folding a departed identity into the one collapsed_dead_count_, see +/// collapse_matured()) may ONLY apply to an entry that has ACTUALLY crossed `miss_grace_` - +/// a confirmed death. Collapsing one that is merely mid-grace would report a death the node +/// has not earned, permanently (the identity is erased, so the node returning cannot un-ring +/// it) - the exact fabrication an earlier shape of this cap produced. An immature departed +/// entry is therefore never evicted by the cap either: it is kept, tracked exactly like any +/// other departed entry, until it either matures (and becomes a collapse candidate) or the +/// node returns (and the entry goes idle again). Under sustained pressure from still-maturing +/// churn the departed set may consequently exceed tracked_key_cap_ for a few ticks - bounded +/// by recent churn rate times miss_grace_, not by how long the process has been running, +/// which is the growth this cap exists to prevent in the first place. +/// +/// Why this differs from LifecycleExpectationTracker's make_room(), despite looking +/// parallel: that tracker keeps TWO clocks per key (a violation streak and an unmeasured +/// clock), so one node's entry can be PRESENT and carrying evidence at the same time - that +/// third state is what makes its idle/collapsible/present-and-evidential three-way eviction +/// order meaningful, and is also why IT can evict a present entry without losing anything +/// (the evidence lives in the clocks, not in presence). This tracker keeps exactly one piece +/// of per-key state (`misses_`), so a key is always either present-and-empty (idle) or +/// absent-and-evidential (departed) - there is no third state, and evicting a present entry +/// here would only ever throw away a key that could simply be re-admitted next tick anyway, +/// at the cost of losing any death that happens to land in the eviction window (only ONLINE +/// nodes re-enter `armed`, so an evicted-while-present node that then dies is never +/// re-admitted at all). The sibling's shape does not transfer. +class NodeLivenessTracker { + public: + /// Sentinel `prune_ticks` meaning "never reclaim anything." The default for callers that + /// only care about update()'s presence/absence bookkeeping (short-lived tests, mostly); + /// real wiring (NodeDeathDetector::configure()) always passes an explicit, clamped value. + static constexpr int kNoPrune = std::numeric_limits::max(); + + /// Default `tracked_key_cap` - same value, for the same reason, as + /// LifecycleExpectationTracker::kDefaultTrackedNodeCap: a large single-robot ROS 2 graph + /// runs to roughly a hundred nodes, a ten-robot fleet sharing one domain to a few hundred, + /// so even an operator whose graph is entirely in scope (every App here, versus only + /// require_active's named subset there) stays comfortably under it at a cost of a few + /// hundred KB. Growth past it can only come from DEPARTED identity churn, which is exactly + /// what the cap exists to bound - a present key never counts against it at all (see the + /// class doc). + static constexpr int kDefaultTrackedKeyCap = 512; + + /// Most DEPARTED, matured (confirmed dead) entries kept individually named when the + /// departed set has to be shrunk; the rest are collapsed into the one count. Mirrors + /// LifecycleExpectationTracker::kMaxNamedDepartedEntries and its reasoning: past three, a + /// name is pure cost (AggregatedFault::kMaxDescriptionChars cannot fit a fourth alongside + /// the freshest entries this tracker already prioritises). + static constexpr int kMaxNamedDepartedEntries = 3; + + /// Key for the collapsed-departed synthetic report entry. '!' sorts below every character + /// a real key can start with here ('/'), matching LifecycleExpectationTracker's identical + /// convention - though this class additionally puts it FIRST in keys_by_freshness itself + /// (see update()) rather than relying on describe_ordered()'s map-order fallback: unlike + /// the sibling's `newly_*` lists (only entries crossing THIS tick), keys_by_freshness here + /// already names every CURRENTLY dead key every tick, so leaving the collapsed entry out + /// of it would place it last, not first, and a capped description could cut the one line + /// that tells the operator identities are being lost to capacity pressure at all. + static constexpr const char * kCollapsedKey = "!collapsed"; + + explicit NodeLivenessTracker(int miss_grace, int prune_ticks = kNoPrune, int tracked_key_cap = kDefaultTrackedKeyCap) + : miss_grace_(miss_grace), prune_ticks_(prune_ticks), tracked_key_cap_(tracked_key_cap < 1 ? 1 : tracked_key_cap) { + } + + NodeDeathReport update(const std::set & present, const std::set & armed) { + NodeDeathReport report; + // Unconditional: a PRESENT/armed key is never refused a slot and never costs an + // existing entry its own - see the class doc for why this differs from the sibling. + for (const auto & key : armed) { + known_.insert(key); // no-op if already tracked + } + + for (const auto & key : known_) { + if (present.count(key) > 0) { + misses_[key] = 0; + } else { + ++misses_[key]; + } + } + + // Bounds the DEPARTED subset only, and only by collapsing entries that have ACTUALLY + // matured - never a present or still-immature one. Runs BEFORE the report below is + // built, so a just-collapsed identity never simultaneously appears both individually and + // inside the collapsed count. + report.tracking_saturated = enforce_departed_cap(); + + std::vector> by_miss_count; + for (const auto & key : known_) { + const int misses = misses_.at(key); + if (misses > miss_grace_) { + report.dead[key] = "node " + key + " disappeared (" + std::to_string(misses) + " missed cycles)"; + by_miss_count.emplace_back(misses, key); + } + } + // Fewest misses first (freshest death first); the key itself breaks ties so the + // ordering - and so the capped description - does not reshuffle tick to tick on its + // own. + std::sort(by_miss_count.begin(), by_miss_count.end()); + report.keys_by_freshness.reserve(by_miss_count.size() + 1); + // See kCollapsedKey's own doc for why this goes first rather than through + // describe_ordered()'s map-order fallback. + if (collapsed_dead_count_ > 0) { + report.dead[kCollapsedKey] = "and " + std::to_string(collapsed_dead_count_) + + " more node(s) disappeared; not named individually (tracked_key_cap is full)"; + report.keys_by_freshness.emplace_back(kCollapsedKey); + } + for (auto & entry : by_miss_count) { + report.keys_by_freshness.push_back(std::move(entry.second)); + } + report.saturation_started = report.tracking_saturated && !saturated_last_tick_; + saturated_last_tick_ = report.tracking_saturated; + return report; + } + + /// Reclaim bookkeeping for a key in `suppressed` once it has been suppressed on more than + /// `prune_ticks_` CONSECUTIVE calls. Call this after update() each tick, with the set of + /// keys a detector's DURABLE suppressors currently vote to suppress - never the raw + /// "removed from this tick's report" set, and never a non-durable suppressor's verdict + /// (see suppressor.hpp's own doc on why only a durable veto makes reclaiming sound). + /// + /// A key missing from `suppressed` has its streak reset to zero on this same call - not + /// merely left alone - so a veto that lifts even once starts the count over rather than + /// merely pausing it. A key that is never suppressed therefore has streak zero forever + /// and can never be reclaimed no matter how long it stays dead: this asymmetry is what + /// makes an unsuppressed death permanent-until-acknowledged while still letting a + /// permanently-vetoed one stop costing memory. + void prune(const std::set & suppressed) { + for (auto it = known_.begin(); it != known_.end();) { + const auto & key = *it; + int & streak = suppressed_streak_[key]; + streak = suppressed.count(key) > 0 ? streak + 1 : 0; + if (streak > prune_ticks_) { + suppressed_streak_.erase(key); + misses_.erase(key); + it = known_.erase(it); + } else { + ++it; + } + } + } + + /// How many keys are currently tracked (known_/misses_ size) - a test seam so a suite can + /// assert the map stays bounded under churn without exposing the map itself. Never + /// counts collapsed-departed identities: they no longer have one to count. Includes + /// present entries, which are not bounded by tracked_key_cap_ at all (see the class doc). + std::size_t tracked_count() const { + return known_.size(); + } + + /// The keys currently tracked. A read-only view, not a copy of any mutable state a caller + /// could corrupt - used by a detector that keeps its OWN per-key bookkeeping alongside + /// this tracker's (e.g. a remembered App::id for allowlist matching after a key dies) and + /// needs to reclaim it in step with prune() without this class exposing prune()'s own + /// internal streak bookkeeping to do it. + const std::set & known_keys() const { + return known_; + } + + private: + /// How many known_ keys currently carry evidence (misses_ > 0) - present (idle) entries + /// never count. What tracked_key_cap_ actually bounds; see the class doc. + std::size_t departed_count() const { + std::size_t n = 0; + for (const auto & [key, misses] : misses_) { + (void)key; + if (misses > 0) { + ++n; + } + } + return n; + } + + /// Shrinks the DEPARTED subset of known_ to at most tracked_key_cap_ where it safely can, + /// by collapsing MATURED entries (misses_ > miss_grace_) - never a present or immature + /// one; see the class doc for why. Two stages, mirroring + /// LifecycleExpectationTracker::make_room()'s identical shape for the same reason: first + /// collapse down to kMaxNamedDepartedEntries (keeping a few individually named when that + /// alone is enough to clear the cap), and only if the departed set is STILL over cap - + /// meaning immature entries alone already account for the excess - collapse the rest of + /// the matured ones too, since every one collapsed still reduces the pressure even when it + /// cannot fully relieve it. + /// + /// Returns true when the departed set remains over tracked_key_cap_ even after collapsing + /// every matured entry available - i.e. immature (still mid-grace) entries alone exceed + /// the cap. That condition is left standing rather than forced down: an immature entry is + /// never a collapse candidate, so there is nothing further this function may safely do + /// about it. + bool enforce_departed_cap() { + if (departed_count() <= static_cast(tracked_key_cap_)) { + return false; + } + collapse_matured(kMaxNamedDepartedEntries); + if (departed_count() <= static_cast(tracked_key_cap_)) { + return false; + } + collapse_matured(0); + return departed_count() > static_cast(tracked_key_cap_); + } + + /// Fold MATURED departed entries (misses_ > miss_grace_) into collapsed_dead_count_ until + /// at most `keep_named` remain individually tracked. NEVER touches an immature departed + /// entry (0 < misses_ <= miss_grace_) or a present one - collapsing either would report + /// (or lose) a death the node has not actually earned yet. Lexicographically LAST first, + /// so the survivors are a stable prefix and the same graph always keeps the same names - + /// mirrors LifecycleExpectationTracker::collapse_departed's identical choice. + void collapse_matured(std::size_t keep_named) { + std::vector matured; + for (const auto & [key, misses] : misses_) { + if (misses > miss_grace_) { + matured.push_back(key); + } + } + for (std::size_t i = matured.size(); i > keep_named; --i) { + const std::string & key = matured[i - 1]; + ++collapsed_dead_count_; + known_.erase(key); + misses_.erase(key); + suppressed_streak_.erase(key); + } + } + + int miss_grace_; + int prune_ticks_; + int tracked_key_cap_; ///< clamped to at least 1 in the constructor; bounds departed_count() only + std::set known_; + std::map misses_; + std::map suppressed_streak_; + bool saturated_last_tick_ = false; ///< for the saturation EDGE (see NodeDeathReport) + /// Departed, MATURED entries folded into a count to shrink the departed set back toward + /// tracked_key_cap_. Monotone within one tracker lifetime by design, mirroring the + /// sibling: a gateway restart or a detector reconfigure both replace this object wholesale + /// (see node_death_detector.cpp's own configure()), which is the only thing that ever + /// resets it. A collapsed identity returning does not decrement it - the identity itself + /// is gone, so there is no way to tell which of possibly-several collapsed departures came + /// back - the aggregate fault this count keeps raised clears only via a reconfigure (which + /// also re-baselines whatever is present at that point) or an operator's explicit + /// acknowledgement. + int collapsed_dead_count_ = 0; +}; + +} // namespace ros2_medkit_graph_watchdog diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/suppressor.hpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/suppressor.hpp new file mode 100644 index 000000000..08a414bcc --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/include/ros2_medkit_graph_watchdog/suppressor.hpp @@ -0,0 +1,89 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include +#include +#include + +#include "ros2_medkit_graph_watchdog/detector.hpp" // DetectorContext + +namespace ros2_medkit_graph_watchdog { + +/// One vote on whether a single candidate entity should be dropped from an aggregated +/// fault this tick. A detector holds a CHAIN of these and drops a candidate the moment +/// any of them votes yes - see apply_suppressors() below. +/// +/// Pure by contract: an implementation reads only the key it is asked about and the +/// DetectorContext it is handed (gate, snapshot, clock), and never raises or clears a +/// fault itself. The interface exists so more than one detector can share the same +/// suppression mechanism (AllowlistSuppressor already serves node_death and tf_stale) +/// without either one owning the other's config surface. +class Suppressor { + public: + virtual ~Suppressor() = default; + + /// True to suppress `entity_key` this tick; false to abstain (say nothing about it). + virtual bool suppresses(const std::string & entity_key, const DetectorContext & ctx) const = 0; + + /// Whether a "yes" from this suppressor is a standing fact about the key rather than a + /// condition that can later stop holding. + /// + /// This is what a detector's own pruning may safely rely on: reclaiming a tracked key's + /// bookkeeping while it is suppressed only avoids a false report FOR AS LONG AS the + /// suppression keeps holding. A veto that CAN lift (a live condition on the current + /// graph, say) offers no such guarantee - the moment it lifts, the key would be a live, + /// unsuppressed death with no tracking left to report it, which is a false heal that + /// silently outlives the very condition that caused it. A veto that is durable never + /// lifts for a given key once it has fired, so reclaiming under it loses nothing: the + /// key would never be reported again regardless of whether its bookkeeping survives. + /// + /// Defaults to false, which is the safe assumption for a suppressor nobody has reasoned + /// about yet. Override to true only for a suppressor whose "yes" is permanent per key. + virtual bool durable() const { + return false; + } +}; + +/// Drop every key in `affected` that any suppressor in `chain` votes to suppress. +/// +/// Order-independent: a key survives only if EVERY suppressor abstains, so which one +/// happens to run first never changes the result. Returns how many keys were dropped, for +/// callers that want to know without re-diffing `affected` themselves. +inline std::size_t apply_suppressors(std::map & affected, + const std::vector & chain, const DetectorContext & ctx) { + if (chain.empty()) { + return 0; + } + std::size_t dropped = 0; + for (auto it = affected.begin(); it != affected.end();) { + bool suppressed = false; + for (const Suppressor * s : chain) { + if (s != nullptr && s->suppresses(it->first, ctx)) { + suppressed = true; + break; + } + } + if (suppressed) { + it = affected.erase(it); + ++dropped; + } else { + ++it; + } + } + return dropped; +} + +} // namespace ros2_medkit_graph_watchdog diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/lifecycle_expectation_detector.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/lifecycle_expectation_detector.cpp index 477fc25d0..290be0395 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/lifecycle_expectation_detector.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/lifecycle_expectation_detector.cpp @@ -60,14 +60,26 @@ constexpr int kDefaultGrace = 5; // that reports a node on its very first not-active tick, with no warning and no default. // // The upper end used to be INT_MAX - 1, which is not a wide tolerance but an off switch with -// no warning attached. `grace` bounds two things at once: how long a node may read not-active -// before being reported, AND how long a node that LEFT the graph while not-active sits in the -// tracker's pending set - during which GRAPH_NODE_INACTIVE's clear is withheld for every -// node, so the fault can neither raise nor heal for anybody. At INT_MAX - 1 that is roughly -// 24 days at the shipped cadence; five minutes is already an extravagant allowance for a -// managed node to reach `active`, and it makes the worst-case withhold something an operator -// can reason about. A deployment that genuinely needs longer wants a slower tick, not a -// detector that is silent for weeks. +// no warning attached. `grace` bounds how long a node that STAYS IN THE GRAPH may read +// not-active before being reported, and how long a node that returns from a departure still +// inactive takes to (re-)mature - both advance on present ticks, which this value directly +// caps. At INT_MAX - 1 that is roughly 24 days at the shipped cadence either way; five minutes +// is already an extravagant allowance for a managed node to reach `active`, and it makes the +// worst-case PRESENT withhold something an operator can reason about. A deployment that +// genuinely needs longer wants a slower tick, not a detector that is silent for weeks. +// +// Whether it also bounds a node that leaves the graph before its streak matures depends on +// whether that node was ever ARMED (see LifecycleExpectationTracker's own class doc). For one +// the reliability gate has armed at least once, `grace` does NOT bound it: absence holds a +// below-grace streak rather than advancing it, so that entry sits in the tracker's pending +// set for as long as the node stays gone, however long that is - GRAPH_NODE_DISAPPEARED can +// report that departure instead, so nothing here needs to hurry it. That is not a new way to +// withhold GRAPH_NODE_INACTIVE's clear for every OTHER node - the clear was already gated on +// every required node's status being settled, so one node this indecisive already blocked it +// before this bound existed. For a node that was NEVER armed - node_death only ever tracks a +// node the gate has armed, so this is the one departure nothing else can report - `grace` +// DOES still bound it: absence keeps advancing the streak too, so it matures within `grace` + +// `absence_grace` + 1 ticks either way. constexpr std::int64_t kMaxGrace = 300; /// Consecutive ticks an entry must match ONLY unmanaged nodes before the typo warning /// fires. One transient tick is not evidence: discover_apps() wraps the per-node service @@ -308,9 +320,19 @@ class LifecycleExpectationDetector : public Detector { if (state.has_value()) { entry_has_managed_match.insert(id); } + // Whether the gate currently allows a fault to be raised for this SAME app.id - the + // exact predicate node_death's own presence detector uses to decide whether it will + // ever be able to track this node at all (reliability_allows(), see + // node_death_detector.cpp's identical call). Reading it here, from the one gate both + // detectors share on the same tick, is what makes "armed" trustworthy rather than a + // guess: it is not derived from this node's own lifecycle label, it IS the fact the + // presence detector would itself consult if asked right now. The tracker needs it to + // tell a node the presence detector could someday report from one it structurally + // never can - see LifecycleExpectationTracker's own class doc. + const bool armed = reliability_allows(ctx.gate, app.id); // One match per (entry, node): the tracker keys violations by NODE, so two // namesakes are both reported instead of one silently replacing the other. - matches.push_back(LifecycleMatch{id, fqn, state}); + matches.push_back(LifecycleMatch{id, fqn, state, armed}); } } std::set matched_entries; diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/node_death_detector.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/node_death_detector.cpp new file mode 100644 index 000000000..4be730eff --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/detectors/node_death_detector.cpp @@ -0,0 +1,541 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "ros2_medkit_gateway/core/providers/introspection_provider.hpp" +#include "ros2_medkit_graph_watchdog/aggregated_fault.hpp" +#include "ros2_medkit_graph_watchdog/allowlist_suppressor.hpp" +#include "ros2_medkit_graph_watchdog/detector_config_keys.hpp" +#include "ros2_medkit_graph_watchdog/detector_registry.hpp" +#include "ros2_medkit_graph_watchdog/graph_fault_codes.hpp" +#include "ros2_medkit_graph_watchdog/lifecycle_shutdown_suppressor.hpp" +#include "ros2_medkit_graph_watchdog/node_liveness_tracker.hpp" +#include "ros2_medkit_graph_watchdog/suppressor.hpp" + +namespace ros2_medkit_graph_watchdog { + +namespace { +// Fault severity scale: SEVERITY_WARN=1, SEVERITY_ERROR=2 (ros2_medkit_msgs/msg/Fault.msg). +// A dead node is a total, silent loss of whatever it provided - as severe as QoS starvation. +constexpr std::uint8_t kNodeDeathSeverity = ros2_medkit_msgs::msg::Fault::SEVERITY_ERROR; + +// Ticks a tracked node may be absent before it is reported dead. Mirrors +// GraphWatchdogPlugin's own kDefaultNodeDeathMissGrace, which needs this value before +// this detector's own configure() has run (see that plugin's +// compute_departed_retention_ticks()). +constexpr int kDefaultMissGrace = 2; + +// Standalone fallback only. A real launch always injects "prune_grace" (the plugin's own +// default, also 60) once wired through GraphWatchdogPlugin::set_context() - this only +// matters for a bare configure() call that never goes through the plugin, e.g. a unit test. +constexpr int kDefaultPruneGrace = 60; + +// Bounds for detectors.node_death.tracked_node_cap - same range, same reasoning, as +// lifecycle_expectation_detector.cpp's identical kMinTrackedNodeCap/kMaxTrackedNodeCap: the +// range check runs on the WIDE integer before narrowing, for the same reason miss_grace and +// prune_grace above do. Zero is refused rather than clamped: a cap of nothing means this +// detector tracks nothing, which is exactly the silence it exists to prevent. +constexpr std::int64_t kMinTrackedNodeCap = 1; +constexpr std::int64_t kMaxTrackedNodeCap = 16384; + +/// Whether `fqn`'s leaf is one of ROS 2's own short-lived CLI helper nodes. +/// +/// `ros2 topic echo`, `ros2 param get` and every other `ros2` CLI invocation spin up a +/// real node named `_ros2cli_` - there is no hidden-node filter in discovery, so the +/// gateway turns each one into an ordinary App. A few seconds of one of those is enough to +/// arm it, and the moment the CLI process exits looks exactly like a death; because every +/// invocation gets a fresh pid, letting these through would accumulate one permanently +/// "dead" entry per CLI invocation for the life of the gateway. The prefix is rcl's own +/// naming convention for hidden nodes, so matching it is an exact structural check, not a +/// heuristic over operator-chosen names. +bool is_ros2cli_node(const std::string & fqn) { + const auto slash = fqn.rfind('/'); + const std::string leaf = slash == std::string::npos ? fqn : fqn.substr(slash + 1); + return leaf.rfind("_ros2cli_", 0) == 0; +} +} // namespace + +/// Watches every App the graph carries for one that was armed and then vanishes, and +/// raises GRAPH_NODE_DISAPPEARED naming it. Zero-config: unlike lifecycle_expectation's +/// operator-declared require_active list, every App in scope is a candidate - see the +/// package design note this mirrors. +/// +/// Liveness is `App::is_online`, never mere membership in the snapshot. In runtime +/// discovery a dead node's App leaves the snapshot entirely, so the two happen to +/// coincide; but a manifest keeps a bound App present with only `is_online` cleared once +/// its node disappears, so counting membership alone would make a manifest node immortal. +/// A managed lifecycle node that merely deactivates keeps `is_online=true` (its process is +/// still alive; only its ROS 2 lifecycle state changed), so that is never mistaken for +/// death either - lifecycle state is lifecycle_expectation's concern, not this one's. +/// +/// A key is tracked - and so can ever be reported dead - only once it has been armed by +/// the reliability gate at least once. A manifest App that never comes online starts +/// `is_online=false` and so is never armed, so it can never be falsely called dead; a node +/// still inside its warmup window gets the same protection. +/// +/// Composable/component nodes hosted in one process die together: if the container +/// process dies, every node it hosted vanishes from the snapshot on the same tick, and all +/// of them are folded into the one aggregated fault rather than one fault each (see +/// AggregatedFault) - individually named up to AggregatedFault::kMaxDescriptionChars and +/// NodeLivenessTracker::kMaxNamedDepartedEntries, whichever runs out first; past either +/// limit the remainder are still counted, just not named (see NodeLivenessTracker's own +/// collapsed-count doc). +class NodeDeathDetector : public Detector { + public: + std::string id() const override { + return "node_death"; + } + + void configure(const nlohmann::json & config) override { + std::vector warnings; + + // tick_interval_ms is plugin plumbing (see plugin_injected_detector_keys()), but the + // miss_grace floor below is computed FROM it, so a malformed value here would corrupt + // that derivation silently unless it too is validated and named. + int tick_interval_ms = kDefaultTickIntervalMs; + if (config.contains("tick_interval_ms")) { + const auto & value = config["tick_interval_ms"]; + // ROS parameters are int64: get the wide value and range-check it BEFORE narrowing, + // or a value above INT_MAX wraps back into the accepted band and passes silently. + const std::int64_t wide = value.is_number_integer() ? value.get() : 0; + if (wide > 0 && wide <= std::numeric_limits::max()) { + tick_interval_ms = static_cast(wide); + } else { + warnings.push_back("'tick_interval_ms' must be a positive integer up to " + + std::to_string(std::numeric_limits::max()) + "; keeping " + + std::to_string(tick_interval_ms)); + } + } + + int miss_grace = kDefaultMissGrace; + if (config.contains("miss_grace")) { + const auto & value = config["miss_grace"]; + const std::int64_t wide = value.is_number_integer() ? value.get() : -1; + if (wide >= 0 && wide <= kMaxNodeDeathGraceTicks) { + miss_grace = static_cast(wide); + } else { + warnings.push_back("'miss_grace' must be an integer in 0.." + std::to_string(kMaxNodeDeathGraceTicks) + + "; keeping " + std::to_string(kDefaultMissGrace)); + } + } + // The wall-clock floor: a death is reported once misses EXCEED miss_grace, i.e. after + // miss_grace + 1 ticks, so raise the tick count until that window spans at least + // kMinNodeDeathWindowMs - and say so, since a silently-shortened tolerance is exactly + // how a fast tick turns one stale graph-cache generation into an immediate false death. + const int floor = min_node_death_miss_grace(tick_interval_ms); + if (miss_grace < floor) { + warnings.push_back("'miss_grace' " + std::to_string(miss_grace) + " spans only " + + std::to_string((miss_grace + 1) * tick_interval_ms) + "ms at a " + + std::to_string(tick_interval_ms) + "ms tick, under the " + + std::to_string(kMinNodeDeathWindowMs) + "ms floor one graph-cache refresh needs; using " + + std::to_string(floor)); + miss_grace = floor; + } + + int prune_grace = kDefaultPruneGrace; + if (config.contains("prune_grace")) { + const auto & value = config["prune_grace"]; + const std::int64_t wide = value.is_number_integer() ? value.get() : -1; + if (wide >= 0 && wide <= kMaxNodeDeathGraceTicks) { + prune_grace = static_cast(wide); + } else { + warnings.push_back("'prune_grace' must be an integer in 0.." + std::to_string(kMaxNodeDeathGraceTicks) + + "; keeping " + std::to_string(kDefaultPruneGrace)); + } + } + + // The hard bound on how many keys tracker_ keeps state for at once. Same validation + // shape as miss_grace/prune_grace above; see NodeLivenessTracker::kDefaultTrackedKeyCap + // for why 512 is comfortable even against this detector's full-graph scope. + int tracked_node_cap = NodeLivenessTracker::kDefaultTrackedKeyCap; + if (config.contains("tracked_node_cap")) { + const auto & value = config["tracked_node_cap"]; + const std::int64_t wide = value.is_number_integer() ? value.get() : -1; + if (wide >= kMinTrackedNodeCap && wide <= kMaxTrackedNodeCap) { + tracked_node_cap = static_cast(wide); + } else { + warnings.push_back("'tracked_node_cap' must be an integer in " + std::to_string(kMinTrackedNodeCap) + ".." + + std::to_string(kMaxTrackedNodeCap) + "; keeping the default (" + + std::to_string(NodeLivenessTracker::kDefaultTrackedKeyCap) + ")"); + } + } + + build_suppressor_chain(config, warnings); + // A remembered id-match belongs to the allowlist that computed it; build_suppressor_chain + // just rebuilt (or dropped) that allowlist above, so a value captured under the OLD + // config would misrepresent the new one - clear rather than carry it forward. + id_allowlisted_.clear(); + // Deliberately NOT reset here, unlike id_allowlisted_ above: ever_raised_ is a fact about + // this PROCESS's history ("has this instance ever put a genuine FAILED on the wire"), + // not about the current config. A live reconfigure (an operator editing the allowlist, + // say) can run with a death still outstanding and absent; resetting the flag would make + // the freshly-rebuilt tracker unable to ever re-create evidence for a node that is not + // currently online and armed, so the standing fault would get neither a further FAILED + // NOR a PASSED once the node genuinely returns - stuck for the rest of the process's + // life. A real process restart needs no reset either: it gets a brand-new + // NodeDeathDetector object, and ever_raised_'s own member initializer already starts + // false - see tick()'s own comment for the guard this flag drives. + collect_unknown_detector_keys(config, known_keys(), warnings); + warnings_ = std::move(warnings); + warnings_logged_ = false; + + // A death can only ever be reported once misses_ exceeds miss_grace, so prune_ticks + // must be at least miss_grace + 1 - otherwise a durably-suppressed key could be + // reclaimed the very tick it would first have been reported, and the report would be + // silently lost rather than silently suppressed. + const int prune_ticks = std::max(prune_grace, miss_grace + 1); + tracker_ = NodeLivenessTracker(miss_grace, prune_ticks, tracked_node_cap); + tracked_node_cap_.store(tracked_node_cap); + } + + std::size_t tracked_count_for_test() const override { + return tracked_count_.load(); + } + + /// status_json() runs on an HTTP handler thread concurrently with tick() on the plugin's + /// tick thread (see the base class doc), and NodeLivenessTracker itself carries no + /// synchronization of its own - reading tracker_ directly from here would race tick()'s + /// writes to it. tick() publishes into these atomics once per sweep instead, so this + /// method touches nothing tick() also touches. + /// + /// `tracking_saturated` is the one an operator acts on: it means the number of DEPARTED + /// (not yet confirmed dead, or already collapsed) identities exceeds tracked_node_cap even + /// after collapsing every confirmed death it safely could - so a newly confirmed death + /// among them may not get an individual name, only a place in the collapsed count. An + /// armed/present key is never refused tracking by this cap (see NodeLivenessTracker's own + /// class doc), so this is never "a node is going unchecked." The fix is a + /// require_active-style narrower allowlist entry set is not available here (this detector + /// is zero-config by design), so it is either less simultaneous departure churn or a larger + /// tracked_node_cap - both of which need tracked_count/tracked_node_cap alongside it to + /// reason about. + nlohmann::json status_json() const override { + return {{"tracked_count", tracked_count_.load()}, + {"tracking_saturated", saturated_.load()}, + {"tracked_node_cap", tracked_node_cap_.load()}}; + } + + void tick(DetectorContext & ctx) override { + if (!ctx.snapshot) { + return; + } + log_warnings_once(ctx); + + std::set present; + std::set armed; + for (const auto & app : ctx.snapshot->apps) { + // A peer-aggregated app carries no ROS binding of its own (PeerClient::parse_app + // never reads x-medkit.ros2.node), so effective_fqn() is empty for every one of + // them. Tracking them would collapse the whole peer fleet onto a single "" key: one + // online peer app would mask every other peer app's departure. + if (app.source.rfind("peer:", 0) == 0) { + continue; + } + // Liveness is the bound node actually running, not mere presence in the snapshot - + // see the class doc for why membership alone would make a manifest node immortal. + if (!app.is_online) { + continue; + } + // Keyed on the STABLE fqn, not App::id: id is recomputed every sweep and only gets a + // namespace prefix once a bare-name collision currently exists anywhere in the + // graph, so a live node's id can change out from under a key built from it. + const std::string key = app.effective_fqn(); + if (key.empty() || is_ros2cli_node(key)) { + continue; + } + present.insert(key); + if (reliability_allows(ctx.gate, app.id)) { // the gate is keyed by App::id + armed.insert(key); + } + // Capture the allowlist's id-form verdict WHILE the entity is still present and + // app.id is in hand - suppresses() alone can never do this for a dead key, since by + // the time one is checked below the entity has already left ctx.snapshot entirely. + // See allowlist_suppressor.hpp's class doc. Overwritten every tick rather than + // latched, so a key whose id later stops colliding (and so stops matching) does not + // stay exempt on a stale reading - App::id, unlike the fqn, is not stable. + if (allowlist_ != nullptr) { + if (allowlist_->allows(app.id)) { + id_allowlisted_.insert(key); + } else { + id_allowlisted_.erase(key); + } + } + } + + auto report = tracker_.update(present, armed); + + std::set keys_before_suppression; + for (const auto & [key, detail] : report.dead) { + (void)detail; + keys_before_suppression.insert(key); + } + // Suppress via the generic chain_ (each suppressor gets just the fqn key - covers + // AllowlistSuppressor's own fqn/leaf forms and LifecycleShutdownSuppressor's fqn-keyed + // departed-state lookup) OR via a remembered allowlist id-match captured above while the + // key was still present. The id form bypasses chain_ dispatch entirely rather than + // going through Suppressor::suppresses() - see allowlist_suppressor.hpp for why. + for (auto it = report.dead.begin(); it != report.dead.end();) { + bool suppressed = id_allowlisted_.count(it->first) > 0; + if (!suppressed) { + for (const Suppressor * s : chain_) { + if (s != nullptr && s->suppresses(it->first, ctx)) { + suppressed = true; + break; + } + } + } + if (suppressed) { + it = report.dead.erase(it); + } else { + ++it; + } + } + + // Only a DURABLE suppressor's veto may feed prune() (see suppressor.hpp). Re-checking + // against durable_chain_ specifically - rather than simply "no longer in report.dead" + // - is required because a non-durable suppressor could remove a key from this tick's + // report even while chain_ also carries a durable one; durability is a property of + // WHICH suppressor vetoed a key, not of whether the key survived the filter. The id-form + // allowlist match is exactly as durable as its fqn/leaf forms (both read the same + // AllowlistSuppressor::durable()==true), so it feeds prune() the same way. + std::set suppressed_for_prune; + for (const auto & key : keys_before_suppression) { + if (report.dead.count(key) > 0) { + continue; // survived the filter - nothing suppressed it this tick + } + if (id_allowlisted_.count(key) > 0) { + suppressed_for_prune.insert(key); + continue; + } + for (const Suppressor * s : durable_chain_) { + if (s != nullptr && s->suppresses(key, ctx)) { + suppressed_for_prune.insert(key); + break; + } + } + } + tracker_.prune(suppressed_for_prune); + // Bound id_allowlisted_ the same way tracker_ itself is bounded: drop any key the + // tracker no longer knows about at all, so a key reclaimed by prune() above does not + // leave a same-tick orphan entry that would otherwise sit here forever (this map is + // never itself consulted for a key outside known_keys(), but an unbounded map under + // identity churn is exactly the failure N11 exists to rule out for tracker_ itself). + for (auto it = id_allowlisted_.begin(); it != id_allowlisted_.end();) { + it = tracker_.known_keys().count(*it) > 0 ? std::next(it) : id_allowlisted_.erase(it); + } + // Published once per sweep, after prune() has settled this tick's count - see + // status_json()'s own doc for why this is the only cross-thread-safe way to read it. + tracked_count_.store(tracker_.tracked_count()); + saturated_.store(report.tracking_saturated); + if (report.saturation_started && ctx.gateway_node) { + // Once per EPISODE, not once per process: the latch re-arms when saturation ends (see + // NodeLivenessTracker's own saturated_last_tick_), so a later, real saturation is not + // silent because an earlier one already spent the warning. Mirrors + // lifecycle_expectation_detector.cpp's identical saturation_started handling. + RCLCPP_WARN(ctx.gateway_node->get_logger(), + "graph_watchdog node_death: %d departed identities exceed tracked_node_cap even after " + "collapsing every confirmed death - a newly confirmed death may not be named " + "individually until some of these clear or are collapsed in turn. Every armed/present " + "node is still being checked; raise 'tracked_node_cap' if this graph genuinely churns " + "this many simultaneous departures at once", + tracked_node_cap_.load()); + } + + // ctx.clear_fault(), unlike ctx.raise_fault(), carries no reliability-gate check of its + // own, so a level-triggered "affected is empty" reaches the fault manager as a genuine + // PASSED unconditionally the moment the fault client is ready - and this detector is + // zero-config, so unlike lifecycle_expectation's require_active it has no fixed, + // enumerable set of entries to ask "have I re-observed all of them yet" before trusting + // its own silence. ever_raised_ stands in for that: this process may only clear + // GRAPH_NODE_DISAPPEARED once it has ITSELF actually handed a FAILED request for it to + // the fault client at least once - not merely decided one was warranted. See + // DetectorContext::raise_fault's own doc for exactly what that return value does and + // does not prove. + // + // Tracked-key COUNT is deliberately not the signal, though it may look like an equally + // good proxy: any App elsewhere in the graph - the gateway's own node included - tends + // to arm within a tick or two of startup, which would make tracker_.tracked_count() go + // non-zero long before the SPECIFIC node a stored, outstanding fault names has been + // re-observed at all. A guard keyed on that count would open on an unrelated node's + // arming and clear a fault this process has not actually re-verified - exactly the + // silent, ungated heal this guard exists to rule out. report.dead's own emptiness says + // nothing that specific either (every dead key is merged into one aggregate), which is + // why the guard is keyed on whether THIS instance has raised at all, not on what it + // currently knows about. + if (report.dead.empty() && !ever_raised_) { + return; + } + // emit_ordered()'s own return is what earns ever_raised_, not report.dead's mere + // non-emptiness: raise_fault() can still decline to send even with a non-empty report - + // Advisory/Off mode, no client wired yet, or (concretely) the fault_manager service not + // being ready right at this tick, none of which are visible from here without reading + // the return value. Setting the flag from intent rather than delivery is the exact + // shape of false-clear this guard exists to rule out, one level deeper: kill a node + // while the service is down, wait past miss_grace, restore the service, let the node + // return - a flag set from "the report was non-empty" would already be true by then + // though async_send_request() had never actually been called for it, and the empty + // report on return would then emit a PASSED for an occurrence never even attempted. + // Retried + // every tick the report stays non-empty (emit_ordered() runs unconditionally above the + // guard once report.dead is non-empty, whether or not ever_raised_ is already true), so + // a raise that fails here is simply attempted again next tick rather than lost. + const bool sent = aggregated_.emit_ordered(ctx, report.dead, report.keys_by_freshness); + if (!report.dead.empty() && sent) { + ever_raised_ = true; + } + } + + private: + static const std::set & known_keys() { + static const std::set keys{"allowlist", "miss_grace", "prune_grace", + "suppress", "tick_interval_ms", "tracked_node_cap"}; + return keys; + } + + void log_warnings_once(const DetectorContext & ctx) { + if (warnings_.empty() || warnings_logged_ || !ctx.gateway_node) { + return; + } + for (const auto & warning : warnings_) { + RCLCPP_WARN(ctx.gateway_node->get_logger(), "graph_watchdog node_death: %s", warning.c_str()); + } + warnings_logged_ = true; + } + + /// Rebuild owned_suppressors_/chain_/durable_chain_ from this detector's own config + /// slice. `suppress` is a list of mechanism names, dispatched here: + /// "allowlist" - an AllowlistSuppressor over the configured `allowlist` set. + /// "lifecycle" - a (stateless) LifecycleShutdownSuppressor. + /// anything else - named but unrecognised; warned about, not built. + /// + /// Both mechanisms are opt-in: a configured `allowlist` that `suppress` never names has + /// NO effect, and says so - naming an entry on the list is not, by itself, a request to + /// suppress it. Every field is re-validated from scratch on every call (type, then array + /// element shape), and every rejection is named rather than silently dropped, so a typo'd + /// key or a malformed entry never reads as "working" from the operator's side. + /// + /// Rebuilds all three containers unconditionally, so a re-configure() never leaves + /// chain_/durable_chain_ holding a pointer into a freed owned_suppressors_ entry. + void build_suppressor_chain(const nlohmann::json & config, std::vector & warnings) { + std::set allow_set; + if (config.contains("allowlist")) { + const auto & value = config["allowlist"]; + if (value.is_array()) { + for (const auto & entry : value) { + if (entry.is_string() && !entry.get().empty()) { + allow_set.insert(entry.get()); + } else { + warnings.push_back("'allowlist' entries must be non-empty strings; skipping " + entry.dump()); + } + } + } else { + warnings.push_back("'allowlist' must be an array of strings; ignoring it"); + } + } + + owned_suppressors_.clear(); + chain_.clear(); + durable_chain_.clear(); + allowlist_ = nullptr; // owned_suppressors_.clear() just freed whatever this pointed to + + bool allowlist_named = false; + if (config.contains("suppress")) { + const auto & value = config["suppress"]; + if (value.is_array()) { + for (const auto & entry : value) { + if (!entry.is_string()) { + warnings.push_back("'suppress' entries must be strings; skipping " + entry.dump()); + continue; + } + const std::string name = entry.get(); + if (name == "allowlist") { + allowlist_named = true; + } else if (name == "lifecycle") { + owned_suppressors_.push_back(std::make_unique()); + } else { + warnings.push_back("unknown 'suppress' entry '" + name + "'; ignoring it"); + } + } + } else { + warnings.push_back("'suppress' must be an array of strings; ignoring it"); + } + } + + if (allowlist_named) { + auto suppressor = std::make_unique(std::move(allow_set)); + // Kept as a second, typed pointer alongside the polymorphic chain_ entry below - + // node_death_detector.cpp's own tick() needs to call allows() directly (an + // AllowlistSuppressor-specific method, not part of the generic Suppressor interface) + // while an entity is still present. See tick()'s own comment and + // allowlist_suppressor.hpp's class doc for why. + allowlist_ = suppressor.get(); + owned_suppressors_.push_back(std::move(suppressor)); + } else if (!allow_set.empty()) { + // A configured allowlist that suppress does not name has no effect - naming it in + // suppress is what opts it in, matching every other suppression mechanism this + // framework carries. + warnings.push_back("'allowlist' has " + std::to_string(allow_set.size()) + " entr" + + (allow_set.size() == 1 ? std::string("y") : std::string("ies")) + + " configured but 'suppress' does not name \"allowlist\"; the list is inert"); + } + + for (const auto & s : owned_suppressors_) { + chain_.push_back(s.get()); + if (s->durable()) { + durable_chain_.push_back(s.get()); + } + } + } + + NodeLivenessTracker tracker_{kDefaultMissGrace}; + std::vector warnings_; + bool warnings_logged_ = false; + AggregatedFault aggregated_{graph_fault_codes::kNodeDisappeared, kNodeDeathSeverity}; + std::vector> owned_suppressors_; + std::vector chain_; + std::vector durable_chain_; ///< Subset of chain_ safe to feed prune(). See tick(). + /// Non-owning; points into owned_suppressors_ when "allowlist" is named in suppress, null + /// otherwise. Typed (not just another chain_ entry) because tick() needs allows(), which + /// is not part of the generic Suppressor interface - see allowlist_suppressor.hpp. + AllowlistSuppressor * allowlist_ = nullptr; + /// fqn -> "the last App::id observed for this key, while present, matched the allowlist". + /// Refreshed every tick a key is present (see tick()); the value from its last live tick + /// is what a dead key is judged by, since id is unavailable once the entity has left + /// ctx.snapshot. Bounded to tracker_.known_keys() at the end of every tick. + std::set id_allowlisted_; + /// Whether THIS detector instance has itself genuinely raised GRAPH_NODE_DISAPPEARED at + /// least once - the ungated-clear guard. See tick()'s own comment for why this, and not + /// tracker_.tracked_count(), is the right granularity. + bool ever_raised_ = false; + std::atomic tracked_count_{0}; ///< Cross-thread-safe mirror of tracker_.tracked_count(). + std::atomic saturated_{false}; ///< tracked_node_cap refused a key on the last tick + std::atomic tracked_node_cap_{NodeLivenessTracker::kDefaultTrackedKeyCap}; ///< as of the last configure() +}; + +REGISTER_DETECTOR(NodeDeathDetector, "node_death") + +} // namespace ros2_medkit_graph_watchdog diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/graph_watchdog_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/graph_watchdog_plugin.cpp index f6a7d54d7..27189fa63 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/graph_watchdog_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/src/graph_watchdog_plugin.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -35,6 +36,7 @@ #include "ros2_medkit_gateway/core/http/error_codes.hpp" #include "ros2_medkit_graph_watchdog/aggregated_fault.hpp" // kGraphWatchdogEntityId #include "ros2_medkit_graph_watchdog/detector_config.hpp" +#include "ros2_medkit_graph_watchdog/detector_config_keys.hpp" // min_node_death_miss_grace #include "ros2_medkit_graph_watchdog/detector_registry.hpp" #include "ros2_medkit_graph_watchdog/reliability_gate.hpp" @@ -73,6 +75,43 @@ typename rclcpp::Client::SharedPtr create_client_in_group(rclcpp::Node return node->create_client(name, rmw_qos_profile_services_default, group); #endif } + +/// What node_death's own configure() will resolve `prune_grace` to, mirroring that +/// detector's exact validation (a non-negative integer no larger than +/// kMaxNodeDeathGraceTicks) so this function and the config actually injected into +/// node_death's own dcfg (set_context()'s detector loop, below) can never disagree - both +/// call this, so there is exactly one place that decision is made. +/// +/// Falls back to `plugin_prune_grace` (clamped into node_death's own accepted range - a +/// defensive clamp, not another validation pass: load_parameters() only checks +/// `>= 0` for its own field) exactly when node_death's own configure() would ALSO fall back: +/// the per-detector value is absent, the wrong JSON type, negative, or past +/// kMaxNodeDeathGraceTicks. Before this existed, the plugin's own retention math used this +/// same fallback while set_context() injected the plugin default only when the per-detector +/// key was ABSENT - so a PRESENT but malformed `detectors.node_death.prune_grace` reached +/// node_death's configure() unfiltered, which falls back to ITS OWN hardcoded default +/// instead of `plugin_prune_grace`. The two fallbacks coincide at their shared default (60) +/// and only diverge when an operator's plugin-scope `prune_grace` differs from it - which is +/// exactly the gap a test that never varies the plugin-scope value off 60 cannot catch. +int resolve_node_death_prune_grace(const nlohmann::json & config_snapshot, int plugin_prune_grace) { + if (config_snapshot.contains("detectors") && config_snapshot["detectors"].is_object()) { + const auto & detectors = config_snapshot["detectors"]; + if (detectors.contains("node_death") && detectors["node_death"].is_object()) { + const auto & node_death_cfg = detectors["node_death"]; + // ROS/JSON integers are wide: read int64_t and range-check it BEFORE narrowing to int, + // mirroring node_death_detector.cpp's own identical read of this same field. A value + // above kMaxNodeDeathGraceTicks (or above INT_MAX) narrowed first can wrap back into + // an accepted-looking band and pass silently. + if (node_death_cfg.contains("prune_grace") && node_death_cfg["prune_grace"].is_number_integer()) { + const std::int64_t wide = node_death_cfg["prune_grace"].get(); + if (wide >= 0 && wide <= kMaxNodeDeathGraceTicks) { + return static_cast(wide); + } + } + } + } + return std::min(plugin_prune_grace, static_cast(kMaxNodeDeathGraceTicks)); +} } // namespace ros2_medkit_gateway::IntrospectionResult @@ -144,31 +183,42 @@ int GraphWatchdogPlugin::compute_departed_retention_ticks(const nlohmann::json & // configure() has not run yet at the point set_context() needs this value, so its // own kDefaultMissGrace/clamping cannot be reused directly here. int node_death_miss_grace = kDefaultNodeDeathMissGrace; - int node_death_prune_grace = prune_grace_; if (config_snapshot.contains("detectors") && config_snapshot["detectors"].is_object()) { const auto & detectors = config_snapshot["detectors"]; if (detectors.contains("node_death") && detectors["node_death"].is_object()) { const auto & node_death_cfg = detectors["node_death"]; + // ROS/JSON integers are wide: read int64_t and range-check it BEFORE narrowing to int, + // mirroring node_death_detector.cpp's own identical read of this same field. A + // value above kMaxNodeDeathGraceTicks (or above INT_MAX) narrowed first can wrap back + // into an accepted-looking band and pass the ">= 0" check silently - node_death's own + // configure() would reject that same value and keep its default, so a plugin that + // narrowed first would size this window from a number the detector never actually + // uses, under- or over-running the retention this function exists to get right. if (node_death_cfg.contains("miss_grace") && node_death_cfg["miss_grace"].is_number_integer()) { - const int candidate = node_death_cfg["miss_grace"].get(); - if (candidate >= 0) { - node_death_miss_grace = candidate; - } - } - // Same reason we read miss_grace here: prune_grace is a per-detector key whose - // plugin-scope value is only a default, so the retention window must be computed from - // whatever node_death will ACTUALLY clamp its prune_ticks_ to. Reading prune_grace_ - // unconditionally would under-run the retention whenever an operator raises - // detectors.node_death.prune_grace, and node_death would then re-raise a - // cleanly-shut-down node it should have silently reclaimed (see the formula below). - if (node_death_cfg.contains("prune_grace") && node_death_cfg["prune_grace"].is_number_integer()) { - const int candidate = node_death_cfg["prune_grace"].get(); - if (candidate >= 0) { - node_death_prune_grace = candidate; + const std::int64_t wide = node_death_cfg["miss_grace"].get(); + if (wide >= 0 && wide <= kMaxNodeDeathGraceTicks) { + node_death_miss_grace = static_cast(wide); } } } } + // prune_grace is a per-detector key whose plugin-scope value is only a default, so the + // retention window must be computed from whatever node_death will ACTUALLY clamp its + // prune_ticks_ to - see resolve_node_death_prune_grace()'s own doc for why this must be + // the SAME function set_context() uses to inject node_death's own dcfg, not a parallel + // re-derivation of the same rule. + const int node_death_prune_grace = resolve_node_death_prune_grace(config_snapshot, prune_grace_); + // node_death's own configure() unconditionally raises miss_grace to its wall-clock floor + // (min_node_death_miss_grace(), detector_config_keys.hpp) whenever the configured tick is fast + // enough to need it - "unconditionally" because that floor applies to node_death's + // DEFAULT miss_grace too, not only an operator-set one. A fast tick is exactly this + // plugin's own tick_interval_ms_, already loaded by the time set_context() calls this + // (load_parameters() runs first). Applied here too, and outside the "detectors.node_death + // exists" check above: mirroring only the RAW config value - or only flooring it when an + // operator happened to configure node_death at all - would leave this window sized for a + // miss_grace smaller than the one node_death will actually use, which is the exact + // under-run the rest of this function's comment describes. + node_death_miss_grace = std::max(node_death_miss_grace, min_node_death_miss_grace(tick_interval_ms_)); // prune_ticks mirrors node_death's OWN prune_ticks_ clamp (node_death_detector.cpp's // configure()): max(prune_grace, miss_grace + 1). The departed-lifecycle label must // survive not just past the FIRST tick node_death evaluates suppression on @@ -177,14 +227,14 @@ int GraphWatchdogPlugin::compute_departed_retention_ticks(const nlohmann::json & // node_death's own RECLAIM tick (T_depart + miss_grace + prune_ticks): only a durable // suppressor (lifecycle_clean_shutdown IS one, see suppressor.hpp) may feed // NodeLivenessTracker::prune(), and that reclaim can only happen while the suppressor - // still actively vetoes the id - i.e. while the label is still cached. If the label - // expired even one tick earlier (the old `max(prune_grace, miss_grace + 1)` retention - // - the SAME length as prune_ticks, but anchored miss_grace ticks later at - // T_depart instead of T_death), the suppressor would abstain right at the reclaim - // tick, node_death would RAISE instead of silently reclaiming, and - because the - // label is now gone for good - the id could never be suppressed again: a permanent - // false GRAPH_NODE_DISAPPEARED for a cleanly-shut-down node. The extra "+1" keeps one - // full tick of margin past the reclaim tick itself. See + // still actively vetoes the id - i.e. while the label is still cached. A retention window + // anchored at T_depart instead and merely as long as prune_ticks itself (`max(prune_grace, + // miss_grace + 1)`, with no `+ node_death_miss_grace + 1` on top) would expire miss_grace + // ticks too early: the suppressor would then abstain right at the reclaim tick, node_death + // would RAISE instead of silently reclaiming, and - because the label is now gone for good + // - the id could never be suppressed again: a permanent false GRAPH_NODE_DISAPPEARED for a + // cleanly-shut-down node. The extra "+1" below keeps one full tick of margin past the + // reclaim tick itself. See // test_node_death_integration.cpp's CleanShutdownDepartureIsReclaimedNotReRaisedPastRetention // for the regression this formula fixes. const int prune_ticks = std::max(node_death_prune_grace, node_death_miss_grace + 1); @@ -268,6 +318,17 @@ void GraphWatchdogPlugin::set_context(ros2_medkit_gateway::PluginContext & conte if (!dcfg.contains("prune_grace")) { dcfg["prune_grace"] = prune_grace_; } + if (id == "node_death") { + // node_death is the one detector this plugin also predicts the behaviour of BEFORE + // configure() runs (compute_departed_retention_ticks(), above, sizes the lifecycle + // watcher's own retention off it) - so what actually reaches its configure() here + // must be byte-for-byte the value that prediction assumed, not merely "the operator's + // value when well-formed, else whatever node_death happens to default to on its own." + // Re-resolving (rather than trusting the injection above) also covers a PRESENT but + // malformed detectors.node_death.prune_grace, which the absence check above leaves + // untouched even though node_death's own configure() will still reject it. + dcfg["prune_grace"] = resolve_node_death_prune_grace(config_snapshot, prune_grace_); + } detector->configure(dcfg); } catch (const std::exception & e) { log_error("detector '" + id + "' configure() threw: " + std::string(e.what()) + "; skipping"); diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/harness.py b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/harness.py index 7378e2a88..a6257df1d 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/harness.py +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/harness.py @@ -585,18 +585,229 @@ def assert_fault_absent_throughout(test_case, port, code, duration, interval=0.5 f'>= interval ({interval}s)') +def assert_fault_persists_throughout(test_case, port, code, duration, interval=0.5): + """Fail unless `code` is present on EVERY poll for `duration` seconds. + + The mirror of assert_fault_absent_throughout, and it owes the same proof: a fault + surface that stops answering must fail the assertion rather than satisfy it. Poll the + same endpoint the same way; a request error, a non-200, or a body that does not parse + is a failure, not a skipped sample. Waiting for a single sighting + (``assertIsNotNone(poll_faults(...))``) proves the fault appeared once, not that it + stayed - it returns on the first match and says nothing about a channel that goes dark + on poll three of twenty. Use this for every scenario whose claim is sustained presence + over a window, not merely "present right now". + + Parameters + ---------- + test_case : unittest.TestCase + Used for the actual assertion calls, so a failure here reports through the normal + unittest failure path rather than a bare ``AssertionError`` from a free function. + port : int + Gateway HTTP port. + code : str + The ``fault_code`` that must be present on every poll. + duration : float + Total seconds to keep polling. + interval : float + Sleep between polls in seconds. + + """ + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + duration + polls = 0 + while time.monotonic() < deadline: + try: + response = requests.get(f'{base}/faults', timeout=5) + except requests.exceptions.RequestException as exc: + test_case.fail( + f'/faults became unreachable {polls} poll(s) into a {duration}s persistence ' + f'window (could not ask, which is not the same as "asked, and {code} is ' + f'still there"): {exc}') + return + if response.status_code != 200: + test_case.fail( + f'/faults answered HTTP {response.status_code} {polls} poll(s) into a ' + f'{duration}s persistence window - the channel died mid-window, which this ' + 'assertion must not read as "still there"') + return + codes = {item.get('fault_code') for item in response.json().get('items', [])} + test_case.assertIn( + code, codes, + f'{code} was missing {polls} poll(s) into a {duration}s window that was supposed ' + 'to stay confirmed the whole way through') + polls += 1 + time.sleep(interval) + test_case.assertGreater( + polls, 0, + f'the {duration}s persistence window never actually polled /faults - duration must ' + f'be >= interval ({interval}s)') + + +def assert_fault_describes_only( + test_case, port, code, required, forbidden, duration, interval=0.5): + """Fail unless `code` is present with a matching description on every poll. + + On every poll for `duration` seconds, `code` must be present and its description must + contain every needle in `required` and none in `forbidden`. `required` and `forbidden` + are iterables of plain substrings. The aggregated fault + names several entities in one description, so this is how a scenario says "this node is + named and that one is not" without depending on ordering. Polls the whole window, not a + single sample: a description this assertion must hold true FOR THE DURATION (one node + named, a sibling never named) is exactly the kind of claim a channel that goes dark + mid-window can falsely satisfy if only checked once. + + Parameters + ---------- + test_case : unittest.TestCase + Used for the actual assertion calls, so a failure here reports through the normal + unittest failure path rather than a bare ``AssertionError`` from a free function. + port : int + Gateway HTTP port. + code : str + The ``fault_code`` that must be present, with a matching description, on every poll. + required : iterable of str + Substrings that must all appear in the fault's description on every poll. + forbidden : iterable of str + Substrings that must never appear in the fault's description on any poll. + duration : float + Total seconds to keep polling. + interval : float + Sleep between polls in seconds. + + """ + required = list(required) + forbidden = list(forbidden) + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + duration + polls = 0 + while time.monotonic() < deadline: + try: + response = requests.get(f'{base}/faults', timeout=5) + except requests.exceptions.RequestException as exc: + test_case.fail( + f'/faults became unreachable {polls} poll(s) into a {duration}s description ' + f'window (could not ask, which is not the same as "asked, and the ' + f'description still holds"): {exc}') + return + if response.status_code != 200: + test_case.fail( + f'/faults answered HTTP {response.status_code} {polls} poll(s) into a ' + f'{duration}s description window - the channel died mid-window, which this ' + 'assertion must not read as "the description still holds"') + return + items = {item.get('fault_code'): item for item in response.json().get('items', [])} + fault = items.get(code) + if fault is None: + test_case.fail( + f'{code} was missing {polls} poll(s) into a {duration}s window that was ' + 'supposed to keep describing it') + description = fault.get('description', '') + for needle in required: + test_case.assertIn( + needle, description, + f"{code}'s description dropped required text {needle!r} {polls} poll(s) " + f'into a {duration}s window: {description!r}') + for needle in forbidden: + test_case.assertNotIn( + needle, description, + f"{code}'s description carries forbidden text {needle!r} {polls} poll(s) " + f'into a {duration}s window: {description!r}') + polls += 1 + time.sleep(interval) + test_case.assertGreater( + polls, 0, + f'the {duration}s description window never actually polled /faults - duration must ' + f'be >= interval ({interval}s)') + + +def assert_fault_never_names(test_case, port, code, forbidden, duration, interval=0.5): + """Fail if `code`'s description EVER names a forbidden needle, whatever `code` itself does. + + Distinct from the three assertions above in what it does NOT require: it takes no position + on whether `code` is raised at all. ``assert_fault_absent_throughout`` is too strong for a + claim like "no disappearance names THIS node" - a correct detector raising `code` for some + OTHER entity would trip it for a reason that has nothing to do with the node under test. + ``assert_fault_describes_only`` is too strong the other way: it REQUIRES `code` present on + every poll, which is wrong for a scenario where the code staying fully absent is itself a + correct outcome. This is the narrower claim in between: whenever `code` happens to be + present, on any poll, none of `forbidden` may appear in its description; `code` being absent + on a given poll is not a failure. + + Same channel-alive discipline as the other window assertions: a request error or a non-200 + is a failure naming which poll and why, never a silently-skipped sample. + + Parameters + ---------- + test_case : unittest.TestCase + Used for the actual assertion calls, so a failure here reports through the normal + unittest failure path rather than a bare ``AssertionError`` from a free function. + port : int + Gateway HTTP port. + code : str + The ``fault_code`` whose description must never name a forbidden needle. Its own + presence or absence is not judged. + forbidden : iterable of str + Substrings that must never appear in `code`'s description, on any poll where `code` is + present. + duration : float + Total seconds to keep polling. + interval : float + Sleep between polls in seconds. + + """ + forbidden = list(forbidden) + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + duration + polls = 0 + while time.monotonic() < deadline: + try: + response = requests.get(f'{base}/faults', timeout=5) + except requests.exceptions.RequestException as exc: + test_case.fail( + f'/faults became unreachable {polls} poll(s) into a {duration}s window (could ' + f'not ask, which is not the same as "asked, and {code} never named a forbidden ' + f'entity"): {exc}') + return + if response.status_code != 200: + test_case.fail( + f'/faults answered HTTP {response.status_code} {polls} poll(s) into a ' + f'{duration}s window - the channel died mid-window, which this assertion must ' + 'not read as "never named a forbidden entity"') + return + items = {item.get('fault_code'): item for item in response.json().get('items', [])} + fault = items.get(code) + if fault is not None: + description = fault.get('description', '') + for needle in forbidden: + test_case.assertNotIn( + needle, description, + f"{code}'s description named forbidden text {needle!r} {polls} poll(s) " + f'into a {duration}s window: {description!r}') + polls += 1 + time.sleep(interval) + test_case.assertGreater( + polls, 0, + f'the {duration}s window never actually polled /faults - duration must be >= interval ' + f'({interval}s)') + + class _FlakyFaultsHandler(http.server.BaseHTTPRequestHandler): """Stands in for a gateway whose ``GET /faults`` answers normally, then dies. - Answers 200 with an empty fault list for the first ``healthy_polls`` requests to - ``{API_BASE_PATH}/faults`` (this class's own attribute, set per instantiation via - ``make_handler``), then drops the connection with no response at all for every + Answers 200 with ``{'items': healthy_items}`` for the first ``healthy_polls`` requests + to ``{API_BASE_PATH}/faults`` (both class attributes, set per instantiation via + `_FlakyFaultsServer`), then drops the connection with no response at all for every request after that - the same "channel gone" shape a crashed or hung fault_manager produces, distinct from an ordinary 503 (also exercised via ``dead_status``). + `healthy_items` defaults to empty (a healthy-but-silent fault surface, what the + absence proof needs); the persistence and describes-only proofs set it to a list + holding the fault they poll for, so the same handler can stand in for a surface that + is healthy AND SAYING SOMETHING before it goes dark. """ healthy_polls = 2 dead_status = None # None = drop the connection; an int = answer with that status instead + healthy_items = [] # the `items` list served while healthy def do_GET(self): if self.path != f'{API_BASE_PATH}/faults': @@ -611,14 +822,14 @@ def do_GET(self): self.send_response(type(self).dead_status) self.end_headers() return - body = json.dumps({'items': []}).encode() + body = json.dumps({'items': type(self).healthy_items}).encode() self.send_response(200) self.send_header('Content-Type', 'application/json') self.send_header('Content-Length', str(len(body))) self.end_headers() self.wfile.write(body) - def log_message(self, log_format, *args): + def log_message(self, format, *args): # noqa: A002 - matches the base class's own name pass # keep test output quiet - this is expected traffic, not diagnostics @@ -626,15 +837,16 @@ class _FlakyFaultsServer: """Context manager for a local `_FlakyFaultsHandler` HTTP server. Starts on a free port in a daemon thread and tears itself down on exit - the - boilerplate every leg of `prove_silence_proof_catches_a_dead_fault_surface` below - needs, factored out once so each leg reads as the claim it is checking rather than - server plumbing. + boilerplate every leg of `prove_silence_proof_catches_a_dead_fault_surface` (and its + persistence/describes-only siblings) below needs, factored out once so each leg reads + as the claim it is checking rather than server plumbing. """ - def __init__(self, healthy_polls, dead_status): + def __init__(self, healthy_polls, dead_status, healthy_items=None): handler = type( '_Handler', (_FlakyFaultsHandler,), - {'healthy_polls': healthy_polls, 'dead_status': dead_status}) + {'healthy_polls': healthy_polls, 'dead_status': dead_status, + 'healthy_items': [] if healthy_items is None else healthy_items}) self._server = http.server.HTTPServer(('127.0.0.1', 0), handler) self.port = self._server.server_address[1] self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) @@ -701,6 +913,171 @@ def prove_silence_proof_catches_a_dead_fault_surface(test_case): test_case, fake.port, 'GRAPH_NODE_INACTIVE', duration=1.0, interval=0.2) +def prove_persistence_proof_catches_a_dead_fault_surface(test_case): + """Prove `assert_fault_persists_throughout` catches a `/faults` that dies mid-window. + + The mirror of `prove_silence_proof_catches_a_dead_fault_surface`, for the opposite + claim: a fault that must stay CONFIRMED for a whole window, not one that must stay + absent. Self-contained, same stand-in server, same three-part proof: + + (1) the naive pattern this fix replaces - `poll_faults` + `assertIsNotNone` - actually + DOES pass against a `/faults` that answers healthily a couple of times and then dies: + it returns the moment `code` is first seen, before the channel goes dark, so it never + observes the death at all; (2) `assert_fault_persists_throughout` raises against the + SAME dying channel, for both dead-channel shapes (a dropped connection and a 503); (3) + a channel that never dies, and keeps answering `code` present the whole time, does NOT + trip the fixed helper - otherwise the RED result above would be meaningless. + + Raises whatever `test_case`'s own assertions raise on failure; callers use it from + inside a test method exactly like any other assertion helper. + """ + code = 'GRAPH_NODE_INACTIVE' + healthy_items = [{'fault_code': code, 'description': f'{code} present'}] + for dead_status, label in ((None, 'a dropped connection'), (503, 'a 503 response')): + with _FlakyFaultsServer(healthy_polls=2, dead_status=dead_status, + healthy_items=healthy_items) as fake: + # (1) The OLD pattern this fix replaces: prove it actually passes on this + # exact dying channel - it returns on the first sighting, before the death. + test_case.assertIsNotNone( + poll_faults(fake.port, code, timeout=2.0, interval=0.2), + f'the OLD pattern (poll_faults + assertIsNotNone) did NOT pass against a ' + f'/faults that later died via {label} - this self-test no longer ' + 'demonstrates the hole the fix closes') + # (2) The fixed helper must fail against the identical dying channel. + with test_case.assertRaises( + AssertionError, + msg=f'assert_fault_persists_throughout did not fail when /faults died ' + f'mid-window via {label} - it would pass on the exact ' + 'channel-death this fix exists to catch'): + assert_fault_persists_throughout( + test_case, fake.port, code, duration=2.0, interval=0.2) + + # (3) The companion proof: a channel that never dies, and keeps answering `code` + # present, must not trip the assertion, or the RED result above would be meaningless. + with _FlakyFaultsServer(healthy_polls=10_000, dead_status=None, + healthy_items=healthy_items) as fake: + assert_fault_persists_throughout(test_case, fake.port, code, duration=1.0, interval=0.2) + + +def prove_describes_only_proof_catches_a_dead_fault_surface(test_case): + """Prove `assert_fault_describes_only` catches a `/faults` that dies mid-window. + + Same shape as the silence and persistence self-tests above, for the third claim this + package's scenarios rely on: a description that must hold ("names alpha, never beta") + for a whole window, not merely on one lucky read before the channel goes dark. + + (1) `poll_fault_describing` (the existing single-shot reader) DOES pass against a + `/faults` that answers correctly a couple of times and then dies - it returns the + moment the description matches, never observing the death; (2) + `assert_fault_describes_only` raises against the SAME dying channel, for both + dead-channel shapes; (3) a channel that never dies, and keeps describing the fault + correctly the whole time, does NOT trip the fixed helper. + """ + code = 'GRAPH_NODE_INACTIVE' + healthy_items = [{'fault_code': code, 'description': 'alpha is gone, beta is fine'}] + for dead_status, label in ((None, 'a dropped connection'), (503, 'a 503 response')): + with _FlakyFaultsServer(healthy_polls=2, dead_status=dead_status, + healthy_items=healthy_items) as fake: + # (1) The OLD pattern this fix replaces: a single successful description read + # proves nothing about the rest of the window. + found, _description = poll_fault_describing( + fake.port, code, ['alpha'], timeout=2.0, interval=0.2) + test_case.assertTrue( + found, + f'the OLD pattern (poll_fault_describing) did NOT pass against a /faults ' + f'that later died via {label} - this self-test no longer demonstrates the ' + 'hole the fix closes') + # (2) The fixed helper must fail against the identical dying channel. + with test_case.assertRaises( + AssertionError, + msg=f'assert_fault_describes_only did not fail when /faults died ' + f'mid-window via {label} - it would pass on the exact ' + 'channel-death this fix exists to catch'): + assert_fault_describes_only( + test_case, fake.port, code, required=['alpha'], forbidden=['gamma'], + duration=2.0, interval=0.2) + + # (3) The companion proof: a channel that never dies, and keeps the description + # correct the whole time, must not trip the assertion. + with _FlakyFaultsServer(healthy_polls=10_000, dead_status=None, + healthy_items=healthy_items) as fake: + assert_fault_describes_only( + test_case, fake.port, code, required=['alpha'], forbidden=['gamma'], + duration=1.0, interval=0.2) + + +def prove_never_names_proof_catches_a_wrongly_scoped_absence(test_case): + """Prove `assert_fault_never_names` succeeds where a whole-code absence check would not. + + The gap this closes: a claim like "no disappearance names THIS node" is not the same claim + as "this code never appears at all". A correct detector raising `code` for some OTHER entity + should leave a "never names X" assertion GREEN - it is exactly the case + `assert_fault_absent_throughout` cannot tell apart from a real defect, since it only ever + looks at whether the code is present, never at what it names. + + (1) A `/faults` that answers healthily throughout, with `code` present but naming only an + unrelated entity, fails the OLD pattern (`assert_fault_absent_throughout` on the bare code) - + the concrete false failure this fix replaces, demonstrated rather than asserted in prose. (2) + The SAME server does not trip `assert_fault_never_names`. (3) A server whose `code` DOES name + the forbidden entity at some point trips the new helper - or it would never fail on the + actual defect it exists to catch. (4) The same channel-alive discipline the other three + window assertions already carry: a dead channel mid-window fails this one too. + + Raises whatever `test_case`'s own assertions raise on failure; callers use it from inside a + test method exactly like any other assertion helper. + """ + code = 'GRAPH_NODE_DISAPPEARED' + forbidden_entity = '/target/allowlisted' + unrelated_items = [ + {'fault_code': code, 'description': f'{code}: node /other/unrelated is gone'}] + naming_items = [ + {'fault_code': code, 'description': f'{code}: node {forbidden_entity} is gone'}] + + with _FlakyFaultsServer(healthy_polls=10_000, dead_status=None, + healthy_items=unrelated_items) as fake: + # (1) The OLD pattern this fix replaces: a whole-code absence check trips on a raise for + # an unrelated entity - the false failure this fix exists to stop, demonstrated rather + # than asserted. + with test_case.assertRaises( + AssertionError, + msg='assert_fault_absent_throughout did NOT fail against a code raised for an ' + 'unrelated entity - this self-test no longer demonstrates the false failure ' + 'the fix replaces'): + assert_fault_absent_throughout(test_case, fake.port, code, duration=1.0, interval=0.2) + + # (2) The fixed helper must NOT fail on the identical server: the code is present, but + # never names the forbidden entity. + assert_fault_never_names( + test_case, fake.port, code, forbidden=[forbidden_entity], duration=1.0, interval=0.2) + + with _FlakyFaultsServer(healthy_polls=10_000, dead_status=None, + healthy_items=naming_items) as fake: + # (3) The fixed helper MUST fail once the description actually names the forbidden + # entity - or it would never catch the defect it exists for. + with test_case.assertRaises( + AssertionError, + msg='assert_fault_never_names did not fail when the description named the ' + 'forbidden entity - it would pass on the exact defect this fix exists to ' + 'catch'): + assert_fault_never_names( + test_case, fake.port, code, forbidden=[forbidden_entity], + duration=1.0, interval=0.2) + + # (4) Channel-alive discipline, matching the other three window assertions: a dead channel + # mid-window must fail this one too, not be read as "never named a forbidden entity". + for dead_status, label in ((None, 'a dropped connection'), (503, 'a 503 response')): + with _FlakyFaultsServer(healthy_polls=2, dead_status=dead_status, + healthy_items=unrelated_items) as fake: + with test_case.assertRaises( + AssertionError, + msg=f'assert_fault_never_names did not fail when /faults died mid-window ' + f'via {label} - it would pass on the exact channel-death this fix ' + 'exists to catch'): + assert_fault_never_names( + test_case, fake.port, code, forbidden=[forbidden_entity], + duration=2.0, interval=0.2) + + def poll_cleared(port, code, timeout=30.0, interval=0.5): """Poll the GLOBAL ``GET /faults`` endpoint until `code` is absent. diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_node_death_boundary_e2e.test.py b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_node_death_boundary_e2e.test.py new file mode 100644 index 000000000..1b983360e --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_node_death_boundary_e2e.test.py @@ -0,0 +1,1429 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""node_death boundary, config and instrument e2e: the seam with lifecycle_expectation. + +Sibling of test_node_death_e2e.test.py and test_node_death_suppression_e2e.test.py, same shape. +What makes this file different is B1 and B3: their claims are about lifecycle_expectation ALONE, +independent of node_death, so an absence assertion there still discriminates a correct detector +from no detector at all - see their own class docstrings for what was actually observed. + +Runs as EIGHT separate CTest targets (see CMakeLists.txt). WATCHDOG_E2E_SCENARIO selects which +launch and which assertions run: + +- "b1_inactive_present": a require_active node (managed_lifecycle, never activated) sits in the + graph past `grace`. GRAPH_NODE_INACTIVE raises and names it - GENUINELY GREEN, because + lifecycle_expectation already ships this. GRAPH_NODE_DISAPPEARED stays absent for the whole + window - absence alone cannot distinguish a detector that correctly stays silent here from + one that raises nothing at all, but staying silent is also the permanently correct behaviour, + since this node never leaves the graph. +- "b2_inactive_below_grace_then_gone": the same node, now driven to "active" FIRST (arming it + for node_death - see TestBoundaryInactiveBelowGraceThenGone's own class docstring for why + that is not optional), then to "inactive" via a real DEACTIVATE transition, then killed after + being observed non-active for only a couple of ticks - comfortably below `grace`. Absence may + continue a violation that has already matured, but must never mature one that has not: this row + pins the defect where GRAPH_NODE_INACTIVE used to mature from evidence gathered entirely after + the node could no longer be observed, proven with assert_fault_absent_throughout rather than + presented as prose. The below-grace precondition is itself proven from an OBSERVABLE + (GRAPH_NODE_INACTIVE still absent from /faults) immediately before the kill, not merely + inferred from how little time has elapsed - status_json() exposes no per-node violation-streak + count (see lifecycle_expectation_detector.cpp), so the fault surface is the only external + evidence available, and with confirmation_threshold=-2 the detector's first FAILED report IS + the confirmation. Both halves of the row are now reachable together: no GRAPH_NODE_INACTIVE is + born from the departure, and GRAPH_NODE_DISAPPEARED raises because the node was armed before + it died. +- "b3_matured_then_gone": a require_active node driven to "active" first (arming it for + node_death - see TestBoundaryMaturedThenGone's own class docstring for why that is not + optional), then to "inactive" via a real DEACTIVATE transition and left there long enough + for GRAPH_NODE_INACTIVE to CONFIRM, before it is killed. The confirmed fault survives the + departure AND keeps naming the node (content follows the clocks, not the snapshot - already + shipped, GENUINELY proven here with assert_fault_describes_only rather than a code-only + persistence check, which a detector that dropped this node while keeping the code raised for + another one would still pass), and GRAPH_NODE_DISAPPEARED now joins it too, since the node + was genuinely armed before it died. +- "b4_healthy_then_gone": the self-activating variant (managed_lifecycle_active) reaches "active" + and is then killed outright. No GRAPH_NODE_INACTIVE is born from a healthy departure + (release_uncorroborated - already shipped, genuinely proven here), but GRAPH_NODE_DISAPPEARED + never raises. RED. +- "b5_restart_loop_still_caught": the same require_active node, now reaching "active" on EVERY + respawn (not merely the first start - see TestBoundaryRestartLoopStillCaught's own class + docstring for why a node that never arms would make this row unsatisfiable by any correct + implementation), restart-looping under this test's own control, killed and confirmed back + three times over. Nothing here checks GRAPH_NODE_INACTIVE - that is B2's and B3's row. This + one is entirely about whether the presence code catches the departure EVERY cycle, which is + what makes it safe to forbid absence from maturing an unmatured streak: once + GRAPH_NODE_DISAPPEARED independently catches a restart loop, lifecycle_expectation no longer + has to evade it via an unmatured streak maturing on absence. Configures + detectors.node_death.miss_grace explicitly (B5_MISS_GRACE, comfortably past the documented 3000 + ms floor) and drives the node's own respawn on a delay + (B5_RESPAWN_DELAY_SEC) safely longer than that nominal grace, so a correct detector CAN + report every cycle - left at the 1.5 s launch-respawn floor, the outage would be shorter than + the floor the detector enforces and this row could never turn green for a right + implementation, only a wrong one lucky enough to report anyway. +- "c4_config_endpoint_e2e": ONE gateway, ONE armed node, killed once, with a LARGE + detectors.node_death.miss_grace configured (C4_MISS_GRACE_LARGE, comfortably under the + documented 3600-tick ceiling). Proves the knob governs an observable by checking the SAME + gateway at two points on ONE timeline rather than comparing two gateways: a window + (C4_EARLY_WINDOW_SEC) long enough that a near-floor config (this suite's own ~4s convention - + B5_MISS_GRACE, D2_MISS_GRACE) would already have raised, in which this large-grace gateway + must stay silent - needle-scoped (assert_fault_never_names), so a raise for some unrelated + entity cannot decide the row - then, once the configured grace has had time to elapse, the + fault must still arrive and name the node: the large value delays the report, it does not + swallow it. RED. +- "b6_never_armed_below_grace_then_gone": TARGET_NODE launched WITHOUT auto_activate, so it + sits at "unconfigured" its entire life and never once reaches "active" - the gate's + per-entity `armed` precondition (LifecycleWatcher::node_ok()) is therefore false for the + whole test, and node_death can never track it (NodeLivenessTracker only ever tracks a key + the gate has armed at least once). Killed after being observed non-active for only a + couple of ticks - comfortably below `grace`, proven the same way B2 proves it (an + immediate, single-instant read of GET /faults right before the kill, not inferred from + elapsed time). With node_death structurally unable to report this departure, + GRAPH_NODE_INACTIVE is the only detector that ever could - so unlike B2, absence maturing + the below-grace streak is not a defect here, it is the whole point: this is the row that + proves the silence a node like this used to fall into is gone. +- "d2_ungated_clear": a death is confirmed, then the GATEWAY is restarted with a generous + warmup_cycles so the pre-arm window is wide enough to sample. During that window - before the + restarted plugin's own gate has armed anything - the stored fault's `last_passed` must stay + unset the whole time: an ungated detector tick must never report PASSED for a node it has not + actually measured in this process's lifetime. The window is bracketed by two single-shot reads + of the watchdog's own global_state, one immediately before it opens and one immediately after + it closes, both asserted != "armed" - restart-plus-recovery eating the whole nominal warmup + would otherwise let the sampled window land AFTER arming, and a legitimate post-arm PASSED + would then be misread as the ungated-clear bug this row exists to catch. + +### Which arming gate, and why B1 alone uses the global form + +The default rule every scenario in this file follows: gate on `app_id=`, and reserve the global form for a scenario that perturbs the gateway itself. B1 and +B6 are the documented exceptions, for a reason specific to a require_active node rather than a +style choice: ONE of +`ReliabilityGate`'s own preconditions for a per-entity `armed` state is +`LifecycleWatcher::node_ok()`, which is false for exactly a tracked node that is not (yet) +"active" - the very state both targets sit in for their whole life, on purpose, since neither +drives its node past "unconfigured" at all - that IS each row's claim (B1: still present; B6: +gone before its own grace). Gating on `app_id=managed_lifecycle` would therefore wait for +something that never becomes true in either scenario. This is not a new call: +test_lifecycle_expectation_e2e.test.py's own "main" scenario already gates the identical fixture +globally, for the identical reason, and this file follows that precedent rather than inventing a +new one. + +B2, B3, B4 and B5 all use the app_id form instead, because for each of them the per-entity +precondition genuinely becomes true: B4's target (managed_lifecycle_active) has always +self-activated on its own; B2, B3 and B5 now launch TARGET_NODE (managed_lifecycle) with +auto_activate too, specifically so node_death can track it at all - see +`_lifecycle_node_action`'s own docstring and B2/B3/B5's own class docstrings for why a target +that never reaches "active" would make GRAPH_NODE_DISAPPEARED structurally unreachable for any +of the three, not merely slower to catch. B6 is the one row where that same unreachability is +not a precondition to avoid but the claim under test, so it deliberately launches TARGET_NODE +the OTHER way - without auto_activate - and asserts GRAPH_NODE_DISAPPEARED stays silent rather +than waiting for it. + +B2 is the row where getting this wrong is easiest to miss: gating globally and never activating +the target would still be correct for the INACTIVE half (which needs the node observed below +`grace`, so it must never mature past it) while being fatal for the DISAPPEARED half (which +needs node_death to have tracked the node at all, and a node that never arms is structurally +invisible to it regardless of what kills it) - a row built that way could pass while silently +proving nothing about the half it got wrong. B2 therefore follows the same rule every other row +in this file uses: reach "active" first, however briefly, before doing anything else to the node +a DISAPPEARED claim depends on. +""" + +import os +import signal +import sys +import time +import unittest + +from launch.actions import TimerAction +import launch_ros.actions +import launch_testing +from lifecycle_msgs.msg import Transition +from lifecycle_msgs.srv import ChangeState, GetState +import rclpy +from rclpy.node import Node +import requests + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +# I100 as well as E402: `harness` is only importable because of the sys.path line above, so this +# import cannot be moved up to where the alphabetical order would put it. +from harness import ( # noqa: E402, I100 + API_BASE_PATH, + assert_fault_absent_throughout, + assert_fault_describes_only, + assert_fault_never_names, + assert_fault_persists_throughout, + create_watchdog_test_launch, + poll_cleared, + poll_detector_status, + poll_faults, + wait_until_faults_endpoint_live, + wait_until_watchdog_armed, +) + +from ros2_medkit_test_utils.constants import ( # noqa: E402 + ALLOWED_EXIT_CODES, + get_test_port, + get_time_scale, +) +from ros2_medkit_test_utils.coverage import get_coverage_env # noqa: E402 +from ros2_medkit_test_utils.launch_helpers import DEMO_NODE_REGISTRY # noqa: E402 + +# No default on purpose - see harness-consuming siblings' identical rationale: a default makes +# this file FAIL OPEN. A KeyError is loud. +SCENARIO = os.environ['WATCHDOG_E2E_SCENARIO'] +PORT = get_test_port() + +FAULT_CODE_INACTIVE = 'GRAPH_NODE_INACTIVE' +FAULT_CODE_DISAPPEARED = 'GRAPH_NODE_DISAPPEARED' +DETECTOR_ID_LIFECYCLE = 'lifecycle_expectation' +_LIFECYCLE_PREFIX = f'plugins.graph_watchdog.detectors.{DETECTOR_ID_LIFECYCLE}' +_NODE_DEATH_PREFIX = 'plugins.graph_watchdog.detectors.node_death' + +TICK_INTERVAL_MS = 200 +WARMUP_CYCLES = 3 + +# The require_active node B1/B2/B3/B5 share: DEMO_NODE_REGISTRY's own 'managed_lifecycle' key - +# stays "unconfigured" (never active) unless driven otherwise, the trigger every one of those +# rows needs. B4 uses the self-activating 'managed_lifecycle_active' sibling instead. +TARGET_NODE = 'managed_lifecycle' +ACTIVE_NODE = 'managed_lifecycle_active' + +# grace small enough that B1 and B3 mature quickly - same value +# test_lifecycle_expectation_e2e.test.py's own GRACE constant uses, for the same reason. +GRACE = 3 +# B2's own grace: large enough that "killed after a couple of observed ticks" is unambiguously +# BELOW it, whatever small overshoot this scenario's own polling interval costs. +B2_GRACE = 15 +# B6's own grace: same magnitude as B2_GRACE and for the identical reason - "killed after a +# couple of observed ticks" must be unambiguously BELOW it. +B6_GRACE = 15 +# B5's own grace: large relative to one restart cycle's uptime. Not load-bearing for what this +# row actually asserts (see the module docstring - B5 checks GRAPH_NODE_DISAPPEARED only), kept +# generous so lifecycle_expectation's own behaviour cannot accidentally become this row's story. +B5_GRACE = 50 +# B5's own node_death.miss_grace, explicitly configured rather than left at whatever default a +# future detector ships with. min_node_death_miss_grace(TICK_INTERVAL_MS) - the wall-clock floor +# node_death's own configure() silently raises an under-sized value to - is 14 at a 200 ms tick +# (detector_config_keys.hpp: ceil(3000 / 200) - 1). "config sweep C1" +# (NodeDeathIntegrationTest.C1_* in test_node_death_integration.cpp) is the suite that pins the +# floor's own boundary values, but it never exercises tick_interval_ms=200 at all (its own cases +# run at 1, 1000 and 3000 ms), so there is no boundary value at THIS tick period to land on by +# accident - 16 carries two ticks of headroom purely to keep this row visibly off the floor +# itself, the same "comfortably past, not exactly on" convention every miss_grace in this +# package follows. Nominal grace is [B5_MISS_GRACE + 1] * TICK_INTERVAL_MS = 3400 ms; +# B5_RESPAWN_DELAY_SEC below is sized against this value. +B5_MISS_GRACE = 16 +# How long B5's own node is held down every cycle, overriding the launch-wide RESPAWN_DELAY_SEC +# floor (1.5 s - too short to ever satisfy the 3000 ms floor above, let alone B5_MISS_GRACE's own +# 3400 ms). respawn_delay is an ENFORCED floor under how soon launch may even attempt to restart +# a respawn=True process (see _lifecycle_node_action's own docstring), so every cycle's actual +# outage is guaranteed to be at least this long - a correct detector can therefore always report +# it, which is the property this row needs to be satisfiable by a right implementation rather +# than only by a wrong one. +# +# The margin over B5_MISS_GRACE's own nominal grace (3400 ms) has to cover two things a tick +# count alone does not see: the entity cache a tick reads is rebuilt on a debounced graph event +# "always serviced within refresh_debounce_ms + one 100 ms tick" (gateway_node.cpp) - 1100 ms +# worst case before either end of the outage is even visible to node_death's own snapshot - and +# the plugin's tick loop wakes on a condition-variable timed wait, not a hardware clock +# (GraphWatchdogPlugin::wait_for_next_tick), with no enforced upper bound on how far CI +# contention can slow it below its configured 200 ms period. Margin under the 1100 ms debounce +# ceiling alone can be erased by a single unlucky refresh regardless of tick-loop speed at all, +# and NodeLivenessTracker::update() resets a key's miss count to zero the instant it is seen +# present again, with no way to recover a miss already lost that way - a cycle that loses this +# race does not report late, it never reports at all. 12.5 s clears 3 * 3400 + 1100 = 11300 ms +# (a tick loop running at a third of its configured rate, plus the full debounce ceiling on top) +# by 1200 ms - worst-case observed window 12500 - 1100 = 11400 ms, 3.35x the nominal grace. +B5_RESPAWN_DELAY_SEC = 12.5 + +# The budgets below are scaled by MEDKIT_TEST_TIME_SCALE, which the sanitizer jobs set to the +# same factor they apply to every declared CTest timeout. A deadline asserted INSIDE a test is +# invisible to that rewrite, so an instrumented graph that takes longer to forget a departed +# node blows a budget here and the failure reads as a detector that never reported - the exact +# red this suite exists to produce for a real defect. Unset elsewhere, so the normal jobs keep +# the tight budgets that give these assertions their falsifying edge. +# +# Poll intervals, enforced respawn delays and the sustained-observation windows are NOT scaled. +# Those are not give-up bounds: stretching a window a scenario watches for silence buys no +# confidence and spends the whole package's test budget to do it. +TIME_SCALE = get_time_scale() +ARM_TIMEOUT_SEC = 60.0 * TIME_SCALE +FAULTS_LIVE_TIMEOUT_SEC = 30.0 * TIME_SCALE +PRESENCE_TIMEOUT_SEC = 30.0 * TIME_SCALE +DEPARTURE_TIMEOUT_SEC = 30.0 * TIME_SCALE +RAISE_TIMEOUT_SEC = 60.0 * TIME_SCALE +CLEAR_TIMEOUT_SEC = 60.0 * TIME_SCALE +# How long an absent or a persisting fault is watched for - measured from the arming gate, not +# process start, so bringup cannot eat it. Matches test_node_death_e2e.test.py's identical +# constant. +SUSTAINED_WINDOW_SEC = 20.0 + +# launch will not even ATTEMPT to restart a respawn=True process before this elapses - an +# enforced floor under every kill-then-return gap this file measures. +RESPAWN_DELAY_SEC = 1.5 + +# ---- "b5_restart_loop_still_caught" scenario's own target -------------------------------------- +# The claim under test is that the presence code catches EVERY cycle of a restart loop, not +# merely the first: three consecutive kill/raise/clear cycles establish that - this is a loop, +# and each turn of it is caught. A fourth or fifth cycle would repeat an already-proven claim at +# the full cost of B5_RESPAWN_DELAY_SEC apiece, buying no additional discriminating power against +# the failure this row exists to catch (a detector that only reports the first departure, or +# stops reporting after one). If a future change ever needs more cycles to expose something this +# one does not, raising this single constant is the whole edit. +B5_CYCLES = 3 + +# ---- "c4_config_endpoint_e2e" scenario's own fixtures ------------------------------------------- +C4_TARGET_NODE = 'calibration' +C4_TARGET_EXECUTABLE = 'demo_calibration_service' +C4_TARGET_NAMESPACE = '/powertrain/engine' +# Comfortably under the documented ceiling (3600 ticks - node_death's own kMaxNodeDeathGraceTicks +# applies to miss_grace and prune_grace alike), chosen only to sit unambiguously outside +# C4_EARLY_WINDOW_SEC below - 500 ticks at TICK_INTERVAL_MS is 100s, four times that window. +C4_MISS_GRACE_LARGE = 500 +# Long enough that a near-floor config would already have raised several times over by the time +# this window ends - the near-floor graces in this package run 16 to 20 ticks, 3.4s to 4.2s +# nominal at TICK_INTERVAL_MS, see B5_MISS_GRACE and D2_MISS_GRACE - so staying silent through it +# is evidence the large value is actually GOVERNING behaviour, not merely accepted and ignored; +# short enough to sit comfortably inside C4_MISS_GRACE_LARGE's own ~100s nominal grace. +C4_EARLY_WINDOW_SEC = 25.0 +# Measured from the END of C4_EARLY_WINDOW_SEC, not from the kill: comfortably covers the +# remaining ~75s to C4_MISS_GRACE_LARGE's own nominal grace-crossing point plus reporting +# latency. +C4_LATE_RAISE_TIMEOUT_SEC = 100.0 * TIME_SCALE + +# ---- "d2_ungated_clear" scenario's own fixtures ------------------------------------------------- +D2_TARGET_NODE = 'calibration' +D2_TARGET_EXECUTABLE = 'demo_calibration_service' +D2_TARGET_NAMESPACE = '/powertrain/engine' +D2_MISS_GRACE = 20 +# Deliberately large so the RESTARTED gateway's own pre-arm window is wide enough to sample +# several times over - the whole point of this scenario is watching what happens DURING that +# window, not merely before and after it. 150 ticks at TICK_INTERVAL_MS is 30s nominal warmup: +# generous headroom over the restart itself, which is not instantaneous - create_gateway_node's +# own respawn_delay (1.0s default) is an ENFORCED floor before launch even attempts to start +# the replacement process, on top of that process's own ROS init and HTTP bind. Measured live at +# 25 ticks (5s nominal): GET /faults was still completely unreachable (connection refused, not +# even a 503) at the very first poll after the old port was confirmed down - the replacement +# gateway had not bound its port yet. 150 leaves room for that plus real bringup variance. +D2_WARMUP_CYCLES = 150 +# How long the post-restart, pre-arm window is watched for a premature PASSED, once the fault +# surface is confirmed reachable (see test_02's own wait_until_faults_endpoint_live call before +# this window opens). Comfortably inside D2_WARMUP_CYCLES * TICK_INTERVAL_MS (~30s nominal), so +# a poll landing here is provably still INSIDE the ungated window rather than after it. +D2_UNGATED_WATCH_SEC = 3.5 + + +def _lifecycle_node_action( + name, *, respawn=False, respawn_delay=RESPAWN_DELAY_SEC, auto_activate=None): + """One managed_lifecycle instance under `name`, with a PID handle the test can signal. + + Uses DEMO_NODE_REGISTRY's own (executable, ros_name, namespace) triple for `name` so this + stays in lockstep with demo_nodes.launch.py, built by hand only because every scenario here + signals the process directly and create_demo_nodes() hands back no PID. + + `auto_activate` is a ROS PARAMETER, not part of the registry triple, so it has to be set + here explicitly rather than inferred from it. Defaults to whether `name` is ACTIVE_NODE - + the historical convention every scenario but B3/B5 relies on - but a caller launching + TARGET_NODE (managed_lifecycle) for a scenario that needs it to actually ARM for + node_death passes `auto_activate=True` explicitly: node_death only ever tracks a node it + has seen armed at least once (reliability_allows() requires "active" for a managed node - + LifecycleWatcher::node_ok()), so a require_active node that never activates is invisible + to it no matter what happens to it afterwards. See TestBoundaryMaturedThenGone and + TestBoundaryRestartLoopStillCaught's own class docstrings. + """ + executable, ros_name, namespace = DEMO_NODE_REGISTRY[name] + if auto_activate is None: + auto_activate = name == ACTIVE_NODE + node_kwargs = { + 'package': 'ros2_medkit_integration_tests', + 'executable': executable, + 'name': ros_name, + 'namespace': namespace, + 'output': 'screen', + 'additional_env': get_coverage_env('ros2_medkit_integration_tests'), + 'sigterm_timeout': '30', + 'sigkill_timeout': '15', + 'respawn': respawn, + 'respawn_delay': respawn_delay, + } + if auto_activate: + node_kwargs['parameters'] = [{'auto_activate': True}] + return launch_ros.actions.Node(**node_kwargs) + + +def generate_test_description(): + detector_params = { + 'plugins.graph_watchdog.tick_interval_ms': TICK_INTERVAL_MS, + 'plugins.graph_watchdog.warmup_cycles': WARMUP_CYCLES, + } + demo_nodes = [] + gateway_respawn = False + + if SCENARIO == 'b1_inactive_present': + detector_params[f'{_LIFECYCLE_PREFIX}.require_active'] = [TARGET_NODE] + detector_params[f'{_LIFECYCLE_PREFIX}.grace'] = GRACE + demo_nodes = [TARGET_NODE] # never killed - the whole point of this row + elif SCENARIO == 'b2_inactive_below_grace_then_gone': + detector_params[f'{_LIFECYCLE_PREFIX}.require_active'] = [TARGET_NODE] + detector_params[f'{_LIFECYCLE_PREFIX}.grace'] = B2_GRACE + elif SCENARIO == 'b3_matured_then_gone': + detector_params[f'{_LIFECYCLE_PREFIX}.require_active'] = [TARGET_NODE] + detector_params[f'{_LIFECYCLE_PREFIX}.grace'] = GRACE + elif SCENARIO == 'b4_healthy_then_gone': + detector_params[f'{_LIFECYCLE_PREFIX}.require_active'] = [ACTIVE_NODE] + detector_params[f'{_LIFECYCLE_PREFIX}.grace'] = GRACE + elif SCENARIO == 'b5_restart_loop_still_caught': + detector_params[f'{_LIFECYCLE_PREFIX}.require_active'] = [TARGET_NODE] + detector_params[f'{_LIFECYCLE_PREFIX}.grace'] = B5_GRACE + detector_params[f'{_NODE_DEATH_PREFIX}.miss_grace'] = B5_MISS_GRACE + elif SCENARIO == 'b6_never_armed_below_grace_then_gone': + detector_params[f'{_LIFECYCLE_PREFIX}.require_active'] = [TARGET_NODE] + detector_params[f'{_LIFECYCLE_PREFIX}.grace'] = B6_GRACE + elif SCENARIO == 'c4_config_endpoint_e2e': + detector_params[f'{_NODE_DEATH_PREFIX}.miss_grace'] = C4_MISS_GRACE_LARGE + elif SCENARIO == 'd2_ungated_clear': + detector_params[f'{_NODE_DEATH_PREFIX}.miss_grace'] = D2_MISS_GRACE + detector_params['plugins.graph_watchdog.warmup_cycles'] = D2_WARMUP_CYCLES + gateway_respawn = True + else: + raise RuntimeError(f'WATCHDOG_E2E_SCENARIO={SCENARIO!r} has no launch configuration') + + launch_description, context = create_watchdog_test_launch( + detector_params=detector_params, + demo_nodes=demo_nodes, + port=PORT, + gateway_respawn=gateway_respawn, + ) + + if SCENARIO == 'b2_inactive_below_grace_then_gone': + # auto_activate=True: this row's SECOND claim (GRAPH_NODE_DISAPPEARED) needs + # node_death to have tracked the node at all, which requires it to have been armed - + # see TestBoundaryInactiveBelowGraceThenGone's own class docstring and the module + # docstring's "Which arming gate" section. + target = _lifecycle_node_action(TARGET_NODE, respawn=False, auto_activate=True) + launch_description.add_action(TimerAction(period=2.0, actions=[target])) + context['target_node'] = target + + if SCENARIO == 'b3_matured_then_gone': + # auto_activate=True: this row's SECOND claim (GRAPH_NODE_DISAPPEARED) needs + # node_death to have tracked the node at all, which requires it to have been armed - + # see TestBoundaryMaturedThenGone's own class docstring and + # _lifecycle_node_action's. + target = _lifecycle_node_action(TARGET_NODE, respawn=False, auto_activate=True) + launch_description.add_action(TimerAction(period=2.0, actions=[target])) + context['target_node'] = target + + if SCENARIO == 'b4_healthy_then_gone': + target = _lifecycle_node_action(ACTIVE_NODE, respawn=False) + launch_description.add_action(TimerAction(period=2.0, actions=[target])) + context['target_node'] = target + + if SCENARIO == 'b6_never_armed_below_grace_then_gone': + # auto_activate deliberately omitted (defaults False for TARGET_NODE): this row's + # whole claim is that the node NEVER arms - see the module docstring's "Which arming + # gate" section and TestBoundaryNeverArmedBelowGraceThenGone's own class docstring. + target = _lifecycle_node_action(TARGET_NODE, respawn=False) + launch_description.add_action(TimerAction(period=2.0, actions=[target])) + context['target_node'] = target + + if SCENARIO == 'b5_restart_loop_still_caught': + # auto_activate=True: every respawned instance needs to reach "active" on its own + # for node_death to ever track it - see TestBoundaryRestartLoopStillCaught's own + # class docstring and _lifecycle_node_action's. + target = _lifecycle_node_action( + TARGET_NODE, respawn=True, respawn_delay=B5_RESPAWN_DELAY_SEC, auto_activate=True) + launch_description.add_action(TimerAction(period=2.0, actions=[target])) + context['target_node'] = target + + if SCENARIO == 'd2_ungated_clear': + target = launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable=D2_TARGET_EXECUTABLE, + name=D2_TARGET_NODE, + namespace=D2_TARGET_NAMESPACE, + output='screen', + additional_env=get_coverage_env('ros2_medkit_integration_tests'), + sigterm_timeout='30', + sigkill_timeout='15', + # No respawn: the node must STAY gone across the gateway restart, or there would be + # nothing "stored" left for the warmup window to wrongly move toward healing. + ) + launch_description.add_action(TimerAction(period=2.0, actions=[target])) + context['target_node'] = target + + if SCENARIO == 'c4_config_endpoint_e2e': + target = launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable=C4_TARGET_EXECUTABLE, + name=C4_TARGET_NODE, + namespace=C4_TARGET_NAMESPACE, + output='screen', + additional_env=get_coverage_env('ros2_medkit_integration_tests'), + sigterm_timeout='30', + sigkill_timeout='15', + ) + launch_description.add_action(TimerAction(period=2.0, actions=[target])) + context['target_node'] = target + + return launch_description, context + + +# --------------------------------------------------------------------------------------- +# Local helpers - read the same endpoints harness.py's own helpers do, or a service this +# file's own scenario needs that no shared helper covers. +# --------------------------------------------------------------------------------------- + +def _poll_apps_absent(port, app_id, timeout=30.0, interval=0.5): + """Poll ``GET /apps`` until `app_id` is no longer listed. ``True`` once it is gone.""" + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + timeout + last_seen = 'GET /apps was never answered at all' + while time.monotonic() < deadline: + try: + response = requests.get(f'{base}/apps', timeout=5) + if response.status_code == 200: + ids = [item.get('id') for item in response.json().get('items', [])] + last_seen = str(ids) + if app_id not in ids: + return True + else: + last_seen = f'HTTP {response.status_code} from GET /apps' + except requests.exceptions.RequestException as exc: + last_seen = f'GET /apps failed: {exc}' + time.sleep(interval) + print(f'_poll_apps_absent({app_id!r}) timed out after {timeout}s; last seen: {last_seen}') + return False + + +def _poll_apps_present(port, app_id, timeout=30.0, interval=0.5): + """Poll ``GET /apps`` until `app_id` IS listed. ``True`` once it appears.""" + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + timeout + last_seen = 'GET /apps was never answered at all' + while time.monotonic() < deadline: + try: + response = requests.get(f'{base}/apps', timeout=5) + if response.status_code == 200: + ids = [item.get('id') for item in response.json().get('items', [])] + last_seen = str(ids) + if app_id in ids: + return True + else: + last_seen = f'HTTP {response.status_code} from GET /apps' + except requests.exceptions.RequestException as exc: + last_seen = f'GET /apps failed: {exc}' + time.sleep(interval) + print(f'_poll_apps_present({app_id!r}) timed out after {timeout}s; last seen: {last_seen}') + return False + + +def _watchdog_global_state(port, timeout=5.0): + """One immediate read of GET /x-medkit-watchdog's own global_state field. + + Not a polling helper: D2's pre-arm proof needs to know the state AT ONE INSTANT + (immediately before opening the ungated window, and again immediately after it closes), + not wait until some condition becomes true - a poll loop would blur exactly the boundary + this is meant to pin. Mirrors ReliabilityGate::status_json()'s own field (see + wait_until_watchdog_armed's docstring in harness.py: "armed" or "warming_up"). + + Returns the string, or ``None`` if the endpoint did not answer 200. + """ + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + try: + response = requests.get(f'{base}/x-medkit-watchdog', timeout=timeout) + except requests.exceptions.RequestException: + return None + if response.status_code != 200: + return None + return response.json().get('x-medkit-watchdog', {}).get('global_state') + + +def _wait_until_port_is_down(port, timeout=60.0, interval=0.2): + """Wait until the gateway's HTTP port stops answering. ``True`` once it does.""" + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + requests.get(f'{base}/health', timeout=2) + except requests.exceptions.RequestException: + return True + time.sleep(interval) + return False + + +def _fault_record(port, code, timeout=30.0, interval=0.5): + """Poll ``GET /faults?status=all`` until `code` appears, whatever its status. + + ``poll_faults`` uses the default (pending+confirmed) filter, so a HEALED or CLEARED fault + disappears from it. D2 needs the record itself, including ``last_passed``, which survives + both. Returns the matching item dict, or ``None`` on timeout. + """ + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + response = requests.get(f'{base}/faults', params={'status': 'all'}, timeout=5) + if response.status_code == 200: + for item in response.json().get('items', []): + if item.get('fault_code') == code: + return item + except requests.exceptions.RequestException: + pass + time.sleep(interval) + return None + + +def _fault_present_now(port, code, timeout=5.0): + """One immediate GET /faults check for whether `code` is in the active-fault list. + + Not a polling helper: B2's below-grace precondition has to be read from an OBSERVABLE AT + ONE INSTANT, immediately before the kill that follows it - a poll loop's own sampling + interval would widen exactly the race this exists to shrink. Uses the same default + (pending+confirmed) filter as poll_faults, so "False" here means the same thing a + poll_faults timeout does: not (yet) raised. + + Returns ``True``/``False``, or ``None`` if the endpoint did not answer 200 - the caller must + tell that apart from ``False``, the same channel-alive discipline every window assertion in + this package's harness already applies. + """ + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + try: + response = requests.get(f'{base}/faults', timeout=timeout) + except requests.exceptions.RequestException: + return None + if response.status_code != 200: + return None + codes = {item.get('fault_code') for item in response.json().get('items', [])} + return code in codes + + +def _assert_never_passed_throughout(test_case, port, code, duration, interval=0.2): + """Fail unless `code`'s stored record has ``last_passed is None`` on EVERY poll. + + D2's own claim ("no clear reaches the fault manager before the detector has measured") is + about a FIELD staying unset across a window, not about the fault's presence/absence - + neither ``assert_fault_absent_throughout`` nor ``assert_fault_persists_throughout`` reads + ``last_passed``, so this file carries its own, narrow local helper rather than stretching + either one to fit. Same channel-alive discipline as harness.py's own window assertions: a + request error or a non-200 fails the assertion naming which poll and why, rather than being + swallowed the way a bare ``poll_cleared``/``assertFalse`` pair would be. + + Parameters + ---------- + test_case : unittest.TestCase + port : int + Gateway HTTP port. + code : str + The ``fault_code`` whose record must show no PASSED report. + duration : float + Total seconds to keep polling. + interval : float + Sleep between polls in seconds. + + """ + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + duration + polls = 0 + while time.monotonic() < deadline: + try: + response = requests.get(f'{base}/faults', params={'status': 'all'}, timeout=5) + except requests.exceptions.RequestException as exc: + test_case.fail( + f'/faults became unreachable {polls} poll(s) into a {duration}s ungated-window ' + f'check (could not ask, which is not the same as "asked, and {code} was never ' + f'reported PASSED"): {exc}') + return + if response.status_code != 200: + test_case.fail( + f'/faults answered HTTP {response.status_code} {polls} poll(s) into a ' + f'{duration}s ungated-window check - the channel died mid-window, which this ' + 'assertion must not read as "never reported PASSED"') + return + record = next( + (item for item in response.json().get('items', []) if item.get('fault_code') == code), + None) + if record is None: + test_case.fail( + f'{code} was missing from the store {polls} poll(s) into a {duration}s ' + 'ungated-window check that was supposed to keep watching its record') + return + test_case.assertIsNone( + record.get('last_passed'), + f'{code} was reported PASSED (last_passed={record.get("last_passed")!r}) ' + f'{polls} poll(s) into a {duration}s window that starts before the restarted ' + "detector's own gate has armed anything - an ungated clear reached the fault " + 'manager before anything was actually measured', + ) + polls += 1 + time.sleep(interval) + test_case.assertGreater( + polls, 0, + f'the {duration}s ungated-window check never actually polled /faults - duration must ' + f'be >= interval ({interval}s)') + + +def _get_lifecycle_label(client_node, service_name, timeout=10.0): + """One ``GetState`` call against `service_name`. Returns the state label, or ``None``. + + Creates its own client rather than taking one, matching + test_node_death_e2e.test.py's identical helper. + """ + client = client_node.create_client(GetState, service_name) + try: + if not client.wait_for_service(timeout_sec=timeout): + return None + future = client.call_async(GetState.Request()) + rclpy.spin_until_future_complete(client_node, future, timeout_sec=timeout) + result = future.result() + return None if result is None else result.current_state.label + finally: + client_node.destroy_client(client) + + +def _poll_lifecycle_label(client_node, service_name, expected_label, timeout=30.0, interval=0.5): + """Poll ``GetState`` until `service_name` answers `expected_label`. ``True`` once it does. + + One client for the whole poll, matching test_node_death_e2e.test.py's identical helper. + """ + client = client_node.create_client(GetState, service_name) + try: + deadline = time.monotonic() + timeout + last_seen = f'{service_name} was never answered at all' + while time.monotonic() < deadline: + if client.wait_for_service(timeout_sec=interval): + future = client.call_async(GetState.Request()) + rclpy.spin_until_future_complete(client_node, future, timeout_sec=interval) + result = future.result() + if result is not None: + last_seen = result.current_state.label + if last_seen == expected_label: + return True + time.sleep(interval) + print(f'_poll_lifecycle_label({service_name!r}, expected={expected_label!r}) timed ' + f'out after {timeout}s; last seen: {last_seen!r}') + return False + finally: + client_node.destroy_client(client) + + +def _call_change_state_once(client_node, service_name, transition_id, timeout=30.0): + """One ``ChangeState`` call against `service_name`. ``True`` on ``result.success``.""" + client = client_node.create_client(ChangeState, service_name) + if not client.wait_for_service(timeout_sec=timeout): + return False + request = ChangeState.Request() + request.transition.id = transition_id + future = client.call_async(request) + rclpy.spin_until_future_complete(client_node, future, timeout_sec=timeout) + result = future.result() + return result is not None and result.success + + +# --------------------------------------------------------------------------------------- +# Scenarios +# --------------------------------------------------------------------------------------- + +class TestBoundaryInactivePresent(unittest.TestCase): + """B1: a required node inactive past grace, still present: INACTIVE only. + + GENUINELY GREEN, not a placeholder: lifecycle_expectation already ships, so the raise below + exercises real, already-merged code (LifecycleExpectationTracker's violation-streak clock + crossing `grace`). The GRAPH_NODE_DISAPPEARED absence half cannot, by itself, distinguish a + correct node_death from no detector at all - but staying silent is also the permanently + correct behaviour here, since this node never leaves the graph. + """ + + def test_inactive_past_grace_raises_no_disappeared(self): + # No target_node fixture: TARGET_NODE is launched via demo_nodes=[...] (create_demo_nodes + # gives no PID handle back, and this row never needs one - it kills nothing). Only the + # scenarios below that hand-build the node populate a 'target_node' context entry. + # Global gate: see the module docstring's "Which arming gate" section. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC), + 'graph_watchdog never reported an armed global state') + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=FAULTS_LIVE_TIMEOUT_SEC), + 'GET /faults never answered 200 - nothing below would prove anything') + self.assertTrue( + _poll_apps_present(PORT, TARGET_NODE, timeout=PRESENCE_TIMEOUT_SEC), + f'{TARGET_NODE} never appeared on GET /apps - there is no present node here for ' + 'this row to measure', + ) + + fault = poll_faults(PORT, FAULT_CODE_INACTIVE, timeout=RAISE_TIMEOUT_SEC) + if fault is None: + self.fail(f'{FAULT_CODE_INACTIVE} never raised for {TARGET_NODE} past grace') + self.assertIn(TARGET_NODE, fault.get('description', '')) + + assert_fault_absent_throughout(self, PORT, FAULT_CODE_DISAPPEARED, SUSTAINED_WINDOW_SEC) + + # The trigger was pinned before the window; confirm it survived to the end too, or most + # of the window measured a graph this row was not actually about. + self.assertTrue( + _poll_apps_present(PORT, TARGET_NODE, timeout=5.0), + f'{TARGET_NODE} is no longer present at the end of the window - the trigger did ' + 'not survive it', + ) + + +class TestBoundaryInactiveBelowGraceThenGone(unittest.TestCase): + """B2: a required node inactive for FEWER ticks than grace, then it vanishes. + + Absence may continue a violation that has already matured, but must never mature one that + has not: the row it exists to pin is a defect where absence used to CONTINUE a violation + streak that had not yet matured when the node was last observed - necessary only because no + presence detector existed to catch a restart loop any other way. Once GRAPH_NODE_DISAPPEARED + has one, that continuation is no longer needed and absence must stop maturing an unmatured + streak. + + Both of this row's claims are satisfiable together, which took getting the arming gate + wrong once to learn: the target must reach "active" FIRST, or GRAPH_NODE_DISAPPEARED can + never report its death, whatever kills it - reliability_allows() requires "active" for a + MANAGED node (LifecycleWatcher::node_ok()), so a managed_lifecycle instance that stays + "unconfigured" its whole life is structurally invisible to node_death, exactly as + TestBoundaryMaturedThenGone's own docstring explains for its target. The launch therefore + drives TARGET_NODE through "active" first, THEN a real DEACTIVATE transition - the same two + steps B3 takes - but where B3 leaves the node inactive long enough to CONFIRM (past + `grace`), B2 kills it almost immediately after: comfortably below `grace`, which is this + row's own claim and the reason it cannot simply reuse B3's fixture wholesale. + + The below-grace precondition is proven from an OBSERVABLE immediately before the kill, not + inferred from how little time elapsed since the DEACTIVATE transition: status_json() + exposes no per-node violation-streak count (see lifecycle_expectation_detector.cpp's own + status_json, which reports only tracking_saturated/tracked_nodes/tracked_node_cap), so the + fault surface itself is the only external evidence of whether the streak has crossed + `grace`, and with confirmation_threshold=-2 the detector's first FAILED report IS the + confirmation - "absent from /faults" and "not yet matured" are the same fact, one HTTP + round trip apart. A failure at that check names a timing problem in this test's OWN setup + (B2_GRACE too tight against how long the DEACTIVATE round trip took this run), never the + absence-must-not-mature claim the kill exists to test - which is the point of reading an + observable immediately before acting instead of trusting a margin. + """ + + @classmethod + def setUpClass(cls): + rclpy.init() + cls._client_node = Node('node_death_boundary_b2_client') + + @classmethod + def tearDownClass(cls): + cls._client_node.destroy_node() + rclpy.shutdown() + + def test_below_grace_departure_never_matures_inactive(self, target_node): + get_state_service = f'/{TARGET_NODE}/get_state' + change_state_service = f'/{TARGET_NODE}/change_state' + + # app_id form: this node reaches "active" on its own now (auto_activate=True), so the + # per-entity armed precondition (LifecycleWatcher::node_ok()) genuinely becomes true - + # see the module docstring's "Which arming gate" section and this class's own + # docstring. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC, app_id=TARGET_NODE), + f'graph_watchdog never reported {TARGET_NODE} armed') + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=FAULTS_LIVE_TIMEOUT_SEC), + 'GET /faults never answered 200 - nothing below would prove anything') + + self.assertTrue( + _poll_lifecycle_label( + type(self)._client_node, get_state_service, 'active', + timeout=PRESENCE_TIMEOUT_SEC), + f'{TARGET_NODE} never reached "active" on its own - the trigger this row needs (a ' + 'node armed for node_death, then driven non-active BELOW grace) was never set up', + ) + self.assertTrue( + _call_change_state_once( + type(self)._client_node, change_state_service, + Transition.TRANSITION_DEACTIVATE, timeout=PRESENCE_TIMEOUT_SEC), + 'the real DEACTIVATE transition was rejected or never answered', + ) + self.assertTrue( + _poll_lifecycle_label( + type(self)._client_node, get_state_service, 'inactive', + timeout=PRESENCE_TIMEOUT_SEC), + f'{TARGET_NODE} never reached "inactive" after DEACTIVATE', + ) + + # tracked_nodes becoming 1 is the tracker's own proof that it matched TARGET_NODE at + # least once - the earliest moment a kill is guaranteed not to land before the detector + # was even permitted to look. B2_GRACE is generous enough that the handful of extra + # ticks the DEACTIVATE round trip and this poll's own interval can cost before the kill + # below still leaves the node comfortably below grace. + self.assertTrue( + poll_detector_status( + PORT, DETECTOR_ID_LIFECYCLE, 'tracked_nodes', 1, timeout=ARM_TIMEOUT_SEC), + f'lifecycle_expectation never reported tracking {TARGET_NODE} - it was never ' + 'matched at all, so killing it below would prove nothing about absence maturing ' + 'an unmatured streak', + ) + + # The row's own precondition, read from an observable immediately before the kill - see + # the class docstring for why this is the tightest proof available without new + # instrumentation. `is` rather than a plain falsy check: None (channel unreachable) must + # not be read as "confirmed absent". + below_grace = _fault_present_now(PORT, FAULT_CODE_INACTIVE) + self.assertIs( + below_grace, False, + f'{FAULT_CODE_INACTIVE} could not be proven absent immediately before the kill ' + f'below (checked value: {below_grace!r}) - either the fault surface is unreachable, ' + f'or B2_GRACE ({B2_GRACE} ticks) already matured before the DEACTIVATE round trip ' + "finished this run, so the kill below would test B3's claim (already matured), not " + "this row's below-grace claim", + ) + + os.kill(target_node.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, TARGET_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'{TARGET_NODE} never left GET /apps after SIGTERM') + + # The row's first claim: GRAPH_NODE_INACTIVE must never appear, however long the node + # stays gone - a below-grace streak is held by absence, never matured by it. + assert_fault_absent_throughout(self, PORT, FAULT_CODE_INACTIVE, SUSTAINED_WINDOW_SEC) + + # The row's second claim, reachable because TARGET_NODE was armed before it died - see + # the class docstring. + fault = poll_faults(PORT, FAULT_CODE_DISAPPEARED, timeout=RAISE_TIMEOUT_SEC) + if fault is None: + self.fail(f'{FAULT_CODE_DISAPPEARED} never raised for the departed {TARGET_NODE}') + self.assertIn(TARGET_NODE, fault.get('description', '')) + + +class TestBoundaryMaturedThenGone(unittest.TestCase): + """B3: a required node reaches active, is driven non-active past grace, then vanishes. + + Two independent claims, BOTH satisfiable now. The first - "content follows the clocks, not + the snapshot" keeps a matured GRAPH_NODE_INACTIVE violation in place through the departure, + still naming the node - is already-shipped lifecycle_expectation behaviour, proven here + with assert_fault_describes_only against the real stack rather than a code-only persistence + check (a detector that dropped TARGET_NODE from the description while keeping some OTHER + node's violation raised under the same code would still pass a bare presence check), the + same claim test_lifecycle_expectation_e2e.test.py's own "departure_keeps" scenario proves + for the UNREADABLE code. + + The second - GRAPH_NODE_DISAPPEARED also joining once the node is gone, still naming it - + needs node_death to have TRACKED the node at all, which only ever happens for a node armed + at least once: reliability_allows() requires "active" for a MANAGED node + (LifecycleWatcher::node_ok()). A managed_lifecycle instance that stays "unconfigured" its + whole life is never armed and so is structurally invisible to node_death whatever happens + to it afterwards - the second claim would be unreachable by ANY correct implementation + against that fixture, not merely an unimplemented one. This is why the launch drives + TARGET_NODE through "active" FIRST (arming it - the same mechanism B4's target uses), THEN + a real DEACTIVATE transition (maturing GRAPH_NODE_INACTIVE the same way the original + "stays unconfigured" fixture did), THEN kills it: the node this row's second claim needs + has to have been alive and armed at some point before it can ever be reported disappeared. + A future edit that reverts TARGET_NODE to launching without auto_activate would silently + make the second half of this row unreachable again. + """ + + @classmethod + def setUpClass(cls): + rclpy.init() + cls._client_node = Node('node_death_boundary_b3_client') + + @classmethod + def tearDownClass(cls): + cls._client_node.destroy_node() + rclpy.shutdown() + + def test_matured_inactive_survives_departure_and_disappeared_joins(self, target_node): + get_state_service = f'/{TARGET_NODE}/get_state' + change_state_service = f'/{TARGET_NODE}/change_state' + + # app_id form: this node reaches "active" on its own now (auto_activate=True), so the + # per-entity armed precondition (LifecycleWatcher::node_ok()) genuinely becomes true - + # see the module docstring's "Which arming gate" section. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC, app_id=TARGET_NODE), + f'graph_watchdog never reported {TARGET_NODE} armed') + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=FAULTS_LIVE_TIMEOUT_SEC), + 'GET /faults never answered 200 - nothing below would prove anything') + + self.assertTrue( + _poll_lifecycle_label( + type(self)._client_node, get_state_service, 'active', + timeout=PRESENCE_TIMEOUT_SEC), + f'{TARGET_NODE} never reached "active" on its own - the trigger this row needs (a ' + 'node armed for node_death, then driven non-active) was never set up', + ) + self.assertTrue( + _call_change_state_once( + type(self)._client_node, change_state_service, + Transition.TRANSITION_DEACTIVATE, timeout=PRESENCE_TIMEOUT_SEC), + 'the real DEACTIVATE transition was rejected or never answered', + ) + self.assertTrue( + _poll_lifecycle_label( + type(self)._client_node, get_state_service, 'inactive', + timeout=PRESENCE_TIMEOUT_SEC), + f'{TARGET_NODE} never reached "inactive" after DEACTIVATE', + ) + + fault = poll_faults(PORT, FAULT_CODE_INACTIVE, timeout=RAISE_TIMEOUT_SEC) + if fault is None: + self.fail( + f'{FAULT_CODE_INACTIVE} never raised for {TARGET_NODE} past grace - there is ' + 'nothing ALREADY CONFIRMED for the kill below to test the survival of') + self.assertIn(TARGET_NODE, fault.get('description', '')) + + os.kill(target_node.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, TARGET_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'{TARGET_NODE} never left GET /apps after SIGTERM') + + # Already-shipped lifecycle_expectation behaviour: proves the matured violation is not + # healed by the departure, and that the surviving content still names TARGET_NODE + # rather than merely keeping the code raised (see the class docstring for why a + # code-only check is not enough here). + assert_fault_describes_only( + self, PORT, FAULT_CODE_INACTIVE, required=[TARGET_NODE], forbidden=[], + duration=SUSTAINED_WINDOW_SEC) + + # Reached only because the assertion above passed, unlike B2's sibling call. Reachable + # AT ALL because TARGET_NODE was armed before it died - see the class docstring. + second = poll_faults(PORT, FAULT_CODE_DISAPPEARED, timeout=RAISE_TIMEOUT_SEC) + if second is None: + self.fail(f'{FAULT_CODE_DISAPPEARED} never raised for the departed {TARGET_NODE}') + self.assertIn(TARGET_NODE, second.get('description', '')) + + +class TestBoundaryHealthyThenGone(unittest.TestCase): + """B4: a required node reaches active, then shuts down: DISAPPEARED only. + + The INACTIVE-absence half is GENUINELY GREEN, already-shipped behaviour + (`release_uncorroborated`: a node last measured healthy that then departs starts no + violation) - proven here against the real stack, checked FIRST so its own result is never + masked by the DISAPPEARED half's own timeout budget. The DISAPPEARED half proves node_death + raises and names a node that was healthy right up to the moment it departed. + """ + + def test_healthy_departure_raises_only_disappeared(self, target_node): + # app_id form: unlike B1's target, this node reaches "active" on its own, so the + # per-entity armed precondition (LifecycleWatcher::node_ok()) genuinely becomes true + # here - see the module docstring's "Which arming gate" section. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC, app_id=ACTIVE_NODE), + f'graph_watchdog never reported {ACTIVE_NODE} armed') + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=FAULTS_LIVE_TIMEOUT_SEC), + 'GET /faults never answered 200 - nothing below would prove anything') + + os.kill(target_node.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, ACTIVE_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'{ACTIVE_NODE} never left GET /apps after SIGTERM') + + # Already-shipped lifecycle_expectation behaviour: an active-then-departed node never + # becomes INACTIVE content. + assert_fault_absent_throughout(self, PORT, FAULT_CODE_INACTIVE, SUSTAINED_WINDOW_SEC) + + fault = poll_faults(PORT, FAULT_CODE_DISAPPEARED, timeout=RAISE_TIMEOUT_SEC) + if fault is None: + self.fail(f'{FAULT_CODE_DISAPPEARED} never raised for the departed {ACTIVE_NODE}') + self.assertIn(ACTIVE_NODE, fault.get('description', '')) + + +class TestBoundaryRestartLoopStillCaught(unittest.TestCase): + """B5: a required node in a restart loop is still caught, every cycle, by the presence code. + + The most important row in this file: it is what makes it safe for B2 to forbid absence from + maturing an unmatured streak. Once GRAPH_NODE_DISAPPEARED independently catches a departure + regardless of how briefly the node was up, lifecycle_expectation no longer has to lean on an + unmatured streak maturing on absence to keep a restart-looping node from evading every code. + This row does not touch GRAPH_NODE_INACTIVE at all - that claim belongs to B2 and B3. + + The target launches with auto_activate (the same mechanism B4's target uses) and keeps that + parameter across every respawn, not merely the first start: node_death only ever tracks a + node it has seen ARMED at least once (reliability_allows() requires "active" for a managed + node - LifecycleWatcher::node_ok()), so a fixture that never activates would make + GRAPH_NODE_DISAPPEARED structurally unreachable for every cycle here, not merely slow to + catch - this row carries the evidence that forbidding absence from maturing an unmatured + streak is safe, so it has to run against a node that can actually be reported. Each cycle's + respawned instance is re-armed (app_id-scoped wait_until_watchdog_armed, not merely + presence) before the NEXT kill for the identical reason a fresh process needs it the first + time: a kill landing before a + just-restarted instance reaches "active" would leave that cycle's death untracked too, and + the following poll_faults would time out for a precondition reason having nothing to do + with the claim this row is about. A future edit that reverts TARGET_NODE to launching + without auto_activate would silently make every cycle of this row unreachable again. + + Every cycle's outage is held down for B5_RESPAWN_DELAY_SEC, safely longer than the + explicitly-configured B5_MISS_GRACE this scenario launches with (see the module docstring): + left at launch's own 1.5 s respawn floor with no miss_grace configured, a correct detector's + own wall-clock floor could make the outage unreportable regardless of how many cycles this + test waits out, which is a row a right implementation cannot satisfy - not evidence of a + defect in one. Every raise is also checked by NAME, not merely by code, so a detector + reporting the same code for some other entity every cycle could not pass in TARGET_NODE's + place. + """ + + def test_every_restart_cycle_raises_and_clears(self, target_node): + # app_id form: this node reaches "active" on its own now (auto_activate=True), so the + # per-entity armed precondition genuinely becomes true - see the module docstring's + # "Which arming gate" section and this class's own docstring. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC, app_id=TARGET_NODE), + f'graph_watchdog never reported {TARGET_NODE} armed') + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=FAULTS_LIVE_TIMEOUT_SEC), + 'GET /faults never answered 200 - nothing below would prove anything') + + pid = target_node.process_details['pid'] + for cycle in range(1, B5_CYCLES + 1): + os.kill(pid, signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, TARGET_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'cycle {cycle}: {TARGET_NODE} never left GET /apps after SIGTERM', + ) + + fault = poll_faults(PORT, FAULT_CODE_DISAPPEARED, timeout=RAISE_TIMEOUT_SEC) + if fault is None: + self.fail(f'cycle {cycle}: {FAULT_CODE_DISAPPEARED} never raised') + self.assertIn( + TARGET_NODE, fault.get('description', ''), + f'cycle {cycle}: {FAULT_CODE_DISAPPEARED} raised but did not name {TARGET_NODE}') + + # app_id-scoped again, not just present: the respawned instance is a NEW process + # node_death has not armed yet, and the next iteration is about to kill it - the + # same precondition the FIRST kill above needed (see the class docstring). + self.assertTrue( + wait_until_watchdog_armed( + PORT, timeout=DEPARTURE_TIMEOUT_SEC + B5_RESPAWN_DELAY_SEC, + app_id=TARGET_NODE), + f'cycle {cycle}: {TARGET_NODE} never came back armed after the SIGTERM - ' + 'launch did not respawn it, or the respawned instance never reached "active"', + ) + # A fresh pid every cycle: read it back rather than assume launch's respawn keeps + # the value this test already has. + pid = target_node.process_details['pid'] + + self.assertTrue( + poll_cleared(PORT, FAULT_CODE_DISAPPEARED, timeout=CLEAR_TIMEOUT_SEC), + f'cycle {cycle}: {FAULT_CODE_DISAPPEARED} never cleared once {TARGET_NODE} ' + 'came back - this row is about EVERY cycle being caught AND released, not a ' + 'single continuous outage', + ) + + +class TestBoundaryNeverArmedBelowGraceThenGone(unittest.TestCase): + """B6: a required node that never arms, killed below grace: INACTIVE only, and it must raise. + + The row B2 cannot cover. B2's target reaches "active" first, so node_death can track it and + GRAPH_NODE_DISAPPEARED is the one to report a below-grace departure - which is exactly why + absence must NOT mature GRAPH_NODE_INACTIVE there. This target never reaches "active" at + all: LifecycleWatcher::node_ok() is false for its whole life, the gate never arms it + (per-entity `armed` requires "active" for a managed node), and node_death only ever tracks a + key the gate has armed at least once - so GRAPH_NODE_DISAPPEARED is structurally unable to + report this departure, whatever kills it. lifecycle_expectation is the only detector that + ever could, and closing that silence means absence has to be allowed to mature a below-grace + streak here, the opposite of B2's claim for the opposite reason. + + Both halves are checked: GRAPH_NODE_INACTIVE must raise and name the node (the silence + closing), and GRAPH_NODE_DISAPPEARED must stay absent throughout (the structural reason the + silence existed at all - if this half ever raised, node_death would have tracked a node the + gate never armed, which would be its own defect, not evidence this row's fix works). + """ + + @classmethod + def setUpClass(cls): + rclpy.init() + cls._client_node = Node('node_death_boundary_b6_client') + + @classmethod + def tearDownClass(cls): + cls._client_node.destroy_node() + rclpy.shutdown() + + def test_never_armed_departure_matures_inactive_disappeared_silent(self, target_node): + get_state_service = f'/{TARGET_NODE}/get_state' + + # Global gate: see the module docstring's "Which arming gate" section - this target + # never reaches "active", so the app_id-scoped form would wait for something that + # never becomes true. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC), + 'graph_watchdog never reported an armed global state') + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=FAULTS_LIVE_TIMEOUT_SEC), + 'GET /faults never answered 200 - nothing below would prove anything') + self.assertTrue( + _poll_apps_present(PORT, TARGET_NODE, timeout=PRESENCE_TIMEOUT_SEC), + f'{TARGET_NODE} never appeared on GET /apps - there is no present node here for ' + 'this row to measure', + ) + self.assertTrue( + _poll_lifecycle_label( + type(self)._client_node, get_state_service, 'unconfigured', + timeout=PRESENCE_TIMEOUT_SEC), + f'{TARGET_NODE} did not sit at "unconfigured" - the trigger this row needs (a ' + 'required node that never arms) was never set up', + ) + + # tracked_nodes becoming 1 is the tracker's own proof that it matched TARGET_NODE at + # least once - the earliest moment a kill is guaranteed not to land before the + # detector was even permitted to look. B6_GRACE is generous enough that the handful + # of extra ticks this poll's own interval can cost before the kill below still leaves + # the node comfortably below grace. + self.assertTrue( + poll_detector_status( + PORT, DETECTOR_ID_LIFECYCLE, 'tracked_nodes', 1, timeout=ARM_TIMEOUT_SEC), + f'lifecycle_expectation never reported tracking {TARGET_NODE} - it was never ' + 'matched at all, so killing it below would prove nothing about absence maturing ' + 'a violation the presence detector could never have reported', + ) + + # The row's own precondition, read from an observable immediately before the kill - + # same instrument as B2's identical check, for the identical reason: `is` rather than + # a plain falsy check, since None (channel unreachable) must not be read as + # "confirmed absent". + below_grace = _fault_present_now(PORT, FAULT_CODE_INACTIVE) + self.assertIs( + below_grace, False, + f'{FAULT_CODE_INACTIVE} could not be proven absent immediately before the kill ' + f'below (checked value: {below_grace!r}) - either the fault surface is ' + f'unreachable, or B6_GRACE ({B6_GRACE} ticks) already matured while the node was ' + 'still present, so the kill below would prove nothing about absence maturing the ' + 'violation', + ) + + os.kill(target_node.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, TARGET_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'{TARGET_NODE} never left GET /apps after SIGTERM') + + # The row this file exists to add: with node_death structurally unable to track a + # node that was never armed, lifecycle_expectation's own absence handling has to be + # the one that matures the below-grace streak, or the departure is reported by + # nothing at all. + fault = poll_faults(PORT, FAULT_CODE_INACTIVE, timeout=RAISE_TIMEOUT_SEC) + if fault is None: + self.fail( + f'{FAULT_CODE_INACTIVE} never raised for the departed {TARGET_NODE} - a node ' + 'the presence detector could never have reported went unreported by every ' + 'detector') + self.assertIn(TARGET_NODE, fault.get('description', '')) + + # The structural half: GRAPH_NODE_DISAPPEARED must never raise for this node. It was + # never armed, so node_death could never have tracked it in the first place - a raise + # here would mean node_death tracked a node the gate never armed, not that this row's + # fix works. + assert_fault_absent_throughout(self, PORT, FAULT_CODE_DISAPPEARED, SUSTAINED_WINDOW_SEC) + + +class TestBoundaryConfigEndpointE2E(unittest.TestCase): + """C4: `detectors.node_death.miss_grace` visibly changes when a death is reported. + + One gateway, one armed node, killed once, with a LARGE miss_grace configured + (C4_MISS_GRACE_LARGE, comfortably under the documented 3600-tick ceiling). Proves the + knob governs an OBSERVABLE, not merely that the plugin accepts it at startup, by checking the + SAME gateway at two points on ONE timeline rather than comparing two gateways: first, a + window (C4_EARLY_WINDOW_SEC) long enough that a near-floor config (this suite's own ~4s + convention - B5_MISS_GRACE, D2_MISS_GRACE) would already have raised, in which this + large-grace gateway must stay silent - needle-scoped (assert_fault_never_names), so a + detector raising for some unrelated entity cannot decide the row; second, once the + configured grace has had time to elapse, the fault must still arrive and name the node - the + large value delays the report, it does not swallow it. + """ + + def test_large_miss_grace_delays_then_still_reports(self, target_node): + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC, app_id=C4_TARGET_NODE), + f'graph_watchdog never reported {C4_TARGET_NODE} armed') + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=FAULTS_LIVE_TIMEOUT_SEC), + 'GET /faults never answered 200 - nothing below would prove anything') + + os.kill(target_node.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, C4_TARGET_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'{C4_TARGET_NODE} never left GET /apps after SIGTERM') + + # Needle-scoped: a raise for some OTHER entity must not decide this row (see + # assert_fault_never_names's own docstring). Long enough that a near-floor config would + # already have raised; short enough to sit well inside C4_MISS_GRACE_LARGE's own grace. + assert_fault_never_names( + self, PORT, FAULT_CODE_DISAPPEARED, forbidden=[C4_TARGET_NODE], + duration=C4_EARLY_WINDOW_SEC) + + # Past the configured grace, the same knob that delayed the raise must not have + # suppressed it outright. + fault = poll_faults(PORT, FAULT_CODE_DISAPPEARED, timeout=C4_LATE_RAISE_TIMEOUT_SEC) + if fault is None: + self.fail( + f'{FAULT_CODE_DISAPPEARED} never raised for {C4_TARGET_NODE} even past the ' + 'configured C4_MISS_GRACE_LARGE grace') + self.assertIn(C4_TARGET_NODE, fault.get('description', '')) + + +class TestBoundaryUngatedClear(unittest.TestCase): + """D2: an ungated clear must not push a stored fault toward healing before anything measured. + + test_01 confirms the death and its stored record. test_02 then restarts the GATEWAY with a + wide warmup_cycles, and watches the pre-arm window that follows for a premature PASSED on + the stored record - the restarted detector must not report anything about a node it has + never actually measured in this process's lifetime. + """ + + def test_01_the_fault_raises_before_the_restart(self, target_node): + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC, app_id=D2_TARGET_NODE), + f'graph_watchdog never reported {D2_TARGET_NODE} armed') + + os.kill(target_node.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, D2_TARGET_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'{D2_TARGET_NODE} never left GET /apps after SIGTERM') + + fault = poll_faults(PORT, FAULT_CODE_DISAPPEARED, timeout=RAISE_TIMEOUT_SEC) + if fault is None: + self.fail( + f'{FAULT_CODE_DISAPPEARED} never raised for the departed {D2_TARGET_NODE} - ' + 'there is nothing STORED for the restart below to wrongly move toward healing') + self.assertIn(D2_TARGET_NODE, fault.get('description', '')) + + record = _fault_record(PORT, FAULT_CODE_DISAPPEARED, timeout=DEPARTURE_TIMEOUT_SEC) + if record is None: + self.fail(f'{FAULT_CODE_DISAPPEARED} vanished from the store entirely') + self.assertIsNone( + record.get('last_passed'), + f'{FAULT_CODE_DISAPPEARED} was already reported PASSED before the restart ' + f'(last_passed={record.get("last_passed")!r}) - there is nothing left here for ' + 'the restart below to wrongly preserve or wrongly heal', + ) + + def test_02_the_pre_arm_window_never_reports_passed(self, gateway_node, target_node): + del target_node # stays gone by design; not signalled again in this method + old_pid = gateway_node.process_details['pid'] + os.kill(old_pid, signal.SIGTERM) + self.assertTrue( + _wait_until_port_is_down(PORT, timeout=60.0), + f'the gateway (pid {old_pid}) kept answering after SIGTERM - nothing restarted') + + # Gate on the fault surface being reachable AT ALL before opening the ungated-window + # clock. A restarted gateway's HTTP server - let alone its round trip to the + # fault_manager, a separate process this restart never touched - is not up the instant + # the OLD port stops answering: create_gateway_node's own respawn_delay is an ENFORCED + # floor before launch even starts the replacement, before that process's own ROS init + # and HTTP bind. A poll issued before either is up cannot ask this row's own question + # ("was PASSED reported") at all - it can only observe a channel that does not exist + # yet, which _assert_never_passed_throughout correctly reports as unreachable rather + # than silently reading as "never reported PASSED". Confirmed live: without this gate, + # the very first poll after the old port went down found GET /faults refusing the + # connection outright. + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=FAULTS_LIVE_TIMEOUT_SEC), + 'GET /faults never answered 200 after the restart - the ungated-window check below ' + 'would have no live channel to ask anything on', + ) + + # The window has to be PROVEN pre-arm, not merely assumed from D2_WARMUP_CYCLES's own + # nominal budget: if restart-plus-HTTP-recovery above ate most or all of that nominal + # warmup, the watchdog could already be armed by the time the window below opens, and + # any PASSED it then observes would be legitimate post-arm behaviour, not the + # ungated-clear bug this row exists to catch. One immediate read, not a poll - the point + # is the state AT THIS INSTANT, immediately before the window starts. + pre_state = _watchdog_global_state(PORT) + self.assertNotEqual( + pre_state, 'armed', + f'the gateway was already armed (global_state={pre_state!r}) before the ungated ' + 'window below could even open - restart-plus-recovery consumed the whole nominal ' + f'warmup (D2_WARMUP_CYCLES={D2_WARMUP_CYCLES}), so any PASSED observed below would ' + 'be legitimate post-arm behaviour, not the ungated-clear bug this row exists to ' + 'catch; widen D2_WARMUP_CYCLES rather than trusting this window without proof', + ) + + # The claim under test: for D2_UNGATED_WATCH_SEC once the surface is reachable, the + # stored fault's last_passed must stay None - comfortably before D2_WARMUP_CYCLES * + # TICK_INTERVAL_MS (~30s nominal) elapses, so the window is provably still INSIDE the + # period during which the restarted plugin's own gate has not armed anything. + _assert_never_passed_throughout( + self, PORT, FAULT_CODE_DISAPPEARED, D2_UNGATED_WATCH_SEC) + + # The other bracket: prove the window closed still inside pre-arm too, or the samples + # above are not provably pre-arm from end to end - only from their own start. + post_state = _watchdog_global_state(PORT) + self.assertNotEqual( + post_state, 'armed', + f'the gateway armed (global_state={post_state!r}) DURING the {D2_UNGATED_WATCH_SEC}s ' + 'ungated window above - the samples it collected are no longer provably pre-arm ' + 'from end to end, so they cannot be read as evidence about the ungated-clear bug ' + 'this row exists to catch', + ) + + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=90.0), + 'the gateway never came back armed after the restart - the warmup window this row ' + 'is about never actually ended, so the check above proved nothing about a window ' + 'that closes') + self.assertNotEqual( + gateway_node.process_details['pid'], old_pid, + 'the gateway process id did not change, so this test never restarted anything') + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=FAULTS_LIVE_TIMEOUT_SEC), + 'GET /faults never answered 200 after the restart - the persistence check below ' + 'would prove nothing', + ) + self.assertFalse( + _poll_apps_present(PORT, D2_TARGET_NODE, timeout=5.0), + f'{D2_TARGET_NODE} is back in GET /apps - this row is about a restart with the ' + 'node STILL gone, and it came back instead', + ) + + assert_fault_persists_throughout(self, PORT, FAULT_CODE_DISAPPEARED, SUSTAINED_WINDOW_SEC) + + record = _fault_record(PORT, FAULT_CODE_DISAPPEARED, timeout=DEPARTURE_TIMEOUT_SEC) + if record is None: + self.fail(f'{FAULT_CODE_DISAPPEARED} vanished from the store after the restart') + self.assertIsNone( + record.get('last_passed'), + f'the restarted gateway reported {FAULT_CODE_DISAPPEARED} PASSED for a node it can ' + f'never have observed in this process (last_passed={record.get("last_passed")!r})', + ) + + +# Each CTest target launches this file with one scenario, so only that scenario's case may +# run. Removing the others from the module (rather than skipping them) means each run reports +# exactly one case, and a missing result is a real failure rather than an expected line of +# output - see test_node_death_e2e.test.py's identical rationale. +_SCENARIO_CASES = { + 'b1_inactive_present': 'TestBoundaryInactivePresent', + 'b2_inactive_below_grace_then_gone': 'TestBoundaryInactiveBelowGraceThenGone', + 'b3_matured_then_gone': 'TestBoundaryMaturedThenGone', + 'b4_healthy_then_gone': 'TestBoundaryHealthyThenGone', + 'b5_restart_loop_still_caught': 'TestBoundaryRestartLoopStillCaught', + 'b6_never_armed_below_grace_then_gone': 'TestBoundaryNeverArmedBelowGraceThenGone', + 'c4_config_endpoint_e2e': 'TestBoundaryConfigEndpointE2E', + 'd2_ungated_clear': 'TestBoundaryUngatedClear', +} +if SCENARIO not in _SCENARIO_CASES: + raise RuntimeError( + f'WATCHDOG_E2E_SCENARIO={SCENARIO!r} is not one of {sorted(_SCENARIO_CASES)}; the ' + 'CTest target and this file disagree about which scenarios exist') +for _scenario, _case_name in _SCENARIO_CASES.items(): + if _scenario != SCENARIO: + del globals()[_case_name] + + +@launch_testing.post_shutdown_test() +class TestShutdown(unittest.TestCase): + """Verify the gateway/fault_manager/demo stack exits cleanly.""" + + def test_exit_codes(self, proc_info): + for info in proc_info: + self.assertIn( + info.returncode, + ALLOWED_EXIT_CODES, + f'Process {info.process_name} exited with {info.returncode}', + ) diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_node_death_e2e.test.py b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_node_death_e2e.test.py new file mode 100644 index 000000000..12468aa41 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_node_death_e2e.test.py @@ -0,0 +1,1492 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""node_death e2e: GRAPH_NODE_DISAPPEARED against the REAL gateway + fault_manager + graph. + +A scenario that asserts a RAISE proves the fault actually appears, naming the missing-fault +case as the failure mode when it does not. A scenario that asserts an ABSENCE (a node that must +never be called dead) cannot, by itself, discriminate a correct detector from no detector at +all, because both produce the same silence; those scenarios are marked as such in each class's +own docstring below. + +Runs as TEN separate CTest targets (see CMakeLists.txt), each launching its OWN +gateway+fault_manager+demo-node stack, exactly as test_lifecycle_expectation_e2e.test.py +does and for the same reason: the plugin reads its config once at set_context() time, so +different configs need different gateway launches. WATCHDOG_E2E_SCENARIO selects which +launch and which assertions run: + +- "raise": a plain node comes up, is armed, then its process exits. GRAPH_NODE_DISAPPEARED + appears and names it. Also runs both new harness self-tests + (prove_persistence_proof_catches_a_dead_fault_surface, + prove_describes_only_proof_catches_a_dead_fault_surface) - self-contained, so they run + alongside the real assertions without depending on them, the way the lifecycle e2e's + "default_config" scenario runs the silence one. +- "clear_on_return": the same, then the node is started again. The fault clears once the + node is back - the occurrence model's own "outage genuinely ended" case: no forced + transition, the detector closes a cycle when the graph actually recovers. +- "no_heal_standalone": the same drive as clear_on_return, but the fault_manager runs with + healing OFF. The fault does NOT clear - not a defect, the documented shape of a debounce + hysteresis latch that only a HEALED-enabled config can cross - proven as a SUSTAINED + claim over a window via assert_fault_persists_throughout, not a single sample. +- "deactivated_not_dead": a managed lifecycle node reaches active on its own + (managed_lifecycle_active's auto_activate) and is then driven to DEACTIVATE by this test + through a real ChangeState call, while its process keeps running throughout. No + GRAPH_NODE_DISAPPEARED for the whole window - node_death is a presence detector; a + lifecycle transition is not a departure, and telling the two apart is + lifecycle_expectation's job, not this one's (see B1-B5 in + test_node_death_boundary_e2e.test.py, none of them written here). See + TestNodeDeathDeactivatedNotDead's own docstring for what this absence does and does not + prove. +- "manifest_never_online": a hybrid-mode manifest declares an App whose ROS binding never + starts. No GRAPH_NODE_DISAPPEARED for it, ever - the promise the public issue leads with: + a manifest node keeps its App in the snapshot with the online flag cleared, so a detector + that counted snapshot membership alone would call it immortal - see + TestNodeDeathManifestNeverOnline's own docstring for what this absence does and does not + prove. +- "ros2cli_ignored": a node renamed to carry the ros2cli hidden-node prefix is armed then + killed, three times over with distinct names. No fault, and the detector's own + tracked-count status field does not grow across the three cycles - a node matching the + naming convention must not accumulate as permanent tracked state, regardless of what + process gave it that name. A second check pins that fixture's prefix against the + installed ros2cli package's own naming constants, so the fixture cannot silently drift + from what ros2cli itself actually uses - see TestNodeDeathRos2cliIgnored's own docstring + for what this absence does and does not prove. +- "bare_name_collision": two nodes named 'calibration' in different namespaces; one exits. + The fault names the one that exited and does not name the one still running - proven with + assert_fault_describes_only so the claim holds over the WHOLE window, not one lucky read. +- "fast_tick_floor": a short tick_interval_ms, with a node continuously present. No fault. + Narrower than the row this is named for: it does not force a stale graph-cache + generation (config sweep C1 owns the miss_grace floor's own boundary values), only that + fast ticking alone, with nothing perturbing the graph, is safe - see + TestNodeDeathFastTickFloor's own docstring for what was investigated and why forcing the + stronger condition was not implemented - see TestNodeDeathDeactivatedNotDead's own note + for what this absence does and does not prove either way. +- "restart_loop_occurrences": a node killed and restarted three times, each cycle closed + with an explicit acknowledge before the next kill. The fault's occurrence_count reaches + 3 - the fault manager's own occurrence model (a FAILED event reactivating a CLEARED + record bumps the count; a re-report on a still-active fault does not) rather than any + trick the detector plays. See TestNodeDeathRestartLoopOccurrences's own docstring for why + this class stops at occurrence_count and does not attempt the per-occurrence RECORDING + half of this row. +- "restart_rebaseline": a node dies, the fault confirms, then the gateway is killed by PID + and comes back with the node still gone. RECORDS the boundary, the way + TestLifecycleExpectationRestartDeparted does for the sibling detector: the fault stays + CONFIRMED, proven as a sustained claim across the restart via + assert_fault_persists_throughout, not discovered by a single sample. + +Every scenario gates on wait_until_watchdog_armed(PORT) BEFORE asserting anything, and +every scenario that asserts an ABSENCE additionally gates on +wait_until_faults_endpoint_live(PORT) - see harness.py's own docstrings for why a stack +that never came up otherwise produces exactly the silence an absence claim is looking for. +""" + +import os +import signal +import subprocess +import sys +import time +import unittest + +from ament_index_python.packages import get_package_prefix, get_package_share_directory +from launch.actions import TimerAction +import launch_ros.actions +import launch_testing +from lifecycle_msgs.msg import Transition +from lifecycle_msgs.srv import ChangeState, GetState +import rclpy +from rclpy.node import HIDDEN_NODE_PREFIX, Node +import requests +from ros2cli.node import NODE_NAME_PREFIX + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +# I100 as well as E402: `harness` is only importable because of the sys.path line above, +# so this import cannot be moved up to where the alphabetical order would put it. +from harness import ( # noqa: E402, I100 + API_BASE_PATH, + assert_fault_absent_throughout, + assert_fault_describes_only, + assert_fault_persists_throughout, + create_watchdog_test_launch, + poll_cleared, + poll_entity_faults, + poll_faults, + prove_describes_only_proof_catches_a_dead_fault_surface, + prove_persistence_proof_catches_a_dead_fault_surface, + wait_until_faults_endpoint_live, + wait_until_watchdog_armed, + watchdog_detector_status, +) + +from ros2_medkit_test_utils.constants import ( # noqa: E402 + ALLOWED_EXIT_CODES, + get_test_port, + get_time_scale, +) +from ros2_medkit_test_utils.coverage import get_coverage_env # noqa: E402 + +# No default on purpose - a default makes this file FAIL OPEN (see harness-consuming +# siblings' identical rationale): if the CTest ENVIRONMENT property never reaches the +# process, the launch and the assertions would degrade together into one scenario and +# still report 1/1 passed. A KeyError is loud. +SCENARIO = os.environ['WATCHDOG_E2E_SCENARIO'] +PORT = get_test_port() + +FAULT_CODE = 'GRAPH_NODE_DISAPPEARED' +# The detector's own id, matched against GET /x-medkit-watchdog's `detectors` block by +# watchdog_detector_status(). node_death has no config to isolate a fault ON (it is +# zero-config: every App the graph carries is in scope, not an explicit require_active-style +# list), so unlike lifecycle_expectation's _DETECTOR_PREFIX-scoped keys, only `miss_grace` +# and `prune_grace` below are ever set under it. +DETECTOR_ID = 'node_death' +_DETECTOR_PREFIX = f'plugins.graph_watchdog.detectors.{DETECTOR_ID}' + +# Fast tick cadence + a miss_grace comfortably PAST the documented 3000 ms floor +# (min_node_death_miss_grace(200) == 14, detector_config_keys.hpp) without sitting exactly on +# it. "config sweep C1" (NodeDeathIntegrationTest.C1_* in test_node_death_integration.cpp) owns +# the floor's own boundary values, but never exercises tick_interval_ms=200 at all (its own +# cases run at 1, 1000 and 3000 ms), so there is no boundary value at this tick period to land +# on by accident - 16 carries two ticks of headroom purely to keep this row visibly off the +# floor itself. 16 ticks * 200 ms = 3400 ms nominal grace; RESPAWN_DELAY_SEC below is sized +# against this value. +TICK_INTERVAL_MS = 200 +WARMUP_CYCLES = 3 +MISS_GRACE = 16 + +# The plain node most scenarios kill: DEMO_NODE_REGISTRY's own executable/name/namespace +# for the 'calibration' key, constructed by hand (not via demo_nodes=[...]) wherever a +# scenario needs a PID to signal - create_demo_nodes() gives no handle back. Single +# occurrence of the bare name 'calibration' in this launch, so the discovery layer's +# collision-avoidance never engages and the App id IS the bare name (verified against +# ros2_runtime_introspection.cpp's own collision rule: only namespace-prefixed when the +# SAME bare name appears in more than one namespace at once). +TARGET_NODE = 'calibration' +TARGET_EXECUTABLE = 'demo_calibration_service' +TARGET_NAMESPACE = '/powertrain/engine' + +# launch will not even ATTEMPT to restart a respawn=True process before this elapses, so +# it is a real, enforced floor under every kill-then-return gap this file measures. Must +# clear the detector's OWN raise threshold, not just be "over one tick interval": a death is +# reported once misses EXCEED miss_grace, i.e. after (MISS_GRACE + 1) * TICK_INTERVAL_MS = +# 17 * 200ms = 3400ms of real absence. A respawn_delay short of that heals the outage before +# a correct detector could ever confirm it, so clear_on_return/no_heal_standalone/ +# restart_loop_occurrences (the three scenarios that actually respawn TARGET_NODE) would +# time out waiting for a raise that is never supposed to happen - not a detector defect. +# Mirrors B5_RESPAWN_DELAY_SEC in the boundary e2e file (same detector, same tick interval, +# same derivation - see that constant's own comment for the full arithmetic). 3400ms nominal +# grace is not the whole budget a tick-counted miss_grace needs: the entity cache is refreshed +# on a debounced graph event, up to 1100ms of latency on either end of the outage +# (gateway_node.cpp), and the plugin's own tick loop can run slower than its configured period +# under CI contention with no enforced upper bound. Margin under the 1100ms debounce ceiling +# alone can be erased by a single unlucky refresh regardless of tick-loop speed, and +# NodeLivenessTracker::update() resets a key's miss count to zero the instant it is seen +# present again, so a cycle that loses this race does not raise late, it never raises at all. +# 12.5s clears 3*3400+1100=11300ms (a tick loop running at a third of its configured rate, +# plus the full debounce ceiling on top) by 1200ms - worst-case observed window +# 12500-1100=11400ms, 3.35x the nominal grace. +RESPAWN_DELAY_SEC = 12.5 + +# The budgets below are scaled by MEDKIT_TEST_TIME_SCALE, which the sanitizer jobs set to the +# same factor they apply to every declared CTest timeout. A deadline asserted INSIDE a test is +# invisible to that rewrite, so an instrumented graph that takes longer to forget a departed +# node blows a budget here and the failure reads as a detector that never reported - the exact +# red this suite exists to produce for a real defect. Unset elsewhere, so the normal jobs keep +# the tight budgets that give these assertions their falsifying edge. +# +# Poll intervals, enforced respawn delays and the sustained-observation windows are NOT scaled. +# Those are not give-up bounds: stretching a window a scenario watches for silence buys no +# confidence and spends the whole package's test budget to do it. +TIME_SCALE = get_time_scale() +ARM_TIMEOUT_SEC = 60.0 * TIME_SCALE +FAULTS_LIVE_TIMEOUT_SEC = 30.0 * TIME_SCALE +PRESENCE_TIMEOUT_SEC = 30.0 * TIME_SCALE +DEPARTURE_TIMEOUT_SEC = 30.0 * TIME_SCALE +RAISE_TIMEOUT_SEC = 60.0 * TIME_SCALE +CLEAR_TIMEOUT_SEC = 60.0 * TIME_SCALE +# How long a fault must stay ABSENT, or CONFIRMED, for the scenarios whose claim is +# sustained rather than momentary. Short (well under a minute) because every window here +# is measured from the arming gate, not process start, so bringup cannot eat it - matching +# SILENT_WINDOW_SEC's own rationale in the lifecycle e2e file. +SUSTAINED_WINDOW_SEC = 20.0 + +# The entity that owns the aggregated fault this plugin raises - proven already by +# test_lifecycle_expectation_e2e.test.py's own poll_entity_faults(PORT, +# 'apps/graph_watchdog', ...) calls, reused here for the REST-scoped clear_fault call +# restart_loop_occurrences needs. +SOURCE_ENTITY_PATH = 'apps/graph_watchdog' + +# ---- the "fast_tick_floor" scenario's own cadence --------------------------------------- +# +# Deliberately tiny: at 50 ms/tick, a miss_grace of 1 tick is 50 ms of NOMINAL grace - far +# under the documented 3000 ms floor (config sweep C1). If the floor did not exist, a +# single stale graph-cache generation between two ticks would be enough to call a healthy, +# present node dead. The claim under test is that it is not enough, because the floor +# raises the EFFECTIVE grace regardless of how small tick_interval_ms is configured. +FAST_TICK_INTERVAL_MS = 50 +FAST_MISS_GRACE = 1 + +# ---- the "bare_name_collision" scenario's own namespaces --------------------------------- +# +# Short and distinctive so they cannot be confused with any area/namespace used elsewhere +# in this file or in the shared demo fixtures. +COLLISION_NAMESPACE_A = '/coll_a' +COLLISION_NAMESPACE_B = '/coll_b' +# ros2_runtime_introspection.cpp's own collision-avoidance rule, applied by hand: two nodes +# named 'calibration' in different namespaces get namespace-prefixed ids +# ('_'), so the bare 'calibration' +# alone is no longer sufficient to tell them apart - which is exactly what this scenario +# means by "the fault names the one that exited and does not name the one still running". +COLLISION_APP_ID_A = 'coll_a_calibration' +COLLISION_APP_ID_B = 'coll_b_calibration' +# How long the "names the dead one, not the alive one" claim is watched for. Short (a few +# ticks at TICK_INTERVAL_MS) because the claim is "never confuses the two", not "forever" - +# but a WINDOW checked on every poll, never a single sample. +MUTUAL_NAMING_WINDOW_SEC = 5.0 + +# ---- the "ros2cli_ignored" scenario's own fixtures ----------------------------------------- +# The naming convention node_death filters on: a leaf node name +# starting with this prefix, regardless of what process created it. The load-bearing half +# of this scenario proves the convention itself, not any particular producer of it - see +# TestNodeDeathRos2cliIgnored's own docstring for why that distinction is the whole point. +# Pinned against the installed `ros2cli` package's own NODE_NAME_PREFIX, not just assumed +# to stay correct - see this class's test_02. +ROS2CLI_FAKE_NODE_PREFIX = '_ros2cli_fake_' +# How many renamed-node arm/kill cycles the scenario runs, and how many distinct +# ROS2CLI_FAKE_NODE_PREFIX-named nodes it exercises. +ROS2CLI_CYCLES = 3 +# How long the before/after tracked_count samples may take to settle (see +# _poll_stable_tracked_count) - generous against gateway-internal infrastructure (confirmed +# live: a hidden `_param_client_node`, visible only because this scenario turns off +# filter_internal_nodes) arming on its own schedule, not against anything this scenario's +# own cycles do. Must comfortably exceed _poll_stable_tracked_count's own stable_seconds +# default (10.0), or the poll could time out before stability was ever even reachable. +STABLE_TRACKED_COUNT_TIMEOUT_SEC = 40.0 * TIME_SCALE + +# ---- the "restart_loop_occurrences" scenario's own target ------------------------------- +# The claim under test is that occurrence_count tracks the number of genuine deaths. Three +# distinct occurrences demonstrate that exactly as well as five: what would falsify the claim +# is a count that stops incrementing (a detector that goes silent after the first cycle) or +# double-counts (an ungated clear or a duplicate FAILED inflating the number), and either +# failure mode shows up by the third cycle - a fourth or fifth would only repeat the same +# proof at the full cost of RESPAWN_DELAY_SEC apiece. If a future change ever needs more +# cycles to expose something this one does not, raising this single constant is the whole +# edit. +RESTART_LOOP_OCCURRENCES_TARGET = 3 + + +def _target_node_action(*, respawn=False, respawn_delay=RESPAWN_DELAY_SEC): + """One manually-constructed TARGET_NODE action, with a PID handle the test can signal. + + Mirrors DEMO_NODE_REGISTRY's own 'calibration' entry exactly (same package, executable, + name, namespace) so this is the identical fixture every other scenario in this package + gets via demo_nodes=['calibration'] - just launched by hand because a scenario that + kills it needs the launch action's own process_details, which create_demo_nodes() does + not hand back. + """ + return launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable=TARGET_EXECUTABLE, + name=TARGET_NODE, + namespace=TARGET_NAMESPACE, + output='screen', + additional_env=get_coverage_env('ros2_medkit_integration_tests'), + sigterm_timeout='30', + sigkill_timeout='15', + respawn=respawn, + respawn_delay=respawn_delay, + ) + + +def generate_test_description(): + detector_params = { + 'plugins.graph_watchdog.tick_interval_ms': TICK_INTERVAL_MS, + 'plugins.graph_watchdog.warmup_cycles': WARMUP_CYCLES, + f'{_DETECTOR_PREFIX}.miss_grace': MISS_GRACE, + } + extra_gateway_params = None + demo_nodes = [] + healing_enabled = True + gateway_respawn = False + + if SCENARIO == 'raise': + pass # target node launched by hand below, killed once, never restarted + elif SCENARIO == 'clear_on_return': + pass # target node launched by hand below, killed then respawned + elif SCENARIO == 'no_heal_standalone': + healing_enabled = False # the whole point of this scenario + elif SCENARIO == 'deactivated_not_dead': + demo_nodes = ['managed_lifecycle_active'] # self-activates; this test deactivates it + elif SCENARIO == 'manifest_never_online': + pkg_share = get_package_share_directory('ros2_medkit_gateway') + manifest_path = os.path.join(pkg_share, 'config', 'examples', 'demo_nodes_manifest.yaml') + extra_gateway_params = { + 'discovery.mode': 'hybrid', + 'discovery.manifest_path': manifest_path, + # The manifest's own validator can turn an informational notice into a load + # failure under strict validation, which in hybrid mode silently degrades the + # gateway to runtime_only - see this package's CLAUDE.md gotcha and + # test_discovery_gap_fill.test.py's identical setting for the same manifest. + 'discovery.manifest_strict_validation': False, + } + # Only 'calibration' comes online - it satisfies 'engine-calibration-service's + # ros_binding, giving the arming gate something to report on. Every OTHER app the + # manifest declares (e.g. 'lidar-sensor', bound to a 'lidar_sensor' node this + # launch never starts) stays in the snapshot with its online flag cleared and + # nothing ever links it - the exact fixture this scenario needs, already sitting in + # the manifest that ships as this repo's own integration-test fixture. + demo_nodes = ['calibration'] + elif SCENARIO == 'ros2cli_ignored': + # No demo_nodes: test_01 spawns and kills its own renamed nodes by hand, and + # test_02 is a pure constant comparison against the installed ros2cli package - + # neither needs anything else running. + extra_gateway_params = { + # OFF for this scenario only. Left at its true default (on), a node named + # like the ros2cli convention would be invisible upstream of EVERYTHING - + # GET /apps, the entity cache, any IntrospectionProvider plugin gets fed + # from it - which would make this scenario pass for the wrong reason: never + # tracked at all, rather than tracked and then correctly excluded by name. + # This scenario is about node_death's OWN name-based exclusion, which needs a + # node the detector's input can actually see. See the class docstring. + 'discovery.runtime.filter_internal_nodes': False, + } + elif SCENARIO == 'bare_name_collision': + pass # two hand-built nodes below, same bare name, different namespaces + elif SCENARIO == 'fast_tick_floor': + detector_params['plugins.graph_watchdog.tick_interval_ms'] = FAST_TICK_INTERVAL_MS + detector_params[f'{_DETECTOR_PREFIX}.miss_grace'] = FAST_MISS_GRACE + demo_nodes = ['calibration'] + elif SCENARIO == 'restart_loop_occurrences': + pass # target node launched by hand below, respawning under this test's control + elif SCENARIO == 'restart_rebaseline': + gateway_respawn = True # this scenario's own subject, like lifecycle's "main" + else: + raise RuntimeError(f'WATCHDOG_E2E_SCENARIO={SCENARIO!r} has no launch configuration') + + launch_description, context = create_watchdog_test_launch( + detector_params=detector_params, + extra_gateway_params=extra_gateway_params, + demo_nodes=demo_nodes, + port=PORT, + healing_enabled=healing_enabled, + gateway_respawn=gateway_respawn, + ) + + if SCENARIO in ('raise', 'restart_rebaseline'): + # No respawn: both scenarios kill the node PERMANENTLY and measure what happens to + # the fault it left behind - a bounce back would defeat the departure each is + # actually about. + target = _target_node_action(respawn=False) + launch_description.add_action(TimerAction(period=2.0, actions=[target])) + context['target_node'] = target + + if SCENARIO in ('clear_on_return', 'no_heal_standalone', 'restart_loop_occurrences'): + # respawn=True: every one of these scenarios kills the node and then needs it back + # - once for clear_on_return/no_heal_standalone, three times in a loop for + # restart_loop_occurrences - and create_demo_nodes() gives no PID handle to SIGTERM + # by hand in the first place, so this fixture is built here regardless. + target = _target_node_action(respawn=True, respawn_delay=RESPAWN_DELAY_SEC) + launch_description.add_action(TimerAction(period=2.0, actions=[target])) + context['target_node'] = target + + if SCENARIO == 'bare_name_collision': + node_a = launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable=TARGET_EXECUTABLE, + name=TARGET_NODE, + namespace=COLLISION_NAMESPACE_A, + output='screen', + additional_env=get_coverage_env('ros2_medkit_integration_tests'), + sigterm_timeout='30', + sigkill_timeout='15', + ) + node_b = launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable=TARGET_EXECUTABLE, + name=TARGET_NODE, + namespace=COLLISION_NAMESPACE_B, + output='screen', + additional_env=get_coverage_env('ros2_medkit_integration_tests'), + sigterm_timeout='30', + sigkill_timeout='15', + ) + launch_description.add_action(TimerAction(period=2.0, actions=[node_a, node_b])) + context['node_a'] = node_a + context['node_b'] = node_b + + return launch_description, context + + +# --------------------------------------------------------------------------------------- +# Local helpers. These read the SAME endpoints harness.py's own helpers do, but with +# either a different status filter (occurrence_count and last_passed survive a HEAL or a +# CLEAR, both of which drop out of the default active-only listing) or a service this +# file's scenarios need that no existing harness helper covers. +# --------------------------------------------------------------------------------------- + +def _fault_record(port, code, timeout=30.0, interval=0.5): + """Poll ``GET /faults?status=all`` until `code` appears, whatever its status. + + ``poll_faults`` uses the default (pending+confirmed) filter, so a HEALED or CLEARED + fault disappears from it - indistinguishable from one that was never raised. The + restart and occurrence scenarios need the record itself, including fields + (``last_passed``, ``occurrence_count``) that survive both. Returns the matching item + dict, or ``None`` on timeout. + """ + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + response = requests.get(f'{base}/faults', params={'status': 'all'}, timeout=5) + if response.status_code == 200: + for item in response.json().get('items', []): + if item.get('fault_code') == code: + return item + except requests.exceptions.RequestException: + pass + time.sleep(interval) + return None + + +def _poll_occurrence_count(port, code, expected, timeout=60.0, interval=0.5): + """Poll until `code`'s stored ``occurrence_count`` equals `expected`. ``True`` once it does. + + Not "at least" - a detector that fires an extra spurious reactivation would satisfy + "at least 5" just as well as one that fires exactly 5, so an exact match is what + actually discriminates. Prints the last-seen record on timeout, which is gone once the + launch tears down. + """ + deadline = time.monotonic() + timeout + last_seen = f'{code} was never in the store at all' + while time.monotonic() < deadline: + record = _fault_record(port, code, timeout=interval) + if record is not None: + last_seen = str(record) + if record.get('occurrence_count') == expected: + return True + time.sleep(interval) + print(f'_poll_occurrence_count({code!r}, expected={expected!r}) timed out after ' + f'{timeout}s; last seen: {last_seen}') + return False + + +def _wait_until_port_is_down(port, timeout=60.0, interval=0.2): + """Wait until the gateway's HTTP port stops answering. True once it does.""" + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + requests.get(f'{base}/health', timeout=2) + except requests.exceptions.RequestException: + return True + time.sleep(interval) + return False + + +def _poll_apps_absent(port, app_id, timeout=30.0, interval=0.5): + """Poll ``GET /apps`` until `app_id` is no longer listed. ``True`` once it is gone.""" + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + timeout + last_seen = 'GET /apps was never answered at all' + while time.monotonic() < deadline: + try: + response = requests.get(f'{base}/apps', timeout=5) + if response.status_code == 200: + ids = [item.get('id') for item in response.json().get('items', [])] + last_seen = str(ids) + if app_id not in ids: + return True + else: + last_seen = f'HTTP {response.status_code} from GET /apps' + except requests.exceptions.RequestException as exc: + last_seen = f'GET /apps failed: {exc}' + time.sleep(interval) + print(f'_poll_apps_absent({app_id!r}) timed out after {timeout}s; last seen: {last_seen}') + return False + + +def _poll_apps_present(port, app_id, timeout=30.0, interval=0.5): + """Poll ``GET /apps`` until `app_id` IS listed. ``True`` once it appears.""" + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + timeout + last_seen = 'GET /apps was never answered at all' + while time.monotonic() < deadline: + try: + response = requests.get(f'{base}/apps', timeout=5) + if response.status_code == 200: + ids = [item.get('id') for item in response.json().get('items', [])] + last_seen = str(ids) + if app_id in ids: + return True + else: + last_seen = f'HTTP {response.status_code} from GET /apps' + except requests.exceptions.RequestException as exc: + last_seen = f'GET /apps failed: {exc}' + time.sleep(interval) + print(f'_poll_apps_present({app_id!r}) timed out after {timeout}s; last seen: {last_seen}') + return False + + +def _clear_fault(port, entity_path, code, timeout=10.0): + """``DELETE {entity_path}/faults/{code}`` - the REST acknowledge, ``~/clear_fault``. + + Unconditional: it writes CLEARED whatever the fault's current status is + (``fault_storage.cpp``'s own ``clear_fault`` takes no status guard), which is exactly + why it - and not a heal race - is what restart_loop_occurrences uses to close one + outage cycle before the next kill reactivates it as a fresh occurrence. Returns + ``True`` on any 2xx response. + """ + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + try: + response = requests.delete(f'{base}/{entity_path}/faults/{code}', timeout=timeout) + except requests.exceptions.RequestException: + return False + return 200 <= response.status_code < 300 + + +def _get_lifecycle_label(client_node, service_name, timeout=10.0): + """One ``GetState`` call against `service_name`. Returns the state label, or ``None``. + + Creates its own client rather than taking one, since this is also called standalone + (the end-of-window check in "deactivated_not_dead") where polling's own reused client + would be the wrong lifetime to share. + """ + client = client_node.create_client(GetState, service_name) + try: + if not client.wait_for_service(timeout_sec=timeout): + return None + future = client.call_async(GetState.Request()) + rclpy.spin_until_future_complete(client_node, future, timeout_sec=timeout) + result = future.result() + return None if result is None else result.current_state.label + finally: + client_node.destroy_client(client) + + +def _poll_lifecycle_label(client_node, service_name, expected_label, timeout=30.0, interval=0.5): + """Poll ``GetState`` until `service_name` answers `expected_label`. ``True`` once it does. + + One client for the whole poll (not one per iteration, unlike a single + `_get_lifecycle_label` call) - a 30 s poll at the default 0.5 s interval is up to 60 + iterations, and creating and destroying a client every single one of them is needless + churn for what is otherwise an identical repeated call. + """ + client = client_node.create_client(GetState, service_name) + try: + deadline = time.monotonic() + timeout + last_seen = f'{service_name} was never answered at all' + while time.monotonic() < deadline: + if client.wait_for_service(timeout_sec=interval): + future = client.call_async(GetState.Request()) + rclpy.spin_until_future_complete(client_node, future, timeout_sec=interval) + result = future.result() + if result is not None: + last_seen = result.current_state.label + if last_seen == expected_label: + return True + time.sleep(interval) + print(f'_poll_lifecycle_label({service_name!r}, expected={expected_label!r}) timed ' + f'out after {timeout}s; last seen: {last_seen!r}') + return False + finally: + client_node.destroy_client(client) + + +def _call_change_state_once(client_node, service_name, transition_id, timeout=30.0): + """One ``ChangeState`` call against `service_name`. ``True`` on ``result.success``.""" + client = client_node.create_client(ChangeState, service_name) + if not client.wait_for_service(timeout_sec=timeout): + return False + request = ChangeState.Request() + request.transition.id = transition_id + future = client.call_async(request) + rclpy.spin_until_future_complete(client_node, future, timeout_sec=timeout) + result = future.result() + return result is not None and result.success + + +def _poll_stable_tracked_count(port, detector_id, timeout, stable_seconds=10.0, interval=1.0): + """Poll the detector's status block until ``tracked_count`` holds steady. + + "Holds steady" means the SAME value for a continuous `stable_seconds` before `timeout` + elapses. A single sample says nothing about whether the graph has settled: node_death is + zero-config and tracks every armed App, not just this scenario's own three renamed + fixtures. With `discovery.runtime.filter_internal_nodes` off (this scenario's own + requirement - see the class docstring), that includes normally-hidden gateway-internal + nodes too: confirmed live, the gateway keeps its own hidden `_param_client_node` for + querying newly-discovered nodes' parameters (the same path that logs "Parameter service + not available for node: ..." for each renamed fixture below), and that node is present + from early on but can still be inside its OWN per-entity warmup - and so absent from + tracked_count - at the moment a single sample happens to land, only to arm and join the + tracked set a few seconds later. A single `before` sample can therefore catch it + mid-warmup while a single `after` sample catches it already armed, reading as + tracked_count GROWTH that has nothing to do with the ros2cli exclusion this row is + actually about. `stable_seconds` has to be long enough to let that settle BEFORE either + sample is trusted, not merely long enough to smooth over network jitter - a handful of + consecutive reads close together would still land inside the SAME few-second warmup + window and call it "stable" too early. + + Returns ``(status, stable)`` - `status` is the LAST status block read (possibly still + unsettled, so a caller can still report what it saw), `stable` is False if the value + never held for a continuous `stable_seconds` inside `timeout`. A status that reads None + on every poll (no `detectors.node_death` block at all - meaning no such detector is + registered) counts as stable at None: the caller's existing ``if before is not None`` + gate still skips the comparison the same way a single-sample None always did. + """ + deadline = time.monotonic() + timeout + status = None + last_count = None + streak_start = None + while time.monotonic() < deadline: + now = time.monotonic() + status = watchdog_detector_status(port, detector_id) + count = status.get('tracked_count') if status is not None else None + if count != last_count: + last_count = count + streak_start = now + elif now - streak_start >= stable_seconds: + return status, True + time.sleep(interval) + return status, False + + +# --------------------------------------------------------------------------------------- +# Scenarios +# --------------------------------------------------------------------------------------- + +class TestNodeDeathRaise(unittest.TestCase): + """A plain node's process exits: GRAPH_NODE_DISAPPEARED names it.""" + + def test_process_exit_raises_naming_the_node(self, target_node): + # app_id=TARGET_NODE, not the global gate: the global gate is satisfied by ANY + # entity being armed, so it can go true from something else in the graph while + # TARGET_NODE itself has not been read even once. Killing it before that read + # happens is a kill a correct detector was never permitted to see - the raise + # below would then time out for a precondition reason and read exactly like the + # detector-missing red this suite is supposed to produce. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC, app_id=TARGET_NODE), + f'graph_watchdog never reported {TARGET_NODE} armed - the plugin did not load, ' + f'{TARGET_NODE} was never discovered, or the bringup grace never elapsed, so ' + 'no raise below could mean anything', + ) + + os.kill(target_node.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, TARGET_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'{TARGET_NODE} never left GET /apps after SIGTERM - this scenario never ' + 'produced the departure it is named for', + ) + + fault = poll_faults(PORT, FAULT_CODE, timeout=RAISE_TIMEOUT_SEC) + if fault is None: + self.fail(f'{FAULT_CODE} never raised after {TARGET_NODE} exited') + self.assertIn(TARGET_NODE, fault.get('description', '')) + + # The flat /faults list carries a fault whatever its source is; only the + # entity-scoped surface proves an operator can OPEN it somewhere. + self.assertIsNotNone( + poll_entity_faults( + PORT, SOURCE_ENTITY_PATH, FAULT_CODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'{FAULT_CODE} is not reachable at /{SOURCE_ENTITY_PATH}/faults - the entity ' + 'the plugin publishes does not own the fault it raises', + ) + + def test_self_tests_catch_a_dead_fault_surface(self): + """The tests of the tests for both new harness helpers, run once per this file. + + Self-contained (a local HTTP server stands in for the gateway) - runs alongside + the real assertions above without depending on them, the way the lifecycle e2e's + "default_config" scenario runs the silence one. + """ + prove_persistence_proof_catches_a_dead_fault_surface(self) + prove_describes_only_proof_catches_a_dead_fault_surface(self) + + +class TestNodeDeathClearOnReturn(unittest.TestCase): + """A dead node coming back clears the fault. + + The occurrence model's own "outage genuinely ended" case. + """ + + def test_node_returning_clears_the_fault(self, target_node): + # app_id=TARGET_NODE: see TestNodeDeathRaise's identical gate for why the global + # gate alone is not enough before a kill. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC, app_id=TARGET_NODE), + f'graph_watchdog never reported {TARGET_NODE} armed') + + old_pid = target_node.process_details['pid'] + os.kill(old_pid, signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, TARGET_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'{TARGET_NODE} never left GET /apps after SIGTERM') + + fault = poll_faults(PORT, FAULT_CODE, timeout=RAISE_TIMEOUT_SEC) + self.assertIsNotNone(fault, f'{FAULT_CODE} never raised after {TARGET_NODE} exited') + + # launch's own respawn (this scenario's target node is launched with respawn=True) + # brings the same node back under the same name. + self.assertTrue( + _poll_apps_present( + PORT, TARGET_NODE, timeout=DEPARTURE_TIMEOUT_SEC + RESPAWN_DELAY_SEC), + f'{TARGET_NODE} never came back after the SIGTERM - launch did not respawn it, ' + 'so there is nothing here that could clear the fault by returning', + ) + + self.assertTrue( + poll_cleared(PORT, FAULT_CODE, timeout=CLEAR_TIMEOUT_SEC), + f'{FAULT_CODE} did not clear after {TARGET_NODE} came back - the outage ended ' + 'but the fault manager still reports it active', + ) + + +class TestNodeDeathNoHealStandalone(unittest.TestCase): + """The same drive as clear_on_return, with the fault_manager's healing turned OFF. + + The fault must NOT clear - the documented shape of the debounce hysteresis latch + (compute_debounce_status: a CONFIRMED fault stays put unless healing is enabled and the + counter reaches the healing threshold), not a defect. Proven as a SUSTAINED claim over + a window via assert_fault_persists_throughout, not a single sample that could just be + "not healed yet". + """ + + def test_node_returning_does_not_clear_the_fault(self, target_node): + # app_id=TARGET_NODE: see TestNodeDeathRaise's identical gate for why the global + # gate alone is not enough before a kill. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC, app_id=TARGET_NODE), + f'graph_watchdog never reported {TARGET_NODE} armed') + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=FAULTS_LIVE_TIMEOUT_SEC), + 'GET /faults never answered 200 - a persistence claim below would prove ' + 'nothing about the detector if the channel itself never came up', + ) + + os.kill(target_node.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, TARGET_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'{TARGET_NODE} never left GET /apps after SIGTERM') + + fault = poll_faults(PORT, FAULT_CODE, timeout=RAISE_TIMEOUT_SEC) + self.assertIsNotNone(fault, f'{FAULT_CODE} never raised after {TARGET_NODE} exited') + + self.assertTrue( + _poll_apps_present( + PORT, TARGET_NODE, timeout=DEPARTURE_TIMEOUT_SEC + RESPAWN_DELAY_SEC), + f'{TARGET_NODE} never came back after the SIGTERM', + ) + + assert_fault_persists_throughout(self, PORT, FAULT_CODE, SUSTAINED_WINDOW_SEC) + + +class TestNodeDeathDeactivatedNotDead(unittest.TestCase): + """A managed node that DEACTIVATES but keeps running is never called dead. + + Absence alone cannot distinguish a detector that correctly ignores lifecycle state + from one that raises nothing at all - both leave GRAPH_NODE_DISAPPEARED silent here. + What this row catches is a detector that conflates "not active" with "gone": + node_death tracks graph PRESENCE only, so a node that deactivates without leaving the + graph must never be named. + + Uses managed_lifecycle_active (self-activates via launch's own auto_activate + parameter) rather than extending droppable_lifecycle_node.cpp: that fixture's own + ChangeState handler is a documented stub ("Never driven: only + find_lifecycle_get_state_path()'s type check needs it to exist") that always returns + success without changing state, so it cannot be driven through a REAL + active-then-deactivate transition at all - it can only ever answer a label FIXED at + launch. managed_lifecycle is a real rclcpp_lifecycle::LifecycleNode and already + supports genuine ChangeState transitions (see test_lifecycle_expectation_e2e.test.py's + "main" scenario), so it needs no fixture changes for this scenario. + """ + + @classmethod + def setUpClass(cls): + rclpy.init() + cls._client_node = Node('node_death_e2e_deactivate_client') + + @classmethod + def tearDownClass(cls): + cls._client_node.destroy_node() + rclpy.shutdown() + + def test_deactivate_while_alive_never_raises(self): + get_state_service = '/managed_lifecycle_active/get_state' + change_state_service = '/managed_lifecycle_active/change_state' + + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC), + 'graph_watchdog never reported an armed global state') + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=FAULTS_LIVE_TIMEOUT_SEC), + 'GET /faults never answered 200 - the absence below would prove nothing') + + self.assertTrue( + _poll_lifecycle_label( + type(self)._client_node, get_state_service, 'active', + timeout=PRESENCE_TIMEOUT_SEC), + 'managed_lifecycle_active never reached "active" on its own - the trigger ' + 'this scenario needs (an active node that then deactivates) was never set up', + ) + + self.assertTrue( + _call_change_state_once( + type(self)._client_node, change_state_service, + Transition.TRANSITION_DEACTIVATE, timeout=PRESENCE_TIMEOUT_SEC), + 'the real DEACTIVATE transition was rejected or never answered', + ) + self.assertTrue( + _poll_lifecycle_label( + type(self)._client_node, get_state_service, 'inactive', + timeout=PRESENCE_TIMEOUT_SEC), + 'managed_lifecycle_active never reached "inactive" after DEACTIVATE', + ) + + assert_fault_absent_throughout(self, PORT, FAULT_CODE, SUSTAINED_WINDOW_SEC) + + # The trigger was pinned at the START of the window; confirm it survived to the + # end too, or most of the window measured an empty graph instead. + self.assertEqual( + _get_lifecycle_label(type(self)._client_node, get_state_service, timeout=15.0), + 'inactive', + 'managed_lifecycle_active is no longer reading "inactive" at the END of the ' + 'silence window - the trigger did not survive it', + ) + + +class TestNodeDeathManifestNeverOnline(unittest.TestCase): + """A manifest-declared App that never comes online is never called dead. + + Absence alone cannot distinguish a detector that correctly reads the online flag from + one that raises nothing at all - see TestNodeDeathDeactivatedNotDead's own note. What + it catches: a manifest node keeps its App in the snapshot with the online flag + cleared, so a detector that counted snapshot membership alone would call it immortal; + node_death instead arms only apps it has read online at least once. + """ + + NEVER_ONLINE_APP_ID = 'lidar-sensor' # declared in demo_nodes_manifest.yaml; its + # ros_binding (node 'lidar_sensor') is never launched by this scenario. + # The manifest's own id for the app bound to the 'calibration' node. In hybrid mode a + # linked App is exposed under the manifest's declared id, not the runtime-derived bare + # node name - unlike the other scenarios in this file, which run runtime_only. + ONLINE_APP_ID = 'engine-calibration-service' + + def test_never_online_manifest_app_never_raises(self): + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC), + 'graph_watchdog never reported an armed global state - if the manifest failed ' + 'to load (see discovery.manifest_strict_validation) this could be silently ' + 'measuring runtime_only instead of the hybrid launch this scenario needs', + ) + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=FAULTS_LIVE_TIMEOUT_SEC), + 'GET /faults never answered 200 - the absence below would prove nothing') + self.assertTrue( + _poll_apps_present(PORT, self.ONLINE_APP_ID, timeout=PRESENCE_TIMEOUT_SEC), + f'{self.ONLINE_APP_ID} never appeared on GET /apps - without at least one ' + 'online app this launch gives the arming gate nothing to report on', + ) + self.assertTrue( + _poll_apps_present(PORT, self.NEVER_ONLINE_APP_ID, timeout=PRESENCE_TIMEOUT_SEC), + f'{self.NEVER_ONLINE_APP_ID} is not even in the manifest-derived snapshot - ' + 'this scenario needs it present-but-offline, not absent from discovery ' + 'entirely, or the trigger it is named for was never set up', + ) + + assert_fault_absent_throughout(self, PORT, FAULT_CODE, SUSTAINED_WINDOW_SEC) + + +class TestNodeDeathRos2cliIgnored(unittest.TestCase): + """A node named like ros2cli's own hidden-node convention is never tracked or raised. + + What node_death actually implements for this case is a NAME test: it takes the leaf + after the last ``/`` and checks whether it starts with + ``_ros2cli_``. Nothing in that check knows or cares that a real `ros2` CLI process + produced the node - so the fixture that measures it should not depend on one either. + + An earlier version of this scenario drove a real ``ros2 topic echo`` process for + every cycle and polled for its ephemeral ``_ros2cli_`` node. That instrument was + wrong for what it was measuring: it put the ros2cli daemon (`NodeStrategy`'s + daemon-vs-direct-node dispatch), ros2cli's own entry-point process layout, and + ephemeral-process DDS discovery timing all between the test and the one branch this + scenario is actually about - three sources of failure that have nothing to do with + the claim. Verified live, repeatedly: cycles failed in different, unrelated-looking + ways across separate runs (a wrong default spin-time, a bare-vs-FQN mismatch, then - + even after both of those were fixed and the mechanism rebuilt around a persistent + rclpy client node - a specific cycle timing out waiting for presence, and which cycle + failed was not even consistent between runs). That instability was in the instrument, + not in anything this suite is trying to prove. + + test_01 (load-bearing) replaces it: an ordinary, long-lived `demo_engine_temp_sensor` + process, renamed via ``-r __node:={ROS2CLI_FAKE_NODE_PREFIX}`` so the ONLY + thing distinguishing it from any other target node in this package is its name. Being + long-lived and entirely under this test's own control (started and killed on + purpose, not exiting on its own schedule), it can be armed and gated exactly like + every other scenario's target - something an ephemeral CLI subprocess's short, racy + lifetime never allowed. Three cycles, three distinct names, each proven present and + armed BEFORE being killed, so a kill never lands on a node the gate never saw. + + This scenario's gateway launches with `discovery.runtime.filter_internal_nodes` + explicitly OFF (see `generate_test_description`). Left at its true default (on), a + ``_ros2cli_``-named node would be invisible upstream of everything - GET /apps, the + entity cache, any `IntrospectionProvider` plugin's own input - which would make this + scenario pass for the exact wrong reason: never tracked at all, rather than tracked + and then correctly excluded by name. This scenario is about the detector's OWN + name-based exclusion (belt-and-braces on top of, + not a substitute for, the gateway's separate and already-covered generic filter), and + that requires a node the detector's input can actually see. + + test_02 does not drive a process at all. Two earlier attempts to keep a live realism + check both failed to hold up on this stack: a per-cycle ``ros2 topic echo`` (one real + CLI process per cycle) failed in different, unrelated-looking ways across separate + runs (see above), and a single, demoted invocation with a 60 s budget - this method's + own prior version - still did not observe its ephemeral node appear in the ROS graph + within that budget. What test_01's fixture actually needs to stay honest is not proof + that a live CLI process exists, but proof that ROS2CLI_FAKE_NODE_PREFIX is still built + from the same prefix the installed `ros2cli` package would use - a plain constant + comparison against `ros2cli.node.NODE_NAME_PREFIX`, deterministic and immediate, that + fails loudly the day the convention changes instead of quietly testing a prefix + nothing real uses anymore. + + The tracked-count half of test_01's claim compares two STABLE reads + (_poll_stable_tracked_count), not two single samples: node_death is zero-config and + tracks every armed App in the graph, not just this scenario's own three renamed + fixtures, and with `discovery.runtime.filter_internal_nodes` off (this scenario's own + requirement, see above) that includes gateway-internal hidden nodes too. Confirmed live: + the gateway keeps a hidden `_param_client_node` for querying newly-discovered nodes' + parameters - present from early on, but still inside its OWN per-entity warmup, and so + briefly absent from tracked_count, at the moment a bare single sample happens to land. A + `before`/`after` pair of single samples straddling that node's own warmup completion + would read as tracked_count growth that has nothing to do with the ros2cli exclusion + this row is actually about - `_poll_stable_tracked_count` requires the value to hold for + a continuous stable_seconds specifically so that kind of late arrival is waited out + before either sample is trusted, not merely smoothed over. If + `watchdog_detector_status(PORT, DETECTOR_ID)` returns None on every read - no + `detectors.node_death` block at all, meaning no such detector is registered - that + counts as "stable at None" and is the one condition allowed to skip the comparison + entirely; once a detector exists the block stops being None and a subsequent + disappearance, a malformed shape, or a tracked_count that never actually settles fails + loudly instead of being silently discarded (see test_01's own comments). + """ + + def _spawn_renamed_node(self, name): + """Start an ordinary demo node renamed to carry the ros2cli hidden-node prefix. + + `demo_engine_temp_sensor` (`DEMO_NODE_REGISTRY`'s 'temp_sensor' executable) needs + no parameters to start; ``-r __node:={name}`` is a plain ROS 2 remap, not a + ros2cli mechanism. + + Invokes the installed binary directly - NOT ``ros2 run`` - so the returned + `Popen`'s own pid IS the node's pid. `ros2run.api.run_executable` + (confirmed directly against the installed `ros2run` package, not assumed) spawns + the target executable as ITS OWN child via a second `subprocess.Popen`, and only + ever forwards `KeyboardInterrupt` (SIGINT) to it; nothing in that function + forwards SIGTERM. A SIGTERM sent to a `ros2 run` wrapper's own pid therefore + kills only the wrapper - by Python's default disposition, not any handler ros2run + installs - and orphans the real node underneath it, which is then never signaled + at all and never leaves the graph. Reproduced live: an earlier version of this + method went through `ros2 run` and left an actual `demo_engine_temp_sensor` + process running with ppid 1 (reparented to init) for over an hour after its + scenario's own SIGTERM. The process this spawns instead knows nothing about + ros2cli or ros2run - it lives exactly as long as the caller lets it, and a + SIGTERM reaches it directly, the same way every other scenario in this file kills + ITS target node. + """ + exe = os.path.join( + get_package_prefix('ros2_medkit_integration_tests'), 'lib', + 'ros2_medkit_integration_tests', 'demo_engine_temp_sensor') + return subprocess.Popen( + [exe, '--ros-args', '-r', f'__node:={name}'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + def test_01_renamed_node_matching_the_convention_is_never_tracked(self): + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC), + 'graph_watchdog never reported an armed global state') + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=FAULTS_LIVE_TIMEOUT_SEC), + 'GET /faults never answered 200 - the absence below would prove nothing') + + before, before_stable = _poll_stable_tracked_count( + PORT, DETECTOR_ID, timeout=STABLE_TRACKED_COUNT_TIMEOUT_SEC) + self.assertTrue( + before_stable or before is None, + f'tracked_count never settled before the renamed-node cycles even started (last ' + f'read: {before!r}) - the before/after comparison below would measure a moving ' + "target, not this row's own claim", + ) + for cycle in range(1, ROS2CLI_CYCLES + 1): + name = f'{ROS2CLI_FAKE_NODE_PREFIX}{cycle}' + proc = self._spawn_renamed_node(name) + try: + self.assertTrue( + _poll_apps_present(PORT, name, timeout=PRESENCE_TIMEOUT_SEC), + f'cycle {cycle}: {name} never appeared on GET /apps even with ' + 'discovery.runtime.filter_internal_nodes off for this scenario - ' + 'the arm/kill sequence below would prove nothing about the ' + 'detector, only that this node failed to start or be discovered', + ) + # app_id=name, not the global gate: see TestNodeDeathRaise's identical + # gate for why the global gate is not enough - it can go true from + # something else in the graph while this cycle's own node has not been + # read even once. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC, app_id=name), + f'cycle {cycle}: graph_watchdog never reported {name} armed - ' + 'killing it before that would be a kill a correct detector was ' + 'never permitted to see', + ) + + os.kill(proc.pid, signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, name, timeout=DEPARTURE_TIMEOUT_SEC), + f'cycle {cycle}: {name} never left GET /apps after SIGTERM - this ' + 'cycle never produced the departure it is about', + ) + proc.wait(timeout=15) + finally: + if proc.poll() is None: + proc.kill() + proc.wait(timeout=15) + after, after_stable = _poll_stable_tracked_count( + PORT, DETECTOR_ID, timeout=STABLE_TRACKED_COUNT_TIMEOUT_SEC) + # `before` reads None only if GET /x-medkit-watchdog carries no `detectors.node_death` + # block at all - the ONLY condition allowed to skip the comparison, kept for + # structural symmetry with `_poll_stable_tracked_count`'s own "stable at None" + # contract, not because it is expected here. With the detector registered, `before` + # is populated and the comparison below always runs: checking `before is not None` + # alone (not also `after is not None`) is what makes a status block that vanishes or + # loses its shape MID-scenario fail loudly instead of being discarded along with the + # genuinely-inapplicable case. + if before is not None: + self.assertTrue( + after_stable, + f'tracked_count never settled after the renamed-node cycles (last read: ' + f'{after!r}) - cannot tell whether the ros2cli exclusion held or something ' + 'else in the graph is simply still arming, so reporting this rather than ' + 'comparing against a moving target', + ) + self.assertIsNotNone( + after, + f'the node_death detector status block was present before the ' + f'renamed-node cycles ({before!r}) and is gone after them - it ' + 'disappeared mid-scenario', + ) + self.assertIn( + 'tracked_count', after, + f"the node_death detector status block lost its 'tracked_count' field " + f'after the renamed-node cycles ({before!r} -> {after!r})', + ) + if before.get('tracked_count') != after.get('tracked_count'): + try: + apps_url = f'http://127.0.0.1:{PORT}{API_BASE_PATH}/apps' + apps_now = requests.get(apps_url, timeout=5).json().get('items', []) + ids_now = [item.get('id') for item in apps_now] + except requests.exceptions.RequestException as exc: + ids_now = f'GET /apps failed: {exc}' + self.fail( + f'the node_death detector status block reports a different tracked_count ' + f'after {ROS2CLI_CYCLES} renamed-node arm/kill cycles ({before!r} -> ' + f'{after!r}) - GET /apps right now: {ids_now!r}', + ) + + assert_fault_absent_throughout(self, PORT, FAULT_CODE, SUSTAINED_WINDOW_SEC) + + def test_02_the_fake_prefix_still_matches_ros2clis_own_convention(self): + """Pins test_01's fixture to ros2cli's OWN naming convention, not a copy of it. + + What this protects against: test_01 kills nodes it names itself + (``ROS2CLI_FAKE_NODE_PREFIX`` + a cycle number). If the installed `ros2cli` + package ever changed what prefix it hands its own hidden nodes, test_01 would + keep passing - proving a detector ignores a prefix WE made up - while the actual + convention moved out from under it, and `node_death`'s exclusion would silently + stop matching reality. Nothing else in this scenario would notice that drift. + + Reads the installed package rather than driving a process because a live check + was tried first, twice, and neither held up on this stack: a per-cycle ``ros2 + topic echo`` (one real CLI process per cycle, three cycles) failed in different, + unrelated-looking ways across separate runs (a wrong default spin-time, a + bare-vs-FQN mismatch, then - even after both were fixed - a specific cycle timing + out waiting for presence, inconsistently between runs); a single, demoted + invocation with a 60 s budget (this method's own prior version) still did not + observe its ephemeral node appear in the ROS graph within that budget. Both tries + were proving a live process exists, which this scenario does not actually need - + it needs to know what NAME ros2cli would give that process, and that is a plain + constant lookup, not a fact about a live graph. Deterministic, no subprocess, no + discovery, and it fails loudly the day the convention changes - exactly what the + two process-driving attempts before it were for, and could not reliably do here. + + `rclpy.node.HIDDEN_NODE_PREFIX` is the general ROS 2 hidden-node convention + (``'_'``); `ros2cli.node.NODE_NAME_PREFIX` is ros2cli's own + ``HIDDEN_NODE_PREFIX + 'ros2cli'`` - both imported from the installed packages, + not hardcoded here, so this check tracks them automatically. See also + `ros2cli.node.direct.DirectNode`, which builds an actual CLI invocation's node + name as ``NODE_NAME_PREFIX + '_%d' % os.getpid()`` - the shape both live attempts + above were trying, and failing, to observe. + """ + self.assertEqual( + NODE_NAME_PREFIX, HIDDEN_NODE_PREFIX + 'ros2cli', + f'ros2cli.node.NODE_NAME_PREFIX ({NODE_NAME_PREFIX!r}) is no longer built ' + f"from rclpy's own hidden-node prefix ({HIDDEN_NODE_PREFIX!r}) the way this " + 'test assumed - the composition itself changed, not just the value', + ) + self.assertTrue( + ROS2CLI_FAKE_NODE_PREFIX.startswith(NODE_NAME_PREFIX), + f'test_01 kills nodes named {ROS2CLI_FAKE_NODE_PREFIX!r}, but the ' + f'installed ros2cli package now builds its own hidden node names from ' + f'{NODE_NAME_PREFIX!r} - these no longer match, so test_01 passing would no ' + 'longer mean what this scenario claims it means', + ) + + +class TestNodeDeathBareNameCollision(unittest.TestCase): + """Two nodes sharing a bare name in different namespaces; one exits. + + The fault names the one that exited and does not name the one still running - proven + with assert_fault_describes_only so the claim holds for the WHOLE window a scenario + watches it, not one lucky read taken before a second tick could have widened (or + corrupted) the description. + + Required/forbidden are the two nodes' FULLY QUALIFIED names, not their discovery-layer + App ids. The collision-avoidance rule that gives the two colliding nodes their + namespace-prefixed ids (`ros2_runtime_introspection.cpp`) is re-evaluated on every + discovery sweep from whichever bare names are CURRENTLY duplicated - it is not a + property assigned once and kept. Once node_a departs, only one 'calibration' remains, + the collision that justified prefixing it resolves, and node_b's own id reverts to the + bare 'calibration' - confirmed live: GET /apps read `coll_a_calibration` and + `coll_b_calibration` while both were up, then just `calibration` once node_a was gone. + A forbidden needle keyed to an id that stops existing the moment the departure this + scenario is about actually happens would not discriminate anything. The FQN carries no + such collision-avoidance and stays valid for both nodes throughout. + """ + + def test_dead_one_named_alive_one_not(self, node_a, node_b): + # Both ids, not the global gate: this scenario perturbs (kills) node_a and its + # claim is about node_b too ("does not name the one still running"), so both must + # have been read by the detector before node_a dies - see TestNodeDeathRaise's + # identical gate for why the global gate alone would not prove that. + for app_id in (COLLISION_APP_ID_A, COLLISION_APP_ID_B): + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC, app_id=app_id), + f'graph_watchdog never reported {app_id} armed - the discovery layer did ' + 'not namespace-disambiguate the bare-name collision the way this scenario ' + 'assumes, or one of the two fixtures never came up, or it did but was ' + 'never read', + ) + + node_b_pid = node_b.process_details['pid'] + os.kill(node_a.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, COLLISION_APP_ID_A, timeout=DEPARTURE_TIMEOUT_SEC), + f'{COLLISION_APP_ID_A} never left GET /apps after SIGTERM') + # node_b's OWN discovery-layer id is not stable across node_a's departure (see the + # class docstring), so its continued survival is checked at the OS level instead - + # the one identity that does not shift when the collision this scenario sets up + # resolves. + try: + os.kill(node_b_pid, 0) + except ProcessLookupError: + self.fail( + f'node_b (pid {node_b_pid}) is gone too - only node_a was meant to die, ' + 'so the pair no longer discriminates') + + fault = poll_faults(PORT, FAULT_CODE, timeout=RAISE_TIMEOUT_SEC) + self.assertIsNotNone(fault, f'{FAULT_CODE} never raised after {COLLISION_APP_ID_A} exited') + + assert_fault_describes_only( + self, PORT, FAULT_CODE, + required=[f'{COLLISION_NAMESPACE_A}/{TARGET_NODE}'], + forbidden=[f'{COLLISION_NAMESPACE_B}/{TARGET_NODE}'], + duration=MUTUAL_NAMING_WINDOW_SEC) + + +class TestNodeDeathFastTickFloor(unittest.TestCase): + """A very short tick_interval_ms, with the node continuously present, raises nothing. + + Narrower than this row's own name suggests, and deliberately so - see below. Absence + alone cannot distinguish a detector that is correctly silent here from one that raises + nothing at all - see TestNodeDeathDeactivatedNotDead's own note. + + What this scenario does NOT prove: that the documented miss_grace floor (config sweep + C1's own concern) is what stands between a fast tick and a false raise. The row exists + because a fast-ticking detector could read the SAME stale entity-cache generation many + times before the cache refreshes, and - absent the floor - a naive per-tick counter + would count each of those reads as an independent miss rather than one. This scenario + never forces that condition: the node is present throughout and the cache is never + caught between a real departure and a real return, so there is no stale generation to + misread in the first place. A detector built WITHOUT the floor would pass this + scenario just as easily as one built with it, as long as no such staleness happened to + occur in the window - which, with nothing perturbing the graph, it never does. + + Investigated, not assumed, whether the precondition can be forced deterministically. + Traced the real mechanism in `gateway_node.cpp`: the entity cache refreshes on a ROS + graph event, coalesced to at most one refresh per `discovery.refresh_debounce_ms` + (default 1000 ms) via `graph_check_timer_`/`decide_graph_refresh` - PLUS an + independent, unconditional `backstop_timer_` on its own `refresh_interval_ms` cadence, + which the shared `create_gateway_node()` factory pins to 1000 ms for every launch in + this test suite. Both are overridable via `extra_gateway_params`, which in principle + opens a multi-second window: kill the node, poll GET /apps until the departure is + reflected (proving a refresh already consumed the debounce budget), then respawn it + by hand (not launch's own `respawn=True`, whose `RESPAWN_DELAY_SEC` floor is + calibrated to be slow elsewhere in this file and is the wrong direction here) fast + enough to land inside the same now-widened window, and confirm the new process is + genuinely alive via a DIRECT service call that bypasses the gateway entirely while + GET /apps still reports it absent. + + Not implemented. Two reasons: (1) it is a real race, not a guaranteed sequence - + "deterministic" has to mean reliable across CI hardware, and nothing pins how long DDS + discovery takes to converge on the manually-respawned process relative to the widened + window, only that it is likely to fit; (2) winning the race would require new + subprocess-lifecycle code (resolving the demo executable's install path, building its + `--ros-args` remapping by hand, coverage-env parity with every other fixture in this + file, guaranteed cleanup on a failed assertion) whose own bugs could leak an orphaned + process - exactly what this package's own test discipline warns leaks into the NEXT run + as a false regression. Forcing the precondition would exercise a real detector rather + than prove something trivially true either way, which is what makes the remaining cost + purely about race reliability and fixture risk, not about whether the result would mean + anything. + """ + + def test_fast_tick_alone_never_raises(self): + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC), + 'graph_watchdog never reported an armed global state') + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=FAULTS_LIVE_TIMEOUT_SEC), + 'GET /faults never answered 200 - the absence below would prove nothing') + self.assertTrue( + _poll_apps_present(PORT, 'calibration', timeout=PRESENCE_TIMEOUT_SEC), + "'calibration' never appeared on GET /apps - there is no present node here " + 'whose non-death this scenario is about', + ) + + assert_fault_absent_throughout(self, PORT, FAULT_CODE, SUSTAINED_WINDOW_SEC) + + self.assertTrue( + _poll_apps_present(PORT, 'calibration', timeout=5.0), + "'calibration' is no longer present at the END of the silence window - the " + 'trigger (a present node, ticked over rapidly) did not survive it', + ) + + +class TestNodeDeathRestartLoopOccurrences(unittest.TestCase): + """A node killed and restarted three times: occurrence_count reaches 3. + + Each cycle: kill, wait for the fault to CONFIRM (a fresh occurrence), wait for the node + to come back, THEN explicitly acknowledge it via DELETE + {SOURCE_ENTITY_PATH}/faults/{code} - the fault manager's own boundary between + occurrences: `~/clear_fault` IS the acknowledge, not a deletion, and a FAILED event + reactivating a CLEARED record is what the fault manager counts as a new occurrence + (`fault_storage.cpp`'s own comment: "a new outage cycle, not a continuation of the one + that just cleared"). A level-triggered heal on return would NOT do this - reconfirming + from HEALED (rather than from CLEARED) does not bump occurrence_count at all + (`fault_storage.cpp`: "a re-report on a still-active fault... the same continuous + occurrence"), which is why this scenario acknowledges explicitly between cycles instead + of waiting for an organic heal. + + Deliberately does NOT assert the "more than one recording" half of this row. The + per-fault rosbag store enforces `fault_code` as UNIQUE (`sqlite_fault_storage.cpp`'s + `store_rosbag_file_locked`: "INSERT OR REPLACE INTO rosbag_files ... (fault_code is + UNIQUE)" - the SAME fault_code can hold at most one recording ROW, structurally, and a + re-confirm deletes the previous bag file from disk). No config key in the fault manager + lifts this cap today: recording more than one rosbag per fault_code needs the storage + schema itself to change. Asserting a recording count here would either assert something + trivially true for the wrong reason (no captures happen at all without a detector) or + something structurally impossible to ever pass - neither is written. + """ + + def test_repeated_kills_reach_matching_occurrence_count(self, target_node): + # app_id=TARGET_NODE: see TestNodeDeathRaise's identical gate for why the global + # gate alone is not enough before a kill. Re-checked before every LATER kill too + # (below), since a respawned instance needs its own read just as much as the first. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC, app_id=TARGET_NODE), + f'graph_watchdog never reported {TARGET_NODE} armed') + + pid = target_node.process_details['pid'] + for cycle in range(1, RESTART_LOOP_OCCURRENCES_TARGET + 1): + os.kill(pid, signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, TARGET_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'cycle {cycle}: {TARGET_NODE} never left GET /apps after SIGTERM', + ) + self.assertTrue( + _poll_occurrence_count(PORT, FAULT_CODE, cycle, timeout=RAISE_TIMEOUT_SEC), + f'cycle {cycle}: {FAULT_CODE} never reached occurrence_count={cycle}', + ) + + if cycle < RESTART_LOOP_OCCURRENCES_TARGET: + # Wait for the RETURN before acknowledging - not merely a style choice: + # clearing while the node is still absent leaves a window in which a live + # detector's own continued FAILED reports (the SAME departure, still being + # measured) would reactivate the just-cleared record too, one cycle early. + # Waiting for the return first means every reactivation this loop measures + # is attributable to the NEXT kill, not a race against this one's tail end. + # app_id-scoped again, not just present: the respawned instance is a NEW + # process the detector has not read yet, and the next iteration is about + # to kill it - the same precondition the FIRST kill above needed. + self.assertTrue( + wait_until_watchdog_armed( + PORT, timeout=DEPARTURE_TIMEOUT_SEC + RESPAWN_DELAY_SEC, + app_id=TARGET_NODE), + f'cycle {cycle}: {TARGET_NODE} never came back armed after the SIGTERM - ' + 'launch did not respawn it, or the respawned instance was never read, ' + 'so this is a single departure, not a loop', + ) + self.assertTrue( + _clear_fault(PORT, SOURCE_ENTITY_PATH, FAULT_CODE), + f'cycle {cycle}: DELETE /{SOURCE_ENTITY_PATH}/faults/{FAULT_CODE} ' + 'did not acknowledge the fault - the next kill could not reactivate ' + 'it as a fresh occurrence', + ) + # A fresh pid every cycle: read it back rather than assume launch's + # respawn keeps the value this test already has. + pid = target_node.process_details['pid'] + + record = _fault_record(PORT, FAULT_CODE, timeout=DEPARTURE_TIMEOUT_SEC) + if record is None: + self.fail(f'{FAULT_CODE} vanished from the store entirely') + self.assertEqual( + record.get('occurrence_count'), RESTART_LOOP_OCCURRENCES_TARGET, + f'{FAULT_CODE} ended the loop with occurrence_count=' + f'{record.get("occurrence_count")!r}, not {RESTART_LOOP_OCCURRENCES_TARGET}', + ) + + +class TestNodeDeathRestartRebaseline(unittest.TestCase): + """A gateway restart with a death outstanding: the re-baseline boundary is pinned. + + RECORDS a boundary, the way TestLifecycleExpectationRestartDeparted does for the + sibling detector - this test does not fix anything. Pin: the fault stays CONFIRMED + across the restart, proven as a SUSTAINED claim (assert_fault_persists_throughout), + not a single sample that could just mean "not yet re-evaluated". + + Reasoning, on file rather than assumed: node_death has no forced state transition - a + violation's evidence closes only when the graph genuinely recovers, or an operator + explicitly acknowledges it. A gateway restart is neither: the node is + STILL gone, and the freshly-started detector process, having never observed it + present in this lifetime, has nothing to report either way. Unlike + lifecycle_expectation's require_active (an explicit, possibly-mistyped list that has + to eventually give up on an entry that never matches anything, or a typo would block + healing forever), node_death is zero-config: it has no static entries to protect + against typos in the first place, so it has no equivalent reason to ever manufacture a + clear for a node it has simply never seen. + """ + + def test_01_the_fault_raises_and_survives_the_node_leaving(self, target_node): + # app_id=TARGET_NODE: see TestNodeDeathRaise's identical gate for why the global + # gate alone is not enough before a kill. test_02 below gates globally instead - + # by then TARGET_NODE is gone BY DESIGN, so an app_id gate on it could never + # succeed; what test_02 checks is that the PLUGIN itself is back up post-restart, + # not that this specific (permanently absent) node is. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC, app_id=TARGET_NODE), + f'graph_watchdog never reported {TARGET_NODE} armed') + + os.kill(target_node.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, TARGET_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'{TARGET_NODE} never left GET /apps after SIGTERM') + + fault = poll_faults(PORT, FAULT_CODE, timeout=RAISE_TIMEOUT_SEC) + self.assertIsNotNone(fault, f'{FAULT_CODE} never raised for the departed {TARGET_NODE}') + + record = _fault_record(PORT, FAULT_CODE, timeout=DEPARTURE_TIMEOUT_SEC) + if record is None: + self.fail(f'{FAULT_CODE} vanished from the store') + self.assertIsNone( + record.get('last_passed'), + f'{FAULT_CODE} was reported PASSED before the restart ' + f'(last_passed={record.get("last_passed")!r}) - there is nothing left here for ' + 'the restart below to wrongly preserve or wrongly heal', + ) + + def test_02_a_gateway_restart_does_not_manufacture_a_clear(self, gateway_node, target_node): + old_pid = gateway_node.process_details['pid'] + os.kill(old_pid, signal.SIGTERM) + self.assertTrue( + _wait_until_port_is_down(PORT, timeout=60.0), + f'the gateway (pid {old_pid}) kept answering after SIGTERM - nothing restarted') + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=90.0), + 'the gateway never came back armed after the restart') + self.assertNotEqual( + gateway_node.process_details['pid'], old_pid, + 'the gateway process id did not change, so this test never restarted anything') + # The armed gate is served by the plugin INSIDE the restarted gateway process and + # proves nothing about its connection to the fault_manager - a SEPARATE process + # that survived the restart, but whose services the freshly restarted gateway has + # to rediscover from scratch. Without this, the persistence window below could + # start against a /faults that has not reconnected yet and fail for that reason + # instead of the boundary this test is actually pinning. + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=FAULTS_LIVE_TIMEOUT_SEC), + 'GET /faults never answered 200 after the restart - the restarted gateway ' + 'never reconnected to the fault_manager, so the persistence window below ' + 'would prove nothing', + ) + self.assertFalse( + _poll_apps_present(PORT, TARGET_NODE, timeout=5.0), + f'{TARGET_NODE} is back in GET /apps - the boundary this test pins is about ' + 'a restart with the node STILL gone, and it came back instead', + ) + + # The pin: sustained across a real window post-restart, not a single sample. + assert_fault_persists_throughout(self, PORT, FAULT_CODE, SUSTAINED_WINDOW_SEC) + + record = _fault_record(PORT, FAULT_CODE, timeout=DEPARTURE_TIMEOUT_SEC) + if record is None: + self.fail(f'{FAULT_CODE} vanished from the store after the restart') + self.assertIsNone( + record.get('last_passed'), + f'the restarted gateway reported {FAULT_CODE} PASSED for a node it can never ' + f'have observed in this process (last_passed={record.get("last_passed")!r}) - ' + 'the boundary this test pins no longer holds', + ) + + +# Each CTest target launches this file with one scenario, so only that scenario's case may +# run. Removing the others from the module (rather than skipping them) means each run +# reports exactly one case, and a missing result is a real failure rather than an expected +# line of output - see test_lifecycle_expectation_e2e.test.py's identical rationale. +_SCENARIO_CASES = { + 'raise': 'TestNodeDeathRaise', + 'clear_on_return': 'TestNodeDeathClearOnReturn', + 'no_heal_standalone': 'TestNodeDeathNoHealStandalone', + 'deactivated_not_dead': 'TestNodeDeathDeactivatedNotDead', + 'manifest_never_online': 'TestNodeDeathManifestNeverOnline', + 'ros2cli_ignored': 'TestNodeDeathRos2cliIgnored', + 'bare_name_collision': 'TestNodeDeathBareNameCollision', + 'fast_tick_floor': 'TestNodeDeathFastTickFloor', + 'restart_loop_occurrences': 'TestNodeDeathRestartLoopOccurrences', + 'restart_rebaseline': 'TestNodeDeathRestartRebaseline', +} +if SCENARIO not in _SCENARIO_CASES: + raise RuntimeError( + f'WATCHDOG_E2E_SCENARIO={SCENARIO!r} is not one of {sorted(_SCENARIO_CASES)}; ' + 'the CTest target and this file disagree about which scenarios exist') +for _scenario, _case_name in _SCENARIO_CASES.items(): + if _scenario != SCENARIO: + del globals()[_case_name] + + +@launch_testing.post_shutdown_test() +class TestShutdown(unittest.TestCase): + """Verify the gateway/fault_manager/demo stack exits cleanly.""" + + def test_exit_codes(self, proc_info): + for info in proc_info: + self.assertIn( + info.returncode, + ALLOWED_EXIT_CODES, + f'Process {info.process_name} exited with {info.returncode}', + ) diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_node_death_suppression_e2e.test.py b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_node_death_suppression_e2e.test.py new file mode 100644 index 000000000..a64ecd555 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/e2e/test_node_death_suppression_e2e.test.py @@ -0,0 +1,788 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""node_death suppression e2e: allowlist, self-suppression and pruning against the REAL stack. + +Sibling of test_node_death_e2e.test.py, same shape: each RAISE row proves the fault actually +appears, naming the missing-fault case as the failure mode when it does not. An ABSENCE row +cannot, by itself, discriminate a correct suppressor from no detector at all, because both +produce the same silence; each such row's own class docstring says so and states what the row +catches instead. + +Runs as FIVE separate CTest targets (see CMakeLists.txt), each launching its OWN +gateway+fault_manager+demo-node stack - the plugin reads its config once at set_context() time, so +different configs need different gateway launches. WATCHDOG_E2E_SCENARIO selects which launch and +which assertions run: + +- "allowlist_suppresses": ``detectors.node_death.allowlist: [TARGET_NODE]`` with + ``suppress: ["allowlist"]`` - the node is armed, then killed. No GRAPH_NODE_DISAPPEARED names + it, for a sustained window - narrower than "the code never appears at all": a correct + detector raising the code for some OTHER entity must leave this row green, so the assertion + is needle-scoped (assert_fault_never_names) rather than a bare absence check. CANNOT + DISCRIMINATE BY ITSELF: silence here is consistent with a suppressor correctly + honouring the allowlist, and equally consistent with one that never consults it, since both + leave the code unraised. +- "allowlist_not_named_is_inert": the SAME allowlist, but ``suppress`` does not name + "allowlist" - naming an entry on the allowlist is not, by itself, a request to suppress it; + only listing the mechanism in ``suppress`` opts it in, unlike the ported source, which + activated a configured allowlist unconditionally. The fault raises and names the node - a + genuine RAISE row - and a startup warning says the list is inert. The warning half is read + straight off the + gateway process's own stderr via launch_testing's ``proc_output`` fixture, the same instrument + test_startup_param_clamp_warnings.test.py already uses for an identical "did we say so" claim - + no new harness helper, just a capability already available at this tier and never exercised in + this package's own e2e suite before. The match itself is a compiled regex, not the bare + substring "allowlist": a line mentioning the word for an unrelated reason (reporting its size, + say) must not satisfy a warning this specific, so the pattern requires "allowlist" and + "suppress" on the SAME line at WARN severity - see _INERT_ALLOWLIST_WARNING_RE. +- "lifecycle_clean_shutdown": self-suppression - a node this package already watches through + lifecycle_expectation is suppressed only when it departed while not active, not a + lifecycle-agnostic notion of "clean". Two identical managed_lifecycle instances, both listed in + ``detectors.lifecycle_expectation.require_active`` (self-suppression needs BOTH "this node is + require_active-owned" and "it departed while not active" - the lifecycle label alone would + over-suppress a node nobody is watching), AND ``detectors.node_death.suppress: ["lifecycle"]`` + naming the self-suppression mechanism this row is about - suppression is opt-in by ruling, so + without that entry a correct detector reports BOTH departed nodes and this row's own "the clean + one is never named" half would be false against a correct implementation. One is driven through + a REAL ``lifecycle_msgs/srv/ChangeState`` UNCONFIGURED_SHUTDOWN transition (reaching + "finalized") before its process is killed; the other reaches "active" (auto_activate) and is + killed outright, still active. The finalized one is never named; the active one is - proven + with assert_fault_describes_only so the claim holds over the whole window, not one lucky read. +- "suppression_is_opt_in": the allowlist key is set, but ``suppress`` is not configured AT ALL - + merely naming a node on the allowlist must not suppress it by itself. The fault raises and + names the node. +- "prune_no_false_heal": TWO independent nodes. TARGET_NODE carries + ``allowlist: [TARGET_NODE]`` with ``suppress: ["allowlist"]`` - the ONLY shape pruning ever + applies to (``NodeLivenessTracker::prune()`` reclaims a key only once it has been DURABLY + suppressed; an unsuppressed death has no reclaim path at all - see node_liveness_tracker.hpp's + own doc). SECOND_NODE carries no suppression at all, so its death is an ordinary, permanently + outstanding fault - the control this row needs, since a single-node shape cannot tell "prune() + reclaimed the right key" from "prune() reclaimed (or healed) something it should not have". + Both are killed; TARGET_NODE's bookkeeping is then observed reclaimed past + ``detectors.node_death.prune_grace`` as a `tracked_count` DELTA of exactly one off a baseline + (never against zero - the aggregate also carries every other armed App in the graph, so it can + never read exactly zero), while SECOND_NODE's fault must stay CONFIRMED and keep naming it, + never TARGET_NODE, for a window spanning straight through the tick TARGET_NODE's own reclaim + happens on. + +Every scenario gates on wait_until_watchdog_armed(PORT, app_id=...) BEFORE asserting anything - +see harness.py's own docstring for why a stack that never came up otherwise produces exactly the +silence an absence claim is looking for. "lifecycle_clean_shutdown" is the one exception: both its +nodes are non-active managed_lifecycle instances, and ReliabilityGate reports a tracked node with +a known non-active label as "warming_up" by design (LifecycleWatcher::node_ok() is false for +exactly the node this whole detector class exists to catch) - so a per-entity armed gate on either +one would wait for something that can never become true. It gates on the GLOBAL armed state +instead, exactly as test_lifecycle_expectation_e2e.test.py's own "main" scenario does for the +identical reason. + +"allowlist_not_named_is_inert" and "lifecycle_clean_shutdown" both configure +``suppress: ["lifecycle"]`` for the second suppression mechanism this port carries (see +lifecycle_shutdown_suppressor.hpp). In "allowlist_not_named_is_inert" it is chosen so the row +is a well-formed "suppress names something, but not allowlist" rather than a malformed-entry +row (C2's own territory); in "lifecycle_clean_shutdown" it is the entry that actually has to be +present for the row's own claim to be testable at all - self-suppression is opt-in like every +other mechanism here, so without naming it a correct detector reports both departed nodes. +""" + +import os +import re +import signal +import sys +import time +import unittest + +from launch.actions import TimerAction +import launch_ros.actions +import launch_testing +from lifecycle_msgs.msg import Transition +from lifecycle_msgs.srv import ChangeState +import rclpy +from rclpy.node import Node +import requests + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +# I100 as well as E402: `harness` is only importable because of the sys.path line above, so this +# import cannot be moved up to where the alphabetical order would put it. +from harness import ( # noqa: E402, I100 + API_BASE_PATH, + assert_fault_describes_only, + assert_fault_never_names, + create_watchdog_test_launch, + poll_detector_status, + poll_faults, + prove_never_names_proof_catches_a_wrongly_scoped_absence, + wait_until_faults_endpoint_live, + wait_until_watchdog_armed, + watchdog_detector_status, +) + +from ros2_medkit_test_utils.constants import ( # noqa: E402 + ALLOWED_EXIT_CODES, + get_test_port, + get_time_scale, +) +from ros2_medkit_test_utils.coverage import get_coverage_env # noqa: E402 + +# No default on purpose - see harness-consuming siblings' identical rationale: a default makes +# this file FAIL OPEN. A KeyError is loud. +SCENARIO = os.environ['WATCHDOG_E2E_SCENARIO'] +PORT = get_test_port() + +FAULT_CODE = 'GRAPH_NODE_DISAPPEARED' +_NODE_DEATH_PREFIX = 'plugins.graph_watchdog.detectors.node_death' +_LIFECYCLE_PREFIX = 'plugins.graph_watchdog.detectors.lifecycle_expectation' + +# Same fast cadence and miss_grace convention test_node_death_e2e.test.py uses: comfortably past +# the documented 3000 ms floor (config sweep C1 owns the floor's own boundary values) without +# accidentally landing on it. +TICK_INTERVAL_MS = 200 +WARMUP_CYCLES = 3 +MISS_GRACE = 20 + +# The plain node every scenario but "lifecycle_clean_shutdown" kills - same DEMO_NODE_REGISTRY +# fixture test_node_death_e2e.test.py uses, built by hand for the same reason: a scenario that +# signals it needs a PID, and create_demo_nodes() gives none back. +TARGET_NODE = 'calibration' +TARGET_EXECUTABLE = 'demo_calibration_service' +TARGET_NAMESPACE = '/powertrain/engine' + +# The budgets below are scaled by MEDKIT_TEST_TIME_SCALE, which the sanitizer jobs set to the +# same factor they apply to every declared CTest timeout. A deadline asserted INSIDE a test is +# invisible to that rewrite, so an instrumented graph that takes longer to forget a departed +# node blows a budget here and the failure reads as a detector that never reported - the exact +# red this suite exists to produce for a real defect. Unset elsewhere, so the normal jobs keep +# the tight budgets that give these assertions their falsifying edge. +# +# Poll intervals, enforced respawn delays and the sustained-observation windows are NOT scaled. +# Those are not give-up bounds: stretching a window a scenario watches for silence buys no +# confidence and spends the whole package's test budget to do it. +TIME_SCALE = get_time_scale() +ARM_TIMEOUT_SEC = 60.0 * TIME_SCALE +FAULTS_LIVE_TIMEOUT_SEC = 30.0 * TIME_SCALE +DEPARTURE_TIMEOUT_SEC = 30.0 * TIME_SCALE +RAISE_TIMEOUT_SEC = 60.0 * TIME_SCALE +# How long an absent or a persisting fault is watched for. Short (well under a minute) because +# every window here is measured from the arming gate, not process start, so bringup cannot eat +# it - matching test_node_death_e2e.test.py's SUSTAINED_WINDOW_SEC. +SUSTAINED_WINDOW_SEC = 20.0 +# How long the "allowlist is inert" startup warning is waited for on the gateway's own stderr. +# Config parsing happens at plugin set_context() time, at process start - well before the demo +# node and fault_manager even come up (create_watchdog_test_launch's own demo_delay) - so this +# only needs to be generous against process-startup jitter, not against anything downstream. +WARNING_TIMEOUT_SEC = 30.0 * TIME_SCALE + +# Matches a single stderr line that names BOTH "allowlist" and "suppress" - the two config +# keys an inert allowlist is about - at WARN severity, not the bare substring "allowlist", +# which a detector could satisfy with an unrelated line ("allowlist contains 1 entry") while +# never claiming anything is inert. Neither token pins node_death's own exact wording, but the +# codebase's own convention (test_startup_param_clamp_warnings.test.py) is to echo a config key +# verbatim in its warning, and an allowlist that suppress does not name must say so - i.e. be +# logged via RCLCPP_WARN, whose default format always stamps the literal "[WARN]" tag. Requiring +# all three on one line, in any order, rules out a routine config-echo (no [WARN]), a bare size +# report (no "suppress"), and an unrelated warning that happens to mention "suppress" for a +# different reason (no "allowlist") - none of which is the inert-allowlist warning this row is +# about. +_INERT_ALLOWLIST_WARNING_RE = re.compile( + r'^(?=.*\[WARN\])(?=.*\ballowlist\b)(?=.*\bsuppress\b).*$', re.IGNORECASE | re.MULTILINE) + +# ---- "lifecycle_clean_shutdown" scenario's own fixtures --------------------------------------- +# Distinct, self-describing names so this scenario's own two nodes are never confused with the +# plain TARGET_NODE above or with anything a shared demo fixture might also be named. +CLEAN_NODE = 'suppress_lifecycle_clean' +KILLED_NODE = 'suppress_lifecycle_active' +LIFECYCLE_EXECUTABLE = 'managed_lifecycle' # DEMO_NODE_REGISTRY's own executable for both variants +LIFECYCLE_GRACE = 5 + +# ---- "prune_no_false_heal" scenario's own prune_grace ------------------------------------------ +# Must be >= miss_grace + 1 (config sweep C1's own clamp) or it is silently raised to that floor. +# Set to exactly the floor so pruning becomes eligible as soon as this scenario's own window can +# show it. +PRUNE_GRACE = MISS_GRACE + 1 + +# ---- "prune_no_false_heal" scenario's own second, UNSUPPRESSED node ---------------------------- +# A second, independent identity carrying no suppression at all, so its death is an ordinary, +# permanently-outstanding fault - the control this row needs to prove that reclaiming +# TARGET_NODE's (suppressed, prune-eligible) bookkeeping never touches it. +SECOND_NODE = 'prune_second_unsuppressed' +SECOND_NAMESPACE = '/powertrain/unsuppressed' +# How long the reclaim-delta baseline is allowed to take to SETTLE (see +# _poll_stable_tracked_count) before either node departs. wait_until_watchdog_armed(app_id=...) +# only proves the gate considers TARGET_NODE/SECOND_NODE armed, not that node_death's own tick +# has already added their keys to tracker_ (see that helper's own docstring) - this is generous +# against that residual lag, not against anything this scenario's own cycle does. Mirrors +# test_node_death_e2e.test.py's identical STABLE_TRACKED_COUNT_TIMEOUT_SEC, same detector, same +# hazard. +STABLE_TRACKED_COUNT_TIMEOUT_SEC = 40.0 * TIME_SCALE + + +def _target_node_action(): + """One manually-constructed TARGET_NODE action, with a PID handle the test can signal. + + Mirrors DEMO_NODE_REGISTRY's own 'calibration' entry exactly, the same way + test_node_death_e2e.test.py's identical helper does - built by hand only because every + scenario in this file needs to SIGTERM it, and create_demo_nodes() hands back no PID. + Every scenario here kills it exactly once and never respawns it, so respawn is not a + parameter. + """ + return launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable=TARGET_EXECUTABLE, + name=TARGET_NODE, + namespace=TARGET_NAMESPACE, + output='screen', + additional_env=get_coverage_env('ros2_medkit_integration_tests'), + sigterm_timeout='30', + sigkill_timeout='15', + ) + + +def _second_node_action(): + """SECOND_NODE: a second, independent, never-allowlisted identity for "prune_no_false_heal". + + Same executable as TARGET_NODE, under its own name/namespace so it is a genuinely distinct + node - built by hand for the same PID-handle reason _target_node_action() is. + """ + return launch_ros.actions.Node( + package='ros2_medkit_integration_tests', + executable=TARGET_EXECUTABLE, + name=SECOND_NODE, + namespace=SECOND_NAMESPACE, + output='screen', + additional_env=get_coverage_env('ros2_medkit_integration_tests'), + sigterm_timeout='30', + sigkill_timeout='15', + ) + + +def _lifecycle_node_action(name, *, auto_activate): + """One managed_lifecycle instance under `name`, optionally self-activating. + + Same executable DEMO_NODE_REGISTRY's 'managed_lifecycle'/'managed_lifecycle_active' entries + use, launched by hand under this scenario's own names (not create_demo_nodes(), which hands + back no PID) so both instances can run side by side without colliding on the bare node name. + """ + node_kwargs = { + 'package': 'ros2_medkit_integration_tests', + 'executable': LIFECYCLE_EXECUTABLE, + 'name': name, + 'namespace': '', + 'output': 'screen', + 'additional_env': get_coverage_env('ros2_medkit_integration_tests'), + 'sigterm_timeout': '30', + 'sigkill_timeout': '15', + } + if auto_activate: + node_kwargs['parameters'] = [{'auto_activate': True}] + return launch_ros.actions.Node(**node_kwargs) + + +def generate_test_description(): + detector_params = { + 'plugins.graph_watchdog.tick_interval_ms': TICK_INTERVAL_MS, + 'plugins.graph_watchdog.warmup_cycles': WARMUP_CYCLES, + } + demo_nodes = [] + + if SCENARIO == 'allowlist_suppresses': + detector_params[f'{_NODE_DEATH_PREFIX}.allowlist'] = [TARGET_NODE] + detector_params[f'{_NODE_DEATH_PREFIX}.suppress'] = ['allowlist'] + elif SCENARIO == 'allowlist_not_named_is_inert': + detector_params[f'{_NODE_DEATH_PREFIX}.allowlist'] = [TARGET_NODE] + # Non-empty and NOT "allowlist" - see the module docstring's own note on why this + # cannot be an empty list (ROS 2 launch crashes on an empty list parameter) and why + # this specific string is a placeholder for the framework's second mechanism rather + # than a malformed entry. + detector_params[f'{_NODE_DEATH_PREFIX}.suppress'] = ['lifecycle'] + elif SCENARIO == 'lifecycle_clean_shutdown': + detector_params[f'{_LIFECYCLE_PREFIX}.require_active'] = [CLEAN_NODE, KILLED_NODE] + detector_params[f'{_LIFECYCLE_PREFIX}.grace'] = LIFECYCLE_GRACE + # Suppression is opt-in by ruling: without naming the self-suppression mechanism here, + # a correct detector reports BOTH departed nodes, and this scenario's own "the clean one + # is never named" half would be false against a correct implementation. + detector_params[f'{_NODE_DEATH_PREFIX}.suppress'] = ['lifecycle'] + elif SCENARIO == 'suppression_is_opt_in': + detector_params[f'{_NODE_DEATH_PREFIX}.allowlist'] = [TARGET_NODE] + # No 'suppress' key at all - the whole point of this scenario. + elif SCENARIO == 'prune_no_false_heal': + detector_params[f'{_NODE_DEATH_PREFIX}.miss_grace'] = MISS_GRACE + detector_params[f'{_NODE_DEATH_PREFIX}.prune_grace'] = PRUNE_GRACE + # Durable suppression is the only path prune() ever reclaims through - see the + # module docstring's own note on why an unsuppressed death cannot exercise it. + detector_params[f'{_NODE_DEATH_PREFIX}.allowlist'] = [TARGET_NODE] + detector_params[f'{_NODE_DEATH_PREFIX}.suppress'] = ['allowlist'] + else: + raise RuntimeError(f'WATCHDOG_E2E_SCENARIO={SCENARIO!r} has no launch configuration') + + launch_description, context = create_watchdog_test_launch( + detector_params=detector_params, + demo_nodes=demo_nodes, + port=PORT, + ) + + if SCENARIO in ('allowlist_suppresses', 'allowlist_not_named_is_inert', + 'suppression_is_opt_in', 'prune_no_false_heal'): + target = _target_node_action() + launch_description.add_action(TimerAction(period=2.0, actions=[target])) + context['target_node'] = target + + if SCENARIO == 'prune_no_false_heal': + second = _second_node_action() + launch_description.add_action(TimerAction(period=2.0, actions=[second])) + context['second_node'] = second + + if SCENARIO == 'lifecycle_clean_shutdown': + clean = _lifecycle_node_action(CLEAN_NODE, auto_activate=False) + killed = _lifecycle_node_action(KILLED_NODE, auto_activate=True) + launch_description.add_action(TimerAction(period=2.0, actions=[clean, killed])) + context['clean_node'] = clean + context['killed_node'] = killed + + return launch_description, context + + +# --------------------------------------------------------------------------------------- +# Local helpers - read the same endpoints harness.py's own helpers do, or a service this +# file's own scenario needs that no shared helper covers. +# --------------------------------------------------------------------------------------- + +def _poll_apps_absent(port, app_id, timeout=30.0, interval=0.5): + """Poll ``GET /apps`` until `app_id` is no longer listed. ``True`` once it is gone.""" + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + timeout + last_seen = 'GET /apps was never answered at all' + while time.monotonic() < deadline: + try: + response = requests.get(f'{base}/apps', timeout=5) + if response.status_code == 200: + ids = [item.get('id') for item in response.json().get('items', [])] + last_seen = str(ids) + if app_id not in ids: + return True + else: + last_seen = f'HTTP {response.status_code} from GET /apps' + except requests.exceptions.RequestException as exc: + last_seen = f'GET /apps failed: {exc}' + time.sleep(interval) + print(f'_poll_apps_absent({app_id!r}) timed out after {timeout}s; last seen: {last_seen}') + return False + + +def _poll_watchdog_entity(port, app_id, lifecycle, timeout=30.0, interval=0.5): + """Poll GET /x-medkit-watchdog until `app_id` appears with this lifecycle label. + + Same instrument, same rationale, as test_lifecycle_expectation_e2e.test.py's identical + local helper: proves the target's label was actually READ by the live stack, which + wait_until_watchdog_armed cannot (an unread label is treated as benign, so a run in which + the label never arrived would look identical to one where it did). '' (empty string) is + "asked, and still waiting", never "no data yet" - LifecycleWatcher seeds a tracked entry's + label to "" and only overwrites it once a real read succeeds. + + Returns the matching entity dict, or ``None`` on timeout (after printing the last-seen + payload, which is gone once the launch tears down). + """ + base = f'http://127.0.0.1:{port}{API_BASE_PATH}' + deadline = time.monotonic() + timeout + last_seen = 'GET /x-medkit-watchdog was never answered at all' + while time.monotonic() < deadline: + try: + response = requests.get(f'{base}/x-medkit-watchdog', timeout=5) + if response.status_code != 200: + last_seen = f'HTTP {response.status_code} from GET /x-medkit-watchdog' + else: + status = response.json().get('x-medkit-watchdog', {}) + last_seen = str(status) + for entity in status.get('entities') or []: + if entity.get('id') == app_id and entity.get('lifecycle') == lifecycle: + return entity + except requests.exceptions.RequestException as exc: + last_seen = f'GET /x-medkit-watchdog failed: {exc}' + time.sleep(interval) + print(f'_poll_watchdog_entity(app_id={app_id!r}, lifecycle={lifecycle!r}) timed out after ' + f'{timeout}s; last watchdog status: {last_seen}') + return None + + +def _call_change_state_once(client_node, service_name, transition_id, timeout=30.0): + """One ``ChangeState`` call against `service_name`. ``True`` on ``result.success``. + + Same shape as test_node_death_e2e.test.py's identical local helper. + """ + client = client_node.create_client(ChangeState, service_name) + try: + if not client.wait_for_service(timeout_sec=timeout): + return False + request = ChangeState.Request() + request.transition.id = transition_id + future = client.call_async(request) + rclpy.spin_until_future_complete(client_node, future, timeout_sec=timeout) + result = future.result() + return result is not None and result.success + finally: + client_node.destroy_client(client) + + +def _poll_stable_tracked_count(port, detector_id, timeout, stable_seconds=10.0, interval=0.5): + """Poll the detector's status block until ``tracked_count`` holds steady. + + "Holds steady" means the SAME value for a continuous `stable_seconds` before `timeout` + elapses. A single sample proves nothing about whether node_death has actually caught up: + wait_until_watchdog_armed(app_id=...) only proves the RELIABILITY GATE considers an entity + armed, not that node_death's own tick has already added its key to tracker_ - see that + helper's own docstring ("What this does NOT prove is that a detector has already READ the + app... Callers that need a captured baseline must still allow the sweep a window after this + returns"). A baseline read immediately after both TARGET_NODE and SECOND_NODE gate-arm can + therefore land before either one (or both) has actually joined tracked_count, understating + the baseline - and since this row's own claim is a tracked_count DELTA off that baseline, an + understated baseline makes the later reclaim assertion fail for a reason that has nothing to + do with prune(). Same helper, same detector, same hazard as + test_node_death_e2e.test.py's identical ``_poll_stable_tracked_count``. + + Returns ``(status, stable)`` - `status` is the LAST status block read (possibly still + unsettled, so a caller can still report what it saw), `stable` is False if the value never + held for a continuous `stable_seconds` inside `timeout`. + """ + deadline = time.monotonic() + timeout + status = None + last_count = None + streak_start = None + while time.monotonic() < deadline: + now = time.monotonic() + status = watchdog_detector_status(port, detector_id) + count = status.get('tracked_count') if status is not None else None + if count != last_count: + last_count = count + streak_start = now + elif now - streak_start >= stable_seconds: + return status, True + time.sleep(interval) + return status, False + + +# --------------------------------------------------------------------------------------- +# Scenarios +# --------------------------------------------------------------------------------------- + +class TestSuppressionAllowlistSuppresses(unittest.TestCase): + """A node named in the allowlist, with suppress opting the allowlist in, is never named. + + The claim is needle-scoped ("no GRAPH_NODE_DISAPPEARED names it"), not "the code never + appears at all": a correct detector raising the code for some OTHER entity in this launch + (the fault_manager's own node, say) must leave this row green, since that has nothing to do + with whether allowlist-suppression works. A bare code-absence check cannot tell the two + apart, so this uses assert_fault_never_names. + + CANNOT DISCRIMINATE BY ITSELF: silence here is consistent with a suppressor correctly + honouring the allowlist, and equally consistent with one that never consults it, since both + leave the code unraised. Recorded as such rather than presented as proof of correct + suppression - ``test_allowlist_suppressor.cpp`` pins the matcher's own discrimination + directly, and ``prune_no_false_heal`` below runs the SAME allowlist against a second, + unlisted node under the identical config to show the two are told apart. + """ + + def test_allowlisted_death_never_raises(self, target_node): + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC, app_id=TARGET_NODE), + f'graph_watchdog never reported {TARGET_NODE} armed - the plugin did not load, ' + f'{TARGET_NODE} was never discovered, or the bringup grace never elapsed, so no ' + 'absence below could mean anything', + ) + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=FAULTS_LIVE_TIMEOUT_SEC), + 'GET /faults never answered 200 - the absence below would prove nothing') + + os.kill(target_node.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, TARGET_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'{TARGET_NODE} never left GET /apps after SIGTERM - this scenario never produced ' + 'the departure it is named for', + ) + + assert_fault_never_names( + self, PORT, FAULT_CODE, forbidden=[TARGET_NODE], duration=SUSTAINED_WINDOW_SEC) + + def test_self_test_catches_a_wrongly_scoped_absence(self): + """The test of the test for assert_fault_never_names, run once per this file. + + Self-contained (a local HTTP server stands in for the gateway) - runs alongside the + real assertion above without depending on it, the same convention + test_node_death_e2e.test.py's own "raise" scenario and + test_lifecycle_expectation_e2e.test.py's "main" scenario already use for their own new + harness helpers. + """ + prove_never_names_proof_catches_a_wrongly_scoped_absence(self) + + +class TestSuppressionAllowlistNotNamedIsInert(unittest.TestCase): + """An allowlist that `suppress` does not name has no effect, and says so.""" + + def test_inert_allowlist_still_raises_and_warns(self, target_node, proc_output, gateway_node): + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC, app_id=TARGET_NODE), + f'graph_watchdog never reported {TARGET_NODE} armed') + + os.kill(target_node.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, TARGET_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'{TARGET_NODE} never left GET /apps after SIGTERM') + + fault = poll_faults(PORT, FAULT_CODE, timeout=RAISE_TIMEOUT_SEC) + if fault is None: + self.fail( + f'{FAULT_CODE} never raised after {TARGET_NODE} exited - an allowlist entry ' + "that 'suppress' does not name must not suppress it") + self.assertIn(TARGET_NODE, fault.get('description', '')) + + # The warning is read straight off the gateway's own stderr - the same instrument + # test_startup_param_clamp_warnings.test.py already uses for an identical "did we say + # so" claim. No exact wording is pinned (node_death's own log text is free to change); + # this asks that a WARN-severity line names BOTH "allowlist" and "suppress" together - + # see _INERT_ALLOWLIST_WARNING_RE for why the bare substring "allowlist" is not enough. + self.assertTrue( + proc_output.waitFor( + _INERT_ALLOWLIST_WARNING_RE, process=gateway_node, stream='stderr', + timeout=WARNING_TIMEOUT_SEC), + "no startup WARNING naming both 'allowlist' and 'suppress' on the same line " + f"appeared on the gateway's stderr within {WARNING_TIMEOUT_SEC}s - a configured " + 'allowlist that suppress does not name must say so at WARN severity', + ) + + +class TestSuppressionLifecycleCleanShutdown(unittest.TestCase): + """Self-suppression: a require_active node that departed while not active is not named. + + Two managed_lifecycle instances, both require_active-owned - self-suppression needs "this + node is owned by lifecycle_expectation", "it departed while not active", AND + ``detectors.node_death.suppress`` naming the mechanism (suppression is opt-in like every + other mechanism this package carries); the lifecycle label alone would over-suppress a node + nobody is watching, and leaving suppress unset would make a correct detector report both + departed nodes. CLEAN_NODE is driven through a real UNCONFIGURED_SHUTDOWN transition + (reaching "finalized", a non-active label) before its process dies; KILLED_NODE reaches + "active" on its own and is killed outright, still active. Only the second is a departure + lifecycle_expectation has nothing else to say about. + + GRAPH_NODE_DISAPPEARED raises and names KILLED_NODE. The "CLEAN_NODE is never named" half + cannot, by itself, discriminate a working self-suppressor from no detector at all - both + leave CLEAN_NODE unnamed - so it is proven alongside the positive KILLED_NODE claim rather + than on its own. + """ + + @classmethod + def setUpClass(cls): + rclpy.init() + cls._client_node = Node('node_death_suppression_e2e_shutdown_client') + + @classmethod + def tearDownClass(cls): + cls._client_node.destroy_node() + rclpy.shutdown() + + def test_clean_shutdown_not_named_active_kill_is(self, clean_node, killed_node): + # Global gate, not app_id: both nodes are require_active-tracked and neither starts + # active, so ReliabilityGate reports each of them "warming_up" (LifecycleWatcher's + # node_ok() is false for exactly this case) until KILLED_NODE activates - a per-entity + # gate on CLEAN_NODE specifically would wait for something that can never become true. + # Same reasoning test_lifecycle_expectation_e2e.test.py's "main" scenario gives for its + # identical choice. + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC), + 'graph_watchdog never reported an armed global state') + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=FAULTS_LIVE_TIMEOUT_SEC), + 'GET /faults never answered 200 - nothing below would prove anything') + + self.assertIsNotNone( + _poll_watchdog_entity(PORT, CLEAN_NODE, 'unconfigured', timeout=ARM_TIMEOUT_SEC), + f'{CLEAN_NODE} never read as "unconfigured" - the fixture was not discovered as a ' + 'managed node, or this scenario never set up the trigger it is named for', + ) + self.assertIsNotNone( + _poll_watchdog_entity(PORT, KILLED_NODE, 'active', timeout=ARM_TIMEOUT_SEC), + f'{KILLED_NODE} never reached "active" on its own - the contrast this scenario ' + 'needs (an active node killed outright) was never set up', + ) + + change_state_service = f'/{CLEAN_NODE}/change_state' + self.assertTrue( + _call_change_state_once( + type(self)._client_node, change_state_service, + Transition.TRANSITION_UNCONFIGURED_SHUTDOWN, timeout=30.0), + f'the real UNCONFIGURED_SHUTDOWN transition on {CLEAN_NODE} was rejected or never ' + 'answered', + ) + self.assertIsNotNone( + _poll_watchdog_entity(PORT, CLEAN_NODE, 'finalized', timeout=30.0), + f'{CLEAN_NODE} never read as "finalized" after its own UNCONFIGURED_SHUTDOWN - the ' + 'clean-shutdown trigger this scenario is about was never actually reached', + ) + + os.kill(clean_node.process_details['pid'], signal.SIGTERM) + os.kill(killed_node.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, CLEAN_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'{CLEAN_NODE} never left GET /apps after SIGTERM') + self.assertTrue( + _poll_apps_absent(PORT, KILLED_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'{KILLED_NODE} never left GET /apps after SIGTERM') + + fault = poll_faults(PORT, FAULT_CODE, timeout=RAISE_TIMEOUT_SEC) + if fault is None: + self.fail( + f'{FAULT_CODE} never raised after {KILLED_NODE} (active, killed outright) ' + 'exited - a departure lifecycle_expectation has nothing to say about must still ' + 'be named') + + assert_fault_describes_only( + self, PORT, FAULT_CODE, + required=[KILLED_NODE], forbidden=[CLEAN_NODE], + duration=SUSTAINED_WINDOW_SEC) + + +class TestSuppressionOptIn(unittest.TestCase): + """An allowlist entry with no `suppress` key at all does not suppress by itself.""" + + def test_no_suppress_key_still_raises(self, target_node): + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC, app_id=TARGET_NODE), + f'graph_watchdog never reported {TARGET_NODE} armed') + + os.kill(target_node.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, TARGET_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'{TARGET_NODE} never left GET /apps after SIGTERM') + + fault = poll_faults(PORT, FAULT_CODE, timeout=RAISE_TIMEOUT_SEC) + if fault is None: + self.fail( + f'{FAULT_CODE} never raised after {TARGET_NODE} exited - an allowlist entry ' + 'with no suppress key at all must not suppress by itself (suppression is ' + 'opt-in)') + self.assertIn(TARGET_NODE, fault.get('description', '')) + + +class TestSuppressionPruneNoFalseHeal(unittest.TestCase): + """Reclaiming an allowlisted key's bookkeeping never heals an unrelated, still-true fault. + + Two independent nodes. TARGET_NODE is on the allowlist (suppress: ["allowlist"]) - the + ONLY shape prune() ever reclaims through: node_liveness_tracker.hpp's own doc states the + consequence directly - an unsuppressed death has no reclaim path at all and so must stay + reported "permanent until acknowledged" for as long as it is actually dead. That is + exactly why this row cannot point its reclaim proof at an unsuppressed node: prune() would + never reclaim it, so a reclaim assertion against it could only ever fail. SECOND_NODE, + carrying no suppression, is instead the permanently-outstanding control, not a second + reclaim candidate. + + Reclaim itself is proven as a tracked_count DELTA off a baseline, not against zero: the + aggregate also carries every other armed App in the graph (the gateway's own entity among + them), so it can never read exactly zero regardless of whether pruning works. + """ + + def test_pruning_the_suppressed_key_never_heals_the_unsuppressed_one( + self, target_node, second_node): + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC, app_id=TARGET_NODE), + f'graph_watchdog never reported {TARGET_NODE} armed') + self.assertTrue( + wait_until_watchdog_armed(PORT, timeout=ARM_TIMEOUT_SEC, app_id=SECOND_NODE), + f'graph_watchdog never reported {SECOND_NODE} armed') + self.assertTrue( + wait_until_faults_endpoint_live(PORT, timeout=FAULTS_LIVE_TIMEOUT_SEC), + 'GET /faults never answered 200 - the claims below would prove nothing about the ' + 'detector if the channel itself never came up', + ) + + # Baseline for the reclaim delta: both nodes are present and tracked NOW, before + # either departs. wait_until_watchdog_armed above proves the GATE considers both + # armed, not that node_death's own tick has already added their keys to tracker_ (see + # _poll_stable_tracked_count's own docstring) - so the baseline is read only once + # tracked_count has genuinely settled, not off the first sample after both gate-arm. + baseline, stable = _poll_stable_tracked_count( + PORT, 'node_death', timeout=STABLE_TRACKED_COUNT_TIMEOUT_SEC) + self.assertTrue( + stable, + f'node_death tracked_count never held steady before either node departed (last ' + f'read: {baseline!r}) - the baseline below could not be trusted to already ' + f'include {TARGET_NODE} and {SECOND_NODE}, which would make the reclaim delta ' + 'below meaningless', + ) + self.assertIsNotNone(baseline, 'no node_death status block on GET /x-medkit-watchdog') + baseline_count = baseline['tracked_count'] + + os.kill(target_node.process_details['pid'], signal.SIGTERM) + os.kill(second_node.process_details['pid'], signal.SIGTERM) + self.assertTrue( + _poll_apps_absent(PORT, TARGET_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'{TARGET_NODE} never left GET /apps after SIGTERM') + self.assertTrue( + _poll_apps_absent(PORT, SECOND_NODE, timeout=DEPARTURE_TIMEOUT_SEC), + f'{SECOND_NODE} never left GET /apps after SIGTERM') + + # SECOND_NODE carries no suppression: it must raise, ordinarily, like any other death. + fault = poll_faults(PORT, FAULT_CODE, timeout=RAISE_TIMEOUT_SEC) + if fault is None: + self.fail(f'{FAULT_CODE} never raised after {SECOND_NODE} exited') + + # PRUNE_GRACE consecutive suppressed ticks reclaims TARGET_NODE's own bookkeeping - + # tracked_count dropping by exactly one from the baseline, proving prune() genuinely + # ran (not merely that TARGET_NODE was never raised, which a deleted prune() would + # also produce). SECOND_NODE's own departed-but-unsuppressed entry stays tracked + # throughout - only TARGET_NODE's disappears - so the delta is exactly one. + self.assertTrue( + poll_detector_status( + PORT, 'node_death', 'tracked_count', baseline_count - 1, + timeout=RAISE_TIMEOUT_SEC), + f'{TARGET_NODE} was never reclaimed - detectors.node_death.tracked_count never ' + f'dropped by exactly one from its baseline ({baseline_count}) even after ' + 'PRUNE_GRACE, so this row cannot tell a working prune() from one that never runs') + + # The reclaim above must never have touched SECOND_NODE's own, unrelated, genuinely + # outstanding fault: it must stay CONFIRMED and keep naming SECOND_NODE - never + # TARGET_NODE - for a window spanning straight through the reclaim tick just observed. + # describes_only, not a bare presence check: a detector that let pruning heal the + # wrong entry, or empty the description, must turn this row red. + assert_fault_describes_only( + self, PORT, FAULT_CODE, required=[SECOND_NODE], forbidden=[TARGET_NODE], + duration=SUSTAINED_WINDOW_SEC) + + +# Each CTest target launches this file with one scenario, so only that scenario's case may +# run. Removing the others from the module (rather than skipping them) means each run reports +# exactly one case, and a missing result is a real failure rather than an expected line of +# output - see test_node_death_e2e.test.py's identical rationale. +_SCENARIO_CASES = { + 'allowlist_suppresses': 'TestSuppressionAllowlistSuppresses', + 'allowlist_not_named_is_inert': 'TestSuppressionAllowlistNotNamedIsInert', + 'lifecycle_clean_shutdown': 'TestSuppressionLifecycleCleanShutdown', + 'suppression_is_opt_in': 'TestSuppressionOptIn', + 'prune_no_false_heal': 'TestSuppressionPruneNoFalseHeal', +} +if SCENARIO not in _SCENARIO_CASES: + raise RuntimeError( + f'WATCHDOG_E2E_SCENARIO={SCENARIO!r} is not one of {sorted(_SCENARIO_CASES)}; the ' + 'CTest target and this file disagree about which scenarios exist') +for _scenario, _case_name in _SCENARIO_CASES.items(): + if _scenario != SCENARIO: + del globals()[_case_name] + + +@launch_testing.post_shutdown_test() +class TestShutdown(unittest.TestCase): + """Verify the gateway/fault_manager/demo stack exits cleanly.""" + + def test_exit_codes(self, proc_info): + for info in proc_info: + self.assertIn( + info.returncode, + ALLOWED_EXIT_CODES, + f'Process {info.process_name} exited with {info.returncode}', + ) diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_allowlist_suppressor.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_allowlist_suppressor.cpp new file mode 100644 index 000000000..e9cf64ebf --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_allowlist_suppressor.cpp @@ -0,0 +1,125 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Pure logic: no rclcpp::init() needed - AllowlistSuppressor::suppresses() ignores ctx +// entirely, so a default-constructed DetectorContext (every pointer null) is enough. +#include + +#include +#include + +#include "ros2_medkit_graph_watchdog/allowlist_suppressor.hpp" + +using ros2_medkit_graph_watchdog::AllowlistSuppressor; +using ros2_medkit_graph_watchdog::DetectorContext; + +TEST(AllowlistSuppressor, ExactMatchSuppresses) { + AllowlistSuppressor s({"/a", "/b"}); + DetectorContext ctx; + EXPECT_TRUE(s.suppresses("/a", ctx)); + EXPECT_TRUE(s.suppresses("/b", ctx)); +} + +TEST(AllowlistSuppressor, NonMemberAbstains) { + AllowlistSuppressor s({"/a"}); + DetectorContext ctx; + EXPECT_FALSE(s.suppresses("/c", ctx)); +} + +TEST(AllowlistSuppressor, PrefixOfAnAllowedEntryIsNotAMatch) { + AllowlistSuppressor s({"/r1/x"}); + DetectorContext ctx; + EXPECT_FALSE(s.suppresses("/r1/x/child", ctx)); + EXPECT_FALSE(s.suppresses("/r1", ctx)); +} + +TEST(AllowlistSuppressor, ASharedSuffixAcrossDifferentNamespacesIsNotAMatch) { + // /r1/x on the list must never reach /r2/x - the whole reason this is an exact-match + // set rather than a leaf/substring comparison. + AllowlistSuppressor s({"/r1/x"}); + DetectorContext ctx; + EXPECT_FALSE(s.suppresses("/r2/x", ctx)); +} + +TEST(AllowlistSuppressor, EmptyAllowSetSuppressesNothing) { + AllowlistSuppressor s({}); + DetectorContext ctx; + EXPECT_FALSE(s.suppresses("", ctx)); + EXPECT_FALSE(s.suppresses("/a", ctx)); +} + +TEST(AllowlistSuppressor, IsDurable) { + AllowlistSuppressor s({"/a"}); + EXPECT_TRUE(s.durable()); +} + +// The three matching forms this class deliberately mirrors from +// lifecycle_expectation_detector.cpp's own require_active matching (id / effective_fqn() / +// bare leaf) - see the class doc. suppresses() itself only ever sees one string (the entity +// key) and so only ever proves the two forms derivable from that string alone; the id form +// is proven separately below through allows(), the method a caller holding a captured +// App::id uses instead - see node_death_detector.cpp's tick(). + +TEST(AllowlistSuppressor, FullFqnFormSuppressesViaExactMatch) { + // The form the ORIGINAL exact-match tests above already exercise, named here so all three + // forms appear together as one group. + AllowlistSuppressor s({"/powertrain/engine/calibration"}); + DetectorContext ctx; + EXPECT_TRUE(s.suppresses("/powertrain/engine/calibration", ctx)); +} + +TEST(AllowlistSuppressor, BareLeafFormSuppressesANamespacedKey) { + // The bug this fix closes: an operator writes the bare node name they see in the graph, + // not its full namespaced fqn. + AllowlistSuppressor s({"calibration"}); + DetectorContext ctx; + EXPECT_TRUE(s.suppresses("/powertrain/engine/calibration", ctx)); +} + +TEST(AllowlistSuppressor, BareLeafFormMatchesEveryNamespaceOfThatName) { + // Deliberately fleet-wide, mirroring lifecycle_expectation's own require_active: a bare + // name matches every namespace's node of that name. An operator who wants to pin one + // namespace uses a full fqn entry instead (see FullFqnFormSuppressesViaExactMatch). + AllowlistSuppressor s({"calibration"}); + DetectorContext ctx; + EXPECT_TRUE(s.suppresses("/coll_a/calibration", ctx)); + EXPECT_TRUE(s.suppresses("/coll_b/calibration", ctx)); +} + +TEST(AllowlistSuppressor, KeyWithNoSlashIsAbstainedWhenNotListed) { + // Exercises the slash == npos branch: entity_key is already its own bare leaf, so there is + // no second form left to try once the exact-match check above has already missed. + AllowlistSuppressor s({"other"}); + DetectorContext ctx; + EXPECT_FALSE(s.suppresses("calibration", ctx)); +} + +TEST(AllowlistSuppressor, IdFormMatchesThroughAllowsDirectly) { + // The id form: a caller (node_death_detector.cpp) that has captured App::id while an + // entity was still present checks it against allows() directly, bypassing suppresses() + // entirely - see the class doc for why suppresses() alone cannot answer this for a dead + // key. The id used here is what a bare-name collision produces + // ('_'), proven end to end (including the sibling that + // must NOT match) in test_node_death_integration.cpp's own collision test. + AllowlistSuppressor s({"coll_a_calibration"}); + EXPECT_TRUE(s.allows("coll_a_calibration")); +} + +TEST(AllowlistSuppressor, AllowsIsAnExactCheckWithNoLeafDerivation) { + // allows() is the raw building block suppresses() itself is built from - it does not also + // derive a bare leaf the way suppresses() does, so a candidate that would only match via + // leaf-derivation must not match here. + AllowlistSuppressor s({"calibration"}); + EXPECT_FALSE(s.allows("/powertrain/engine/calibration")); +} diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_graph_watchdog_plugin.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_graph_watchdog_plugin.cpp index 702322dd8..bd284d61a 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_graph_watchdog_plugin.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_graph_watchdog_plugin.cpp @@ -280,6 +280,36 @@ class FakeContextChurningApps : public FakeContext { mutable std::atomic counter_{0}; }; +// Reports a permanently-present "/anchor" app (so the snapshot is never empty - an empty +// snapshot re-arms ReliabilityGate's global bringup grace, which would delay arming for a +// reason unrelated to what this context exists to drive) plus one "/victim" app on the VERY +// FIRST call only - departed on every call after that. Built for +// MalformedNodeDeathPruneGraceUsesThePluginScopeFallbackNotTheDetectorsOwnDefault, which +// needs a real death to drive real reclaim timing, not merely a config-read seam. +class FakeContextVictimDepartsAfterFirstTick : public FakeContext { + public: + using FakeContext::FakeContext; + ros2_medkit_gateway::IntrospectionInput get_entity_snapshot() const override { + ros2_medkit_gateway::IntrospectionInput input; + ros2_medkit_gateway::App anchor; + anchor.id = "/anchor"; + anchor.bound_fqn = "/anchor"; + anchor.is_online = true; + input.apps.push_back(anchor); + if (calls_.fetch_add(1) == 0) { + ros2_medkit_gateway::App victim; + victim.id = "/victim"; + victim.bound_fqn = "/victim"; + victim.is_online = true; + input.apps.push_back(victim); + } + return input; + } + + private: + mutable std::atomic calls_{0}; +}; + class GraphWatchdogPluginTest : public ::testing::Test { protected: void SetUp() override { @@ -332,6 +362,34 @@ TEST_F(GraphWatchdogPluginTest, PluginScopePruneGraceStillAppliesWhenTheDetector EXPECT_EQ(g_captured_config["prune_grace"].get(), 42) << "the plugin-scope value is still the default"; } +// compute_departed_retention_ticks() needs no ROS node at all - it reads only its own +// config_snapshot parameter plus tick_interval_ms_/prune_grace_, both left at their +// constructor defaults (1000, 60) when configure()/set_context() are never called. +TEST(ComputeDepartedRetentionTicks, OversizedNodeDeathPruneGraceIsRejectedNotTruncated) { + ros2_medkit_graph_watchdog::GraphWatchdogPlugin plugin; + const int ticks = plugin.compute_departed_retention_ticks_for_test( + nlohmann::json{{"detectors", {{"node_death", {{"prune_grace", 4294967296LL}}}}}}); + // 4294967296 (2^32) narrowed to int BEFORE a range check wraps to exactly 0, which passes + // a bare ">= 0" check silently - node_death_detector.cpp's own configure() rejects this + // same value and keeps its default (60), so the correct retention here is computed from + // that same default: prune_ticks=max(60, miss_grace+1), miss_grace floored to 2 at the + // 1000ms default tick (unaffected, since the floor only bites below ~1500ms), so + // 60+2+1=63. A narrow-then-validate bug would instead compute from the wrapped 0: + // max(0,3)+2+1=6. + EXPECT_EQ(ticks, 63); +} + +TEST(ComputeDepartedRetentionTicks, OversizedNodeDeathMissGraceIsRejectedNotTruncated) { + ros2_medkit_graph_watchdog::GraphWatchdogPlugin plugin; + const int ticks = plugin.compute_departed_retention_ticks_for_test( + nlohmann::json{{"detectors", {{"node_death", {{"miss_grace", 4294967296LL}}}}}}); + // Same shape as the prune_grace case above, for the other field this function reads the + // same way. miss_grace stays at its default (2, unaffected by the floor at the 1000ms + // default tick), prune_grace stays at the plugin's own default (60): + // prune_ticks=max(60,3)=60, retention=60+2+1=63. + EXPECT_EQ(ticks, 63); +} + TEST_F(GraphWatchdogPluginTest, ExportsAdvertiseCorrectApiVersion) { EXPECT_EQ(plugin_api_version(), ros2_medkit_gateway::PLUGIN_API_VERSION); } @@ -507,6 +565,107 @@ TEST_F(GraphWatchdogPluginTest, WatchdogStatusRouteReportsEntityLifecycleAndArme spin.join(); } +// The plugin injects its own prune_grace default only when the +// per-detector key is ABSENT (see set_context()'s configure loop), so a PRESENT but +// malformed detectors.node_death.prune_grace used to reach node_death's own configure() +// unfiltered, which falls back to ITS OWN hardcoded default (60) rather than the plugin's. +// compute_departed_retention_ticks() independently falls back to the plugin's own +// prune_grace_ for the exact same malformed value, so the two disagreed whenever an +// operator's plugin-scope prune_grace was anything other than the coincidentally-matching +// 60 - sizing the lifecycle-departed retention window for a reclaim tick node_death would +// not actually reach for roughly another 57 ticks. Proven here through REAL reclaim timing +// (an allowlisted, durably-suppressed death) rather than by inspecting either side's +// computation in isolation - either side alone can look right while still disagreeing with +// the other, which is exactly why OversizedNodeDeathPruneGraceIsRejectedNotTruncated below +// (plugin-scope prune_grace left at the coincidentally-matching default 60) could not catch +// this. +TEST_F(GraphWatchdogPluginTest, MalformedNodeDeathPruneGraceUsesThePluginScopeFallbackNotTheDetectorsOwnDefault) { + ros2_medkit_graph_watchdog::GraphWatchdogPlugin plugin; + FakeContextVictimDepartsAfterFirstTick ctx(gateway_node_.get()); + + rclcpp::executors::SingleThreadedExecutor exec; + exec.add_node(gateway_node_); + std::thread spin([&exec] { + exec.spin(); + }); + + // tick_interval_ms MUST be 3000: node_death's own wall-clock floor + // (min_node_death_miss_grace) bumps a fast-tick miss_grace up regardless of what is + // configured, and prune_ticks = max(prune_grace, miss_grace + 1) - a bumped-up miss_grace + // would dominate that max() identically whether prune_grace fell back to 1 or to 60, + // masking the exact divergence this test exists to catch. At 3000ms the floor is exactly + // 0, so the configured miss_grace(0) is used as written. + plugin.configure( + nlohmann::json{{"tick_interval_ms", 3000}, + {"warmup_cycles", 0}, + {"prune_grace", 0}, // plugin-scope, deliberately far from node_death's own hardcoded 60 + {"detectors", + {{"node_death", + {{"miss_grace", 0}, + {"prune_grace", "invalid"}, // malformed: must fall back, not pass through unfiltered + {"allowlist", nlohmann::json::array({"/victim"})}, + {"suppress", nlohmann::json::array({"allowlist"})}}}}}}); + plugin.set_context(ctx); + + const auto routes = plugin.get_routes(); + const auto route_it = find_watchdog_route(routes); + ASSERT_NE(route_it, routes.end()); + + // node_death's tracked_count_ atomic default-initializes to 0 - the SAME value a reclaimed + // /victim would leave it at - so reading it before the first tick has actually run would + // make the deadline below pass vacuously, having proven nothing. Wait for the first tick to + // publish "both anchor and victim are tracked" (2) before timing anything past it. + const auto armed_deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + std::size_t tracked_count = 0; + while (std::chrono::steady_clock::now() < armed_deadline) { + TestResponseSink armed_sink; + ros2_medkit_gateway::PluginRequest armed_req(nullptr); + ros2_medkit_gateway::PluginResponse armed_res(&armed_sink); + route_it->handler(armed_req, armed_res); + const auto & armed_detectors = armed_sink.body["x-medkit-watchdog"]["detectors"]; + if (armed_detectors.contains("node_death")) { + tracked_count = armed_detectors["node_death"]["tracked_count"].get(); + if (tracked_count >= 2) { + break; + } + } + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + ASSERT_EQ(tracked_count, 2u) << "precondition: anchor and victim must both have been observed " + "tracked at least once, or the reclaim timing below proves " + "nothing"; + + // prune_ticks = max(0, miss_grace(0)+1=1) = 1 if the plugin-scope fallback is used: victim + // departs and mis-suppresses on tick 2 (streak 1), reclaimed on tick 3 (streak 2 > 1) - two + // more 3000ms ticks past the precondition above, ~6-7s including scheduling slack. If + // node_death instead fell back to its own hardcoded 60, reclaim needs streak 61 - roughly + // 180s at this tick rate. The deadline sits well inside that gap: generous for the fixed + // case, a small fraction of the hardcoded-fallback case, so this discriminates rather than + // merely giving both enough time to pass. + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(12); + while (std::chrono::steady_clock::now() < deadline) { + TestResponseSink sink; + ros2_medkit_gateway::PluginRequest req(nullptr); + ros2_medkit_gateway::PluginResponse res(&sink); + route_it->handler(req, res); + const auto & detectors = sink.body["x-medkit-watchdog"]["detectors"]; + if (detectors.contains("node_death")) { + tracked_count = detectors["node_death"]["tracked_count"].get(); + if (tracked_count == 1) { + break; + } + } + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + EXPECT_EQ(tracked_count, 1u) << "the allowlisted, durably-suppressed /victim must be reclaimed within a few prune " + "ticks sized off the plugin's OWN prune_grace default, not node_death's unrelated " + "hardcoded fallback - only the anchor (never departed) should remain tracked"; + + EXPECT_NO_THROW(plugin.shutdown()); + exec.cancel(); + spin.join(); +} + // The plugin's own fault client, end to end against a REAL ReportFault server. Two // separate claims, and the test would be worth little without both: // diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_expectation_integration.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_expectation_integration.cpp index a4e06a7dc..4e70500f5 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_expectation_integration.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_expectation_integration.cpp @@ -1752,14 +1752,18 @@ TEST_F(LifecycleExpectationIntegrationTest, ANodeCrossingAfterTheCapIsFullIsName // broke - and the operator is then told about the one that is gone instead of the one that // needs attention. // -// Paced so both crossings land on one tick: the departed batch is measured not-active once -// and then vanishes, its streaks resuming past the 3-tick absence grace, while the present -// node starts its own streak exactly late enough that the two reach `grace + 1` together. The -// batch is sized by fill_count_past_cap from the REAL detail builder, so its details alone -// exceed the 480-char cap - which is what makes "which one is named" a real choice rather -// than a cosmetic ordering, and the present node's id sorts LAST among them all. +// A departed node can no longer cross INTO GRAPH_NODE_INACTIVE from a below-grace +// streak (see the tracker's own class doc), so "both cross on the same tick" is no longer a +// reachable shape for this fault - a departed entry only ever carries content it EARNED +// before it left. What is still reachable, and still the real question, is whether a node +// that just broke is named ahead of a pile of nodes that broke earlier and left: the departed +// batch matures FIRST while present, THEN leaves for good, THEN present_id joins and crosses +// grace fresh, on its own tick, while the batch is still absent-and-content. The batch is +// sized by fill_count_past_cap from the REAL detail builder, so its details alone exceed the +// 480-char cap - which is what makes "which one is named" a real choice rather than a +// cosmetic ordering - and the present node's id sorts LAST among them all. TEST_F(LifecycleExpectationIntegrationTest, PresentNodeCrossingWithDepartedOnesIsNamedAheadOfThem) { - constexpr int kGrace = 5; + constexpr int kGrace = 1; std::vector departed_ids; const std::size_t departed_count = fill_count_past_cap("g", departed_ids); ASSERT_GT(departed_count, 1u); @@ -1783,19 +1787,27 @@ TEST_F(LifecycleExpectationIntegrationTest, PresentNodeCrossingWithDepartedOnesI gate.set_lifecycle_state_for_test(id, "inactive"); } - // Tick 1: the soon-to-depart batch is measured not-active (streak 1). The present node is - // not in the graph yet, so it is not tracked at all. + // The departed batch matures FIRST, on its own: present_id is not in the snapshot yet. set_apps(departed_ids); - det->tick(ctx); - // Ticks 2-3: everything absent. Inside the 3-tick absence grace, so the batch's streaks are - // simply held. + for (int i = 0; i < kGrace + 1; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(200ms); + ASSERT_TRUE(any_failed_desc_contains(kGraphSource, departed_ids.front())) + << "the departed batch never matured, so nothing below tests what an ALREADY-CONTENT entry " + "does once it leaves - only what a fresh crossing does"; + + // The batch leaves for good, past the absence grace. It is already matured, so absence + // continues it unconditionally - "has since left the graph" and all. set_apps({}); - det->tick(ctx); - det->tick(ctx); + for (int i = 0; i < kDefaultAbsenceGrace + 2; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } - // Ticks 4-9: only the present node is in the graph, reading not-active. Its streak runs - // 1..6 across these six ticks; the batch's absence passes the grace on tick 5 and its - // streaks resume 2..6 across ticks 5-9. Both reach grace + 1 on tick 9 - the same tick. + // present_id joins and crosses grace fresh, on its own tick, while the departed batch is + // still absent-and-content. const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); set_apps({present_id}); for (int i = 0; i < kGrace + 1; ++i) { @@ -1803,7 +1815,7 @@ TEST_F(LifecycleExpectationIntegrationTest, PresentNodeCrossingWithDepartedOnesI std::this_thread::sleep_for(5ms); } ASSERT_TRUE(wait_for_count(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before + 1)) - << "nothing crossed at all, so this test never reached the tick it is about"; + << "present_id never crossed grace, so this test never reached the tick it is about"; std::this_thread::sleep_for(300ms); const std::string desc = last_failed_description(kGraphSource); @@ -1813,6 +1825,10 @@ TEST_F(LifecycleExpectationIntegrationTest, PresentNodeCrossingWithDepartedOnesI "nodes that had already left the graph - the operator is told about the departures and not " "about the node that just broke. Full description: " << desc; + ASSERT_NE(desc.find(departed_ids.front()), std::string::npos) + << "the departed batch left no trace at all, so the ordering claim below would be vacuous - its " + "evidence must survive the departure, just behind present_id. Full description: " + << desc; EXPECT_LT(desc.find(present_id), desc.find(departed_ids.front())) << "the present node is named, but behind a departed one - a departure is never more urgent " "than a node that is still there and has just gone bad. Full description: " @@ -3168,11 +3184,15 @@ TEST_F(LifecycleExpectationIntegrationTest, NotManagedNodeInARestartLoopIsStillR "never raised GRAPH_NODE_NOT_MANAGED"; } -// The pair no test at any tier covered: a MEASURED not-active read alternating with a -// NOT-MANAGED one, the two separated by absence runs longer than the absence grace. Driven -// through the REAL gate rather than the injection seam, which can only ever SET a label and -// never remove tracking: toggling whether "a" carries GetState/ChangeState services is what -// produces a genuine nullopt for a previously-tracked fqn. +// A MEASURED not-active read alternating with a NOT-MANAGED one, the two separated by +// absence runs longer than the absence grace. Driven through the REAL gate rather than the +// injection seam, which can only ever SET a label and never remove tracking: toggling +// whether "a" carries GetState/ChangeState services is what produces a genuine nullopt for a +// previously-tracked fqn. Neither the not-managed legs (which never touch the +// violation streak) nor the absence gaps (which hold a below-grace streak rather than +// advancing it) contribute anything on their own - only the repeated MEASURED not-active +// legs do, one real tick at a time - so this still raises, from real evidence accumulated +// across several cycles of presence rather than from any of the gaps in between. TEST_F(LifecycleExpectationIntegrationTest, InactiveAlternatingWithNotManagedAcrossAbsenceGapsRaisesInactive) { set_apps({}); ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); @@ -3263,6 +3283,15 @@ TEST_F(LifecycleExpectationIntegrationTest, HealthyNodeThatVanishesRaisesNothing EXPECT_EQ(det->tracked_count_for_test(), 0u) << "an idle entry for a departed healthy node was never reclaimed"; } +// Unlike its UNREADABLE and NOT-MANAGED siblings above, GRAPH_NODE_INACTIVE no longer treats +// this shape as an evasion to close on its own: absence contributes nothing to a +// below-grace violation streak (see the tracker's own class doc), so a node that is inactive +// for one tick and then gone for a run, forever, only accumulates evidence from its PRESENT +// ticks - one per cycle here. It is still eventually reported, because it really was measured +// not-active repeatedly; it just now takes grace + 1 cycles of presence rather than grace + 1 +// ticks of any kind. Closing the evasion in the ABSENCE itself is GRAPH_NODE_DISAPPEARED's job +// now (this package's own node_death detector), not this streak leaning on a departure it +// cannot corroborate. TEST_F(LifecycleExpectationIntegrationTest, InactiveNodeInARestartLoopIsStillReported) { set_apps({"a"}); ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); @@ -3892,16 +3921,21 @@ TEST(LifecycleExpectationConfig, LiveClockIsNotErasedAtTheTightestGraceAndPruneG } // The combination the old clamp existed for - a wide `grace` next to the tightest -// `prune_grace` - is now simply safe: a node still climbing toward that wide grace is -// carrying evidence, so the hair-trigger prune horizon never reaches it, and it goes on to -// be confirmed at exactly the tick it would have been confirmed at anyway. +// `prune_grace` - is where a below-grace streak's two guarantees have to meet: it must not be +// reclaimed as IDLE (it is not - violation_streak != 0 - even though it is small), and it must +// not be silently matured by the absence run either (only a present tick may still advance it +// - see the tracker's own class doc). So the entry survives, HELD, exactly where it was: never +// confirmed while the node stays gone, never pruned either, and ready to resume - not restart +// - the moment the node returns. It is a fixture test rather than a bare configure()-level one +// because only the fake ReportFault sink can see the (absence of a) raise. // -// The instrument is the CONFIRMATION, not the map size. A map size of 1 is satisfied by an -// implementation that keeps the entry but freezes its clock through absence - which is the -// bug that makes a node vanishing while not-active permanently invisible, i.e. exactly the -// thing this test is named for. It is a fixture test rather than a bare configure()-level one -// for that reason: only the fake ReportFault sink can see the raise. -TEST_F(LifecycleExpectationIntegrationTest, WideGraceWithTheTightestPruneGraceStillConfirmsAVanishedNode) { +// The node is armed (read "active") once before anything else: this claim holds only for a +// node the presence detector could itself have reported (node_death tracks any node the gate +// has armed at least once), so the fixture has to BE one, or the absence run below would be +// proving something about a different node than the one this test names. A node that never +// arms gets absence-driven maturation instead - see +// NeverArmedBelowGraceStreakMaturesFromAbsenceInsteadOfHoldingIndefinitely below for that one. +TEST_F(LifecycleExpectationIntegrationTest, BelowGraceStreakSurvivesTheTightestPruneGraceWithoutConfirmingWhileAbsent) { set_apps({"a"}); ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); arm_global_grace(gate); @@ -3910,6 +3944,77 @@ TEST_F(LifecycleExpectationIntegrationTest, WideGraceWithTheTightestPruneGraceSt ASSERT_TRUE(det); det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 4}, {"prune_grace", 0}}); auto ctx = make_ctx(DetectorMode::Raise, &gate); + + gate.set_lifecycle_state_for_test("a", "active"); + ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_PASSED, 0, *det, ctx)) + << "the healthy baseline never cleared, so the node below cannot be shown to have armed"; + gate.set_lifecycle_state_for_test("a", "inactive"); + + const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); + det->tick(ctx); // streak 1, well below grace(4) + ASSERT_EQ(det->tracked_count_for_test(), 1u); + std::this_thread::sleep_for(200ms); + ASSERT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), failed_before) + << "the node was confirmed on its first not-active tick, so grace(4) was never in force and " + "the absence run below proves nothing about a below-grace streak"; + + // "a" vanishes with a streak of 1 out of 4, at the tightest prune_grace (0) - an IDLE entry + // would be reclaimed on the very next absent tick. + set_apps({}); + for (int i = 0; i < 20; ++i) { + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(200ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), failed_before) + << "GRAPH_NODE_INACTIVE was confirmed from a below-grace streak while the node was absent the " + "whole time - a fault born from evidence gathered while nobody could observe the node"; + ASSERT_EQ(det->tracked_count_for_test(), 1u) + << "the entry was reclaimed by the tightest prune_grace despite carrying a live (if below-grace) " + "streak - is_idle() must not treat a non-zero streak as nothing to lose"; + + // It resumes rather than restarts: held at 1, three more present ticks reach grace(4), the + // fourth crosses it - due on that exact tick and no later, so a streak that had restarted + // from zero (needing a fifth) would still read grace here, not past it. + set_apps({"a"}); + for (int i = 0; i < 3; ++i) { + det->tick(ctx); // resumed streak 2, 3, 4 (== grace, not past it) + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(200ms); + ASSERT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), failed_before) + << "resumed streak reached grace(4) but was already confirmed one tick early"; + det->tick(ctx); // resumed streak 5 > grace: due on this exact tick + EXPECT_TRUE(wait_for_count(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before + 1)) + << "resumed streak 5 > grace(4) was not confirmed on this exact tick - the streak restarted " + "from zero on return instead of resuming where the absence had held it"; + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "a")) + << "the confirmation did not name the node once it finally raised"; +} + +// The row the test immediately above cannot cover: a node that is NEVER armed, because it +// never once reads "active" - exactly a require_active node that comes up unconfigured and +// stays there. node_death only ever tracks a key the reliability gate has armed at least +// once, so nothing else in the plugin can ever report this node's departure; through the +// real gate and the real (fake-service-backed) fault client, a below-grace streak on such a +// node must still mature from absence alone, or the departure is reported by nothing at all. +// Same grace, same streak-at-departure, same fixture shape as +// BelowGraceStreakSurvivesTheTightestPruneGraceWithoutConfirmingWhileAbsent immediately +// above, with arming the only difference - together the two rows prove the split is keyed on +// arming and nothing else. +TEST_F(LifecycleExpectationIntegrationTest, NeverArmedBelowGraceStreakMaturesFromAbsenceInsteadOfHoldingIndefinitely) { + set_apps({"a"}); + ReliabilityGate gate(kWarmupCycles, gateway_.get(), &node_mutex_); + arm_global_grace(gate); + + auto det = make_lifecycle_expectation(); + ASSERT_TRUE(det); + det->configure(nlohmann::json{{"require_active", nlohmann::json::array({"a"})}, {"grace", 4}}); + auto ctx = make_ctx(DetectorMode::Raise, &gate); + + // Never armed: "a" reads non-active from the very first tick and never once reads "active", + // so LifecycleWatcher::node_ok() is false for its whole life here and the gate never arms + // it. gate.set_lifecycle_state_for_test("a", "inactive"); const auto failed_before = count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED); @@ -3918,18 +4023,15 @@ TEST_F(LifecycleExpectationIntegrationTest, WideGraceWithTheTightestPruneGraceSt std::this_thread::sleep_for(200ms); ASSERT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), failed_before) << "the node was confirmed on its first not-active tick, so grace(4) was never in force and " - "the absence run below proves nothing about a clock that has to keep climbing"; + "the absence run below proves nothing about a below-grace streak"; - // "a" vanishes with a streak of 1 out of 4. Absence must go on advancing it, so it is - // confirmed while GONE - and the tightest prune horizon must not reach it on the way. + // "a" vanishes with a streak of 1 out of 4. Nothing else will ever report this departure, + // so absence itself has to mature the streak. set_apps({}); ASSERT_TRUE(poll_for_new(kGraphSource, ReportFault::Request::EVENT_FAILED, failed_before, *det, ctx)) - << "a node that left the graph with a streak below a wide grace was never confirmed at all - " - "its clock was frozen or its entry pruned while absent, so a node that vanishes while " - "not-active is permanently invisible"; - EXPECT_EQ(det->tracked_count_for_test(), 1u) - << "the entry was pruned by the tightest prune_grace despite carrying evidence"; + << "a below-grace violation on a node the presence detector could never have tracked did " + "not mature from absence alone - its departure is reported by nothing at all"; EXPECT_TRUE(any_failed_desc_contains(kGraphSource, "has since left the graph")) - << "the confirmation did not say the node had left the graph, so an operator is sent looking " - "for a node that is no longer there"; + << "the fault matured from absence must say the node has since left the graph, or the " + "operator is sent looking for a node that is not there"; } diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_expectation_tracker.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_expectation_tracker.cpp index c8fa77492..9e507ef7e 100644 --- a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_expectation_tracker.cpp +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_expectation_tracker.cpp @@ -39,9 +39,12 @@ using ros2_medkit_graph_watchdog::LifecycleMatch; using M = std::vector; // Violations are keyed by the NODE, so a match carries both the config entry that -// matched and the node's stable fqn. -LifecycleMatch match(const std::string & entry, const std::string & fqn, std::optional state) { - return LifecycleMatch{entry, fqn, std::move(state)}; +// matched and the node's stable fqn. `armed` defaults true, matching LifecycleMatch's own +// default: a call that does not pass it exercises a node the presence detector could +// report, and only a test about the never-armed split passes `false` explicitly. +LifecycleMatch match(const std::string & entry, const std::string & fqn, std::optional state, + bool armed = true) { + return LifecycleMatch{entry, fqn, std::move(state), armed}; } // ---- Violation streak: unchanged behaviour from before this slice's redesign ---- @@ -259,37 +262,39 @@ TEST(LifecycleExpectation, AlternatingEveryTickStillMaturesTheClockEventually) { // The pair the two clocks have to divide between them: a node alternating between a REAL // not-active read and a not-managed one, with runs of ABSENCE longer than the absence grace -// in between - so every tick belongs to one of the three cases and the alternation crosses -// both ownership and presence. kNotManaged never touches the violation streak and kInactive -// always resets the unmeasured clock, so the streak is the only clock that can accumulate -// here, and it must: whichever way the node flaps, it was MEASURED not-active repeatedly. -TEST(LifecycleExpectation, InactiveAlternatingWithNotManagedAcrossAbsenceGapsStillConfirmsTheViolation) { +// in between - so every tick belongs to one of the three cases. kNotManaged never touches the +// violation streak (only its own unmeasured clock), kInactive always resets the unmeasured +// clock, and absence below grace contributes to neither (see the kInactive case in update()'s +// absence loop) - so the ONLY thing that can ever advance the violation streak here is a +// genuine matched "inactive" read, and it takes exactly grace + 1 of them, however many +// not-managed legs and absence gaps sit in between. +TEST(LifecycleExpectation, InactiveAlternatingWithNotManagedAcrossAbsenceGapsCountsOnlyMatchedReads) { constexpr int kGap = kDefaultAbsenceGrace + 1; - LifecycleExpectationTracker t({"a"}, /*grace=*/5, kDefaultAbsenceGrace); - bool reported = false; - for (int cycle = 0; cycle < 20 && !reported; ++cycle) { + constexpr int kGrace = 3; + LifecycleExpectationTracker t({"a"}, kGrace, kDefaultAbsenceGrace); + for (int matched_tick = 1; matched_tick <= kGrace; ++matched_tick) { auto measured = t.update(M{match("a", "/a", "inactive")}); - reported = !measured.affected.empty(); - for (int i = 0; i < kGap && !reported; ++i) { - reported = !t.update(M{}).affected.empty(); - } - if (reported) { - break; + EXPECT_TRUE(measured.affected.empty()) + << "matched tick " << matched_tick << " crossed grace(" << kGrace << ") early"; + for (int i = 0; i < kGap; ++i) { + EXPECT_TRUE(t.update(M{}).affected.empty()) + << "matched tick " << matched_tick << ", absence gap " << i << ": crossed on an ABSENT tick"; } auto unmanaged = t.update(M{match("a", "/a", std::nullopt)}); - EXPECT_TRUE(unmanaged.affected.empty()) << "cycle " << cycle - << ": a node inside an unmeasured spell is not confirmed content, " - "however far its held streak had already climbed"; + EXPECT_TRUE(unmanaged.affected.empty()) + << "matched tick " << matched_tick << ": a not-managed read crossed grace on its own"; EXPECT_EQ(unmanaged.pending_violation.count("/a"), 1u) - << "cycle " << cycle << ": the streak was RESET by a not-managed tick instead of merely held"; - for (int i = 0; i < kGap && !reported; ++i) { - reported = !t.update(M{}).affected.empty(); + << "matched tick " << matched_tick << ": the streak was RESET by a not-managed tick instead of held"; + for (int i = 0; i < kGap; ++i) { + EXPECT_TRUE(t.update(M{}).affected.empty()) + << "matched tick " << matched_tick << ", trailing gap " << i << ": crossed on an ABSENT tick"; } } - EXPECT_TRUE(reported) << "a node alternating between a measured not-active read and a not-managed one, " - "separated by absence runs longer than the absence grace, was never confirmed - " - "invisible to GRAPH_NODE_INACTIVE and, since the unmeasured clock keeps being reset " - "by the real reads, to its siblings too"; + auto crossed = t.update(M{match("a", "/a", "inactive")}); // the (grace + 1)-th matched inactive read + EXPECT_EQ(crossed.affected.count("/a"), 1u) + << "never crossed grace after exactly grace + 1 matched inactive reads - the not-managed legs and " + "absence gaps in between must have contributed nothing to either side of the count"; + EXPECT_EQ(crossed.newly_affected, (std::vector{"/a"})); } // The violation streak resets ONLY on kActive. A mutant that zeroed it on every unmeasured @@ -584,26 +589,140 @@ TEST(LifecycleExpectation, NotManagedInterleavedWithAbsenceRunsStillMaturesAndRe } } -TEST(LifecycleExpectation, InactiveInterleavedWithAbsenceRunsStillCrossesGraceAndReports) { +// Companion to UnreadableInterleavedWithAbsenceRunsStillMaturesAndReports and its +// not-managed sibling above, but for the VIOLATION streak the claim is the opposite: an +// absence run - however many, however long, however far past kDefaultAbsenceGrace - must +// contribute NOTHING to a streak that has not yet crossed `grace`. Crossing happens at +// exactly grace + 1 MATCHED "inactive" reads, never earlier and never later, whatever the +// interleaved absence pattern looks like - absence must never mature an unmatured streak, +// pinned directly against the exact shape that used to defeat that rule. +TEST(LifecycleExpectation, InactiveInterleavedWithAbsenceRunsNeverCrossesGraceFromAbsenceAlone) { for (const int run_length : kAbsenceRunLengths) { - LifecycleExpectationTracker t({"a"}, /*grace=*/5, kDefaultAbsenceGrace); - bool reported = false; - for (int cycle = 0; cycle < kDefaultUnmeasuredHoldTicks + 5 && !reported; ++cycle) { + constexpr int kGrace = 3; + LifecycleExpectationTracker t({"a"}, kGrace, kDefaultAbsenceGrace); + for (int matched_tick = 1; matched_tick <= kGrace; ++matched_tick) { auto present = t.update(M{match("a", "/a", "inactive")}); - reported = !present.affected.empty(); + EXPECT_TRUE(present.affected.empty()) << "run_length=" << run_length << ": matched tick " << matched_tick + << " crossed grace(" << kGrace << ") early"; EXPECT_TRUE(present.unreadable_affected.empty() && present.not_managed_affected.empty()) << "run_length=" << run_length << ": a MEASURED read must never feed an unmeasured code"; - for (int i = 0; i < run_length && !reported; ++i) { - reported = !t.update(M{}).affected.empty(); + for (int i = 0; i < run_length; ++i) { + auto absent = t.update(M{}); + EXPECT_TRUE(absent.affected.empty()) << "run_length=" << run_length + << ": crossed grace on an ABSENT tick - a below-grace violation " + "must never mature while nobody can observe the node"; } } - EXPECT_TRUE(reported) << "run_length=" << run_length - << ": a node measured inactive whenever it is present and absent the rest " - "of the time never crossed grace - the violation streak was discarded " - "by every absence run"; + // The (grace + 1)-th matched tick, after grace full interleaved cycles that must have + // contributed nothing: this crosses if and only if every absence run above truly added + // zero to the streak. + auto crossed = t.update(M{match("a", "/a", "inactive")}); + EXPECT_EQ(crossed.affected.count("/a"), 1u) + << "run_length=" << run_length + << ": never crossed after exactly grace + 1 matched ticks - an " + "absence run either stole progress from a match or silently added some of its own"; + EXPECT_EQ(crossed.newly_affected, (std::vector{"/a"})); + } +} + +// ---- Never armed: nothing else can ever report the departure, so absence must mature it ---- + +// The node the row above cannot cover: one the reliability gate never armed at all. Compare +// directly against InactiveInterleavedWithAbsenceRunsNeverCrossesGraceFromAbsenceAlone above - +// same shape, same grace, same absence budget - with the single difference being `armed`. +// There the presence detector could report the departure instead, so absence merely holds a +// below-grace streak; here nothing else in the plugin ever will, so absence has to be the one +// that matures it, or the node's departure is reported by nothing at all. +TEST(LifecycleExpectation, NeverArmedNodeMaturesFromAbsenceAloneAndStaysGone) { + constexpr int kGrace = 5; + constexpr int kAbsenceGrace = 2; + LifecycleExpectationTracker t({"a"}, kGrace, kAbsenceGrace); + ASSERT_TRUE(t.update(M{match("a", "/a", "inactive", /*armed=*/false)}).affected.empty()) + << "sanity: one present tick, streak 1 <= grace"; + + // Node vanishes forever. Inside the blink tolerance nothing moves - a never-armed node + // gets the same blink tolerance as any other. + for (int i = 0; i < kAbsenceGrace; ++i) { + EXPECT_TRUE(t.update(M{}).affected.empty()) << "absence " << i << ": still inside the blink tolerance"; + } + // Past the blink, absence itself keeps the streak climbing. It was 1 after the one present + // tick above, so exactly (kGrace - 1) further absent ticks reach kGrace (still not past it) + // and the next one crosses. + for (int i = 0; i < kGrace - 1; ++i) { + EXPECT_TRUE(t.update(M{}).affected.empty()) << "past the blink, iteration " << i; + } + auto matured = t.update(M{}); + ASSERT_EQ(matured.affected.count("/a"), 1u) + << "a below-grace violation on a node the presence detector could never have reported " + "must still mature from absence alone, or its departure is reported by nothing"; + EXPECT_NE(matured.affected.at("/a").find("has since left the graph"), std::string::npos); + EXPECT_EQ(matured.newly_affected, (std::vector{"/a"})); + + // And it stays matured while the node remains gone - absence never un-matures a violation, + // never-armed or not. + for (int i = 0; i < 5; ++i) { + auto still = t.update(M{}); + EXPECT_EQ(still.affected.count("/a"), 1u) << "iteration " << i; + EXPECT_TRUE(still.newly_affected.empty()) << "iteration " << i << ": no re-raise churn while merely absent"; } } +// The narrowing itself, restated for the case it must keep working for: a node armed once +// (however briefly) before it went non-active. `ever_armed` is sticky, so the CURRENT tick +// reading non-active - which, against a real gate, means NOT currently armed, since a managed +// node's node_ok() is false whenever it reads anything but "active" - must not un-arm it. This +// is the fact GRAPH_NODE_DISAPPEARED needs in order to be ABLE to report this exact node, so +// GRAPH_NODE_INACTIVE must stay out of the way, exactly as +// InactiveInterleavedWithAbsenceRunsNeverCrossesGraceFromAbsenceAlone already pins for the +// unqualified (default-armed) case. +TEST(LifecycleExpectation, ArmedNodeBelowGraceThenGoneStillDoesNotMatureFromAbsence) { + constexpr int kGrace = 5; + constexpr int kAbsenceGrace = 2; + LifecycleExpectationTracker t({"a"}, kGrace, kAbsenceGrace); + ASSERT_TRUE(t.update(M{match("a", "/a", "active", /*armed=*/true)}).affected.empty()) << "sanity: arms the node"; + ASSERT_TRUE(t.update(M{match("a", "/a", "inactive", /*armed=*/false)}).affected.empty()) + << "sanity: one inactive tick, streak 1 <= grace"; + + for (int i = 0; i < kAbsenceGrace + 20; ++i) { + auto report = t.update(M{}); + EXPECT_TRUE(report.affected.empty()) << "iteration " << i + << ": a node the presence detector could report must never have " + "its below-grace streak matured by absence alone"; + EXPECT_EQ(report.pending_violation.count("/a"), 1u) + << "iteration " << i << ": the streak must stay HELD, not erased either"; + } +} + +// A never-armed node that RESUMES rather than restarts: absence-driven progress survives the +// return exactly like a present-tick streak already does (see +// ViolationStreakSurvivesNonMaturingUnmeasuredTicksAndIsNotRestarted for the unmeasured-clock +// analogue), counted exactly enough that a mutant which restarted the streak at zero - either +// on absence or on return - would need more matched ticks than this test gives it. +TEST(LifecycleExpectation, NeverArmedNodeResumesRatherThanRestartingAfterReturning) { + constexpr int kGrace = 5; + constexpr int kAbsenceGrace = 2; + LifecycleExpectationTracker t({"a"}, kGrace, kAbsenceGrace); + t.update(M{match("a", "/a", "inactive", /*armed=*/false)}); // streak 1 + ASSERT_TRUE(t.update(M{match("a", "/a", "inactive", /*armed=*/false)}).affected.empty()); // streak 2 + + // Vanishes long enough to pass the blink and advance on absence alone a few times, but + // stops short of grace: 2 held (absent_ticks 1..kAbsenceGrace) + 1 advancing tick + // (absent_ticks kAbsenceGrace+1, streak 2 -> 3). + for (int i = 0; i < kAbsenceGrace + 1; ++i) { + EXPECT_TRUE(t.update(M{}).affected.empty()) << "absence " << i; + } + + // Node returns, still inactive. If absence above had genuinely advanced the streak to 3 + // (not restarted it), exactly 2 more matched ticks reach grace(5) and the 3rd crosses. + EXPECT_TRUE(t.update(M{match("a", "/a", "inactive", /*armed=*/false)}).affected.empty()) << "streak 3 -> 4"; + EXPECT_TRUE(t.update(M{match("a", "/a", "inactive", /*armed=*/false)}).affected.empty()) << "streak 4 -> 5 == grace"; + auto crossed = t.update(M{match("a", "/a", "inactive", /*armed=*/false)}); // streak 5 -> 6 > grace + EXPECT_EQ(crossed.affected.count("/a"), 1u) + << "the streak restarted at zero across the absence run instead of resuming what absence " + "had already advanced - a node that departs and returns while never armed would then " + "need far longer than grace + 1 present ticks to ever be reported"; +} + // The tightest prune horizon the documented config space can produce (grace: 0 with // prune_grace: 0). A node carrying a LIVE clock must survive it: reclaiming bookkeeping by // age erases evidence, and at this endpoint the age is one tick. @@ -946,35 +1065,74 @@ TEST(LifecycleExpectation, NodeCrossingGraceDoesNotDisturbAnothersAbsenceBookkee EXPECT_EQ(crossed.newly_affected[0], "/a"); } -// Sustained absence CONTINUES the streak instead of discarding it: the node was measured -// not-active, nothing has said otherwise, and being gone is not an answer. -TEST(LifecycleExpectation, SustainedAbsenceContinuesTheStreakAndEventuallyConfirmsIt) { - LifecycleExpectationTracker t({"a"}, /*grace=*/2, /*absence_grace=*/1); - t.update(M{match("a", "/a", "inactive")}); // streak 1 <= grace - EXPECT_TRUE(t.update(M{}).affected.empty()) << "absent 1 == absence_grace: held, not advanced"; - EXPECT_TRUE(t.update(M{}).affected.empty()) << "absent 2 > absence_grace: streak 2 == grace, not past it"; - auto crossed = t.update(M{}); // absent 3: streak 3 > grace - ASSERT_EQ(crossed.affected.count("/a"), 1u) - << "absence discarded the streak instead of continuing it, so a node that leaves while " - "violating is never confirmed"; - EXPECT_EQ(crossed.newly_affected, (std::vector{"/a"})); - EXPECT_NE(crossed.affected.at("/a").find("inactive"), std::string::npos) +// Sustained absence CONTINUES an ALREADY-MATURED streak instead of discarding it: the node +// was CONFIRMED not-active, nothing has said otherwise, and being gone is not an answer. It +// does not create one that was not already there - see +// BelowGraceStreakStaysPendingForeverWhileAbsentAndNeverBecomesContent for the below-grace +// half of the same absence branch. +TEST(LifecycleExpectation, MaturedStreakSurvivesAbsenceUnconditionally) { + LifecycleExpectationTracker t({"a"}, /*grace=*/1, /*absence_grace=*/1); + t.update(M{match("a", "/a", "inactive")}); // streak 1 <= grace + auto confirmed = t.update(M{match("a", "/a", "inactive")}); // streak 2 > grace: confirmed + ASSERT_EQ(confirmed.affected.count("/a"), 1u) << "sanity: must be confirmed before it can survive a departure"; + + t.update(M{}); // absent 1 == absence_grace: held through the blink, unchanged + for (int i = 0; i < 10; ++i) { + auto still = t.update(M{}); // absent 2.. > absence_grace + ASSERT_EQ(still.affected.count("/a"), 1u) + << "iteration " << i + << ": absence discarded an already-confirmed streak instead of continuing " + "it, so a node that leaves while violating is not confirmed any more"; + EXPECT_TRUE(still.newly_affected.empty()) << "iteration " << i << ": no re-raise churn while merely absent"; + } + auto report = t.update(M{}); + const std::string & detail = report.affected.at("/a"); + EXPECT_NE(detail.find("inactive"), std::string::npos) << "the detail must still name the state the node was last measured in"; - EXPECT_NE(crossed.affected.at("/a").find("has since left the graph"), std::string::npos); + EXPECT_NE(detail.find("has since left the graph"), std::string::npos); } -// `pending` gives way to CONTENT, never to silence: the withheld-clear hold ends because -// the tracker finally settled the node's status, not because it gave up on it. -TEST(LifecycleExpectation, PendingBecomesContentWhenAbsenceOutlivesTheAbsenceGrace) { +// The other half: a streak that had NOT yet crossed grace when the node left is held at +// whatever it already reached, resuming rather than restarting. Counted exactly - a streak +// that had restarted at zero would still read `grace` (not past it) after these same three +// return ticks, one short of the four a fresh climb needs. +TEST(LifecycleExpectation, BelowGraceStreakHeldByAbsenceResumesRatherThanRestartsOnReturn) { + constexpr int kGrace = 3; + LifecycleExpectationTracker t({"a"}, kGrace, /*absence_grace=*/1); + auto first = t.update(M{match("a", "/a", "inactive")}); // streak 1 <= grace + ASSERT_EQ(first.pending_violation.count("/a"), 1u); + + for (int i = 0; i < 30; ++i) { + t.update(M{}); // absent, well past absence_grace: held at streak 1 the whole time + } + + EXPECT_TRUE(t.update(M{match("a", "/a", "inactive")}).affected.empty()) << "resumed streak 2 <= grace"; + EXPECT_TRUE(t.update(M{match("a", "/a", "inactive")}).affected.empty()) << "resumed streak 3 == grace"; + auto confirmed = t.update(M{match("a", "/a", "inactive")}); // resumed streak 4 > grace + ASSERT_EQ(confirmed.affected.count("/a"), 1u) + << "the streak restarted from zero on return instead of resuming where the absence had held it"; + EXPECT_EQ(confirmed.newly_affected, (std::vector{"/a"})); +} + +// `pending` NEVER becomes content merely by waiting: a below-grace streak that goes absent +// stays exactly where it was for as long as the node is gone, so the withheld-clear hold has +// no timeout of its own - it ends only when the tracker actually SETTLES the node's status (a +// fresh present tick reading active or inactive), never because enough absent ticks went by. +// Maturing a below-grace streak on absence alone would raise GRAPH_NODE_INACTIVE from ticks +// gathered while nobody could observe the node at all. +TEST(LifecycleExpectation, BelowGraceStreakStaysPendingForeverWhileAbsentAndNeverBecomesContent) { LifecycleExpectationTracker t({"a"}, /*grace=*/3, /*absence_grace=*/2); - EXPECT_EQ(t.update(M{match("a", "/a", "inactive")}).pending.count("/a"), 1u); // streak 1 - EXPECT_EQ(t.update(M{}).pending.count("/a"), 1u) << "absent 1: inside the blink tolerance, held"; - EXPECT_EQ(t.update(M{}).pending.count("/a"), 1u) << "absent 2 == absence_grace: still held"; - EXPECT_EQ(t.update(M{}).pending.count("/a"), 1u) << "absent 3: streak 2, still below grace"; - EXPECT_EQ(t.update(M{}).pending.count("/a"), 1u) << "absent 4: streak 3 == grace, still below"; - auto settled = t.update(M{}); // absent 5: streak 4 > grace - EXPECT_TRUE(settled.pending.empty()) << "the hold must end by SETTLING, not by discarding"; - EXPECT_EQ(settled.affected.count("/a"), 1u); + EXPECT_EQ(t.update(M{match("a", "/a", "inactive")}).pending.count("/a"), 1u); // streak 1 <= grace + + for (int i = 0; i < 50; ++i) { + auto report = t.update(M{}); // absent, past absence_grace for every iteration but the first two + EXPECT_EQ(report.pending.count("/a"), 1u) << "iteration " << i + << ": the hold ended without the node ever being measured active or " + "confirmed inactive"; + EXPECT_EQ(report.pending_violation.count("/a"), 1u) << "iteration " << i; + EXPECT_TRUE(report.affected.empty()) << "iteration " << i + << ": a below-grace streak matured purely from waiting while absent"; + } } // A node the detector already reported keeps its content through a blink AND past it - it @@ -1204,18 +1362,29 @@ TEST(LifecycleExpectation, CorroboratedUnmeasuredRunBeforeADepartureIsStillRepor } // A real measurement needs no corroboration at all: one not-active read is a fact about the -// node, so a node that departs immediately after it still confirms. Without this the -// settling rule would swallow the very case the detector exists for at grace: 0. +// node, so it is content the instant it is read (grace: 0 makes that one tick the crossing +// tick), and a node that departs immediately after keeps that content unconditionally, the +// same as any other already-matured entry. Without instant settling for a real measurement, +// the corroboration rule built for the unmeasured clock would swallow the very case grace: 0 +// exists for. grace: 0 also keeps this test meaningful now that absence alone may not mature +// a below-grace streak: at any grace > 0 a single read leaves the streak BELOW grace, and a +// below-grace streak is now held rather than +// confirmed by a departure - see BelowGraceStreakStaysPendingForeverWhileAbsentAndNever +// BecomesContent for that (deliberately different) claim. TEST(LifecycleExpectation, OneMeasuredNotActiveReadBeforeADepartureStillConfirms) { - LifecycleExpectationTracker t({"a"}, /*grace=*/2); - t.update(M{match("a", "/a", "inactive")}); // exactly one real measurement, then gone - bool confirmed = false; - for (int i = 0; i < kDefaultAbsenceGrace + 10 && !confirmed; ++i) { - confirmed = !t.update(M{}).affected.empty(); + LifecycleExpectationTracker t({"a"}, /*grace=*/0); + auto confirmed_before_leaving = t.update(M{match("a", "/a", "inactive")}); // grace 0: confirmed on this tick + ASSERT_EQ(confirmed_before_leaving.affected.count("/a"), 1u) + << "sanity: the node must already be confirmed before it departs, or nothing below is tested"; + + for (int i = 0; i < kDefaultAbsenceGrace + 10; ++i) { + auto report = t.update(M{}); // gone, immediately + ASSERT_EQ(report.affected.count("/a"), 1u) + << "iteration " << i + << ": a node measured not-active once, already confirmed, and then gone lost " + "its fault - a lifecycle label is a measurement, not something a sweep can invent, so it needs " + "no corroborating, and a departure must not un-confirm it either"; } - EXPECT_TRUE(confirmed) << "a node measured not-active once and then gone was never confirmed - a " - "lifecycle label is a measurement, not something a sweep can invent, so it " - "needs no corroborating"; } // ---- The cap: a present node always wins a slot ---- @@ -1298,6 +1467,36 @@ TEST(LifecycleExpectation, ANodeReturningAfterItsEntryWasCollapsedIsMeasuredAfre << AggregatedFault::describe(returned.affected); } +// The one cap/collapse shape no OTHER test in this file can tell apart: every sibling above +// uses grace=0, where violation_streak > 0 and violation_streak > grace_ are the SAME +// condition, so they cannot distinguish count_collapsed() counting "any non-zero streak" from +// counting "only an already-matured one". Here grace is wide enough that the departed entry +// is genuinely BELOW it when the cap forces its collapse: under the design this replaces, +// absence kept advancing it regardless, so folding it into collapsed_inactive_ merely +// anticipated a maturity it would have reached anyway. That is no longer true now that +// absence alone never matures a below-grace streak - so collapsing it as content would +// fabricate a violation the node never earned. +TEST(LifecycleExpectation, BelowGraceDepartedEntryCollapsedAtTheCapContributesNothingToTheCount) { + constexpr int kCap = 1; + LifecycleExpectationTracker t({"/held", "/live"}, /*grace=*/5, /*absence_grace=*/1, kDefaultNoMatchWarnTicks, + LifecycleExpectationTracker::kNoPrune, kDefaultUnmeasuredHoldTicks, kCap); + t.update(M{match("/held", "/held", "inactive")}); // streak 1, well below grace(5) + ASSERT_EQ(t.tracked_count(), 1u); + for (int i = 0; i < 3; ++i) { + t.update(M{}); // "/held" leaves for good, past the absence grace - HELD, never matures + } + + // "/live" needs the only slot. "/held" is departed, so it is collapsed rather than "/live" + // being refused - but "/held" was never content, so nothing may be fabricated for it. + auto report = t.update(M{match("/live", "/live", "inactive")}); + ASSERT_EQ(t.tracked_count(), 1u) << "the slot was never freed - a departed entry blocked a present node"; + EXPECT_FALSE(report.tracking_saturated) << "a departed, below-grace entry blocked a present node's slot"; + EXPECT_TRUE(report.affected.empty()) + << "the collapsed below-grace entry fabricated a violation it never earned ('/live' itself is " + "only at streak 1, well below grace(5)): " + << AggregatedFault::describe(report.affected); +} + // ---- Entries matching nothing: unrelated per-entry mechanism, unaffected by this slice ---- TEST(LifecycleExpectation, EntryMatchingNothingIsReportedOnce) { diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_shutdown_suppressor.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_shutdown_suppressor.cpp new file mode 100644 index 000000000..d2dfd3fc6 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_lifecycle_shutdown_suppressor.cpp @@ -0,0 +1,122 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Drives a REAL ReliabilityGate (needs a real rclcpp::Node for its LifecycleWatcher), fed +// through its set_departed_lifecycle_state_for_test() seam rather than a live managed +// node - that seam exists precisely so a suppressor test can drive +// departed_lifecycle_state_of() without a real GetState/transition_event round trip. +#include + +#include +#include +#include + +#include + +#include "ros2_medkit_graph_watchdog/lifecycle_shutdown_suppressor.hpp" +#include "ros2_medkit_graph_watchdog/reliability_gate.hpp" + +using ros2_medkit_graph_watchdog::DetectorContext; +using ros2_medkit_graph_watchdog::LifecycleShutdownSuppressor; +using ros2_medkit_graph_watchdog::ReliabilityGate; + +class LifecycleShutdownSuppressorTest : public ::testing::Test { + protected: + static void SetUpTestSuite() { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + } + + void SetUp() override { + node_ = std::make_shared("lifecycle_shutdown_suppressor_test_node"); + gate_ = std::make_unique(0, node_.get(), &mtx_); + } + + void TearDown() override { + gate_.reset(); + node_.reset(); + } + + DetectorContext ctx_with_gate() { + DetectorContext ctx; + ctx.gate = gate_.get(); + return ctx; + } + + rclcpp::Node::SharedPtr node_; + std::mutex mtx_; + std::unique_ptr gate_; +}; + +TEST_F(LifecycleShutdownSuppressorTest, NullGateAbstains) { + LifecycleShutdownSuppressor s; + DetectorContext ctx; // ctx.gate stays nullptr + EXPECT_FALSE(s.suppresses("/anything", ctx)); +} + +TEST_F(LifecycleShutdownSuppressorTest, AFqnThatNeverDepartedAbstains) { + LifecycleShutdownSuppressor s; + EXPECT_FALSE(s.suppresses("/never/departed", ctx_with_gate())); +} + +TEST_F(LifecycleShutdownSuppressorTest, ShuttingdownLabelAloneSuppresses) { + LifecycleShutdownSuppressor s; + gate_->set_departed_lifecycle_state_for_test("/clean", "shuttingdown"); + EXPECT_TRUE(s.suppresses("/clean", ctx_with_gate())); +} + +TEST_F(LifecycleShutdownSuppressorTest, FinalizedWithAnObservedTransitionAndNoErrorSuppresses) { + LifecycleShutdownSuppressor s; + gate_->set_departed_lifecycle_state_for_test("/clean", "finalized", /*saw_transition=*/true, + /*error_terminated=*/false); + EXPECT_TRUE(s.suppresses("/clean", ctx_with_gate())); +} + +TEST_F(LifecycleShutdownSuppressorTest, FinalizedWithoutAnObservedTransitionIsNotSuppressed) { + // An unobserved "finalized" cannot be told apart from a node that failed on_error before + // this watcher ever saw a transition - the departure stays unclassified, so it is + // reported rather than trusted. + LifecycleShutdownSuppressor s; + gate_->set_departed_lifecycle_state_for_test("/unseen", "finalized", /*saw_transition=*/false, + /*error_terminated=*/false); + EXPECT_FALSE(s.suppresses("/unseen", ctx_with_gate())); +} + +TEST_F(LifecycleShutdownSuppressorTest, FinalizedThroughTheErrorBranchIsNotSuppressed) { + LifecycleShutdownSuppressor s; + gate_->set_departed_lifecycle_state_for_test("/crashed", "finalized", /*saw_transition=*/true, + /*error_terminated=*/true); + EXPECT_FALSE(s.suppresses("/crashed", ctx_with_gate())); +} + +TEST_F(LifecycleShutdownSuppressorTest, UnconfiguredIsNeverTreatedAsClean) { + // unconfigured is also the resting state of a failed configure() or a node that never + // activated at all - treating it as clean would hide exactly that startup failure. + LifecycleShutdownSuppressor s; + gate_->set_departed_lifecycle_state_for_test("/never/started", "unconfigured"); + EXPECT_FALSE(s.suppresses("/never/started", ctx_with_gate())); +} + +TEST_F(LifecycleShutdownSuppressorTest, InactiveIsNotSuppressed) { + // A managed node paused mid-lifecycle (not shut down at all) is not a clean departure. + LifecycleShutdownSuppressor s; + gate_->set_departed_lifecycle_state_for_test("/paused", "inactive"); + EXPECT_FALSE(s.suppresses("/paused", ctx_with_gate())); +} + +TEST_F(LifecycleShutdownSuppressorTest, IsDurable) { + LifecycleShutdownSuppressor s; + EXPECT_TRUE(s.durable()); +} diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_node_death_integration.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_node_death_integration.cpp new file mode 100644 index 000000000..69c905b43 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_node_death_integration.cpp @@ -0,0 +1,1261 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Drives the REAL mechanism: a real NodeDeathDetector (pulled from the registry, exactly +// as test_qos_mismatch_integration.cpp and test_lifecycle_expectation_integration.cpp do +// for their own detectors) against hand-built IntrospectionInput snapshots and a REAL +// ReliabilityGate, raising/clearing through a real ReportFault service round-trip to a +// fake fault_manager. NOT a full-gateway e2e; this proves the detector + suppression +// framework end to end within their own scope. +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "ros2_medkit_gateway/core/providers/introspection_provider.hpp" +#include "ros2_medkit_graph_watchdog/detector.hpp" +#include "ros2_medkit_graph_watchdog/detector_config_keys.hpp" +#include "ros2_medkit_graph_watchdog/detector_registry.hpp" +#include "ros2_medkit_graph_watchdog/graph_fault_codes.hpp" +#include "ros2_medkit_graph_watchdog/reliability_gate.hpp" + +using ros2_medkit_gateway::App; +using ros2_medkit_gateway::IntrospectionInput; +using ros2_medkit_graph_watchdog::Detector; +using ros2_medkit_graph_watchdog::DetectorContext; +using ros2_medkit_graph_watchdog::DetectorMode; +using ros2_medkit_graph_watchdog::DetectorRegistry; +using ros2_medkit_graph_watchdog::ReliabilityGate; +using ros2_medkit_graph_watchdog::graph_fault_codes::kNodeDisappeared; +using ReportFault = ros2_medkit_msgs::srv::ReportFault; +using namespace std::chrono_literals; + +namespace { +// The detector class is file-local in node_death_detector.cpp but self-registers via +// REGISTER_DETECTOR, which runs when that .cpp is linked into this test. +std::unique_ptr make_node_death() { + for (auto & d : DetectorRegistry::instance().create_all()) { + if (d->id() == "node_death") { + return std::move(d); + } + } + return nullptr; +} + +// Aggregated fault source: no Component in the test snapshot, so graph_source_id() falls +// back to this literal (see aggregated_fault.hpp). +constexpr const char * kGraphSource = "graph_watchdog"; + +/// Captures rcutils log output for as long as it is alive and restores the console handler +/// on every exit path. +class LogCapture { + public: + LogCapture() { + active().store(this); + rcutils_logging_set_output_handler(&LogCapture::handler); + } + ~LogCapture() { + rcutils_logging_set_output_handler(rcutils_logging_console_output_handler); + active().store(nullptr); + } + LogCapture(const LogCapture &) = delete; + LogCapture & operator=(const LogCapture &) = delete; + LogCapture(LogCapture &&) = delete; + LogCapture & operator=(LogCapture &&) = delete; + + int count(const std::string & needle) const { + std::lock_guard lk(mutex_); + int n = 0; + for (const auto & line : lines_) { + if (line.find(needle) != std::string::npos) { + ++n; + } + } + return n; + } + + private: + static std::atomic & active() { + static std::atomic current{nullptr}; + return current; + } + + static void handler(const rcutils_log_location_t * /*location*/, int /*severity*/, const char * /*name*/, + rcutils_time_point_value_t /*timestamp*/, const char * format, va_list * args) { + char buf[1024]; + va_list copy; + va_copy(copy, *args); + // The format string and args arrive from the logging call site through the handler + // signature - there is no literal to write here. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-nonliteral" + vsnprintf(buf, sizeof(buf), format, copy); +#pragma GCC diagnostic pop + va_end(copy); + LogCapture * capture = active().load(); + if (capture == nullptr) { + return; + } + std::lock_guard lk(capture->mutex_); + capture->lines_.emplace_back(buf); + } + + mutable std::mutex mutex_; + std::vector lines_; +}; +} // namespace + +class NodeDeathIntegrationTest : public ::testing::Test { + protected: + // The ROS plumbing (both nodes, the executor, its spin thread, the fake service and the + // client) is built ONCE for this whole binary, in SetUpTestSuite() - not per TEST_F, the + // way every sibling integration test in this package builds it. None of this file's ~27 + // cases needs its OWN node identity: what makes a case independent is its own detector + // instance and its own config, both already fresh per test, and the fixture's own + // received_/snapshot_ state, cleared in SetUp() below. A full rclcpp::Node + executor + + // OS thread create/destroy cycle is comparatively expensive and asynchronous on the DDS + // side, and every sibling integration test in this package happens to run few enough + // cases, each spending multiple SECONDS of its own on polling loops, that the cost never + // shows up. This file's cases run in single-digit milliseconds with nothing to wait on, + // so cycling that same plumbing ~27 times over in a few hundred milliseconds is a rate no + // other test in this package produces - and doing so measurably stalled this binary + // indefinitely at an arbitrary case, confirmed to move to a different case (never a fixed + // one) across repeated runs of the identical binary with nothing else on the machine. + // Building it once removes the cycling entirely rather than merely slowing it down. + static void SetUpTestSuite() { + if (!rclcpp::ok()) { + rclcpp::init(0, nullptr); + } + gateway_ = std::make_shared("nd_it_gateway"); + sink_ = std::make_shared("nd_it_sink"); + srv_ = sink_->create_service( + "/fault_manager/report_fault", + [](const std::shared_ptr & req, const std::shared_ptr & resp) { + { + std::lock_guard lk(mtx_); + received_.push_back(*req); + } + resp->accepted = true; // ReportFault.srv response field is `bool accepted` + }); + client_ = gateway_->create_client("/fault_manager/report_fault"); + // Constructed here, not as a default-initialized static member: MultiThreadedExecutor's + // constructor needs a valid rclcpp context (it creates its own guard condition), and a + // static member's default initializer runs at static-initialization time - before + // main(), so before rclcpp::init() above ever has a chance to run. + exec_ = std::make_unique(rclcpp::ExecutorOptions(), 2); + exec_->add_node(gateway_); + exec_->add_node(sink_); + spin_ = std::thread([]() { + exec_->spin(); + }); + ASSERT_TRUE(client_->wait_for_service(5s)); + } + + static void TearDownTestSuite() { + exec_->cancel(); + if (spin_.joinable()) { + spin_.join(); + } + exec_->remove_node(gateway_); + exec_->remove_node(sink_); + exec_.reset(); + client_.reset(); + srv_.reset(); + sink_.reset(); + gateway_.reset(); + } + + // Per-test isolation for the shared plumbing above: a stale response delivered late from + // a PREVIOUS case must never be read as evidence by the NEXT one. + void SetUp() override { + std::lock_guard lk(mtx_); + received_.clear(); + } + + DetectorContext make_ctx(ReliabilityGate * gate) { + DetectorContext ctx; + ctx.gateway_node = gateway_.get(); + ctx.node_mutex = &node_mutex_; + ctx.mode = DetectorMode::Raise; + ctx.gate = gate; + ctx.fault_client = client_; + ctx.snapshot = &snapshot_; + return ctx; + } + + static App app_of(const std::string & id, bool online = true, const std::string & source = "runtime") { + App a; + a.id = id; + a.is_online = online; + a.source = source; + a.bound_fqn = "/" + id; + return a; + } + + /// A permanently-present, never-under-test App any case whose subject departs must still + /// carry in its snapshot alongside the departure. + /// + /// A live gateway's entity snapshot is never actually empty just because one node exits - + /// the graph_watchdog App itself, the fault_manager's own node and whatever else is + /// running are still in it. ReliabilityGate::update() reads that as a real signal: an + /// EMPTY snapshot re-arms the global bringup grace (graph_seen_ = false), documented and + /// deliberate for a genuine full-stack restart. The aggregated fault's own source_id + /// ("graph_watchdog", see graph_source_id()) is never itself an app in any test snapshot, + /// so it is always evaluated through that SAME global grace inside raise_fault() - and a + /// snapshot that drops to truly empty the instant the app under test departs would + /// silently re-gate every raise this file checks for, for a reason that has nothing to do + /// with node_death or any suppressor. Real gateways never produce that snapshot shape, so + /// no test here should either. + static App anchor_app() { + return app_of("anchor"); + } + + void set_apps(const std::vector & apps) { + snapshot_.apps = apps; + } + + std::size_t count_faults(const std::string & source_id, uint8_t event_type) { + std::lock_guard lk(mtx_); + std::size_t n = 0; + for (const auto & r : received_) { + if (r.source_id == source_id && r.fault_code == kNodeDisappeared && r.event_type == event_type) { + ++n; + } + } + return n; + } + + bool any_failed_desc_contains(const std::string & source_id, const std::vector & needles) { + std::lock_guard lk(mtx_); + for (const auto & r : received_) { + if (r.source_id != source_id || r.fault_code != kNodeDisappeared || + r.event_type != ReportFault::Request::EVENT_FAILED) { + continue; + } + bool all = true; + for (const auto & needle : needles) { + if (r.description.find(needle) == std::string::npos) { + all = false; + break; + } + } + if (all) { + return true; + } + } + return false; + } + + /// Configure a fresh node_death, tick it once with an empty snapshot against a real + /// node, and report whether the captured log carries `needle` - the shared shape every + /// C1/C2/C3 case below needs. + bool configure_warns(const nlohmann::json & config, const std::string & needle) { + auto det = make_node_death(); + if (!det) { + return false; + } + det->configure(config); + LogCapture capture; + IntrospectionInput empty; + DetectorContext ctx; + ctx.gateway_node = gateway_.get(); + ctx.snapshot = ∅ + det->tick(ctx); + return capture.count(needle) > 0; + } + + // Suite-scoped ROS plumbing (see SetUpTestSuite()'s own doc for why this is not + // per-fixture-instance). `inline` (C++17): each is defined here, once, with no separate + // out-of-class definition. The executor's thread count is explicit, not the default (0): + // MultiThreadedExecutor's own header documents 0 as "the number of cpu cores found + // (minimum of 2)" - this fixture's actual concurrency need is at most two (the fake + // service callback and the client's own response callback). + static inline rclcpp::Node::SharedPtr gateway_, sink_; + static inline rclcpp::Service::SharedPtr srv_; + static inline rclcpp::Client::SharedPtr client_; + // unique_ptr, not a by-value member: see SetUpTestSuite()'s own note on why this cannot + // be constructed via a default member initializer. + static inline std::unique_ptr exec_; + static inline std::thread spin_; + static inline std::mutex mtx_; // guards received_ (both the service callback and every reader below) + static inline std::vector received_; + + // Per-test state: fresh for every TEST_F, unlike the plumbing above. + std::mutex node_mutex_; + IntrospectionInput snapshot_; +}; + +// ============================================================================================= +// N9: peer-aggregated apps are never tracked. +// ============================================================================================= + +TEST_F(NodeDeathIntegrationTest, N9_PeerAggregatedAppsAreNeverTracked) { + ReliabilityGate gate(/*warmup_cycles=*/0, gateway_.get(), &node_mutex_); + auto det = make_node_death(); + ASSERT_TRUE(det); + det->configure({{"miss_grace", 0}}); + auto ctx = make_ctx(&gate); + + // No anchor app here: this test's own assertions are all negative + // (tracked_count_for_test()==0, no FAILED report) and so are unaffected by whether + // ReliabilityGate's global bringup grace re-arms on an empty snapshot - see anchor_app()'s + // own doc for why a case that checks a POSITIVE raise needs one and this one does not. + set_apps({app_of("peer_app", /*online=*/true, /*source=*/"peer:robot2")}); + for (int i = 1; i <= 5; ++i) { + gate.update(snapshot_, static_cast(i)); + det->tick(ctx); + } + EXPECT_EQ(det->tracked_count_for_test(), 0u) << "a peer app must never enter the tracked map"; + + set_apps({}); // peer app "departs" - if it had ever been tracked, this would report it dead + for (int i = 6; i <= 12; ++i) { + gate.update(snapshot_, static_cast(i)); + det->tick(ctx); + std::this_thread::sleep_for(5ms); + } + std::this_thread::sleep_for(50ms); + EXPECT_EQ(det->tracked_count_for_test(), 0u); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u); +} + +TEST_F(NodeDeathIntegrationTest, IsOnlineFalseIsNeverTracked) { + // The manifest/hybrid case the class doc describes: an App present in the snapshot but + // not (yet) online must never be armed, however many sweeps see it. + ReliabilityGate gate(/*warmup_cycles=*/0, gateway_.get(), &node_mutex_); + auto det = make_node_death(); + ASSERT_TRUE(det); + det->configure({{"miss_grace", 0}}); + auto ctx = make_ctx(&gate); + + set_apps({app_of("manifest_app", /*online=*/false)}); + for (int i = 1; i <= 10; ++i) { + gate.update(snapshot_, static_cast(i)); + det->tick(ctx); + } + EXPECT_EQ(det->tracked_count_for_test(), 0u); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u); +} + +// ============================================================================================= +// status_json() exposes tracked_count on GET /x-medkit-watchdog (detectors.node_death) - the +// field the ros2cli_ignored e2e row reads to prove tracked state does not grow across renamed- +// node churn cycles. Without this override the field is simply absent (Detector's own default), +// which that row's own comment treats as a condition to skip the comparison under rather than +// fail on - so this is required for that row to check anything at all. +// ============================================================================================= + +TEST_F(NodeDeathIntegrationTest, StatusJsonExposesTrackedCountMatchingTheTestSeam) { + ReliabilityGate gate(/*warmup_cycles=*/0, gateway_.get(), &node_mutex_); + auto det = make_node_death(); + ASSERT_TRUE(det); + det->configure({{"miss_grace", 0}}); + auto ctx = make_ctx(&gate); + + // Before any tick: present from construction, not only after the first sweep. + auto status = det->status_json(); + ASSERT_TRUE(status.is_object()); + ASSERT_TRUE(status.contains("tracked_count")); + EXPECT_EQ(status.at("tracked_count").get(), 0u); + + set_apps({app_of("a"), app_of("b")}); + gate.update(snapshot_, 1); + det->tick(ctx); + + status = det->status_json(); + EXPECT_EQ(status.at("tracked_count").get(), 2u); + EXPECT_EQ(status.at("tracked_count").get(), det->tracked_count_for_test()); +} + +TEST_F(NodeDeathIntegrationTest, StatusJsonTrackedCountStaysStableAcrossNeverArmingChurn) { + // Mirrors the e2e row's own claim: N churn cycles of never-armed apps (present for one + // tick each, then gone) must never move tracked_count - the SAME property N11 pins + // through tracked_count_for_test(), checked here through the PUBLIC status route instead. + ReliabilityGate gate(/*warmup_cycles=*/3, gateway_.get(), &node_mutex_); + auto det = make_node_death(); + ASSERT_TRUE(det); + det->configure({{"miss_grace", 0}}); + auto ctx = make_ctx(&gate); + + set_apps({app_of("survivor")}); + for (std::uint64_t t = 1; t <= 4; ++t) { // 4 ticks: armed at t=4 (4-1>=3) + gate.update(snapshot_, t); + det->tick(ctx); + } + const auto before = det->status_json().at("tracked_count").get(); + ASSERT_EQ(before, 1u); + + for (int cycle = 0; cycle < 5; ++cycle) { + set_apps({app_of("survivor"), app_of("churn_" + std::to_string(cycle))}); + gate.update(snapshot_, static_cast(5 + cycle)); + det->tick(ctx); + } + + const auto after = det->status_json().at("tracked_count").get(); + EXPECT_EQ(before, after) << "five renamed-node cycles must not move tracked_count"; +} + +// ============================================================================================= +// N11: the tracked map stays bounded under churn, and shrinks once a durable veto reclaims +// it (also corroborating S5's "a durable veto may reclaim bookkeeping" through the real +// detector, not just the pure tracker - test_node_liveness_tracker.cpp's own +// NodeLivenessTrackerPrune cases prove both halves of S5 precisely, at the framework level). +// ============================================================================================= + +TEST_F(NodeDeathIntegrationTest, N11_TrackedCountStaysBoundedUnderChurnAndShrinksAfterReclaim) { + ReliabilityGate gate(/*warmup_cycles=*/3, gateway_.get(), &node_mutex_); + auto det = make_node_death(); + ASSERT_TRUE(det); + det->configure({{"tick_interval_ms", 3000}, // floor(3000ms) == 0, so miss_grace=0 is legal + {"miss_grace", 0}, + {"prune_grace", 2}, + {"allowlist", nlohmann::json::array({"/victim"})}, + {"suppress", nlohmann::json::array({"allowlist"})}}); + auto ctx = make_ctx(&gate); + + std::size_t max_seen = 0; + std::uint64_t tick = 0; + const auto run_tick = [&](const std::vector & apps) { + ++tick; + set_apps(apps); + gate.update(snapshot_, tick); + det->tick(ctx); + max_seen = std::max(max_seen, det->tracked_count_for_test()); + }; + + // Arm the two survivors. is_armed() requires (tick - first_seen) >= warmup_cycles, so + // first_seen's own tick (1) plus warmup_cycles(3) MORE ticks - 4 calls, not 3 - before the + // gate considers them armed (see WarmupTracker::is_armed()). + for (int i = 0; i < 4; ++i) { + run_tick({app_of("alive_a"), app_of("alive_b")}); + } + ASSERT_EQ(det->tracked_count_for_test(), 2u); + + // 100 ticks of churn. alive_a/alive_b stay present throughout. "victim" is present for + // the first 4 ticks (enough to arm under warmup_cycles=3: present at ticks 5-8, armed at + // tick 8 where 8-5>=3), then permanently gone. Every tick also carries ONE uniquely-named + // "noise" app present for exactly that one tick alone and never again - well under + // warmup_cycles, so none of the ~100 distinct noise apps can ever arm. + for (int i = 0; i < 100; ++i) { + std::vector apps{app_of("alive_a"), app_of("alive_b"), app_of("noise_" + std::to_string(i))}; + if (i < 4) { + apps.push_back(app_of("victim")); + } + run_tick(apps); + } + + EXPECT_LE(max_seen, 3u) << "noise apps (never armed) and the victim's own eventual reclaim " + "must never push tracked_count past the 3 entities that were ever " + "genuinely armed"; + EXPECT_GE(max_seen, 3u) << "the victim must have been tracked at some point, or this test " + "would trivially pass by never tracking it at all"; + + // prune_ticks = max(prune_grace=2, miss_grace(0)+1=1) = 2. The victim, durably suppressed + // by the allowlist on every tick it is dead, must be reclaimed well before the churn loop + // above even finished - confirmed here with a few more clean ticks. + for (int i = 0; i < 10; ++i) { + run_tick({app_of("alive_a"), app_of("alive_b")}); + } + EXPECT_EQ(det->tracked_count_for_test(), 2u) << "the allowlisted, durably-suppressed victim must have been reclaimed"; + std::this_thread::sleep_for(50ms); + EXPECT_FALSE(any_failed_desc_contains(kGraphSource, {"/victim"})) + << "an allowlisted death must never be reported in the first place"; +} + +// N11's own churn above never varies scale past NodeLivenessTracker's tracked_key_cap - its +// noise apps stay well under warmup_cycles, so none of them are ever armed at all, and the +// one entry that IS tracked and reclaimed (/victim) is reclaimed via prune()'s +// suppression-streak path, not the cap. An unsuppressed death has NO other reclaim path - +// prune() requires a durable suppressor - so this row exists specifically to prove the cap +// itself engages end to end through configure(), not merely inside the tracker in isolation +// (see test_node_liveness_tracker.cpp's own NodeLivenessTrackerCap suite for that). +TEST_F(NodeDeathIntegrationTest, N11_TrackedNodeCapBoundsUnsuppressedChurnAndKeepsTheFaultRaised) { + ReliabilityGate gate(/*warmup_cycles=*/0, gateway_.get(), &node_mutex_); + auto det = make_node_death(); + ASSERT_TRUE(det); + det->configure({{"tick_interval_ms", 3000}, {"miss_grace", 0}, {"tracked_node_cap", 3}}); + auto ctx = make_ctx(&gate); + + std::size_t max_seen = 0; + std::uint64_t tick = 0; + const auto run_tick = [&](const std::vector & apps) { + ++tick; + set_apps(apps); + gate.update(snapshot_, tick); + det->tick(ctx); + max_seen = std::max(max_seen, det->tracked_count_for_test()); + }; + + // 20 distinct, unsuppressed identities - no allowlist, no suppress configured. Each is + // armed on its own tick (PRESENT, so it never competes for the departed cap that tick), + // then given a tick fully absent, where it matures instantly (miss_grace=0) and becomes a + // departed-cap candidate. warmup_cycles=0 makes the fully-empty ticks harmless - the + // global bringup grace re-arms on the very next non-empty tick regardless. + for (int i = 0; i < 20; ++i) { + run_tick({app_of("churn_" + std::to_string(i))}); + run_tick({}); + } + + // kCap + 1, not kCap: a present/armed key is never refused a slot or evicted for one (see + // node_liveness_tracker.hpp's class doc) - only the DEPARTED subset is bounded. This + // churn's own admission tick therefore transiently coexists with a departed set already + // at the cap: kCap matured entries plus the one brand-new present arrival, until THAT one + // departs on the very next tick and the departed set's own collapse brings it back to + // kCap. Without the cap this would grow past 20, unbounded for the life of the process. + constexpr std::size_t kCap = 3; + EXPECT_LE(max_seen, kCap + 1) << "tracked_node_cap must actually engage under scale past it"; + std::this_thread::sleep_for(50ms); + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, {"more node(s) disappeared"})) + << "identities the cap forced out of individual tracking must still count as content, " + "or a death the cap collapsed would silently heal instead of staying raised"; +} + +// ============================================================================================= +// Cap redesign regressions. Both sit PAST tracked_node_cap, in the two directions a cap that +// bounds the wrong thing gets wrong: a graph LARGER than the cap (every death must still be +// reported), and departed-set CHURN below miss_grace (capacity pressure must never fabricate a +// death). See node_liveness_tracker.hpp's own class doc for the ruling these pin. +// ============================================================================================= + +TEST_F(NodeDeathIntegrationTest, CapC1_GraphLargerThanTheDefaultCapStillReportsADeathAmongItsPresentMajority) { + // Reproduces the sequence that exposes the defect: at the shipped default tracked_node_cap + // (512), 513 armed nodes used to leave only one survivor once make_room() evicted every + // present entry to admit the newcomer - and evicting a present entry can permanently lose + // any death landing in the eviction window, because only ONLINE nodes re-enter `armed`. + ReliabilityGate gate(/*warmup_cycles=*/0, gateway_.get(), &node_mutex_); + auto det = make_node_death(); + ASSERT_TRUE(det); + det->configure({{"tick_interval_ms", 3000}, {"miss_grace", 0}}); // default tracked_node_cap (512) + auto ctx = make_ctx(&gate); + + // 513 armed, present nodes - one more than the default cap - lexically ordered zero-padded + // ids so the very first admitted is also lexically first, reproducing the same eviction + // order the defect above depends on. + std::vector apps; + apps.reserve(514); + for (int i = 0; i < 513; ++i) { + std::string idx = std::to_string(i); + idx.insert(idx.begin(), 3 - idx.size(), '0'); + apps.push_back(app_of("n" + idx)); + } + apps.push_back(anchor_app()); + set_apps(apps); + gate.update(snapshot_, 1); + det->tick(ctx); + ASSERT_EQ(det->tracked_count_for_test(), 514u) + << "every armed/present node must be tracked - none refused for capacity, however many " + "there are"; + + // Kill only the first one; every other node (512 of them) plus the anchor stay present. + apps.erase(apps.begin()); + set_apps(apps); + gate.update(snapshot_, 2); + det->tick(ctx); // misses(1) > miss_grace(0): dead + + std::this_thread::sleep_for(50ms); + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, {"n000"})) + << "the one node that actually died among 513 present ones must still be reported - a " + "cap that evicts present entries to make room can lose exactly this death"; +} + +TEST_F(NodeDeathIntegrationTest, CapC2_ChurnUnderCapPressureNeverFabricatesADeathBeforeMissGrace) { + // Reproduces the sequence that exposes the defect: at cap(2) with three entries carrying + // only one below-grace miss each, a make_room() that collapsed every non-idle entry used + // to fold all three into the collapsed count, raising GRAPH_NODE_DISAPPEARED before any + // of them had actually crossed miss_grace, and unhealably (their identities were erased). + ReliabilityGate gate(/*warmup_cycles=*/0, gateway_.get(), &node_mutex_); + auto det = make_node_death(); + ASSERT_TRUE(det); + det->configure({{"tick_interval_ms", 3000}, {"miss_grace", 2}, {"tracked_node_cap", 2}}); + auto ctx = make_ctx(&gate); + + set_apps({app_of("a"), app_of("b"), app_of("c"), anchor_app()}); + gate.update(snapshot_, 1); + det->tick(ctx); // arms a, b, c + + set_apps({anchor_app()}); // a, b, c all depart on the same tick + gate.update(snapshot_, 2); + det->tick(ctx); // each misses 1 of 2 (miss_grace) - below grace, but the departed COUNT + // (3) exceeds tracked_node_cap(2): the exact capacity pressure that used + // to force make_room() to collapse entries before grace. + + std::this_thread::sleep_for(50ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "none of a/b/c has crossed miss_grace yet - capacity pressure alone must never " + "report any of them dead early"; + EXPECT_EQ(det->tracked_count_for_test(), 4u) + << "the anchor (idle) plus all three of a/b/c must still be individually tracked - none " + "had matured, so none was an eligible collapse candidate despite the departed count " + "exceeding the cap"; + + // "/a" returns, relieving the pressure before anything matured; "/b" and "/c" stay gone and + // now genuinely cross miss_grace - proving the pressure above did not silently erase either + // identity (which would make it un-reportable, since only ONLINE nodes re-enter `armed`). + set_apps({app_of("a"), anchor_app()}); + gate.update(snapshot_, 3); + det->tick(ctx); // b, c miss 2 == miss_grace, not yet; a is present again (idle) + gate.update(snapshot_, 4); + det->tick(ctx); // b, c miss 3 > miss_grace: genuinely, individually dead - the departed + // count (2) no longer exceeds the cap(2), so nothing is collapsed + + std::this_thread::sleep_for(50ms); + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, {"/b"})); + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, {"/c"})); + EXPECT_EQ(det->tracked_count_for_test(), 4u) << "anchor and a (idle), b and c (departed) - none collapsed"; +} + +TEST_F(NodeDeathIntegrationTest, CapCollapsedDeathClearsOnceTheGraphRecoversAndTheDetectorIsReconfigured) { + // A cap-forced collapse is a confirmed death that can no longer be named individually - it + // still keeps GRAPH_NODE_DISAPPEARED raised, by design (collapsed_dead_count_ is monotone + // within one tracker lifetime: the identities are gone, so there is no way to tell which of + // possibly-several collapsed departures came back). What was untested is whether such a + // fault can EVER clear again once the graph genuinely recovers - proven here through the + // same reconfigure path ReconfigureWhileAbsentDoesNotWithholdAnEarnedClearOnceTheNodeReturns + // already proves for an ordinary (uncollapsed) outstanding fault: a live reconfigure rebuilds + // tracker_ (and so collapsed_dead_count_) from scratch while preserving ever_raised_, so a + // clean re-observation of an all-present graph clears it. + ReliabilityGate gate(/*warmup_cycles=*/0, gateway_.get(), &node_mutex_); + auto det = make_node_death(); + ASSERT_TRUE(det); + det->configure({{"tick_interval_ms", 3000}, {"miss_grace", 0}, {"tracked_node_cap", 1}}); + auto ctx = make_ctx(&gate); + + set_apps({app_of("a"), app_of("b"), anchor_app()}); + gate.update(snapshot_, 1); + det->tick(ctx); // arms a, b + + set_apps({anchor_app()}); // a, b both depart + gate.update(snapshot_, 2); + det->tick(ctx); // both miss 1 > miss_grace(0): matured; departed count(2) > cap(1): collapsed + + std::this_thread::sleep_for(50ms); + ASSERT_EQ(det->tracked_count_for_test(), 1u) + << "precondition: both identities collapsed - only the anchor (idle, uncapped) remains"; + ASSERT_TRUE(any_failed_desc_contains(kGraphSource, {"more node(s) disappeared"})) + << "precondition: the collapse must itself have been reported as a genuine raise"; + + set_apps({app_of("a"), app_of("b"), anchor_app()}); // the graph recovers: both return + gate.update(snapshot_, 3); + det->tick(ctx); + + std::this_thread::sleep_for(50ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), 0u) + << "merely returning must NOT by itself clear a collapsed death - collapsed_dead_count_ " + "stays raised until an operator actually acts on the cap-saturation warning"; + + // The operator's actual remedy: reconfigure (raising tracked_node_cap, say - the cap value + // itself does not matter here, only that configure() rebuilds tracker_ from scratch). + det->configure({{"tick_interval_ms", 3000}, {"miss_grace", 0}, {"tracked_node_cap", 16}}); + gate.update(snapshot_, 4); // a, b are present in this same, unchanged snapshot + det->tick(ctx); + + std::this_thread::sleep_for(50ms); + EXPECT_GT(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), 0u) + << "once reconfigured, a clean re-observation of an all-present graph must clear the " + "fault - ever_raised_ survives reconfigure (see ReconfigureWhileAbsentDoesNot... " + "above) precisely so this recovery path is not permanently stuck"; +} + +// ============================================================================================= +// C1: config sweep - miss_grace / prune_grace endpoints and degenerate values, +// tick_interval_ms edge values, the prune_grace clamp, and the 3000ms floor's own boundary. +// ============================================================================================= + +TEST_F(NodeDeathIntegrationTest, C1_MissGraceAcceptsTheLowerBoundZero) { + // tick_interval_ms=3000 -> floor is 0, so miss_grace=0 needs no bump. + EXPECT_FALSE(configure_warns({{"tick_interval_ms", 3000}, {"miss_grace", 0}}, "'miss_grace'")); +} + +TEST_F(NodeDeathIntegrationTest, C1_MissGraceAcceptsTheUpperBoundThirtySixHundred) { + EXPECT_FALSE(configure_warns({{"tick_interval_ms", 3000}, {"miss_grace", 3600}}, "'miss_grace' must")); +} + +TEST_F(NodeDeathIntegrationTest, C1_MissGraceRejectsNegativeOne) { + EXPECT_TRUE(configure_warns({{"miss_grace", -1}}, "'miss_grace' must be an integer")); +} + +TEST_F(NodeDeathIntegrationTest, C1_MissGraceRejectsOnePastTheUpperBound) { + EXPECT_TRUE(configure_warns({{"miss_grace", 3601}}, "'miss_grace' must be an integer")); +} + +TEST_F(NodeDeathIntegrationTest, C1_PruneGraceAcceptsTheLowerBoundZero) { + EXPECT_FALSE(configure_warns({{"prune_grace", 0}}, "'prune_grace' must")); +} + +TEST_F(NodeDeathIntegrationTest, C1_PruneGraceAcceptsTheUpperBoundThirtySixHundred) { + EXPECT_FALSE(configure_warns({{"prune_grace", 3600}}, "'prune_grace' must")); +} + +TEST_F(NodeDeathIntegrationTest, C1_PruneGraceRejectsNegativeOne) { + EXPECT_TRUE(configure_warns({{"prune_grace", -1}}, "'prune_grace' must be an integer")); +} + +TEST_F(NodeDeathIntegrationTest, C1_PruneGraceRejectsOnePastTheUpperBound) { + EXPECT_TRUE(configure_warns({{"prune_grace", 3601}}, "'prune_grace' must be an integer")); +} + +TEST_F(NodeDeathIntegrationTest, C1_TickIntervalMsZeroIsRejectedAndWarns) { + EXPECT_TRUE(configure_warns({{"tick_interval_ms", 0}}, "'tick_interval_ms'")); +} + +TEST_F(NodeDeathIntegrationTest, C1_TickIntervalMsNegativeIsRejectedAndWarns) { + EXPECT_TRUE(configure_warns({{"tick_interval_ms", -200}}, "'tick_interval_ms'")); +} + +TEST_F(NodeDeathIntegrationTest, C1_TickIntervalMsOneIsAcceptedAndFloorsMissGraceHeavily) { + // At a 1ms tick the floor is (3000+1-1)/1 - 1 = 2999 ticks - the default miss_grace(2) is + // nowhere close, so this must warn about the bump rather than accept the default quietly. + EXPECT_TRUE(configure_warns({{"tick_interval_ms", 1}}, "'miss_grace'")); +} + +TEST(MinNodeDeathMissGrace, DoesNotOverflowAtTickIntervalMsDocumentedValidCeiling) { + // tick_interval_ms's own documented-valid ceiling (node_death_detector.cpp's configure(): + // "must be a positive integer up to INT_MAX") - kMinNodeDeathWindowMs + tick_interval_ms + // overflows a 32-bit int here before the division ever runs unless computed in a wider + // type, undefined behaviour at a value the config contract explicitly accepts. The + // returned floor must still be small and non-negative, not a wraparound artifact. + const int floor = ros2_medkit_graph_watchdog::min_node_death_miss_grace(std::numeric_limits::max()); + EXPECT_GE(floor, 0); + EXPECT_LT(floor, 10) << "at a multi-billion-ms tick, kMinNodeDeathWindowMs is already spanned " + "by a single tick, so the floor must be tiny"; +} + +TEST_F(NodeDeathIntegrationTest, C1_MissGraceExactlyAtTheFloorIsAcceptedWithoutWarning) { + // At the 1000ms default tick the floor is (3000+999)/1000 - 1 = 2, exactly the default - + // the row either side of the floor this case and the next one pin. + EXPECT_FALSE(configure_warns({{"tick_interval_ms", 1000}, {"miss_grace", 2}}, "'miss_grace'")); +} + +TEST_F(NodeDeathIntegrationTest, C1_MissGraceOneBelowTheFloorWarnsAndIsBumped) { + EXPECT_TRUE(configure_warns({{"tick_interval_ms", 1000}, {"miss_grace", 1}}, "'miss_grace'")); +} + +TEST_F(NodeDeathIntegrationTest, C1_PruneGraceBelowMissGracePlusOneIsSilentlyClampedUp) { + // The clamp is not a rejection (no warning is owed for it - prune_grace=0 is itself a + // perfectly legal value), so this is a BEHAVIOURAL proof, in two halves: an allowlisted, + // durably dead key must not be reclaimed before miss_grace + 1 = 3 consecutive suppressed + // ticks, even though prune_grace=0 was configured (which would reclaim after just 1 if + // unclamped) - AND it must actually BE reclaimed once that clamped horizon passes, or this + // row could not tell the claimed clamp from prune() being broken entirely (neither would + // move tracked_count off 1 at the first-eligible tick either). + ReliabilityGate gate(/*warmup_cycles=*/0, gateway_.get(), &node_mutex_); + auto det = make_node_death(); + ASSERT_TRUE(det); + det->configure({{"tick_interval_ms", 3000}, + {"miss_grace", 2}, + {"prune_grace", 0}, + {"allowlist", nlohmann::json::array({"/x"})}, + {"suppress", nlohmann::json::array({"allowlist"})}}); + auto ctx = make_ctx(&gate); + + set_apps({app_of("x")}); + gate.update(snapshot_, 1); + det->tick(ctx); // arm + set_apps({}); + for (std::uint64_t t = 2; t <= 4; ++t) { // misses 1, 2, 3 (dead once misses > 2) + gate.update(snapshot_, t); + det->tick(ctx); + } + ASSERT_EQ(det->tracked_count_for_test(), 1u) << "the clamp must not have reclaimed it yet " + "at the very tick it first became eligible"; + + // prune_ticks = max(prune_grace=0, miss_grace(2)+1=3) = 3. Suppression is first evaluated + // at t=4 (streak 1); reclaim once the streak exceeds 3, i.e. streak 4 at t=7. Ticked to 9 + // for margin. + for (std::uint64_t t = 5; t <= 9; ++t) { + gate.update(snapshot_, t); + det->tick(ctx); + } + EXPECT_EQ(det->tracked_count_for_test(), 0u) + << "past the clamped horizon the durably-suppressed key must actually be reclaimed - a " + "clamp that silently became 'never prune' would leave this at 1 forever"; +} + +// ============================================================================================= +// X2: the 3000ms floor makes two otherwise-identical configurations behave DIFFERENTLY, not +// merely both eventually raise. +// ============================================================================================= + +TEST_F(NodeDeathIntegrationTest, X2_TheFloorMakesAConfiguredMissGraceBehaveDifferentlyFromOneAboveIt) { + const int tick_interval_ms = 200; // floor here is (3000+199)/200 - 1 = 14 ticks + + auto det_below = make_node_death(); + ASSERT_TRUE(det_below); + det_below->configure({{"tick_interval_ms", tick_interval_ms}, {"miss_grace", 1}}); // bumped to 14 + + auto det_above = make_node_death(); + ASSERT_TRUE(det_above); + det_above->configure({{"tick_interval_ms", tick_interval_ms}, {"miss_grace", 20}}); // already above 14 + + ReliabilityGate gate_below(0, gateway_.get(), &node_mutex_); + ReliabilityGate gate_above(0, gateway_.get(), &node_mutex_); + IntrospectionInput snap_below, snap_above; + + DetectorContext ctx_below; + ctx_below.gateway_node = gateway_.get(); + ctx_below.node_mutex = &node_mutex_; + ctx_below.mode = DetectorMode::Raise; + ctx_below.gate = &gate_below; + ctx_below.fault_client = client_; + ctx_below.snapshot = &snap_below; + DetectorContext ctx_above = ctx_below; + ctx_above.gate = &gate_above; + ctx_above.snapshot = &snap_above; + + snap_below.apps = {app_of("floor_below_node"), anchor_app()}; + snap_above.apps = {app_of("floor_above_node"), anchor_app()}; + gate_below.update(snap_below, 1); + det_below->tick(ctx_below); + gate_above.update(snap_above, 1); + det_above->tick(ctx_above); + + snap_below.apps = {anchor_app()}; + snap_above.apps = {anchor_app()}; + + int below_raised_at = -1; + int above_raised_at = -1; + for (int tick = 2; tick <= 25 && (below_raised_at < 0 || above_raised_at < 0); ++tick) { + gate_below.update(snap_below, static_cast(tick)); + det_below->tick(ctx_below); + gate_above.update(snap_above, static_cast(tick)); + det_above->tick(ctx_above); + std::this_thread::sleep_for(5ms); + if (below_raised_at < 0 && any_failed_desc_contains(kGraphSource, {"floor_below_node"})) { + below_raised_at = tick; + } + if (above_raised_at < 0 && any_failed_desc_contains(kGraphSource, {"floor_above_node"})) { + above_raised_at = tick; + } + } + + ASSERT_GT(below_raised_at, 0) << "the floor-bumped detector never reported the death at all"; + ASSERT_GT(above_raised_at, 0) << "the already-above-floor detector never reported the death at all"; + // Absent from tick 2 onward: reported once misses (tick - 1) exceeds the EFFECTIVE + // miss_grace, i.e. at tick = effective_miss_grace + 2. + EXPECT_EQ(below_raised_at, 16) << "the floor must have raised the effective miss_grace to " + "14, not left it at the configured 1"; + EXPECT_EQ(above_raised_at, 22) << "20 is already above the floor and must be used as-is"; + EXPECT_NE(below_raised_at, above_raised_at) + << "the two configurations must behave DIFFERENTLY, not merely both eventually raise"; +} + +// ============================================================================================= +// C2: malformed allowlist/suppress warn; an allowlist that `suppress` does not name has no +// effect and says so at WARN severity - naming an entry on the allowlist is not, by itself, +// a request to suppress it. +// ============================================================================================= + +TEST_F(NodeDeathIntegrationTest, C2_AllowlistWrongTopLevelTypeWarns) { + EXPECT_TRUE(configure_warns({{"allowlist", "not-an-array"}}, "'allowlist' must be an array")); +} + +TEST_F(NodeDeathIntegrationTest, C2_AllowlistBadElementWarns) { + EXPECT_TRUE(configure_warns({{"allowlist", nlohmann::json::array({42, "/ok"})}}, "'allowlist' entries must")); +} + +TEST_F(NodeDeathIntegrationTest, C2_SuppressWrongTopLevelTypeWarns) { + EXPECT_TRUE(configure_warns({{"suppress", "not-an-array"}}, "'suppress' must be an array")); +} + +TEST_F(NodeDeathIntegrationTest, C2_SuppressBadElementWarns) { + EXPECT_TRUE(configure_warns({{"suppress", nlohmann::json::array({42, "allowlist"})}}, "'suppress' entries must")); +} + +TEST_F(NodeDeathIntegrationTest, C2_UnknownSuppressEntryWarnsNamingIt) { + EXPECT_TRUE(configure_warns({{"suppress", nlohmann::json::array({"bogus_mechanism"})}}, "bogus_mechanism")); +} + +TEST_F(NodeDeathIntegrationTest, C2_AllowlistNotNamedInSuppressWarnsAndDoesNotSuppress) { + EXPECT_TRUE(configure_warns({{"allowlist", nlohmann::json::array({"/x"})}}, "inert")); + + // Behavioural half of the above: an allowlist entry that `suppress` does not name must + // never actually suppress, whatever the ported source did. + ReliabilityGate gate(/*warmup_cycles=*/0, gateway_.get(), &node_mutex_); + auto det = make_node_death(); + ASSERT_TRUE(det); + det->configure({{"tick_interval_ms", 3000}, {"miss_grace", 0}, {"allowlist", nlohmann::json::array({"/x"})}}); + auto ctx = make_ctx(&gate); + set_apps({app_of("x"), anchor_app()}); + gate.update(snapshot_, 1); + det->tick(ctx); + set_apps({anchor_app()}); + for (std::uint64_t t = 2; t <= 3; ++t) { + gate.update(snapshot_, t); + det->tick(ctx); + } + std::this_thread::sleep_for(100ms); + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, {"/x"})) + << "an allowlist that suppress does not name must not suppress by itself"; +} + +TEST_F(NodeDeathIntegrationTest, C2_TrackedNodeCapBelowRangeWarnsAndKeepsTheDefault) { + EXPECT_TRUE(configure_warns({{"tracked_node_cap", 0}}, "'tracked_node_cap' must be an integer in 1..")); +} + +TEST_F(NodeDeathIntegrationTest, C2_TrackedNodeCapAboveRangeWarnsAndKeepsTheDefault) { + EXPECT_TRUE(configure_warns({{"tracked_node_cap", 16385}}, "'tracked_node_cap' must be an integer in 1..")); +} + +TEST_F(NodeDeathIntegrationTest, C2_TrackedNodeCapFarAboveIntMaxIsRejectedNotWrapped) { + // The wide-then-validate pattern every other numeric key here uses: get() on this + // value would truncate it to 0 before any range check ever ran, and 0 would then read as + // "below range" only by accident - the real risk is a value that truncates to something + // INSIDE 1..16384 and is silently accepted as a completely different cap than what was + // configured. + EXPECT_TRUE(configure_warns({{"tracked_node_cap", 4294967296LL}}, "'tracked_node_cap' must be an integer in 1..")); +} + +// ============================================================================================= +// C3: an unknown config key warns naming it. +// ============================================================================================= + +TEST_F(NodeDeathIntegrationTest, C3_UnknownConfigKeyWarnsNamingIt) { + EXPECT_TRUE(configure_warns({{"miz_grace", 5}}, "miz_grace")); +} + +// ============================================================================================= +// The plugin's GraphWatchdogPlugin::compute_departed_retention_ticks() sizes the gate's +// departed-lifecycle retention window for exactly this pair of properties. Both tests +// construct the gate with the SAME value that function would compute for this config (a +// 200ms tick with an unconfigured, floor-bumped miss_grace and prune_grace=3), proving the +// formula is sufficient rather than merely asserting it in a comment. +// ============================================================================================= + +TEST_F(NodeDeathIntegrationTest, CleanShutdownDepartureAtMissGraceBoundaryIsSuppressed) { + const int tick_interval_ms = 200; + const int floored_miss_grace = 14; // detector_config_keys::min_node_death_miss_grace(200) + const int prune_ticks = 15; // max(prune_grace=3, floored_miss_grace+1=15) + const int retention_ticks = prune_ticks + floored_miss_grace + 1; // 30, the plugin's own formula + + ReliabilityGate gate(/*warmup_cycles=*/0, gateway_.get(), &node_mutex_, retention_ticks); + auto det = make_node_death(); + ASSERT_TRUE(det); + det->configure( + {{"tick_interval_ms", tick_interval_ms}, {"prune_grace", 3}, {"suppress", nlohmann::json::array({"lifecycle"})}}); + auto ctx = make_ctx(&gate); + + set_apps({app_of("clean"), anchor_app()}); + gate.update(snapshot_, 0); + det->tick(ctx); // arms it (present, warmup_cycles=0) + gate.set_departed_lifecycle_state_for_test("/clean", "finalized", /*saw_transition=*/true, + /*error_terminated=*/false); + set_apps({anchor_app()}); + + // Absent from tick 1: dead once misses(tick) > 14, i.e. at tick 15 - the boundary where + // node_death evaluates suppression against this key for the very first time. + for (std::uint64_t t = 1; t <= 15; ++t) { + gate.update(snapshot_, t); + det->tick(ctx); + } + std::this_thread::sleep_for(50ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "a clean departure must not be reported even at the very first tick suppression is " + "evaluated for it"; +} + +TEST_F(NodeDeathIntegrationTest, CleanShutdownDepartureIsReclaimedNotReRaisedPastRetention) { + const int tick_interval_ms = 200; + const int floored_miss_grace = 14; + const int prune_ticks = 15; + const int retention_ticks = prune_ticks + floored_miss_grace + 1; // 30 + + ReliabilityGate gate(/*warmup_cycles=*/0, gateway_.get(), &node_mutex_, retention_ticks); + auto det = make_node_death(); + ASSERT_TRUE(det); + det->configure( + {{"tick_interval_ms", tick_interval_ms}, {"prune_grace", 3}, {"suppress", nlohmann::json::array({"lifecycle"})}}); + auto ctx = make_ctx(&gate); + + set_apps({app_of("clean"), anchor_app()}); + gate.update(snapshot_, 0); + det->tick(ctx); + gate.set_departed_lifecycle_state_for_test("/clean", "finalized", /*saw_transition=*/true, + /*error_terminated=*/false); + set_apps({anchor_app()}); + + // Boundary at tick 15 (streak starts there), reclaimed once the streak exceeds + // prune_ticks(15), i.e. streak 16 at tick 15+15=30. Ticked to 35 for margin. + for (std::uint64_t t = 1; t <= 35; ++t) { + gate.update(snapshot_, t); + det->tick(ctx); + } + std::this_thread::sleep_for(50ms); + // 1, not 0: anchor_app() stays present and armed for the whole run and is never pruned - + // only "/clean" is ever eligible for reclaim. + EXPECT_EQ(det->tracked_count_for_test(), 1u) + << "a permanently clean departure must be RECLAIMED (forgotten), not merely stay " + "suppressed forever"; + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "the departed-lifecycle label must survive at least until the reclaim tick - if the " + "gate's retention window under-runs node_death's own reclaim tick, the label expires " + "first and this cleanly-shut-down node gets re-raised instead of silently reclaimed"; +} + +// ============================================================================================= +// AllowlistSuppressor's id form: a bare-name collision gives two nodes the same leaf, so an +// allowlist entry written in the collision-prefixed id shape must suppress only the node that +// id actually names - never its bare-name sibling, which is reachable only through the exact +// FQN or bare-leaf forms proven directly on AllowlistSuppressor in test_allowlist_suppressor.cpp. +// ============================================================================================= + +TEST_F(NodeDeathIntegrationTest, AllowlistIdFormSuppressesOnlyTheCollisionPrefixedNode) { + ReliabilityGate gate(/*warmup_cycles=*/0, gateway_.get(), &node_mutex_); + auto det = make_node_death(); + ASSERT_TRUE(det); + // The id form specifically: neither node's bare leaf ("calibration") nor either full fqn + // is on this list, so a suppression must be reaching the durable, id-shaped entry via + // AllowlistSuppressor::allows() - the id captured while each node was still present (see + // node_death_detector.cpp's tick()) - not via suppresses()'s own exact/bare-leaf checks. + det->configure({{"tick_interval_ms", 3000}, // floor(3000ms) == 0, so miss_grace=0 is legal + {"miss_grace", 0}, + {"allowlist", nlohmann::json::array({"coll_a_calibration"})}, + {"suppress", nlohmann::json::array({"allowlist"})}}); + auto ctx = make_ctx(&gate); + + // Two nodes sharing the bare leaf "calibration" in different namespaces - the shape + // ros2_runtime_introspection.cpp's own collision rule produces: same-bare-name nodes each + // get a namespace-prefixed id, while their fqn (namespace + bare leaf) stays exactly what + // it always was. Built by hand rather than via app_of(), which only ever derives bound_fqn + // from id itself and so cannot produce an id that DIFFERS from the fqn's own leaf. + App node_a; + node_a.id = "coll_a_calibration"; + node_a.bound_fqn = "/coll_a/calibration"; + node_a.is_online = true; + node_a.source = "runtime"; + App node_b; + node_b.id = "coll_b_calibration"; + node_b.bound_fqn = "/coll_b/calibration"; + node_b.is_online = true; + node_b.source = "runtime"; + + set_apps({node_a, node_b, anchor_app()}); + gate.update(snapshot_, 0); + det->tick(ctx); // arms both (present, warmup_cycles=0) and captures each one's id verdict + + set_apps({anchor_app()}); // both depart on the same tick + gate.update(snapshot_, 1); + det->tick(ctx); // misses(1) > miss_grace(0): both would be dead absent suppression + + std::this_thread::sleep_for(50ms); + EXPECT_TRUE(any_failed_desc_contains(kGraphSource, {"/coll_b/calibration"})) + << "the sibling the allowlist entry does not name must still be reported dead - " + "otherwise this test cannot tell a working id-match from a suppressor that " + "swallows everything"; + EXPECT_FALSE(any_failed_desc_contains(kGraphSource, {"/coll_a/calibration"})) + << "an allowlist entry naming the collision-prefixed id must suppress exactly the node " + "that id names"; +} + +// ============================================================================================= +// The ungated-clear guard (ever_raised_): ctx.clear_fault() carries no reliability-gate check +// of its own, so this detector may only clear GRAPH_NODE_DISAPPEARED once it has itself +// genuinely raised it at least once - never merely because SOMETHING, anything, is tracked. +// ============================================================================================= + +TEST_F(NodeDeathIntegrationTest, UngatedClearStaysSuppressedWhileAnUnrelatedNodeArmsAlone) { + ReliabilityGate gate(/*warmup_cycles=*/0, gateway_.get(), &node_mutex_); + auto det = make_node_death(); + ASSERT_TRUE(det); + det->configure({{"tick_interval_ms", 3000}, {"miss_grace", 0}}); // floor(3000ms) == 0 + auto ctx = make_ctx(&gate); + + // "other" arms and stays present for every tick - it never dies, so report.dead is never + // non-empty because of it. It stands in for anything elsewhere in the graph (the gateway's + // own App among real candidates) that becomes tracked without being what a stored, + // outstanding fault is actually about - tracker_.tracked_count() going non-zero here must + // not be read as "safe to clear". + for (std::uint64_t t = 0; t < 10; ++t) { + set_apps({app_of("other"), anchor_app()}); + gate.update(snapshot_, t); + det->tick(ctx); + } + ASSERT_GT(det->tracked_count_for_test(), 0u) + << "precondition this test needs: something IS tracked, without this detector having " + "ever raised anything"; + + std::this_thread::sleep_for(50ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), 0u) + << "a fresh instance that has never itself raised GRAPH_NODE_DISAPPEARED must never " + "clear it either, however many unrelated entities become tracked"; +} + +TEST_F(NodeDeathIntegrationTest, ClearFlowsOnceThisInstanceHasGenuinelyRaised) { + ReliabilityGate gate(/*warmup_cycles=*/0, gateway_.get(), &node_mutex_); + auto det = make_node_death(); + ASSERT_TRUE(det); + det->configure({{"tick_interval_ms", 3000}, {"miss_grace", 0}}); + auto ctx = make_ctx(&gate); + + set_apps({app_of("victim"), anchor_app()}); + gate.update(snapshot_, 0); + det->tick(ctx); // arms "victim" + + set_apps({anchor_app()}); // victim departs + gate.update(snapshot_, 1); + det->tick(ctx); // misses(1) > miss_grace(0): genuinely dead, genuinely raised + + std::this_thread::sleep_for(50ms); + ASSERT_GT(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "precondition: this instance must have genuinely raised before the recovery below " + "can prove anything about clearing"; + + set_apps({app_of("victim"), anchor_app()}); // victim returns + gate.update(snapshot_, 2); + det->tick(ctx); + + std::this_thread::sleep_for(50ms); + EXPECT_GT(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), 0u) + << "a genuine recovery after this instance has itself raised must still clear - the " + "ungated-clear guard must suppress an UNEARNED clear, not an earned one"; +} + +TEST_F(NodeDeathIntegrationTest, EverRaisedTracksDeliveryNotIntentSoAnUndeliveredRaiseNeverEarnsAClear) { + // ever_raised_ must be set from emit_ordered()'s own return (did raise_fault() actually + // call async_send_request()), not from report.dead's mere non-emptiness. Advisory mode is + // the simplest of several ways raise_fault() can decline to send despite a non-empty + // report (mode_emits, no client, empty source_id, the reliability gate, and a + // not-yet-ready service all take the identical silent-decline path) - any one of them + // proves the same gap, since ever_raised_ has no way to tell them apart from a genuine + // send without reading raise_fault()'s own return value. + ReliabilityGate gate(/*warmup_cycles=*/0, gateway_.get(), &node_mutex_); + auto det = make_node_death(); + ASSERT_TRUE(det); + det->configure({{"tick_interval_ms", 3000}, {"miss_grace", 0}}); + auto ctx = make_ctx(&gate); + ctx.mode = DetectorMode::Advisory; // every raise_fault()/clear_fault() call declines to send + + set_apps({app_of("victim"), anchor_app()}); + gate.update(snapshot_, 0); + det->tick(ctx); // arms "victim" + + set_apps({anchor_app()}); // victim departs + gate.update(snapshot_, 1); + det->tick(ctx); // report.dead is non-empty, but Advisory mode declines to send it + + std::this_thread::sleep_for(50ms); + ASSERT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "precondition: Advisory mode must have genuinely suppressed the raise"; + + // Switch to Raise mode and let victim return. If ever_raised_ had already been set from + // report.dead's mere non-emptiness on the tick above (the bug this test pins), it would + // read true here and this recovery would emit an unearned PASSED for an occurrence the + // fault manager never heard about in the first place. + ctx.mode = DetectorMode::Raise; + set_apps({app_of("victim"), anchor_app()}); + gate.update(snapshot_, 2); + det->tick(ctx); + + std::this_thread::sleep_for(50ms); + EXPECT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), 0u) + << "no FAILED for this occurrence ever reached the wire, so no PASSED may either"; +} + +TEST_F(NodeDeathIntegrationTest, AdvisoryModeStillObservesAndAccumulatesMissesWithoutEmittingAnything) { + // README's own contract: "advisory = observe, do not push". The sibling test above proves + // the "do not push" half; this proves the "observe" half is not a no-op masquerading as + // one - a detector that skipped tracker_.update() entirely in Advisory mode would ALSO + // send nothing, and would be indistinguishable from a correct one by that test alone. + // + // Proof: accumulate misses PAST miss_grace while in Advisory (so the node is genuinely + // dead from the tracker's own point of view, just never reported), then switch to Raise + // mode WITHOUT letting the node return. A detector that truly kept observing raises on + // this very first Raise-mode tick, because the miss count already crossed the threshold + // during the Advisory window; one that had paused observation would need miss_grace + 1 + // MORE ticks from here to reach the same threshold from zero. + ReliabilityGate gate(/*warmup_cycles=*/0, gateway_.get(), &node_mutex_); + auto det = make_node_death(); + ASSERT_TRUE(det); + det->configure({{"tick_interval_ms", 3000}, {"miss_grace", 2}}); + auto ctx = make_ctx(&gate); + ctx.mode = DetectorMode::Advisory; + + set_apps({app_of("victim"), anchor_app()}); + gate.update(snapshot_, 0); + det->tick(ctx); // arms "victim" + + set_apps({anchor_app()}); // victim departs + for (std::uint64_t t = 1; t <= 3; ++t) { // misses 1, 2, 3 (> miss_grace(2)): genuinely dead + gate.update(snapshot_, t); + det->tick(ctx); + } + EXPECT_EQ(det->tracked_count_for_test(), 2u) + << "Advisory mode must still track victim and anchor exactly as Raise mode would"; + + std::this_thread::sleep_for(50ms); + ASSERT_EQ(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "precondition: Advisory mode must have genuinely suppressed the raise so far"; + + ctx.mode = DetectorMode::Raise; + gate.update(snapshot_, 4); // victim still absent - no recovery here + det->tick(ctx); + + std::this_thread::sleep_for(50ms); + EXPECT_GT(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "the very first Raise-mode tick must already raise, proving the miss count had " + "already crossed miss_grace DURING the Advisory window rather than starting fresh"; +} + +TEST_F(NodeDeathIntegrationTest, ReconfigureWhileAbsentDoesNotWithholdAnEarnedClearOnceTheNodeReturns) { + // ever_raised_ is a fact about this PROCESS's history, not about the currently-loaded + // config - configure() rebuilds tracker_ fresh (an operator editing the allowlist, say, + // with a death still outstanding and absent), and if ever_raised_ reset along with it, the + // freshly-rebuilt tracker could never re-create evidence for a node it has not itself + // re-observed, leaving the standing fault able to get neither a further FAILED nor a + // PASSED for the rest of the process's life once the node genuinely returns. + ReliabilityGate gate(/*warmup_cycles=*/0, gateway_.get(), &node_mutex_); + auto det = make_node_death(); + ASSERT_TRUE(det); + det->configure({{"tick_interval_ms", 3000}, {"miss_grace", 0}}); + auto ctx = make_ctx(&gate); + + set_apps({app_of("victim"), anchor_app()}); + gate.update(snapshot_, 0); + det->tick(ctx); // arms "victim" + + set_apps({anchor_app()}); // victim departs + gate.update(snapshot_, 1); + det->tick(ctx); // genuinely raised + + std::this_thread::sleep_for(50ms); + ASSERT_GT(count_faults(kGraphSource, ReportFault::Request::EVENT_FAILED), 0u) + << "precondition: this instance must have genuinely raised before the reconfigure below"; + + // Reconfigure WHILE the node is still absent - tracker_ is rebuilt empty, and this + // (otherwise identical) config has never itself observed "victim" at all. + det->configure({{"tick_interval_ms", 3000}, {"miss_grace", 0}}); + + set_apps({app_of("victim"), anchor_app()}); // victim returns + gate.update(snapshot_, 2); + det->tick(ctx); + + std::this_thread::sleep_for(50ms); + EXPECT_GT(count_faults(kGraphSource, ReportFault::Request::EVENT_PASSED), 0u) + << "a genuine recovery after a live reconfigure must still clear - the standing fault " + "must not be stuck unable to ever heal for the rest of the process's life"; +} diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_node_liveness_tracker.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_node_liveness_tracker.cpp new file mode 100644 index 000000000..d9f0ce210 --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_node_liveness_tracker.cpp @@ -0,0 +1,329 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Pure logic: no rclcpp::Node, no rclcpp::init() anywhere in this file - both +// NodeLivenessTracker and AggregatedFault::describe_ordered() are ROS-free. +#include + +#include +#include +#include + +#include "ros2_medkit_graph_watchdog/aggregated_fault.hpp" +#include "ros2_medkit_graph_watchdog/node_liveness_tracker.hpp" + +using ros2_medkit_graph_watchdog::AggregatedFault; +using ros2_medkit_graph_watchdog::NodeLivenessTracker; + +// ---- update(): the presence/absence state machine ---------------------------------------- + +TEST(NodeLivenessTrackerUpdate, PresentButNeverArmedIsNeverTracked) { + NodeLivenessTracker tracker(/*miss_grace=*/0); + auto report = tracker.update({"/a"}, /*armed=*/{}); + EXPECT_TRUE(report.dead.empty()); + EXPECT_EQ(tracker.tracked_count(), 0u); +} + +TEST(NodeLivenessTrackerUpdate, OnceArmedPresenceAloneKeepsItAlive) { + NodeLivenessTracker tracker(/*miss_grace=*/0); + tracker.update({"/a"}, {"/a"}); // arms it + auto report = tracker.update({"/a"}, {}); // still present; arm state no longer matters + EXPECT_TRUE(report.dead.empty()); + EXPECT_EQ(tracker.tracked_count(), 1u); +} + +TEST(NodeLivenessTrackerUpdate, ArmedThenAbsentIsReportedOnlyPastMissGrace) { + NodeLivenessTracker tracker(/*miss_grace=*/2); + tracker.update({"/a"}, {"/a"}); + EXPECT_TRUE(tracker.update({}, {}).dead.empty()) << "miss 1"; + EXPECT_TRUE(tracker.update({}, {}).dead.empty()) << "miss 2 == miss_grace, not yet"; + auto report = tracker.update({}, {}); // miss 3 > miss_grace + ASSERT_EQ(report.dead.count("/a"), 1u); + EXPECT_NE(report.dead.at("/a").find("/a"), std::string::npos) << "the detail must name the key"; +} + +TEST(NodeLivenessTrackerUpdate, ReturningAfterBeingReportedDeadClearsIt) { + NodeLivenessTracker tracker(/*miss_grace=*/0); + tracker.update({"/a"}, {"/a"}); + ASSERT_EQ(tracker.update({}, {}).dead.count("/a"), 1u); + auto report = tracker.update({"/a"}, {}); // it came back + EXPECT_TRUE(report.dead.empty()); +} + +TEST(NodeLivenessTrackerUpdate, AnUnsuppressedDeathIsNeverForgottenWithoutPrune) { + // The anti-false-heal guarantee update() alone provides: a dead, never-pruned key stays + // reported however many further ticks pass. + NodeLivenessTracker tracker(/*miss_grace=*/0); + tracker.update({"/a"}, {"/a"}); + for (int i = 0; i < 50; ++i) { + EXPECT_EQ(tracker.update({}, {}).dead.count("/a"), 1u) << "tick " << i; + } + EXPECT_EQ(tracker.tracked_count(), 1u); +} + +TEST(NodeLivenessTrackerUpdate, KeysByFreshnessOrdersTheMostRecentDeathFirst) { + NodeLivenessTracker tracker(/*miss_grace=*/0); + tracker.update({"/old", "/new"}, {"/old", "/new"}); + tracker.update({"/new"}, {}); // /old dies now (miss 1); /new stays present + auto report = tracker.update({}, {}); // /old miss 2; /new miss 1 (just died - freshest) + ASSERT_EQ(report.dead.size(), 2u); + ASSERT_EQ(report.keys_by_freshness.size(), 2u); + EXPECT_EQ(report.keys_by_freshness.front(), "/new"); + EXPECT_EQ(report.keys_by_freshness.back(), "/old"); +} + +TEST(NodeLivenessTrackerUpdate, TwoIdsWithTheSameMissCountBreakTiesByKey) { + NodeLivenessTracker tracker(/*miss_grace=*/0); + tracker.update({"/b", "/a"}, {"/b", "/a"}); + auto report = tracker.update({}, {}); // both die on the same tick, same miss count + ASSERT_EQ(report.keys_by_freshness.size(), 2u); + EXPECT_EQ(report.keys_by_freshness.front(), "/a") << "ties break lexicographically, deterministically"; +} + +// ---- prune(): S5 - only a veto that never lifts may safely reclaim ----------------------- + +TEST(NodeLivenessTrackerPrune, APermanentlySuppressedKeyIsReclaimedOnceItsStreakPassesPruneTicks) { + NodeLivenessTracker tracker(/*miss_grace=*/0, /*prune_ticks=*/2); + tracker.update({}, {"/durable"}); + ASSERT_EQ(tracker.update({}, {}).dead.count("/durable"), 1u); + ASSERT_EQ(tracker.tracked_count(), 1u); + + tracker.prune({"/durable"}); // streak 1 + EXPECT_EQ(tracker.tracked_count(), 1u); + tracker.prune({"/durable"}); // streak 2 == prune_ticks: not yet past it + EXPECT_EQ(tracker.tracked_count(), 1u); + tracker.prune({"/durable"}); // streak 3 > prune_ticks(2): reclaimed + EXPECT_EQ(tracker.tracked_count(), 0u); +} + +TEST(NodeLivenessTrackerPrune, AVetoThatLiftsEvenOnceResetsTheStreakAndDelaysReclaim) { + // This is the property that makes it SAFE for a detector to feed prune() only its + // durable suppressors' verdicts: a condition-based (non-durable) veto can stop matching + // on any tick, and when it does, the streak must restart from zero rather than merely + // pause - or a key could be reclaimed by ACCUMULATED, non-consecutive suppressed ticks + // even though the condition was false in between, which is exactly the false-heal + // suppressor.hpp's durable() doc warns against. + NodeLivenessTracker tracker(/*miss_grace=*/0, /*prune_ticks=*/2); + tracker.update({}, {"/lifts"}); + ASSERT_EQ(tracker.update({}, {}).dead.count("/lifts"), 1u); + + tracker.prune({"/lifts"}); // streak 1 + tracker.prune({}); // the veto lifts for one call -> streak resets to 0 + tracker.prune({"/lifts"}); // streak 1 again + tracker.prune({"/lifts"}); // streak 2 + EXPECT_EQ(tracker.tracked_count(), 1u) + << "two consecutive suppressed calls after a lift must not equal three from a permanent veto"; + tracker.prune({"/lifts"}); // streak 3 > prune_ticks(2): only now reclaimed + EXPECT_EQ(tracker.tracked_count(), 0u); +} + +TEST(NodeLivenessTrackerPrune, AnUnsuppressedDeathIsNeverReclaimedNoMatterHowManyPruneCalls) { + NodeLivenessTracker tracker(/*miss_grace=*/0, /*prune_ticks=*/2); + tracker.update({}, {"/never_suppressed"}); + ASSERT_EQ(tracker.update({}, {}).dead.count("/never_suppressed"), 1u); + for (int i = 0; i < 20; ++i) { + tracker.prune({}); // nothing is ever suppressed + } + EXPECT_EQ(tracker.tracked_count(), 1u); +} + +TEST(NodeLivenessTrackerPrune, PruneNeverErasesAKeyFromAReportAlreadyHandedBack) { + // Every real caller runs prune() strictly AFTER update() within the same tick - this + // pins that pruning can never retroactively take back a key this tick's own report + // already earned. + NodeLivenessTracker tracker(/*miss_grace=*/0, /*prune_ticks=*/0); + tracker.update({}, {"/a"}); + auto report = tracker.update({}, {}); + ASSERT_EQ(report.dead.count("/a"), 1u); + tracker.prune({"/a"}); // reclaims it for the NEXT tick, not this one + EXPECT_EQ(report.dead.count("/a"), 1u); +} + +// ---- D1: the capped description survives a fresh entry among many stale ones ------------- + +TEST(NodeLivenessTrackerFreshness, AFreshDeathSurvivesTheDescriptionCapAmongManyStaleOnes) { + NodeLivenessTracker tracker(/*miss_grace=*/0); + std::set stale_ids; + // Ten long, alphabetically-EARLY ids already exceed kMaxDescriptionChars by themselves - + // proves the freshness ordering actually matters, not merely that a short text happens + // to fit regardless of order. + for (int i = 0; i < 10; ++i) { + stale_ids.insert("/aaa_stale_node_padded_to_be_long_enough_" + std::to_string(1000 + i)); + } + // Armed and immediately dead: absent from `present` on the very tick they are armed + // already counts as one miss, which exceeds miss_grace(0). + tracker.update({}, stale_ids); + + // A fresh id that sorts alphabetically LAST, armed a tick later while still present (so + // it is not dead yet), then departs on the tick after - the genuinely most recent death. + const std::string fresh_id = "/zzz_fresh_node"; + tracker.update({fresh_id}, {fresh_id}); + auto report = tracker.update({}, {}); + ASSERT_EQ(report.dead.count(fresh_id), 1u); + ASSERT_FALSE(report.keys_by_freshness.empty()); + EXPECT_EQ(report.keys_by_freshness.front(), fresh_id); + + const std::string description = AggregatedFault::describe_ordered(report.dead, report.keys_by_freshness); + EXPECT_LE(description.size(), AggregatedFault::kMaxDescriptionChars); + EXPECT_NE(description.find(fresh_id), std::string::npos) + << "a fresh death must survive the capped description even though it sorts last " + "alphabetically among the stale entries"; +} + +TEST(NodeLivenessTrackerFreshness, PlainLexicographicOrderWouldHaveCutTheFreshEntry) { + // The control case for the test above: build the SAME description from `dead` alone (the + // uncapped, alphabetical helper's own default order), proving the fresh id really would + // have been lost without keys_by_freshness - this is not a description that happens to + // fit either way. + NodeLivenessTracker tracker(/*miss_grace=*/0); + std::set stale_ids; + for (int i = 0; i < 10; ++i) { + stale_ids.insert("/aaa_stale_node_padded_to_be_long_enough_" + std::to_string(1000 + i)); + } + tracker.update({}, stale_ids); + const std::string fresh_id = "/zzz_fresh_node"; + tracker.update({fresh_id}, {fresh_id}); + auto report = tracker.update({}, {}); + + const std::string lexicographic = AggregatedFault::describe(report.dead); + EXPECT_EQ(lexicographic.find(fresh_id), std::string::npos) + << "this pins that the freshness ordering is load-bearing: without it, the " + "alphabetically-last fresh id is the one the cap would have cut"; +} + +// ---- tracked_key_cap: the bound that keeps the DEPARTED subset of known_/misses_/ +// suppressed_streak_ from growing without limit under identity churn. A present entry never +// counts against it and is never evicted to make room - see node_liveness_tracker.hpp's own +// class doc for why this tracker's cap does not mirror LifecycleExpectationTracker's. + +TEST(NodeLivenessTrackerCap, TrackedCountNeverExceedsTheConfiguredCapUnderChurn) { + NodeLivenessTracker tracker(/*miss_grace=*/0, NodeLivenessTracker::kNoPrune, /*tracked_key_cap=*/5); + std::size_t max_seen = 0; + // 200 distinct, never-repeating identities, never present again once armed - every one + // instantly departed (miss_grace=0), so this exercises exactly the DEPARTED-subset cap at a + // scale that pushes past it, with no present entries in the mix to blur the measurement. + for (int i = 0; i < 200; ++i) { + tracker.update({}, {"/churn_" + std::to_string(i)}); + max_seen = std::max(max_seen, tracker.tracked_count()); + } + EXPECT_LE(max_seen, 5u) << "the cap must actually engage under scale past it, not merely " + "exist unexercised"; +} + +TEST(NodeLivenessTrackerCap, PresentEntriesExceedingTheCapAreAllStillIndividuallyReportableOnDeath) { + // C1 regression, at the tracker level: the earlier shape of this cap evicted PRESENT + // entries to admit a newcomer once the map (not merely the departed subset) reached + // tracked_key_cap - at 513 armed nodes against the default 512, only one survived. A + // present key must never be refused a slot or evicted for one, at any map size. + NodeLivenessTracker tracker(/*miss_grace=*/0, NodeLivenessTracker::kNoPrune, /*tracked_key_cap=*/5); + std::set ids; + for (int i = 0; i < 50; ++i) { + ids.insert("/n" + std::to_string(i)); + } + auto report = tracker.update(ids, ids); // 50 present, armed keys - ten times the cap + EXPECT_TRUE(report.dead.empty()); + EXPECT_EQ(tracker.tracked_count(), 50u) << "a cap on the DEPARTED subset must never refuse " + "or evict a present key, however many there are"; + + ids.erase(ids.begin()); // kill exactly one of the fifty + report = tracker.update(ids, ids); + EXPECT_EQ(report.dead.size(), 1u) << "the one node that actually died must be reported - " + "the other 49, still present, never competed for the " + "departed cap at all"; +} + +TEST(NodeLivenessTrackerCap, ImmatureDepartedEntriesAreNeverCollapsedOrReportedUnderCapPressure) { + // C2 regression, at the tracker level: the earlier shape of this cap collapsed every + // NON-idle entry to make room, which included entries carrying only one below-grace miss - + // fabricating a death the node had not actually earned, permanently (the identity is + // erased, so the node returning cannot un-report it). An entry that has not crossed + // miss_grace_ must never be a collapse candidate, however much departed-set pressure exists. + NodeLivenessTracker tracker(/*miss_grace=*/2, NodeLivenessTracker::kNoPrune, /*tracked_key_cap=*/1); + tracker.update({"/a", "/b"}, {"/a", "/b"}); // both present, armed + auto report = tracker.update({}, {}); // both miss 1 of 2 - departed count(2) > cap(1) + EXPECT_TRUE(report.dead.empty()) << "neither has crossed miss_grace yet"; + EXPECT_EQ(report.dead.count(NodeLivenessTracker::kCollapsedKey), 0u) + << "capacity pressure from two BELOW-GRACE entries must never manufacture a collapsed " + "death - the earlier shape of this cap did exactly that"; + EXPECT_EQ(tracker.tracked_count(), 2u) << "both identities must survive the pressure " + "untouched, so they can still mature genuinely"; + EXPECT_TRUE(report.tracking_saturated) << "the departed set (2) still exceeds the cap (1) " + "even though nothing was eligible to collapse - an " + "accurate, non-fabricating pressure signal"; +} + +TEST(NodeLivenessTrackerCap, MaturedDepartedEntriesAreCollapsedIntoACountUnderCapPressure) { + NodeLivenessTracker tracker(/*miss_grace=*/0, NodeLivenessTracker::kNoPrune, /*tracked_key_cap=*/1); + tracker.update({}, {"/victim"}); + ASSERT_EQ(tracker.update({}, {}).dead.count("/victim"), 1u); + ASSERT_EQ(tracker.tracked_count(), 1u); + + // "/newcomer" arms already absent - departed on its very first tick, same as "/victim" was. + // Both are MATURED (miss_grace=0), and the departed set (2) now exceeds the cap (1): there + // is no present entry to spare (there never is, for this tracker) and no immature entry to + // leave alone either, so admitting the newcomer must collapse matured entries instead of + // refusing it. + auto report = tracker.update({}, {"/newcomer"}); + EXPECT_EQ(tracker.tracked_count(), 0u) << "cap(1) is below kMaxNamedDepartedEntries, so even " + "the two-stage collapse cannot leave either one " + "individually named"; + EXPECT_EQ(report.dead.count("/victim"), 0u); + EXPECT_EQ(report.dead.count("/newcomer"), 0u); + ASSERT_EQ(report.dead.count(NodeLivenessTracker::kCollapsedKey), 1u) + << "both confirmed deaths must still count as content, or the cap forcing them out of " + "individual tracking would silently heal the fault instead of collapsing it"; + EXPECT_NE(report.dead.at(NodeLivenessTracker::kCollapsedKey).find('2'), std::string::npos); + ASSERT_FALSE(report.keys_by_freshness.empty()); + EXPECT_EQ(report.keys_by_freshness.front(), NodeLivenessTracker::kCollapsedKey) + << "the collapsed count must never be the entry a capped description cuts, since it is " + "the one line telling the operator identities are being lost at all"; +} + +TEST(NodeLivenessTrackerCap, CollapsedCountIsMonotoneAndKeepsAccumulating) { + NodeLivenessTracker tracker(/*miss_grace=*/0, NodeLivenessTracker::kNoPrune, /*tracked_key_cap=*/1); + tracker.update({}, {"/first"}); + tracker.update({}, {}); // "/first" confirmed dead, occupies the only slot + auto after_second = tracker.update({}, {"/second"}); // "/second" arms already-departed: both collapse + ASSERT_EQ(after_second.dead.count(NodeLivenessTracker::kCollapsedKey), 1u); + EXPECT_NE(after_second.dead.at(NodeLivenessTracker::kCollapsedKey).find('2'), std::string::npos); + ASSERT_EQ(tracker.tracked_count(), 0u); + + auto after_third = tracker.update({}, {"/third"}); // one free slot: "/third" stays named alone + EXPECT_EQ(tracker.tracked_count(), 1u); + EXPECT_NE(after_third.dead.at(NodeLivenessTracker::kCollapsedKey).find('2'), std::string::npos) + << "the count from the first collapse must persist even on a tick that collapses nothing new"; + + auto after_fourth = tracker.update({}, {"/fourth"}); // "/fourth" arms already-departed: collapses both + ASSERT_EQ(after_fourth.dead.count(NodeLivenessTracker::kCollapsedKey), 1u); + EXPECT_NE(after_fourth.dead.at(NodeLivenessTracker::kCollapsedKey).find('4'), std::string::npos) + << "two more distinct identities have now been collapsed - the count must not have reset " + "when the slot changed hands again"; +} + +TEST(NodeLivenessTrackerCap, AtZeroMissGraceEveryDepartedEntryIsInstantlyMaturedSoSaturationNeverFires) { + // NOT a general property of this tracker - see + // ImmatureDepartedEntriesAreNeverCollapsedOrReportedUnderCapPressure above for the case + // that DOES saturate. At miss_grace=0 every departed entry matures on its very first + // missed tick, so there is never an immature entry the cap has to leave alone: collapsing + // every matured entry always succeeds in bringing the departed set back under the cap. + NodeLivenessTracker tracker(/*miss_grace=*/0, NodeLivenessTracker::kNoPrune, /*tracked_key_cap=*/1); + bool ever_saturated = false; + for (int i = 0; i < 50; ++i) { + // Two brand-new, simultaneously-arming keys every tick, cap=1: the tightest possible + // squeeze this tracker can be put under. + auto report = tracker.update({}, {"/a_" + std::to_string(i), "/b_" + std::to_string(i)}); + ever_saturated = ever_saturated || report.tracking_saturated; + } + EXPECT_FALSE(ever_saturated); +} diff --git a/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_suppressor.cpp b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_suppressor.cpp new file mode 100644 index 000000000..7a2f608aa --- /dev/null +++ b/src/ros2_medkit_plugins/ros2_medkit_graph_watchdog/test/test_suppressor.cpp @@ -0,0 +1,108 @@ +// Copyright 2026 bburda +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Pure interface/logic tests: the Suppressor contract and apply_suppressors() over +// hand-built stand-in suppressors. No rclcpp::init() needed anywhere here - every +// DetectorContext used is default-constructed (all pointers null), which the interface +// contract already requires a Suppressor to tolerate (see suppresses()'s own doc). +#include + +#include +#include +#include +#include +#include + +#include "ros2_medkit_graph_watchdog/suppressor.hpp" + +using ros2_medkit_graph_watchdog::apply_suppressors; +using ros2_medkit_graph_watchdog::DetectorContext; +using ros2_medkit_graph_watchdog::Suppressor; + +namespace { +/// Votes yes for every key in `veto`, abstains otherwise. `durable_flag` lets a test +/// stand in for either shape the interface describes without needing a real suppressor. +class FixedSuppressor : public Suppressor { + public: + explicit FixedSuppressor(std::set veto, bool durable_flag = false) + : veto_(std::move(veto)), durable_flag_(durable_flag) { + } + bool suppresses(const std::string & entity_key, const DetectorContext & /*ctx*/) const override { + return veto_.count(entity_key) > 0; + } + bool durable() const override { + return durable_flag_; + } + + private: + std::set veto_; + bool durable_flag_; +}; +} // namespace + +TEST(SuppressorInterface, DurableDefaultsFalse) { + FixedSuppressor s({"/a"}); + EXPECT_FALSE(s.durable()); +} + +TEST(SuppressorInterface, DurableCanBeOverriddenTrue) { + FixedSuppressor s({"/a"}, /*durable_flag=*/true); + EXPECT_TRUE(s.durable()); +} + +TEST(ApplySuppressors, EmptyChainSuppressesNothing) { + std::map affected{{"/a", "reason a"}, {"/b", "reason b"}}; + DetectorContext ctx; + EXPECT_EQ(apply_suppressors(affected, {}, ctx), 0u); + EXPECT_EQ(affected.size(), 2u); +} + +TEST(ApplySuppressors, DropsOnlyTheVetoedKeys) { + std::map affected{{"/a", "reason a"}, {"/b", "reason b"}, {"/c", "reason c"}}; + FixedSuppressor vetoes_b({"/b"}); + DetectorContext ctx; + const std::vector chain{&vetoes_b}; + EXPECT_EQ(apply_suppressors(affected, chain, ctx), 1u); + EXPECT_EQ(affected.count("/a"), 1u); + EXPECT_EQ(affected.count("/b"), 0u); + EXPECT_EQ(affected.count("/c"), 1u); +} + +TEST(ApplySuppressors, AnySingleSuppressorInTheChainIsEnough) { + std::map affected{{"/a", "reason a"}, {"/b", "reason b"}}; + FixedSuppressor abstains_on_everything({}); + FixedSuppressor vetoes_b({"/b"}); + DetectorContext ctx; + const std::vector chain{&abstains_on_everything, &vetoes_b}; + apply_suppressors(affected, chain, ctx); + EXPECT_EQ(affected.count("/a"), 1u); + EXPECT_EQ(affected.count("/b"), 0u); +} + +TEST(ApplySuppressors, ANullEntryInTheChainIsSkippedNotDereferenced) { + std::map affected{{"/a", "reason a"}}; + DetectorContext ctx; + const std::vector chain{nullptr}; + EXPECT_EQ(apply_suppressors(affected, chain, ctx), 0u); + EXPECT_EQ(affected.size(), 1u); +} + +TEST(ApplySuppressors, ReturnsTheDroppedCount) { + std::map affected{{"/a", "x"}, {"/b", "y"}, {"/c", "z"}}; + FixedSuppressor vetoes_everything({"/a", "/b", "/c"}); + DetectorContext ctx; + const std::vector chain{&vetoes_everything}; + EXPECT_EQ(apply_suppressors(affected, chain, ctx), 3u); + EXPECT_TRUE(affected.empty()); +}